diff --git a/RELEASES.json b/RELEASES.json index e6244d23bc38322b1840177e1be4c2dc13122270..df54929807057eb7fd23fe2414f8b721b27c3abe 100644 --- a/RELEASES.json +++ b/RELEASES.json @@ -1,6 +1,18 @@ { - "current": "v54 (6a8e99a)", + "current": "v56 (2970b72)", "releases": [ + { + "version": "v56", + "sha": "2970b72", + "date": "2026-08-28", + "subject": "release v56" + }, + { + "version": "v55", + "sha": "b766b90", + "date": "2026-08-28", + "subject": "release v55" + }, { "version": "v54", "sha": "6a8e99a", diff --git a/VERSION b/VERSION index a9b2c41ce46ced52cceac9087332a965810f81c9..1bd3b1d7563f9779f04c3e075290c24fd0993a52 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -v54 (6a8e99a) +v56 (2970b72) diff --git a/api/automation_control.py b/api/automation_control.py index 95d87e61c04d352791a6200ed6192443f2153e89..dd4562a21783e4b832e83c87a6cd492a8f6588e0 100644 --- a/api/automation_control.py +++ b/api/automation_control.py @@ -19,6 +19,7 @@ import os import re import time import uuid +from zoneinfo import ZoneInfo, ZoneInfoNotFoundError import requests @@ -124,6 +125,21 @@ def _base_job(tenant: str, defn: dict, mode: str) -> dict: "scheduledTime": ""} +def _recurring_timezone(defn: dict) -> str: + """Use a Route's frozen wall-clock Zone; every other automation remains UTC.""" + if str(defn.get("kind") or "") != "route_delivery": + return "UTC" + schedule = defn.get("schedule") if isinstance(defn.get("schedule"), dict) else {} + config = defn.get("config") if isinstance(defn.get("config"), dict) else {} + snapshot = config.get("routeSnapshot") if isinstance(config.get("routeSnapshot"), dict) else {} + timezone = str(schedule.get("timezone") or snapshot.get("timezone") or "").strip() + try: + ZoneInfo(timezone) + except (ZoneInfoNotFoundError, ValueError): + raise ValueError("a scheduled route needs a recognized IANA timezone") from None + return timezone + + def recurring_specs(tenant: str, defn: dict) -> list[dict]: scope = schedule_scope(tenant, defn.get("id")) specs: list[dict] = [] @@ -131,7 +147,7 @@ def recurring_specs(tenant: str, defn: dict) -> list[dict]: if schedule.get("enabled"): for index, expression in enumerate(aws_cron_expressions(schedule.get("cron") or "")): specs.append({"name": f"{scope}-schedule-{index}", "mode": "schedule", - "expression": expression, "timezone": "UTC", + "expression": expression, "timezone": _recurring_timezone(defn), "job": _base_job(tenant, defn, "schedule")}) trigger = defn.get("trigger") or {} if (trigger.get("key") == "email" and trigger.get("enabled", True) diff --git a/api/automation_engine.py b/api/automation_engine.py index 896ee84422dc7c1ccd6e875276ee3cdfbaf28b32..f54777c885b7fbc8a3bc0e2fefc3386ac86f48a9 100644 --- a/api/automation_engine.py +++ b/api/automation_engine.py @@ -35,6 +35,7 @@ import socket import threading import time from urllib.parse import urljoin, urlparse +from zoneinfo import ZoneInfo, ZoneInfoNotFoundError import requests @@ -1481,8 +1482,14 @@ def field_def(text_key, label, ftype="text", **extra): """ description = " ".join(str(extra.pop( "description", IG_FIELD_DESCRIPTIONS.get(text_key, "")) or "").split()) - out = {"key": text_key, "label": label, "type": ftype, "source": "overlay", - "editRole": "admins"} + # W44 C3: machine-spawned columns must declare the same full field contract emitted by the + # user-table validator. A generic engine field is source-owned; the two derived kinds are + # computed. Keeping this in the one constructor keeps every preset family exact under + # `_clean_field` instead of letting a new contract key drift across hundreds of declarations. + value_authority = "computed" if ftype in {"formula", "rollup"} else "source" + out = {"key": text_key, "label": "Table" if ftype == "json" else label, + "type": ftype, "source": "overlay", "kind": ftype, "origin": "automation", + "valueAuthority": value_authority, "scope": "tenant", "editRole": "admins"} if description: out["description"] = description out.update(extra) @@ -1664,14 +1671,19 @@ def clean_config(kind, raw, previous=None): revision = str(snap.get("sourceViewRevision") or "").strip()[:128] recipient = str(raw.get("recipient") or "").strip()[:120] timezone = str(snap.get("timezone") or raw.get("timezone") or "UTC").strip()[:80] + try: + ZoneInfo(timezone) + except (ZoneInfoNotFoundError, ValueError): + return None, "a scheduled route needs a recognized IANA timezone" source_name = " ".join(str(snap.get("sourceViewName") or "").split())[:120] + route_id = str(snap.get("routeId") or "").strip()[:96] route_field = str(snap.get("routeField") or "").strip()[:120] try: links_per_map = int(snap.get("mapsPerLink", raw.get("mapsPerLink", 10))) except (TypeError, ValueError): links_per_map = 10 # R5: Google directions requires at least an origin and destination. The map UI exposes - # the same 2–10 bound; this server normalisation is the wall for hand-written bodies too. + # the same 2-10 bound; this server normalisation is the wall for hand-written bodies too. links_per_map = max(2, min(10, links_per_map)) ordered = [] seen = set() @@ -1694,6 +1706,7 @@ def clean_config(kind, raw, previous=None): return { "recipient": recipient, "routeSnapshot": { + "routeId": route_id, "sourceViewId": source_view, "sourceViewRevision": revision, "sourceViewName": source_name or source_view, @@ -2334,6 +2347,10 @@ def clean_definition(raw, previous=None, username="", notes=None, rt=None): else prev.get("schedule"), prev.get("schedule")) except ValueError as e: return None, str(e) + if kind == "route_delivery": + # Native cron interprets this wall clock in the exact validated Zone frozen beside the + # Route. Ordinary automations intentionally retain their historical UTC default. + schedule["timezone"] = config["routeSnapshot"]["timezone"] # (`clean_trigger` ran at the top — law 1 needs the trigger before the kind.) flow_raw = raw.get("flow") if "flow" in raw else prev.get("flow") # ⭐ EVERY CLEAN, NOT ONLY THE EDGE (owner ruling 6 — see `_ensure_ig_action`). The edge-only @@ -9259,6 +9276,7 @@ WEB_REQUIRED = { #: run-time refusal rather than becoming a store read on the automations LIST path (W30-T12 took #: that read off this route; putting it back to draw a label would undo the wave before this one). ACTION_REQUIRED = dict(WEB_REQUIRED) +ACTION_REQUIRED["voice_agent"] = (("a Voice agent", "voiceAgentId"),) #: ⭐ W31-T38 (C5) — the web step's own clamps. `20000` matches the default E's seam documents; #: the ceiling exists because this blocks the record walk (~9-32 s of cold start ALREADY), and an #: automation that can be configured to wait ten minutes per record is a stalled run, not a slow one. @@ -9268,7 +9286,7 @@ WEB_READ_TIMEOUT_MAX_MS = 120000 #: and never by a literal list of group names: a client-side ordering is a second copy of this #: table, and the way it fails is that a group added here renders last, or not at all, with #: nothing red anywhere. -ACTION_GROUP_ORDER = {"Web action": 1, "Database": 2, "Connected": 3, "Advanced logic": 4} +ACTION_GROUP_ORDER = {"Agent": 1, "Database": 2, "Connected": 3, "Advanced logic": 4} ACTION_CATALOG = [ # ── 1. WEB ACTION (owner rulings R1-R5) — DECLARED HERE, NOT YET BUILT (DEBT D-51). ──────── # ⛔ NOT A TEASE. `clean_actions` refuses an unready kind with a sentence, so the faded row @@ -9326,16 +9344,19 @@ ACTION_CATALOG = [ # hidden row's label is still rendered on a STORED step's card, so leaving the six captions in # place would have kept them on screen for exactly the people who already use them. They now # name the mechanism instead, and no old caption survives as a substring in any case. - {"kind": "ai_agent", "label": "Web agent", "group": "Web action", "ready": True, + {"kind": "ai_agent", "label": "Web agent", "group": "Agent", "ready": True, "detail": "Describe a job on a website in words; the agent works out the steps, runs them in " "a browser and reports what it actually did"}, - {"kind": "web_goto", "label": "Web agent step (navigate)", "group": "Web action", + {"kind": "voice_agent", "label": "Voice agent", "group": "Agent", "ready": True, + "detail": "Use a configured Loopable Voice agent. Production calls remain unavailable; " + "the no-dial harness is the only executable Voice path"}, + {"kind": "web_goto", "label": "Web agent step (navigate)", "group": "Agent", "ready": True, "menu": False, "detail": "Opens a page in a browser job and reports the title it landed on"}, - {"kind": "web_fill", "label": "Web agent step (type)", "group": "Web action", + {"kind": "web_fill", "label": "Web agent step (type)", "group": "Agent", "ready": True, "menu": False, "detail": "Types a value into a field. Mark it secret and the value is masked in the log"}, - {"kind": "web_click", "label": "Web agent step (click)", "group": "Web action", + {"kind": "web_click", "label": "Web agent step (click)", "group": "Agent", "ready": True, "menu": False, "detail": "Clicks the element a selector names, and reports where it landed"}, # ⚠ `"kind": "web_read"` AND `"ready": True` STAY ON ONE LINE. `verify_wiring`'s C5 row matches @@ -9345,9 +9366,9 @@ ACTION_CATALOG = [ # the client can add must be one the runner will execute); the layout is what it happens to # depend on, so the layout is preserved here rather than the assertion weakened there. {"kind": "web_read", "ready": True, "label": "Web agent step (extract)", - "group": "Web action", "menu": False, + "group": "Agent", "menu": False, "detail": "Reads one value off a live page in a browser job. Expect ~10-30 s per step"}, - {"kind": "web_repair", "label": "Web agent step (relocate)", "group": "Web action", + {"kind": "web_repair", "label": "Web agent step (relocate)", "group": "Agent", "ready": True, "menu": False, "detail": "Follows a label to its control when a selector has gone stale, and proposes one"}, # ── 2. DATABASE ────────────────────────────────────────────────────────────────────────── @@ -9432,7 +9453,7 @@ ACTION_CATALOG = [ "ready": True, "menu": False, "detail": "Pull the connected Odoo databases on a schedule. Configured on the connector, not " "here"}, - {"kind": "ai_enrich", "label": "Enrich this column with AI", "group": "Connected", + {"kind": "ai_enrich", "label": "AI field agent", "group": "Agent", "ready": True, "menu": False, "detail": "Fill an AI column for the records this flow walks. Configured on the column, not " "here"}, @@ -10027,6 +10048,15 @@ def _clean_action_config(kind, cfg, depth, seen, count, notes=None, rt=None): if limit < 1 or limit > FIND_LIMIT_MAX: return None, f"the find records limit is between 1 and {FIND_LIMIT_MAX}" return {"table": table, "cond": cond, "limit": limit}, None + if kind == "voice_agent": + # Voice configuration lives in the dedicated, permissioned control plane. The action + # stores one Loopable-local reference only: never a provider id, phone number, SIP route, + # or credential. An empty draft is addable and is then named Unconfigured by + # ACTION_REQUIRED; the runner arm below still refuses every ordinary dispatch. + voice_id = _s(cfg.get("voiceAgentId"), 120).strip() + if voice_id and not re.fullmatch(r"va_[A-Za-z0-9_\-]{4,110}", voice_id): + return None, "select a Voice agent from Loopable rather than entering a provider id" + return {"voiceAgentId": voice_id}, None if kind == "ai_agent": # ⭐⭐ W33-T56 — THE FUZZY STEP'S CONFIG. Same posture as the web arm below in every # respect that matters: an ALLOWLIST, a shape test on the url, a clamped timeout, and an @@ -11101,6 +11131,19 @@ def apply_actions(rt, defn, table_key, row_ids, username="automation", log=print "editor, not from this canvas. Nothing was done") counts["webBlocked"] += 1 continue + elif kind == "voice_agent": + # Voice has a dedicated no-dial harness and no ordinary automation dispatcher. + # Keeping the refusal in the walk is load-bearing: this dispatcher has no final + # unknown-kind else, so a catalog row without an arm would otherwise report a + # successful run that silently did nothing. A future activation must replace this + # wall only after consent, capacity lease, callback route, webhook verification, + # provider tests, and an explicit owner authorization all exist. + if "voice_agent_not_activated" not in web_notes: + web_notes.append("voice_agent_not_activated") + log("[aios-auto] voice_agent: production Voice dispatch is not activated. " + "Use the no-dial harness; no call was created") + counts["webBlocked"] += 1 + continue elif kind == "ai_agent": # ⭐⭐ W33-T56 (owner item 7, ruling R3) — THE FUZZY STEP, AT RUN TIME. # diff --git a/api/routes_admin.py b/api/routes_admin.py index 46cb67f9c80538a0d194b0eeba6f3d2b09ca9eb7..d7e4c45f9dc66bdc2f8a552ab0109b4816b7c7db 100644 --- a/api/routes_admin.py +++ b/api/routes_admin.py @@ -843,7 +843,7 @@ def _module_fields(key, session=None): if isinstance(_uf, dict) and _uf.get("key") and _uf["key"] not in _have: _have.add(_uf["key"]) src.append(_uf) - known = _server_side_vocabularies() + known = _server_side_vocabularies(session=session) out = [] for f in src: if not isinstance(f, dict) or not f.get("key"): @@ -1019,7 +1019,7 @@ BLANK = "(none)" _VOCAB_CACHE = None -def _server_side_vocabularies(): +def _server_side_vocabularies(session=None): """`{field key: [choice, …]}` for choice columns whose vocabulary is CLOSED and known here, but which the grid contract does not declare. @@ -1090,8 +1090,36 @@ def _server_side_vocabularies(): it. Recorded because the ask, not a guess, is what settled it. """ global _VOCAB_CACHE + + def _product_shared_choices(): + """Tenant-owned options for the editable Product taxonomy columns. + + These nine fields are ordinary shared Product fields. Their vocabulary is admitted by + the same atomic cell+choice write that stores an edit, so the permission editor must read + the tenant's shared definition rather than inventing a global list or loading the whole + Product pool. A missing runtime honestly supplies nothing; a value from another tenant + must never be borrowed from the process cache below. + """ + if session is None: + return {} + import core.shared_overlay as shared_overlay + import modules.product_data as product_data + + stored = shared_overlay.fields(product_data.TABLE_KEY, st=session.runtime) or {} + keys = ("collection", "color", "finish", "main_category", "material", + "occassion", "shape_style", "size", "sub_category") + return {key: (stored.get(key) or {}).get("options") or () for key in keys} + if _VOCAB_CACHE is not None: - return dict(_VOCAB_CACHE) + out = dict(_VOCAB_CACHE) + try: + for key, values in _product_shared_choices().items(): + clean = sorted({str(v) for v in values if str(v or "").strip()}) + if clean: + out[key] = clean + except Exception: # noqa: BLE001 + pass + return out out = {} @@ -1199,6 +1227,10 @@ def _server_side_vocabularies(): _agents, _salespeople)]) if complete: _VOCAB_CACHE = dict(out) + # Tenant-owned choices are deliberately added AFTER the process cache is stamped. Caching + # them beside Odoo's shared Royal vocabulary would leak one tenant's Product taxonomy into + # another tenant's permission editor for the life of the worker. + arm(_product_shared_choices) return out @@ -2029,7 +2061,14 @@ def _store_binding(session): import core.store as store # noqa: PLC0415 bound = getattr(session.runtime, "store_handle", None) repo = getattr(bound, "repo", None) or store.REPO - return data_binding.describe(repo) + result = data_binding.describe(repo) + if result.get("backend") == "pg": + import core.store_pg as store_pg # noqa: PLC0415 + slug = getattr(bound, "tenant_slug", None) or session.tenant + refusal = store_pg.write_refusal(slug) + result.update(writable=refusal is None, refusal=refusal, + allowlist=sorted(store_pg.sandbox_tenants())) + return result except Exception as e: # noqa: BLE001 return {"deployment": "unknown", "production": False, "repo": "", "writable": False, "refusal": f"could not resolve the store binding: {e}", diff --git a/api/routes_automation.py b/api/routes_automation.py index 78c12961b6b0907cf7e33ce9de452a209ac68eb4..927e72ba076ef18038ce6bf885658c84029a5053 100644 --- a/api/routes_automation.py +++ b/api/routes_automation.py @@ -19,6 +19,7 @@ from fastapi import APIRouter, Body, Depends, Header, Request import ai_review import automation_control import automation_engine as engine +import voice_control_plane as voice import oauth_connect import routes_oauth import scope_cache @@ -53,6 +54,186 @@ MODULE = "automation" _GATE = module_gate(MODULE) +def _voice_view_revision(view): + """One stable revision string for a server-owned saved View choice.""" + if not isinstance(view, dict): + return "" + cfg = view.get("config") if isinstance(view.get("config"), dict) else {} + for key in ("revision", "version", "updatedAt", "updated_at"): + value = view.get(key, cfg.get(key)) + if value not in (None, ""): + return str(value)[:120] + return "" + + +def _voice_option(key, label, fields, views): + phone = [{"key": str(field.get("key") or ""), + "label": str(field.get("label") or field.get("key") or "")} + for field in (fields or []) if isinstance(field, dict) + and field.get("key") and field.get("type") == "phone"] + saved = [{"id": str(view.get("id") or ""), + "label": str(view.get("name") or view.get("label") or view.get("id") or ""), + "revision": _voice_view_revision(view)} + for view in (views or []) if isinstance(view, dict) and view.get("id")] + if not phone or not saved: + return None + return {"key": str(key), "label": str(label), "views": saved, "phoneFields": phone} + + +def _voice_options(session): + """Compose only databases, saved Views, and phone fields this session may already open.""" + tables = [] + # Built-in customer data has its own module wall. An Automation grant is not permission to + # enumerate its schema, so failure simply withholds this choice. + try: + session.require("customer_data") + from routes_customers import grid_assembly + customer = grid_assembly(session, consume_corrections=False) + option = _voice_option("customer_data", "Customers", customer.get("fields"), + customer.get("views")) + if option: + tables.append(option) + except Exception: # noqa: BLE001 - permission/absence. + pass + # User tables are permissioned by their own object wall inside automation_tables(). + for table in automation_tables(session).get("tables") or []: + option = _voice_option(table.get("key"), table.get("label") or table.get("key"), + table.get("fields"), table.get("views")) + if option: + tables.append(option) + return {"tables": tables} + + +def _voice_rows_from_user_table(session, table_key): + import aios_grid + from routes_tables import ut_assembly + + g = ut_assembly(session, table_key, consume_corrections=False, with_rows=True) + overlays = (g.get("ws") or {}).get("overlays") or {} + merged = {} + for row in g.get("rows_src") or []: + pid = str(row.get("pid")) + cells = dict(overlays.get(pid) or overlays.get(row.get("pid")) or {}) + shared = (g.get("shared_cells") or {}).get(pid) + if isinstance(shared, dict): + cells.update(shared) + merged[pid] = cells + rows = aios_grid.rows_from_pool(g.get("rows_src") or [], g.get("fields") or [], merged, + derived=g.get("derived") or {}) + return g, rows + + +def _voice_rows_from_customers(session): + import aios_grid + from core import route_system + from routes_customers import grid_assembly + + session.require("customer_data") + g = grid_assembly(session, consume_corrections=False) + rows = aios_grid.rows_from_pool(g["rows_src"], g["fields"], g["ws"].get("overlays"), + derived=g["derived"]) + return g, route_system.apply_customer_projections(rows, g["routes"], g["zones"]) + + +def _voice_view_input(session, audience): + """Resolve an authorized saved View to redacted recipient counts, never phone values.""" + from harness import filter_eval + + table_key = str((audience or {}).get("tableKey") or "") + if table_key == "customer_data": + g, rows = _voice_rows_from_customers(session) + elif table_key.startswith("ut_"): + g, rows = _voice_rows_from_user_table(session, table_key) + else: + raise voice.VoiceRefusal("select a database available to this account") + view_id = str((audience or {}).get("viewId") or "") + phone_key = str((audience or {}).get("phoneFieldKey") or "") + view = next((row for row in (g.get("views") or []) if isinstance(row, dict) + and str(row.get("id") or "") == view_id), None) + if not isinstance(view, dict): + raise voice.VoiceRefusal("select a saved View you are authorized to open") + revision = _voice_view_revision(view) + requested_revision = str((audience or {}).get("viewRevision") or "") + if requested_revision and revision and requested_revision != revision: + raise voice.VoiceRefusal("that saved View changed; review the audience before running") + field = next((row for row in (g.get("fields") or []) if isinstance(row, dict) + and str(row.get("key") or "") == phone_key), None) + if not isinstance(field, dict) or field.get("type") != "phone": + raise voice.VoiceRefusal("select a phone-type field") + cfg = dict(view.get("config") or {}) + cohorts = {str(item.get("id")): set(item.get("pids") or []) + for item in (g.get("lists") or []) if isinstance(item, dict) and item.get("id")} + context = filter_eval.EvalCtx(cohort_sets=cohorts, + measure_sets=g.get("measure_sets") or {}, + today=g.get("today") or time.strftime("%Y-%m-%d")) + tree = {"nodes": cfg.get("filters") or [], "conj": cfg.get("filterConj") or "and"} + pids = set(filter_eval.visible_pids(tree, rows, g.get("fields") or [], context, + member_pids=cfg.get("memberPids") or [], + group_by=cfg.get("groupBy"))) + selected = [row for row in rows if row.get("pid") in pids] + eligible = [row for row in selected if row.get(phone_key) not in (None, "")] + return {"tableKey": table_key, "viewId": view_id, "viewRevision": revision, + "phoneFieldKey": phone_key, "selectedCount": len(selected), + "recipientCount": len(eligible), "rawPhoneValuesReturned": False} + + +@router.get("/voice-agents") +def list_voice_agents(session: Session = Depends(_GATE)): + options = _voice_options(session) + rows = voice.list_browser_dtos(session.runtime) + for row in rows: + row["audienceOptions"] = options + return {"voiceAgents": rows} + + +@router.get("/voice-agents/options") +def voice_agent_options(session: Session = Depends(_GATE)): + return {**voice.browser_options(session.runtime), "audienceOptions": _voice_options(session)} + + +@router.post("/voice-agents") +def create_voice_agent(body: dict = Body(default=None), session: Session = Depends(_GATE)): + try: + saved = voice.save_draft(session.runtime, body or {}, session.uname) + row = voice.get_browser_dto(session.runtime, saved["id"]) + row["audienceOptions"] = _voice_options(session) + return {"voiceAgent": row} + except voice.VoiceRefusal as exc: + raise err(400, "voice_refused", str(exc)) + + +@router.patch("/voice-agents/{voice_id}") +def patch_voice_agent(voice_id: str, body: dict = Body(default=None), session: Session = Depends(_GATE)): + try: + voice.patch_draft(session.runtime, voice_id, body or {}, session.uname) + row = voice.get_browser_dto(session.runtime, voice_id) + row["audienceOptions"] = _voice_options(session) + return {"voiceAgent": row} + except voice.VoiceRefusal as exc: + raise err(400 if "unknown" not in str(exc) else 404, "voice_refused", str(exc)) + + +@router.post("/voice-agents/{voice_id}/harness/dry-run") +def voice_dry_run(voice_id: str, body: dict = Body(default=None), session: Session = Depends(_GATE)): + body = body or {} + try: + if isinstance(body.get("audience"), dict): + audience = body.get("audience") + else: + audience = {} + for source, target in (("table", "tableKey"), ("viewId", "viewId"), + ("viewRevision", "viewRevision"), + ("phoneField", "phoneFieldKey"), + ("emptyPolicy", "emptyPolicy"), + ("maxRecipients", "maxRecipients")): + if source in body: + audience[target] = body.get(source) + return voice.dry_run(session.runtime, voice_id, audience, + lambda selected: _voice_view_input(session, selected)) + except voice.VoiceRefusal as exc: + raise err(400 if "unknown" not in str(exc) else 404, "voice_refused", str(exc)) + + def _sync_schedule_or_503(tenant, defn): """Apply the exact native schedules after a definition write, never silently drift.""" try: @@ -1037,7 +1218,8 @@ def _table_views(session, key): return None merged = {**shared, **own} # a view lives in ONE home; the merge is belt-and-braces return sorted( - ({"id": str(vid), "label": str((v or {}).get("name") or vid)} + ({"id": str(vid), "label": str((v or {}).get("name") or vid), + "revision": _voice_view_revision(v)} for vid, v in merged.items() if isinstance(v, dict)), key=lambda r: r["label"].lower()) diff --git a/api/routes_customers.py b/api/routes_customers.py index c437d7abe9a10ece80a0301972c01b10dac7d33d..1db214db9671a852ac675554fd4c3beaa8757938 100644 --- a/api/routes_customers.py +++ b/api/routes_customers.py @@ -190,7 +190,11 @@ def shared_fields(st=None): try: field_permissions.migrate_legacy_fields( _shared_key(), st=st, grant_topic="customer_data", shared_key=_shared_key()) - return shared_overlay.fields(_shared_key(), st=st) + fields = shared_overlay.fields(_shared_key(), st=st) + # Normalized Routes use an invisible field-shaped bridge so the established Share dialog + # and grant registry can be reused. It is object metadata, never a customer column. + return {key: value for key, value in fields.items() + if not _is_route_share_bridge(value)} except Exception: # noqa: BLE001 # Lenient like every other display read: an unreachable store degrades to "nothing is # shared yet", never to a 500 on a grid that would otherwise render. The WALL does not @@ -198,6 +202,11 @@ def shared_fields(st=None): return {} +def _is_route_share_bridge(definition): + route = definition.get("route") if isinstance(definition, dict) else None + return bool(isinstance(route, dict) and route.get("normalizedId")) + + def shared_cells(pids, st=None): """`{"": {key: value}}` for the rows named by `pids`, and ONLY those. @@ -213,7 +222,14 @@ def shared_cells(pids, st=None): return {} -def _merge_shared_fields(fields, defs, session=None): +def _canonical_field_keys(): + """Keys declared by the product contract, not by a user's workspace stratum.""" + import aios_grid + return {str(field.get("key") or "").strip() + for field in (aios_grid.FIELDS or ()) if isinstance(field, dict)} + + +def _merge_shared_fields(fields, defs, session=None, strata=None): """`fields` PLUS the tenant-wide columns this topic declares — `routes_tables._ut_shared_fields` on the customer topic. @@ -225,12 +241,33 @@ def _merge_shared_fields(fields, defs, session=None): ⚠ A key the canonical contract already declares WINS. A shared column is an ADDITION to this database's contract, never a redefinition of a column it already has. """ + incoming = list(fields or ()) + canonical = _canonical_field_keys() + lent = strata if isinstance(strata, dict) else None + have = {} + for index, field in enumerate(incoming): + if not isinstance(field, dict): + continue + key = str(field.get("key") or "").strip() + if not key or key in have: + continue + have[key] = index + if lent is not None: + lent[key] = "contract" if key in canonical else "personal" + if not defs: return fields - have = {f.get("key") for f in (fields or ()) if isinstance(f, dict)} projected = [] for k, f in defs.items(): - if k in have: + if _is_route_share_bridge(f): + continue + k = str(k or "").strip() + if not k: + continue + index = have.get(k) + # A tenant-wide definition may never redefine a built-in contract field. For any other + # same-key collision (including a saved route), the shared definition wins in place. + if index is not None and k in canonical: continue item = dict(f, source="overlay", shared=True) if session is not None: @@ -240,8 +277,29 @@ def _merge_shared_fields(fields, defs, session=None): is_admin=session.admin, st=session.runtime) if role: item["sharedRole"] = role - projected.append(item) - return list(fields or ()) + projected + if index is None: + projected.append(item) + else: + local = incoming[index] if isinstance(incoming[index], dict) else {} + item["superseded"] = { + "stratum": "personal", + "owner": str(local.get("createdBy") or "").strip() or None, + "label": str(local.get("label") or "").strip() or None, + } + incoming[index] = item + if lent is not None: + lent[k] = "shared" + return incoming + projected + + +def _mark_offerability(fields): + """Withhold stored route-order fields from pickers without deleting them from the payload.""" + return [dict(field, **{OFFERABLE_KEY: False}) + if (isinstance(field, dict) + and field.get("kind") == ROUTE_KIND + and str(field.get("key") or "").strip() != ROUTE_PROJECTION_KEY) + else field + for field in (fields or ())] def grid_assembly(session: Session, scope: str = "customer", storage_key: str = "", @@ -305,9 +363,10 @@ def grid_assembly(session: Session, scope: str = "customer", storage_key: str = # dict on one request (the write ctx's wall, the read merge, and the closure), and letting each # take its own copy is the shape D-214 spent a whole ticket removing one document over. _defs = shared_fields(st=rt) + _workspace_ctx = _ctx_for(session, pids, defs=_defs) ws = grid_events.table_workspace( - _ctx_for(session, pids, defs=_defs), allowed_pids=pids, - consume_corrections=consume_corrections) + _workspace_ctx, allowed_pids=pids, + consume_corrections=consume_corrections, capture_document=True) # ⭐⭐ W38-T20 / D-425 — THE TENANT-WIDE CELLS, LAYERED OVER THE PER-USER ONES, IN THE # ASSEMBLY SO EVERY CONSUMER SEES ONE TRUTH. `routes_grid`'s /workspace route serves # `workspace["overlays"] = g["ws"].get("overlays")` verbatim and `_payload` hands the same @@ -333,7 +392,18 @@ def grid_assembly(session: Session, scope: str = "customer", storage_key: str = # ⭐⭐ W38-T20 — AND THE COLUMN DEFINITIONS, BEFORE THE WALL. See `_merge_shared_fields`: this # position is load-bearing twice, once for the transitive closure and once because it is the # only order in which the per-field grant marker is ever presented to `hidden_keys`. - fields = _merge_shared_fields(fields, _defs, session=session) + _strata = {} + fields = _merge_shared_fields(fields, _defs, session=session, strata=_strata) + # Legacy `Destination N` definitions remain in the shared document for rollback/audit, but + # once the dedicated migration recorded its independent parity proof they are display-hidden. + # This is intentionally a projection only: no field or cell is deleted here. + from core import route_system + _legacy_migrations = route_system.read(rt).get("legacyMigrations") or {} + fields = [field for field in fields if not ( + isinstance(field, dict) and (_legacy_migrations.get(str(field.get("key"))) or {}).get("verified"))] + # Presentation only: legacy route-order definitions remain served/manageable where W44 has + # not migrated them, but no generic field picker may offer one as a data dimension. + fields = _mark_offerability(fields) # ⭐⭐ W41-T01 / RULING R5 / CONTRACTS C1 + C8 — **THE THREE BADGES, STAMPED ONCE, HERE.** # # C1 makes `field_permissions.field_class` the ONE producer of `{origin, audience, sharedBy, @@ -377,7 +447,8 @@ def grid_assembly(session: Session, scope: str = "customer", storage_key: str = try: from core import field_permissions as _fp _classes = _fp.field_classes(fields, session.uname, table_key=_shared_key(), - grant_topic=SHARE_TOPIC, values_shared=set(_defs), st=rt) + grant_topic=SHARE_TOPIC, values_shared=set(_defs), + strata=_strata, st=rt) fields = [dict(f, **{"class": _classes[f["key"]]}) if isinstance(f, dict) and _classes.get(f.get("key")) else f for f in fields] @@ -399,6 +470,22 @@ def grid_assembly(session: Session, scope: str = "customer", storage_key: str = # makes this degrade safe: a client renders NOTHING for a missing `class` rather than a # wrong badge, so a registry that will not open costs the badges and not the grid. pass + # C4 parity: the topic grid reuses the same whole-workspace usage producer as user databases. + # The captured document avoids another store read; MODULE remains the memo/permission topic, + # while `_shared_key()` remains the independently named workspace document. + try: + from routes_tables import _field_usage as _fu + _usage_source = getattr(_workspace_ctx, "workspace_document", None) + if not isinstance(_usage_source, dict): + _usage_source = rt + _usage = _fu(session, MODULE, fields, None, + _usage_source, _shared_key(), st=rt) + except Exception: # noqa: BLE001 + _usage = {} + if _usage: + fields = [dict(f, usage=dict(_usage[f["key"]])) + if isinstance(f, dict) and _usage.get(f.get("key")) else f + for f in fields] # THE FIELD WALL — a TRANSITIVE closure (C-PERM amendment 5), so hiding a field also hides # every formula computed FROM it. Formulas evaluate in the browser from `{ref}`s, so # shipping a dependent formula while withholding its input either leaks the input through @@ -474,14 +561,19 @@ def grid_assembly(session: Session, scope: str = "customer", storage_key: str = on_error=_measure_err) if may_metrics else {}).items(): derived.setdefault(pid, {}).update(cells) - # Zones are table-scoped tenant data, surfaced as the established comma-joined multiselect - # contract so saved Views and FilterBuilder evaluate them like every other membership field. - from core import user_tables as _zones - fields = _zones.with_zone_field("customer_data", fields, st=rt) + # Route stops and Zone membership are normalized system relations. They project into two + # read-only customer fields; neither is a generic Link or an editable user-table cell. + # Route names and membership are object data too: a private Route must not leak through the + # derived customer column while its navigator correctly hides it. + normalized_routes = [route for route, _mark in _visible_normalized_routes(session)] + normalized_zones = route_system.list_zones(rt, rows_src) + fields = [field for field in fields if field.get("key") not in {"routes", "zone"}] + fields.extend(route_system.customer_projection_fields(normalized_routes, normalized_zones)) + fields = [route_system.field_axes(field) for field in fields] return {"rows_src": rows_src, "pids": pids, "ws": ws, "workspace": workspace, "fields": fields, "views": views, "lists": lists, "derived": derived, "measures": measures, "measure_sets": measure_sets, "today": today, - "team_id": team_id} + "team_id": team_id, "routes": normalized_routes, "zones": normalized_zones} def _payload(session: Session): @@ -501,9 +593,8 @@ def _payload(session: Session): g = grid_assembly(session) rows = aios_grid.rows_from_pool( g["rows_src"], g["fields"], g["ws"].get("overlays"), derived=g["derived"]) - from core import user_tables as _zones - rows = _zones.zone_cells("customer_data", rows, allowed_row_ids=g["pids"], - st=session.runtime) + from core import route_system + rows = route_system.apply_customer_projections(rows, g["routes"], g["zones"]) # ⭐ C4 / D-138 (W30-T37) — THE DOCUMENTS PRODUCER FOR THE CUSTOMER SCOPE. The write door # (`doc_add`/`doc_fetch`/`doc_delete`) never stopped working and every client half is # complete; what vanished with `app.py` at EXIT-6 was the only thing that ever set this key. @@ -604,6 +695,673 @@ def customers(session: Session = Depends(module_gate(MODULE))): return _payload(session) +# ── Wave 44: normalized, Loopable-owned Route/Zone APIs ───────────────────── +# These are deliberately ahead of `/customers/{pid}`. Route stops are a protected relation; +# no generic Link field, generic table writer, or legacy Destination column is a write door. +def _route_view_can_edit(session, view_id): + """Route mutations inherit the source View editor wall, never an invented role.""" + if not view_id: + return bool(session.admin) + try: + import core.table_store as table_store + ws = _customer_table(session).workspace(session.uname, consume_corrections=False) + view = (ws.get("views") or {}).get(str(view_id)) + return bool(view and table_store._may_edit(view, session.uname, session.admin)) + except Exception: # unavailable store is fail-closed + return False + + +def _route_input_pids(raw): + offered = [] + for stop in (raw or {}).get("stops") or (): + candidate = stop.get("customerPid") if isinstance(stop, dict) else stop + if isinstance(candidate, bool): + raise err(400, "invalid_route_stop", "a route stop needs a customer id") + try: + offered.append(int(candidate)) + except (TypeError, ValueError): + raise err(400, "invalid_route_stop", "a route stop needs a customer id") + return offered + + +def _reject_client_create_id(raw, subject): + """Keep POST as create-only; object identities are minted by the server. + + ``route_system.put_*`` intentionally remains an upsert primitive for PATCH and migrations. + Passing any browser-supplied ``id`` key from POST into that primitive would turn the create + door into an undocumented update door, including across owners. Key presence is the wall: + null, false, zero, blank, whitespace, arrays, and objects are all refused. Refuse loudly + instead of silently stripping the id so a malformed caller learns to use PATCH. + """ + if isinstance(raw, dict) and "id" in raw: + slug = str(subject or "object").strip().lower().replace(" ", "_") or "object" + raise err(400, f"server_managed_{slug}_id", + f"{subject} ids are server-managed; omit id when creating and use PATCH " + f"to update an existing {subject}") + + +def _legacy_route_defs_readonly(session): + """Visible legacy route definitions from the shared stratum, with no reconciliation write.""" + import aios_grid + import core.perm_scope as perm_scope + from core import route_system, shared_overlay + + try: + all_defs = shared_overlay.fields(_shared_key(), st=session.runtime) or {} + except Exception: + all_defs = {} + migrated = route_system.read(session.runtime).get("legacyMigrations") or {} + defs = {key: definition for key, definition in all_defs.items() + if isinstance(definition, dict) and definition.get("kind") == ROUTE_KIND + and not _is_route_share_bridge(definition) + and not bool((migrated.get(key) or {}).get("verified"))} + if not defs: + return {} + hidden = perm_scope.hidden_keys( + session.user, MODULE, _merge_shared_fields(list(aios_grid.FIELDS), all_defs), + st=session.runtime) + return {key: definition for key, definition in defs.items() if key not in hidden} + + +def _route_share_records(session): + """One normalized registry read for list visibility and every share mark.""" + from core import field_permissions + return field_permissions.grant_records(SHARE_TOPIC, st=session.runtime) or {} + + +def _route_share_mark(session, route, records=None): + """C6's exact owner/shared/shared-out projection from the generic field registry.""" + from core import route_system, shares + records = records if isinstance(records, dict) else _route_share_records(session) + key = str(route.get("shareKey") or route_system.route_share_key(route.get("id"))) + rec = records.get(key) or {} + entries = rec.get("entries") or [] + me = str(session.uname or "").strip().lower() + owner = (str(rec.get("owner") or "").strip().lower() + or str(route.get("createdBy") or "").strip().lower()) + role = None + if me and owner != me: + for entry in entries: + if str(entry.get("user") or "").lower() not in (me, shares.EVERYONE): + continue + if entry.get("role") == "edit": + role = "edit" + break + role = role or "view" + mine = bool(me and owner == me) + return {"shareId": f"customer_data:{key}", "owner": owner, + "shared": role is not None, "sharedOut": bool(mine and entries), + "hasShares": bool(entries), "sharedRole": role, + "visible": bool(session.admin or mine or role)} + + +def _visible_normalized_routes(session, records=None): + from core import route_system + records = records if isinstance(records, dict) else _route_share_records(session) + visible = [] + for route in route_system.list_routes(session.runtime): + mark = _route_share_mark(session, route, records) + if mark["visible"]: + visible.append((route, mark)) + return visible + + +def _compatible_routes(session, permitted=None, records=None): + """Visible normalized Routes plus a read-only projection of legacy route-order fields. + + Wave 44 introduced a normalized Route store while production already held saved route-order + columns. Requiring an administrator to run the migration before a GET made those existing + Routes disappear from the Map. This projection uses the same field and row walls as the old + route-order picker and performs no store update; the explicit migration remains a separate, + independently verified write operation. + """ + from core import route_system + permitted = allowed_pids(session) if permitted is None else frozenset(permitted) + normalized = [route for route, _mark in _visible_normalized_routes(session, records)] + return route_system.compatible_routes( + normalized, + _legacy_route_defs_readonly(session), + shared_cells(permitted, st=session.runtime), + tenant_id=session.tenant, + created_by=session.uname, + ) + + +def _legacy_route_navigation(session): + """Wave 42 per-reader Route folders, used only as a no-write migration fallback.""" + try: + ws = _customer_table(session).workspace(session.uname, consume_corrections=False) + except Exception: + return [], {} + folders = ws.get("folders") or {} + return (list(folders.get("routes") or []) if isinstance(folders, dict) else [], + dict((ws.get("itemFolders") or {}).get("routes") or {})) + + +def _route_navigation(session, routes): + from core import route_system + folders, placements = _legacy_route_navigation(session) + return route_system.route_navigation( + session.runtime, session.uname, routes, + legacy_folders=folders, legacy_placements=placements) + + +def _route_dto(route, allowed, active, session, share_mark=None): + from core import route_system + allowed = {int(pid) for pid in allowed} + stops = [stop for stop in route_system.ordered_stops(route) + if int(stop.get("customerPid")) in allowed] + visible = [int(stop["customerPid"]) for stop in stops] + active = {int(pid) for pid in active if int(pid) in allowed} + compatibility = route.get("compatibility") + mark = share_mark if isinstance(share_mark, dict) else _route_share_mark(session, route) + owns_route = bool(session.admin or ( + mark.get("owner") and mark.get("owner") == str(session.uname or "").strip().lower())) + dto = {"id": route["id"], "name": route["name"], "color": route["color"], + "number": route["number"], "visibleStopIds": visible, + "outsideFilter": bool(active) and not bool(active.intersection(visible)), + "permission": {"canEdit": (False if compatibility else owns_route and + _route_view_can_edit(session, route.get("sourceViewId"))), + "sourceViewId": route.get("sourceViewId")}, + "source": {"viewId": route.get("sourceViewId"), + "revision": route.get("sourceViewRevision")}, + "owner": mark["owner"], "shared": mark["shared"], + "sharedOut": mark["sharedOut"], "hasShares": mark["hasShares"], + "sharedRole": mark["sharedRole"], "shareId": mark["shareId"], + "folderId": route.get("folderId"), "navOrder": route.get("navOrder", 0)} + if compatibility == "legacy-route-order": + dto["compatibility"] = compatibility + if route.get("mapsUrl"): + dto["mapsUrl"] = route["mapsUrl"] + return dto + + +def _query_pids(raw): + values = [] + for value in str(raw or "").split(","): + try: + values.append(int(value)) + except (TypeError, ValueError): + continue + return values + + +def _route_or_404(session, route_id): + route = next((item for item, _mark in _visible_normalized_routes(session) + if item.get("id") == str(route_id)), None) + if route is None: + raise err(404, "route_not_found", "that Route does not exist") + return route + + +def _route_mutation_allowed(session, route, requested_source=None): + owner = str(_route_share_mark(session, route).get("owner") or "").strip().lower() + if not (session.admin or (owner and owner == str(session.uname or "").strip().lower())): + raise err(403, "route_not_owner", "only the Route owner or an administrator can change it") + source = requested_source if requested_source is not None else route.get("sourceViewId") + if not _route_view_can_edit(session, source): + raise err(403, "route_view_read_only", "only an editor of the Route source View can change it") + + +def _ensure_route_share_bridge(session, route): + """Materialize the invisible field-shaped object consumed by the existing Share dialog.""" + from core import route_system, shared_overlay + key = str(route.get("shareKey") or route_system.route_share_key(route.get("id"))) + shared_overlay.put_field(_shared_key(), key, { + "key": key, "label": route.get("name") or "Route", "type": "text", + "source": "overlay", "kind": ROUTE_KIND, "custom": True, + "createdBy": route.get("createdBy") or session.uname, + "note": "Access bridge for a saved Route.", "shared": True, "granted": True, + "route": {"normalizedId": route.get("id"), "routeObject": True}, + }, st=session.runtime) + return key + + +def _drop_route_share_bridge(session, route): + from core import route_system, shared_overlay, shares + key = str(route.get("shareKey") or route_system.route_share_key(route.get("id"))) + shared_overlay.drop_field(_shared_key(), key, st=session.runtime) + shares.drop_objects([("field", shares.field_oid(SHARE_TOPIC, key))], st=session.runtime) + + +def _route_nav_mutation_context(session): + routes = _compatible_routes(session) + folders, placements = _legacy_route_navigation(session) + return routes, folders, placements + + +@router.get("/customers/routes") +def routes(visiblePids: str = "", currentPids: str = "", + session: Session = Depends(module_gate(MODULE))): + """Route picker DTO consumed by C1/C3; only permitted stops ever leave the server.""" + allowed = allowed_pids(session) + active = _query_pids(currentPids or visiblePids) + records = _route_share_records(session) + navigation = _route_navigation(session, _compatible_routes(session, allowed, records)) + return {"routes": [_route_dto( + route, allowed, active, session, + _route_share_mark(session, route, records)) + for route in navigation["routes"]], + "folders": navigation["folders"]} + + +@router.post("/customers/routes", status_code=201) +def create_route(body: dict = Body(default=None), session: Session = Depends(module_gate(MODULE))): + from core import route_system + raw = dict(body or {}) + _reject_client_create_id(raw, "Route") + if not _route_view_can_edit(session, raw.get("sourceViewId")): + raise err(403, "route_view_read_only", "a Route requires an editable source View") + permitted = allowed_pids(session) + if any(pid not in permitted for pid in _route_input_pids(raw)): + raise err(403, "out_of_scope", "a Route stop must be in your customer book") + route = route_system.put_route(session.runtime, raw, tenant_id=session.tenant, created_by=session.uname) + if route is None: + raise err(400, "invalid_route", "a Route needs a name and an optional HTTPS Google Maps URL") + _ensure_route_share_bridge(session, route) + return {"route": _route_dto(route, permitted, (), session)} + + +@router.post("/customers/routes/{route_id}/duplicate", status_code=201) +def duplicate_route(route_id: str, body: dict = Body(default=None), + session: Session = Depends(module_gate(MODULE))): + from core import route_system + prior = _route_or_404(session, route_id) + if not _route_view_can_edit(session, prior.get("sourceViewId")): + raise err(403, "route_view_read_only", "an editable source View is required to duplicate this Route") + route = route_system.duplicate_route( + session.runtime, route_id, tenant_id=session.tenant, created_by=session.uname, + name=(body or {}).get("name")) + if route is None: + raise err(404, "route_not_found", "that Route does not exist") + _ensure_route_share_bridge(session, route) + return {"route": _route_dto(route, allowed_pids(session), (), session)} + + +@router.post("/customers/routes/{route_id}/share-ready") +def route_share_ready(route_id: str, session: Session = Depends(module_gate(MODULE))): + route = _route_or_404(session, route_id) + _route_mutation_allowed(session, route) + key = _ensure_route_share_bridge(session, route) + return {"kind": "field", "id": f"customer_data:{key}", "label": route.get("name")} + + +@router.patch("/customers/routes/{route_id}") +def update_route(route_id: str, body: dict = Body(default=None), + session: Session = Depends(module_gate(MODULE))): + from core import route_system + prior = _route_or_404(session, route_id) + raw = dict(body or {}) + _route_mutation_allowed(session, prior, raw.get("sourceViewId", prior.get("sourceViewId"))) + raw["id"] = prior["id"] + if "stops" in raw and any(pid not in allowed_pids(session) for pid in _route_input_pids(raw)): + raise err(403, "out_of_scope", "a Route stop must be in your customer book") + route = route_system.put_route(session.runtime, raw, tenant_id=session.tenant, created_by=session.uname) + if route is None: + raise err(400, "invalid_route", "a Route needs a name and an optional HTTPS Google Maps URL") + _ensure_route_share_bridge(session, route) + return {"route": _route_dto(route, allowed_pids(session), (), session)} + + +@router.post("/customers/routes/{route_id}/stops/move") +def move_route_stop(route_id: str, body: dict = Body(default=None), + session: Session = Depends(module_gate(MODULE))): + from core import route_system + prior = _route_or_404(session, route_id) + _route_mutation_allowed(session, prior) + raw = dict(body or {}) + try: + pid, order = int(raw.get("customerPid")), int(raw.get("order")) + except (TypeError, ValueError): + raise err(400, "invalid_route_stop", "customerPid and order are required") + if pid not in allowed_pids(session): + raise err(403, "out_of_scope", "that customer is not in your book") + route = route_system.move_stop(session.runtime, route_id, pid, order, + tenant_id=session.tenant, created_by=session.uname) + if route is None: + # A Route may intentionally have no stops. We never turn that harmless state into a + # synthetic generic Link row; the caller can render the honest no-stop state. + raise err(409, "no_route_stop", "that customer is not a stop on this Route") + return {"route": _route_dto(route, allowed_pids(session), (), session)} + + +@router.post("/customers/routes/{route_id}/stops/membership") +def set_route_stop_membership(route_id: str, body: dict = Body(default=None), + session: Session = Depends(module_gate(MODULE))): + """Add or remove one permitted customer while preserving every other ordered stop.""" + from core import route_system + prior = _route_or_404(session, route_id) + _route_mutation_allowed(session, prior) + raw = dict(body or {}) + candidate = raw.get("customerPid") + present = raw.get("present") + if isinstance(candidate, bool) or not isinstance(present, bool): + raise err(400, "invalid_route_membership", + "customerPid and a true or false present value are required") + try: + pid = int(candidate) + except (TypeError, ValueError): + raise err(400, "invalid_route_membership", + "customerPid and a true or false present value are required") + permitted = allowed_pids(session) + if pid not in permitted: + raise err(403, "out_of_scope", "that customer is not in your book") + route = route_system.set_stop_membership( + session.runtime, route_id, pid, present, + tenant_id=session.tenant, created_by=session.uname) + if route is None: + raise err(404, "route_not_found", "that Route does not exist") + return {"route": _route_dto(route, permitted, (), session)} + + +@router.delete("/customers/routes/{route_id}") +def remove_route(route_id: str, session: Session = Depends(module_gate(MODULE))): + from core import route_system + route = _route_or_404(session, route_id) + _route_mutation_allowed(session, route) + linked_delivery_ids = _linked_route_delivery_ids(session.runtime, route_id) + if linked_delivery_ids: + visible_delivery_ids = { + str(delivery.get("id") or "") + for delivery in _route_delivery_definitions(session, route_id) + if isinstance(delivery, dict) + } + if not session.admin and not set(linked_delivery_ids).issubset(visible_delivery_ids): + raise err(409, "route_deliveries_exist", + "Ask an administrator to remove this Route's scheduled deliveries, " + "then delete the Route") + raise err(409, "route_deliveries_exist", + "Remove this Route's scheduled deliveries first, then delete the Route") + if not route_system.delete_route(session.runtime, route_id): + raise err(404, "route_not_found", "that Route does not exist") + _drop_route_share_bridge(session, route) + return {"deleted": True} + + +@router.patch("/customers/routes/{route_id}/folder") +def move_route_to_folder(route_id: str, body: dict = Body(default=None), + session: Session = Depends(module_gate(MODULE))): + from core import route_system + routes, legacy_folders, legacy_placements = _route_nav_mutation_context(session) + if str(route_id) not in {str(route.get("id")) for route in routes}: + raise err(404, "route_not_found", "that Route does not exist") + folder_id = (body or {}).get("folderId") + if folder_id is not None and not isinstance(folder_id, str): + raise err(400, "invalid_route_folder", "folderId must be a folder id or null") + if folder_id and not any(folder.get("id") == folder_id for folder in _route_navigation(session, routes)["folders"]): + raise err(409, "route_folder_not_found", "that Route folder no longer exists") + route_system.place_route( + session.runtime, session.uname, route_id, folder_id, routes, + legacy_folders=legacy_folders, legacy_placements=legacy_placements) + navigation = _route_navigation(session, routes) + moved = next(route for route in navigation["routes"] if route.get("id") == str(route_id)) + return {"route": _route_dto(moved, allowed_pids(session), (), session), + "folders": navigation["folders"]} + + +@router.post("/customers/routes/reorder") +def reorder_route_navigation(body: dict = Body(default=None), + session: Session = Depends(module_gate(MODULE))): + from core import route_system + routes, legacy_folders, legacy_placements = _route_nav_mutation_context(session) + order = (body or {}).get("order") + if not isinstance(order, list): + raise err(400, "invalid_route_order", "order must be a list of visible Route ids") + result = route_system.reorder_routes( + session.runtime, session.uname, order, routes, + legacy_folders=legacy_folders, legacy_placements=legacy_placements) + if result is None: + raise err(409, "invalid_route_order", "Route order contains an unknown or duplicate id") + return {"order": result} + + +@router.post("/customers/route-folders", status_code=201) +def create_route_folder(body: dict = Body(default=None), + session: Session = Depends(module_gate(MODULE))): + from core import route_system + routes, legacy_folders, legacy_placements = _route_nav_mutation_context(session) + folder = route_system.put_route_folder( + session.runtime, session.uname, body or {}, routes, + legacy_folders=legacy_folders, legacy_placements=legacy_placements) + if folder is None: + raise err(400, "invalid_route_folder", "a Route folder needs a name and a valid parent") + return {"folder": folder, "folders": _route_navigation(session, routes)["folders"]} + + +@router.patch("/customers/route-folders/{folder_id}") +def update_route_folder(folder_id: str, body: dict = Body(default=None), + session: Session = Depends(module_gate(MODULE))): + from core import route_system + routes, legacy_folders, legacy_placements = _route_nav_mutation_context(session) + current = next((folder for folder in _route_navigation(session, routes)["folders"] + if folder.get("id") == folder_id), None) + if current is None: + raise err(404, "route_folder_not_found", "that Route folder does not exist") + raw = dict(current) + raw.update(body or {}) + raw["id"] = folder_id + folder = route_system.put_route_folder( + session.runtime, session.uname, raw, routes, + legacy_folders=legacy_folders, legacy_placements=legacy_placements) + if folder is None: + raise err(409, "invalid_route_folder", "that folder move would create an invalid path") + return {"folder": folder, "folders": _route_navigation(session, routes)["folders"]} + + +@router.delete("/customers/route-folders/{folder_id}") +def remove_route_folder(folder_id: str, session: Session = Depends(module_gate(MODULE))): + from core import route_system + routes, legacy_folders, legacy_placements = _route_nav_mutation_context(session) + if not route_system.delete_route_folder( + session.runtime, session.uname, folder_id, routes, + legacy_folders=legacy_folders, legacy_placements=legacy_placements): + raise err(404, "route_folder_not_found", "that Route folder does not exist") + return {"deleted": True, "folders": _route_navigation(session, routes)["folders"]} + + +@router.post("/customers/route-folders/reorder") +def reorder_route_folder_navigation(body: dict = Body(default=None), + session: Session = Depends(module_gate(MODULE))): + from core import route_system + routes, legacy_folders, legacy_placements = _route_nav_mutation_context(session) + order = (body or {}).get("order") + if not isinstance(order, list): + raise err(400, "invalid_route_folder_order", "order must be a list of Route folder ids") + result = route_system.reorder_route_folders( + session.runtime, session.uname, order, routes, + legacy_folders=legacy_folders, legacy_placements=legacy_placements) + if result is None: + raise err(409, "invalid_route_folder_order", "folder order contains an unknown or duplicate id") + return {"order": result} + + +@router.post("/customers/routes/selection") +def route_selection(body: dict = Body(default=None), session: Session = Depends(module_gate(MODULE))): + """Dedicated managed-OR mutation; user filter clauses are returned byte-for-byte intact.""" + from core import route_system + raw = dict(body or {}) + chosen = [str(item) for item in raw.get("routeIds") or ()] + compatible = _compatible_routes(session) + known = {route["id"] for route in compatible} + if any(route_id not in known for route_id in chosen): + raise err(404, "route_not_found", "a selected Route does not exist") + state = route_system.apply_managed_route_selection( + raw.get("viewState"), chosen, compatible) + return {"viewState": state, "managedBy": route_system.MANAGED_BY} + + +def _export_sort_value(row, field): + value = row.get(field.get("key")) + if field.get("type") in {"currency", "int", "pct", "rating", "formula"}: + try: + return (value is None or value == "", float(value)) + except (TypeError, ValueError): + return (True, 0.0) + return (value is None or value == "", str(value or "").lower()) + + +@router.post("/customers/view-export") +def selected_view_export(body: dict = Body(default=None), + session: Session = Depends(module_gate(MODULE))): + """Authoritative selected-View export data; the browser owns XLSX rendering, not selection. + + The request names saved View ids only. Its field order/visibility, filter, sort and cohort + membership are resolved independently per tab, so the caller cannot accidentally duplicate + the active View's rows under several View names. + """ + import aios_grid + from core import route_system + from harness import filter_eval + + raw_ids = (body or {}).get("viewIds") + if not isinstance(raw_ids, list) or not raw_ids or len(raw_ids) > 20: + raise err(400, "invalid_view_ids", "viewIds must name between one and twenty visible Views") + ids = [str(value).strip()[:120] for value in raw_ids] + if any(not value for value in ids) or len(set(ids)) != len(ids): + raise err(400, "invalid_view_ids", "viewIds must be unique non-empty View ids") + g = grid_assembly(session) + views = {str(view.get("id")): view for view in g["views"] if isinstance(view, dict)} + if any(view_id not in views for view_id in ids): + raise err(404, "view_not_found", "an export View is not visible to this account") + rows = aios_grid.rows_from_pool(g["rows_src"], g["fields"], g["ws"].get("overlays"), + derived=g["derived"]) + rows = route_system.apply_customer_projections(rows, g["routes"], g["zones"]) + by_key = {field["key"]: field for field in g["fields"] if isinstance(field, dict) and field.get("key")} + cohort_sets = {str(item.get("id")): set(item.get("pids") or []) + for item in g["lists"] if isinstance(item, dict) and item.get("id")} + exports = [] + for view_id in ids: + view, cfg = views[view_id], dict(views[view_id].get("config") or {}) + context = filter_eval.EvalCtx(cohort_sets=cohort_sets, measure_sets=g["measure_sets"], + today=g["today"]) + tree = {"nodes": cfg.get("filters") or [], "conj": cfg.get("filterConj") or "and"} + pids = set(filter_eval.visible_pids(tree, rows, g["fields"], context, + member_pids=cfg.get("memberPids") or [], + group_by=cfg.get("groupBy"))) + selected = [dict(row) for row in rows if row.get("pid") in pids] + for sort in reversed(cfg.get("sorts") or []): + field = by_key.get(sort.get("colId")) if isinstance(sort, dict) else None + if field is not None: + selected.sort(key=lambda row, f=field: _export_sort_value(row, f), + reverse=sort.get("dir") == "desc") + ordered = [key for key in cfg.get("order") or [] if key in by_key] + ordered += [key for key in by_key if key not in ordered] + visible = set(cfg.get("visible") or ordered) + columns = [dict(by_key[key]) for key in ordered if key in visible] + exports.append({"viewId": view_id, "name": view.get("name") or view_id, + "columns": columns, "rows": selected, + "view": {"filters": cfg.get("filters") or [], + "filterConj": cfg.get("filterConj") or "and", + "sorts": cfg.get("sorts") or [], + "memberPids": cfg.get("memberPids") or []}}) + return {"today": g["today"], "exports": exports} + + +@router.post("/customers/routes/migrate-legacy") +def migrate_legacy_destination_fields(session: Session = Depends(module_gate(MODULE))): + """Copy legacy Destination N ranks, prove parity, then hide the retained columns. + + The explicit whole-book check is important: a partial manager scope may copy a useful subset, + but it can never prove a tenant-wide route complete and therefore may not hide anything. + """ + if not session.admin: + raise err(403, "legacy_migration_admin_only", "only an administrator may migrate legacy Routes") + from core import route_system + raw_rows = _pool_rows(session) + whole_book = {int(row["pid"]) for row in raw_rows if row.get("pid") is not None} + if whole_book != set(allowed_pids(session)): + raise err(409, "legacy_migration_scope_limited", + "legacy Route migration needs an unrestricted customer scope to prove parity") + defs = shared_fields(st=session.runtime) + result = route_system.migrate_legacy_routes( + session.runtime, defs, shared_cells(whole_book, st=session.runtime), + tenant_id=session.tenant, created_by=session.uname, + zones=route_system.list_zones(session.runtime, raw_rows)) + # `legacyMigrations` lives with the normalized system data. The source field definition and + # every scalar rank cell are retained byte-for-byte; the persisted proof is the read-side + # hide/read-only marker rather than a generic field rewrite or a delete. + return result + + +def _zone_can_edit(session, source_view_id): + return bool(session.admin) or _route_view_can_edit(session, source_view_id) + + +def _zone_dto(zone, allowed, session): + return {"id": zone["id"], "name": zone["name"], "color": zone["color"], + "geometryGeoJson": zone.get("geometryGeoJson"), "zipCodes": zone.get("zipCodes") or [], + # MapView and the read-only Customer ``Zone`` field consume this same normalized + # membership. Returning a second ``visibleMemberIds`` dialect made the map paint an + # empty Zone while the grid projection named the customer as a member. + "memberships": [int(pid) for pid in zone.get("memberPids") or [] if int(pid) in allowed], + "membershipRevision": zone.get("membershipRevision", 0), + "sourceViewId": zone.get("sourceViewId"), + "geographicField": str(zone.get("geographicField") or ""), + "createdBy": str(zone.get("createdBy") or ""), + "createdAt": str(zone.get("createdAt") or ""), + "visible": zone.get("visible", True), + "permission": { + "canEdit": _zone_can_edit(session, zone.get("sourceViewId")), + "canDelete": bool(session.admin), + }} + + +@router.get("/customers/zones") +def zones(session: Session = Depends(module_gate(MODULE))): + from core import route_system + rows = _pool_rows(session) + allowed = allowed_pids(session) + return {"zones": [_zone_dto(zone, allowed, session) + for zone in route_system.list_zones(session.runtime, rows)]} + + +@router.post("/customers/zones", status_code=201) +def create_zone(body: dict = Body(default=None), session: Session = Depends(module_gate(MODULE))): + from core import route_system + raw = dict(body or {}) + _reject_client_create_id(raw, "Zone") + if not _zone_can_edit(session, raw.get("sourceViewId")): + raise err(403, "zone_view_read_only", "a Zone requires an editable source View") + zone = route_system.put_zone(session.runtime, raw, rows=_pool_rows(session), + tenant_id=session.tenant, created_by=session.uname) + if zone is None: + raise err(400, "invalid_zone", "a Zone needs a name and a polygon or ZIP code") + return {"zone": _zone_dto(zone, allowed_pids(session), session)} + + +@router.patch("/customers/zones/{zone_id}") +def update_zone(zone_id: str, body: dict = Body(default=None), + session: Session = Depends(module_gate(MODULE))): + from core import route_system + prior = next((zone for zone in route_system.list_zones(session.runtime) + if zone.get("id") == str(zone_id)), None) + if prior is None: + raise err(404, "zone_not_found", "that Zone does not exist") + raw = dict(body or {}) + # A Zone keeps the View it was authored from. Editing its colour or boundary is not a + # second door for silently rebasing its permission wall onto another View. + source_view_id = prior.get("sourceViewId") or raw.get("sourceViewId") + if not _zone_can_edit(session, source_view_id): + raise err(403, "zone_view_read_only", "a Zone requires an editable source View") + raw["sourceViewId"] = source_view_id + raw["id"] = prior["id"] + zone = route_system.put_zone(session.runtime, raw, rows=_pool_rows(session), + tenant_id=session.tenant, created_by=session.uname) + if zone is None: + raise err(400, "invalid_zone", "a Zone needs a name and a polygon or ZIP code") + return {"zone": _zone_dto(zone, allowed_pids(session), session)} + + +@router.delete("/customers/zones/{zone_id}") +def remove_zone(zone_id: str, session: Session = Depends(module_gate(MODULE))): + from core import route_system + if not session.admin: + raise err(403, "zone_view_read_only", "only an administrator can delete a Zone") + if not route_system.delete_zone(session.runtime, zone_id): + raise err(404, "zone_not_found", "that Zone does not exist") + return {"deleted": True} + + def _split_route_updates(session, updates): """Split a cell PATCH into `(route order updates, everything else)`. @@ -614,7 +1372,9 @@ def _split_route_updates(session, updates): route, rest = {}, {} for key, value in (updates or {}).items(): defn = defs.get(key) - if isinstance(defn, dict) and defn.get("kind") == ROUTE_KIND: + from core import route_system + migrated = bool((route_system.read(session.runtime).get("legacyMigrations") or {}).get(key, {}).get("verified")) + if isinstance(defn, dict) and defn.get("kind") == ROUTE_KIND and not migrated: route[key] = value else: rest[key] = value @@ -652,9 +1412,10 @@ def _patch_route_ranks(session, ctx, pid, pool, updates): `patch_customer` routes everything else through `grid_events` rather than writing the store. ⚠ UNIQUENESS IS MAINTAINED OVER THE ROWS THIS CALLER CAN SEE, and that is a limit worth - stating rather than hiding. `shared_overlay.cells` takes the scoped pool BY SIGNATURE (there is - deliberately no "every shared cell in the tenant" call), so the holder of a displaced number is - looked for inside this session's book. In practice the whole column lives there anyway -- + stating rather than hiding. `shared_overlay.transpose_rank` requires the scoped pool BY + SIGNATURE, then reads it inside the same store-update callback that writes both rows. The + holder of a displaced number is therefore looked for inside this session's book, but never + from a stale pre-write snapshot. In practice the whole column lives there anyway: `route_order_write` REFUSES a body carrying a pid outside the writer's pool, so every number in a route order was written by somebody whose book contained all of them. """ @@ -670,49 +1431,30 @@ def _patch_route_ranks(session, ctx, pid, pool, updates): #: value the caller did NOT ask for. taken, swaps = set(), [] - def _held(): - """`{pid: rank}` for this column, over the rows this caller can see. + def _stored(key): + """The current rank for a refused key, scoped to this one admitted row. - Re-read per key rather than hoisted, because a multi-key patch writes between iterations - and the second key must see the first one's result. + Valid nonblank assignments never use this read: ``transpose_rank`` reads the whole scoped + column inside the atomic update that writes it. This helper exists only so a refusal can + repaint the store's truth in the optimistic client. """ - out = {} - for raw_pid, cells in (shared_cells(pool, st=session.runtime) or {}).items(): - if not isinstance(cells, dict): - continue - cell = cells.get(key) - if cell in (None, ""): - continue - try: - out[int(raw_pid)] = int(str(cell).strip()) - except (TypeError, ValueError): - continue - return out + row = ((shared_cells([pid], st=session.runtime) or {}).get(str(pid)) or {}) + cell = row.get(key) if isinstance(row, dict) else None + if cell in (None, ""): + return "" + try: + return str(int(str(cell).strip())) + except (TypeError, ValueError): + return "" for key, raw in updates.items(): defn = defs.get(key) or {} - held = _held() - - def _stored(): - """⛔⛔ A REFUSAL REPORTS WHAT THE CELL ACTUALLY HOLDS, AND THAT IS NOT TIDINESS. - - `patchTopicRow` adopts `body.updates` into the browser's optimistic copy and rolls - back only on a non-2xx. A refused key that is simply ABSENT from `updates` is a key - the client never hears about: the response is 200, the rollback never fires, `adopt` - has nothing to write, and the typed text goes on painting a value the store rejected - until something unrelated forces a refetch. On a `select` the option normaliser made - this unreachable; item 7 made the column an `int`, so `14.5` or a stray letter is now - an ordinary typo away. `patchTopicRow`'s own docstring already states the rule — - keep only what the server actually took — and this is that rule with its hole closed. - """ - n = held.get(pid) - return "" if n is None else str(n) if key in hidden: - accepted[key] = _stored() + accepted[key] = _stored(key) continue if not grid_events._may_edit_field_value(ctx, key, defn): - accepted[key] = _stored() + accepted[key] = _stored(key) continue # An empty cell is "not on this route". Legal, and the only way to take a stop off the # day without re-solving the whole column. @@ -724,30 +1466,22 @@ def _patch_route_ranks(session, ctx, pid, pool, updates): # ⛔ A BOOL IS AN `int` IN PYTHON, so `True` would store as visit number 1. Excluded by # name here exactly as `route_order_write` excludes it, rather than by hoping. if isinstance(raw, bool): - accepted[key] = _stored() + accepted[key] = _stored(key) continue try: want = int(str(raw).strip()) except (TypeError, ValueError): - accepted[key] = _stored() + accepted[key] = _stored(key) continue if want < 1: - accepted[key] = _stored() + accepted[key] = _stored(key) continue - prior = held.get(pid) - if prior == want: - accepted[key] = str(want) - taken.add(key) - continue - writes = {pid: {key: str(want)}} - holder = next((p for p, n in held.items() if n == want and p != pid), None) - if holder is not None: - moved = prior if prior is not None else max(held.values()) + 1 - writes[holder] = {key: str(moved)} - swaps.append({"pid": holder, "field": key, "value": str(moved)}) - shared_overlay.put_rows(_shared_key(), writes, st=session.runtime) - accepted[key] = str(want) + moved = shared_overlay.transpose_rank( + _shared_key(), pid, key, want, pool, st=session.runtime) + if moved.get("swap"): + swaps.append(moved["swap"]) + accepted[key] = moved["value"] taken.add(key) return accepted, swaps, taken @@ -768,6 +1502,12 @@ def patch_customer(pid: int, body: dict = Body(default=None), updates = dict(body or {}) if not updates: raise err(400, "empty_patch", "no fields to update") + from core import route_system + retired = {str(key) for key, proof in (route_system.read(session.runtime).get("legacyMigrations") or {}).items() + if isinstance(proof, dict) and proof.get("verified")} + if retired.intersection(updates): + raise err(403, "legacy_route_hidden", + "this legacy Destination field was migrated; edit the named Route instead") pool = allowed_pids(session) if pid not in pool: # 403, not 404: the pid may well exist — it is simply not in this session's book, and @@ -875,6 +1615,8 @@ ROUTE_KEY_PREFIX = "route_" #: the client; never inferred from the key, because a prefix is a naming convention and a #: convention is not a declaration. ROUTE_KIND = "route_order" +OFFERABLE_KEY = "offerable" +ROUTE_PROJECTION_KEY = "routes" #: ⭐⭐ W41-T05 / CONTRACT C3 — THE DESCRIPTION EVERY ROUTE COLUMN IS BORN WITH. #: @@ -991,8 +1733,11 @@ def _route_defs(session: Session): import core.perm_scope as perm_scope all_defs = shared_fields(st=session.runtime) or {} + from core import route_system + migrated = route_system.read(session.runtime).get("legacyMigrations") or {} defs = {k: v for k, v in all_defs.items() - if isinstance(v, dict) and v.get("kind") == ROUTE_KIND} + if isinstance(v, dict) and v.get("kind") == ROUTE_KIND + and not bool((migrated.get(k) or {}).get("verified"))} if not defs: return {} hide = perm_scope.hidden_keys( @@ -1254,6 +1999,9 @@ def route_order_write(body: dict = Body(default=None), "source": "overlay", "shared": True, "kind": ROUTE_KIND, + # Stored as well as projected so every read door starts from the same explicit contract. + # This is presentation metadata: the route stays visible/manageable and its values remain. + OFFERABLE_KEY: False, # ⭐⭐ W41-T05 — THE DESCRIPTION, CARRIED BY **PRESENCE** AND NOT BY TRUTHINESS. # `ROUTE_FIELD_NOTE` says why the column cannot be born without one and why `""` has to # survive a re-solve: `route_order_rename` clears a description by storing exactly that, @@ -1585,6 +2333,7 @@ def _route_delivery_wire(defn): "recipient": str(cfg.get("recipient") or ""), "schedule": dict(defn.get("schedule") or {}), "snapshot": { + "routeId": str(snap.get("routeId") or ""), "sourceViewId": str(snap.get("sourceViewId") or ""), "sourceViewRevision": str(snap.get("sourceViewRevision") or ""), "sourceViewName": str(snap.get("sourceViewName") or ""), @@ -1598,11 +2347,19 @@ def _route_delivery_wire(defn): } -def _route_delivery_recipient_or_400(raw): +def _route_delivery_recipient_or_400(session, raw, directory=None): + """Resolve one active Inbox account behind the narrow directory seam. + + ``session`` is intentionally part of the seam even though the ordinary registry does not + consume it yet. Staging-only identities can extend this one helper without leaking fixture + logic into Route persistence or falling through to a different account plane. + """ recipient = str(raw or "").strip() try: - import core.users as users - user = (users.registry() or {}).get(recipient) + if directory is None: + import core.users as users + directory = users.registry + user = (directory() or {}).get(recipient) except Exception: # registry unavailable is not permission to queue a phantom Inbox delivery user = None if not recipient or not isinstance(user, dict) or not user.get("active", True): @@ -1624,89 +2381,141 @@ def _route_view_revision(view_id, config): return "view-config-v2-" + hashlib.sha256(canonical.encode("utf-8")).hexdigest()[:24] -def _route_snapshot_for_view_or_409(snapshot, source_view): - """Accept a route only when the caller's View config is the View visible at this request. +def _route_delivery_snapshot_for_route_or_400( + session, route, body, *, source_views=None, permitted=None, recipient_directory=None): + """Build the immutable delivery exclusively from a normalized Route and server reads. - Ordered stops remain the actual frozen work list. The source View is its separately auditable - provenance, so a forged revision must never be stored beside an otherwise valid View id. - Copy only canonical server facts forward; the submitted config is a compare-and-discard - concurrency guard, not another persisted configuration copy. + The browser may choose the recipient, timezone, and bounded Maps chunk size. It cannot send + View configuration, revision, Route name, or stop ids: all of those are resolved again here + from the saved Route and its real source View at the moment the schedule is created. """ - source_view_id = str(source_view.get("id") or "").strip() - source_config = source_view.get("config") - submitted_config = snapshot.get("sourceViewConfig") - if not source_view_id or not isinstance(source_config, dict): - raise err(409, "stale_route_source_view", - "the saved route View changed; refresh it before scheduling this route") - if not isinstance(submitted_config, dict) or submitted_config != source_config: - raise err(409, "stale_route_source_view", - "the saved route View changed; refresh it before scheduling this route") - return { - **snapshot, + from core import route_system + from zoneinfo import ZoneInfo, ZoneInfoNotFoundError + + body = body if isinstance(body, dict) else {} + recipient = _route_delivery_recipient_or_400( + session, body.get("recipient"), directory=recipient_directory) + source_view_id = str(route.get("sourceViewId") or "").strip() + if not source_view_id: + raise err(409, "route_source_view_missing", + "this Route has no saved source View and cannot be scheduled") + + if source_views is None: + source_views = grid_assembly(session, consume_corrections=False).get("views") or [] + source_by_id = {str(view.get("id") or ""): view for view in source_views + if isinstance(view, dict) and view.get("id")} + source_view = source_by_id.get(source_view_id) + source_config = source_view.get("config") if isinstance(source_view, dict) else None + if not isinstance(source_config, dict): + raise err(409, "route_source_view_unavailable", + "the Route source View is no longer visible; restore it before scheduling") + + ordered_stop_ids = [int(stop["customerPid"]) for stop in route_system.ordered_stops(route)] + if len(ordered_stop_ids) < 2: + raise err(400, "route_delivery_needs_stops", + "add at least two ordered stops before scheduling this Route") + permitted = allowed_pids(session) if permitted is None else frozenset(int(pid) for pid in permitted) + if any(pid not in permitted for pid in ordered_stop_ids): + raise err(409, "route_delivery_scope_changed", + "one or more Route stops are outside your current customer book; update the Route first") + + timezone = str(body.get("timezone") or "UTC").strip()[:80] or "UTC" + try: + ZoneInfo(timezone) + except (ZoneInfoNotFoundError, ValueError): + raise err(400, "invalid_route_timezone", "choose a recognized timezone for this delivery") + + raw_maps_per_link = body.get("mapsPerLink", 10) + if isinstance(raw_maps_per_link, bool): + raise err(400, "invalid_route_maps_per_link", "Maps links must contain between 2 and 10 stops") + try: + maps_per_link = int(raw_maps_per_link) + except (TypeError, ValueError): + raise err(400, "invalid_route_maps_per_link", "Maps links must contain between 2 and 10 stops") + if maps_per_link < 2 or maps_per_link > 10: + raise err(400, "invalid_route_maps_per_link", "Maps links must contain between 2 and 10 stops") + + return recipient, { + "routeId": str(route.get("id") or ""), "sourceViewId": source_view_id, "sourceViewRevision": _route_view_revision(source_view_id, source_config), "sourceViewName": str(source_view.get("name") or source_view_id), + "timezone": timezone, + "routeField": str(route.get("id") or ""), + "orderedStopIds": ordered_stop_ids, + "mapsPerLink": maps_per_link, } -@router.get("/customers/route-deliveries") -def route_delivery_list(session: Session = Depends(module_gate(MODULE))): - """Saved C2 route deliveries owned by the caller (or all for an administrator).""" +def _route_delivery_definitions(session, route_id=None): import automation_engine as engine out = [] for defn in (engine.all_definitions(session.runtime) or {}).values(): if not isinstance(defn, dict) or defn.get("kind") != "route_delivery": continue - # A recipient can see the delivery arriving in their Inbox; the edit index is the owner's - # configuration surface. Admins can support either one. if not session.admin and str(defn.get("createdBy") or "") != session.uname: continue - out.append(_route_delivery_wire(defn)) - return {"deliveries": sorted(out, key=lambda row: (row["name"].casefold(), row["id"]))} + wire = _route_delivery_wire(defn) + if route_id is not None and wire["snapshot"]["routeId"] != str(route_id): + continue + out.append(wire) + return sorted(out, key=lambda row: (row["name"].casefold(), row["id"])) -@router.post("/customers/route-deliveries") -def route_delivery_create(body: dict = Body(default=None), - session: Session = Depends(module_gate(MODULE))): - """Save a daily route delivery as an immutable C2 snapshot and arm its native schedule.""" +def _linked_route_delivery_ids(runtime, route_id): + """Return every tenant delivery linked to a Route, regardless of its creator. + + The user-facing delivery list is intentionally filtered by caller. A Route delete guard is a + referential-integrity question instead: a collaborator-owned delivery is still a live link and + must block deletion just like an owner-created one. + """ + import automation_engine as engine + + wanted = str(route_id or "") + linked = [] + for definition_id, defn in (engine.all_definitions(runtime) or {}).items(): + if not isinstance(defn, dict) or defn.get("kind") != "route_delivery": + continue + snapshot = ((defn.get("config") or {}).get("routeSnapshot") + if isinstance(defn.get("config"), dict) else None) + if not isinstance(snapshot, dict) or str(snapshot.get("routeId") or "") != wanted: + continue + linked.append(str(defn.get("id") or definition_id)) + return sorted(set(linked)) + + +@router.get("/customers/route-deliveries") +def route_delivery_list(session: Session = Depends(module_gate(MODULE))): + """Saved C2 route deliveries owned by the caller (or all for an administrator).""" + return {"deliveries": _route_delivery_definitions(session)} + + +@router.get("/customers/routes/{route_id}/deliveries") +def normalized_route_delivery_list(route_id: str, + session: Session = Depends(module_gate(MODULE))): + """Deliveries for the one normalized Route whose detail is open.""" + _route_or_404(session, route_id) + return {"deliveries": _route_delivery_definitions(session, route_id)} + + +def _create_route_delivery_for_route(session, route, body): + """Persist and arm one server-authored snapshot; never execute its runner here.""" import automation_control import automation_engine as engine if not session.runtime.available(): raise err(503, "store_unavailable", "the tenant store is unavailable. Nothing was saved") - body = body if isinstance(body, dict) else {} - recipient = _route_delivery_recipient_or_400(body.get("recipient")) - snapshot = body.get("routeSnapshot") if isinstance(body.get("routeSnapshot"), dict) else {} - # A route snapshot is allowed to freeze its *stops*, never to invent its View provenance. - # `map-current`/a route-plan fingerprint were a synthetic fallback that made an automation - # impossible to audit back to the actual filtered View. Resolve the submitted id against - # this reader's visible View contract, compare the complete config, and mint its revision - # server-side; a valid View id with a forged/stale revision is still not an honest snapshot. - source_view_id = str(snapshot.get("sourceViewId") or "").strip() - if not source_view_id or source_view_id == "map-current": - raise err(400, "invalid_route_source_view", - "choose an active saved View for this frozen route") - source_views = {str(view.get("id") or ""): view for view in - (grid_assembly(session, consume_corrections=False).get("views") or []) - if isinstance(view, dict) and view.get("id")} - source_view = source_views.get(source_view_id) - if source_view is None: - raise err(400, "unknown_route_source_view", - "the route source View is no longer visible to this account") - snapshot = _route_snapshot_for_view_or_409(snapshot, source_view) + _route_mutation_allowed(session, route) + recipient, snapshot = _route_delivery_snapshot_for_route_or_400(session, route, body) + route_name = " ".join(str(route.get("name") or "Route").split())[:120] or "Route" raw = { - "name": " ".join(str(body.get("name") or snapshot.get("sourceViewName") or "Scheduled route").split())[:120], + "name": f"{route_name} delivery"[:120], "kind": "route_delivery", "trigger": {"key": "manual"}, - "schedule": body.get("schedule") if isinstance(body.get("schedule"), dict) - else {"cron": "0 8 * * *", "enabled": True}, - "config": { - "recipient": recipient, - "routeSnapshot": { - **snapshot, - "mapsPerLink": snapshot.get("mapsPerLink", body.get("mapsPerLink", 10)), - }, - }, + # The normalized Route UI promises one deliberately boring cadence. A browser cannot + # smuggle a different cron or an immediate run into this creation request. + "schedule": {"cron": "0 8 * * *", "enabled": True}, + "config": {"recipient": recipient, "routeSnapshot": snapshot}, "flow": {"actions": []}, } defn, problem = engine.create(session.runtime, raw, username=session.uname) @@ -1715,22 +2524,41 @@ def route_delivery_create(body: dict = Body(default=None), try: automation_control.sync_definition(session.tenant, defn) except automation_control.ControlPlaneError as exc: - # The definition exists but is not allowed to read as armed if its native schedule did not. raise err(503, "automation_control_unavailable", f"the route was saved, but its daily schedule could not be armed: {exc}") return {"delivery": _route_delivery_wire(defn)} -@router.delete("/customers/route-deliveries/{delivery_id}") -def route_delivery_delete(delivery_id: str, +@router.post("/customers/routes/{route_id}/deliveries", status_code=201) +def normalized_route_delivery_create(route_id: str, body: dict = Body(default=None), + session: Session = Depends(module_gate(MODULE))): + """Schedule a named normalized Route from its server-resolved View and stop relation.""" + return _create_route_delivery_for_route( + session, _route_or_404(session, route_id), body if isinstance(body, dict) else {}) + + +@router.post("/customers/route-deliveries") +def route_delivery_create(body: dict = Body(default=None), session: Session = Depends(module_gate(MODULE))): - """Remove one scheduled delivery without touching its immutable route field or rows.""" + """Compatibility door: a normalized Route id is still mandatory and server-resolved.""" + body = body if isinstance(body, dict) else {} + route_id = str(body.get("routeId") or "").strip() + if not route_id: + raise err(400, "normalized_route_required", + "choose a saved Route before scheduling its delivery") + return _create_route_delivery_for_route(session, _route_or_404(session, route_id), body) + + +def _delete_route_delivery(session, delivery_id, route_id=None): import automation_control import automation_engine as engine defn = (engine.all_definitions(session.runtime) or {}).get(str(delivery_id)) if not isinstance(defn, dict) or defn.get("kind") != "route_delivery": raise err(404, "unknown_route_delivery", "no scheduled route with that id") + wire = _route_delivery_wire(defn) + if route_id is not None and wire["snapshot"]["routeId"] != str(route_id): + raise err(404, "unknown_route_delivery", "no scheduled delivery for this Route with that id") if not session.admin and str(defn.get("createdBy") or "") != session.uname: raise err(403, "forbidden", "only the route creator or an administrator can remove its schedule") engine.remove(session.runtime, str(delivery_id)) @@ -1740,3 +2568,19 @@ def route_delivery_delete(delivery_id: str, raise err(503, "automation_control_unavailable", f"the route schedule was removed, but its native schedule could not be removed: {exc}") return {"deleted": str(delivery_id)} + + +@router.delete("/customers/routes/{route_id}/deliveries/{delivery_id}") +def normalized_route_delivery_delete(route_id: str, delivery_id: str, + session: Session = Depends(module_gate(MODULE))): + """Remove one delivery from the normalized Route detail that owns it.""" + route = _route_or_404(session, route_id) + _route_mutation_allowed(session, route) + return _delete_route_delivery(session, delivery_id, route_id) + + +@router.delete("/customers/route-deliveries/{delivery_id}") +def route_delivery_delete(delivery_id: str, + session: Session = Depends(module_gate(MODULE))): + """Compatibility removal for existing schedule links.""" + return _delete_route_delivery(session, delivery_id) diff --git a/api/routes_geo.py b/api/routes_geo.py index 1de2d31d6aef1686dd6c35e16b2c42618af00390..c3b7ee46865f303dec8d71d84005d3b2c409690f 100644 --- a/api/routes_geo.py +++ b/api/routes_geo.py @@ -123,7 +123,9 @@ _CACHE_MAX = 4096 ZONES_KEY = "map_zones" MAX_ZONES = 200 MAX_ZONE_MEMBERSHIPS = 10_000 +MAX_ZONE_ZIP_CODES = 500 _ZONE_COLORS = ("#9DBFF2", "#A5D8B4", "#F5D989", "#F0A8A0", "#C7B7E8") +_ZONE_ZIP_RE = re.compile(r"^\d{5}(?:-\d{4})?$") def _zone_id(raw): @@ -173,6 +175,25 @@ def _clean_geometry(raw): "type": "MultiPolygon", "coordinates": cleaned_polys} +def _clean_zip_codes(raw): + """A bounded US ZIP list. Invalid input is refused rather than silently losing a boundary.""" + if raw is None: + return [] + if not isinstance(raw, list): + raise ValueError("Zone ZIP codes must be a list") + if len(raw) > MAX_ZONE_ZIP_CODES: + raise ValueError(f"a Zone can contain at most {MAX_ZONE_ZIP_CODES:,} ZIP codes") + out, seen = [], set() + for value in raw: + zip_code = str(value or "").strip().upper() + if not _ZONE_ZIP_RE.fullmatch(zip_code): + raise ValueError("each Zone ZIP code must be five digits or ZIP+4") + if zip_code not in seen: + seen.add(zip_code) + out.append(zip_code) + return out + + def _clean_zone(raw, previous=None, *, zone_id=""): raw = raw if isinstance(raw, dict) else {} previous = previous if isinstance(previous, dict) else {} @@ -182,8 +203,13 @@ def _clean_zone(raw, previous=None, *, zone_id=""): color = str(raw.get("color") if "color" in raw else previous.get("color") or _ZONE_COLORS[0]).upper() if not re.fullmatch(r"#[0-9A-F]{6}", color): raise ValueError("a Zone color is a six-digit hex color") - geometry = _clean_geometry(raw.get("geometryGeoJson") if "geometryGeoJson" in raw - else previous.get("geometryGeoJson")) + geometry_raw = (raw.get("geometryGeoJson") if "geometryGeoJson" in raw + else previous.get("geometryGeoJson")) + geometry = _clean_geometry(geometry_raw) if geometry_raw is not None else None + zip_codes = _clean_zip_codes(raw.get("zipCodes") if "zipCodes" in raw + else previous.get("zipCodes")) + if geometry is None and not zip_codes: + raise ValueError("a Zone needs GeoJSON Polygon or MultiPolygon geometry or a ZIP code") geographic = str(raw.get("geographicField") if "geographicField" in raw else previous.get("geographicField") or "").strip()[:80] memberships = raw.get("memberships") if "memberships" in raw else previous.get("memberships") or [] @@ -209,6 +235,7 @@ def _clean_zone(raw, previous=None, *, zone_id=""): "name": name, "color": color, "geometryGeoJson": geometry, + "zipCodes": zip_codes, "visible": bool(raw.get("visible")) if "visible" in raw else bool(previous.get("visible", True)), "geographicField": geographic, "memberships": member_ids, @@ -217,7 +244,8 @@ def _clean_zone(raw, previous=None, *, zone_id=""): def _zone_wire(zone): wire = {key: zone.get(key) for key in ("id", "name", "color", "geometryGeoJson", "visible", - "geographicField", "memberships", "createdBy", "createdAt")} + "geographicField", "zipCodes", "memberships", + "createdBy", "createdAt")} # Storage keeps ids as strings because table records may originate in several backends; Map # selection is numeric `pid`s. The wire performs that one boundary conversion explicitly. members = [] @@ -229,9 +257,50 @@ def _zone_wire(zone): if pid > 0: members.append(pid) wire["memberships"] = members + wire["zipCodes"] = list(zone.get("zipCodes") or ()) return wire +def _zone_membership_union(zone, rows, allowed_pids, *, refuse_overflow=False): + """Return a scoped selected/polygon/ZIP union without mutating the stored Zone. + + C3's durable ``memberships`` snapshot is retained for existing Zones and explicit geography + groups. Wave 44 adds current polygon/ZIP matches from the already permission-walled rows. + GET calls this pure projection too, so an address change can update the visible chooser while + tenant data remains untouched. + """ + from core import route_system + + allowed = {str(pid) for pid in allowed_pids or ()} + prepared = [] + geographic = str(zone.get("geographicField") or "") + geographic_is_zip = bool(re.search(r"zip|postal", geographic, re.IGNORECASE)) + for raw in rows or (): + if not isinstance(raw, dict): + continue + item = raw + if geographic_is_zip and geographic in raw and not any( + raw.get(key) not in (None, "") + for key in ("zip", "zip_code", "postal_code", "postalCode")): + item = {**raw, "zip": raw.get(geographic)} + prepared.append(item) + derived = route_system.zone_members(zone, prepared) + members, seen = [], set() + for value in list(zone.get("memberships") or ()) + list(derived): + try: + pid = int(value) + except (TypeError, ValueError): + continue + marker = str(pid) + if pid <= 0 or marker in seen or marker not in allowed: + continue + seen.add(marker) + members.append(pid) + if refuse_overflow and len(members) > MAX_ZONE_MEMBERSHIPS: + raise ValueError(f"a Zone can contain at most {MAX_ZONE_MEMBERSHIPS:,} record IDs") + return {**zone, "memberships": members} + + def _zone_context(session, raw_table_key): """One table-scoped Zone context, using the existing grid assemblies as permission walls.""" table_key = str(raw_table_key or "").strip() @@ -250,7 +319,7 @@ def _zone_context(session, raw_table_key): raise err(400, "invalid_zone_table", "choose a database before working with Zones") fields = {str(field.get("key") or "") for field in (grid.get("fields") or []) if isinstance(field, dict) and field.get("key")} - return table_key, fields, set(grid.get("pids") or ()) + return table_key, fields, set(grid.get("pids") or ()), list(grid.get("rows_src") or ()) def _zone_subject(session, table_key, zone_id): @@ -437,13 +506,14 @@ def geo_providers(session: Session = Depends(require_session)): def geo_zones(tableKey: str = Query(default=""), session: Session = Depends(require_session)): """The canonical table-scoped Zone DTOs a Map may draw or filter by.""" from core import user_tables - table_key, _fields, _pids = _zone_context(session, tableKey) + table_key, _fields, pids, rows = _zone_context(session, tableKey) zones = user_tables.zones(table_key, st=session.runtime) # A hidden Zone remains recoverable to its creator/admin, but cannot become visible through a # second table simply because row ids happen to collide there. visible = [zone for zone in zones if zone.get("visible", True) or session.admin or str(zone.get("createdBy") or "") == session.uname] - return {"zones": [_zone_wire(zone) for zone in sorted(visible, + projected = [_zone_membership_union(zone, rows, pids) for zone in visible] + return {"zones": [_zone_wire(zone) for zone in sorted(projected, key=lambda zone: (str(zone.get("name") or "").casefold(), str(zone.get("id") or "")))]} @@ -452,9 +522,10 @@ def geo_zone_create(body: dict = Body(default=None), session: Session = Depends( """Persist one Zone in the existing table-scoped Zone store and field/filter contract.""" from core import user_tables body = body if isinstance(body, dict) else {} - table_key, field_keys, pids = _zone_context(session, body.get("tableKey")) + table_key, field_keys, pids, rows = _zone_context(session, body.get("tableKey")) try: zone = _clean_zone(body) + zone = _zone_membership_union(zone, rows, pids, refuse_overflow=True) except ValueError as exc: raise err(400, "invalid_zone", str(exc)) if not zone["id"]: @@ -476,10 +547,11 @@ def geo_zone_update(zone_id: str, body: dict = Body(default=None), """Edit a table-scoped Zone while retaining creator, geometry and membership invariants.""" from core import user_tables body = body if isinstance(body, dict) else {} - table_key, field_keys, pids = _zone_context(session, body.get("tableKey")) + table_key, field_keys, pids, rows = _zone_context(session, body.get("tableKey")) previous = _zone_subject(session, table_key, zone_id) try: zone = _clean_zone(body, previous, zone_id=str(zone_id)) + zone = _zone_membership_union(zone, rows, pids, refuse_overflow=True) except ValueError as exc: raise err(400, "invalid_zone", str(exc)) saved = user_tables.upsert_zone(table_key, zone, field_keys=field_keys, @@ -495,7 +567,7 @@ def geo_zone_delete(zone_id: str, body: dict = Body(default=None), session: Session = Depends(require_session)): """Delete exactly one table-scoped Zone; record values are never mutated here.""" body = body if isinstance(body, dict) else {} - table_key, _fields, _pids = _zone_context(session, body.get("tableKey")) + table_key, _fields, _pids, _rows = _zone_context(session, body.get("tableKey")) _zone_subject(session, table_key, zone_id) from core import user_tables if not user_tables.delete_zone(table_key, zone_id, st=session.runtime): diff --git a/api/routes_grid.py b/api/routes_grid.py index 822c01bcb7a226838b5d3d01d22f96a431d3ecbe..91fd673a8bef28b3ac26e6b8056a845a8e3c8353 100644 --- a/api/routes_grid.py +++ b/api/routes_grid.py @@ -24,7 +24,7 @@ back. A 200 over a write that evaporated is the failure this rule exists to prev import datetime as dt import time -from fastapi import APIRouter, Body, Depends +from fastapi import APIRouter, Body, Depends, HTTPException from deps import Session, err, module_gate, perms, require_session @@ -167,7 +167,8 @@ def workspace(scope: str = "customer", # The field contract rides the cheap re-read on EVERY topic — see the customer branch # below for why (a user table's first cohort changes its contract too: `_cohorts(ctx)` # is scope-parameterized, so `ut_` surfaces have their own sets). - workspace["fields"] = g["fields"] + from core import route_system + workspace["fields"] = [route_system.field_axes(field) for field in g["fields"]] workspace["measures"] = g["measures"] workspace["measureSets"] = g["measure_sets"] # ⚠ `g["derived"]`, NOT `{}` (2026-08-04). R9 gave every topic its own cohort sets, and @@ -267,6 +268,10 @@ def workspace(scope: str = "customer", except grid_events.StoreUnavailable: raise err(503, "store_unavailable", "the tenant store is unavailable") workspace = g["workspace"] + # C1: this is a server projection, not an editable workspace preference. A customer user + # can reorder private folders but cannot remove or relocate Loopable's protected system one. + from core import route_system + workspace = route_system.inject_loopable_folder(workspace) # ADDITIVE to the wire: the caller's own overlay cells. The grid reads overlays from the # /customers rows, but this route is the cheap read-back a write can be verified against # (verify_api's overlay probe) without paying the pool call. Per-user by construction — @@ -286,7 +291,7 @@ def workspace(scope: str = "customer", # same permission wall (`hidden_keys` already applied) — so the two wires cannot disagree # about what a column is. Taking it from anywhere else would fork the contract that # verify_fields_contract.py exists to keep single-sourced. - workspace["fields"] = g["fields"] + workspace["fields"] = [route_system.field_axes(field) for field in g["fields"]] # ── the STANDALONE measure channel (owner item 1, 2026-07-31) ──────────────────────────── # The embed receives these as top-level render args; standalone lifts them off THIS route # (useCustomerData merges them into the payload slots the grid already reads). They ride @@ -1066,6 +1071,9 @@ def grid_events_route(body: dict = Body(default=None), #: [[date-window-vocabulary]]. This is a RENAME and nothing else. `all` resolves to `(None, None)`. PIVOT_WINDOWS = {"all": "all_time", "last12m": "ltm", "ytd": "ytd"} +# The required subset of a stored semantic `via` bag consumed by the pivot door. +_PIVOT_VIA_REQUIRED = ("topic", "measure", "scope_key", "source_dim", "target_dim", "target_key") + #: The DECLARED pivot routes: `(source, target) -> the edge that answers it`. A pair with no entry #: is a 400, never an empty grid — "there is no way to get there" and "nothing is related" are two #: different answers and only one of them is true. @@ -1089,13 +1097,75 @@ _PIVOT_ROUTES = { "measure": "revenue", "scope_key": "customer", # the cohort bucket the SOURCE tree's cohort leaves name "source_dim": "order_partner", - "source_key": "pid", "target_dim": "product_code", "target_key": "code", }, } +def _pivot_route(fields, source, target, via_field): + """Resolve a visible link field first; allow only the single dated A23 fallback.""" + try: + return _pivot_route_via(fields, source, target, via_field) + except HTTPException as exc: + detail = exc.detail if isinstance(exc.detail, dict) else {} + if (detail.get("error") or {}).get("code") != "no_pivot_route": + raise + tombstone = _PIVOT_ROUTES.get((source, target)) + if tombstone is None: + raise + return dict(tombstone) + + +def _pivot_route_via(fields, source, target, via_field): + """Return the `via` bag from the link field named by the request, or refuse by name.""" + via_field = str(via_field or "").strip() + if not via_field: + raise err(400, "no_pivot_route", + f"this pivot did not name a link field, so there is no edge to answer it with. " + f"Send 'viaField' naming a link field on '{source}' that points at '{target}'") + + match = None + for field in fields or (): + if not isinstance(field, dict): + continue + if str(field.get("key") or "") == via_field and field.get("type") == "link": + match = field + break + if match is None: + raise err(400, "no_pivot_route", + f"this account cannot resolve a link field named '{via_field}' on '{source}', " + f"so there is no declared edge to '{target}'. Pivot from a link field this " + f"account can see on that grid") + + link = match.get("link") if isinstance(match.get("link"), dict) else {} + points_at = str(link.get("table") or "") + if points_at != target: + raise err(400, "no_pivot_route", + f"the link field '{via_field}' points at " + f"'{points_at or 'nothing this door can name'}', not at '{target}'. Refusing " + f"rather than pivoting through an edge nobody asked for") + + via = link.get("via") if isinstance(link.get("via"), dict) else None + if not via: + raise err(400, "no_pivot_route", + f"the link field '{via_field}' joins '{source}' to '{target}' directly and " + f"carries no 'via' declaration, so this door has no fact table to ask") + missing = [key for key in _PIVOT_VIA_REQUIRED if not via.get(key)] + if missing: + raise err(400, "no_pivot_route", + f"the 'via' declaration on link field '{via_field}' is missing " + f"{', '.join(missing)}, so the edge is not fully described") + return via + + +_PIVOT_AUTHORING = ( + "unknown metric", "unknown topic", "belong to another topic", "is company-level", + "at least one measure required", "filter tree", "filter rule", "must reference", + "grain must be", "is store-only", "unknown dim", "has no name column", +) + + def _pivot_today(): """The tenant's today, ISO. One clock, so the window and any date leaf agree.""" import core.periods as periods @@ -1228,12 +1298,12 @@ def grid_pivot(body: dict = Body(default=None), f"window must be one of {', '.join(sorted(PIVOT_WINDOWS))}. Refusing to guess " f"which period was meant, because guessing 'all time' would widen the answer " f"under a count nobody would doubt") - route = _PIVOT_ROUTES.get((source, target)) - if route is None: - raise err(400, "no_pivot_route", - f"there is no declared way to get from '{source}' to '{target}'. " - f"Declared: " + - "; ".join(f"{a} to {b}" for a, b in sorted(_PIVOT_ROUTES))) + # Permission and path validity are decided before either database is fetched. + scope = perm_scope.pivot_scope(session.user, source, target, st=session.runtime) + if not scope.permitted: + return _pivot_reply(source, target, [], [], 0, 0, + refusal=scope.refusal(), + limits=scope.limits(st=session.runtime)) # ── THE WALL, ON BOTH SIDES OF THE JOIN (R12) ──────────────────────────────────────────── src, refusal = _pivot_side(session, source) @@ -1246,6 +1316,9 @@ def grid_pivot(body: dict = Body(default=None), src_fields, src_rows = src tgt_fields, tgt_rows = tgt + # Resolve the relationship from the visible link field the request actually named. + route = _pivot_route(src_fields, source, target, body.get("viaField")) + today = _pivot_today() columns = filter_eval._columns_map(src_fields) unanswerable = _pivot_unanswerable(filters, columns) @@ -1270,10 +1343,6 @@ def grid_pivot(body: dict = Body(default=None), for cid, c in cohort_mod.scoped(route["scope_key"]).visible( session.uname, pool_pids).items()} - if route["source_key"] != "pid": - raise err(500, "pivot_source_key", - f"this pivot names '{route['source_key']}' as the source identity and the row " - f"engine reads 'pid'. Refusing rather than pivoting on the wrong column") member = sorted({p for p in filter_eval.visible_pids( {"conj": filter_conj, "nodes": filters}, src_rows, src_fields, ctx=filter_eval.EvalCtx(cohort_sets=cohort_sets, today=today)) @@ -1295,8 +1364,20 @@ def grid_pivot(body: dict = Body(default=None), filters={route["source_dim"]: member}, date_from=date_from, date_to=date_to, team_id=team_id, today=today, limit=semantic.MAX_GROUPS) + except semantic.UnknownColumnError as exc: + others = [column for column in (exc.columns or ()) if column != exc.column] + also = f" It also names {', '.join(others)}." if others else "" + raise err(400, "pivot_unknown_column", + f"{str(exc).rstrip('. ')}. The column this pivot could not resolve is " + f"'{exc.column}'.{also} Fix the link field's 'via' declaration, or pivot " + f"through a link field whose dims this topic actually carries") except semantic.ModelError as exc: - raise err(503, "pivot_unavailable", str(exc)) + sentence = str(exc) + if any(needle in sentence.lower() for needle in _PIVOT_AUTHORING): + raise err(400, "pivot_bad_edge", + f"{sentence.rstrip('. ')}. This pivot's edge comes from the link " + f"field's 'via' declaration, so that declaration is what needs correcting") + raise err(503, "pivot_unavailable", sentence) # ⛔ `truncated` IS READ, NOT ASSUMED FALSE. `store_query` returns groups plus a flag, and a # grouped query's groups ARE the answer, so a dropped one is missing data with nothing to @@ -1396,3 +1477,84 @@ def _register_topic_rows(): _PRODUCT_MODULE = "product_data" _C1_ROW_SOURCES = _register_topic_rows() + + +# W42-T07 / C2: model-derived operands for the one-hop link editor. This deliberately reads +# model files, not a tenant store, so a cold/locked store cannot silently empty the picker. +_HOP_CACHE = {} +_HOP_FIELD_SOURCE = "topic" +_HOP_FIELD_TYPES = ("text", "int", "date", "currency", "pct") + + +def _hop_dim_type(columns, dim_key): + field_type = ((columns or {}).get(dim_key) or {}).get("type") + return field_type if field_type in _HOP_FIELD_TYPES else "text" + + +def _hop_topic_offer(): + """Return answerable topics plus the exact windows accepted by `_clean_via`.""" + if _HOP_CACHE.get("offer"): + return _HOP_CACHE["offer"] + from harness import semantic as sem + from harness import windows as window_model + import core.user_tables as user_tables + + topics, metrics = sem.topics(), sem.metrics() + by_topic = {} + for key, metric in metrics.items(): + if metric.get("agg") in ("ratio", "derived"): + continue + by_topic.setdefault(metric.get("topic"), []).append({ + "key": key, + "label": metric.get("label") or key, + "format": metric.get("format") or "usd", + "description": metric.get("description") or "", + }) + + offered, omissions = [], [] + for topic_key, topic in sorted(topics.items()): + dims = ((topic.get("store") or {}).get("dims")) or {} + measures = by_topic.get(topic_key) or [] + label = topic.get("label") or topic_key + if not dims or not measures: + reasons = [] + if not dims: + reasons.append("no_dims") + if not measures: + reasons.append("no_measures") + omissions.append({"topic": topic_key, "label": label, "reasons": reasons}) + continue + try: + columns = sem.store_columns(topic_key, include_measures=False) + except Exception: + omissions.append({"topic": topic_key, "label": label, + "reasons": ["no_store_columns"]}) + continue + offered.append({ + "key": topic_key, + "label": label, + "grain": topic.get("grain") or "", + "dims": [{ + "key": dim_key, + "label": dim.get("label") or dim_key, + "type": _hop_dim_type(columns, dim_key), + "source": _HOP_FIELD_SOURCE, + "keyedBy": "id" if dim.get("name_col") else "value", + } for dim_key, dim in dims.items()], + "measures": sorted(measures, key=lambda measure: measure["label"].lower()), + }) + + windows = [{"key": key, "label": window_model.WINDOW_LABELS.get(key, key).format(n="N")} + for key in user_tables.LINK_VIA_WINDOWS] + offer = {"topics": offered, "windows": windows, "omissions": omissions} + _HOP_CACHE["offer"] = offer + return offer + + +@router.get("/grid/hop-topics") +def grid_hop_topics(session: Session = Depends(require_session)): + """Serve the one-hop authoring offer with an unconditional response shape.""" + import odoo_relational + if not odoo_relational.is_royal(session.tenant): + return {"topics": [], "windows": [], "omissions": []} + return _hop_topic_offer() diff --git a/api/routes_nav.py b/api/routes_nav.py index 5d1f0d2d1dea2feb4809201c5a7944642db17620..65ffa54eccfe4a22d1d5d6d86b60826db392b2de 100644 --- a/api/routes_nav.py +++ b/api/routes_nav.py @@ -914,7 +914,10 @@ def nav_schema(key: str, session: Session = Depends(require_session)): # shared stratum and this ticket is not where it starts; widening the list handed to the # closure can only ever ADD to the hidden set, never remove from it. import routes_tables as _rt - base = list(defn.get("fields") or []) + # Zone is a canonical derived field on the same pre-wall contract the grid uses. Shared + # tenant-wide fields remain schema-private here; this only prevents the two real doors + # from evaluating the Zone field on opposite sides of their common wall. + base = _rt._ut_contract_fields(key, defn.get("fields") or [], st=session.runtime) hidden = _rt._ut_hidden(session, key, _rt._ut_shared_fields(session, key, base), st=session.runtime) diff --git a/api/routes_shares.py b/api/routes_shares.py index e2b6710e79912f060bd9e114cdc2b7c0f041d8f3..84b27a6cec0bf10acdc0b4b8075c1a5a4a0d40aa 100644 --- a/api/routes_shares.py +++ b/api/routes_shares.py @@ -415,6 +415,415 @@ def _unremovable(held_rows, entries, uname): stuck.add(user) return stuck +# ── ⭐⭐ WAVE 42 · T25 — SHARING A VIEW SHARES THE FIELDS IT NAMES ────────────────────────────── +# +# A saved view is a SELECTION expressed in columns: an order, a visible set, a group, a colour, +# a sort and a filter tree, every one of which names a field key. Granting the view alone hands +# the receiver a rail row whose columns resolve to nothing they may read, so the feature that +# looked shipped puts an empty grid on their screen. This block closes that, and it refuses +# LOUDLY in the three places where a grant would otherwise be a lie. +# +# ⛔⛔ TRAP 1 — A FIELD GRANT IS TOPIC-WIDE, NOT VIEW-SCOPED. It is keyed +# `shares.field_oid(topic, key)`, so it hands the receiver that column on EVERY view of that +# database, for as long as the grant stands. That is the existing grant primitive and this +# ticket does not narrow it; what it does is refuse to let the response PRETEND otherwise, which +# is why `sharedFieldScope` rides back beside the list. +# +# ⛔⛔ TRAP 2 — THE SHARER CANNOT GRANT A COLUMN THEY DO NOT OWN. `field` is absent from +# `shares.RESHARE_KINDS` and `_owns_object`'s field branch demands `createdBy == session.uname`, +# so a view built over a colleague's column simply cannot carry that column with it. Silently +# skipping it ships an empty column; silently granting it is a permission hole. It is REFUSED +# BY NAME, inside the 200 response, with the sentence that says who to ask. +# +# ⛔⛔ TRAP 3 — A GRANT CANNOT UN-HIDE AN ADMIN-WALLED COLUMN. `perm_scope.hidden_keys` UNIONS +# the administrator's `hiddenFields` with the field-grant source, so granting a walled column +# answers 200, writes the record, and the receiver still sees nothing: granted with no effect, +# which is the failure this ticket exists to end. Detected BEFORE the grant, refused by name, +# and the receiver it is walled from is named too. +# +# ⚠ THE WALL IS READ FROM THE ADMIN SOURCE DIRECTLY AND NEVER THROUGH `hidden_keys`, AND THAT IS +# NOT AN OPTIMISATION. `hidden_keys` unions `field_grant_hidden` — "hidden because you were not +# granted it" — which is true of every column this route is ABOUT to grant. Asking it here would +# refuse the whole set on the grounds that the grant has not happened yet. + + +def _view_configs(view): + """The dicts a saved view keeps its column state in. + + `SavedView.config` is the contract (`types.ts::ViewConfig`) and is where every key below + actually lives. The record ITSELF is read as well because a legacy row that stored the same + members flat costs one dict lookup to cover and would otherwise hand back an empty key set, + which is the shape that grants nothing and reports nothing wrong. + """ + out = [] + if not isinstance(view, dict): + return out + cfg = view.get("config") + if isinstance(cfg, dict): + out.append(cfg) + out.append(view) + return out + + +def _view_field_keys(view): + """Every field key this view NAMES, in the order a reader would meet them. + + ⚠ `filters[].rhs` IS ONE OF THEM AND IS THE ONE THAT GETS DROPPED. A field-vs-field leaf + (`{colId: price, op: lt, rhs: {kind: field, colId: cost}}`) names TWO columns, and a share + carrying only the left one leaves the receiver a filter that cannot evaluate. `kind` `stat` + is not a column reference and is skipped, matching `perm_scope`'s own reader of the tree. + + ⚠ AND `filters` IS A TREE, NOT A LIST. Condition groups nest under `children`, so the walk + recurses rather than iterating once: a column named only inside a group is still named. + """ + keys, seen = [], set() + + def add(raw): + key = str(raw or "").strip() + if key and key not in seen: + seen.add(key) + keys.append(key) + + def walk(node): + if isinstance(node, list): + for child in node: + walk(child) + return + if not isinstance(node, dict): + return + kids = node.get("children") + if isinstance(kids, list): + walk(kids) + add(node.get("colId")) + rhs = node.get("rhs") + if isinstance(rhs, dict) and rhs.get("kind") != "stat": + add(rhs.get("colId")) + + for cfg in _view_configs(view): + for key in (cfg.get("order") or ()): + add(key) + for key in (cfg.get("visible") or ()): + add(key) + walk(cfg.get("filters")) + add(cfg.get("groupBy")) + colour = cfg.get("colorBy") + add(colour.get("colId") if isinstance(colour, dict) else colour) + sorts = cfg.get("sorts") + if isinstance(sorts, dict): + sorts = [sorts] + for spec in (sorts or ()): + if isinstance(spec, dict): + add(spec.get("colId")) + return keys + + +def _find_view_topic(session, oid): + """`(topic, view)` for a saved view living in any personal stratum of any topic, else + `(None, None)`. + + ⚠ THE TOPIC IS HALF THE ANSWER. A field grant is keyed by TOPIC, so resolving the view + without remembering which workspace answered would leave the caller guessing at the oid, + which is exactly how two arms of one route come to write two different keys for one column. + Same loop and same finder as `_owns_object`, which answers a different question about the + same lookup and is deliberately left alone. + """ + try: + import core.table_store as table_store + except Exception: # noqa: BLE001 + return None, None + for topic in _topics(session): + try: + hit = table_store.make(f"{topic}_table_workspace", st=session.runtime).find_view(oid) + except Exception: # noqa: BLE001 + continue + if hit: + return str(topic), (hit[1] if len(hit) > 1 else {}) + return None, None + + +def _promote_field_for_grant(session, table_key, field_key): + """Promote a private column exactly once, stamp it shared, and answer its GRANT OID. + + ⛔⛔ ONE SEQUENCE, TWO ARMS, AND THAT IS THE WHOLE REASON THIS IS A FUNCTION. `put_share`'s + `field` arm and its `view` arm (T25) both hand a column to somebody. Two spellings of + "promote, stamp, key the oid" is how two surfaces start disagreeing about what a grant + means, and the oid is the exact member that would drift: it is normalised through + `_field_storage_keys(...)[2]`, never built from whatever key the caller happened to hold. + + Answers `None` for a key with no definition on that database, so the caller decides whether + that is a 404 (the `field` arm, where the column IS the request) or a named refusal (the + `view` arm, where it is one entry in a list). + """ + from core import field_permissions, shared_overlay + defn, already_shared, workspace_key, shared_key, grant_topic = _field_definition( + session, table_key, field_key) + if not isinstance(defn, dict): + return None + if not already_shared: + defn = field_permissions.promote_field( + workspace_key, shared_key, grant_topic, + session.uname, defn, st=session.runtime) + stamped = dict(defn) + stamped["shared"] = True + stamped["granted"] = True + shared_overlay.put_field(shared_key, field_key, stamped, st=session.runtime) + return shares.field_oid(grant_topic, field_key) + + +def _grant_receivers(session, entries): + """`{username: display name}` for everybody a grant list actually reaches. + + `shares.EVERYONE` is a real audience, not a placeholder, so it expands to the tenant roster + `_people` already computes for the picker. Reading it from a second place would be a second + answer to "who is in this workspace". + """ + people = None + out = {} + for entry in shares._clean_entries(entries): + who = str(entry.get("user") or "").strip().lower() + if not who: + continue + if who == shares.EVERYONE: + if people is None: + people = _people(session.tenant) + for person in people: + out.setdefault(str(person["username"]).strip().lower(), person["name"]) + continue + out.setdefault(who, who) + if out: + try: + reg = users.registry() or {} + except Exception: # noqa: BLE001 + reg = {} + for uname in list(out): + rec = reg.get(uname) + if isinstance(rec, dict) and rec.get("name"): + out[uname] = str(rec["name"]) + return out + + +def _admin_walled(session, grant_topic, defs, receivers): + """`{field key: [display names]}` for receivers an ADMINISTRATOR has already walled off. + + ⛔ THE ADMIN SOURCE ONLY. `perm_scope.entry(rec, module)['hiddenFields']` is the half of + `hidden_keys` a field grant cannot overrule; the other half IS the field grant, so consulting + the closure here would report every column this route is about to grant as hidden and refuse + the lot (see the block note above). + + ⭐ THE MEASURE BINDING IS BORROWED, NEVER RESPELLED. An administrator ticking + "Metric - Gross margin" stores `measure_margin`, while the COLUMNS carrying that number are + keyed `measure_margin_`; `perm_scope._measure_bound_keys` is the one evaluator that + joins the two, and a second copy here would be the first to drift. + + ⚠ AN ADMINISTRATOR RECEIVER IS NEVER WALLED — `hidden_keys` answers an empty set for one + before it reads anything — so the same short circuit runs here, or this function would refuse + a grant to somebody who can already see the column. + + ⚠ A RECEIVER WITH NO REGISTRY RECORD IS SKIPPED RATHER THAN TREATED AS WALLED: `perm_scope` + reads a permission block off the RECORD (`_rec` answers `{}` for a bare username), so a name + this tenant cannot resolve carries no admin wall to find. `_entries_or_400` has already + refused a grant list naming an account that does not exist. + """ + import core.perm_scope as perm_scope + import core.perms as perms + try: + reg = users.registry() or {} + except Exception: # noqa: BLE001 + reg = {} + field_list = [dict(defn or {}, key=key) for key, defn in defs.items() + if isinstance(defn, dict)] + walled = {} + for uname, display in sorted(receivers.items()): + rec = reg.get(uname) + if not isinstance(rec, dict) or perms.is_admin(rec): + continue + try: + entry = perm_scope.entry(rec, grant_topic) or {} + except Exception: # noqa: BLE001 + continue + hidden = {str(k) for k in (entry.get("hiddenFields") or ()) if k} + if not hidden: + continue + try: + hidden |= set(perm_scope._measure_bound_keys(hidden, field_list)) + except Exception: # noqa: BLE001 + pass + for key in defs: + if key in hidden: + walled.setdefault(key, []).append(display) + return walled + + +#: ⭐ THE THREE REASONS A VIEW'S COLUMN IS NOT GRANTED, EACH ITS OWN CODE. One fused +#: "could not share" would make the three walls unprovable apart, which is the +#: [[a-declared-gate-is-an-unchecked-claim]] failure `verify_field_doors` section 8 exists to +#: prevent: every cause fails by its own name, or it is not gated at all. +#: ⛔ STANDING RULE 2 — no em dash and no en dash in any sentence below; these reach a screen. +FIELD_REFUSAL_CODES = ("not_a_shared_column", "not_owned", "admin_hidden") + +#: ⭐ TRAP 1, SAID OUT LOUD. A column grant is keyed by DATABASE, so it is not scoped to the view +#: that occasioned it. The response carries this sentence rather than letting a caller infer a +#: narrower promise than the store actually makes. +FIELD_GRANT_SCOPE = ("A column shared this way is shared on the whole database, so the people " + "you named can see it on every view of it, not only on this one.") + + +def _refusal(key, code, names=()): + """One refused column, with the sentence a person reads.""" + if code == "not_a_shared_column": + why = ('"{k}" belongs to the database itself rather than to a person, so there was ' + 'nothing to grant. Anybody who can open the database already sees it.') + elif code == "not_owned": + why = ('"{k}" is somebody else\'s column, and only its owner (or an administrator) can ' + 'share it. Ask them to share it, then share this view again.') + else: + why = ('"{k}" is hidden from {who} by an administrator, so sharing it would change ' + 'nothing. Ask an administrator to unhide it first.') + return {"field": key, "reason": code, + "message": why.format(k=key, who=_named(names) if names else "them")} + + +def _classify_view_fields(keys, owned, defined, walled): + """`(sharedFieldKeys, refusedFieldKeys)` for the columns a view names. PURE. + + ⛔ EVERY KEY IS DECIDED BEFORE ANY OF THEM IS WRITTEN, which is the same property the ceiling + block above claims for the three role walls: a refusal must not be able to leave part of the + set promoted and part of it untouched. + + ⚠ THE ORDER OF THE CAUSES IS FIXED AND IT DECIDES WHICH SENTENCE A PERSON READS. A built-in + column is owned by nobody, so "there was nothing to grant" has to be asked before "you do not + own it", or every database column in `visible` would be reported as somebody else's. + """ + granted, refused = [], [] + for key in keys: + if key not in defined: + refused.append(_refusal(key, "not_a_shared_column")) + elif key not in owned: + refused.append(_refusal(key, "not_owned")) + elif walled.get(key): + refused.append(_refusal(key, "admin_hidden", walled.get(key))) + else: + granted.append(key) + return granted, refused + + +def _merge_field_entries(existing, incoming): + """A column's grant list with `incoming` ADDED to it. Nobody is removed, nobody is lowered. + + ⛔⛔ THE SHARE PUT REPLACES, AND A COLUMN'S AUDIENCE IS NOT THE VIEW'S AUDIENCE. This route's + contract is a REPLACING set (`PUT ... entries` at the top of the file), which is right for the + object the caller is looking at and catastrophic for the columns underneath it: a column grant + is TOPIC-WIDE, so handing the view's list straight to `set_grants` would rewrite the column's + audience to whoever this view happens to be shared with. A colleague granted that column + through the `field` door would lose it, on every view of the database, silently, inside a 200. + That is exactly the harm D-474 closed at the view door, arriving one door over. + + ⚠ AND A LENGTH CHECK DOES NOT SEE IT. Replacing a one-person list with a different one-person + list keeps the count identical, so the gate asserts the HELD USER IS STILL THERE AT THEIR OWN + ROLE rather than counting rows. + + ⭐ RAISED, NEVER LOWERED. A receiver already held at `view` who arrives on an `edit` share is + raised, because only the column's owner (or an administrator) reaches this path at all and + that is the access they just asked to hand over. An `edit` holder meeting a `view` share keeps + `edit`: a share is an addition, and nothing here is a revocation. + + ⚠ `by` RIDES ON EXISTING ROWS AND IS NEVER TAKEN FROM AN INCOMING ONE. Provenance decides who + may revoke a grant (`_unremovable`), so an incoming payload must not be able to claim it; + `set_grants` stamps a genuinely new row from `granter`. + """ + rank = {"view": 0, "edit": 1} + order, merged = [], {} + for row in (existing or ()): + if not isinstance(row, dict): + continue + who = str(row.get("user") or "").strip().lower() + role = str(row.get("role") or "").strip().lower() + if not who or role not in rank or who in merged: + continue + kept = {"user": who, "role": role} + stamp = str(row.get("by") or "").strip().lower() + if stamp: + kept["by"] = stamp + merged[who] = kept + order.append(who) + for row in (incoming or ()): + if not isinstance(row, dict): + continue + who = str(row.get("user") or "").strip().lower() + role = str(row.get("role") or "").strip().lower() + if not who or role not in rank: + continue + if who not in merged: + merged[who] = {"user": who, "role": role} + order.append(who) + elif rank[role] > rank[merged[who]["role"]]: + merged[who]["role"] = role + return [merged[who] for who in order] + + +def _share_view_fields(session, oid, entries): + """The `view` arm's field half: classify every column the view names, then grant the ones + that survive. `(sharedFieldKeys, refusedFieldKeys)`. + + ⛔ CLASSIFY EVERYTHING, THEN WRITE. `_classify_view_fields` sees the whole key set before + `_promote_field_for_grant` touches a single definition, so a refusal cannot leave half the + view's columns promoted. + + ⚠ THE VIEW'S OWN GRANT HAS ALREADY BEEN WRITTEN when this runs, and that order is the safe + one: a `409 grants_changed` on the view must not be preceded by a set of column promotions + for a share that then did not happen. + """ + topic, view = _find_view_topic(session, oid) + if not topic or not isinstance(view, dict): + return [], [] + keys = _view_field_keys(view) + if not keys: + return [], [] + table_key = f"{topic}_table_workspace" + grant_topic = _field_storage_keys(table_key)[2] + defs, defined, owned, owners = {}, set(), set(), {} + for key in keys: + defn, already_shared = _field_definition(session, table_key, key)[:2] + if not isinstance(defn, dict): + continue + defs[key] = defn + defined.add(key) + # ⚠ THE COLUMN'S OWNER, NOT THIS SESSION. `set_grants`' owner is sticky, so stamping the + # SHARER here would hand an administrator sharing somebody else's column the ownership + # of it as a side effect of sharing a view. Same resolver the claim wall uses. + owners[key] = _field_owner(session, defn, already_shared) + # ⛔ ONE OWNERSHIP PREDICATE. `_owns_object` is the wall the `field` arm claims through + # and the DELETE door reads; a second "does this person own this column" here is how the + # two answers come apart. It short circuits True for an administrator, which is right: + # an administrator may already share any column through the `field` door. + if _owns_object(session, "field", shares.field_oid(grant_topic, key)): + owned.add(key) + walled = _admin_walled(session, grant_topic, defs, _grant_receivers(session, entries)) + granted, refused = _classify_view_fields(keys, owned, defined, walled) + written = [] + for key in granted: + field_oid = _promote_field_for_grant(session, table_key, key) + if not field_oid: + refused.append(_refusal(key, "not_a_shared_column")) + continue + # ⛔⛔ ADDED TO, NEVER REPLACED. `set_grants` writes the list it is given, and this route + # PUTs a REPLACING set, so handing it the view's `entries` would make the view's audience + # the COLUMN's audience: a colleague granted this column through the `field` door would + # lose it, on every view of the database, inside a 200. `_merge_field_entries` says what + # that sentence means once, and it is the only writer of a column list on this path. + held = shares.grants("field", field_oid, st=session.runtime) + # ⚠ NO `expect` HERE, AND THAT IS NOT AN OVERSIGHT. The compare-and-set on the view's own + # grant guards a record TWO PEOPLE ARE EDITING through the same dialog. A column's record + # was just read (and possibly created) inside this call, nobody is looking at it in a + # dialog, and a `409` raised at this point would fail a PUT whose view grant has ALREADY + # been written, which is the half-applied state the refusals above exist to avoid. + shares.set_grants("field", field_oid, + _merge_field_entries(held.get("entries"), entries), + owner=held.get("owner") or owners.get(key) or session.uname, + granter=session.uname, st=session.runtime) + written.append(key) + return written, refused + @router.get("/share/mine") def my_shares(session: Session = Depends(require_session)): @@ -662,21 +1071,15 @@ def put_share(kind: str, oid: str, body: dict = Body(default=None), # A field grant is a visibility and edit wall. Promote a private custom # field exactly once, then keep the requested Share field role as the # authoritative override for the legacy permissions bag. - from core import field_permissions, shared_overlay + # ⭐⭐ W42-T25 — THE SEQUENCE MOVED INTO `_promote_field_for_grant` AND DID NOT CHANGE. + # The `view` arm below now hands columns to people as well, and one function answering + # "promote, stamp, key the oid" is what keeps the two arms writing the SAME oid for the + # same column. `None` here still means the column does not exist on that database. table_key, field_key = shares.split_field_oid(oid) - defn, already_shared, workspace_key, shared_key, grant_topic = _field_definition( - session, table_key, field_key) - if not isinstance(defn, dict): + promoted = _promote_field_for_grant(session, table_key, field_key) + if promoted is None: raise err(404, "no_object", "no such field, or it is not shared with this account") - if not already_shared: - defn = field_permissions.promote_field( - workspace_key, shared_key, grant_topic, - session.uname, defn, st=session.runtime) - stamped = dict(defn) - stamped["shared"] = True - stamped["granted"] = True - shared_overlay.put_field(shared_key, field_key, stamped, st=session.runtime) - oid = shares.field_oid(grant_topic, field_key) + oid = promoted # ⭐⭐ R4 / W40-T02 (D3) — OWNERSHIP NEVER MOVES ON A RE-SHARE, AND IT IS SAID HERE RATHER # THAN LEFT TO FALL OUT. `set_grants`' owner is sticky, so the old `rec["owner"] or # session.uname` already happened not to transfer ownership — incidentally, as a property of @@ -706,6 +1109,20 @@ def put_share(kind: str, oid: str, body: dict = Body(default=None), except shares.GrantsChanged as exc: raise err(409, "grants_changed", str(exc)) _notify_new_grantees(session, kind, oid, before=rec["entries"], after=out.get("entries") or []) + # ⭐⭐ W42-T25 — SHARING A VIEW SHARES THE FIELDS IT NAMES. A view is a selection expressed + # in columns, so a grant that carries the view and none of its columns puts a rail row on + # the receiver's screen over an empty grid. The three refusals are named rather than thrown: + # this PUT succeeded, the view IS shared, and what could not follow it is reported by name + # with the reason. Throwing instead would make the ordinary case (a view over one built-in + # column) unshareable. + # ⛔ THE EXISTING KEYS ARE UNTOUCHED. `{owner, entries}` is what every current client reads + # off this route; the field report is additive, and a caller that ignores it is unaffected. + if kind == "view": + shared_fields, refused_fields = _share_view_fields(session, oid, entries) + return {**out, + "sharedFieldKeys": shared_fields, + "refusedFieldKeys": refused_fields, + "sharedFieldScope": FIELD_GRANT_SCOPE} return out diff --git a/api/routes_tables.py b/api/routes_tables.py index 1c59f3d6b93279949da73a2cad0ddf9cf2a3f602..8ffa880e88c4e4aebb463c054203d8f1878bce9d 100644 --- a/api/routes_tables.py +++ b/api/routes_tables.py @@ -52,6 +52,38 @@ def _ut(): return user_tables +def _managed_or_refuse(session, table_key, *, mutation=False): + """C1 generic-door policy. Dedicated Route/Zone handlers use their own capability.""" + from core import route_system + definition = _ut().get(table_key, st=session.runtime) + # Route/Zone are virtual system descriptors until their first dedicated write, so looking + # only in `user_tables` would leave a generic mutation path open during that empty state. + managed = route_system.descriptor(table_key) + if managed is not None: + if mutation: + raise err(403, "loopable_managed", "Loopable system databases can only be changed from Route or Zone tools") + return managed + if definition and (_ut().is_managed_system_table(table_key, st=session.runtime, definition=definition) + or route_system.is_managed(definition)): + if mutation: + raise err(403, "loopable_managed", "Loopable system databases can only be changed from Route or Zone tools") + return definition + return definition + + +def _field_axes(fields): + """C3 wire projection; the four axes are explicit on every server field payload.""" + from core import route_system + out = [] + for field in fields or (): + try: + out.append(route_system.field_axes(field)) + except ValueError: + # Stored historical fields may not invent a new vocabulary just by being read. + raise err(500, "invalid_field_contract", "stored field has an unapproved field axis") + return out + + def _ops(session, table_key, st=None): """This database's per-user workspace store, bound to the tenant. @@ -800,7 +832,19 @@ def _ut_hidden(session, table_key, fields, st=None): return perm_scope.hidden_keys(session.user, table_key, fields, st=st) -def _ut_shared_fields(session, table_key, fields): +def _ut_contract_fields(table_key, fields, st=None): + """The base contract every `ut_*` read door presents before its field wall. + + Zone is a real derived field, not a map-only decoration: saved filters and views address its + canonical key. It therefore joins the contract before the one field-wall evaluator runs. + Keeping this boundary named prevents `/nav/schema` and the grid from attaching it on + opposite sides of that wall. + """ + return _ut().with_zone_field(table_key, fields, st=st) + + +def _ut_shared_fields(session, table_key, fields, workspace_document=None, + shared_document=None, return_document=False): """`fields` PLUS this database's TENANT-WIDE columns — the read-side merge, W38-T16. ⛔⛔ WHY THIS DID NOT EXIST AND WHY THAT WAS A HOLE. `patch_shared_cell` has written into @@ -817,12 +861,15 @@ def _ut_shared_fields(session, table_key, fields): contract, never a redefinition of a column that database already has. """ from core import field_permissions, shared_overlay + document = (shared_document if isinstance(shared_document, dict) + else shared_overlay.document(table_key, st=session.runtime)) field_permissions.migrate_legacy_fields( f"{table_key}_table_workspace", st=session.runtime, - grant_topic=table_key, shared_key=table_key) - defs = shared_overlay.fields(table_key, st=session.runtime) or {} + grant_topic=table_key, shared_key=table_key, + workspace_document=workspace_document, shared_document=document) + defs = shared_overlay.fields(table_key, st=session.runtime, document=document) or {} if not defs: - return fields + return (fields, document) if return_document else fields have = {f.get("key") for f in (fields or ()) if isinstance(f, dict)} from core import shares projected = [] @@ -836,7 +883,8 @@ def _ut_shared_fields(session, table_key, fields): if role: item["sharedRole"] = role projected.append(item) - return list(fields or ()) + projected + merged = list(fields or ()) + projected + return (merged, document) if return_document else merged def _ut_shared_cells(session, table_key, pids): @@ -1050,7 +1098,7 @@ def _usage_column_refs(field): return rollup, automation -def _field_usage(session, table_key, fields, defn, ops, st=None): +def _field_usage(session, table_key, fields, defn, ws_st, ws_key, st=None): """`{field_key: {views, filters, rollups, automations, permRules}}` — C8's `usage`, computed over the WHOLE workspace, or `{}` when it cannot be established. @@ -1099,18 +1147,16 @@ def _field_usage(session, table_key, fields, defn, ops, st=None): return {} lent = st if st is not None else session.runtime memo_key = (str(getattr(session, "tenant", "") or ""), str(table_key)) - token = _usage_token(lent, ops.table_key) + token = _usage_token(lent, ws_key) if token is not None: hit = _USAGE_MEMO.get(memo_key) if hit and hit[0] == token: return hit[1] try: - # ⚠ `ops.st` and `ops.table_key`, not a rebuilt literal: the store key `_ops` closed over - # is the one this database's strata actually live under, and a second spelling of the - # suffix is a second thing to keep in step. A lend passes `_table_workspace` straight - # through to the runtime (it serves only `user_tables` and `object_shares`), so this is - # the tenant's own document. - ws_doc = ops.st.get(ops.table_key) or {} + # The caller lends either its already-captured whole workspace document or the exact store + # handle plus key that produced it. Customer's document is `customer_table_workspace`, + # which cannot be reconstructed from its topic name `customer_data`. + ws_doc = (ws_st if isinstance(ws_st, dict) else (ws_st.get(ws_key) or {})) except Exception: # noqa: BLE001 return {} try: @@ -1254,7 +1300,8 @@ def ut_assembly(session: Session, table_key: str, storage_key: str = "", st=session.runtime, scope_key=table_key, table=ops) ws = grid_events.table_workspace(ctx, allowed_pids=pids, - consume_corrections=consume_corrections) + consume_corrections=consume_corrections, + capture_document=True) workspace, fields, views, lists = aios_grid.workspace_wire( ws, session.uname, set(pids), defs={}, scope_key=table_key, storage_key=storage_key, fields_base=fields_base, st=session.runtime) @@ -1262,7 +1309,9 @@ def ut_assembly(session: Session, table_key: str, storage_key: str = "", # `workspace_wire` and the wall, which is `routes_odoo_tables`' own position for the same two # lines and for the same reason: the closure must be recomputed on the MERGED contract or a # formula in a shared column reaches past it. - fields = _ut_shared_fields(session, table_key, fields) + fields, _shared_document = _ut_shared_fields( + session, table_key, fields, + workspace_document=getattr(ctx, "workspace_document", None), return_document=True) # W43C C2: store compact recurrence configuration, then derive next/preview under the # request's server day only after every field stratum is merged. _recurrence_today = time.strftime("%Y-%m-%d") @@ -1306,13 +1355,30 @@ def ut_assembly(session: Session, table_key: str, storage_key: str = "", # ⚠ A COPY OF THE BAG, because `_field_usage` MEMOISES its answer: handing the memo's own dict # out would let anything downstream write into the number the next request reads. try: - _usage = _field_usage(session, table_key, fields, defn, ops, st=st) + _usage_source = getattr(ctx, "workspace_document", None) + if not isinstance(_usage_source, dict): + _usage_source = ops.st + _usage = _field_usage(session, table_key, fields, defn, + _usage_source, ops.table_key, st=st) except Exception: # noqa: BLE001 _usage = {} if _usage: fields = [dict(f, usage=dict(_usage[f["key"]])) if isinstance(f, dict) and _usage.get(f.get("key")) else f for f in fields] + # C4 parity: user databases carry the same field-class badges as topic grids. The batch + # producer reads the grant registry once and the shared-value stratum once for the full list. + try: + from core import field_permissions as _fp + _classes = _fp.field_classes( + fields, session.uname, table_key=table_key, grant_topic=table_key, + values_shared=((_shared_document.get("fields") or {}).keys() + if isinstance(_shared_document, dict) else ()), st=st) + fields = [dict(f, **{"class": _classes[f["key"]]}) + if isinstance(f, dict) and _classes.get(f.get("key")) else f + for f in fields] + except Exception: # noqa: BLE001 + pass # ⭐⭐ C8's `descriptionEdited`, AND IT CLOSES A CONSUMER THAT WAS ALREADY WRITTEN. # W41-T11 built the producer (`user_tables.description_edited`, R19's custody mark scoped to the # description) and correctly stopped at its own fence, which is one core file. Measured across @@ -1339,10 +1405,8 @@ def ut_assembly(session: Session, table_key: str, storage_key: str = "", shared_cells = _ut_shared_cells(session, table_key, pids) if with_rows else {} # ⭐⭐ W36-T21 / R6 — the READ half, in `grid_assembly`'s own position: after `workspace_wire`, # so the closure covers this user's `custom_` and `measure_` columns too. + fields = _ut_contract_fields(table_key, fields, st=st) fields, rows_src, hidden = _ut_field_wall(session, table_key, fields, rows_src, st=st) - # The canonical Zone field is derived from the same table-scoped store as the map DTO, so - # FilterBuilder and saved Views use their ordinary multiselect path rather than a map-only one. - fields = _ut().with_zone_field(table_key, fields, st=st) if hidden and shared_cells: # ⛔ BOTH WIRES, AND THE SHARED STRATUM IS A THIRD ONE. `strip_row` above cleaned the # DEFINITION rows; these cells arrived by a different door and one narrowing cannot speak @@ -1688,6 +1752,18 @@ def list_tables(brief: bool = Query(False), # picker renders this order verbatim. out.extend(_module_link_targets(session, meta, st=lent)) out.sort(key=lambda r: (r.get("label") or "").lower()) + else: + # C1's Loopable folder has virtual descriptor rows even before the first Route/Zone + # exists. They deliberately do NOT enter `brief=1`: stops are a protected relation, + # never a generic Link target. + try: + session.require("customer_data") + except Exception: + pass + else: + from core import route_system + out.extend(route_system.system_table_descriptors()) + out.sort(key=lambda r: (r.get("label") or "").lower()) return {"tables": out} @@ -1837,6 +1913,15 @@ def create_table(body: dict = Body(default=None), if not session.runtime.available(): raise err(503, "store_unavailable", "the tenant store is unavailable. Nothing was created") + # C8b: table creation is the bulk field-definition door. `clean_fields` deliberately drops + # one invalid field while preserving valid siblings, so the route must narrate an over-cap + # option list before that validator runs; otherwise the whole column disappears under 201. + for raw_field in (body.get("fields") or []): + if not isinstance(raw_field, dict): + continue + option_refusal = ut.option_cap_refusal(raw_field.get("options")) + if option_refusal: + raise err(400, "refused", option_refusal) source = body.get("source") try: key = ut.create(label, session.uname, fields=body.get("fields"), @@ -1862,6 +1947,7 @@ def patch_table(table_key: str, body: dict = Body(default=None), picker that is confidently wrong rather than obviously stale. Posted to the wave doc as an amendment for whoever owns that door. """ + _managed_or_refuse(session, table_key, mutation=True) defn = _defn_or_refuse(session, table_key) ut = _ut() if not (session.admin or defn.get("createdBy") == session.uname): @@ -1935,6 +2021,7 @@ def delete_table(table_key: str, session: Session = Depends(require_session)): DISABLES bound automations with a status note — never deletes them. The client's confirm dialog disclosed `/footprint` first; the server cannot tell a click from a plan, so the dialog is a product requirement, not a formality.""" + _managed_or_refuse(session, table_key, mutation=True) defn = _defn_or_refuse(session, table_key) if not (session.admin or defn.get("createdBy") == session.uname): raise err(403, "forbidden", "only the database's creator or an admin can delete it") @@ -2137,7 +2224,7 @@ def table_rows(table_key: str, session: Session = Depends(require_session)): _zones = [dict(zone, memberships=[pid for pid in zone.get("memberships", []) if str(pid) in _zone_pids]) for zone in _ut().zones(table_key, st=_lent)] - return {"fields": g["fields"], "rows": rows, "today": g["today"], + return {"fields": _field_axes(g["fields"]), "rows": rows, "today": g["today"], "zones": _zones, "docs": _ge.docs_for(g["pids"], scope_key=table_key, uname=session.uname, admin=session.admin, st=session.runtime), @@ -2227,6 +2314,7 @@ def add_row(table_key: str, body: dict = Body(default=None), that requested `rid: 7` and got 12 because 7 had been re-used must find that out from the response rather than assume; the client re-anchors on what came back. """ + _managed_or_refuse(session, table_key, mutation=True) _records_or_refuse(session, table_key) ut = _ut() values = (body or {}).get("values") or {} @@ -2274,6 +2362,7 @@ def import_rows(table_key: str, body: dict = Body(default=None), profile cell validates, because a half-imported file is the worst outcome available: the user cannot tell which rows landed without reconciling the spreadsheet by hand. """ + _managed_or_refuse(session, table_key, mutation=True) _records_or_refuse(session, table_key) ut = _ut() rows_in = (body or {}).get("rows") @@ -2377,6 +2466,7 @@ def _field_or_refuse(session, table_key, fkey=""): @router.post("/tables/{table_key}/fields", status_code=201) def add_field(table_key: str, body: dict = Body(default=None), session: Session = Depends(require_session)): + _managed_or_refuse(session, table_key, mutation=True) _field_or_refuse(session, table_key) ut = _ut() field = ut.add_field(table_key, body or {}, st=session.runtime) @@ -2444,6 +2534,9 @@ def _refusal_sentence(ut, session, body, table_key="", fkey=""): # column would otherwise be explained by the flow law -- "pick a flow, or make this an # ordinary column" -- which is the D-46 misdirection exactly, pointing at a control the user # never touched. + option_refusal = ut.option_cap_refusal((body or {}).get("options")) + if option_refusal: + return option_refusal if str((body or {}).get("type") or "").strip().lower() == "ai_enrich": bag = (body or {}).get("aiEnrich") if not isinstance(bag, dict) or not str(bag.get("prompt") or "").strip(): @@ -2495,9 +2588,12 @@ def patch_field(table_key: str, fkey: str, body: dict = Body(default=None), Blue to Navy" from "deleted Blue, added Navy" — and guessing wrong empties the column and every saved view that filtered on it. """ + _managed_or_refuse(session, table_key, mutation=True) _field_or_refuse(session, table_key, fkey) ut = _ut() body = body or {} + if 'deletionLocked' in body and not session.admin: + raise err(403, "owner_required", "only an Owner can change a field deletion lock") migrated = None renames = body.get("renames") if renames: @@ -2620,6 +2716,7 @@ def enrich_field(table_key: str, fkey: str, body: dict = Body(default=None), @router.delete("/tables/{table_key}/fields/{fkey}") def delete_field(table_key: str, fkey: str, session: Session = Depends(require_session)): + _managed_or_refuse(session, table_key, mutation=True) _field_or_refuse(session, table_key, fkey) if not _ut().delete_field(table_key, fkey, st=session.runtime): raise err(400, "refused", @@ -2628,6 +2725,108 @@ def delete_field(table_key: str, fkey: str, session: Session = Depends(require_s return {"deleted": fkey} +_BULK_DELETE_SENTENCES = { + "unknown_field": ("that column is not on this database. It may already have been removed, " + "so reload and check before trying again"), + "field_hidden": "this database has no column by that name that you may remove", + "last_field": ("a database must keep at least one column, so this one cannot be removed. " + "Add another column first, then remove this one"), + "delete_failed": "that column could not be removed. Reload the database and try again", +} + + +def _bulk_delete_refusal(key, code): + return {"key": key, "status": "refused", "code": code, + "reason": _BULK_DELETE_SENTENCES.get(code, _BULK_DELETE_SENTENCES["delete_failed"])} + + +def _contract_holds_field(ut, table_key, fkey, st): + fields = (ut.get(table_key, st=st) or {}).get("fields") or [] + return any(isinstance(field, dict) and str(field.get("key")) == str(fkey) + for field in fields) + + +@router.post("/tables/{table_key}/fields/delete") +@router.delete("/tables/{table_key}/fields") +def delete_fields(table_key: str, body: dict = Body(default=None), + session: Session = Depends(require_session)): + """Delete N schema fields once and return one explicit verdict for every requested key.""" + # Preserve W44's managed-database lifecycle wall on the plural door too. + _managed_or_refuse(session, table_key, mutation=True) + defn = _defn_or_refuse(session, table_key, scope_applied=True) + ut = _ut() + if not ut.is_user_table(table_key, st=session.runtime): + raise err(400, "not_a_user_table", + "only a user-created database has an editable schema. A connected source " + "owns its own columns") + raw = (body or {}).get("keys") + if not isinstance(raw, list): + raise err(400, "bad_keys", "send `keys`, the list of columns to remove") + ordered = [] + for candidate in raw: + key = str(candidate or "").strip() + if key and key not in ordered: + ordered.append(key) + if not ordered: + raise err(400, "bad_keys", "name at least one column to remove") + + from core import field_permissions, shared_overlay + declared = [field for field in (defn.get("fields") or []) if isinstance(field, dict)] + contract_keys = {str(field.get("key")) for field in declared} + hidden = _ut_hidden(session, table_key, declared) + bucket_read = True + try: + shared_defs = shared_overlay.fields(table_key, st=session.runtime) or {} + except Exception: # noqa: BLE001 + shared_defs, bucket_read = {}, False + + results = [] + landed = [] + for key in ordered: + field = next((item for item in declared if str(item.get("key")) == key), None) + if field is None and isinstance(shared_defs.get(key), dict): + field = shared_defs[key] + if field is None: + results.append(_bulk_delete_refusal( + key, "unknown_field" if bucket_read else "delete_failed")) + continue + if key in hidden: + results.append(_bulk_delete_refusal(key, "field_hidden")) + continue + refusal = ut.delete_field_refusal( + table_key, key, session.uname, session.admin, session.runtime, + field=field, grant_topic=table_key) + if refusal is not None: + results.append({"key": key, "status": "refused", "code": refusal, + "reason": _delete_field_sentence( + refusal, str(field.get("createdBy") or ""))}) + continue + try: + dropped = False + if key in contract_keys: + dropped = bool(ut.delete_field(table_key, key, st=session.runtime)) + if not dropped and _contract_holds_field(ut, table_key, key, session.runtime): + results.append(_bulk_delete_refusal(key, "last_field")) + continue + removed = field_permissions.delete_field_everywhere( + f"{table_key}_table_workspace", table_key, table_key, key, + st=session.runtime) + except Exception: # noqa: BLE001 + results.append(_bulk_delete_refusal(key, "delete_failed")) + continue + if not (dropped or removed.get("shared") or removed.get("personal")): + results.append(_bulk_delete_refusal(key, "delete_failed")) + continue + landed.append(key) + results.append({"key": key, "status": "deleted", "contract": dropped, + "removed": removed}) + + if landed: + _refresh_relations(session) + return {"ok": True, "rerender": True, "results": results, "deleted": landed, + "refused": [item["key"] for item in results if item["status"] == "refused"]} + + @router.delete("/tables/{table_key}/rows/{rid}") def delete_row(table_key: str, rid: str, session: Session = Depends(require_session)): # ⭐⭐ W31 QA — ONE SNAPSHOT FOR THE WHOLE DELETE, and the owner reported what it cost. @@ -2643,6 +2842,7 @@ def delete_row(table_key: str, rid: str, session: Session = Depends(require_sess # ⚠ `flush='sync'` IS DELIBERATELY UNTOUCHED (D-118): nobody spams a delete, and an eventually # consistent delete is indistinguishable from one that did not work. This makes the guard # cheap; it does not make the commit optimistic. + _managed_or_refuse(session, table_key, mutation=True) lent = _ut().lend(session.runtime) _records_or_refuse(session, table_key, st=lent) # ⭐⭐ W36-T21 — THE ROW SCOPE, ON THE DELETE DOOR. `patch_row` has asked this since it was @@ -2678,6 +2878,7 @@ def patch_row(table_key: str, pid: int, body: dict = Body(default=None), updates = dict(body or {}) if not updates: raise err(400, "empty_patch", "no fields to update") + _managed_or_refuse(session, table_key, mutation=True) _records_or_refuse(session, table_key) g = ut_assembly(session, table_key, consume_corrections=False) if pid not in g["pids"]: diff --git a/api/voice_control_plane.py b/api/voice_control_plane.py new file mode 100644 index 0000000000000000000000000000000000000000..c0ae2310ad73a3fd82cb57a14f8dcd6708549f40 --- /dev/null +++ b/api/voice_control_plane.py @@ -0,0 +1,745 @@ +"""Provider-neutral Voice records, redacted browser DTOs, and a no-dial harness. + +Voice configuration is inert by construction. Browser drafts cannot carry credentials, raw +numbers, SIP/routing material, webhook controls, or raw provider payloads. Provider inventory is +reconciled only through :mod:`voice_provider_adapters`' injected GET-only seam. Ordinary +automation/webhook/schedule execution has no dispatch door here; a dry-run resolves a saved View, +records a normalized local event, and always reports ``noExternalDispatch: true``. +""" +from __future__ import annotations + +import copy +import hashlib +import re +import secrets +import time +from typing import Any, Callable + +import voice_provider_adapters as adapters + + +STORE_KEY = "voice_control_plane" +SCHEMA_VERSION = 1 +RECORD_TYPES = ( + "voice_provider_connection", + "voice_agent", + "voice_number", + "voice_number_route", + "voice_knowledge_profile", + "voice_provider_sync", + "voice_outbound_lease", + "voice_provider_event", +) +SAFE_LIFECYCLES = frozenset({"draft", "needs_attention", "inactive"}) +EMPTY_POLICIES = frozenset({"skip", "refuse"}) +MAX_RECIPIENTS = 1000 +MAX_LEASE_SECONDS = 300 + +_SENSITIVE_KEYS = frozenset({ + "credential", "credentials", "secret", "secrets", "apikey", "api_key", "token", + "authorization", "sip", "sipcredential", "sipcredentials", "webhook", "webhookurl", + "rawnumber", "phonenumber", "phone_number", "e164", "inboundroute", "routing", +}) +_PHONE = re.compile(r"\+?\d[\d .()\-]{5,}\d") +_ID = re.compile(r"^[a-z][a-z0-9_\-]{0,119}$") + + +class VoiceRefusal(ValueError): + """A request was structurally unsafe; its message is fit for the API client.""" + + +def _text(value: Any, limit: int = 160) -> str: + return " ".join(str(value or "").split())[:limit] + + +def _now(value: Any = None) -> int: + try: + return int(value if value is not None else time.time()) + except (TypeError, ValueError): + return int(time.time()) + + +def _mint(prefix: str) -> str: + return f"{prefix}_{secrets.token_urlsafe(9)}" + + +def _empty_document() -> dict[str, Any]: + return {"schemaVersion": SCHEMA_VERSION, **{name: {} for name in RECORD_TYPES}} + + +def _normalise_document(raw: Any) -> dict[str, Any]: + """Read the v1 document and project the pre-v1 flat draft bucket without data loss.""" + if isinstance(raw, dict) and raw.get("schemaVersion") == SCHEMA_VERSION: + doc = copy.deepcopy(raw) + for name in RECORD_TYPES: + if not isinstance(doc.get(name), dict): + doc[name] = {} + return doc + doc = _empty_document() + if not isinstance(raw, dict): + return doc + for key, legacy in raw.items(): + if str(key).startswith("_") or not isinstance(legacy, dict): + continue + agent_id = str(legacy.get("id") or key) + if not agent_id: + continue + doc["voice_agent"][agent_id] = { + "id": agent_id, + "tenant_id": "", + "display_name": _text(legacy.get("displayName"), 120), + "lifecycle": str(legacy.get("activationState") or "draft"), + "provider_connection_id": "", + "external_agent_ref": "", + "direction_policy": ["inbound"] if legacy.get("callbackOnly", True) + else ["inbound", "outbound"], + "core_config": { + "purpose": _text(legacy.get("purpose"), 240), + "callbackPolicy": "callbacks_only" if legacy.get("callbackOnly", True) else "", + }, + "provider_extension": {"provider": "retell", "version": 1, + "config": dict(legacy.get("supportedConfiguration") or {})}, + "knowledge_profile_id": "", + "audience": _clean_audience({}), + "created_at": _now(legacy.get("createdAt")), + "updated_at": _now(legacy.get("updatedAt")), + } + return doc + + +def _store(rt: Any) -> dict[str, Any]: + try: + return _normalise_document(rt.get(STORE_KEY)) + except Exception: + return _empty_document() + + +def _update(rt: Any, fn: Callable[[dict[str, Any]], dict[str, Any]]) -> Any: + def wrapped(raw: Any) -> dict[str, Any]: + return fn(_normalise_document(raw)) + return rt.update(STORE_KEY, wrapped, flush="async") + + +def _reject_sensitive(value: Any) -> None: + if isinstance(value, dict): + for key, item in value.items(): + norm = str(key).replace("-", "").lower() + if norm in _SENSITIVE_KEYS: + raise VoiceRefusal( + "Voice credentials, SIP, webhooks, routing, and raw phone controls are server-only") + _reject_sensitive(item) + elif isinstance(value, list): + for item in value: + _reject_sensitive(item) + elif isinstance(value, str) and _PHONE.search(value): + raise VoiceRefusal("raw phone numbers are not accepted by the browser Voice API") + + +def _safe_id(value: Any, label: str, allow_empty: bool = True) -> str: + result = str(value or "").strip()[:120] + if not result and allow_empty: + return "" + if not result or not _ID.fullmatch(result): + raise VoiceRefusal(f"{label} is not a valid Loopable reference") + return result + + +def _clean_directions(value: Any) -> list[str]: + if value in (None, ""): + return ["inbound"] + if not isinstance(value, list): + raise VoiceRefusal("direction policy must be a list") + result: list[str] = [] + for item in value: + key = str(item or "").strip().lower() + if key not in {"inbound", "outbound"}: + raise VoiceRefusal("direction policy accepts inbound and outbound only") + if key not in result: + result.append(key) + if not result: + raise VoiceRefusal("select at least one permitted Voice direction") + return result + + +def _clean_string_list(value: Any, label: str, limit: int = 12) -> list[str]: + if value in (None, ""): + return [] + if not isinstance(value, list) or len(value) > limit: + raise VoiceRefusal(f"{label} must be a list of at most {limit} items") + return [item for item in (_text(v, 160) for v in value) if item] + + +def _clean_core(raw: Any, previous: Any = None) -> dict[str, Any]: + previous = previous if isinstance(previous, dict) else {} + if raw in (None, ""): + return copy.deepcopy(previous) + if not isinstance(raw, dict): + raise VoiceRefusal("portable Voice configuration must be an object") + allowed = { + "role", "purpose", "businessIdentity", "languagePolicy", "transferPolicy", + "callbackPolicy", "safetyBoundaries", "consentMode", "outcomeSchema", + } + unknown = sorted(set(raw) - allowed) + if unknown: + raise VoiceRefusal("unsupported portable Voice setting: " + ", ".join(unknown)) + merged = {**previous, **raw} + out: dict[str, Any] = {} + for key in ("role", "purpose", "businessIdentity", "languagePolicy", "transferPolicy", + "callbackPolicy", "consentMode"): + out[key] = _text(merged.get(key), 500 if key == "purpose" else 240) + out["safetyBoundaries"] = _clean_string_list( + merged.get("safetyBoundaries"), "safety boundaries") + schema = merged.get("outcomeSchema") + if schema in (None, ""): + out["outcomeSchema"] = [] + elif isinstance(schema, list) and len(schema) <= 20: + out["outcomeSchema"] = [ + {"key": _safe_id((row or {}).get("key"), "outcome field", allow_empty=False), + "label": _text((row or {}).get("label"), 80), + "type": str((row or {}).get("type") or "text") + if str((row or {}).get("type") or "text") in {"text", "number", "boolean"} + else "text"} + for row in schema if isinstance(row, dict) + ] + else: + raise VoiceRefusal("outcome schema must be a list of at most 20 fields") + return out + + +def _clean_audience(raw: Any, previous: Any = None) -> dict[str, Any]: + previous = previous if isinstance(previous, dict) else {} + if raw in (None, ""): + raw = {} + if not isinstance(raw, dict): + raise VoiceRefusal("Voice audience must be an object") + allowed = {"tableKey", "viewId", "viewRevision", "phoneFieldKey", + "emptyPolicy", "maxRecipients"} + unknown = sorted(set(raw) - allowed) + if unknown: + raise VoiceRefusal("unsupported Voice audience setting: " + ", ".join(unknown)) + merged = {**previous, **raw} + empty_policy = str(merged.get("emptyPolicy") or "skip").strip().lower() + if empty_policy not in EMPTY_POLICIES: + raise VoiceRefusal("empty phone values must be skipped or refused") + try: + maximum = int(merged.get("maxRecipients") or 100) + except (TypeError, ValueError): + raise VoiceRefusal("maximum recipients must be a whole number") from None + if maximum < 1 or maximum > MAX_RECIPIENTS: + raise VoiceRefusal(f"maximum recipients must be between 1 and {MAX_RECIPIENTS}") + revision = merged.get("viewRevision") + if revision not in (None, "") and not isinstance(revision, (str, int)): + raise VoiceRefusal("View revision must be text or a number") + return { + "tableKey": _safe_id(merged.get("tableKey"), "database", allow_empty=True), + "viewId": _safe_id(merged.get("viewId"), "View", allow_empty=True), + "viewRevision": str(revision or "")[:120], + "phoneFieldKey": _safe_id(merged.get("phoneFieldKey"), "phone field", allow_empty=True), + "emptyPolicy": empty_policy, + "maxRecipients": maximum, + } + + +def _provider_for(doc: dict[str, Any], connection_id: str, extension: Any) -> str: + connection = doc["voice_provider_connection"].get(connection_id) if connection_id else None + if connection_id and not isinstance(connection, dict): + raise VoiceRefusal("select a Voice provider connection available to this workspace") + provider = str((connection or {}).get("provider") or "").lower() + if isinstance(extension, dict): + requested = str(extension.get("provider") or "").strip().lower() + if requested and provider and requested != provider: + raise VoiceRefusal("provider extension does not match the selected connection") + provider = provider or requested + return provider + + +def _agent_record(doc: dict[str, Any], raw: Any, previous: Any = None, + actor: str = "") -> dict[str, Any]: + raw = raw if isinstance(raw, dict) else {} + previous = previous if isinstance(previous, dict) else {} + _reject_sensitive(raw) + forbidden = {"phoneBindings", "numberBinding", "externalAgentRef", "provider", "route"} + if forbidden & set(raw): + raise VoiceRefusal("provider identity and number routing are server-managed") + name = _text(raw.get("displayName", previous.get("display_name")), 120) + if not name: + raise VoiceRefusal("name the Voice agent") + lifecycle = str(raw.get("lifecycle", raw.get("activationState", + previous.get("lifecycle", "draft")))).strip().lower() + if lifecycle not in SAFE_LIFECYCLES: + raise VoiceRefusal("Voice activation is unavailable until an explicit verified activation gate") + connection_id = _safe_id(raw.get("providerConnectionId", + previous.get("provider_connection_id")), + "provider connection", allow_empty=True) + extension_raw = raw.get("providerExtension", previous.get("provider_extension") or {}) + provider = _provider_for(doc, connection_id, extension_raw) + if provider: + try: + extension = adapters.clean_extension(provider, extension_raw) + except adapters.ProviderAdapterRefusal as exc: + raise VoiceRefusal(str(exc)) from exc + else: + extension = {"provider": "", "version": 1, "config": {}} + core_raw = raw.get("portableConfig") + if core_raw is None and any(key in raw for key in ("purpose", "callbackOnly")): + core_raw = dict(previous.get("core_config") or {}) + if "purpose" in raw: + core_raw["purpose"] = raw.get("purpose") + if "callbackOnly" in raw: + core_raw["callbackPolicy"] = "callbacks_only" if raw.get("callbackOnly") else "" + core = _clean_core(core_raw, previous.get("core_config")) + directions_raw = raw.get("directionPolicy") + if directions_raw is None and "callbackOnly" in raw: + directions_raw = ["inbound"] if raw.get("callbackOnly") else ["inbound", "outbound"] + directions = _clean_directions(directions_raw if directions_raw is not None + else previous.get("direction_policy")) + audience = _clean_audience(raw.get("audience"), previous.get("audience")) + knowledge_id = _safe_id(raw.get("knowledgeProfileId", + previous.get("knowledge_profile_id")), + "knowledge profile", allow_empty=True) + if knowledge_id and knowledge_id not in doc["voice_knowledge_profile"]: + raise VoiceRefusal("select a Voice knowledge profile available to this workspace") + stamp = _now() + return { + "id": str(previous.get("id") or _mint("va")), + "tenant_id": str(previous.get("tenant_id") or ""), + "display_name": name, + "lifecycle": lifecycle, + "provider_connection_id": connection_id, + "external_agent_ref": str(previous.get("external_agent_ref") or ""), + "direction_policy": directions, + "core_config": core, + "provider_extension": extension, + "knowledge_profile_id": knowledge_id, + "audience": audience, + "created_by": str(previous.get("created_by") or actor or "")[:120], + "updated_by": str(actor or previous.get("updated_by") or "")[:120], + "created_at": _now(previous.get("created_at")) if previous else stamp, + "updated_at": stamp, + } + + +def save_draft(rt: Any, raw: Any, actor: str = "") -> dict[str, Any]: + result: dict[str, Any] = {} + def up(doc: dict[str, Any]) -> dict[str, Any]: + rec = _agent_record(doc, raw, actor=actor) + doc["voice_agent"][rec["id"]] = rec + result.update(rec) + return doc + _update(rt, up) + return result + + +def patch_draft(rt: Any, voice_id: str, raw: Any, actor: str = "") -> dict[str, Any]: + result: dict[str, Any] = {} + def up(doc: dict[str, Any]) -> dict[str, Any]: + previous = doc["voice_agent"].get(str(voice_id)) + if not isinstance(previous, dict): + raise VoiceRefusal("unknown Voice agent") + rec = _agent_record(doc, raw, previous, actor=actor) + doc["voice_agent"][rec["id"]] = rec + result.update(rec) + return doc + _update(rt, up) + return result + + +def server_register_provider_connection(rt: Any, provider: str, secret_ref: str, + tenant_id: str = "", connection_id: str = "") -> dict[str, Any]: + """Internal server seam. ``secret_ref`` names a secret; it is never a browser value.""" + key = str(provider or "").strip().lower() + if not adapters.provider_label(key): + raise VoiceRefusal("select Retell or ElevenLabs") + ref = str(secret_ref or "").strip() + if not re.fullmatch(r"(?:env|keychain|secret):[A-Z0-9_./\-]{3,160}", ref): + raise VoiceRefusal("provider credential must be a server-side secret reference") + rec = { + "id": _safe_id(connection_id, "provider connection") or _mint("vpc"), + "tenant_id": str(tenant_id or "")[:120], + "provider": key, "secret_ref": ref, "status": "unconfigured", + "last_probe_at": 0, "created_at": _now(), "updated_at": _now(), + } + def up(doc: dict[str, Any]) -> dict[str, Any]: + doc["voice_provider_connection"][rec["id"]] = rec + return doc + _update(rt, up) + return copy.deepcopy(rec) + + +def server_reconcile_provider(rt: Any, connection_id: str, + transport: adapters.Transport) -> dict[str, Any]: + """Perform GET-only inventory and persist redacted sync outcomes, never provider payloads.""" + doc = _store(rt) + connection = doc["voice_provider_connection"].get(str(connection_id)) + if not isinstance(connection, dict): + raise VoiceRefusal("unknown Voice provider connection") + try: + observed = adapters.reconcile(connection.get("provider"), transport) + except adapters.ProviderAdapterRefusal as exc: + raise VoiceRefusal(str(exc)) from exc + at = _now() + def up(current: dict[str, Any]) -> dict[str, Any]: + rec = current["voice_provider_connection"].get(str(connection_id)) + if not isinstance(rec, dict): + raise VoiceRefusal("unknown Voice provider connection") + rec = {**rec, "status": observed["status"], "last_probe_at": at, "updated_at": at} + current["voice_provider_connection"][str(connection_id)] = rec + for row in observed["checks"]: + sync_id = _mint("vps") + current["voice_provider_sync"][sync_id] = { + "id": sync_id, "tenant_id": rec.get("tenant_id") or "", + "provider": rec.get("provider") or "", "resource_type": row["resource"], + "external_ref": "", "profile_version": "", "status": row["status"], + "count": row["count"], "error_class": row.get("errorClass") or "", + "observed_at": at, "raw_payload_stored": False, + } + return current + _update(rt, up) + return copy.deepcopy(observed) + + +def server_register_knowledge_profile(rt: Any, version: str, content_hash: str, + classification: str, approval_state: str, + tenant_id: str = "", profile_id: str = "") -> dict[str, Any]: + if approval_state not in {"draft", "approved", "revoked"}: + raise VoiceRefusal("knowledge approval state is invalid") + digest = str(content_hash or "").lower() + if not re.fullmatch(r"[0-9a-f]{32,64}", digest): + raise VoiceRefusal("knowledge profile needs a content hash, never raw content") + rec = {"id": _safe_id(profile_id, "knowledge profile") or _mint("vkp"), + "tenant_id": str(tenant_id or "")[:120], "version": _text(version, 80), + "content_hash": digest, "classification": _text(classification, 80), + "approval_state": approval_state, "created_at": _now(), "updated_at": _now()} + def up(doc: dict[str, Any]) -> dict[str, Any]: + doc["voice_knowledge_profile"][rec["id"]] = rec + return doc + _update(rt, up) + return copy.deepcopy(rec) + + +def server_bind_agent_ref(rt: Any, voice_id: str, external_ref: str) -> dict[str, Any]: + """Internal opaque provider binding. The external reference never serializes to the browser.""" + ref = str(external_ref or "").strip()[:240] + if not ref or _PHONE.search(ref): + raise VoiceRefusal("provider agent binding must be an opaque server reference") + result: dict[str, Any] = {} + def up(doc: dict[str, Any]) -> dict[str, Any]: + rec = doc["voice_agent"].get(str(voice_id)) + if not isinstance(rec, dict): + raise VoiceRefusal("unknown Voice agent") + rec = {**rec, "external_agent_ref": ref, "updated_at": _now()} + doc["voice_agent"][str(voice_id)] = rec + result.update(rec) + return doc + _update(rt, up) + return result + + +def _masked_display(value: Any) -> str: + display = _text(value, 80) + if not display or _PHONE.search(display) or not any(ch in display for ch in "*•xX"): + raise VoiceRefusal("phone display must already be masked") + return display + + +def server_bind_number(rt: Any, voice_id: str, opaque_id: str, masked_number: str, + active: bool = False, carrier_provider: str = "server") -> dict[str, Any]: + """Internal-only number/route seam enforcing exactly one active inbound owner per DID.""" + external = str(opaque_id or "").strip()[:240] + if not external or _PHONE.search(external): + raise VoiceRefusal("a phone binding needs an opaque server reference") + masked = _masked_display(masked_number) + result: dict[str, Any] = {} + def up(doc: dict[str, Any]) -> dict[str, Any]: + agent = doc["voice_agent"].get(str(voice_id)) + if not isinstance(agent, dict): + raise VoiceRefusal("unknown Voice agent") + number = next((row for row in doc["voice_number"].values() + if isinstance(row, dict) and row.get("external_number_ref") == external), None) + if not number: + nid = _mint("vn") + number = {"id": nid, "tenant_id": agent.get("tenant_id") or "", + "carrier_provider": _text(carrier_provider, 40), + "external_number_ref": external, "masked_display": masked, + "status": "verified", "created_at": _now(), "updated_at": _now()} + doc["voice_number"][nid] = number + existing = [row for row in doc["voice_number_route"].values() + if isinstance(row, dict) and row.get("voice_number_id") == number["id"] + and row.get("direction") == "inbound" and row.get("status") == "active" + and row.get("voice_agent_id") != str(voice_id)] + if active and existing: + raise VoiceRefusal("that phone already has an active inbound owner") + route = next((row for row in doc["voice_number_route"].values() + if isinstance(row, dict) and row.get("voice_number_id") == number["id"] + and row.get("direction") == "inbound" + and row.get("voice_agent_id") == str(voice_id)), None) + rid = str((route or {}).get("id") or _mint("vnr")) + provider = str((agent.get("provider_extension") or {}).get("provider") or "") + route = {"id": rid, "tenant_id": agent.get("tenant_id") or "", + "voice_number_id": number["id"], "direction": "inbound", + "selected_provider": provider, "voice_agent_id": str(voice_id), + "status": "active" if active else "inactive", "callback_window": "", + "created_at": _now((route or {}).get("created_at")), "updated_at": _now()} + doc["voice_number_route"][rid] = route + result.update({"voiceAgentId": str(voice_id), "numberId": number["id"], + "maskedDisplay": masked, "active": bool(active), "routeId": rid}) + return doc + _update(rt, up) + return result + + +def acquire_outbound_lease(rt: Any, voice_id: str, number_id: str, run_ref: str, + capacity: int = 1, ttl_seconds: int = 60, now: Any = None) -> dict[str, Any]: + """Atomically acquire one per-DID permit. No dispatcher exists in this module.""" + stamp = _now(now) + try: + cap, ttl = int(capacity), int(ttl_seconds) + except (TypeError, ValueError): + raise VoiceRefusal("Voice lease capacity and duration must be whole numbers") from None + if cap < 1 or cap > 100 or ttl < 1 or ttl > MAX_LEASE_SECONDS: + raise VoiceRefusal("Voice lease capacity or duration is outside the safe bound") + run = _safe_id(run_ref, "run", allow_empty=False) + result: dict[str, Any] = {} + def up(doc: dict[str, Any]) -> dict[str, Any]: + agent = doc["voice_agent"].get(str(voice_id)) + number = doc["voice_number"].get(str(number_id)) + if not isinstance(agent, dict) or not isinstance(number, dict): + raise VoiceRefusal("unknown Voice agent or number") + if "outbound" not in (agent.get("direction_policy") or []): + raise VoiceRefusal("this Voice agent is not permitted for outbound work") + for lid, lease in list(doc["voice_outbound_lease"].items()): + if isinstance(lease, dict) and lease.get("state") == "active" \ + and _now(lease.get("expires_at")) <= stamp: + doc["voice_outbound_lease"][lid] = {**lease, "state": "expired"} + active = [lease for lease in doc["voice_outbound_lease"].values() + if isinstance(lease, dict) and lease.get("voice_number_id") == str(number_id) + and lease.get("state") == "active" and _now(lease.get("expires_at")) > stamp] + if len(active) >= cap: + raise VoiceRefusal("outbound capacity for that phone is already leased") + lid = _mint("vol") + lease = {"id": lid, "tenant_id": agent.get("tenant_id") or "", + "voice_number_id": str(number_id), + "provider": str((agent.get("provider_extension") or {}).get("provider") or ""), + "voice_agent_id": str(voice_id), "run_ref": run, + "permit": secrets.token_urlsafe(18), "expires_at": stamp + ttl, + "state": "active", "created_at": stamp, "updated_at": stamp} + doc["voice_outbound_lease"][lid] = lease + result.update(lease) + return doc + _update(rt, up) + return result + + +def release_outbound_lease(rt: Any, lease_id: str, permit: str, now: Any = None) -> dict[str, Any]: + result: dict[str, Any] = {} + def up(doc: dict[str, Any]) -> dict[str, Any]: + lease = doc["voice_outbound_lease"].get(str(lease_id)) + if not isinstance(lease, dict) or not secrets.compare_digest( + str(lease.get("permit") or ""), str(permit or "")): + raise VoiceRefusal("unknown outbound lease permit") + lease = {**lease, "state": "released", "updated_at": _now(now)} + doc["voice_outbound_lease"][str(lease_id)] = lease + result.update(lease) + return doc + _update(rt, up) + return result + + +def _connection_dto(connection: Any) -> dict[str, Any] | None: + if not isinstance(connection, dict): + return None + key = str(connection.get("provider") or "") + return {"id": str(connection.get("id") or ""), "provider": key, + "label": adapters.provider_label(key), + "status": str(connection.get("status") or "unconfigured"), + "lastProbeAt": _now(connection.get("last_probe_at"))} + + +def _knowledge_dto(profile: Any) -> dict[str, Any] | None: + if not isinstance(profile, dict): + return None + return {"id": str(profile.get("id") or ""), "version": _text(profile.get("version"), 80), + "classification": _text(profile.get("classification"), 80), + "approvalState": str(profile.get("approval_state") or "draft")} + + +def _binding_dto(doc: dict[str, Any], voice_id: str) -> dict[str, Any] | None: + routes = [row for row in doc["voice_number_route"].values() + if isinstance(row, dict) and row.get("voice_agent_id") == str(voice_id) + and row.get("direction") == "inbound"] + if not routes: + return None + route = sorted(routes, key=lambda row: (row.get("status") != "active", + -_now(row.get("updated_at"))))[0] + number = doc["voice_number"].get(str(route.get("voice_number_id") or "")) or {} + return {"numberId": str(number.get("id") or ""), + "maskedDisplay": _text(number.get("masked_display"), 80), + "numberStatus": str(number.get("status") or "unconfigured"), + "routeStatus": str(route.get("status") or "inactive")} + + +def browser_dto(record: Any, document: Any = None) -> dict[str, Any]: + """The complete allow-list for data that may cross into the browser.""" + rec = record if isinstance(record, dict) else {} + doc = _normalise_document(document) + connection = doc["voice_provider_connection"].get(str(rec.get("provider_connection_id") or "")) + knowledge = doc["voice_knowledge_profile"].get(str(rec.get("knowledge_profile_id") or "")) + binding = _binding_dto(doc, str(rec.get("id") or "")) + audience = _clean_audience(rec.get("audience") or {}) + reasons: list[str] = [] + if not connection or connection.get("status") != "connected": + reasons.append("provider_connection") + if not all(audience.get(key) for key in ("tableKey", "viewId", "phoneFieldKey")): + reasons.append("audience") + if not knowledge or knowledge.get("approval_state") != "approved": + reasons.append("knowledge_profile") + if not binding or binding.get("routeStatus") != "active": + reasons.append("inbound_binding") + if rec.get("lifecycle") != "inactive": + reasons.append("activation_gate") + core = copy.deepcopy(rec.get("core_config") or {}) + extension = copy.deepcopy(rec.get("provider_extension") or + {"provider": "", "version": 1, "config": {}}) + callback_only = (rec.get("direction_policy") or ["inbound"]) == ["inbound"] + return { + "schemaVersion": SCHEMA_VERSION, + "automationId": str(rec.get("id") or ""), "voiceAgentId": str(rec.get("id") or ""), + "displayName": _text(rec.get("display_name", rec.get("displayName")), 120), + "agentType": "voice", "lifecycle": str(rec.get("lifecycle") or "draft"), + "readiness": {"state": "configured" if not reasons else "needs_attention", + "reasons": reasons, "productionDispatch": False}, + "providerConnectionId": str(rec.get("provider_connection_id") or ""), + "providerConnection": _connection_dto(connection), "portableConfig": core, + "providerExtension": extension, + "opaqueProviderRefs": {"agentBound": bool(rec.get("external_agent_ref"))}, + "directionPolicy": list(rec.get("direction_policy") or ["inbound"]), + "knowledgeProfileId": str(rec.get("knowledge_profile_id") or ""), + "knowledgeProfile": _knowledge_dto(knowledge), "phoneBinding": binding, + "audience": audience, + "capabilities": {"dryRun": True, "numberProvisioning": "contact_loopable", + "activation": False, "externalDispatch": False}, + "connectionStatus": str((connection or {}).get("status") or "unconfigured"), + "supportedConfiguration": copy.deepcopy(extension.get("config") or {}), + "purpose": _text(core.get("purpose"), 240), "callbackOnly": callback_only, + "activationState": str(rec.get("lifecycle") or "draft"), + "phoneBindings": ([{"opaqueId": binding["numberId"], + "maskedNumber": binding["maskedDisplay"], + "active": binding["routeStatus"] == "active"}] + if binding else []), + } + + +def list_browser_dtos(rt: Any) -> list[dict[str, Any]]: + doc = _store(rt) + return [browser_dto(rec, doc) for _, rec in sorted(doc["voice_agent"].items()) + if isinstance(rec, dict)] + + +def get_browser_dto(rt: Any, voice_id: str) -> dict[str, Any]: + doc = _store(rt) + rec = doc["voice_agent"].get(str(voice_id)) + if not isinstance(rec, dict): + raise VoiceRefusal("unknown Voice agent") + return browser_dto(rec, doc) + + +def browser_options(rt: Any) -> dict[str, Any]: + """Server-stored choices only; callers add permissioned database/View/field choices.""" + doc = _store(rt) + connections = [_connection_dto(row) for _, row in sorted( + doc["voice_provider_connection"].items()) if isinstance(row, dict)] + knowledge = [_knowledge_dto(row) for _, row in sorted( + doc["voice_knowledge_profile"].items()) if isinstance(row, dict)] + return {"schemaVersion": SCHEMA_VERSION, "providers": adapters.providers(), + "providerConnections": [row for row in connections if row], + "knowledgeProfiles": [row for row in knowledge if row], + "emptyPolicies": [{"key": "skip", "label": "Skip empty phone values"}, + {"key": "refuse", "label": "Refuse if any phone is empty"}], + "maxRecipientsCeiling": MAX_RECIPIENTS} + + +def _event_key(provider: str, event_ref: str) -> str: + return hashlib.sha256(f"{provider}:{event_ref}".encode("utf-8")).hexdigest()[:32] + + +def record_provider_event(rt: Any, provider: str, event_ref: str, voice_id: str, + verification_state: str, normalized_outcome: Any, + tenant_id: str = "", received_at: Any = None) -> dict[str, Any]: + """Idempotent normalized event storage. Raw transcripts/payloads have no parameter.""" + key = str(provider or "local").lower() + ref = str(event_ref or "").strip()[:240] + if not ref: + raise VoiceRefusal("provider event needs an opaque reference") + if verification_state not in {"verified", "local", "refused"}: + raise VoiceRefusal("provider event verification state is invalid") + if not isinstance(normalized_outcome, dict): + raise VoiceRefusal("provider outcome must be a normalized object") + _reject_sensitive(normalized_outcome) + eid = "vpe_" + _event_key(key, ref) + rec = {"id": eid, "provider": key, "event_ref_hash": _event_key(key, ref), + "tenant_id": str(tenant_id or "")[:120], "voice_agent_id": str(voice_id or ""), + "received_at": _now(received_at), "verification_state": verification_state, + "normalized_outcome": copy.deepcopy(normalized_outcome), "raw_payload_stored": False} + def up(doc: dict[str, Any]) -> dict[str, Any]: + doc["voice_provider_event"].setdefault(eid, rec) + return doc + _update(rt, up) + return rec + + +def dry_run(rt: Any, voice_id: str, audience: Any, + resolve_view: Callable[[dict[str, Any]], dict[str, Any]]) -> dict[str, Any]: + """Resolve one allowed saved View and record a redacted local event; never dispatch.""" + doc = _store(rt) + rec = doc["voice_agent"].get(str(voice_id)) + if not isinstance(rec, dict): + raise VoiceRefusal("unknown Voice agent") + if rec.get("lifecycle") not in SAFE_LIFECYCLES: + raise VoiceRefusal("Voice activation is unavailable until an explicit verified activation gate") + chosen = _clean_audience(audience, rec.get("audience")) + if not all(chosen.get(key) for key in ("tableKey", "viewId", "phoneFieldKey")): + raise VoiceRefusal("select a database, saved View, and phone field before dry-run") + if not callable(resolve_view): + raise VoiceRefusal("Voice dry-run needs the server View resolver") + resolved = resolve_view(copy.deepcopy(chosen)) + if not isinstance(resolved, dict): + raise VoiceRefusal("Voice View resolver returned no result") + try: + recipients = int(resolved.get("recipientCount")) + selected = int(resolved.get("selectedCount", recipients)) + except (TypeError, ValueError): + raise VoiceRefusal("Voice View resolver returned an invalid recipient count") from None + if recipients < 0 or selected < recipients: + raise VoiceRefusal("Voice View resolver returned an invalid audience") + if recipients > chosen["maxRecipients"]: + raise VoiceRefusal( + f"that View resolves to {recipients} eligible recipients; the saved maximum is " + f"{chosen['maxRecipients']}") + if chosen["emptyPolicy"] == "refuse" and selected != recipients: + raise VoiceRefusal("that View contains empty phone values and this audience refuses them") + at = _now() + event_ref = f"dry:{voice_id}:{at}:{secrets.token_urlsafe(5)}" + outcome = {"kind": "dry_run", "state": "complete", "recipientCount": recipients, + "selectedCount": selected, "externalDispatches": 0, + "noExternalDispatch": True} + event = record_provider_event( + rt, "local", event_ref, str(voice_id), "local", outcome, + tenant_id=str(rec.get("tenant_id") or ""), received_at=at) + return { + "schemaVersion": SCHEMA_VERSION, "noExternalDispatch": True, + "externalDispatches": 0, "state": "dry_run_complete", + "audience": {**chosen, "viewRevision": str(resolved.get("viewRevision") + or chosen.get("viewRevision") or "")}, + "recipientCount": recipients, "selectedCount": selected, + "maskedPreview": {"displayName": _text(rec.get("display_name"), 120), + "recipientCount": recipients}, + "preview": {"displayName": _text(rec.get("display_name"), 120), + "maskedBindingCount": 1 if _binding_dto(doc, str(voice_id)) else 0}, + "eventId": event["id"], + } + + +def refuse_external_dispatch(source: str = "automation") -> None: + """The explicit ordinary-run/webhook/schedule wall until production activation is built.""" + raise VoiceRefusal( + f"Voice {str(source or 'automation')} dispatch is not activated; use the no-dial harness") diff --git a/api/voice_provider_adapters.py b/api/voice_provider_adapters.py new file mode 100644 index 0000000000000000000000000000000000000000..a251d11252bdbbb32d6b77e9324b098e06acb758 --- /dev/null +++ b/api/voice_provider_adapters.py @@ -0,0 +1,189 @@ +"""Read-only, dependency-injected provider reconciliation for Loopable Voice. + +The adapter boundary deliberately has no HTTP implementation. A caller must inject a transport +that is already scoped to one server-side secret reference. Only documented inventory GETs are +accepted; no create, update, publish, assignment, dial, test, or delete verb can cross this module. +Results contain counts and readiness classes only, never provider identifiers or raw payloads. +""" +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any, Callable + + +class ProviderAdapterRefusal(ValueError): + """A provider or transport tried to leave the read-only reconciliation contract.""" + + +Transport = Callable[[str, str], Any] + + +@dataclass(frozen=True) +class InventoryCheck: + name: str + path: str + list_key: str | None = None + + +_CHECKS: dict[str, tuple[InventoryCheck, ...]] = { + "retell": ( + InventoryCheck("agents", "/list-agents?limit=5&is_latest=true"), + InventoryCheck("response_engines", "/v2/list-retell-llms?limit=5", "items"), + InventoryCheck("phone_numbers", "/v2/list-phone-numbers?limit=5", "items"), + InventoryCheck("knowledge_bases", "/list-knowledge-bases"), + ), + "elevenlabs": ( + InventoryCheck("agents", "/v1/convai/agents?page_size=5", "agents"), + InventoryCheck("phone_numbers", "/v1/convai/phone-numbers"), + InventoryCheck("knowledge_base_documents", + "/v1/convai/knowledge-base?page_size=5", "documents"), + ), +} + +_EXTENSIONS: dict[str, dict[str, tuple[type, ...]]] = { + "retell": { + "responseEngineRef": (str,), + "voiceRef": (str,), + "conversationFlowRef": (str,), + "responsiveness": (int, float), + "interruptionSensitivity": (int, float), + }, + "elevenlabs": { + "voiceRef": (str,), + "language": (str,), + "firstMessageMode": (str,), + "stability": (int, float), + "similarityBoost": (int, float), + }, +} + +_LABELS = {"retell": "Retell", "elevenlabs": "ElevenLabs"} + + +def providers() -> list[dict[str, Any]]: + """Browser-safe provider choices and deliberately supported extension fields.""" + return [ + { + "key": key, + "label": _LABELS[key], + "extensionVersion": 1, + "extensionFields": [ + {"key": name, "type": "number" if float in types else "text"} + for name, types in fields.items() + ], + } + for key, fields in _EXTENSIONS.items() + ] + + +def provider_label(provider: str) -> str: + return _LABELS.get(str(provider or "").lower(), "") + + +def clean_extension(provider: str, raw: Any) -> dict[str, Any]: + """Validate one typed provider bag and discard no unknown setting silently.""" + key = str(provider or "").strip().lower() + allowed = _EXTENSIONS.get(key) + if allowed is None: + raise ProviderAdapterRefusal("select Retell or ElevenLabs") + if raw in (None, ""): + return {"provider": key, "version": 1, "config": {}} + if not isinstance(raw, dict): + raise ProviderAdapterRefusal("provider extension must be an object") + version = raw.get("version", 1) + if version != 1: + raise ProviderAdapterRefusal("provider extension version must be 1") + config = raw.get("config", raw) + if not isinstance(config, dict): + raise ProviderAdapterRefusal("provider extension config must be an object") + config = {k: v for k, v in config.items() if k not in {"provider", "version"}} + unknown = sorted(set(config) - set(allowed)) + if unknown: + raise ProviderAdapterRefusal( + "unsupported provider setting: " + ", ".join(unknown)) + out: dict[str, Any] = {} + for name, value in config.items(): + if isinstance(value, bool) or not isinstance(value, allowed[name]): + expected = "a number" if float in allowed[name] else "text" + raise ProviderAdapterRefusal(f"{name} must be {expected}") + if isinstance(value, str): + value = " ".join(value.split())[:160] + elif isinstance(value, (int, float)): + value = max(0.0, min(float(value), 1.0)) + out[name] = value + return {"provider": key, "version": 1, "config": out} + + +def _status_payload(result: Any) -> tuple[int, Any]: + if isinstance(result, tuple) and len(result) == 2: + try: + return int(result[0]), result[1] + except (TypeError, ValueError): + return 0, None + if isinstance(result, dict) and "status" in result: + try: + return int(result.get("status") or 0), result.get("payload") + except (TypeError, ValueError): + return 0, None + raise ProviderAdapterRefusal("provider transport must return (status, payload)") + + +def _count(payload: Any, list_key: str | None) -> int: + if isinstance(payload, list): + return len(payload) + if isinstance(payload, dict): + candidates = payload.get(list_key) if list_key else payload.get("items") + if isinstance(candidates, list): + return len(candidates) + # Several provider list endpoints answer an object keyed by resource ids. We expose only + # its cardinality, never a key or value. + if list_key is None and payload and all(isinstance(k, str) for k in payload): + return len(payload) + return 0 + + +def reconcile(provider: str, transport: Transport) -> dict[str, Any]: + """Run the exact safe inventory reads for one provider through an injected transport.""" + key = str(provider or "").strip().lower() + checks = _CHECKS.get(key) + if checks is None: + raise ProviderAdapterRefusal("select Retell or ElevenLabs") + if not callable(transport): + raise ProviderAdapterRefusal("a server-side read transport is required") + rows: list[dict[str, Any]] = [] + for check in checks: + method = "GET" + if method != "GET": # structural tripwire: this module has no mutation verb. + raise ProviderAdapterRefusal("provider reconciliation is read-only") + try: + status, payload = _status_payload(transport(method, check.path)) + except ProviderAdapterRefusal: + raise + except Exception as exc: # noqa: BLE001 - classify, never serialize provider errors. + rows.append({"resource": check.name, "status": 0, + "count": 0, "errorClass": type(exc).__name__}) + continue + row = {"resource": check.name, "status": status, + "count": _count(payload, check.list_key)} + if status != 200: + row["errorClass"] = "http" + rows.append(row) + required = {"agents", "phone_numbers"} + healthy = all(row["status"] == 200 for row in rows) + foundation = all(any(row["resource"] == name and row["status"] == 200 + and row["count"] > 0 for row in rows) for name in required) + return { + "provider": key, + "providerLabel": _LABELS[key], + "readOnly": True, + "requests": len(rows), + "status": "connected" if healthy and foundation else "needs_attention", + "checks": rows, + "rawPayloadStored": False, + } + + +def assert_read_only_request(method: str) -> None: + """Public negative control for transports that want an explicit verb guard.""" + if str(method or "").upper() != "GET": + raise ProviderAdapterRefusal("provider reconciliation accepts GET inventory only") diff --git a/platform/aios_grid.py b/platform/aios_grid.py index 2d40b70ab58420199ed7970acd33606c1c761ac3..9c01a1f1b44c0c7f4161460ccb9e0a0843698036 100644 --- a/platform/aios_grid.py +++ b/platform/aios_grid.py @@ -211,7 +211,9 @@ def _clean_code(raw): #: Emitted with the cohort column's read-only mechanism — `source: 'odoo'` + `derived` — so #: the client never offers an editor and the host never accepts a cell write for them. READONLY_CUSTOM_TYPES = {"created_time", "formula"} -MAX_FIELD_OPTIONS = 50 +# Mirrored server persistence ceiling. ``core.route_system.MAX_OPTIONS`` owns what is published +# to clients; the focused contract gate keeps this import-free helper equal to that value. +MAX_FIELD_OPTIONS = 200 MAX_FORMULA_LEN = 500 #: `rating` bounds. Airtable caps at 10; below 2 a rating is a checkbox. RATING_MAX_DEFAULT, RATING_MAX_MIN, RATING_MAX_MAX = 5, 2, 10 @@ -226,12 +228,25 @@ MEASURE_FIELD_PREFIX = "measure_" MEASURE_FIELD_TYPES = {"currency", "int", "pct"} +class OptionLimitExceeded(ValueError): + """A choice vocabulary cannot be represented without discarding entered values.""" + + def __init__(self, entered): + self.entered = int(entered) + self.limit = MAX_FIELD_OPTIONS + self.rejected = max(0, self.entered - self.limit) + super().__init__( + f"This list has {self.entered} choices; the server limit is {self.limit}, so " + f"{self.rejected} would be rejected. Remove {self.rejected} choices, then save." + ) + + def _clean_options(raw): """Choices for a `select`: strings, trimmed, de-duplicated case-insensitively, capped. Mirrors types.ts parseOptions — a choice list that means one thing in the picker and another in the filter dropdown is a column with two vocabularies.""" out, seen = [], set() - for v in list(raw or [])[:MAX_FIELD_OPTIONS * 2]: + for v in list(raw or []): if not isinstance(v, (str, int, float)) or isinstance(v, bool): continue s = str(v).strip()[:120] @@ -239,7 +254,9 @@ def _clean_options(raw): continue seen.add(s.lower()) out.append(s) - return out[:MAX_FIELD_OPTIONS] + if len(out) > MAX_FIELD_OPTIONS: + raise OptionLimitExceeded(len(out)) + return out def merge_choice_options(*sources): @@ -440,19 +457,71 @@ def _clean_permissions(raw): #: drift risk this file's docstring names. The charset says which characters may appear; the #: engine says what they mean. `_quotes_balanced` below is the one structural rule the quote #: character brings with it. -_FORMULA_CHARS = re.compile(r"^[\w\s{}()+\-*/.,<>=!^&\"]*$") +#: WAVE 42 G7 (2026-08-25) -- THE SAME SCAR, WIDENED IN THE SAME CHANGE THIS TIME. +#: +#: The client engine gained REGEX_EXTRACT, REGEX_REPLACE, SUBSTITUTE, FIND, SWITCH, DATEADD, +#: DATETIME_DIFF and DATETIME_FORMAT. Each added character below is admitted because a token +#: the client's tokenizer parses would otherwise die here: +#: +#: ' a SINGLE-QUOTED literal. Airtable documents units and date formats that way. +#: [ ] a regex character class, the commonest construct there is: "[0-9]+". +#: | ? \\ : regex alternation, optional/non-capturing groups, and escapes. +#: @ SUBSTITUTE/FIND over an email or URL. +#: # $ % TEXT number formats already accepted by the client; `$` is also a regex anchor. +#: +#: `;` stays refused: no legal formula token needs a statement separator. +_FORMULA_CHARS = re.compile(r"^[\w\s{}()+\-*/.,<>=!^&\"'\[\]|?:\\#$%@]*$") _FORMULA_REF = re.compile(r"\{([^{}]*)\}") +#: Longest single regex-pattern literal. The client independently caps both pattern and subject. +MAX_FORMULA_LITERAL_LEN = 200 + +#: Catastrophic-backtracking shape, applied to STRING LITERALS only. Applying this to the whole +#: source would also match ordinary arithmetic such as `({a} + {b}) * 2`. +_NESTED_QUANT = re.compile(r"\((?:[^()\\]|\\.)*[*+](?:[^()\\]|\\.)*\)\??\s*[*+]") + + +def _string_literals(s): + """Return literal bodies, or None for an unterminated quote. + + This mirrors the client tokenizer's structural rule: single and double quoted strings are + equivalent, a doubled opening quote escapes itself, and the other quote kind is ordinary + text. It assigns no formula meaning; it only locates literal boundaries. + """ + out = [] + i = 0 + n = len(s) + while i < n: + quote = s[i] + if quote not in ('"', "'"): + i += 1 + continue + j = i + 1 + body = [] + while True: + if j >= n: + return None + if s[j] == quote: + if j + 1 < n and s[j + 1] == quote: + body.append(quote) + j += 2 + continue + break + body.append(s[j]) + j += 1 + out.append("".join(body)) + i = j + 1 + return out + def _quotes_balanced(s): - """An even number of `"` — the structural half of string support. + """Whether every single- or double-quoted literal is structurally closed. - Sound because the engine's own escape is Excel's: `""` inside a string is one quote, and it - contributes TWO characters. So a well-formed expression always has an even count and an - unterminated string always has an odd one. What a balanced pair MEANS is the engine's - business, exactly as with parentheses. + A parity count is not sound now that both spellings are legal: the double quote inside + ``'5" pipe'`` is ordinary text, not an opener. The shared scanner answers exactly the same + boundary question as the client tokenizer without assigning meaning to the literal. """ - return s.count('"') % 2 == 0 + return _string_literals(s) is not None #: Wave-18 C5-AUTOFIELD. `kind` is a whitelist because an unknown kind would be a column that @@ -537,6 +606,19 @@ def _clean_formula(raw, valid_keys=None): return None if not _quotes_balanced(s): return None + # Regex patterns are user input reaching JavaScript's regex engine. The client owns the + # load-bearing runtime bounds (pattern AND subject); this structural wall rejects an obvious + # unsafe literal before it is stored. Run it on read too: literal shape is intrinsic and + # cannot become invalid merely because the table's field set later changes. + literals = _string_literals(s) + if literals is None: + return None + regexy = 'REGEX_' in s.upper() + for literal in literals: + if regexy and len(literal) > MAX_FORMULA_LITERAL_LEN: + return None + if _NESTED_QUANT.search(literal): + return None refs = _FORMULA_REF.findall(s) leftover = _FORMULA_REF.sub("", s) if "{" in leftover or "}" in leftover: # unbalanced / nested braces @@ -1247,6 +1329,7 @@ def rows_from_pool(pool_rows, fields=None, overlays=None, derived=None): out = [] for r in pool_rows: pid = r.get("pid") + saved = overlays.get(str(pid), {}) or {} # `_created` (wave-5 item 11) rides every row like `pid` does — the datum the # `created_time` field type renders, regardless of that field's own key. Not a Field: # it has no column of its own until a user creates one. `lat`/`lon` (wave-7 W11) ride @@ -1258,11 +1341,12 @@ def rows_from_pool(pool_rows, fields=None, overlays=None, derived=None): if field.get("derived"): continue # not on the pool row — filled from `derived` below v = r.get(k) + if k == "maps_url" and saved.get(k): + v = saved[k] # D-153 — three roundings, not two: text/status/date pass through verbatim, currency # and pct keep two decimals (`_round2`), everything else stays a whole number. row[k] = v if field["type"] in {"text", "status", "date"} else ( _round2(v) if field["type"] in {"currency", "pct"} else _round(v)) - saved = overlays.get(str(pid), {}) or {} for field in overlay_fields: row[field["key"]] = saved.get(field["key"], "") got = derived.get(pid) or {} @@ -2618,8 +2702,11 @@ def clean_filter_tree(raw, valid_keys, depth=1, budget=None, cohort_ids=None, # is the same all-or-nothing the single-cohort leaf already had, extended to a set. if op in COHORT_OPS and named and all(c in (cohort_ids or ()) for c in named): budget[0] -= 1 - out.append({'colId': COHORT_FIELD, 'op': op, - 'value': ','.join(named), 'value2': ''}) + leaf = {'colId': COHORT_FIELD, 'op': op, + 'value': ','.join(named), 'value2': ''} + if node.get('locked') is True: + leaf['locked'] = True + out.append(leaf) continue if node.get('colId') == VIEW_FIELD: # C4: a view-membership leaf # ⛔ THE SAME ALL-OR-NOTHING THE COHORT ARM ABOVE APPLIES, AND FOR THE SAME REASON. @@ -2634,7 +2721,10 @@ def clean_filter_tree(raw, valid_keys, depth=1, budget=None, cohort_ids=None, v_id = str(node.get('value') or '').strip()[:MAX_VIEW_ID] if v_op in VIEW_OPS and v_id and v_id in (visible_view_ids or ()): budget[0] -= 1 - out.append({'colId': VIEW_FIELD, 'op': v_op, 'value': v_id, 'value2': ''}) + leaf = {'colId': VIEW_FIELD, 'op': v_op, 'value': v_id, 'value2': ''} + if node.get('locked') is True: + leaf['locked'] = True + out.append(leaf) continue if node.get('colId') in valid_keys and node.get('op') in FILTER_OPS: budget[0] -= 1 @@ -2690,10 +2780,49 @@ def clean_filter_tree(raw, valid_keys, depth=1, budget=None, cohort_ids=None, rhs = _clean_rhs(node.get('rhs'), valid_keys) if rhs is not None: leaf['rhs'] = rhs + # C3: a record lock is explicit on the leaf, not a view-wide presentation flag. + # Once stored, grid_events preserves the containing filter config unless it is copied + # to a new View; a later caller cannot turn this `True` into an editable-looking false. + if node.get('locked') is True: + leaf['locked'] = True out.append(leaf) return out +def preserve_locked_view_records(prior_config, filters, sorts): + """Return authoritative filter/sort records when an in-place lock was touched. + + A lock protects its semantic position as well as its leaf text: moving a locked condition + between AND/OR groups changes the question even when the leaf itself is byte-identical. We + therefore retain the full corresponding list on any attempted change. A new View has no + prior config, so copying is the explicit alteration path. + """ + prior = prior_config if isinstance(prior_config, dict) else {} + old_filters = prior.get('filters') if isinstance(prior.get('filters'), list) else [] + old_sorts = prior.get('sorts') if isinstance(prior.get('sorts'), list) else [] + + def _has_locked_filter(nodes): + for node in nodes or (): + if not isinstance(node, dict): + continue + if node.get('locked') is True: + return True + if _has_locked_filter(node.get('children') if isinstance(node.get('children'), list) else []): + return True + return False + + def _same(left, right): + return json.dumps(left, sort_keys=True, separators=(',', ':')) == json.dumps( + right, sort_keys=True, separators=(',', ':')) + + if _has_locked_filter(old_filters) and not _same(old_filters, filters): + filters = json.loads(json.dumps(old_filters)) + if any(isinstance(rule, dict) and rule.get('locked') is True for rule in old_sorts) \ + and not _same(old_sorts, sorts): + sorts = json.loads(json.dumps(old_sorts)) + return filters, sorts + + def _clean_rank_scope(raw, valid_keys): """The persisted rank scope, or ``None`` when absent/malformed. diff --git a/platform/aios_grid_fields.json b/platform/aios_grid_fields.json index dd107e0999fd4f77f3f590578cf2882143f1df28..b2b8674dcbb2b7092dce9249a71651fc0f6c6006 100644 --- a/platform/aios_grid_fields.json +++ b/platform/aios_grid_fields.json @@ -1,5 +1,6 @@ { "_comment": "CANONICAL field contract for the AIOS Airtable-style grid — the SINGLE source of truth. Consumed by platform/aios_grid.py (embed/Space host) and aios-web/api/main.py (standalone API), and regenerated into aios-web/web/public/sample_customers.json. Edit HERE only, then run aios-web/verify_fields_contract.py. source=odoo is READ-ONLY; source=overlay is the editable stratum (notes/tags) outside Odoo. type in {text,status,select,multiselect,currency,int,date,pct,url} (select = a fixed-choice READ-ONLY brand attribute; dba is the first, wave 2026-08-02. multiselect = one cell holding a comma-joined SET, so a choice list offers the MEMBERS, not the joined string; customer tags and product supplier are the first, wave 40 W40-T06. url = a link cell, GridCellKind.Uri, drawn by cells.ts and opened by CustomerGrid's scheme-guarded click path; maps_url is the first, wave 40 W40-T08). W40-T08 also adds created_on, the uniform records-created DATE column every database carries (owner I24). It is a plain date field and NOT the created_time TYPE on purpose: created_time renders the row-level _created datum and its value never lives at the field's own key, so it is invisible to the semantic topic, to a stored-cell read and to any host-side export, while a date field is an ordinary cell that all three can see. The _created datum still rides every row for a created_time column a user creates. `description` (wave 5) is the CANONICAL per-field description — every field must carry one, and since wave 7 (owner W8, 2026-07-28) every description is ONE SHORT PLAIN sentence (two only when a fact would otherwise mislead): what the field IS, nothing else — no filter tips, no '(none)' coaching, no rationale; the user's workspace NOTE overrides it in the (i) hover, never in this file. BUILDER FACT (documented here, deliberately NOT in user-facing text): blank connector attributes display as '(none)' (they are no longer all `text`: W40-T06 retyped eight of them to `select` and `tags` to `multiselect`, and the sentinel is a real member of every one of those vocabularies), so `is '(none)'` — not `is empty` — finds the blanks on agent/city/state/country/zip/payment_terms/pricelist/tags. filterable:false = the CONDITION BUILDER does not offer it (still displayed, still sortable); every such field must have a replacement declared in aios-web/verify_fields_contract.py. 2026-07-27 partner attributes: country/zip/payment_terms/pricelist/tags/customer_since all ship default:false. zip is TEXT because a postal code has leading zeros. Odoo's credit_limit (1% populated) and user_id salesperson (2%) are deliberately ABSENT; agent_ids is the salesperson field and AR is where credit exposure comes from. Wave-5 item 8 (2026-07-27): ltm_rev and at_risk are DELETED — LTM's replacement is a creatable Sales measure column (the demo column IS Sales · the last 12 months), at_risk's replacement is a formula field, e.g. MAX(0, {revenue_ly} - {revenue_ytd}). Wave-6 item 8 (2026-07-27, the no-buildable-presets rule): revenue_ytd, revenue_ly, orders_24m, aov and yoy_pct are DELETED — every one is self-buildable, so a frozen pre-set beside the builder was two ways to ask one question. Replacements (recorded in verify_fields_contract.py): creatable measure columns for Sales / Orders / Avg order $ over any period (harness/measure_filter.py ADMITTED carries revenue, orders and the composite aov), and a formula over two measure columns for YoY, e.g. ({sales_ytd} - {sales_ly}) / {sales_ly}. Stale view colIds naming the five self-heal on the next autosave (the established rule).", + "_valueOverride": "A source field remains immutable unless its exact contract carries valueOverride=overlay; maps_url is the only admitted source-backed override.", "fields": [ { "key": "customer", @@ -113,6 +114,7 @@ "label": "Google Maps", "type": "url", "source": "odoo", + "valueOverride": "overlay", "default": true, "description": "A Google Maps link to this customer's address." }, diff --git a/platform/core/field_permissions.py b/platform/core/field_permissions.py index f31053f92521918b783a06843f9162b9abd2423c..472fe0b1f792350ec9367dd6bd2fcf78964bc1c2 100644 --- a/platform/core/field_permissions.py +++ b/platform/core/field_permissions.py @@ -26,6 +26,26 @@ FIELD_ORIGINS = ("preset", "user") FIELD_AUDIENCES = ("private", "users", "everyone") FIELD_VALUE_STRATA = ("shared", "personal") +#: ⭐⭐ W42-T47 / RULING R15 / CONTRACT C10 — WHICH STRATUM A DEFINITION WAS READ FROM. +#: +#: C10: *"`field_classes` reports, beside `{origin, audience, sharedBy, values, owner}`, WHICH +#: STRATUM the definition was read from, so the client sections on ownership without re-deriving +#: it."* The three are the three places a field definition can physically live in this codebase: +#: +#: * ``contract`` — the canonical contract file (`aios_grid_fields.json`) or a module's declared +#: columns. Nobody made it here; it is the same document for every account. +#: * ``shared`` — `shared_overlay`'s tenant-wide bucket (`_table_workspace__shared`), +#: which is where `mint_field` and `promote_field` put a definition. ONE document, read by +#: every account. +#: * ``personal`` — the reader's OWN member of the `_table_workspace` document. A +#: per-user definition: no other account ever receives it. +#: +#: ⛔ IT IS NOT `FIELD_VALUE_STRATA` WEARING A SECOND HAT, AND THE TWO MUST NOT BE COMPARED. That +#: tuple says where a column's VALUES live; this one says where its DEFINITION was read from. A +#: column can be `stratum='personal'` with `values='shared'` — that pair is R15's report, not R15's +#: bug (see `field_stratum`). +FIELD_DEFINITION_STRATA = ("contract", "shared", "personal") + #: `permissions_from_grants`' edit vocabulary -> C1's audience vocabulary. The two answer the SAME #: question in different words (who can reach this column), so this map is the whole of the #: translation and there is no second derivation to drift from it @@ -200,6 +220,82 @@ def field_origin(field): return "preset" +def _grant_mark(): + """The name of the wall's own mark, read from `perm_scope` and never re-spelled locally. + + ⛔ `_governed`'s note forbids a second reader by name: *"Re-spelling `f.get('granted') is True` + locally would be a second reader of the mark, and a second reader is how the two come apart."* + The stratum fallback below needs the same mark, so it takes the CONSTANT rather than the + literal, and the two cannot drift apart on a rename. + + ⚠ THE FAILURE DIRECTION IS DELIBERATE AND IS THE OPPOSITE OF `_governed`'S. `_governed` + degrades to `False` (not governed), which is right for a WALL question. Here an unresolvable + import must not tip a definition toward `personal`, because `personal` zeroes the grant record + in `field_class`; falling back to the mark's own spelling keeps the conservative answer + (`shared`, i.e. trust the registry) when `core.perm_scope` cannot be loaded at all. + """ + try: + import core.perm_scope as perm_scope + return str(perm_scope.FIELD_GRANT_MARK) + except Exception: # noqa: BLE001 + return "granted" + + +def field_stratum(field): + """⭐⭐ W42-T47 / R15 / C10 — THE **FALLBACK** DERIVATION OF WHICH STRATUM A DEFINITION CAME + FROM, for a caller that cannot lend the fact. See `FIELD_DEFINITION_STRATA`. + + ⛔⛔ IT IS A FALLBACK AND NOT THE ANSWER, AND THE REASON IS THE WHOLE OF THIS TICKET: **NOTHING + ON A FIELD DEFINITION RECORDS WHICH DOCUMENT IT WAS READ FROM.** Every mark that looks like + evidence survives being COPIED OUT of that document: + + * `routes_customers._merge_shared_fields` projects a shared definition as + `dict(f, source="overlay", shared=True)`; + * that merged list reaches `grid_events.field_upsert`, whose fork-minting branch is + `base = field_by_key[key]` then `field = {**base, 'note': ...}` — `{**base}` copies EVERY + key, `shared` and `granted` and `source` included; + * `TableStore.save_field` then writes that copy into THIS user's own workspace member. + + So a personal fork of a tenant-wide route carries `shared: True` verbatim, and a mark-based + test calls it `shared`. That is R15's own reproduction (gabriel mints `route_x`; another + account forks it through Format or Summary), which is exactly the case this member exists to + report, so the mark test cannot be the producer of the answer. + + ⛔ ADDING MORE MARKS DOES NOT RESCUE IT. `source == "overlay"` is copied by the same `{**base}` + and fails identically. A key-keyed store (the grant registry, `shared_overlay.fields()` + membership, the lent `values_shared`) fails the other way: it answers about whichever + definition of that key lives in ITS document, which on a fork is a DIFFERENT definition from + the dict in hand, so it can never report the conflict either. + + ⚠ THE BLIND SPOT, STATED RATHER THAN HIDDEN: **this fallback cannot see a fork, and names one + `shared`.** A documented blind spot beats a silent one; the honest answer is lent by the + assembly that DID the reading, through `field_class(stratum=...)` / `field_classes(strata=...)`. + + The marks are still the best available evidence for everything else, and the polarity matches + `field_origin`'s: unmarked plus a demonstrable author is `personal`, everything else `contract`. + """ + if not isinstance(field, dict): + return "contract" + if field.get("shared") is True or field.get(_grant_mark()) is True: + return "shared" + if field_origin(field) == "user": + return "personal" + return "contract" + + +def _clean_stratum(value, field=None): + """One lent stratum, canonicalised, falling back to `field_stratum` on anything unrecognised. + + ⛔ AN UNKNOWN TOKEN FALLS BACK RATHER THAN FAILING CLOSED TO `personal`. `personal` is not the + safe direction here: it ZEROES the grant record in `field_class`, so a typo in a lent map would + silently strip the owner and the audience off every column it named. + """ + token = str(value or "").strip().lower() + if token in FIELD_DEFINITION_STRATA: + return token + return field_stratum(field) + + def _shared_value_keys(table_key, st=None): """The columns of `table_key` whose VALUES are one tenant-wide stratum.""" key = str(table_key or "").strip() @@ -272,12 +368,24 @@ def grant_records(grant_topic, st=None): def field_class(field, viewer, grants=None, *, table_key=None, grant_topic=None, - values_shared=None, st=None): + values_shared=None, stratum=None, st=None): """⭐⭐ W41-T01 / R5 / CONTRACT C1 — **THE ONE PRODUCER OF A FIELD'S THREE BADGES.** - Returns `{origin, audience, sharedBy, values, owner}` and nothing else derives a badge - independently. The five member spellings are C1's and they are the wire's: the payload key is - `class`, its members are `origin`, `audience`, `sharedBy`, `values`, `owner`. + Returns `{origin, audience, sharedBy, values, owner, stratum}` and nothing else derives a badge + independently. The member spellings are C1's and C10's, and they are the wire's: the payload key + is `class`, its members are `origin`, `audience`, `sharedBy`, `values`, `owner`, `stratum`. + + **stratum** — ⭐⭐ W42-T47 / R15 / C10, and it is the member the other five are now ANCHORED to. + `contract` | `shared` | `personal`: WHICH DOCUMENT this definition was read from. The client + sections on ownership (My fields / From my colleagues / Built in) from `stratum` plus `owner` + WITHOUT re-deriving either. + ⚠ IT IS **LENT BY THE CALLER**, for `values_shared`'s reason turned up one notch: not merely + that `core` never imports up, but that the fact is not present in ANY of this function's + inputs. Nothing on a definition records the document it came from, because every mark on it + survives being copied out of that document (`field_stratum` carries the proof and the copying + doors). Only the assembly that DID the reading knows, so it lends + `field_classes(strata={key: stratum})`, and absent that the derivation falls back to + `field_stratum`, whose blind spot is stated there rather than hidden here. `viewer` is a USERNAME STRING (`session.uname`), never a user dict — a dict compares unequal to every stored name, which would make `sharedBy` silently never match and `owner` never equal the @@ -326,6 +434,19 @@ def field_class(field, viewer, grants=None, *, table_key=None, grant_topic=None, contract-declared one), else `personal`. This is R5's new information and it is INDEPENDENT of audience: a route column is `private` with `shared` values, Supplier is `everyone` with `shared` values, and a column you made is `private` with `personal` ones. + ⛔⛔ W42-T47 / R15 — `values` AND `audience` KEEP READING GENUINELY DIFFERENT STORES, AND THEY + ARE STILL INCAPABLE OF DESCRIBING DIFFERENT DEFINITIONS. The independence is required (one is + membership of the shared-overlay stratum, the other is `shares.grants`), so it is not removed; + what is removed is the second opinion about WHICH DEFINITION is being described. `stratum` is + computed once, from the definition in hand, and both legs are anchored to it: the registry is + not consulted at all on a `personal` definition, and `values` no longer decides an audience for + one. R15's *"`Private` + `Shared values` is NOT a state to preserve, it IS the bug"* is + therefore answered by making the pair UNAMBIGUOUS rather than by suppressing either member: a + column whose cells come from the shared pool but whose definition is the reader's own now reads + `{stratum: 'personal', values: 'shared', audience: 'private', owner: }`, which + is one coherent sentence and the exact fact the owner reported, instead of two badges that + contradict each other about who has this column. + ⚠ `values_shared` IS LENT BY THE CALLER because the key set is a MODULE-level fact (`product_data.SHARED_KEYS()`), and `core` never imports up. Absent, it is derived from `shared_overlay.fields(table_key)`, which is right for an overlay topic and blind to a @@ -342,8 +463,45 @@ def field_class(field, viewer, grants=None, *, table_key=None, grant_topic=None, me = _uname(viewer) origin = field_origin(field) + # ⛔⛔ W42-T47 — AN **ASSERTED** STRATUM ANCHORS THE BAG; A GUESSED ONE REPORTS AND NOTHING MORE. + # `user_tables._field_badges` calls `field_class` on a RIGHTS path: `delete_field_refusal` reads + # `bag['owner']` and returns `NOT_FIELD_OWNER` on a mismatch, so anything that moves `owner` + # moves a delete. That caller lends no stratum, and the fallback derivation is a guess by its + # own docstring, so letting a guess re-source `owner` could hand somebody a delete they did not + # have. This ticket makes a REPORT honest; it moves no wall, and this line is where that + # promise is kept rather than asserted. A caller that KNOWS which document it read (C10's lent + # map) gets the anchored answer; every existing caller keeps today's behaviour byte for byte. + asserted = str(stratum or "").strip().lower() in FIELD_DEFINITION_STRATA + stratum = _clean_stratum(stratum, field) + anchored_personal = asserted and stratum == "personal" record = grants if isinstance(grants, dict) else _grant_record( key, table_key=table_key, grant_topic=grant_topic, st=st) + # ⛔⛔ W42-T47 / R15 / C10 — THE REGISTRY RECORD IS KEYED BY FIELD **KEY**, SO ON A PERSONAL + # DEFINITION IT DESCRIBES A DIFFERENT DEFINITION AND IS DROPPED ENTIRELY. + # + # `shares.field_oid(topic, key)` names a KEY, not a document. A grant record for `route_x` was + # written about the definition in the tenant-wide bucket; when the dict in hand came from the + # reader's own workspace member instead, those are two different definitions of one key, and + # `routes_customers._merge_shared_fields` makes that routine rather than exotic — its rule is + # `if k in have: continue`, so a reader holding ANY same-key entry silently keeps their own and + # never sees the shared one (that merge is W42-T46's, not this ticket's). + # + # Reading `owner`, `sharedBy` and `audience` off that record while reading `origin`, `values` + # and the marks off the personal dict is how ONE bag came to describe TWO definitions, which is + # R15's *"Gabriel created the Route and it ends up being MY field"* — the owner fact and the + # audience were answering about different documents. Zeroing the record here is what makes the + # anchoring STRUCTURAL: after this line there is no path by which a registry-derived member can + # describe a definition the `stratum` member does not name. + # + # ⚠ NO PERMISSION MOVES, AND IT IS CHECKED RATHER THAN REASONED — BUT NOT FOR THE REASON IT + # FIRST LOOKS LIKE. A fork carries `granted: True` copied verbatim, so a `personal` definition + # CAN be one the wall does consult the registry about; this arm is exactly where R15's fork + # lands. What makes it safe is that the wall reads the registry ITSELF + # (`perm_scope.field_grant_hidden` -> `shares.role_for`) and never reads this bag, so zeroing + # here changes the REPORT and nothing else. The report is the thing that was wrong: a personal + # definition's grant record describes a document the reader is not reading. + if anchored_personal: + record = _empty_record() owner = _uname(record.get("owner")) or None entries = [e for e in (record.get("entries") or ()) if isinstance(e, dict)] @@ -354,10 +512,19 @@ def field_class(field, viewer, grants=None, *, table_key=None, grant_topic=None, if _governed(field): audience = _AUDIENCE_FROM_EDIT.get( permissions_from_grants(entries).get("edit"), "private") - elif values == "shared" or origin == "preset": + elif not anchored_personal and (values == "shared" or origin == "preset"): # Ungoverned AND tenant-wide: the database's holders all hold this column. `origin` covers # a contract-declared column whose VALUES are per-user (a canonical Odoo column); the # `values` leg covers a shared-stratum column the contract never named. + # + # ⛔ W42-T47 — THE `stratum != "personal"` GUARD IS THIS TICKET'S BEHAVIOUR CHANGE, AND IT + # FIRES ON EXACTLY ONE SHAPE: a definition read from the reader's OWN workspace member + # whose key ALSO has a definition (and therefore shared cells) in the tenant-wide bucket. + # Without it, `values` — a fact about the SHARED document — promoted a definition nobody + # else can even receive to `everyone`. The reader would be told the whole tenant can see a + # column that exists only in their own stratum, which is a promise the merge cannot keep. + # ⚠ It cannot touch a shared or contract definition: both answer a `stratum` this test + # admits, so Supplier and every minted route badge exactly as they did before. audience = "everyone" else: audience = "private" @@ -372,13 +539,30 @@ def field_class(field, viewer, grants=None, *, table_key=None, grant_topic=None, if shared_by == me: shared_by = None + # ⚠ THE `createdBy` FALLBACK IS NOW THE WHOLE ANSWER ON A PERSONAL DEFINITION, not merely the + # create-window stopgap it was written as, because `record` was zeroed above. That is the + # intended reading of C10's *"whose it is"*: the author mark rides on the definition that was + # read, so `owner` and `stratum` cannot name different documents. return {"origin": origin, "audience": audience, "sharedBy": shared_by, - "values": values, "owner": owner or _uname(field.get("createdBy")) or None} + "values": values, "owner": owner or _uname(field.get("createdBy")) or None, + "stratum": stratum} def field_classes(fields, viewer, *, table_key=None, grant_topic=None, values_shared=None, - st=None): - """`{field_key: }` for a whole field list, on ONE registry read. See `grant_records`. + strata=None, st=None): + """`{field_key: }` for a whole field list, on ONE registry read. See `grant_records`. + + ⭐⭐ W42-T47 / C10 — every bag carries `stratum` (`contract` | `shared` | `personal`), so a + caller sections on ownership without re-deriving it. + + ⛔ `strata` IS THE LENT MAP AND IT IS THE ONLY HONEST SOURCE: `{field_key: }`, supplied by the assembly that READ each definition and therefore + knows which document each entry of `fields` came out of. Partial maps are fine — a key the map + does not name falls back to `field_stratum`, per key, so a caller can lend only the keys it is + sure about. ⚠ THE PER-FIELD ANSWER IS NOT DERIVABLE FROM ANYTHING THIS FUNCTION HOLDS: + `records` and `values_shared` are both keyed by field key, so they describe whichever + definition of that key lives in THEIR store, which on a forked column is a different definition + from the one in `fields`. That is why the fact is lent rather than computed here. ⚠ EVERY COLUMN GETS A BAG, INCLUDING ONE THE REGISTRY HAS NEVER HEARD OF: the missing record is substituted explicitly rather than left to `field_class` to re-read, which is what keeps the @@ -389,6 +573,7 @@ def field_classes(fields, viewer, *, table_key=None, grant_topic=None, values_sh if values_shared is None: values_shared = _shared_value_keys(table_key, st=st) shared_keys = {str(k) for k in (values_shared or ())} + lent = {str(k).strip(): v for k, v in strata.items()} if isinstance(strata, dict) else {} out = {} for field in (fields or ()): if not isinstance(field, dict): @@ -398,7 +583,7 @@ def field_classes(fields, viewer, *, table_key=None, grant_topic=None, values_sh continue out[key] = field_class(field, viewer, grants=records.get(key) or _empty_record(), table_key=table_key, grant_topic=topic, - values_shared=shared_keys, st=st) + values_shared=shared_keys, stratum=lent.get(key), st=st) return out @@ -416,7 +601,7 @@ def _same_classification(left, right): return set(left.get("users") or ()) == set(right.get("users") or ()) -def reconcile_shared_permissions(shared_key, grant_topic, st=None): +def reconcile_shared_permissions(shared_key, grant_topic, st=None, shared_document=None): """Re-derive every SHARED definition's `permissions` bag from the grant registry. Returns the number of definitions actually rewritten. @@ -463,7 +648,8 @@ def reconcile_shared_permissions(shared_key, grant_topic, st=None): if not bucket or st is None: return 0 rewritten = 0 - for field_key, defn in list(shared_overlay.fields(bucket, st=st).items()): + for field_key, defn in list(shared_overlay.fields( + bucket, st=st, document=shared_document).items()): # ⚠ EACH DEFINITION IS GUARDED ON ITS OWN. `shares.field_oid` RAISES on an empty key, and # the caller wraps this whole function in one swallowing except — so a single malformed # entry would otherwise skip every REMAINING column, including the one the owner reported, @@ -494,6 +680,8 @@ def reconcile_shared_permissions(shared_key, grant_topic, st=None): updated = dict(defn) updated["permissions"] = derived shared_overlay.put_field(bucket, key, updated, st=st) + if isinstance(shared_document, dict): + shared_document.setdefault("fields", {})[key] = updated rewritten += 1 except Exception: continue @@ -648,12 +836,14 @@ def _field_is_migratable(field): and field.get("source") == "overlay") -def migrate_legacy_fields(table_key, st=None, known_users=None, grant_topic=None, shared_key=None): +def migrate_legacy_fields(table_key, st=None, known_users=None, grant_topic=None, shared_key=None, + workspace_document=None, shared_document=None): """Promote legacy shared-looking custom fields into the shared field stratum. The operation is intentionally idempotent and copy-before-remove. A failed request can therefore leave a private shadow behind, but it cannot lose the creator's values. The next - read retries the promotion and the shared definition wins. + read retries the promotion and the shared definition wins. A caller that already captured + the workspace document may lend that read-only snapshot to avoid reopening the same key. """ key = str(table_key or "").strip() grant_key = str(grant_topic or key).strip() @@ -679,14 +869,25 @@ def migrate_legacy_fields(table_key, st=None, known_users=None, grant_topic=None # finds no grant record, the "no record" arm fires for every column, and the reconcile # silently does nothing at all. try: - reclassified = reconcile_shared_permissions(shared_key, grant_key, st=st) + reclassified = reconcile_shared_permissions( + shared_key, grant_key, st=st, shared_document=shared_document) except Exception: reclassified = 0 - try: - document = st.get(key) or {} - except Exception: - return {"promoted": 0, "normalized": 0, "reclassified": reclassified, - "suppressed": 0, "purged": 0} + # W44 predecessor replay: a table assembly has already read this exact document through + # `TableStore.workspace(..., include_document=True)`. Reopening it here paid a third whole + # `_table_workspace` copy for the same rows request. Accept that captured snapshot when + # supplied; every historical/cold caller still takes the original `st.get(key)` path. + # + # A non-dict is NOT treated as an empty captured document. It falls back to the store so a + # malformed optional argument cannot suppress an existing migration by manufacturing `{}`. + if isinstance(workspace_document, dict): + document = workspace_document + else: + try: + document = st.get(key) or {} + except Exception: + return {"promoted": 0, "normalized": 0, "reclassified": reclassified, + "suppressed": 0, "purged": 0} if not isinstance(document, dict): return {"promoted": 0, "normalized": 0, "reclassified": reclassified, "suppressed": 0, "purged": 0} @@ -730,7 +931,8 @@ def migrate_legacy_fields(table_key, st=None, known_users=None, grant_topic=None normalized += 1 continue - existing = shared_overlay.fields(shared_key, st=st).get(field_key) + existing = shared_overlay.fields( + shared_key, st=st, document=shared_document).get(field_key) if isinstance(existing, dict): # A completed promotion is authoritative. Remove a duplicate personal copy # only when it has the same creator; a collision from another owner fails closed. @@ -744,6 +946,8 @@ def migrate_legacy_fields(table_key, st=None, known_users=None, grant_topic=None "createdBy": original.get("createdBy") or owner, "source": "overlay", "custom": True}) shared_overlay.put_field(shared_key, field_key, shared, st=st) + if isinstance(shared_document, dict): + shared_document.setdefault("fields", {})[field_key] = dict(shared) promoted += 1 rows = {} @@ -790,7 +994,7 @@ def migrate_legacy_fields(table_key, st=None, known_users=None, grant_topic=None purged = 0 if tombstoned: try: - live = shared_overlay.fields(shared_key, st=st) + live = shared_overlay.fields(shared_key, st=st, document=shared_document) except Exception: # noqa: BLE001 live = {} for dead in tombstoned: diff --git a/platform/core/grid_events.py b/platform/core/grid_events.py index c1d6237e7a9df1889858a94ba949755efda3c802..51682bc926695e42e07da5fb9b7acb66344ba801 100644 --- a/platform/core/grid_events.py +++ b/platform/core/grid_events.py @@ -333,6 +333,18 @@ def _may_edit_field_value(ctx, key, definition): str(user).lower() for user in (perms.get('users') or [])}) +def _may_override_source_value(definition): + """One source-backed value may write through the ordinary overlay door. + + The canonical field contract must carry the capability and the field identity must still be + Maps. Requiring both prevents a copied or caller-forged flag from widening every Odoo field. + """ + return (isinstance(definition, dict) + and definition.get('key') == 'maps_url' + and definition.get('source') == 'odoo' + and definition.get('valueOverride') == 'overlay') + + def _cohorts(ctx): """The COHORT store this ctx operates on — `_tops`'s sibling, and here for the same reason. @@ -507,7 +519,7 @@ def _grant_marker(ctx, ws): return _has_grants -def table_workspace(ctx, allowed_pids=None, consume_corrections=True): +def table_workspace(ctx, allowed_pids=None, consume_corrections=True, capture_document=False): """Durable Airtable-style view/schema/overlay state, with an honest session fallback. Wave-9 I17: the returned `views` are this user's OWN views MERGED with every view SHARED @@ -534,8 +546,13 @@ def table_workspace(ctx, allowed_pids=None, consume_corrections=True): """ uname = ctx.uname if _store_of(ctx).available(): - ws = _tops(ctx).workspace( - uname, consume_corrections=bool(consume_corrections)) + read = _tops(ctx).workspace( + uname, consume_corrections=bool(consume_corrections), + include_document=bool(capture_document)) + if capture_document: + ws, ctx.workspace_document = read + else: + ws = read try: shared = _tops(ctx).shared_views(uname, bool(ctx.admin)) except Exception as _se: @@ -1261,7 +1278,8 @@ def handle_one(event, ctx): # vocabulary is absent (ordinary customer and legacy in-memory contexts have none). pass valid_keys = set(field_by_key) - overlay_keys = {f['key'] for f in fields if f.get('source') == 'overlay'} + overlay_keys = {f['key'] for f in fields + if f.get('source') == 'overlay' or _may_override_source_value(f)} # Event validation is not a render payload. It must not consume a one-shot label # correction before the browser has had a chance to receive it. ws = table_workspace( @@ -1371,7 +1389,10 @@ def handle_one(event, ctx): for rule in list(cfg.get('sorts') or [])[:20]: if (isinstance(rule, dict) and rule.get('colId') in valid_keys and rule.get('dir') in ('asc', 'desc')): - sorts.append({'colId': rule['colId'], 'dir': rule['dir']}) + clean_sort = {'colId': rule['colId'], 'dir': rule['dir']} + if rule.get('locked') is True: + clean_sort['locked'] = True + sorts.append(clean_sort) order = list(dict.fromkeys( key for key in list(cfg.get('order') or []) if key in valid_keys)) order += [field['key'] for field in fields if field['key'] not in order] @@ -1469,6 +1490,14 @@ def handle_one(event, ctx): if _raw_shared is not None and not _tstore._may_edit(_raw_shared, uname, admin): return False # visible but not editable by this caller is_new = not prior + if not is_new: + # W44 C3: filters/sorts locked in an existing View are immutable records. This + # runs after the visibility/edit wall has established `prior`, so a guessed foreign + # view cannot be used as a source of lock state. A copied View has no prior and is + # consequently the only alteration path. + filters, sorts = _ag.preserve_locked_view_records( + prior.get('config') or {}, filters, sorts) + config['filters'], config['sorts'] = filters, sorts # C4: the creator is HOST-STAMPED and never read from the browser — the same rule as # the field-level createdBy. An existing view keeps whoever made it. creator = prior.get('createdBy') or uname @@ -1787,7 +1816,10 @@ def handle_one(event, ctx): # its list and visual treatment rather than silently preserving a # read-only source projection. if field.get('type') in ('select', 'multiselect') and 'options' in raw: - options = _ag2._clean_options(raw.get('options')) + try: + options = _ag2._clean_options(raw.get('options')) + except _ag2.OptionLimitExceeded as exc: + return _refuse(ctx, 'option_limit', str(exc), key) if not options: return _refuse( ctx, 'no_options', 'a select column needs at least one choice', key) @@ -1895,7 +1927,10 @@ def handle_one(event, ctx): # choices are the tenant's accounts, resolved fresh on every render, so persisting # a snapshot here would go stale the moment somebody joins or leaves. if ftype in ('select', 'multiselect'): - field['options'] = _ag2._clean_options(raw.get('options')) + try: + field['options'] = _ag2._clean_options(raw.get('options')) + except _ag2.OptionLimitExceeded as exc: + return _refuse(ctx, 'option_limit', str(exc), key) if not field['options']: # a select with no choices can never hold a value return _refuse( @@ -3105,7 +3140,7 @@ def handle_one(event, ctx): # A choice value is DATA, not a precondition the field editor must have visited. # Pasted/imported/API values therefore mint an option through this same write # door. It is deliberately still bounded and case-canonical: "ET" then "et" - # is one option, and the 51st distinct label is refused rather than creating a + # is one option, and the 201st distinct label is refused rather than creating a # picker the contract cannot represent. import aios_grid as _ag_choices options = _ag_choices.merge_choice_options(definition.get('options')) diff --git a/platform/core/route_system.py b/platform/core/route_system.py new file mode 100644 index 0000000000000000000000000000000000000000..c3581ab9a1b7337862107b98ddac666cf0ae616b --- /dev/null +++ b/platform/core/route_system.py @@ -0,0 +1,886 @@ +"""Loopable-owned Route/Zone system data (Wave 44 C1/C2). + +The browser never writes the two system databases through the generic table surface. This module +owns their normalized document and exposes narrow Route/Zone writers. ``InternalWriterCapability`` +remains for the generic-policy seam; it is not a browser credential. +""" +from __future__ import annotations + +import copy +import datetime as _dt +import hashlib +import math +import re +import uuid + +MANAGED_BY = "loopable" +LOOPABLE_FOLDER_ID = "loopable" +LOOPABLE_FOLDER = {"id": LOOPABLE_FOLDER_ID, "name": "Loopable", "managedBy": MANAGED_BY} +ROUTES_KEY, ZONES_KEY = "ut_odoo_customers_routes", "ut_odoo_customers_zones" +SYSTEM_TABLES = {ROUTES_KEY: "Odoo Customers - Routes", ZONES_KEY: "Odoo Customers - Zones"} + +# TenantRuntime namespaces this bucket. User-table rows are scalar user data; route stops are a +# protected relation (route id + customer id + order), so they cannot be a generic Link field. +STORE_KEY = "loopable_route_system" +# C3 owns this ceiling on the server. Every field payload publishes it, and clients that do not +# receive it must refuse choice edits rather than guessing a local value. +MAX_OPTIONS = 200 +_ROUTE_COLORS = ("#6E4AC7", "#1F4E78", "#C85A54", "#2F8F83", "#B57716", "#A14B8B") +_COLOR_RE = re.compile(r"^#[0-9A-Fa-f]{6}$") +_ZIP_RE = re.compile(r"^[A-Za-z0-9 -]{1,20}$") +_GOOGLE_MAPS_URL_RE = re.compile( + r"^https://(?:(?:www\.)?google\.(?:com|co\.[a-z]{2}|[a-z]{2,3})|maps\.app\.goo\.gl|goo\.gl)/", re.I) + + +class InternalWriterCapability: + """Opaque capability intentionally constructible only by this module.""" + __slots__ = ("_token",) + + def __init__(self, token): + self._token = token + + +_TOKEN = object() + + +def internal_writer_capability(): + return InternalWriterCapability(_TOKEN) + + +def is_internal_writer(capability): + return isinstance(capability, InternalWriterCapability) and capability._token is _TOKEN + + +def descriptor(key): + label = SYSTEM_TABLES.get(str(key)) + if not label: + return None + return {"key": str(key), "label": label, "managedBy": MANAGED_BY, + "folderId": LOOPABLE_FOLDER_ID, "protected": True, "source": "Loopable", + "recordsMutable": False, "canDelete": False, "canMove": False} + + +def system_table_descriptors(): + """Virtual Data-nav rows that exist before the first Route is created.""" + return [dict(descriptor(key)) for key in (ROUTES_KEY, ZONES_KEY)] + + +def is_managed(definition_or_key): + if isinstance(definition_or_key, dict): + return definition_or_key.get("managedBy") == MANAGED_BY + return str(definition_or_key) in SYSTEM_TABLES + + +def may_generic_mutate(definition_or_key, capability=None): + return not is_managed(definition_or_key) or is_internal_writer(capability) + + +def field_axes(field): + """Return the closed C3 axes without accepting caller-invented vocabulary.""" + field = dict(field or {}) + ftype = str(field.get("type") or "text") + source = str(field.get("source") or "odoo") + origin = field.get("origin") or ({"odoo": "odoo", "overlay": "user"}.get(source, "system")) + if origin not in {"odoo", "loopable", "system", "user", "automation"}: + raise ValueError("unapproved field origin") + kind = field.get("kind") or ftype + authority = field.get("valueAuthority") or ( + "derived" if source == "odoo" or field.get("derived") else "user-entered") + scope = field.get("scope") or ("source" if source == "odoo" else "tenant") + if authority not in {"source", "derived", "user-entered", "computed"} or scope not in {"source", "tenant", "view", "record"}: + raise ValueError("unapproved field axis") + field.update({"kind": kind, "origin": origin, "valueAuthority": authority, "scope": scope}) + # Publish the server-owned limit on EVERY field. The Add-field pane needs the contract before + # the choice field it is creating exists; restricting it to existing choice fields forces a + # client literal (or an unsafe guess) on an empty table. Overwrite caller-supplied values: + # stored metadata is not allowed to mint a different product limit. + limits = dict(field.get("limits") or {}) + limits["maxOptions"] = MAX_OPTIONS + field["limits"] = limits + if ftype == "json": + field["label"] = "Table" + return field + + +def inject_loopable_folder(workspace): + """Server projection: personal preferences cannot remove or relocate this folder.""" + out = copy.deepcopy(workspace or {}) + folders = out.setdefault("folders", []) + existing = next((row for row in folders if isinstance(row, dict) + and row.get("id") == LOOPABLE_FOLDER_ID), None) + if existing is None: + folders.append(dict(LOOPABLE_FOLDER)) + else: + existing.update(LOOPABLE_FOLDER) + return out + + +def _now(): + return _dt.datetime.now(_dt.timezone.utc).replace(microsecond=0).isoformat() + + +def _store(st): + try: + return dict(st.get(STORE_KEY) or {}) + except Exception: + return {} + + +def _pid(raw): + if isinstance(raw, bool): + return None + try: + value = int(raw) + except (TypeError, ValueError): + return None + return value if value >= 0 else None + + +def _color(raw, ordinal=0): + value = str(raw or "").upper() + return value if _COLOR_RE.fullmatch(value) else _ROUTE_COLORS[ordinal % len(_ROUTE_COLORS)] + + +def _name(raw): + return " ".join(str(raw or "").split())[:120] + + +def _stops(raw): + """Canonical dense stop relation. One customer can occur once in a Route.""" + offered, seen = [], set() + for item in raw or (): + candidate = item.get("customerPid") if isinstance(item, dict) else item + pid = _pid(candidate) + if pid is not None and pid not in seen: + seen.add(pid) + offered.append(pid) + return [{"customerPid": pid, "position": index + 1} + for index, pid in enumerate(offered)] + + +def ordered_stops(route): + return [dict(stop) for stop in sorted(route.get("stops") or [], + key=lambda stop: int(stop.get("position") or 0))] + + +def _route(raw, *, ordinal=0, created_by="", tenant_id="", prior=None): + raw = raw if isinstance(raw, dict) else {} + prior = prior if isinstance(prior, dict) else {} + name = _name(raw.get("name", prior.get("name"))) + if not name: + return None + rid = str(raw.get("id") or prior.get("id") or "route_" + uuid.uuid4().hex)[:96] + source_view = raw.get("sourceViewId", raw.get("source_view_id", prior.get("sourceViewId"))) + source_view = str(source_view or "").strip()[:120] or None + revision = raw.get("sourceViewRevision", prior.get("sourceViewRevision")) + revision = str(revision or "").strip()[:128] or None + stops_raw = raw.get("stops") if "stops" in raw else prior.get("stops") + maps_url = str(raw.get("mapsUrl", prior.get("mapsUrl")) or "").strip()[:2048] + if maps_url and not _GOOGLE_MAPS_URL_RE.match(maps_url): + return None + stamp = _now() + share_key = str(raw.get("shareKey") or prior.get("shareKey") or route_share_key(rid))[:80] + out = {"id": rid, "tenantId": str(tenant_id or prior.get("tenantId") or ""), "name": name, + "color": _color(raw.get("color", prior.get("color")), ordinal), + "number": int(prior.get("number") or ordinal + 1), "sourceViewId": source_view, + "sourceViewRevision": revision, "stops": _stops(stops_raw), "managedBy": MANAGED_BY, + "createdBy": str(prior.get("createdBy") or created_by or "")[:120], + "shareKey": share_key, + "createdAt": prior.get("createdAt") or stamp, "updatedAt": stamp} + if maps_url: + out["mapsUrl"] = maps_url + migration = raw.get("migration", prior.get("migration")) + if isinstance(migration, dict): + out["migration"] = {"legacyField": str(migration.get("legacyField") or "")[:80], + "verified": migration.get("verified") is True} + return out + + +def _point_in_ring(lon, lat, ring): + inside = False + if len(ring) < 4: + return False + for index, point in enumerate(ring): + previous = ring[index - 1] + try: + x1, y1, x2, y2 = float(previous[0]), float(previous[1]), float(point[0]), float(point[1]) + except (TypeError, ValueError, IndexError): + return False + intersects = (y1 > lat) != (y2 > lat) + if intersects and lon < (x2 - x1) * (lat - y1) / ((y2 - y1) or 1e-15) + x1: + inside = not inside + return inside + + +def _polygon_contains(geometry, lon, lat): + if not isinstance(geometry, dict): + return False + kind, coords = geometry.get("type"), geometry.get("coordinates") + polygons = [coords] if kind == "Polygon" else coords if kind == "MultiPolygon" else [] + for polygon in polygons or (): + if not isinstance(polygon, list) or not polygon: + continue + if _point_in_ring(lon, lat, polygon[0]) and not any( + _point_in_ring(lon, lat, ring) for ring in polygon[1:]): + return True + return False + + +def _zone(raw, *, ordinal=0, created_by="", tenant_id="", prior=None): + raw = raw if isinstance(raw, dict) else {} + prior = prior if isinstance(prior, dict) else {} + name = _name(raw.get("name", prior.get("name"))) + geometry = raw.get("geometryGeoJson", prior.get("geometryGeoJson")) + zips_raw = raw.get("zipCodes", prior.get("zipCodes", ())) + zips, seen = [], set() + for value in zips_raw or (): + text = " ".join(str(value or "").upper().split())[:20] + if text and text not in seen and _ZIP_RE.fullmatch(text): + seen.add(text) + zips.append(text) + if not name or (geometry is not None and not isinstance(geometry, dict)) or (geometry is None and not zips): + return None + zid = str(raw.get("id") or prior.get("id") or "zone_" + uuid.uuid4().hex)[:96] + source_view = raw.get("sourceViewId", prior.get("sourceViewId")) + source_view = str(source_view or "").strip()[:120] or None + geographic_field = raw.get("geographicField", prior.get("geographicField")) + geographic_field = str(geographic_field or "").strip()[:120] + stamp = _now() + return {"id": zid, "tenantId": str(tenant_id or prior.get("tenantId") or ""), "name": name, + "color": _color(raw.get("color", prior.get("color")), ordinal), + "geometryGeoJson": copy.deepcopy(geometry) if isinstance(geometry, dict) else None, + "zipCodes": zips, "visible": raw.get("visible", prior.get("visible", True)) is not False, + "sourceViewId": source_view, "geographicField": geographic_field, + "membershipRevision": int(prior.get("membershipRevision") or 0), + "memberPids": [int(pid) for pid in prior.get("memberPids") or [] if _pid(pid) is not None], + "managedBy": MANAGED_BY, "createdBy": str(prior.get("createdBy") or created_by or "")[:120], + "createdAt": prior.get("createdAt") or stamp, "updatedAt": stamp} + + +def _customer_zip(row): + for key in ("zip", "zip_code", "postal_code", "postalCode"): + value = row.get(key) if isinstance(row, dict) else None + text = " ".join(str(value or "").upper().split())[:20] + if text: + return text + return "" + + +def zone_members(zone, rows): + """Polygon-or-ZIP UNION, recalculated from the current customer address projection.""" + zips = set(zone.get("zipCodes") or ()) + out = [] + for row in rows or (): + pid = _pid((row or {}).get("pid")) + if pid is None: + continue + zip_match = _customer_zip(row) in zips + try: + lon, lat = float(row.get("lon")), float(row.get("lat")) + polygon_match = math.isfinite(lon) and math.isfinite(lat) and _polygon_contains( + zone.get("geometryGeoJson"), lon, lat) + except (TypeError, ValueError): + polygon_match = False + if zip_match or polygon_match: + out.append(pid) + return out + + +def refresh_zone_memberships(state, rows): + """Pure state projection used by dedicated Zone writers and reads.""" + out = copy.deepcopy(state if isinstance(state, dict) else {}) + next_zones = [] + for zone in out.get("zones") or []: + if not isinstance(zone, dict): + continue + members = zone_members(zone, rows) + previous = [int(pid) for pid in zone.get("memberPids") or [] if _pid(pid) is not None] + if members != previous: + zone["memberPids"] = members + zone["membershipRevision"] = int(zone.get("membershipRevision") or 0) + 1 + zone["updatedAt"] = _now() + next_zones.append(zone) + out["zones"] = next_zones + return out + + +def _persist(st, mutate): + result = {} + + def apply(current): + state = copy.deepcopy(current) if isinstance(current, dict) else {} + state.setdefault("routes", []) + state.setdefault("zones", []) + state.setdefault("legacyMigrations", {}) + state.setdefault("routeNavigation", {}) + next_state, value = mutate(state) + result["value"] = copy.deepcopy(value) + return next_state + + st.update(STORE_KEY, apply, flush="sync") + return result.get("value") + + +def read(st, rows=None): + state = _store(st) + state.setdefault("routes", []) + state.setdefault("zones", []) + state.setdefault("legacyMigrations", {}) + state.setdefault("routeNavigation", {}) + return refresh_zone_memberships(state, rows or ()) if rows is not None else state + + +def list_routes(st): + return [copy.deepcopy(route) for route in read(st).get("routes") or [] if isinstance(route, dict)] + + +def route_share_key(route_id): + """Stable generic-field bridge id for the existing share registry. + + Route records are not generic columns, but the shipped share dialog and grant registry already + own the access workflow. A bounded digest keeps arbitrary Route ids out of field keys while + remaining deterministic for old normalized records that predate the stored ``shareKey``. + """ + digest = hashlib.sha256(str(route_id or "").encode("utf-8")).hexdigest()[:24] + return "route_object_" + digest + + +def duplicate_route(st, route_id, *, tenant_id, created_by, name=None): + """Copy one normalized Route without aliasing its identity, ownership, or share grants.""" + source = next((item for item in list_routes(st) if item.get("id") == str(route_id)), None) + if source is None: + return None + raw = {"name": _name(name) or f"{source.get('name') or 'Route'} copy", + "color": source.get("color"), "sourceViewId": source.get("sourceViewId"), + "sourceViewRevision": source.get("sourceViewRevision"), + "stops": [stop.get("customerPid") for stop in ordered_stops(source)]} + if source.get("mapsUrl"): + raw["mapsUrl"] = source["mapsUrl"] + return put_route(st, raw, tenant_id=tenant_id, created_by=created_by) + + +def _nav_user(username): + return str(username or "").strip().lower()[:120] + + +_NAV_REFUSED = object() + + +def _folder_name(raw): + return " ".join(str(raw or "").split())[:80] + + +def _clean_nav(raw, routes=(), legacy_folders=(), legacy_placements=None): + """Pure, fail-closed current-reader Route navigation projection. + + Old Wave 42 folders are accepted only as a read fallback. Invalid parents, cycles, excess + nesting, dangling placements and stale order ids resolve to root; a GET never writes repairs. + """ + raw = raw if isinstance(raw, dict) else {} + offered = raw.get("folders") if isinstance(raw.get("folders"), list) else legacy_folders + folders, seen = [], set() + for index, item in enumerate(offered or ()): + if not isinstance(item, dict): + continue + fid = str(item.get("id") or "").strip()[:96] + name = _folder_name(item.get("name")) + if not fid or not name or fid in seen: + continue + seen.add(fid) + try: + order = int(item.get("order", index)) + except (TypeError, ValueError): + order = index + parent = str(item.get("parent") or "").strip()[:96] or None + folders.append({"id": fid, "name": name, "parent": parent, "order": order}) + by_id = {item["id"]: item for item in folders} + for item in folders: + parent, trail, depth = item.get("parent"), {item["id"]}, 0 + while parent: + if parent not in by_id or parent in trail or depth >= 9: + item["parent"] = None + break + trail.add(parent) + parent = by_id[parent].get("parent") + depth += 1 + folders.sort(key=lambda item: (int(item.get("order") or 0), item["name"].lower(), item["id"])) + for index, item in enumerate(folders): + item["order"] = index + + placement_raw = raw.get("placements") if isinstance(raw.get("placements"), dict) else legacy_placements or {} + known_folders = set(by_id) + route_keys = set() + for route in routes or (): + rid = str((route or {}).get("id") or "") + if rid: + route_keys.add(rid) + legacy = ((route or {}).get("migration") or {}).get("legacyField") + if legacy: + route_keys.add(str(legacy)) + placements = {} + for key, folder_id in placement_raw.items(): + key, folder_id = str(key), str(folder_id or "") + if key in route_keys and folder_id in known_folders: + placements[key] = folder_id + + offered_order = raw.get("order") if isinstance(raw.get("order"), list) else [] + canonical_ids = [str((route or {}).get("id") or "") for route in routes or ()] + order = [] + for route_id in [*offered_order, *canonical_ids]: + route_id = str(route_id) + if route_id in canonical_ids and route_id not in order: + order.append(route_id) + return {"folders": folders, "placements": placements, "order": order} + + +def route_navigation(st, username, routes=(), *, legacy_folders=(), legacy_placements=None): + """This reader's folders, placement and ordering; route object data remains tenant-wide.""" + user = _nav_user(username) + raw = (read(st).get("routeNavigation") or {}).get(user) if user else None + nav = _clean_nav(raw, routes, legacy_folders, legacy_placements) + folders = nav["folders"] + placement = nav["placements"] + out_routes = [] + rank = {rid: index for index, rid in enumerate(nav["order"])} + for route in routes or (): + item = copy.deepcopy(route) + legacy_key = ((item.get("migration") or {}).get("legacyField")) + item["folderId"] = placement.get(item.get("id"), placement.get(str(legacy_key))) + item["navOrder"] = rank.get(str(item.get("id")), len(rank)) + out_routes.append(item) + out_routes.sort(key=lambda item: (int(item.get("navOrder") or 0), int(item.get("number") or 0))) + return {"folders": folders, "routes": out_routes} + + +def _mutate_nav(st, username, routes, change, *, legacy_folders=(), legacy_placements=None): + user = _nav_user(username) + if not user: + return None + + def mutate(state): + navigation = dict(state.get("routeNavigation") or {}) + current = _clean_nav(navigation.get(user), routes, legacy_folders, legacy_placements) + result = change(current) + if result is _NAV_REFUSED: + return state, None + navigation[user] = _clean_nav(current, routes) + state["routeNavigation"] = navigation + return state, result + return _persist(st, mutate) + + +def put_route_folder(st, username, raw, routes=(), *, legacy_folders=(), legacy_placements=None): + folder_id = str((raw or {}).get("id") or "route_folder_" + uuid.uuid4().hex)[:96] + name, parent = _folder_name((raw or {}).get("name")), str((raw or {}).get("parent") or "")[:96] or None + if not name: + return None + def change(nav): + folders = nav["folders"] + prior = next((item for item in folders if item["id"] == folder_id), None) + if parent == folder_id or (parent and not any(item["id"] == parent for item in folders)): + return _NAV_REFUSED + if parent: + by_id = {item["id"]: item for item in folders} + cursor, walked, depth = parent, set(), 1 + while cursor: + if cursor == folder_id or cursor in walked or depth >= 10: + return _NAV_REFUSED + walked.add(cursor) + cursor = (by_id.get(cursor) or {}).get("parent") + depth += 1 + if prior: + prior.update({"name": name, "parent": parent}) + else: + prior = {"id": folder_id, "name": name, "parent": parent, "order": len(folders)} + folders.append(prior) + return copy.deepcopy(prior) + return _mutate_nav(st, username, routes, change, + legacy_folders=legacy_folders, legacy_placements=legacy_placements) + + +def delete_route_folder(st, username, folder_id, routes=(), *, legacy_folders=(), legacy_placements=None): + folder_id = str(folder_id or "") + def change(nav): + if not any(item["id"] == folder_id for item in nav["folders"]): + return _NAV_REFUSED + nav["folders"] = [item for item in nav["folders"] if item["id"] != folder_id] + for item in nav["folders"]: + if item.get("parent") == folder_id: + item["parent"] = None + nav["placements"] = {key: value for key, value in nav["placements"].items() if value != folder_id} + return True + return bool(_mutate_nav(st, username, routes, change, + legacy_folders=legacy_folders, legacy_placements=legacy_placements)) + + +def place_route(st, username, route_id, folder_id, routes=(), *, legacy_folders=(), legacy_placements=None): + route_id, folder_id = str(route_id or ""), str(folder_id or "") or None + def change(nav): + if route_id not in {str((item or {}).get("id")) for item in routes or ()}: + return _NAV_REFUSED + if folder_id and not any(item["id"] == folder_id for item in nav["folders"]): + return _NAV_REFUSED + if folder_id: + nav["placements"][route_id] = folder_id + else: + nav["placements"].pop(route_id, None) + return folder_id + result = _mutate_nav(st, username, routes, change, + legacy_folders=legacy_folders, legacy_placements=legacy_placements) + return result + + +def reorder_routes(st, username, order, routes=(), *, legacy_folders=(), legacy_placements=None): + offered = [str(value) for value in order or ()] + def change(nav): + known = {str((item or {}).get("id")) for item in routes or ()} + if len(offered) != len(set(offered)) or any(value not in known for value in offered): + return _NAV_REFUSED + nav["order"] = [*offered, *(value for value in nav["order"] if value not in offered)] + return list(nav["order"]) + return _mutate_nav(st, username, routes, change, + legacy_folders=legacy_folders, legacy_placements=legacy_placements) + + +def reorder_route_folders(st, username, order, routes=(), *, legacy_folders=(), legacy_placements=None): + offered = [str(value) for value in order or ()] + def change(nav): + known = {item["id"] for item in nav["folders"]} + if len(offered) != len(set(offered)) or any(value not in known for value in offered): + return _NAV_REFUSED + rank = [*offered, *(item["id"] for item in nav["folders"] if item["id"] not in offered)] + positions = {folder_id: index for index, folder_id in enumerate(rank)} + nav["folders"].sort(key=lambda item: positions[item["id"]]) + for index, item in enumerate(nav["folders"]): + item["order"] = index + return rank + return _mutate_nav(st, username, routes, change, + legacy_folders=legacy_folders, legacy_placements=legacy_placements) + + +def list_zones(st, rows=None): + return [copy.deepcopy(zone) for zone in read(st, rows).get("zones") or [] if isinstance(zone, dict)] + + +def put_route(st, raw, *, tenant_id, created_by): + """Dedicated normalized Route writer. Generic table endpoints cannot call this.""" + wanted = str((raw or {}).get("id") or "").strip() + + def mutate(state): + routes = list(state.get("routes") or []) + prior_index = next((index for index, item in enumerate(routes) + if isinstance(item, dict) and item.get("id") == wanted), None) + prior = routes[prior_index] if prior_index is not None else None + clean = _route(raw, ordinal=prior_index if prior_index is not None else len(routes), + created_by=created_by, tenant_id=tenant_id, prior=prior) + if clean is None: + return state, None + if prior_index is None: + routes.append(clean) + else: + routes[prior_index] = clean + state["routes"] = routes + return state, clean + + return _persist(st, mutate) + + +def move_stop(st, route_id, customer_pid, order, *, tenant_id, created_by): + """Move one protected stop. Empty routes are valid and never manufacture a stop.""" + pid = _pid(customer_pid) + try: + target = max(1, int(order)) + except (TypeError, ValueError): + return None + if pid is None: + return None + route = next((item for item in list_routes(st) if item.get("id") == str(route_id)), None) + if route is None: + return None + ids = [stop.get("customerPid") for stop in ordered_stops(route)] + if pid not in ids: + return None + ids.remove(pid) + ids.insert(min(target - 1, len(ids)), pid) + return put_route(st, {**route, "stops": ids}, tenant_id=tenant_id, created_by=created_by) + + +def set_stop_membership(st, route_id, customer_pid, present, *, tenant_id, created_by): + """Add or remove exactly one protected stop without replacing the Route snapshot. + + The mutation runs inside the store update so two concurrent pin confirmations cannot read the + same old stop list and then overwrite one another. In particular, callers never submit the + filtered ``visibleStopIds`` DTO as the full Route: every stop outside the current View remains + in its original relative order, and ``_route`` re-densifies the resulting relation. + """ + pid = _pid(customer_pid) + route_id = str(route_id or "").strip() + if pid is None or not isinstance(present, bool) or not route_id: + return None + + def mutate(state): + routes = list(state.get("routes") or []) + prior_index = next((index for index, item in enumerate(routes) + if isinstance(item, dict) and item.get("id") == route_id), None) + if prior_index is None: + return state, None + prior = routes[prior_index] + ids = [stop.get("customerPid") for stop in ordered_stops(prior)] + if present: + if pid not in ids: + ids.append(pid) + else: + ids = [item for item in ids if item != pid] + clean = _route({**prior, "stops": ids}, ordinal=prior_index, created_by=created_by, + tenant_id=tenant_id, prior=prior) + if clean is None: + return state, None + routes[prior_index] = clean + state["routes"] = routes + return state, clean + + return _persist(st, mutate) + + +def delete_route(st, route_id): + route_id = str(route_id or "").strip() + + def mutate(state): + prior = list(state.get("routes") or []) + removed_keys = {route_id} + for item in prior: + if isinstance(item, dict) and item.get("id") == route_id: + legacy_key = (item.get("migration") or {}).get("legacyField") + if legacy_key: + removed_keys.add(str(legacy_key)) + state["routes"] = [item for item in prior if not (isinstance(item, dict) and item.get("id") == route_id)] + navigation = dict(state.get("routeNavigation") or {}) + for username, raw in navigation.items(): + if not isinstance(raw, dict): + continue + nav = dict(raw) + nav["placements"] = {key: value for key, value in (nav.get("placements") or {}).items() + if str(key) not in removed_keys} + nav["order"] = [value for value in nav.get("order") or () if str(value) != route_id] + navigation[username] = nav + state["routeNavigation"] = navigation + return state, len(prior) != len(state["routes"]) + + return bool(_persist(st, mutate)) + + +def put_zone(st, raw, *, rows, tenant_id, created_by): + wanted = str((raw or {}).get("id") or "").strip() + + def mutate(state): + zones = list(state.get("zones") or []) + prior_index = next((index for index, item in enumerate(zones) + if isinstance(item, dict) and item.get("id") == wanted), None) + prior = zones[prior_index] if prior_index is not None else None + clean = _zone(raw, ordinal=prior_index if prior_index is not None else len(zones), + created_by=created_by, tenant_id=tenant_id, prior=prior) + if clean is None: + return state, None + clean["memberPids"] = zone_members(clean, rows) + old_members = prior.get("memberPids") if isinstance(prior, dict) else None + clean["membershipRevision"] = int((prior or {}).get("membershipRevision") or 0) + ( + 1 if old_members != clean["memberPids"] else 0) + if prior_index is None: + zones.append(clean) + else: + zones[prior_index] = clean + state["zones"] = zones + return state, clean + + return _persist(st, mutate) + + +def delete_zone(st, zone_id): + zone_id = str(zone_id or "").strip() + + def mutate(state): + prior = list(state.get("zones") or []) + state["zones"] = [item for item in prior if not (isinstance(item, dict) and item.get("id") == zone_id)] + return state, len(prior) != len(state["zones"]) + + return bool(_persist(st, mutate)) + + +def customer_projections(routes, zones): + """Derived read-only customer multi-select values; stops are never generic Link cells.""" + projections = {} + for route in routes or []: + for stop in ordered_stops(route): + projections.setdefault(stop.get("customerPid"), {"Routes": [], "Zone": []})["Routes"].append(route.get("name")) + for zone in zones or []: + for pid in zone.get("memberPids", zone.get("member_pids", ())) or (): + projections.setdefault(pid, {"Routes": [], "Zone": []})["Zone"].append(zone.get("name")) + return projections + + +def customer_projection_fields(routes, zones): + """The two read-only customer projection fields, never generic Link definitions.""" + route_names = [str(route.get("name") or "") for route in routes or [] + if str(route.get("name") or "")] + zone_names = [str(zone.get("name") or "") for zone in zones or [] + if str(zone.get("name") or "")] + return [field_axes({"key": "routes", "label": "Routes", "type": "multiselect", + "source": "odoo", "derived": True, "readOnly": True, + "options": route_names}), + field_axes({"key": "zone", "label": "Zone", "type": "multiselect", + "source": "odoo", "derived": True, "readOnly": True, + "options": zone_names})] + + +def apply_customer_projections(rows, routes, zones): + """Layer normalized Route/Zone projections onto already scoped customer rows.""" + projection = customer_projections(routes, zones) + out = [] + for row in rows or []: + item = dict(row) + values = projection.get(item.get("pid"), projection.get(str(item.get("pid")), {})) or {} + item["routes"] = ", ".join(str(name) for name in values.get("Routes", ()) if name) + item["zone"] = ", ".join(str(name) for name in values.get("Zone", ()) if name) + out.append(item) + return out + + +def apply_managed_route_selection(view_state, route_ids, routes): + """Replace only Loopable's managed OR clause in an otherwise caller-owned View state. + + The generic View writer remains the persistence authority. This narrow helper makes a + Route selection safe to compose: it neither reads nor rewrites the user's ordinary filter + tree and an empty selection removes only the managed clause it previously produced. + """ + out = copy.deepcopy(view_state if isinstance(view_state, dict) else {}) + out.pop("managedRouteClause", None) + wanted = {str(value) for value in route_ids or () if str(value)} + if not wanted: + return out + pids, known = [], set() + for route in routes or (): + if str(route.get("id")) not in wanted: + continue + for stop in ordered_stops(route): + pid = _pid(stop.get("customerPid")) + if pid is not None and pid not in known: + known.add(pid) + pids.append(pid) + out["managedRouteClause"] = {"managedBy": MANAGED_BY, "kind": "route-membership", + "operator": "or", "routeIds": sorted(wanted), "pids": pids} + return out + + +def legacy_route_plan(fields, cells, *, tenant_id, created_by, ordinal_offset=0): + """Project legacy route-order fields without writing the normalized store. + + The same pure projection feeds the explicit parity migration and the compatibility read path. + A field key becomes the stable Route id, so repeated GETs cannot manufacture duplicate Routes. + """ + plans = [] + for key, definition in sorted((fields or {}).items(), key=lambda item: str(item[0])): + if not isinstance(definition, dict) or definition.get("kind") != "route_order": + continue + migration_mark = definition.get("legacyRouteMigration") + if isinstance(migration_mark, dict) and migration_mark.get("verified"): + continue + ranks = [] + for raw_pid, values in (cells or {}).items(): + try: + rank = int(str((values or {}).get(key) or "").strip()) + except (TypeError, ValueError): + continue + pid = _pid(raw_pid) + if pid is not None and rank > 0: + ranks.append((rank, pid)) + ranks.sort() + if len({rank for rank, _pid_value in ranks}) != len(ranks): + continue + plan = _route({"id": "legacy_" + str(key)[:72], "name": definition.get("label") or key, + "color": definition.get("color"), + "shareKey": key, + "sourceViewId": (definition.get("route") or {}).get("sourceViewId"), + "sourceViewRevision": (definition.get("route") or {}).get("inputsHash"), + "stops": [pid for _rank, pid in ranks], + "migration": {"legacyField": key, "verified": True}}, + ordinal=max(0, int(ordinal_offset)) + len(plans), + tenant_id=tenant_id, + created_by=definition.get("createdBy") or created_by) + if plan is not None: + plans.append({"field": key, "route": plan, + "expected": {pid: rank for rank, pid in ranks}}) + return plans + + +def compatible_routes(routes, fields, cells, *, tenant_id, created_by): + """Read-through view of normalized Routes plus still-live legacy route-order fields. + + This function is intentionally store-free. Listing a Route must never mutate tenant data or + require an administrator to run a migration first. If an independently verified migration + already produced the same stable ``legacy_`` id, the normalized record wins and the + compatibility projection disappears rather than rendering a duplicate. + """ + out = [copy.deepcopy(route) for route in routes or () if isinstance(route, dict)] + existing_ids = {str(route.get("id") or "") for route in out} + last_number = max( + (int(route.get("number") or 0) for route in out + if str(route.get("number") or "").lstrip("-").isdigit()), + default=0, + ) + plans = legacy_route_plan( + fields, cells, tenant_id=tenant_id, created_by=created_by, + ordinal_offset=last_number, + ) + for plan in plans: + route = plan.get("route") if isinstance(plan, dict) else None + if not isinstance(route, dict) or str(route.get("id") or "") in existing_ids: + continue + existing_ids.add(str(route.get("id") or "")) + # Transient marker only: a compatibility projection has no normalized write target yet. + # A successfully migrated Route with this same stable id remains an ordinary editable + # normalized record because it wins above and never receives this marker. + out.append({**copy.deepcopy(route), "compatibility": "legacy-route-order"}) + return out + + +def migrate_legacy_routes(st, fields, cells, *, tenant_id, created_by, zones=()): + """Persist copied Routes and prove count/member/order/projection parity before hiding fields.""" + current_routes = list_routes(st) + last_number = max((int(route.get("number") or 0) for route in current_routes), default=0) + plans = legacy_route_plan( + fields, cells, tenant_id=tenant_id, created_by=created_by, + ordinal_offset=last_number, + ) + if not plans: + return {"migrated": [], "verified": {}, "hide": []} + + def mutate(state): + routes = list(state.get("routes") or []) + migrations = dict(state.get("legacyMigrations") or {}) + migrated, verified, hide = [], {}, [] + for plan in plans: + field, route = plan["field"], plan["route"] + existing_index = next((index for index, item in enumerate(routes) + if isinstance(item, dict) and item.get("id") == route["id"]), None) + if existing_index is None: + routes.append(route) + candidate = route + else: + candidate = routes[existing_index] + expected = plan["expected"] + observed = {stop["customerPid"]: stop["position"] for stop in ordered_stops(candidate)} + projections = customer_projections(routes, zones) + projection_ok = all(candidate.get("name") in (projections.get(pid) or {}).get("Routes", []) + for pid in expected) + parity = {"routeCount": len(expected) == len(observed), + "memberCount": set(expected) == set(observed), "order": expected == observed, + "projections": projection_ok} + verified[field] = parity + if all(parity.values()): + migrations[field] = {"routeId": candidate["id"], "verified": True, "at": _now()} + migrated.append(candidate["id"]) + hide.append(field) + state["routes"], state["legacyMigrations"] = routes, migrations + return state, {"migrated": migrated, "verified": verified, "hide": hide} + + return _persist(st, mutate) diff --git a/platform/core/shared_overlay.py b/platform/core/shared_overlay.py index 9b89f4d1d469cd535afb48bd650a53b4bd7c159c..9c83c2e460220dc691b3155a1c5324c68f5add26 100644 --- a/platform/core/shared_overlay.py +++ b/platform/core/shared_overlay.py @@ -101,7 +101,8 @@ ALLOWED_FIELD_KEYS = frozenset({ 'note', 'description', '_note', # display and behaviour 'format', 'agg', 'options', 'optionColors', 'colorCodeOptions', 'default', 'pinned', - 'filterable', 'multi', 'max', 'derived', 'preset', 'custom', 'profile', 'link', 'recurrence', + 'filterable', 'offerable', 'multi', 'max', 'derived', 'preset', 'custom', 'profile', 'link', + 'recurrence', # the created strata's computation bags 'formula', 'measure', 'metric', 'rollup', 'geocode', 'code', 'automation', 'aiEnrich', 'scope', @@ -148,6 +149,17 @@ def _read(table_key, st=None): 'cells': dict(data.get('cells') or {})} +def document(table_key, st=None): + """One captured shared-stratum document for consumers in a single request. + + A field-contract assembly asks three questions of the same small bucket: reconcile its + stored permission word, merge its definitions, and classify the resulting fields. They are + one request's view of one document, so reopening it for each question buys no freshness and + pays three deep copies. Callers that do not hold one keep the historical read path. + """ + return _read(table_key, st=st) + + def _write(table_key, change, st=None, flush='async'): """Read-modify-write one shared stratum. @@ -188,14 +200,15 @@ def _value(value): # ----------------------------------------------------------------- the column DEFINITIONS -def fields(table_key, st=None): +def fields(table_key, st=None, document=None): """`{field_key: Field}` — the columns this topic shares tenant-wide. Unscoped on purpose, and it is the one thing here that is: a shared column's EXISTENCE is tenant-wide by definition — that is the whole feature, and it is what stops a shared view naming a column half the workspace lacks. Its VALUES are scoped by `cells()`. """ - return _read(table_key, st)['fields'] + data = document if isinstance(document, dict) else _read(table_key, st) + return dict(data.get('fields') or {}) def is_shared(table_key, field_key, st=None): @@ -482,6 +495,90 @@ def put_cells_and_choice_options(table_key, pid, values, choice_values, st=None) return clean +def transpose_rank(table_key, pid, field_key, rank, pids, st=None): + """Atomically assign one positive integer rank within an explicitly scoped row set. + + If another scoped row holds ``rank``, that row receives this row's prior rank. A row that did + not previously have a rank sends the displaced holder to ``max + 1``. The returned ``swap`` + is shaped for the HTTP caller that must invalidate the other row it changed. + + The scope is required for the same reason it is required by :func:`cells`: this persistence + primitive must never discover or mutate a row the caller was not already allowed to see. + + The read and both writes deliberately live inside ONE ``store.update`` callback. Reading the + ranks first and then calling :func:`put_rows` is not equivalent: two concurrent requests can + both choose a holder from the same stale snapshot, serialize their precomputed writes, and + leave duplicate ranks. Both the Postgres row lock and the HF conflict-replay path protect the + callback itself, so the second request computes against the first request's landed result. + """ + row_id = _pid(pid) + key = str(field_key or '').strip() + if not key: + raise ValueError('shared_overlay.transpose_rank: a field key is required') + if isinstance(rank, bool): + raise ValueError('shared_overlay.transpose_rank: a rank is a positive whole number') + try: + want = int(rank) + except (TypeError, ValueError) as exc: + raise ValueError( + 'shared_overlay.transpose_rank: a rank is a positive whole number') from exc + if want < 1: + raise ValueError('shared_overlay.transpose_rank: a rank is a positive whole number') + try: + scope = {_pid(item) for item in pids} + except TypeError as exc: + raise ValueError( + 'shared_overlay.transpose_rank: scoped pids are required') from exc + if row_id not in scope: + raise ValueError( + 'shared_overlay.transpose_rank: the target pid must be in the scoped row set') + + outcome = {} + + def _transpose(data): + # Numeric pid order makes the holder deterministic if an already-corrupt document has + # more than one. A healthy route has at most one; this function preserves that invariant. + ordered_scope = sorted(scope, key=int) + held = {} + for scoped_pid in ordered_scope: + row = data['cells'].get(scoped_pid) + if not isinstance(row, dict): + continue + value = row.get(key) + if value in (None, ''): + continue + try: + held[scoped_pid] = int(str(value).strip()) + except (TypeError, ValueError): + continue + + prior = held.get(row_id) + max_rank = max(held.values(), default=0) + holder = None if prior == want else next( + (other_pid for other_pid in ordered_scope + if other_pid != row_id and held.get(other_pid) == want), + None) + moved = None + + data['cells'].setdefault(row_id, {})[key] = str(want) + if holder is not None: + moved = prior if prior is not None else max_rank + 1 + data['cells'].setdefault(holder, {})[key] = str(moved) + + # ``store.update`` may replay this callback over a newer HF document. Replace, rather + # than append to, the captured outcome so the caller receives the final replay's truth. + outcome.clear() + outcome.update({ + 'value': str(want), + 'prior': '' if prior is None else str(prior), + 'swap': ({'pid': int(holder), 'field': key, 'value': str(moved)} + if holder is not None else None), + }) + + _write(table_key, _transpose, st, flush='sync') + return dict(outcome) + + def put_rows(table_key, rows, st=None): """Write shared cells on MANY rows in ONE store update. Returns `{row_id: {key: value}}`. diff --git a/platform/core/store_pg.py b/platform/core/store_pg.py index 0be96a7314e4d86fb43e98fad880ec4cd8abb8d8..e43fb9c8c4aa16d1210817326f81e13e9c7f5bff 100644 --- a/platform/core/store_pg.py +++ b/platform/core/store_pg.py @@ -39,6 +39,7 @@ empty dict gets written back over real data. import json import os import threading +from urllib.parse import urlparse import core.data_binding as data_binding @@ -137,6 +138,8 @@ PRODUCTION_TENANTS = frozenset({ #: one it did not name". That is `--data-repo`'s exact failure, which is the failure this file's #: existence is a response to. _SANDBOX_TENANTS_VAR = 'AIOS_SANDBOX_TENANTS' +_ISOLATED_DATABASE_HOST_VAR = 'AIOS_ISOLATED_DATABASE_HOST' +_STAGING_SPACE = 'fsanyoto/loopable' def sandbox_tenants(): @@ -145,6 +148,18 @@ def sandbox_tenants(): return frozenset(s.strip().lower() for s in raw.split(',') if s.strip()) +def isolated_staging_database(): + """Whether this process is staging on the exact isolated database host it was handed.""" + if data_binding.deployment_id() != _STAGING_SPACE: + return False + allowed_host = str(os.environ.get(_ISOLATED_DATABASE_HOST_VAR) or '').strip().lower() + try: + current_host = (urlparse(str(os.environ.get('DATABASE_URL') or '')).hostname or '').lower() + except ValueError: + return False + return bool(allowed_host and current_host == allowed_host) + + def write_refusal(tenant_slug=None): """`None` if this deployment may write this tenant's schema, else the sentence explaining why. @@ -152,19 +167,19 @@ def write_refusal(tenant_slug=None): thing being classified changes, from a dataset repo to a tenant slug: 1. the PRODUCTION deployment -> may write any tenant - 2. any other deployment writing a PRODUCTION tenant -> REFUSED (a laptop may opt in) + 2. any other deployment writing a PRODUCTION tenant -> REFUSED, except exact isolated staging 3. any other deployment writing a tenant it was HANDED -> allowed 4. anything else -> REFUSED - ⭐ RULE 2 BEATS RULE 3, deliberately, which is why the order above is the order below: an - allowlist must not be able to re-open tenant #0. ⚠ And the Space/laptop asymmetry is carried - over unchanged — a Space that is not production gets NO override, because that is the case the - owner said "EVER" about. + Rule 2 permits one narrow exception: the exact staging Space, an explicitly handed tenant, + and a DATABASE_URL host matching the isolated host stamped by the deploy must all agree. """ slug = _effective_slug(tenant_slug) if data_binding.is_production_deployment(): return None # rule 1 if slug in PRODUCTION_TENANTS: # rule 2 + if slug in sandbox_tenants() and isolated_staging_database(): + return None if data_binding.local_override(): return None who = data_binding.deployment_id() or 'this machine (no Space identity)' @@ -172,8 +187,8 @@ def write_refusal(tenant_slug=None): if data_binding.is_space(): return (f'{who} is not the production deployment, so it may not write tenant {where}, ' f'which holds real customer data. Point this deployment at its own database, ' - f'or name the sandbox tenants it may write in {_SANDBOX_TENANTS_VAR}. There is ' - f'no override for a Space: a build that is not live never writes live data.') + f'or bind exact isolated staging and name its writable tenants in ' + f'{_SANDBOX_TENANTS_VAR}. A non-live build never writes the live database.') return (f'{who} may not write tenant {where}, which holds real customer data. Set ' f'AIOS_TENANT to a sandbox tenant, or set ' f'{data_binding._LOCAL_OPT_IN_VAR}=1 to opt in deliberately.') diff --git a/platform/core/table_store.py b/platform/core/table_store.py index ed8f588db2172775b3bb5445b2604565c239d681..fb6d716569c603a34992b69282580c41ad31079e 100644 --- a/platform/core/table_store.py +++ b/platform/core/table_store.py @@ -595,7 +595,7 @@ class TableStore: return None # ---------------------------------------------------------------- read - def workspace(self, username, consume_corrections=True): + def workspace(self, username, consume_corrections=True, include_document=False): """One user's durable workspace: always the full three-strata shape.""" try: data = self._st.get(self.table_key) or {} @@ -673,7 +673,10 @@ class TableStore: # stratum, absent until the user first reorders — exactly "default order". if isinstance(ws.get('recordLayout'), dict): out['recordLayout'] = dict(ws['recordLayout']) - return out + # A `/rows` assembly already needs the whole workspace document to count field usage. + # Returning this request-local snapshot avoids a third identical store read; callers that + # only render the personal projection retain the original dict return shape. + return (out, data) if include_document else out # ---------------------------------------------------------------- write def _update(self, username, change, shared=None): @@ -1150,25 +1153,44 @@ class TableStore: """Persist cell updates and newly admitted choice labels in one workspace update. The event wall has already checked field visibility, edit permission, label size, and the - 50-choice cap. This method owns only residency and atomicity: an overlay value must never - commit while the field vocabulary remains stale, because a refresh would render a cell the - picker cannot select. + 50-choice cap. This method owns only residency and atomicity: each cell and its vocabulary + must land together in the stratum that owns the field. A shared value written into the + caller's private workspace is shadowed by the shared read layer and therefore appears to + reset after refresh (D-423); a private option written into the shared schema leaks another + user's vocabulary. Choice additions are admitted only beside an update for the same key, + so a stale or mismatched additions bag cannot resurrect a field in either stratum. """ clean = dict(updates or {}) + clean_keys = {str(key) for key in clean if str(key)} additions = {str(key): list(values or ()) for key, values in - (choice_values or {}).items() if str(key) and values} + (choice_values or {}).items() + if str(key) in clean_keys and values} if not clean: return + wide = self._tenant_wide_keys() + shared = {key: value for key, value in clean.items() if key in wide} + personal = {key: value for key, value in clean.items() if key not in wide} + shared_additions = {key: values for key, values in additions.items() if key in shared} + personal_additions = {key: values for key, values in additions.items() if key in personal} + + if shared: + import core.shared_overlay as shared_overlay + shared_overlay.put_cells_and_choice_options( + self.table_key, pid, shared, shared_additions, st=self._st) + + if not personal: + return + def _patch(ws): - ws['overlays'].setdefault(str(int(pid)), {}).update(clean) - if not additions: + ws['overlays'].setdefault(str(int(pid)), {}).update(personal) + if not personal_additions: return # Import here rather than at module load: `aios_grid` is intentionally independent # of `core`, while this lower persistence layer may use its pure canonical cleaner. from aios_grid import merge_choice_options fields = ws.setdefault('fields', {}) - for key, values in additions.items(): + for key, values in personal_additions.items(): prior = fields.get(key) entry = dict(prior) if isinstance(prior, dict) else {} entry['options'] = merge_choice_options(entry.get('options'), values) diff --git a/platform/core/user_tables.py b/platform/core/user_tables.py index 75030e7492985bf763b1d146ffdf6e7d52153f78..7d43ae991b03b93665e7b7efbeb876e9dc16fbef 100644 --- a/platform/core/user_tables.py +++ b/platform/core/user_tables.py @@ -52,6 +52,7 @@ import re import uuid as _uuid import core.store as store +from core.route_system import MAX_OPTIONS STORE_KEY = 'user_tables' ZONE_STORE_KEY = 'table_zones' @@ -491,6 +492,9 @@ def records_mutable(table_key, st=None): UT_FIELD_TYPES = {'text', 'select', 'multiselect', 'user', 'int', 'currency', 'pct', 'date', 'checkbox', 'phone', 'email', 'url', 'rating', 'automation', 'json', 'link', 'rollup', 'code', 'image', 'formula', 'ai_enrich'} +FIELD_ORIGINS = {'odoo', 'loopable', 'system', 'user', 'automation'} +FIELD_AUTHORITIES = {'source', 'derived', 'user-entered', 'computed'} +FIELD_SCOPES = {'source', 'tenant', 'view', 'record'} #: A cell is still a scalar on the wire (`Row` values are strings/numbers) — what this bounds is #: the DOCUMENT inside it. Source data is one already-paid provider response; keep it whole so a #: provider field is never silently discarded merely because the response is rich. 32 MiB matches @@ -811,10 +815,19 @@ FIELD_AGGS = ('sum', 'average', 'median', 'min', 'max', 'count') #: A choice colour, as the client's editor spells one. `#RRGGBB`, upper-cased on the way in so a #: stored value has exactly ONE spelling and two fields cannot disagree about `#ebf6ef`. _OPTION_COLOR_RE = re.compile(r'#[0-9A-F]{6}') -_MAX_CHOICE_OPTIONS = 50 _MAX_CHOICE_LABEL = 60 +def option_cap_refusal(raw_options): + """Name an over-cap choice declaration, including entered, limit, and rejected counts.""" + entered = len([option for option in (raw_options or []) if str(option).strip()]) + if entered <= MAX_OPTIONS: + return None + rejected = entered - MAX_OPTIONS + return (f'That column declares {entered} options; the server limit is {MAX_OPTIONS}, so ' + f'{rejected} would be rejected. Remove {rejected} options, then save.') + + def _clean_option_colors(raw, options): """`{option label → #RRGGBB}`, capped to the field's own option vocabulary (wave-29 C1). @@ -887,7 +900,7 @@ def _hydrate_choice_options(fields, value_maps): continue for option in _choice_values(field, values[key]): folded = option.casefold() - if folded in known or len(options) >= _MAX_CHOICE_OPTIONS: + if folded in known or len(options) >= MAX_OPTIONS: continue options.append(option) known.add(folded) @@ -947,7 +960,17 @@ def clean_fields(raw): if not key or key in seen or ftype not in UT_FIELD_TYPES or not label: continue seen.add(key) - entry = {'key': key, 'label': label, 'type': ftype, 'source': 'overlay'} + origin = str(f.get('origin') or 'user') + authority = str(f.get('valueAuthority') or 'user-entered') + scope = str(f.get('scope') or 'tenant') + # C3: these are closed product vocabularies. Do not silently persist a word no + # renderer/permission door understands. + if origin not in FIELD_ORIGINS or authority not in FIELD_AUTHORITIES or scope not in FIELD_SCOPES: + continue + entry = {'key': key, 'label': label, 'type': ftype, 'source': 'overlay', + 'kind': ftype, 'origin': origin, 'valueAuthority': authority, 'scope': scope} + if ftype == 'json' and not f.get('label'): + entry['label'] = 'Table' # ⭐ 2026-08-10 — the DISPLAY format, carried through the CREATE door. See `_clean_format`: # this is the fifth time an entry built key-by-key has silently dropped a bag the editor # sends, and the fourth of those was `agg` on this very line one wave ago. @@ -967,8 +990,12 @@ def clean_fields(raw): if note: entry['note'] = note if ftype in ('select', 'multiselect'): + # Refuse the field instead of storing a silently shortened vocabulary. This list + # validator already drops one malformed field while preserving valid siblings. opts = [' '.join(str(o).split())[:60] for o in (f.get('options') or []) - if str(o).strip()][:50] + if str(o).strip()] + if len(opts) > MAX_OPTIONS: + continue entry['options'] = opts if f.get('default') is True: entry['default'] = True @@ -1146,7 +1173,9 @@ def get(table_key, st=None): _ZONE_ID_RE = re.compile(r'^[A-Za-z0-9_-]{1,96}$') _ZONE_FIELD_RE = re.compile(r'^[A-Za-z][A-Za-z0-9_]{0,79}$') _ZONE_COLOR_RE = re.compile(r'^#[0-9A-Fa-f]{6}$') +_ZONE_ZIP_RE = re.compile(r'^\d{5}(?:-\d{4})?$') _ZONE_GEOMETRIES = frozenset({'Polygon', 'MultiPolygon'}) +MAX_ZONE_ZIP_CODES = 500 def _zone_position(raw): @@ -1205,11 +1234,25 @@ def clean_zone(raw, field_keys=None, allowed_row_ids=None): # cannot be legal inside one Zone label without making one Zone appear as two filter choices. name = ' '.join(str(raw.get('name') or '').replace(',', ' ').split())[:120] geometry = _clean_zone_geometry(raw.get('geometryGeoJson')) - if not name or geometry is None: + raw_zips = raw.get('zipCodes') or [] + if not isinstance(raw_zips, (list, tuple)) or len(raw_zips) > MAX_ZONE_ZIP_CODES: + return None + zip_codes, seen_zips = [], set() + for value in raw_zips: + zip_code = str(value or '').strip().upper() + if not _ZONE_ZIP_RE.fullmatch(zip_code): + return None + if zip_code not in seen_zips: + seen_zips.add(zip_code) + zip_codes.append(zip_code) + # Existing polygon Zones remain valid byte-for-byte. Wave 44 also permits a ZIP-only Zone; + # the two shapes share this record rather than inventing a second persistence model. + if not name or (geometry is None and not zip_codes): return None out = {'name': name, 'color': (str(raw.get('color') or '#1F4E78').upper()), 'geometryGeoJson': geometry, + 'zipCodes': zip_codes, 'visible': raw.get('visible') is not False} if not _ZONE_COLOR_RE.fullmatch(out['color']): return None @@ -1371,12 +1414,22 @@ def is_user_table(table_key, st=None): return str(table_key or '').startswith(KEY_PREFIX) and get(table_key, st) is not None -def create(label, username, fields=None, source=None, st=None): +def is_managed_system_table(table_key, st=None, definition=None): + """C1 policy predicate, intentionally separate from ordinary record mutability.""" + from core import route_system + return route_system.is_managed(definition if definition is not None else (get(table_key, st) or table_key)) + + +def create(label, username, fields=None, source=None, st=None, *, managed_by=None, capability=None): """Create a blank table. Returns its key, or None if refused. Refuses rather than raises on the ordinary conditions (no name, duplicate, cap reached) — the caller is a UI form and a refusal is a message, not an exception. """ + if managed_by is not None: + from core import route_system + if managed_by != route_system.MANAGED_BY or not route_system.is_internal_writer(capability): + return None label = str(label or '').strip()[:MAX_LABEL] if not label: return None @@ -1404,6 +1457,10 @@ def create(label, username, fields=None, source=None, st=None): 'source': 'overlay', 'default': True}], 'rows': {}, } + if managed_by: + from core import route_system + defn.update({'managedBy': route_system.MANAGED_BY, 'folderId': route_system.LOOPABLE_FOLDER_ID, + 'protected': True}) def _add(cur): cur[key] = defn @@ -1429,6 +1486,8 @@ def delete(table_key, st=None): Every family after the definition is best-effort: a store blip mid-sweep must not resurrect the table, and an orphaned empty bucket is residue, not a leak.""" key = str(table_key) + if is_managed_system_table(key, st=st): + return False s = _st(st) def _drop(cur): @@ -1773,6 +1832,42 @@ def _clean_metric(raw): #: ~3 KB — comfortably inside a text cell, and `MAX_ROWS` bounds the absolute worst case anyway. LINK_MAX_IDS = 500 +# Link picker filters are schema, not a View selection: the leaf below constrains candidate rows +# in the linked table. `sourceField` is explicitly a field on the current record; absence means +# this is the static/literal form. Keeping the two shapes disjoint prevents a dynamic picker +# value being mistaken for a saved View lock or a stale literal. +LINK_FILTER_OPS = ('eq', 'neq', 'contains', 'not_contains', 'is_empty', 'is_not_empty') + + +def _clean_link_filter(raw): + if not isinstance(raw, dict): + return None + field = _field_key(raw.get('field', raw.get('colId'))) + op = str(raw.get('op') or '').strip() + if not field or op not in LINK_FILTER_OPS: + return None + source = _field_key(raw.get('sourceField', raw.get('currentRecordField'))) + if source: + # A dynamic leaf has no literal fallback. Retaining one would let an old/current record + # value silently answer when the source field goes blank, a different filter under one UI. + return {'field': field, 'op': op, 'sourceField': source} + if op in ('is_empty', 'is_not_empty'): + return {'field': field, 'op': op} + if 'value' not in raw or isinstance(raw.get('value'), (dict, list, tuple, set)): + return None + return {'field': field, 'op': op, 'value': str(raw.get('value'))[:500]} + + +def dynamic_link_filter_valid(field, fields): + """Does every dynamic Link-filter source name another field on this current record?""" + bag = field.get('link') if isinstance(field, dict) else None + leaf = bag.get('filter') if isinstance(bag, dict) else None + source = leaf.get('sourceField') if isinstance(leaf, dict) else None + if not source: + return True + names = {str(item.get('key') or '') for item in fields or () if isinstance(item, dict)} + return source in names and source != str(field.get('key') or '') + #: Airtable's 17 aggregations, minus the two that are meaningless over scalar cells. #: ⛔ `arrayflatten` and `arrayslice` are NOT here: both operate on NESTED arrays, and a cell in #: this product is a scalar — they could only ever return their own input, so offering them would @@ -1883,6 +1978,117 @@ def is_linkable_target(table_key): return key.startswith(KEY_PREFIX) and len(key) > len(KEY_PREFIX) +# W42-T03 / R1 / C1: a user-created link may carry exactly one governed semantic hop. +# Keep this tuple tied to the literal reads in `_clean_via`: verify_pivot extracts those reads +# from the AST and compares them to both this allow-list and the client emitter. +LINK_VIA_KEYS = ('topic', 'source_dim', 'target_dim', 'target_key', 'scope_key', 'measure', + 'filter', 'filterConj', 'window') +LINK_VIA_CONJ = ('and', 'or') +LINK_VIA_WINDOWS = ROLLUP_SOURCE_WINDOWS + + +def _topic_dims(topic): + """Return model-declared dimensions without opening a tenant store; fail closed.""" + if not topic: + return None + try: + from harness import semantic as _sem + definition = (_sem.topics() or {}).get(topic) + except Exception: + return None + if not definition: + return None + dims = ((definition.get('store') or {}).get('dims')) or {} + return frozenset(dims) or None + + +def _clean_via_filter(raw, depth=1, budget=None): + """Shape-check a hop filter without silently trimming a widening condition.""" + import aios_grid as _agv + if budget is None: + budget = [_agv.MAX_FILTER_NODES] + if not isinstance(raw, list) or len(raw) > _agv.MAX_FILTER_SIBLINGS: + return None + out = [] + for node in raw: + if not isinstance(node, dict): + return None + budget[0] -= 1 + if budget[0] < 0: + return None + children = node.get('children') + if isinstance(children, list): + if depth >= _agv.MAX_FILTER_DEPTH: + return None + kids = _clean_via_filter(children, depth + 1, budget) + if kids is None: + return None + out.append({'conj': 'or' if node.get('conj') == 'or' else 'and', + 'children': kids}) + continue + leaf = {} + for key, value in node.items(): + if not isinstance(key, str): + return None + if isinstance(value, str): + leaf[key] = value[:500] + elif isinstance(value, (int, float, bool)) or value is None: + leaf[key] = value + elif isinstance(value, (list, dict)): + leaf[key] = value + else: + return None + out.append(leaf) + return out + + +def _clean_via(raw): + """One semantic `via` bag, or None. Nested hops and partial bags are refused.""" + if not isinstance(raw, dict): + return None + if 'via' in raw: + return None + for key in raw: + if key not in LINK_VIA_KEYS: + return None + # These are deliberately literal reads: verify_pivot mechanically extracts the accepted set. + topic = str(raw.get('topic') or '').strip() + dims = _topic_dims(topic) + if not dims: + return None + source_dim = str(raw.get('source_dim') or '').strip() + target_dim = str(raw.get('target_dim') or '').strip() + if source_dim not in dims or target_dim not in dims: + return None + # Required target/scope identities are not topic dimensions and are never defaulted. + target_key = _field_key(raw.get('target_key')) + scope_key = _field_key(raw.get('scope_key')) + measure = _field_key(raw.get('measure')) + if not target_key or not scope_key or not measure: + return None + out = {'topic': topic, 'source_dim': source_dim, 'target_dim': target_dim, + 'target_key': target_key, 'scope_key': scope_key, 'measure': measure} + raw_filter = raw.get('filter') + if raw_filter is not None: + nodes = _clean_via_filter(raw_filter) + if nodes is None: + return None + out['filter'] = nodes + raw_conj = raw.get('filterConj') + if raw_conj is not None: + conj = str(raw_conj).strip().lower() + if conj not in LINK_VIA_CONJ: + return None + out['filterConj'] = conj + raw_window = raw.get('window') + if raw_window is not None: + window = str(raw_window).strip().lower() + if window not in LINK_VIA_WINDOWS: + return None + out['window'] = window + return out + + def _clean_link(raw): """One `link` bag → the stored shape, or None (refused). @@ -1942,10 +2148,25 @@ def _clean_link(raw): # than dropped: a bag half of which is silently ignored is [[wrong-parent-not-broken-control]] # with the control still on screen. return None + # A semantic hop is meaningful only on a derived link with a first join to travel from. + # Its absence preserves the historical one-hop bag byte-for-byte. + if raw.get('via') is not None: + if 'on' not in out: + return None + via = _clean_via(raw.get('via')) + if via is None: + return None + out['via'] = via if reciprocal: out['reciprocal'] = reciprocal if raw.get('single') is True: out['single'] = True + supplied_filter = raw.get('filter', raw.get('where')) + if supplied_filter is not None: + leaf = _clean_link_filter(supplied_filter) + if leaf is None: + return None + out['filter'] = leaf return out @@ -2425,7 +2646,17 @@ def _clean_field(raw, previous=None): ftype = str(raw.get('type') or prev.get('type') or 'text').strip().lower() if not key or not label or ftype not in UT_FIELD_TYPES: return None - out = {'key': key, 'label': label, 'type': ftype, 'source': 'overlay'} + origin = str(raw.get('origin') if 'origin' in raw else prev.get('origin') or 'user') + authority = str(raw.get('valueAuthority') if 'valueAuthority' in raw else prev.get('valueAuthority') or 'user-entered') + scope = str(raw.get('scope') if 'scope' in raw else prev.get('scope') or 'tenant') + if origin not in FIELD_ORIGINS or authority not in FIELD_AUTHORITIES or scope not in FIELD_SCOPES: + return None + out = {'key': key, 'label': label, 'type': ftype, 'source': 'overlay', 'kind': ftype, + 'origin': origin, 'valueAuthority': authority, 'scope': scope} + if ftype == 'json': + out['label'] = 'Table' + if bool(raw.get('deletionLocked') if 'deletionLocked' in raw else prev.get('deletionLocked')): + out['deletionLocked'] = True # ⭐ 2026-08-10 — the DISPLAY format, on the add/patch door too. ⚠ `'format' in raw` rather than # `raw.get('format')`, exactly like `description` below: an explicit `{}`/None CLEARS the # format, while omitting the key keeps what is stored. A PATCH that sends only a label must not @@ -2477,7 +2708,9 @@ def _clean_field(raw, previous=None): # a valid, new select indistinguishable from a malformed text field on its read path. # Keep the empty list deliberately: the type is the contract; options are its current # vocabulary, not a prerequisite for the column to exist. - opts = [' '.join(str(o).split())[:60] for o in (opts_raw or []) if str(o).strip()][:50] + opts = [' '.join(str(o).split())[:60] for o in (opts_raw or []) if str(o).strip()] + if len(opts) > MAX_OPTIONS: + return None out['options'] = opts # ⭐⭐ WAVE-29 CONTRACT C1 (owner item 13) — **THE OPTION COLOURS, WHICH THIS DOOR DROPPED.** # @@ -3191,6 +3424,8 @@ def delete_field_refusal(table_key, fkey, viewer, is_admin=False, st=None, *, fi grant_topic=grant_topic) if bag is None: return FIELD_NOT_FOUND + if bool((field or {}).get('deletionLocked')): + return 'field_deletion_locked' # ⛔ ABOVE THE ADMIN BRANCH. Moving this below it is the mutation W41-T02's proof breaks on # purpose, and R6a is the sentence it would violate. if bag.get('origin') == 'preset': @@ -3287,6 +3522,8 @@ def add_field(table_key, raw, st=None): field = _clean_field(raw) if not field or any(f.get('key') == field['key'] for f in have): return None + if not dynamic_link_filter_valid(field, have): + return None if not flow_bound(field.get('automation'), st): return None # C8: no field without a flow (400 at the route) # C3 (wave 25, R7's first mitigation): AT MOST ONE profile field per table. Without it the @@ -3490,6 +3727,8 @@ def patch_field(table_key, fkey, raw, st=None): field = _clean_field(merged, fields[idx]) if not field: return None + if not dynamic_link_filter_valid(field, fields): + return None if 'automation' in merged and not flow_bound(field.get('automation'), st): return None # C8: a patch cannot un-bind a column either # C3: the one-per-table law on the PATCH door too — flagging a second column is the same act @@ -3560,6 +3799,8 @@ def delete_field(table_key, fkey, st=None): if len(fields) <= 1 or not any(f.get('key') == str(fkey) for f in fields): return False doomed = next((f for f in fields if f.get('key') == str(fkey)), {}) + if doomed.get('deletionLocked') is True: + return False doomed_link = doomed.get('link') if isinstance(doomed.get('link'), dict) else {} inverse_source = None if doomed_link.get('inverse'): diff --git a/platform/harness/semantic.py b/platform/harness/semantic.py index 70cda04f9c5ed6c37d37c5b10faa4c86bf599052..f9b6e5e77c53eabf826e3e3a82ecd2edf0f6b40f 100644 --- a/platform/harness/semantic.py +++ b/platform/harness/semantic.py @@ -71,6 +71,15 @@ class ModelError(Exception): """A model-file problem (unknown key, bad reference, unparseable expr).""" +class UnknownColumnError(ModelError): + """A filter/group request named a column the topic does not declare.""" + + def __init__(self, message, column, columns=None): + super().__init__(message) + self.column = str(column) + self.columns = sorted({str(item) for item in (columns or [column])}) + + # ---------------------------------------------------------------- loading @functools.lru_cache(maxsize=1) @@ -456,7 +465,7 @@ def _post_compute(row, key): _FORMAT_TYPE = {"usd": "currency", "int": "int", "pct": "pct"} -def _clean_tree(tree, cols): +def _clean_tree(tree, cols, tolerate=()): """Run a filter tree through the SAME validator the client's view state goes through. `filter_sql`'s documented input contract is "feed me CLEANED trees" — it is faithful to the @@ -470,6 +479,9 @@ def _clean_tree(tree, cols): module with zero imports of its own, so this cannot cycle. """ from aios_grid import (clean_filter_tree, COHORT_FIELD as _COHORT_FIELD, + VIEW_FIELD as _VIEW_FIELD, FILTER_OPS as _FILTER_OPS, + COHORT_OPS as _COHORT_OPS, COHORT_OP_ALIASES as _COHORT_ALIASES, + VIEW_OPS as _VIEW_OPS, MAX_FILTER_DEPTH, MAX_FILTER_NODES, MAX_FILTER_SIBLINGS) # REFUSE an over-large tree rather than TRUNCATE it. clean_filter_tree caps siblings per @@ -544,6 +556,53 @@ def _clean_tree(tree, cols): f" — those are resolved to a set of ids by harness.measure_filter, not compiled into " f"this query. Refusing rather than dropping or mis-compiling them, either of which " f"would silently answer a different question under an authoritative count.") + # The UI cleaner drops unknown columns/operators for stale saved views. On a server query + # that would silently widen the result, so authoring faults are refused before normalization. + # `tolerate` covers deliberately narrower row projections. + tolerated = {str(column) for column in (tolerate or ())} + known = set(cols) | tolerated | {_COHORT_FIELD, _VIEW_FIELD} + unknown = [] + bad_ops = [] + + def _inspect(nodes): + for node in nodes or []: + if not isinstance(node, dict): + continue + if isinstance(node.get('children'), list): + _inspect(node['children']) + continue + column = node.get('colId') + if column in (None, ''): + continue + column = str(column) + if column not in known: + unknown.append(column) + continue + if tolerated and column in tolerated and column not in cols: + continue + op = node.get('op') + if op in (None, ''): + continue + legal = (_COHORT_OPS | set(_COHORT_ALIASES) if column == _COHORT_FIELD + else _VIEW_OPS if column == _VIEW_FIELD + else _FILTER_OPS) + if str(op) not in legal: + bad_ops.append((column, str(op), sorted(legal))) + + _inspect(tree) + if unknown: + bad = sorted(set(unknown)) + extra = f" (and {bad[1:]})" if len(bad) > 1 else "" + raise UnknownColumnError( + f"filter rule names column {bad[0]!r}{extra}, which this topic does not have. " + f"Refusing rather than dropping the rule, which would silently widen the result.", + column=bad[0], columns=bad) + if bad_ops: + column, op, legal = bad_ops[0] + raise ModelError( + f"filter rule on column {column!r} carries operator {op!r}, which is not one this " + f"filter understands (use: {legal}). Refusing rather than dropping the rule, which " + f"would silently widen the result.") return clean_filter_tree(tree, set(cols)) @@ -782,7 +841,8 @@ def store_query(topic, measures, group_by=None, grain=None, date_from=None, date gb = [g for g in (group_by or []) if g] for g in gb: if g not in dims: - raise ModelError(f"group_by {g!r} not a dim of {topic!r} (allowed: {list(dims)})") + raise UnknownColumnError( + f"group_by {g!r} not a dim of {topic!r} (allowed: {list(dims)})", column=g) d = dims[g] select.append(f"{d['col']} AS {g}_id" if d.get("name_col") else f"{d['col']} AS {g}") group_cols.append(d["col"]) @@ -821,7 +881,8 @@ def store_query(topic, measures, group_by=None, grain=None, date_from=None, date where.append(render_scope(s["service_filter"])) for dim_key, vals in (filters or {}).items(): if dim_key not in dims: - raise ModelError(f"filter dim {dim_key!r} not allowed (use: {list(dims)})") + raise UnknownColumnError( + f"filter dim {dim_key!r} not allowed (use: {list(dims)})", column=dim_key) # id dims filter by int; TEXT dims (e.g. city) filter by the value itself — either way # every VALUE stays parameterized (the whitelist covers identifiers only) def _coerce(v): @@ -953,7 +1014,9 @@ def store_rows(topic, date_from=None, date_to=None, team_id=None, filter_tree=No # --- the user's narrowing, on top of the scope -------------------------------------- narrow, narrow_params = [], [] - pred = _fs.compile_filter_tree(_clean_tree(filter_tree or [], cols), filter_conj, cols, + pred = _fs.compile_filter_tree(_clean_tree(filter_tree or [], cols, + tolerate=store_columns(topic)), + filter_conj, cols, today=today, member_ids=member_ids or None, id_sql=id_sql) if pred is not None: @@ -1027,7 +1090,7 @@ def store_field_values(topic, dim, search=None, limit=25): raise ModelError(f"topic {topic!r} has no store binding") d = (t["store"].get("dims") or {}).get(dim) if not d: - raise ModelError(f"unknown dim {dim!r} for topic {topic!r}") + raise UnknownColumnError(f"unknown dim {dim!r} for topic {topic!r}", column=dim) if not d.get("name_col"): raise ModelError(f"dim {dim!r} has no name column (filter by id)") s = t["store"] diff --git a/platform/model/topics/sales_lines.yml b/platform/model/topics/sales_lines.yml index 5552441627296279752319bedcff6fef7f0c536e..240ddd19003a033a630e43735bf23f97413fc736 100644 --- a/platform/model/topics/sales_lines.yml +++ b/platform/model/topics/sales_lines.yml @@ -74,6 +74,14 @@ store: # 2,056 customers have a non-agent as agent_ids[0]. Do NOT "fix" it by reading the commission # table, which mixes agents and salespeople. agent: {col: "rp.agent_id", name_col: "ag.name", label: "Agent"} + # W42-T01: status operands live on the parent order already joined as `o`. These are text + # codes, so the value and name expressions are intentionally identical; this keeps filtering + # and grouping on the exact raw Odoo state without inventing a numeric companion. + invoice_status: {col: "o.invoice_status", name_col: "o.invoice_status", + label: "Invoice status"} + # `scope_sql` already limits this to confirmed/locked orders (`sale`/`done`). This dimension + # distinguishes those states; it must never widen the topic to quotations or cancellations. + order_state: {col: "o.state", name_col: "o.state", label: "Order state"} ai_context: > The wholesale sales dataset of Royal Imports / Fisch Floral Supply, at order-LINE grain. @@ -117,3 +125,7 @@ ai_context: > has no address on file — present it as "no state on file", keep it in totals) and duplicate state records exist (two ids can share one name, e.g. New Jersey twice) — list them as the data shows and note the duplication rather than silently merging. + Invoicing questions use `invoice_status`, inherited from the parent order. Its raw values are + 'invoiced', 'to invoice', 'no', and 'upselling'. `order_state` is limited to 'sale' or 'done' + because this topic is confirmed-order scope; a missing quotation bucket is not evidence that + the tenant has no quotations. diff --git a/web/public/sample_customers.json b/web/public/sample_customers.json index 3b9df6c8f0d7069bd62e3c541170b3b9fc5b6433..db9241bebe0f15a610c0227f3eacf601fba277d8 100644 --- a/web/public/sample_customers.json +++ b/web/public/sample_customers.json @@ -112,6 +112,7 @@ "label": "Google Maps", "type": "url", "source": "odoo", + "valueOverride": "overlay", "default": true, "description": "A Google Maps link to this customer's address." }, diff --git a/web/src/automation/AutomationBuilder.tsx b/web/src/automation/AutomationBuilder.tsx index 96f1bd99ffecff61b16d127bf44a60ff6520c2a7..e82a4dcad34457e41c9dc7e9226783ee3cc3737e 100644 --- a/web/src/automation/AutomationBuilder.tsx +++ b/web/src/automation/AutomationBuilder.tsx @@ -31,7 +31,7 @@ // AutomationTrigger). // --------------------------------------------------------------------------- import type { ReactNode } from "react"; -import { useEffect, useRef, useState } from "react"; +import { Fragment, useEffect, useRef, useState } from "react"; // ⭐ WAVE 30 · ITEM 7 (R3 + R4) — connector identity, from the ONE module that owns it. Not a // second copy of two SVGs: the directory, this menu and the trigger picker all answer "which @@ -50,6 +50,7 @@ import type { TriggerOption, UserTable, } from "./automationApi"; +import type { VoiceAgent } from "./voiceAgentTypes"; // ⭐ WAVE 26 · ITEM 22 / D-70 — the Run guard, imported rather than re-derived. This file and // `AutomationDetail` each paint a Run button onto the same paid action; ONE function decides. // ITEM 9 / R11 — `createTable` + `AutomationError` for the in-place "+ New database". @@ -100,8 +101,13 @@ interface Props { /** The server's trigger vocabulary (C3). Absent is a state this file states, never fills in. */ triggers?: TriggerOption[]; catalog: ActionCatalogRow[]; + /** Existing synthetic Field agents, already composed and permission-filtered by the server. */ + fieldAgents: Automation[]; + /** Open one of those existing agents; AI enrichment cannot be authored as a direct action. */ + onOpenFieldAgent: (id: string) => void; vocab?: FlowVocab; tables: UserTable[]; + voiceAgents?: VoiceAgent[]; oauth: OAuthStatus | null; /** True while a write is in flight — drives the "All changes saved" stamp and disables edits. */ busy: boolean; @@ -197,13 +203,6 @@ interface Props { * (duplicate names are legal), and the automation still points at neither. */ onTablesChanged: () => Promise | void; - /** - * Is there an edit the HEADER's Save still owns? REQUIRED, and it exists so the panel's - * "All changes saved" stamp can stop being a claim about fields it does not cover — the - * machine steps' panels moved into Properties this wave and they commit with Save, not on - * change. - */ - dirty: boolean; } /* @@ -436,8 +435,11 @@ export default function AutomationBuilder({ onNameCommit, triggers, catalog, + fieldAgents, + onOpenFieldAgent, vocab, tables, + voiceAgents, oauth, busy, onPatch, @@ -454,7 +456,6 @@ export default function AutomationBuilder({ schedules, walkTable, onTablesChanged, - dirty, }: Props) { const [sel, setSel] = useState({ kind: "trigger", id: "" }); const [picking, setPicking] = useState(false); @@ -686,6 +687,8 @@ export default function AutomationBuilder({ : null; if (kind === "find_records") return firstTable ? { table: firstTable.key, cond: null, limit: 25 } : null; + if (kind === "voice_agent") + return { voiceAgentId: "" }; /* ⭐⭐ WAVE 31 · T38 (contract C5, owner ruling R10) — THE WEB READ, and this branch is here because a gate demanded it rather than because anybody remembered. @@ -943,6 +946,77 @@ export default function AutomationBuilder({ `verify_steps` exercises the nest, which no amount of JSX here could be tested against. */ const menuGroups = groupActions(catalog); + /* + * `ai_enrich` deliberately rides the catalog with `menu:false`: it names synthetic steps but + * the ordinary action door refuses authorship because enrichment belongs to a field. Hiding it + * from the whole Agent catalog was the bug. This separate consumer uses the row's server-owned + * label/group/detail and opens only the synthetic Field-agent choices the list route supplied. + */ + const fieldAgentCatalog = catalog.find((entry) => entry.kind === "ai_enrich") || null; + + const fieldAgentChoice = fieldAgentCatalog ? ( + fieldAgentCatalog.ready && fieldAgents.length ? ( +
+ + + + + + {fieldAgentCatalog.label} + {fieldAgentCatalog.detail} + + + {fieldAgents.map((agent) => ( + + ))} +
+ ) : ( + + ) + ) : null; + /** ONE action row, so the nested and un-nested lists cannot render differently. */ const actionRow = (c: ActionCatalogRow) => ( + + + {saving || triggerBusy + ? "Saving…" + : touched + ? "Unsaved changes. Press Save" + : "All changes saved"} + + + + + + ); + })} {/* ONE LINE (R13). It used to describe the kinds — "one reads a public web page… the other fills an automation column…" — which was two sentences, wrong by the time a third kind shipped, and printed inside a `--lp-rail-w` column. What the @@ -671,6 +756,11 @@ export default function AutomationSurface() { {error} ) : null} + {voiceLoadError ? ( +
+ {voiceLoadError} +
+ ) : null} {data && !data.storeAvailable ? (
The tenant store is unavailable, so nothing can be saved right now. @@ -694,7 +784,17 @@ export default function AutomationSurface() { */} {/* T21: the editor's own Suspense boundary — see the `lazy()` above for why it cannot be `Shell.tsx`'s. `fallback={null}`, matching `Lazily`. */} - {active ? ( + {activeVoice ? ( + + { + setVoiceAgents((current) => replaceVoiceAgent(current, fresh)); + }} + /> + + ) : active ? ( void; +} + +type PersistVoiceAgent = (voiceId: string, draft: VoiceAgentDraft) => Promise; + +const READINESS_REASON_LABELS: Record = { + connection: "Connection", + provider_connection: "Provider connection", + audience: "Audience", + knowledge_profile: "Knowledge profile", + inbound_binding: "Inbound binding", + activation_gate: "Activation gate", +}; + +function statusLabel(value: string): string { + return String(value || "not reported") + .replace(/[_-]+/g, " ") + .replace(/\b\w/g, (letter) => letter.toUpperCase()); +} + +function readinessReasonLabel(value: string): string { + return READINESS_REASON_LABELS[value] || "Additional setup"; +} + +export function voiceDraftFromAgent(agent: VoiceAgent): VoiceAgentDraft { + return { + portableConfig: { ...agent.portableConfig }, + audience: { ...agent.audience }, + }; +} + +export function voiceDraftWithTable( + draft: VoiceAgentDraft, + tableKey: string +): VoiceAgentDraft { + return { + ...draft, + audience: { + ...draft.audience, + tableKey, + viewId: "", + viewRevision: "", + phoneFieldKey: "", + }, + }; +} + +export function voiceDraftWithView( + draft: VoiceAgentDraft, + table: VoiceAudienceOption | undefined, + viewId: string +): VoiceAgentDraft { + const view = table?.views.find((candidate) => candidate.id === viewId); + return { + ...draft, + audience: { + ...draft.audience, + viewId, + viewRevision: view?.revision || "", + phoneFieldKey: "", + }, + }; +} + +/** Validate against the vocabulary that accompanied this exact DTO, never a client limit table. */ +export function voiceDraftValidationIssue( + agent: VoiceAgent, + draft: VoiceAgentDraft +): string { + const policyIsOffered = agent.audienceOptions.emptyPolicies.some( + (option) => option.key === draft.audience.emptyPolicy + ); + if (!policyIsOffered) { + return "Choose an empty-value policy currently offered by the server."; + } + const ceiling = agent.audienceOptions.maxRecipientsCeiling; + const maximum = draft.audience.maxRecipients; + if (!Number.isInteger(maximum) || ceiling < 1 || maximum < 1 || maximum > ceiling) { + return ceiling > 0 + ? `Maximum recipients must be between 1 and ${ceiling}.` + : "The recipient limit is not available from the server."; + } + return ""; +} + +/** One write, followed by adoption of the exact DTO returned by that write. */ +export async function saveVoiceAgentDraft( + agent: VoiceAgent, + draft: VoiceAgentDraft, + onSaved: (fresh: VoiceAgent) => void, + persist: PersistVoiceAgent = patchVoiceAgent +): Promise { + const issue = voiceDraftValidationIssue(agent, draft); + if (issue) throw new AutomationError(issue, 400); + const next = await persist(agent.automationId, draft); + onSaved(next); + return next; +} + +export default function VoiceAgentProperties({ agent, onSaved }: Props) { + const [draft, setDraft] = useState(() => voiceDraftFromAgent(agent)); + const [input, setInput] = useState(() => ({ + table: agent.audience.tableKey, + viewId: agent.audience.viewId, + phoneField: agent.audience.phoneFieldKey, + })); + const [result, setResult] = useState(null); + const [error, setError] = useState(""); + const [saveError, setSaveError] = useState(""); + const [saveReceipt, setSaveReceipt] = useState(""); + const [checking, setChecking] = useState(false); + const [saving, setSaving] = useState(false); + + const tables = agent.audienceOptions.tables; + const selectedTable = tables.find((table) => table.key === draft.audience.tableKey); + const views = selectedTable?.views || []; + const selectedView = views.find((view) => view.id === draft.audience.viewId); + const phoneFields = selectedTable?.phoneFields || []; + const harnessTable = tables.find((table) => table.key === input.table); + const harnessViews = harnessTable?.views || []; + const harnessView = harnessViews.find((view) => view.id === input.viewId); + const harnessPhoneFields = harnessTable?.phoneFields || []; + const canCheck = !!input.table.trim() && !!input.viewId.trim() && !!input.phoneField.trim(); + const dirty = JSON.stringify(draft) !== JSON.stringify(voiceDraftFromAgent(agent)); + const emptyPolicies = agent.audienceOptions.emptyPolicies; + const recipientCeiling = agent.audienceOptions.maxRecipientsCeiling; + const selectedPolicyIsOffered = emptyPolicies.some( + (option) => option.key === draft.audience.emptyPolicy + ); + const validationIssue = voiceDraftValidationIssue(agent, draft); + const visibleSaveError = saveError || (dirty ? validationIssue : ""); + + const changeDraft = (next: VoiceAgentDraft) => { + setDraft(next); + setSaveError(""); + setSaveReceipt(""); + }; + + const saveConfiguration = async () => { + if (!dirty || validationIssue || saving) return; + setSaving(true); + setSaveError(""); + setSaveReceipt(""); + try { + const next = await saveVoiceAgentDraft(agent, draft, onSaved); + setDraft(voiceDraftFromAgent(next)); + setInput({ + table: next.audience.tableKey, + viewId: next.audience.viewId, + phoneField: next.audience.phoneFieldKey, + }); + setSaveReceipt("Configuration saved"); + } catch (reason) { + setSaveError( + reason instanceof AutomationError + ? reason.message + : "The Voice configuration could not be saved." + ); + } finally { + setSaving(false); + } + }; + + const updateHarnessInput = (key: keyof VoiceHarnessInput, value: string) => { + setInput((current) => ({ ...current, [key]: value })); + setError(""); + setResult(null); + }; + + const checkHarness = async () => { + if (!canCheck || checking) return; + setChecking(true); + setError(""); + setResult(null); + try { + const next = await dryRunVoiceAgent(agent.automationId, { + table: input.table.trim(), + viewId: input.viewId.trim(), + phoneField: input.phoneField.trim(), + }); + setResult(next); + } catch (reason) { + setError( + reason instanceof AutomationError + ? reason.message + : "The Voice harness could not validate this selection." + ); + } finally { + setChecking(false); + } + }; + + return ( +
+
+
+

Agent

+

{agent.displayName}

+
+ + {statusLabel(agent.readiness.state)} + +
+ +
+
+

Status

+
+
+
Lifecycle
+
{statusLabel(agent.lifecycle)}
+
+
+
Readiness
+
{statusLabel(agent.readiness.state)}
+
+
+
Connection
+
{statusLabel(agent.connectionStatus)}
+
+
+ {agent.readiness.reasons.length ? ( +
    + {agent.readiness.reasons.map((reason) => ( +
  • {readinessReasonLabel(reason)}
  • + ))} +
+ ) : ( +

No readiness issues reported.

+ )} + +

+ Voice stays inactive until Loopable explicitly activates it. +

+
+ +
+

Knowledge

+ {agent.knowledgeProfile ? ( +
+
+
Approval
+
{statusLabel(agent.knowledgeProfile.approvalState)}
+
+
+
Version
+
{agent.knowledgeProfile.version || "Not set"}
+
+
+
Classification
+
{agent.knowledgeProfile.classification || "Not set"}
+
+
+ ) : ( +

No knowledge profile is assigned.

+ )} +
+
+ +
+
+

Configuration

+ Draft settings +
+
+