diff --git a/RELEASES.json b/RELEASES.json index 330352123c0cf585139a4ffcdfc397ca34b5ee92..574e5f48f0f7ed5b4a01acb628b3d02a6a29871a 100644 --- a/RELEASES.json +++ b/RELEASES.json @@ -1,6 +1,12 @@ { - "current": "2970b72", + "current": "ba24d8b", "releases": [ + { + "version": "v56", + "sha": "2970b72", + "date": "2026-08-28", + "subject": "release v56" + }, { "version": "v55", "sha": "b766b90", diff --git a/VERSION b/VERSION index c3a817e183809e5543a053e50c602c4d3c71faab..25dca9b87dc61496f55b23fe9d9f52041aa4a0cd 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2970b72 +ba24d8b diff --git a/api/application_registry.py b/api/application_registry.py new file mode 100644 index 0000000000000000000000000000000000000000..ee2cc6dc423a87f734bdc64d7e6413e32cad54a7 --- /dev/null +++ b/api/application_registry.py @@ -0,0 +1,290 @@ +"""Tenant-level reusable Application assets and deterministic default resolution.""" +from __future__ import annotations + +import copy +import datetime as dt +import uuid + + +APPLICATION_TYPES = ("chart", "route", "map", "timeseries") +_KEY = "applications_v1" +_MAX = 500 + + +class ApplicationError(ValueError): + pass + + +def blank_doc(): + return {"schema": 1, "applications": {}, "defaults": {}} + + +def _clean_doc(raw): + doc = blank_doc() + if isinstance(raw, dict): + if isinstance(raw.get("applications"), dict): + doc["applications"] = copy.deepcopy(raw["applications"]) + if isinstance(raw.get("defaults"), dict): + doc["defaults"] = copy.deepcopy(raw["defaults"]) + return doc + + +def load(runtime): + try: return _clean_doc(runtime.get(_KEY)) + except Exception: return blank_doc() + + +def _users(raw): + out=[] + for item in raw if isinstance(raw, (list, tuple)) else (): + got=str(item or "").strip()[:120] + if got and got not in out: out.append(got) + return out + + +def may_read(app, user, is_admin=False): + if not isinstance(app, dict): return False + if is_admin or app.get("createdBy") == user: return True + share=app.get("share") if isinstance(app.get("share"), dict) else {} + return share.get("mode") == "tenant" or user in set(share.get("users") or ()) + + +def may_edit(app, user, is_admin=False): + return bool(is_admin or (isinstance(app, dict) and app.get("createdBy") == user)) + + +def _compat(raw): + values=[] + for item in raw if isinstance(raw, (list, tuple)) else (): + key=str(item or "").strip()[:120] + if key and key not in values: values.append(key) + if not values: raise ApplicationError("at least one compatible database is required") + return values + + +def _governed_bindings(raw, compatible, semantic_records): + """Normalize role -> database -> approved field, refusing optimistic bindings.""" + if raw in (None, {}): + return {} + if not isinstance(raw, dict): + raise ApplicationError("semanticRoleBindings must be an object") + compatible = tuple(compatible or ()) + out = {} + for raw_role, raw_binding in raw.items(): + role = str(raw_role or "").strip()[:80] + if not role or not isinstance(raw_binding, dict): + raise ApplicationError("every semantic role needs a binding object") + # The first Wave shape was one flat binding. Admit it only for one compatible database, + # then store the canonical per-database shape. + if "field" in raw_binding: + database = str(raw_binding.get("database") or "").strip() + if not database and len(compatible) == 1: + database = compatible[0] + bindings = {database: raw_binding} if database else {} + else: + bindings = raw_binding + if set(bindings) != set(compatible): + raise ApplicationError( + f"semantic role {role!r} must be confirmed for every compatible database") + per_database = {} + for database in compatible: + binding = bindings.get(database) + if not isinstance(binding, dict) or binding.get("confirmed") is not True: + raise ApplicationError(f"semantic role {role!r} is not confirmed for {database}") + field = str(binding.get("field") or "").strip()[:120] + records = semantic_records.get(database) if isinstance(semantic_records, dict) else None + record = (records or {}).get(f"{database}::{field}") if field else None + if not isinstance(record, dict) or record.get("state") != "approved": + raise ApplicationError( + f"semantic role {role!r} needs an approved field in {database}") + per_database[database] = {"field": field, "confirmed": True} + out[role] = per_database + return out + + +_CONFIG_MODE = {"chart": "chart", "route": "map", "map": "map", "timeseries": "timeseries"} + + +def _typed_config(kind, raw): + if not isinstance(raw, dict): + raise ApplicationError("config must be an object") + if set(raw) - {"schemaVersion", "display"}: + raise ApplicationError("config contains fields outside the typed Application contract") + if raw.get("schemaVersion") != 1: + raise ApplicationError("config schemaVersion must be 1") + display = raw.get("display") + if not isinstance(display, dict): + raise ApplicationError("config.display must be an object") + if str(display.get("mode") or "") != _CONFIG_MODE[kind]: + raise ApplicationError(f"{kind} config has the wrong display mode") + return {"schemaVersion": 1, "display": copy.deepcopy(display)} + + +def _allowed(compatible, allowed_databases): + return (set(compatible or ()) if allowed_databases is None + else set(compatible or ()) & set(allowed_databases)) + + +def _public(app, user, is_admin=False, *, allowed_databases=None, target_database=None, + is_default=False): + row=copy.deepcopy(app) + compatible = [database for database in row.get("compatibleDatabases") or () + if database in _allowed(row.get("compatibleDatabases"), allowed_databases)] + row["compatibleDatabases"] = compatible + row["capabilities"]={"read":may_read(app,user,is_admin),"edit":may_edit(app,user,is_admin), + "share":may_edit(app,user,is_admin),"delete":may_edit(app,user,is_admin), + "duplicate":may_read(app,user,is_admin),"star":may_read(app,user,is_admin), + "default":bool(is_admin)} + row["starred"] = user in (app.get("starredBy") or ()) + row["isDefault"] = bool(is_default) + target_database = target_database if target_database in compatible else (compatible[0] if compatible else None) + row["openTarget"]={"kind":"application","id":row["id"],"type":row["type"], + "database":target_database} + return row + + +def list_apps(runtime, user, is_admin=False, *, database=None, kind=None, starred=None, + allowed_databases=None): + doc = load(runtime) + rows=[] + for app in doc["applications"].values(): + if not may_read(app,user,is_admin): continue + if database and database not in (app.get("compatibleDatabases") or ()): continue + if not _allowed(app.get("compatibleDatabases"), allowed_databases): continue + if kind and app.get("type") != kind: continue + if starred is not None and bool(user in (app.get("starredBy") or ())) != bool(starred): continue + default_key = f"{database}::{app.get('type')}" if database else "" + rows.append(_public(app,user,is_admin,allowed_databases=allowed_databases, + target_database=database, + is_default=bool(default_key and doc["defaults"].get(default_key)==app.get("id")))) + return sorted(rows,key=lambda x:(str(x.get("updatedAt") or ""),str(x.get("id"))),reverse=True) + + +def create(runtime, tenant, user, payload, *, semantic_records=None, allowed_databases=None): + payload=payload if isinstance(payload,dict) else {} + kind=str(payload.get("type") or "").lower() + if kind not in APPLICATION_TYPES: raise ApplicationError("application type is not recognized") + name=" ".join(str(payload.get("name") or "").split())[:120] + if not name: raise ApplicationError("application name is required") + app_id=f"app_{uuid.uuid4().hex[:16]}"; now=dt.datetime.now(dt.timezone.utc).isoformat() + compatible = _compat(payload.get("compatibleDatabases")) + if _allowed(compatible, allowed_databases) != set(compatible): + raise PermissionError("an Application cannot bind a database you cannot open") + app={"id":app_id,"tenant":str(tenant),"type":kind,"name":name, + "compatibleDatabases":compatible, + "semanticRoleBindings":_governed_bindings( + payload.get("semanticRoleBindings"), compatible, semantic_records or {}), + "config":_typed_config(kind, payload.get("config")), + "share":{"mode":"private","users":[]},"starredBy":[],"createdBy":user, + "createdAt":now,"updatedAt":now,"revision":1} + def apply(raw): + doc=_clean_doc(raw) + if len(doc["applications"])>=_MAX: raise ApplicationError("application limit reached") + doc["applications"][app_id]=app; return doc + runtime.update(_KEY,apply) + return _public(app,user,allowed_databases=allowed_databases) + + +def mutate(runtime, app_id, user, is_admin, patch, *, semantic_records=None, + allowed_databases=None, target_database=None): + patch=patch if isinstance(patch,dict) else {}; landed={} + def apply(raw): + doc=_clean_doc(raw); app=doc["applications"].get(app_id) + if not app: raise ApplicationError("application does not exist") + if not _allowed(app.get("compatibleDatabases"), allowed_databases): + raise ApplicationError("application does not exist") + star_only = set(patch) <= {"starred"} + if star_only: + if not may_read(app,user,is_admin): raise PermissionError("application is not available") + elif not may_edit(app,user,is_admin): + raise PermissionError("application belongs to another user") + if "name" in patch: + name=" ".join(str(patch["name"] or "").split())[:120] + if not name: raise ApplicationError("application name is required") + app["name"]=name + if "config" in patch: + app["config"]=_typed_config(app["type"],patch["config"]) + compatible = (_compat(patch["compatibleDatabases"]) + if "compatibleDatabases" in patch else app.get("compatibleDatabases") or []) + if _allowed(compatible, allowed_databases) != set(compatible): + raise PermissionError("an Application cannot bind a database you cannot open") + if "semanticRoleBindings" in patch or "compatibleDatabases" in patch: + bindings = patch.get("semanticRoleBindings", app.get("semanticRoleBindings")) + app["semanticRoleBindings"]=_governed_bindings( + bindings, compatible, semantic_records or {}) + app["compatibleDatabases"] = list(compatible) + if "share" in patch: + share=patch["share"] if isinstance(patch["share"],dict) else {} + mode=str(share.get("mode") or "private") + if mode not in ("private","users","tenant"): raise ApplicationError("share mode is not recognized") + app["share"]={"mode":mode,"users":_users(share.get("users")) if mode=="users" else []} + if "starred" in patch: + stars=set(app.get("starredBy") or ()) + (stars.add if patch["starred"] else stars.discard)(user); app["starredBy"]=sorted(stars) + app["revision"]=int(app.get("revision") or 0)+1 + app["updatedAt"]=dt.datetime.now(dt.timezone.utc).isoformat(); landed.update(app) + return doc + runtime.update(_KEY,apply) + return _public(landed,user,is_admin,allowed_databases=allowed_databases, + target_database=target_database) + + +def duplicate(runtime, app_id, tenant, user, is_admin=False, *, semantic_records=None, + allowed_databases=None): + app=load(runtime)["applications"].get(app_id) + if not may_read(app,user,is_admin): raise ApplicationError("application does not exist") + if not _allowed(app.get("compatibleDatabases"), allowed_databases): + raise ApplicationError("application does not exist") + return create(runtime,tenant,user,{"type":app["type"],"name":f"{app['name']} copy", + "compatibleDatabases":app.get("compatibleDatabases"), + "semanticRoleBindings":app.get("semanticRoleBindings"),"config":app.get("config")}, + semantic_records=semantic_records, allowed_databases=allowed_databases) + + +def delete(runtime, app_id, user, is_admin=False, *, allowed_databases=None): + def apply(raw): + doc=_clean_doc(raw); app=doc["applications"].get(app_id) + if not app: raise ApplicationError("application does not exist") + if not _allowed(app.get("compatibleDatabases"), allowed_databases): + raise ApplicationError("application does not exist") + if not may_edit(app,user,is_admin): raise PermissionError("application belongs to another user") + doc["applications"].pop(app_id) + for key in list(doc["defaults"]): + if doc["defaults"][key]==app_id: doc["defaults"].pop(key) + return doc + runtime.update(_KEY,apply); return {"deleted":app_id} + + +def set_default(runtime, database, kind, app_id, user, is_admin=False, *, allowed_databases=None): + if kind not in APPLICATION_TYPES: raise ApplicationError("application type is not recognized") + if not is_admin: raise PermissionError("only a tenant admin can change Application defaults") + if database not in set(allowed_databases or ()): + raise PermissionError("that database is not available") + key=f"{database}::{kind}" + def apply(raw): + doc=_clean_doc(raw); app=doc["applications"].get(app_id) + if (not isinstance(app, dict) or not may_edit(app,user,is_admin) + or database not in (app.get("compatibleDatabases") or ()) + or app.get("type")!=kind): + raise ApplicationError("application is not compatible with this default") + doc["defaults"][key]=app_id; return doc + runtime.update(_KEY,apply); return resolve_default( + runtime,database,kind,user,is_admin,allowed_databases=allowed_databases) + + +def resolve_default(runtime, database, kind, user, is_admin=False, *, allowed_databases=None): + if kind not in APPLICATION_TYPES: + raise ApplicationError("application type is not recognized") + if database not in set(allowed_databases or ()): + raise PermissionError("that database is not available") + doc=load(runtime); explicit=doc["defaults"].get(f"{database}::{kind}") + app=doc["applications"].get(explicit) + if app and may_read(app,user,is_admin) and database in (app.get("compatibleDatabases") or ()) and app.get("type")==kind: + return {"resolution":"explicit","application":_public( + app,user,is_admin,allowed_databases=allowed_databases,target_database=database, + is_default=True)} + matches=[x for x in doc["applications"].values() if x.get("type")==kind and database in (x.get("compatibleDatabases") or ()) and may_read(x,user,is_admin)] + matches.sort(key=lambda x:(str(x.get("updatedAt") or ""),str(x.get("id"))),reverse=True) + return {"resolution":"latest" if matches else "blank", + "application":_public(matches[0],user,is_admin,allowed_databases=allowed_databases, + target_database=database) if matches else None} diff --git a/api/automation_engine.py b/api/automation_engine.py index f54777c885b7fbc8a3bc0e2fefc3386ac86f48a9..f9fe4ef5b6878238c46ef692601d919fa165b67e 100644 --- a/api/automation_engine.py +++ b/api/automation_engine.py @@ -58,6 +58,9 @@ except Exception: # pragma: no cover — deploy lag; see m #: The per-tenant store key. Colocated with its reader, the `routes_nav._NAV_PREFS_KEY` pattern. STORE_KEY = "automations" +AGENT_CONTRACT_KEY = "agent_contracts" +AGENT_EVENT_TYPES = ("transcript", "message", "gathered_data", "run", "consent") +AGENT_ACTIONS = ("read", "create", "update", "overwrite", "delete") #: The user-table bucket. ⚠ Read/written HERE through `TenantRuntime`, never through #: `core.user_tables` — that module writes the UNPREFIXED module-global key, which is correct for #: tenant #0 (empty prefix) and silently cross-tenant for every R2 tenant after it. Booked in the @@ -2681,6 +2684,329 @@ def all_definitions(rt, cached=False): return out +def _agent_contracts(rt): + """Private Agent metadata and events, stored outside normal database navigation.""" + try: + raw = rt.get(AGENT_CONTRACT_KEY) or {} + except Exception: + return {} + return copy.deepcopy(raw) if isinstance(raw, dict) else {} + + +def _agent_event_schema(): + return { + "transcript": {"required": ["text"], "optional": ["speaker", "channel"]}, + "message": {"required": ["text"], "optional": ["role", "channel"]}, + "gathered_data": {"required": ["data"], "optional": ["source", "citations"]}, + "run": {"required": ["state"], "optional": ["workflowId", "evidence"]}, + "consent": {"required": ["state"], "optional": ["subjectRef", "purpose"]}, + } + + +def _new_agent_contract(agent_id, actor="", name=""): + aid = str(agent_id or "").strip() + stamp = int(time.time()) + workflow_id = "wf_main" + return { + "schemaVersion": 1, + "agentId": aid, + "eventStore": { + "id": "aes_" + hashlib.sha256(aid.encode("utf-8")).hexdigest()[:18], + "kind": "agent_event_store", + "visibility": "private", + "hiddenFromData": True, + "eventSchemas": _agent_event_schema(), + }, + "events": [], + "workflows": [{"id": workflow_id, "name": "Main", "kind": "event_store_view", + "eventStoreId": "aes_" + hashlib.sha256(aid.encode("utf-8")).hexdigest()[:18], + "purpose": "", "createdAt": stamp, "updatedAt": stamp}], + "sharing": {"visibility": "private", "sharedWith": []}, + "capabilities": {"defaultMode": "preview", "workflows": {workflow_id: {}}}, + "semantic": {"fields": [], "retrievable": []}, + "setup": {"state": "unconfigured", "answers": {}, "disabledFields": []}, + "createdBy": str(actor or "")[:120], "createdAt": stamp, + "displayName": str(name or "")[:120], + } + + +def ensure_agent_contract(rt, agent_id, actor="", name=""): + """Mint exactly one private event store and default Workflow for a stored Agent.""" + aid = str(agent_id or "").strip() + if not aid or aid.startswith("field:") or aid == "system:odoo_sync": + return None + found = _agent_contracts(rt).get(aid) + if isinstance(found, dict): + return found + fresh = _new_agent_contract(aid, actor, name) + result = {} + def up(raw): + doc = dict(raw or {}) if isinstance(raw, dict) else {} + rec = doc.get(aid) + if not isinstance(rec, dict): + rec = copy.deepcopy(fresh) + doc[aid] = rec + result.update(copy.deepcopy(rec)) + return doc + rt.update(AGENT_CONTRACT_KEY, up, flush="sync") + return result + + +def _agent_update(rt, agent_id, fn): + aid = str(agent_id or "").strip() + result = {} + def up(raw): + doc = dict(raw or {}) if isinstance(raw, dict) else {} + rec = copy.deepcopy(doc.get(aid)) + if not isinstance(rec, dict): + raise ValueError("unknown stored Agent") + rec = fn(rec) + doc[aid] = rec + result.update(copy.deepcopy(rec)) + return doc + rt.update(AGENT_CONTRACT_KEY, up, flush="sync") + return result + + +def agent_contract_dto(rt, agent_id): + rec = _agent_contracts(rt).get(str(agent_id or "")) + if not isinstance(rec, dict): + return None + store = copy.deepcopy(rec.get("eventStore") or {}) + workflows = copy.deepcopy(rec.get("workflows") or []) + schemas = copy.deepcopy(store.get("eventSchemas") or {}) + toc = {"eventSchemas": [{"type": key, "required": value.get("required", []), + "optional": value.get("optional", [])} + for key, value in sorted(schemas.items())], + "workflows": [{"id": row.get("id"), "name": row.get("name"), + "eventStoreId": row.get("eventStoreId")} for row in workflows], + "setupState": str((rec.get("setup") or {}).get("state") or "unconfigured")} + return { + "schemaVersion": 1, "agentId": str(rec.get("agentId") or ""), + "eventStore": store, + "workflows": workflows, + "sharing": copy.deepcopy(rec.get("sharing") or {"visibility": "private", "sharedWith": []}), + "capabilities": copy.deepcopy(rec.get("capabilities") or {}), + "semanticState": copy.deepcopy(rec.get("semantic") or {}), + "setupInterview": setup_interview(rec), + "harnessTableOfContents": toc, + "eventCount": len(rec.get("events") or []), + } + + +def authorize_agent_principal(rt, agent_id, actor, is_admin=False): + rec = _agent_contracts(rt).get(str(agent_id or "")) or {} + sharing = rec.get("sharing") or {} + if is_admin or rec.get("createdBy") == str(actor or "") or str(actor or "") in set(sharing.get("sharedWith") or []): + return True + raise PermissionError("this Agent is private or not shared with this principal") + + +def append_agent_event(rt, agent_id, event_type, payload, actor="agent"): + kind = str(event_type or "").strip().lower() + schemas = _agent_event_schema() + if kind not in schemas: + raise ValueError("unsupported Agent event type") + if not isinstance(payload, dict): + raise ValueError("Agent event payload must be an object") + missing = [key for key in schemas[kind]["required"] if key not in payload] + if missing: + raise ValueError("Agent event is missing: " + ", ".join(missing)) + allowed = set(schemas[kind]["required"] + schemas[kind]["optional"]) + unknown = sorted(set(payload) - allowed) + if unknown: + raise ValueError("unsupported Agent event fields: " + ", ".join(unknown)) + result = {} + def mutate(rec): + events = list(rec.get("events") or []) + event = {"id": f"evt_{len(events) + 1}", "type": kind, + "payload": copy.deepcopy(payload), "actor": str(actor or "agent")[:120], + "createdAt": int(time.time())} + events.append(event) + rec["events"] = events[-1000:] + result.update(copy.deepcopy(event)) + return rec + _agent_update(rt, agent_id, mutate) + return result + + +def create_agent_workflow(rt, agent_id, name, purpose=""): + label = " ".join(str(name or "").split())[:80] + if not label: + raise ValueError("name the Workflow") + result = {} + def mutate(rec): + rows = list(rec.get("workflows") or []) + wid = f"wf_{len(rows) + 1}" + while any(row.get("id") == wid for row in rows): + wid = f"wf_{len(rows) + int(time.time())}" + stamp = int(time.time()) + row = {"id": wid, "name": label, "kind": "event_store_view", + "eventStoreId": (rec.get("eventStore") or {}).get("id"), + "purpose": " ".join(str(purpose or "").split())[:240], + "createdAt": stamp, "updatedAt": stamp} + rows.append(row) + rec["workflows"] = rows + caps = dict(rec.get("capabilities") or {}) + caps["workflows"] = {**dict(caps.get("workflows") or {}), wid: {}} + rec["capabilities"] = caps + result.update(copy.deepcopy(row)) + return rec + _agent_update(rt, agent_id, mutate) + return result + + +def configure_agent_governance(rt, agent_id, raw, allowed_databases=None, allowed_users=None): + """Store private sharing and named-field grants after the caller's wall has narrowed them.""" + body = raw if isinstance(raw, dict) else {} + visibility = str(body.get("visibility") or "private") + if visibility not in {"private", "shared"}: + raise ValueError("Agent visibility is private or shared") + users = sorted({str(v).strip()[:120] for v in body.get("sharedWith", []) if str(v).strip()}) + if allowed_users is not None and not set(users).issubset({str(v) for v in allowed_users}): + raise PermissionError("Agent share recipient is outside the tenant user wall") + field_wall = ({str(key): {str(value) for value in values} + for key, values in allowed_databases.items()} + if isinstance(allowed_databases, dict) else {}) + allowed = set(field_wall) if field_wall else set(str(v) for v in (allowed_databases or [])) + workflows = body.get("workflows") if isinstance(body.get("workflows"), dict) else {} + result = {} + def mutate(rec): + known = {row.get("id") for row in rec.get("workflows") or []} + clean_workflows = {} + for wid, databases in workflows.items(): + if wid not in known or not isinstance(databases, dict): + raise ValueError("unknown Workflow") + clean_dbs = {} + for table, grant in databases.items(): + if allowed and table not in allowed: + raise PermissionError("database permission denied before Agent execution") + if not isinstance(grant, dict): + raise ValueError("database capability must be an object") + mode = str(grant.get("mode") or "preview") + if mode not in {"preview", "autonomous"}: + raise ValueError("Workflow mode is preview or autonomous") + clean = {"mode": mode, "read": bool(grant.get("read"))} + for action in AGENT_ACTIONS[1:]: + value = grant.get(action) or [] + if not isinstance(value, list): + raise ValueError(f"{action} capability needs named fields") + clean[action] = sorted({str(v).strip()[:120] for v in value if str(v).strip()}) + if field_wall and not set(clean[action]).issubset(field_wall.get(str(table), set())): + raise PermissionError("field permission denied before Agent execution") + clean_dbs[str(table)] = clean + clean_workflows[str(wid)] = clean_dbs + rec["sharing"] = {"visibility": visibility, "sharedWith": users if visibility == "shared" else []} + rec["capabilities"] = {"defaultMode": "preview", "workflows": clean_workflows} + result.update(copy.deepcopy(rec)) + return rec + _agent_update(rt, agent_id, mutate) + return result + + +def authorize_agent_action(rt, agent_id, workflow_id, table, action, field="", + actor_databases=None, actor_fields=None): + """Capability intersection. This must run before a model or provider is invoked.""" + rec = _agent_contracts(rt).get(str(agent_id or "")) or {} + actor_dbs = set(str(v) for v in (actor_databases or [])) + if str(table) not in actor_dbs: + raise PermissionError("database permission denied before Agent execution") + grant = ((((rec.get("capabilities") or {}).get("workflows") or {}) + .get(str(workflow_id)) or {}).get(str(table)) or {}) + act = str(action or "") + if act == "read": + if not grant.get("read"): + raise PermissionError("Agent read capability denied before execution") + if actor_fields is not None and str(field) not in set(str(v) for v in actor_fields): + raise PermissionError("field permission denied before Agent execution") + elif act in AGENT_ACTIONS[1:]: + if str(field) not in set(grant.get(act) or []): + raise PermissionError(f"Agent {act} capability denied before execution") + if actor_fields is not None and str(field) not in set(str(v) for v in actor_fields): + raise PermissionError("field permission denied before Agent execution") + if grant.get("mode") == "preview" and act in {"create", "update"}: + return {"allowed": True, "mode": "preview", "action": act} + else: + raise PermissionError("unknown Agent capability") + return {"allowed": True, "mode": str(grant.get("mode") or "preview"), "action": act} + + +def governed_agent_query_dry_run(rt, agent_id, workflow_id, table, field, prompt, query_tool, + actor_databases=None, actor_fields=None): + """Authorization and semantic readiness run before the injected C2 Query callable.""" + authorize_agent_action(rt, agent_id, workflow_id, table, "read", field, + actor_databases=actor_databases, actor_fields=actor_fields) + ready = (agent_contract_dto(rt, agent_id) or {}).get("semanticState", {}).get("retrievable", []) + if {"database": str(table), "field": str(field)} not in ready: + raise PermissionError("semantic field is not approved before Query execution") + if not callable(query_tool): + raise ValueError("governed Query is unavailable") + result = query_tool(str(prompt or ""), {"agentId": str(agent_id), "workflowId": str(workflow_id), + "database": str(table), "field": str(field)}) + if not isinstance(result, dict): + raise ValueError("governed Query returned no typed result") + append_agent_event(rt, agent_id, "message", {"text": str(prompt or "")[:2000], "role": "user", + "channel": "voice"}) + append_agent_event(rt, agent_id, "run", {"state": "dry_run", "workflowId": str(workflow_id), + "evidence": "governed_query"}) + return {"query": copy.deepcopy(result), "noProviderCalls": True, "providerCalls": 0} + + +def set_semantic_readiness(rt, agent_id, fields): + rows = fields if isinstance(fields, list) else [] + clean = [{"database": str(row.get("database") or ""), "field": str(row.get("field") or ""), + "state": str(row.get("state") or "draft")} + for row in rows if isinstance(row, dict) and row.get("database") and row.get("field") + and str(row.get("state") or "draft") in {"draft", "approved", "disabled"}] + def mutate(rec): + rec["semantic"] = {"fields": clean, + "retrievable": [{"database": row["database"], "field": row["field"]} + for row in clean if row["state"] == "approved"]} + return rec + return _agent_update(rt, agent_id, mutate) + + +def setup_interview(rec): + setup = rec.get("setup") if isinstance(rec, dict) else {} + setup = setup if isinstance(setup, dict) else {} + answers = setup.get("answers") if isinstance(setup.get("answers"), dict) else {} + questions = ( + ("purpose", "What should this Agent help with?", "Summarize the work this Agent should handle."), + ("source", "Which approved database should it use?", "Choose one approved source."), + ("workflow", "What should the first Workflow do?", "Describe one repeatable outcome."), + ) + pending = next((row for row in questions if row[0] not in answers), None) + if not pending: + return {"state": "configured", "question": None, "progress": len(questions), + "total": len(questions)} + return {"state": "unconfigured", "question": {"key": pending[0], "text": pending[1], + "suggestion": pending[2], "allowDisable": pending[0] == "source"}, + "progress": len(answers), "total": len(questions)} + + +def answer_setup_interview(rt, agent_id, key, answer="", disable=False): + result = {} + def mutate(rec): + current = setup_interview(rec).get("question") + if not current or current.get("key") != str(key or ""): + raise ValueError("answer the current setup question first") + if disable and not current.get("allowDisable"): + raise ValueError("this setup question cannot be disabled") + text = " ".join(str(answer or "").split())[:240] + if not text and not disable: + raise ValueError("answer the setup question or disable this source") + setup = dict(rec.get("setup") or {}) + answers = dict(setup.get("answers") or {}) + answers[str(key)] = {"answer": text, "disabled": bool(disable)} + setup["answers"] = answers + setup["state"] = "configured" if setup_interview({"setup": setup}).get("state") == "configured" else "unconfigured" + rec["setup"] = setup + result.update(copy.deepcopy(rec)) + return rec + _agent_update(rt, agent_id, mutate) + return setup_interview(result) + + def tick_definitions(rt): """Definitions for the external scheduler, cached until `_store_update` changes them. @@ -11111,7 +11437,7 @@ def apply_actions(rt, defn, table_key, row_ids, username="automation", log=print if "odoo_sync_not_run_here" not in web_notes: web_notes.append("odoo_sync_not_run_here") log("[aios-auto] odoo_sync: the Odoo pull runs on the connector's own " - "schedule, not from this canvas. Nothing was done") + "schedule, not from this Workflow. Nothing was done") counts["webBlocked"] += 1 continue elif kind == "ai_enrich": @@ -11128,7 +11454,7 @@ def apply_actions(rt, defn, table_key, row_ids, username="automation", log=print if "ai_enrich_not_run_here" not in web_notes: web_notes.append("ai_enrich_not_run_here") log("[aios-auto] ai_enrich: an AI column is filled from the column's own " - "editor, not from this canvas. Nothing was done") + "editor, not from this Workflow. Nothing was done") counts["webBlocked"] += 1 continue elif kind == "voice_agent": diff --git a/api/formula_runtime.py b/api/formula_runtime.py new file mode 100644 index 0000000000000000000000000000000000000000..cfb38e2c2c3235816771f3858c24b394301541c9 --- /dev/null +++ b/api/formula_runtime.py @@ -0,0 +1,750 @@ +"""Versioned, allowlisted server formula AST and evaluator. + +The browser may preview the same fixtures, but values shipped by the product are produced here. +No Python ``eval`` is used; parsed nodes are interpreted by the closed grammar below. +""" +from __future__ import annotations + +import ast +import calendar +import datetime as dt +import hashlib +import json +import math +import re +import warnings +from decimal import Decimal, ROUND_FLOOR, ROUND_HALF_UP + + +CONTRACT_VERSION = 1 +AST_VERSION = 1 +MAX_SOURCE = 500 +MAX_NODES = 200 +MAX_DEPTH = 24 +MAX_FIELD_KEY = 120 +MAX_DEFINITIONS = 100 +MAX_BATCH_ROWS = 1000 +MANY = 30 +AUTHORITY = "server" +CONSUMERS = ("preview", "grid", "filter", "export", "automation", "query", "timeseries") +FN_ARITY = { + "ABS": (1, 1), "ROUND": (1, 2), "ROUNDUP": (1, 2), "ROUNDDOWN": (1, 2), + "INT": (1, 1), "MOD": (2, 2), "SQRT": (1, 1), "POWER": (2, 2), "EXP": (1, 1), + "LN": (1, 1), "SUM": (1, MANY), "AVERAGE": (1, MANY), "COUNT": (1, MANY), + "COUNTA": (1, MANY), "MIN": (1, MANY), "MAX": (1, MANY), "IF": (2, 3), + "AND": (1, MANY), "OR": (1, MANY), "NOT": (1, 1), "IFERROR": (2, 2), + "ISBLANK": (1, 1), "SWITCH": (3, MANY), "CONCATENATE": (1, MANY), + "LEFT": (1, 2), "RIGHT": (1, 2), "MID": (3, 3), "LEN": (1, 1), + "TRIM": (1, 1), "UPPER": (1, 1), "LOWER": (1, 1), "PROPER": (1, 1), + "VALUE": (1, 1), "TEXT": (2, 2), "SUBSTITUTE": (3, 4), "FIND": (2, 3), + "REGEX_EXTRACT": (2, 2), "REGEX_REPLACE": (3, 3), "TODAY": (0, 0), + "YEAR": (1, 1), "MONTH": (1, 1), "DAY": (1, 1), "DAYS": (2, 2), + "DATEADD": (3, 3), "DATETIME_DIFF": (2, 3), "DATETIME_FORMAT": (2, 2), +} +ALIASES = {"CONCAT": "CONCATENATE"} + +# Every accepted source token has a fixture. The AST stores the canonical token (``=`` and +# ``!=``), while the grammar publishes the spelling aliases so a browser never has to infer them. +OPERATOR_PARITY_FIXTURES = { + "neg": ("-5", {}, -5.0), + "add": ("{a} + {b}", {"a": 2, "b": 3}, 5.0), + "subtract": ("{a} - {b}", {"a": 7, "b": 2}, 5.0), + "multiply": ("{a} * {b}", {"a": 3, "b": 4}, 12.0), + "divide": ("{a} / {b}", {"a": 9, "b": 3}, 3.0), + "power": ("{a} ^ {b}", {"a": 2, "b": 3}, 8.0), + "concatenate": ("{a} & {b}", {"a": "A", "b": "B"}, "AB"), + "less_than": ("{a} < {b}", {"a": 2, "b": 3}, "TRUE"), + "less_or_equal": ("{a} <= {b}", {"a": 3, "b": 3}, "TRUE"), + "greater_than": ("{a} > {b}", {"a": 4, "b": 3}, "TRUE"), + "greater_or_equal": ("{a} >= {b}", {"a": 3, "b": 3}, "TRUE"), + "equal": ("{a} = {b}", {"a": "A", "b": "a"}, "TRUE"), + "equal_alias": ("{a} == {b}", {"a": 3, "b": 3}, "TRUE"), + "not_equal": ("{a} != {b}", {"a": 2, "b": 3}, "TRUE"), + "not_equal_alias": ("{a} <> {b}", {"a": 2, "b": 3}, "TRUE"), +} + +PARSE_ERROR_FIXTURES = { + "empty": "", + "legacy_square_reference": "[amount] + 1", + "unknown_function": "UNKNOWN(1)", + "wrong_arity": "ABS(1, 2)", + "unclosed_reference": "{amount + 1", + "invalid_reference": "{amount-due} + 1", + "unterminated_text": '"amount', + "chained_comparison": "1 < 2 < 3", + "unsupported_operator": "5 % 2", +} + +EVALUATION_ERROR_FIXTURES = { + "unknown_ref": ("{missing} + 1", {}, "unknown_ref"), + "div_zero": ("{amount} / 0", {"amount": 4}, "div_zero"), +} + +# Shared contract fixture corpus. The API publishes this exact data so the browser preview gate +# can evaluate the same source/row/today triples without maintaining a second handwritten grammar. +PARITY_FIXTURES = { + "ABS": ("ABS(-2)", 2.0), "ROUND": ("ROUND(2.345, 2)", 2.35), + "ROUNDUP": ("ROUNDUP(2.341, 2)", 2.35), "ROUNDDOWN": ("ROUNDDOWN(2.349, 2)", 2.34), + "INT": ("INT(2.9)", 2), "MOD": ("MOD(7, 4)", 3.0), "SQRT": ("SQRT(9)", 3.0), + "POWER": ("POWER(2, 3)", 8.0), "EXP": ("EXP(0)", 1.0), "LN": ("LN(1)", 0.0), + "SUM": ("SUM(1, 2, 3)", 6.0), "AVERAGE": ("AVERAGE(2, 4)", 3.0), + "COUNT": ("COUNT(1, \"x\", 3)", 2), "COUNTA": ("COUNTA(1, \"x\", \"\")", 2), + "MIN": ("MIN(4, 2)", 2.0), "MAX": ("MAX(4, 2)", 4.0), "IF": ("IF(TRUE, 1, 2)", 1), + "AND": ("AND(TRUE, TRUE)", "TRUE"), "OR": ("OR(FALSE, TRUE)", "TRUE"), + "NOT": ("NOT(FALSE)", "TRUE"), "IFERROR": ("IFERROR(1 / 0, 7)", 7), + "ISBLANK": ("ISBLANK({empty})", "TRUE"), "SWITCH": ("SWITCH(\"A\", \"A\", 1, 0)", 1), + "CONCATENATE": ("CONCATENATE(\"A\", \"B\")", "AB"), + "LEFT": ("LEFT(\"ABC\", 2)", "AB"), "RIGHT": ("RIGHT(\"ABC\", 2)", "BC"), + "MID": ("MID(\"ABCD\", 2, 2)", "BC"), "LEN": ("LEN(\"ABC\")", 3), + "TRIM": ("TRIM(\" A B \" )", "A B"), "UPPER": ("UPPER(\"Ab\")", "AB"), + "LOWER": ("LOWER(\"Ab\")", "ab"), "PROPER": ("PROPER(\"hello world\")", "Hello World"), + "VALUE": ("VALUE(\"12.5\")", 12.5), "TEXT": ("TEXT(0.25, \"0%\")", "25%"), + "SUBSTITUTE": ("SUBSTITUTE(\"a-b\", \"-\", \"+\")", "a+b"), + "FIND": ("FIND(\"b\", \"abc\")", 2), + "REGEX_EXTRACT": ("REGEX_EXTRACT(\"SO-42\", \"SO-[0-9]+\")", "SO-42"), + "REGEX_REPLACE": ("REGEX_REPLACE(\"SO-42\", \"[0-9]+\", \"#\")", "SO-#"), + "TODAY": ("TODAY()", "2024-01-02"), "YEAR": ("YEAR(\"2024-01-02\")", 2024), + "MONTH": ("MONTH(\"2024-01-02\")", 1), "DAY": ("DAY(\"2024-01-02\")", 2), + "DAYS": ("DAYS(\"2024-01-05\", \"2024-01-02\")", 3), + "DATEADD": ("DATEADD(\"2024-01-31\", 1, \"months\")", "2024-02-29"), + "DATETIME_DIFF": ("DATETIME_DIFF(\"2024-01-16\", \"2024-01-02\", \"weeks\")", 2), + "DATETIME_FORMAT": ("DATETIME_FORMAT(\"2024-01-02\", \"YYYY-MM-DD\")", "2024-01-02"), +} + + +class FormulaError(ValueError): + pass + + +def _rewrite(source): + """Translate Excel punctuation to Python AST punctuation without touching strings.""" + out, i, quote = [], 0, None + while i < len(source): + c = source[i] + if quote: + out.append(c) + if c == quote: + if i + 1 < len(source) and source[i + 1] == quote: + out.append(source[i + 1]); i += 2; continue + quote = None + i += 1; continue + if c in ("'", '"'): + quote = c; out.append(c); i += 1; continue + if c == "{": + end = source.find("}", i + 1) + if end < 0: + raise FormulaError("a field reference is missing its closing }") + key = source[i + 1:end].strip() + if not re.fullmatch(r"[A-Za-z0-9_]+", key) or len(key) > MAX_FIELD_KEY: + raise FormulaError("a field reference may contain only letters, digits and _") + out.append(f"REF({key!r})"); i = end + 1; continue + two = source[i:i + 2] + if two in ("<=", ">=", "!=", "=="): + out.append(two); i += 2; continue + if two == "<>": + out.append("!="); i += 2; continue + if c == "=": + out.append("==") + elif c == "^": + out.append("**") + else: + out.append(c) + i += 1 + if quote: + raise FormulaError("a text value is missing its closing quote") + return "".join(out) + + +def _normal(node, depth=0): + if depth > MAX_DEPTH: + raise FormulaError("formula is nested too deeply") + if isinstance(node, ast.Constant) and isinstance(node.value, (int, float, str, bool)): + if isinstance(node.value, float) and not math.isfinite(node.value): + raise FormulaError("number is not finite") + return {"t": "const", "v": node.value} + if isinstance(node, ast.Name) and node.id.upper() in ("TRUE", "FALSE"): + return {"t": "const", "v": node.id.upper() == "TRUE"} + if isinstance(node, ast.UnaryOp) and isinstance(node.op, ast.USub): + return {"t": "neg", "e": _normal(node.operand, depth + 1)} + if isinstance(node, ast.BinOp): + ops = {ast.Add: "+", ast.Sub: "-", ast.Mult: "*", ast.Div: "/", ast.Pow: "^", + ast.BitAnd: "&"} + op = ops.get(type(node.op)) + if not op: + raise FormulaError("that operator is not supported") + return {"t": "bin", "op": op, "l": _normal(node.left, depth + 1), + "r": _normal(node.right, depth + 1)} + if isinstance(node, ast.Compare): + if len(node.ops) != 1 or len(node.comparators) != 1: + raise FormulaError("comparisons cannot be chained") + ops = {ast.Lt: "<", ast.LtE: "<=", ast.Gt: ">", ast.GtE: ">=", ast.Eq: "=", + ast.NotEq: "!="} + op = ops.get(type(node.ops[0])) + if not op: + raise FormulaError("that comparison is not supported") + return {"t": "cmp", "op": op, "l": _normal(node.left, depth + 1), + "r": _normal(node.comparators[0], depth + 1)} + if isinstance(node, ast.Call) and isinstance(node.func, ast.Name): + fn = ALIASES.get(node.func.id.upper(), node.func.id.upper()) + if fn == "REF": + if len(node.args) != 1 or not isinstance(node.args[0], ast.Constant): + raise FormulaError("field reference is malformed") + return {"t": "ref", "k": str(node.args[0].value)} + if fn not in FN_ARITY or node.keywords: + raise FormulaError(f"{fn} is not a supported function") + lo, hi = FN_ARITY[fn] + if not lo <= len(node.args) <= hi: + raise FormulaError(f"{fn} takes {lo} to {hi} arguments") + return {"t": "call", "fn": fn, "args": [_normal(x, depth + 1) for x in node.args]} + raise FormulaError("formula contains unsupported syntax") + + +def parse_formula(source): + source = str(source or "") + if not source.strip(): + raise FormulaError("the formula is empty") + if len(source) > MAX_SOURCE: + raise FormulaError(f"formula longer than {MAX_SOURCE} characters") + try: + # Regex patterns such as ``"\\s"`` are data for REGEX_* rather than Python escapes. + # Python 3.12 warns while parsing them even though the string is valid for this grammar. + with warnings.catch_warnings(): + warnings.simplefilter("ignore", SyntaxWarning) + parsed = ast.parse(_rewrite(source), mode="eval") + except (SyntaxError, ValueError) as exc: + raise FormulaError("the formula could not be parsed") from exc + if sum(1 for _ in ast.walk(parsed)) > MAX_NODES: + raise FormulaError(f"formula has more than {MAX_NODES} nodes") + root = _normal(parsed.body) + refs = [] + def walk(row): + if row["t"] == "ref" and row["k"] not in refs: + refs.append(row["k"]) + for key in ("e", "l", "r"): + if isinstance(row.get(key), dict): walk(row[key]) + for child in row.get("args") or (): walk(child) + walk(root) + return {"version": AST_VERSION, "source": source, "ast": root, "refs": refs} + + +def _num(v): + if isinstance(v, bool) or v is None or v == "": return None + try: n = float(v) + except (TypeError, ValueError): return None + return n if math.isfinite(n) else None + + +def _text(v): + if v is None: return "" + if isinstance(v, bool): return "TRUE" if v else "FALSE" + if isinstance(v, float) and v.is_integer(): return str(int(v)) + return str(v) + + +def _date(v): + try: return dt.date.fromisoformat(str(v)[:10]) + except (TypeError, ValueError): return None + + +def _round(v, places=0, mode=ROUND_HALF_UP): + n, p = _num(v), _num(places) + if n is None or p is None: return None + q = Decimal("1").scaleb(-int(p)) + return float(Decimal(str(n)).quantize(q, rounding=mode)) + + +def _safe_regex(pattern, subject): + pattern, subject = str(pattern), str(subject) + if len(pattern) > 120 or len(subject) > 10000 or "{" in pattern: + raise FormulaError("regular expression exceeds the safe bounds") + if re.search(r"\([^)]*[+*][^)]*\)[+*]", pattern): + raise FormulaError("nested unbounded regular expression is refused") + return pattern, subject + + +def _require_node_keys(node, expected): + if set(node) != set(expected): + raise FormulaError("formula AST node fields do not match the versioned schema") + + +def _validate_ast(node, depth=0, refs=None, count=None): + """Validate a serialized AST independently of the parser before strict API evaluation.""" + refs = [] if refs is None else refs + count = [0] if count is None else count + count[0] += 1 + if depth > MAX_DEPTH or count[0] > MAX_NODES or not isinstance(node, dict): + raise FormulaError("formula AST exceeds the safe bounds") + kind = node.get("t") + if kind == "const": + _require_node_keys(node, ("t", "v")) + value = node.get("v") + if not isinstance(value, (int, float, str, bool)) or ( + isinstance(value, float) and not math.isfinite(value)): + raise FormulaError("formula AST constant is invalid") + elif kind == "ref": + _require_node_keys(node, ("t", "k")) + key = node.get("k") + if (not isinstance(key, str) or not re.fullmatch(r"[A-Za-z0-9_]+", key) + or len(key) > MAX_FIELD_KEY): + raise FormulaError("formula AST field reference is invalid") + if key not in refs: + refs.append(key) + elif kind == "neg": + _require_node_keys(node, ("t", "e")) + _validate_ast(node.get("e"), depth + 1, refs, count) + elif kind in ("bin", "cmp"): + _require_node_keys(node, ("t", "op", "l", "r")) + allowed = {"+", "-", "*", "/", "^", "&"} if kind == "bin" else { + "<", "<=", ">", ">=", "=", "!="} + if node.get("op") not in allowed: + raise FormulaError("formula AST operator is invalid") + _validate_ast(node.get("l"), depth + 1, refs, count) + _validate_ast(node.get("r"), depth + 1, refs, count) + elif kind == "call": + _require_node_keys(node, ("t", "fn", "args")) + fn, args = node.get("fn"), node.get("args") + if fn not in FN_ARITY or not isinstance(args, list): + raise FormulaError("formula AST function is invalid") + lo, hi = FN_ARITY[fn] + if not lo <= len(args) <= hi: + raise FormulaError("formula AST function arity is invalid") + for child in args: + _validate_ast(child, depth + 1, refs, count) + else: + raise FormulaError("formula AST node is not supported") + return refs + + +def validate_definition(definition, *, require_source=False): + """Refuse any serialized definition that is not exactly reproducible by this parser. + + Checking the AST alone would let a caller display one ``source`` while asking the server to + execute a different tree. API consumers therefore send the parser-issued envelope and strict + evaluation reparses its source before accepting it. + """ + if not isinstance(definition, dict) or isinstance(definition.get("version"), bool) or ( + definition.get("version") != AST_VERSION): + raise FormulaError("formula AST version is not supported") + refs = _validate_ast(definition.get("ast")) + metadata_refs = definition.get("refs") + if not isinstance(metadata_refs, list) or metadata_refs != refs: + raise FormulaError("formula AST references do not match its metadata") + source = definition.get("source") + if require_source and not isinstance(source, str): + raise FormulaError("formula source is required for authoritative evaluation") + if isinstance(source, str): + reparsed = parse_formula(source) + if reparsed["ast"] != definition.get("ast") or reparsed["refs"] != metadata_refs: + raise FormulaError("formula source does not match its AST") + return refs + + +def evaluate_result(definition, row, *, today=None, strict=False): + """Evaluate one definition and return the shared value/error DTO.""" + diagnostics = [] + try: + if strict: + validate_definition(definition, require_source=True) + elif not isinstance(definition, dict) or definition.get("version") != AST_VERSION: + return {"value": None, "error": "invalid_definition"} + value = _eval(definition.get("ast"), row if isinstance(row, dict) else {}, + today or dt.date.today().isoformat(), diagnostics) + if diagnostics: + value = None + if isinstance(value, bool): + value = "TRUE" if value else "FALSE" + elif isinstance(value, float) and not math.isfinite(value): + value = None + elif value is not None and not isinstance(value, (int, float, str)): + value = None + diagnostics.append("evaluation_error") + return {"value": value, + "error": None if value is not None else (diagnostics[0] if diagnostics else "blank")} + except FormulaError: + if strict: + raise + return {"value": None, "error": "invalid_definition"} + except (ArithmeticError, TypeError, ValueError, OverflowError, KeyError, re.error): + return {"value": None, "error": "evaluation_error"} + + +def evaluate(definition, row, *, today=None, strict=False): + """Backward-compatible value projection of :func:`evaluate_result`.""" + return evaluate_result(definition, row, today=today, strict=strict)["value"] + + +def _diagnose(diagnostics, code): + if diagnostics is not None and code not in diagnostics: + diagnostics.append(code) + + +def _eval(node, row, today, diagnostics=None): + kind = node.get("t") if isinstance(node, dict) else None + if kind == "const": return node.get("v") + if kind == "ref": + key = node.get("k") + if key not in row: + _diagnose(diagnostics, "unknown_ref") + return row.get(key) + if kind == "neg": + n = _num(_eval(node["e"], row, today, diagnostics)); return None if n is None else -n + if kind == "bin": + left = _eval(node["l"], row, today, diagnostics) + right = _eval(node["r"], row, today, diagnostics) + if node["op"] == "&": return _text(left) + _text(right) + a, b = _num(left), _num(right) + if a is None or b is None: return None + if node["op"] == "/" and b == 0: + _diagnose(diagnostics, "div_zero") + return None + return {"+": lambda: a + b, "-": lambda: a - b, "*": lambda: a * b, + "/": lambda: a / b, "^": lambda: a ** b}[node["op"]]() + if kind == "cmp": + a = _eval(node["l"], row, today, diagnostics) + b = _eval(node["r"], row, today, diagnostics) + if a is None or b is None: return None + na, nb = _num(a), _num(b) + if (na is None) != (nb is None): return None + x, y = (na, nb) if na is not None else (str(a).casefold(), str(b).casefold()) + return {"<": x < y, "<=": x <= y, ">": x > y, ">=": x >= y, + "=": x == y, "!=": x != y}[node["op"]] + if kind == "call": + fn = node["fn"] + # IF/IFERROR are lazy, so an unselected bad branch cannot blank the result. + if fn == "IF": + cond = _eval(node["args"][0], row, today, diagnostics) + if not isinstance(cond, bool): return None + pick = 1 if cond else 2 + return (_eval(node["args"][pick], row, today, diagnostics) + if pick < len(node["args"]) else None) + if fn == "IFERROR": + # A handled failure is not an error of the whole formula, so diagnostics from the + # losing branch remain local. + try: + first = _eval(node["args"][0], row, today, []) + except (ArithmeticError, TypeError, ValueError, OverflowError, KeyError, re.error, + FormulaError): + first = None + return first if first is not None else _eval( + node["args"][1], row, today, diagnostics) + args = [_eval(x, row, today, diagnostics) for x in node.get("args") or ()] + return _call(fn, args, today) + raise FormulaError("formula AST node is not supported") + + +def _call(fn, a, today): + nums = [_num(x) for x in a]; present = [x for x in nums if x is not None] + if fn == "ABS": return abs(nums[0]) if nums[0] is not None else None + if fn == "ROUND": return _round(a[0], a[1] if len(a) > 1 else 0) + if fn == "ROUNDUP": + n, p = nums[0], int(nums[1] or 0); return None if n is None else math.copysign(math.ceil(abs(n)*10**p)/10**p, n) + if fn == "ROUNDDOWN": + n, p = nums[0], int(nums[1] or 0); return None if n is None else math.copysign(math.floor(abs(n)*10**p)/10**p, n) + if fn == "INT": return math.floor(nums[0]) if nums[0] is not None else None + if fn == "MOD": return None if None in nums or nums[1] == 0 else nums[0] % nums[1] + if fn == "SQRT": return math.sqrt(nums[0]) if nums[0] is not None and nums[0] >= 0 else None + if fn == "POWER": return None if None in nums else nums[0] ** nums[1] + if fn == "EXP": return math.exp(nums[0]) if nums[0] is not None else None + if fn == "LN": return math.log(nums[0]) if nums[0] is not None and nums[0] > 0 else None + if fn == "SUM": return sum(present) + if fn == "AVERAGE": return sum(present)/len(present) if present else None + if fn == "COUNT": return len(present) + if fn == "COUNTA": return sum(x is not None and x != "" for x in a) + if fn == "MIN": return min(present) if present else None + if fn == "MAX": return max(present) if present else None + if fn in ("AND", "OR"): + if not all(isinstance(x, bool) for x in a): return None + return all(a) if fn == "AND" else any(a) + if fn == "NOT": return not a[0] if isinstance(a[0], bool) else None + if fn == "ISBLANK": return a[0] is None or a[0] == "" + if fn == "SWITCH": + pairs = a[1:]; fallback = None + if len(pairs) % 2: fallback = pairs.pop() + for i in range(0, len(pairs), 2): + if str(a[0]).casefold() == str(pairs[i]).casefold(): return pairs[i+1] + return fallback + if fn == "CONCATENATE": return "".join(_text(x) for x in a) + if fn == "LEFT": return _text(a[0])[:int(nums[1] if len(a)>1 and nums[1] is not None else 1)] + if fn == "RIGHT": return _text(a[0])[-int(nums[1] if len(a)>1 and nums[1] is not None else 1):] + if fn == "MID": return _text(a[0])[max(0,int(nums[1] or 1)-1):max(0,int(nums[1] or 1)-1)+int(nums[2] or 0)] + if fn == "LEN": return len(_text(a[0])) + if fn == "TRIM": return " ".join(_text(a[0]).split()) + if fn == "UPPER": return _text(a[0]).upper() + if fn == "LOWER": return _text(a[0]).lower() + if fn == "PROPER": return _text(a[0]).title() + if fn == "VALUE": return _num(a[0]) + if fn == "TEXT": + n = _num(a[0]); fmt = str(a[1]); + if n is None: return None + if fmt == "0%": return f"{round(n*100):.0f}%" + if fmt in ("0.00", "#,##0.00", "$#,##0.00"): return ("$" if fmt.startswith("$") else "") + f"{n:,.2f}" + return None + if fn == "SUBSTITUTE": + text, old, new = map(_text, a[:3]) + if len(a) == 3: return text.replace(old, new) + occurrence = int(nums[3] or 0); parts = text.split(old) + return old.join(parts[:occurrence]) + (new + old.join(parts[occurrence:]) if 0 < occurrence < len(parts) else "") + if fn == "FIND": + start = int(nums[2] or 1)-1 if len(a)>2 else 0; pos = _text(a[1]).find(_text(a[0]), start) + return pos+1 if pos >= 0 else None + if fn in ("REGEX_EXTRACT", "REGEX_REPLACE"): + pattern, subject = _safe_regex(a[1], a[0]); m = re.search(pattern, subject) + if fn == "REGEX_EXTRACT": return m.group(0) if m else None + return re.sub(pattern, _text(a[2]), subject) + if fn == "TODAY": return str(today)[:10] + if fn in ("YEAR", "MONTH", "DAY"): + d = _date(a[0]); return getattr(d, fn.lower()) if d else None + if fn == "DAYS": + x,y=_date(a[0]),_date(a[1]); return (x-y).days if x and y else None + if fn == "DATEADD": + d, n, unit = _date(a[0]), nums[1], _text(a[2]).lower() + if not d or n is None: return None + n=int(n) + if unit in ("day","days"): d += dt.timedelta(days=n) + elif unit in ("week","weeks"): d += dt.timedelta(weeks=n) + elif unit in ("month","months"): + month=d.month-1+n; year=d.year+month//12; month=month%12+1; d=d.replace(year=year,month=month,day=min(d.day,calendar.monthrange(year,month)[1])) + elif unit in ("year","years"): d=d.replace(year=d.year+n,day=min(d.day,calendar.monthrange(d.year+n,d.month)[1])) + else: return None + return d.isoformat() + if fn == "DATETIME_DIFF": + x,y=_date(a[0]),_date(a[1]); unit=_text(a[2] if len(a)>2 else "days").lower() + if not x or not y: return None + days=(x-y).days + return days if unit in ("day","days") else math.trunc(days/7) if unit in ("week","weeks") else None + if fn == "DATETIME_FORMAT": + d=_date(a[0]); fmt=_text(a[1]); + if not d: return None + tokens={"YYYY":f"{d.year:04d}","MM":f"{d.month:02d}","DD":f"{d.day:02d}","M":str(d.month),"D":str(d.day)} + for key in sorted(tokens,key=len,reverse=True): fmt=fmt.replace(key,tokens[key]) + return fmt + raise FormulaError("formula function is not implemented") + + +def explain_recalculation(definition, field_revisions): + """Compact provenance; formula history stores definitions, never redundant cell snapshots.""" + definition = definition if isinstance(definition, dict) else {} + field_revisions = field_revisions if isinstance(field_revisions, dict) else {} + canonical = {key: definition.get(key) for key in ("version", "source", "ast", "refs")} + digest = hashlib.sha256(json.dumps( + canonical, sort_keys=True, separators=(",", ":"), ensure_ascii=False + ).encode("utf-8")).hexdigest() + def revision(value, default): + if isinstance(value, bool): + return default + try: + return max(0, int(value)) + except (TypeError, ValueError, OverflowError): + return default + + return {"formulaRevision": revision(definition.get("revision"), 1) or 1, + "astVersion": revision(definition.get("version"), AST_VERSION) or AST_VERSION, + "definitionHash": f"sha256:{digest}", + "inputs": {key: revision(field_revisions.get(key), 0) + for key in definition.get("refs") or ()}, + "valueSnapshotStored": False} + + +def evaluate_fields(definitions, row, *, today=None): + """Evaluate formula-over-formula dependencies once, refusing every member of a cycle.""" + definitions = definitions if isinstance(definitions, dict) else {} + row = row if isinstance(row, dict) else {} + # Stored/client materialisations are never authority. A definition present in this batch + # always replaces a same-named row value through the server evaluator. + values = {key: value for key, value in row.items() if key not in definitions} + errors, active, resolved = {}, [], set() + + def resolve(key): + if key in resolved or (key in values and key not in definitions): + return values[key] + if key in active: + start = active.index(key) + for member in active[start:]: + errors[member] = "cycle" + values[member] = None + return None + definition = definitions.get(key) + if not isinstance(definition, dict): + return None + active.append(key) + inputs = dict(values) + dependency_error = False + for ref in definition.get("refs") or (): + if ref in definitions: + inputs[ref] = resolve(ref) + dependency_error = dependency_error or ref in errors + elif ref in row: + inputs[ref] = row[ref] + if errors.get(key) == "cycle": + active.pop() + resolved.add(key) + return None + result = evaluate_result(definition, inputs, today=today) + active.pop() + values[key] = result["value"] + if result["error"]: + errors[key] = ("dependency_error" + if result["error"] == "blank" and dependency_error + else result["error"]) + resolved.add(key) + return values.get(key) + + for field_key in definitions: + resolve(field_key) + return {"values": {key: values.get(key) for key in definitions}, "errors": errors} + + +def evaluate_batch(definitions, rows, *, consumer, today=None, field_revisions=None): + """Evaluate a bounded set of formula definitions over typed consumer rows. + + Grid, filter, export, automation, Query and Time Series all use this DTO. ``consumer`` is + attribution, never an evaluator switch: the result is deliberately identical for every + declared surface. + """ + if consumer not in CONSUMERS: + raise FormulaError("formula consumer is not supported") + if not isinstance(definitions, dict) or not definitions: + raise FormulaError("formula definitions must be a non-empty object") + if len(definitions) > MAX_DEFINITIONS: + raise FormulaError(f"formula batch has more than {MAX_DEFINITIONS} definitions") + clean = {} + for key, definition in definitions.items(): + if not isinstance(key, str) or not re.fullmatch(r"[A-Za-z0-9_]+", key): + raise FormulaError("formula definition key is invalid") + validate_definition(definition, require_source=True) + clean[key] = definition + if not isinstance(rows, list) or not rows: + raise FormulaError("formula rows must be a non-empty list") + if len(rows) > MAX_BATCH_ROWS: + raise FormulaError(f"formula batch has more than {MAX_BATCH_ROWS} rows") + output, seen_ids = [], set() + for item in rows: + if not isinstance(item, dict) or not isinstance(item.get("values"), dict): + raise FormulaError("each formula row needs an id and values object") + row_id = item.get("id") + if isinstance(row_id, bool) or not isinstance(row_id, (str, int)) or row_id in seen_ids: + raise FormulaError("each formula row id must be a unique string or integer") + seen_ids.add(row_id) + result = evaluate_fields(clean, item["values"], today=today) + output.append({"id": row_id, "values": result["values"], "errors": result["errors"]}) + return { + "contractVersion": CONTRACT_VERSION, + "astVersion": AST_VERSION, + "authority": AUTHORITY, + "consumer": consumer, + "definitions": [ + {"key": key, **explain_recalculation(definition, field_revisions or {})} + for key, definition in clean.items() + ], + "rows": output, + } + + +def contract_dto(): + """Return the complete JSON-safe grammar and executable browser parity corpus.""" + inverse_aliases = {} + for alias, canonical in ALIASES.items(): + inverse_aliases.setdefault(canonical, []).append(alias) + functions = [ + {"name": name, "minArgs": bounds[0], "maxArgs": bounds[1], + "aliases": sorted(inverse_aliases.get(name, []))} + for name, bounds in sorted(FN_ARITY.items()) + ] + operators = [ + {"kind": "unary", "canonical": "-", "aliases": []}, + *({"kind": "binary", "canonical": token, "aliases": []} + for token in ("+", "-", "*", "/", "^", "&")), + *({"kind": "comparison", "canonical": token, "aliases": aliases} + for token, aliases in (("<", []), ("<=", []), (">", []), (">=", []), + ("=", ["=="]), ("!=", ["<>"]))), + ] + fixtures = [ + {"id": f"function:{name.lower()}", "kind": "function", "function": name, + "source": source, "row": {"empty": None}, "today": "2024-01-02", + "expected": expected, "expectedError": None} + for name, (source, expected) in PARITY_FIXTURES.items() + ] + fixtures.extend( + {"id": f"operator:{name}", "kind": "operator", "operator": name, + "source": source, "row": row, "today": "2024-01-02", + "expected": expected, "expectedError": None} + for name, (source, row, expected) in OPERATOR_PARITY_FIXTURES.items() + ) + errors = [ + {"id": f"parse:{name}", "phase": "parse", "source": source, + "expectedStatus": 400, "expectedError": "bad_formula"} + for name, source in PARSE_ERROR_FIXTURES.items() + ] + errors.extend( + {"id": f"evaluate:{name}", "phase": "evaluate", "source": source, + "row": row, "today": "2024-01-02", "expected": None, + "expectedStatus": 200, "expectedError": expected_error} + for name, (source, row, expected_error) in EVALUATION_ERROR_FIXTURES.items() + ) + errors.extend(( + {"id": "definition:source_ast_mismatch", "phase": "definition", + "source": "{amount} + 1", "tamperedSource": "{amount} + 2", + "row": {"amount": 4}, "expectedStatus": 400, "expectedError": "bad_formula"}, + {"id": "batch:cycle", "phase": "batch", + "sources": {"a": "{b} + 1", "b": "{a} + 1"}, + "rows": [{"id": "2024-01", "values": {}}], "consumer": "timeseries", + "expectedStatus": 200, "expectedErrors": {"a": "cycle", "b": "cycle"}}, + )) + return { + "contractVersion": CONTRACT_VERSION, + "astVersion": AST_VERSION, + "authority": AUTHORITY, + "consumers": list(CONSUMERS), + "grammar": { + "name": "aios-formula", + "reference": {"open": "{", "close": "}", + "pattern": r"\{[A-Za-z0-9_]+\}", + "example": "{amount}", + "legacySquareReferenceAccepted": False}, + "literals": ["number", "text", "TRUE", "FALSE"], + "functionCall": "NAME(argument, ...)", + "operators": operators, + }, + "ast": { + "definitionFields": ["version", "source", "ast", "refs"], + "nodes": [ + {"type": "const", "fields": ["t", "v"]}, + {"type": "ref", "fields": ["t", "k"]}, + {"type": "neg", "fields": ["t", "e"]}, + {"type": "bin", "fields": ["t", "op", "l", "r"]}, + {"type": "cmp", "fields": ["t", "op", "l", "r"]}, + {"type": "call", "fields": ["t", "fn", "args"]}, + ], + }, + "dto": { + "parseRequestRequired": ["source"], + "parseResponseRequired": ["contractVersion", "astVersion", "authority", "definition"], + "singleEvaluateRequestRequired": ["definition", "row"], + "singleEvaluateResponseRequired": ["contractVersion", "astVersion", "authority", + "value", "error", "recalculation"], + "batchEvaluateRequestRequired": ["consumer", "definitions", "rows"], + "batchEvaluateResponseRequired": ["contractVersion", "astVersion", "authority", + "consumer", "definitions", "rows"], + "batchRequestRowRequired": ["id", "values"], + "batchResponseRowRequired": ["id", "values", "errors"], + }, + "batchFixture": { + "id": "timeseries:formula_dependency", + "consumer": "timeseries", + "sources": {"double": "{amount} * 2", "total": "{double} + 1"}, + "rows": [{"id": "2024-01", "values": {"amount": 4}}, + {"id": "2024-02", "values": {"amount": 7}}], + "expected": [ + {"id": "2024-01", "values": {"double": 8.0, "total": 9.0}, "errors": {}}, + {"id": "2024-02", "values": {"double": 14.0, "total": 15.0}, "errors": {}}, + ], + }, + "limits": {"sourceCharacters": MAX_SOURCE, "astNodes": MAX_NODES, + "astDepth": MAX_DEPTH, "definitionsPerBatch": MAX_DEFINITIONS, + "rowsPerBatch": MAX_BATCH_ROWS}, + "functions": functions, + "errorCodes": ["bad_formula", "unknown_ref", "div_zero", "cycle", "blank", + "dependency_error", "evaluation_error"], + "fixtures": fixtures, + "errors": errors, + } diff --git a/api/main.py b/api/main.py index dd0b4eb668b9f3896772db912466a81120d0d35f..9871eb99a995a68ebc57352d32178c12ea82cda1 100644 --- a/api/main.py +++ b/api/main.py @@ -832,7 +832,13 @@ def _rebuild_odoo_relational(why, previous_meta_fingerprint=None, meta_state_out if not _rel.is_royal("royal-imports"): return False _rt = _runtime.get_runtime("royal-imports") + from core import user_tables as _ut + _ut.record_odoo_sync_status(phase="syncing", st=_rt) counts = _rel.refresh(_rt, "royal-imports") + raw_models, schema_errors = _rel.authoritative_schema_snapshot() + _ut.record_odoo_schema_snapshot(raw_models, complete=not schema_errors, + errors=schema_errors, st=_rt) + rebuild_errors = [] # ⭐⭐ AND THEN COMPUTE THE CELLS, which `refresh` does NOT do. # # ⛔ THE FAILURE THIS CLOSES IS THE WORST-LOOKING KIND. `refresh` writes rows and field @@ -853,15 +859,21 @@ def _rebuild_odoo_relational(why, previous_meta_fingerprint=None, meta_state_out import automation_engine as _engine _engine.refresh_relations(_rt, log=lambda *_a: None) except Exception as e: # noqa: BLE001 + rebuild_errors.append(f"relations: {type(e).__name__}: {e}") print(f"[aios-api] odoo relation cells failed ({why}): {type(e).__name__}: {e}") try: import rollup_sql as _rollup for _b, _key, _l, _f in _rel.TABLES: _rollup.compute(_rt, _key) except Exception as e: # noqa: BLE001 + rebuild_errors.append(f"rollups: {type(e).__name__}: {e}") print(f"[aios-api] odoo source rollups failed ({why}): {type(e).__name__}: {e}") print(f"[aios-api] odoo relational rebuild done ({why}): " + ", ".join(f"{k}={v}" for k, v in sorted((counts or {}).items()))) + if rebuild_errors: + _ut.record_odoo_sync_status(phase="error", errors=rebuild_errors, st=_rt) + else: + _ut.record_odoo_sync_status(phase="ok", st=_rt) meta_state = _rebuild_meta_relational( why, _rt, previous_fingerprint=previous_meta_fingerprint, force_relations=True) if isinstance(meta_state_out, dict): @@ -876,6 +888,12 @@ def _rebuild_odoo_relational(why, previous_meta_fingerprint=None, meta_state_out # `BinderException` about a missing column is a different action from a timeout or an auth # failure, and the text alone often does not say which it was. print(f"[aios-api] odoo relational rebuild failed ({why}): {type(e).__name__}: {e}") + try: + if '_rt' in locals(): + from core import user_tables as _ut + _ut.record_odoo_sync_status(phase="error", errors=[f"{type(e).__name__}: {e}"], st=_rt) + except Exception: + pass return False diff --git a/api/odoo_relational.py b/api/odoo_relational.py index 0bfd7f34e7a6fd89f55df2d5ee94463b7a45710e..84754e2022f4173824011caad4b0adc860990508 100644 --- a/api/odoo_relational.py +++ b/api/odoo_relational.py @@ -1404,6 +1404,33 @@ TABLES = ( ("gl_lines", GL_LINES_KEY, "Odoo GL lines", gl_line_fields), ) +# The grid declarations above are the stable mirror contract, not Odoo's schema. T03 snapshots +# must come from Odoo's authoritative ``fields_get`` response so empty/default-off fields are not +# silently lost. This helper is injectable in hermetic tests and keeps the live read narrow. +ODOO_TABLE_MODELS = { + INVOICES_KEY: "account.move", ORDERS_KEY: "sale.order", AGENTS_KEY: "res.partner", + ACCOUNTS_KEY: "account.account", VENDORS_KEY: "res.partner", BILLS_KEY: "account.move", + ORDER_LINES_KEY: "sale.order.line", GL_LINES_KEY: "account.move.line", +} + + +def authoritative_schema_snapshot(): + from core.odoo import get_odoo + from harness.datastore import ENTITIES + models = {} + errors = [] + seen = set() + for spec in ENTITIES.values(): + model = str(spec.get("model") or "") + if model and model not in seen: + seen.add(model) + try: + models[model] = get_odoo().fields_get( + model, attributes=("string", "type", "required", "readonly", "relation")) + except Exception as exc: + errors.append(f"{model}: {type(exc).__name__}: {exc}") + return models, errors + #: store key -> the REAL-WORLD POPULATION that table presents (`core.registry`'s `subject` #: vocabulary, same strings, one namespace). ⭐ W33-T41 / item 12a. #: diff --git a/api/operator_control.py b/api/operator_control.py new file mode 100644 index 0000000000000000000000000000000000000000..65a8c091315dbaaee62c335d1a61b67d42ca49f8 --- /dev/null +++ b/api/operator_control.py @@ -0,0 +1,206 @@ +"""Same-origin, capability-scoped Operator grants and audited setup records. + +This module is deliberately transport-neutral. The browser and a future MCP adapter call the same +authorization function; neither receives a secret, provider credential, or cross-tenant wildcard. +""" +from __future__ import annotations + +import copy +import hashlib +import secrets +import time +from typing import Any + + +STORE_KEY = "operator_control" +SETUP_KEY = "operator_setup" +AUDIENCE = "loopable-operator" +CAPABILITIES = frozenset({"tenant.read", "tenant.configure", "voice.read", "voice.configure"}) +SETUP_FIELDS = frozenset({"displayName", "timezone", "voicePurpose", "consentPolicy", + "queryEnabled", "providerLabel"}) + + +class OperatorRefusal(ValueError): + pass + + +def _now() -> int: + return int(time.time()) + + +def _hash(value: str) -> str: + return hashlib.sha256(str(value or "").encode("utf-8")).hexdigest() + + +def _origin(value: Any) -> str: + text = str(value or "").strip().lower().rstrip("/") + if not text.startswith(("https://", "http://localhost", "http://127.0.0.1")): + raise OperatorRefusal("Operator grants require the current Loopable origin") + if any(ch in text for ch in ("?", "#", "@")): + raise OperatorRefusal("Operator origin is invalid") + return text[:240] + + +def _document(rt: Any) -> dict: + try: + raw = rt.get(STORE_KEY) or {} + except Exception: + raw = {} + if not isinstance(raw, dict): + raw = {} + return {"grants": copy.deepcopy(raw.get("grants") or {}), + "audit": copy.deepcopy(raw.get("audit") or [])} + + +def _update(rt: Any, fn): + return rt.update(STORE_KEY, lambda raw: fn({ + "grants": copy.deepcopy((raw or {}).get("grants") or {}), + "audit": copy.deepcopy((raw or {}).get("audit") or []), + }), flush="sync") + + +def _audit(rt: Any, *, actor: str, tenant: str, capability: str, action: str, + outcome: str, grant_id: str = "", source: str = "browser") -> dict: + event = {"id": f"opa_{_now()}_{secrets.token_hex(4)}", "at": _now(), + "actor": str(actor or "")[:120], "tenant": str(tenant or "")[:120], + "capability": str(capability or "")[:80], "action": str(action or "")[:120], + "outcome": str(outcome or "refused")[:40], "grantId": str(grant_id or "")[:120], + "source": "mcp" if source == "mcp" else "browser"} + def up(doc): + doc["audit"] = (list(doc.get("audit") or []) + [event])[-2000:] + return doc + _update(rt, up) + return event + + +def mint_grant(rt: Any, *, actor: str, tenant: str, capabilities: Any, origin: str, + audience: str = AUDIENCE, ttl_seconds: int = 900) -> dict: + target = str(tenant or "").strip().lower() + gid, outcome = "", "refused" + try: + if not target or target == "*": + raise OperatorRefusal("Operator grant needs one exact tenant") + requested = sorted({str(value) for value in (capabilities or [])}) + if not requested or any(value not in CAPABILITIES for value in requested): + raise OperatorRefusal("Operator grant contains an unsupported capability") + if str(audience or "") != AUDIENCE: + raise OperatorRefusal("Operator grant audience is invalid") + bound_origin = _origin(origin) + ttl = max(60, min(int(ttl_seconds or 900), 3600)) + token = secrets.token_urlsafe(32) + gid = "opg_" + secrets.token_hex(8) + rec = {"id": gid, "actor": str(actor or "")[:120], "tenant": target, + "capabilities": requested, "origin": bound_origin, "audience": AUDIENCE, + "tokenHash": _hash(token), "createdAt": _now(), "expiresAt": _now() + ttl, + "revokedAt": 0} + _update(rt, lambda doc: {**doc, "grants": {**doc["grants"], gid: rec}}) + outcome = "allowed" + return {"grant": grant_dto(rec), "token": token} + finally: + _audit(rt, actor=actor, tenant=target, capability="grant.mint", action="mint", + outcome=outcome, grant_id=gid) + + +def grant_dto(rec: Any) -> dict: + row = rec if isinstance(rec, dict) else {} + return {key: copy.deepcopy(row.get(key)) for key in + ("id", "actor", "tenant", "capabilities", "origin", "audience", + "createdAt", "expiresAt", "revokedAt")} + + +def revoke_grant(rt: Any, grant_id: str, actor: str) -> dict: + result = {} + outcome = "refused" + def up(doc): + rec = doc["grants"].get(str(grant_id)) + if not isinstance(rec, dict): + raise OperatorRefusal("unknown Operator grant") + rec = {**rec, "revokedAt": _now()} + doc["grants"][str(grant_id)] = rec + result.update(rec) + return doc + try: + _update(rt, up) + outcome = "allowed" + return grant_dto(result) + finally: + _audit(rt, actor=actor, tenant=result.get("tenant", ""), capability="grant.revoke", + action="revoke", outcome=outcome, grant_id=str(grant_id)) + + +def authorize(rt: Any, *, token: str, actor: str, tenant: str, capability: str, + origin: str, audience: str = AUDIENCE, action: str = "read", + source: str = "browser") -> dict: + target, cap = str(tenant or "").strip().lower(), str(capability or "") + outcome, matched = "refused", None + try: + bound_origin = _origin(origin) + now = _now() + for rec in _document(rt)["grants"].values(): + if isinstance(rec, dict) and secrets.compare_digest(str(rec.get("tokenHash") or ""), + _hash(token)): + matched = rec + break + if not matched or matched.get("actor") != str(actor or ""): + raise OperatorRefusal("Operator grant is invalid") + if matched.get("tenant") != target: + raise OperatorRefusal("Operator grant does not cover that tenant") + if matched.get("audience") != audience or audience != AUDIENCE: + raise OperatorRefusal("Operator grant audience is invalid") + if matched.get("origin") != bound_origin: + raise OperatorRefusal("Operator grant origin is invalid") + if matched.get("revokedAt") or int(matched.get("expiresAt") or 0) <= now: + raise OperatorRefusal("Operator grant is expired or revoked") + if cap not in set(matched.get("capabilities") or []): + raise OperatorRefusal("Operator capability is denied") + outcome = "allowed" + return grant_dto(matched) + finally: + _audit(rt, actor=actor, tenant=target, capability=cap, action=action, + outcome=outcome, grant_id=str((matched or {}).get("id") or ""), source=source) + + +def authorize_mcp(rt: Any, **kwargs) -> dict: + """Optional adapter seam. MCP gets the identical grant, audience, origin, and audit wall.""" + return authorize(rt, source="mcp", **kwargs) + + +def setup_dto(rt: Any, tenant: str) -> dict: + try: + raw = rt.get(SETUP_KEY) or {} + except Exception: + raw = {} + row = raw if isinstance(raw, dict) else {} + safe = {key: copy.deepcopy(row.get(key)) for key in SETUP_FIELDS if key in row} + return {"tenant": str(tenant or ""), "setup": safe, "secretsPresent": False, + "rawProviderBindings": False} + + +def update_setup(rt: Any, tenant: str, raw: Any, actor: str) -> dict: + body = raw if isinstance(raw, dict) else {} + unknown = sorted(set(body) - SETUP_FIELDS) + if unknown: + raise OperatorRefusal("unsupported setup properties: " + ", ".join(unknown)) + clean = {} + for key, value in body.items(): + if key == "queryEnabled": + if not isinstance(value, bool): + raise OperatorRefusal("queryEnabled must be true or false") + clean[key] = value + else: + clean[key] = " ".join(str(value or "").split())[:240] + clean.update({"updatedBy": str(actor or "")[:120], "updatedAt": _now()}) + rt.update(SETUP_KEY, lambda old: {**dict(old or {}), **clean}, flush="sync") + return setup_dto(rt, tenant) + + +def audit_rows(rt: Any, limit: int = 200) -> list[dict]: + rows = _document(rt)["audit"][-max(1, min(int(limit or 200), 500)):] + return [copy.deepcopy(row) for row in reversed(rows)] + + +def audit_attempt(rt: Any, *, actor: str, tenant: str, capability: str, action: str, + outcome: str, grant_id: str = "", source: str = "browser") -> dict: + """Public final-operation audit seam; authorization is not operation success.""" + return _audit(rt, actor=actor, tenant=tenant, capability=capability, action=action, + outcome=outcome, grant_id=grant_id, source=source) diff --git a/api/routes_admin.py b/api/routes_admin.py index d7e4c45f9dc66bdc2f8a552ab0109b4816b7c7db..1833e3a3936fed3395980ed57cd1fc6657afed38 100644 --- a/api/routes_admin.py +++ b/api/routes_admin.py @@ -46,6 +46,22 @@ router = APIRouter(prefix="/api/v1") #: (`data[username]`), so it is constrained rather than trusted: lowercase, no separators, no #: whitespace, nothing that could traverse or collide once it becomes part of a path or a filename. _UNAME_RE = re.compile(r"^[a-z0-9][a-z0-9._-]{1,31}$") + + +@router.get("/admin/system-data") +def system_data_inspector(session: Session = Depends(require_session)): + """Definition-only Loopable system data, visible to platform admins and never mutable.""" + if not platform_admin.is_platform_admin(session.user): + raise err(403, "platform_admin_only", + "System data is available only to a Loopable platform administrator") + from core import route_system, user_tables + definitions = user_tables.all_defs(st=session.runtime) + rows = [] + for descriptor in route_system.system_table_descriptors(): + stored = definitions.get(descriptor["key"]) or {} + rows.append({**descriptor, "fields": [dict(field) for field in stored.get("fields") or ()], + "readOnly": True}) + return {"databases": rows, "readOnly": True, "source": "Loopable"} #: ⭐⭐ WAVE 37 · T03 (owner ruling R8) — AN EMAIL ADDRESS IS ALSO A USERNAME. Owner: the username #: IS the email, one identity. ⛔ THIS IS NOT A NEW CAPABILITY, IT IS THE DOOR CATCHING UP WITH THE #: DATA: measured on deployed staging before the change, FIVE of tenant #0's eight accounts already @@ -818,6 +834,11 @@ def _module_fields(key, session=None): f"{key!r} has no permission-editor field schema. The editor governs " f"{sorted(providers)} and this tenant's own databases") src = list(provider()) + if key == "product_data" and session is not None: + from core import user_tables + raw = user_tables.enabled_odoo_schema_fields(session.runtime, model="product.product") + raw_have = {f.get("key") for f in src if isinstance(f, dict)} + src.extend(f for f in raw if f.get("key") not in raw_have) import core.perm_scope as perm_scope # ⭐⭐ W40-T05 / OWNER I16 — THE USER-GENERATED COLUMNS JOIN THE TOPIC'S VOCABULARY. # *"Permission Filters must be able to filter on user-generated Fields too."* The two topic diff --git a/api/routes_alerts.py b/api/routes_alerts.py index c651d88cf37431431a9588f38c540b71c89597df..1d653f2bc2209bc6f4955fcc017fd5c83873af85 100644 --- a/api/routes_alerts.py +++ b/api/routes_alerts.py @@ -84,6 +84,11 @@ AUTOMATION_TOPIC = "automation" _UT_TOPIC = re.compile(r"ut_[A-Za-z0-9_]+\Z") +def _tenant_today(session): + from core import route_system + return route_system.tenant_day_for_runtime(session.runtime) + + def route_for_topic(topic): """A grid SCOPE key -> the registry route that renders it, or None. @@ -417,7 +422,8 @@ def _evaluate(session: Session, rec: dict, assemblies=None): measure_sets=g.get("measure_sets") or {}, today=g.get("today")) pids = filter_eval.visible_pids(config.get("filters") or [], rows, g["fields"], ctx, - member_pids=config.get("memberPids")) + member_pids=config.get("memberPids"), + group_by=config.get("groupBy")) labels = {str(r.get("pid")): str(r.get("name") or r.get("pid")) for r in rows} return alerts.evaluate(rec.get("id"), [str(p) for p in pids], labels=labels, partial=False, st=session.runtime) @@ -629,7 +635,15 @@ def notifications(session: Session = Depends(require_session)): continue # one bad alert must not empty the pane # ⭐ W32-T20 (C3): every item leaves through `inbox_view`, so a notification queued before # this wave carries a `target` too. See `notification_view`'s header for why it is derived. - return inbox_view(alerts.inbox(session.uname, st=session.runtime)) + box = inbox_view(alerts.inbox(session.uname, st=session.runtime)) + from core import route_system + route_tasks = route_system.inbox_tasks( + session.runtime, session.uname, today=_tenant_today(session)) + items = list(box.get("items") or []) + route_tasks + items.sort(key=lambda item: str(item.get("at") or ""), reverse=True) + box["items"] = items + box["unread"] = sum(1 for item in items if not item.get("read")) + return box @router.post("/notifications/read") @@ -642,8 +656,17 @@ def read_notifications(body: dict = Body(default=None), # ⚠ THE SAME ENRICHMENT ON BOTH DOORS. `mark_read` returns a fresh inbox, and the Inbox # module re-renders from it — an un-enriched answer here would strip `target` off every row # the moment somebody marked one read, i.e. the feature would work until first use. - return inbox_view(alerts.mark_read(session.uname, ids, read=bool(body.get("read", True)), - st=session.runtime)) + read = bool(body.get("read", True)) + box = inbox_view(alerts.mark_read(session.uname, ids, read=read, st=session.runtime)) + from core import route_system + route_system.reconcile_inbox_tasks(session.runtime, today=_tenant_today(session)) + route_tasks = route_system.mark_inbox_tasks( + session.runtime, session.uname, ids, read=read) + items = list(box.get("items") or []) + route_tasks + items.sort(key=lambda item: str(item.get("at") or ""), reverse=True) + box["items"] = items + box["unread"] = sum(1 for item in items if not item.get("read")) + return box def after_write(session: Session, topic_key: str): diff --git a/api/routes_assets.py b/api/routes_assets.py index 01d0914916d2b1d32ec90de6f0fc470e984284f9..64a5ea96af0e5e42a455e33a393f5a6323c2375d 100644 --- a/api/routes_assets.py +++ b/api/routes_assets.py @@ -37,9 +37,158 @@ from fastapi.responses import FileResponse from deps import Session, err, module_gate, require_session from routes_admin import admin_gate +import application_registry as applications +import semantic_registry as semantic_rt router = APIRouter(prefix="/api/v1") + +def _application_error(exc): + if isinstance(exc, PermissionError): + raise err(403, "forbidden", str(exc)) + message = str(exc) + status = 404 if "does not exist" in message else 400 + raise err(status, "application_not_found" if status == 404 else "bad_application", message) + + +def application_access_context(session, databases, *, strict=False): + """Return readable databases plus governed semantic records, without exposing refused names.""" + from core import perm_scope + from harness import semantic as sem + allowed, records = set(), {} + for database in dict.fromkeys(str(value or "").strip() for value in databases or ()): + if not database: + continue + topic = next((row for row in sem.topics().values() + if isinstance(row, dict) and row.get("grid") == database), None) + readable = bool(topic and perm_scope.may_read( + session.user, database, st=session.runtime)) + if not readable: + if strict: + raise PermissionError("that database is not available") + continue + allowed.add(database) + baseline = semantic_rt.topic_records(database, topic) + records[database] = semantic_rt.load(session.runtime, baseline) + return allowed, records + + +def _application_databases(runtime, application_id=None): + doc = applications.load(runtime) + if application_id: + app = doc["applications"].get(application_id) + return list(app.get("compatibleDatabases") or ()) if isinstance(app, dict) else [] + return list(dict.fromkeys( + database for app in doc["applications"].values() if isinstance(app, dict) + for database in (app.get("compatibleDatabases") or ()))) + + +@router.get("/applications") +def application_index(database: str = "", type: str = "", starred: bool | None = None, + session: Session = Depends(require_session)): + if type and type not in applications.APPLICATION_TYPES: + raise err(400, "bad_application", "application type is not recognized") + candidates = [database] if database else _application_databases(session.runtime) + try: + allowed, _ = application_access_context(session, candidates, strict=bool(database)) + except PermissionError as exc: + _application_error(exc) + return {"applications": applications.list_apps( + session.runtime, session.uname, session.admin, database=database or None, + kind=type or None, starred=starred, allowed_databases=allowed)} + + +@router.post("/applications", status_code=201) +def create_application(body: dict = Body(default=None), + session: Session = Depends(require_session)): + if not session.runtime.available(): + raise err(503, "store_unavailable", "the tenant store is unavailable. Nothing was saved.") + try: + compatible = (body or {}).get("compatibleDatabases") + allowed, records = application_access_context(session, compatible, strict=True) + return {"application": applications.create( + session.runtime, session.tenant, session.uname, body, + semantic_records=records, allowed_databases=allowed)} + except (applications.ApplicationError, PermissionError) as exc: + _application_error(exc) + + +@router.patch("/applications/{application_id}") +def update_application(application_id: str, body: dict = Body(default=None), + session: Session = Depends(require_session)): + try: + current = _application_databases(session.runtime, application_id) + if not current: + raise applications.ApplicationError("application does not exist") + allowed, _ = application_access_context(session, current) + if not allowed: + raise applications.ApplicationError("application does not exist") + proposed = ((body or {}).get("compatibleDatabases") + if "compatibleDatabases" in (body or {}) else current) + proposed_allowed, records = application_access_context(session, proposed, strict=True) + allowed |= proposed_allowed + return {"application": applications.mutate( + session.runtime, application_id, session.uname, session.admin, body, + semantic_records=records, allowed_databases=allowed, + target_database=next((database for database in proposed + if database in proposed_allowed), None))} + except (applications.ApplicationError, PermissionError) as exc: + _application_error(exc) + + +@router.post("/applications/{application_id}/duplicate", status_code=201) +def duplicate_application(application_id: str, + session: Session = Depends(require_session)): + try: + current = _application_databases(session.runtime, application_id) + if not current: + raise applications.ApplicationError("application does not exist") + allowed, records = application_access_context(session, current, strict=True) + return {"application": applications.duplicate( + session.runtime, application_id, session.tenant, session.uname, session.admin, + semantic_records=records, allowed_databases=allowed)} + except (applications.ApplicationError, PermissionError) as exc: + _application_error(exc) + + +@router.delete("/applications/{application_id}") +def delete_application(application_id: str, + session: Session = Depends(require_session)): + try: + current = _application_databases(session.runtime, application_id) + allowed, _ = application_access_context(session, current) + if not allowed: + raise applications.ApplicationError("application does not exist") + return applications.delete(session.runtime, application_id, session.uname, session.admin, + allowed_databases=allowed) + except (applications.ApplicationError, PermissionError) as exc: + _application_error(exc) + + +@router.put("/applications/defaults/{database}/{type}") +def set_application_default(database: str, type: str, body: dict = Body(default=None), + session: Session = Depends(require_session)): + try: + allowed, _ = application_access_context(session, [database], strict=True) + return applications.set_default(session.runtime, database, type, + str((body or {}).get("applicationId") or ""), + session.uname, session.admin, + allowed_databases=allowed) + except (applications.ApplicationError, PermissionError) as exc: + _application_error(exc) + + +@router.get("/applications/defaults/{database}/{type}") +def resolve_application_default(database: str, type: str, + session: Session = Depends(require_session)): + try: + allowed, _ = application_access_context(session, [database], strict=True) + return applications.resolve_default(session.runtime, database, type, + session.uname, session.admin, + allowed_databases=allowed) + except (applications.ApplicationError, PermissionError) as exc: + _application_error(exc) + #: Same grant as the product surface these images decorate (routes_products.MODULE, restated #: here to avoid importing that module for one string). MODULE = "product_data" @@ -110,7 +259,7 @@ def _web_derivative(src, code): from PIL import Image except ImportError: raise err(503, "pillow_missing", - "image derivatives need Pillow — `pip install pillow` on this host") + "image derivatives need Pillow. Install it with `pip install pillow` on this host") with Image.open(src) as im: im.load() if im.mode not in ("RGBA", "RGB", "LA", "L"): @@ -268,7 +417,7 @@ def editorial_upload(body: dict = Body(default=None), from PIL import Image except ImportError: raise err(503, "pillow_missing", - "image validation needs Pillow — `pip install pillow` on this host") + "image validation needs Pillow. Install it with `pip install pillow` on this host") try: with Image.open(io.BytesIO(blob)) as im: im.load() @@ -293,7 +442,7 @@ def editorial_upload(body: dict = Body(default=None), _assets().put_editorial(slug, blob, ext, buf.getvalue()) except Exception: raise err(503, "asset_repo_unavailable", - "the asset repo did not accept the upload — try again") + "the asset repo did not accept the upload. Try again") return {"slug": slug, "source": "repo"} @@ -368,7 +517,7 @@ def record_upload(body: dict = Body(default=None), from PIL import Image except ImportError: raise err(503, "pillow_missing", - "image validation needs Pillow — `pip install pillow` on this host") + "image validation needs Pillow. Install it with `pip install pillow` on this host") try: with Image.open(io.BytesIO(blob)) as im: im.load() @@ -396,7 +545,7 @@ def record_upload(body: dict = Body(default=None), _assets().put_record(tenant, asset_id, blob, ext, buf.getvalue()) except Exception: raise err(503, "asset_repo_unavailable", - "the asset repo did not accept the upload — try again") + "the asset repo did not accept the upload. Try again") return {"ref": f"rec:{asset_id}", "source": "repo"} diff --git a/api/routes_automation.py b/api/routes_automation.py index 927e72ba076ef18038ce6bf885658c84029a5053..46e99fbe2f741cd4621ca9a0ff644aa2a97122c3 100644 --- a/api/routes_automation.py +++ b/api/routes_automation.py @@ -12,6 +12,8 @@ one finite worker immediately. No shared tick secret or polling clock remains. import os import re import time +import hashlib +import json from datetime import datetime, timezone from fastapi import APIRouter, Body, Depends, Header, Request @@ -20,6 +22,7 @@ import ai_review import automation_control import automation_engine as engine import voice_control_plane as voice +import voice_provider_adapters as voice_adapters import oauth_connect import routes_oauth import scope_cache @@ -54,6 +57,34 @@ MODULE = "automation" _GATE = module_gate(MODULE) +def _governed_query_tool(session, database=""): + """Adapt C4 dry-runs to B's mounted C2 Query door without owning Query authority.""" + def invoke(prompt, context): + query_route = getattr(__import__("routes_query"), "governed_query", None) + if not callable(query_route): + raise ValueError("governed Query seam is unavailable until its mounted route is present") + context = context if isinstance(context, dict) else {} + result = query_route({ + "database": str(context.get("database") or database or ""), + "prompt": str(prompt or ""), + }, session) + if not isinstance(result, dict): + raise ValueError("governed Query returned no typed result") + # C's no-dispatch dry-run intentionally sends no selected fields or resume token. B's + # governed door must therefore stop at its authenticated clarification boundary, before + # its planner/provider path. Any other result fails closed instead of becoming a fallback. + if result.get("status") != "clarification": + raise ValueError("governed Query dry-run did not stop before provider execution") + alignment = result.get("alignmentQuestion") + question = str((alignment or {}).get("question") or "") if isinstance(alignment, dict) else "" + if not question: + raise ValueError("governed Query clarification is incomplete") + return {"state": "clarification", "question": question, + "providerCalls": 0, "noExternalDispatch": True} + + return invoke + + def _voice_view_revision(view): """One stable revision string for a server-owned saved View choice.""" if not isinstance(view, dict): @@ -194,7 +225,7 @@ def voice_agent_options(session: Session = Depends(_GATE)): @router.post("/voice-agents") def create_voice_agent(body: dict = Body(default=None), session: Session = Depends(_GATE)): try: - saved = voice.save_draft(session.runtime, body or {}, session.uname) + saved = voice.save_draft(session.runtime, body or {}, session.uname, tenant_id=session.tenant) row = voice.get_browser_dto(session.runtime, saved["id"]) row["audienceOptions"] = _voice_options(session) return {"voiceAgent": row} @@ -234,6 +265,116 @@ def voice_dry_run(voice_id: str, body: dict = Body(default=None), session: Sessi raise err(400 if "unknown" not in str(exc) else 404, "voice_refused", str(exc)) +@router.post("/voice-agents/{voice_id}/whatsapp/telnyx/dry-run") +def telnyx_whatsapp_dry_run(voice_id: str, request: Request, body: dict = Body(default=None), + session: Session = Depends(_GATE)): + """Recorded-fixture proof only. No public webhook or provider transport is mounted.""" + body = body or {} + payload = body.get("payload") + headers = body.get("headers") if isinstance(body.get("headers"), dict) else {} + verifier = getattr(request.app.state, "telnyx_verify_signature", None) + def query_tool(prompt, context): + # The provider adapter invokes this only after signature/timestamp verification. + ready = ((engine.agent_contract_dto(session.runtime, voice_id) or {}) + .get("semanticState", {}).get("retrievable") or []) + return _governed_query_tool( + session, (ready[0] if ready else {}).get("database"))(prompt, context) + if not callable(verifier): + raise err(503, "voice_seam_unavailable", "verified Telnyx signature seam is unavailable") + try: + return voice.whatsapp_query_dry_run( + session.runtime, voice_id, payload, + headers, verifier, query_tool) + except (ValueError, voice.VoiceRefusal, voice_adapters.ProviderAdapterRefusal) as exc: + raise err(400, "whatsapp_fixture_refused", str(exc)) + + +@router.post("/voice-agents/{voice_id}/whatsapp/telnyx/receipt") +async def telnyx_whatsapp_receipt(voice_id: str, request: Request, + ): + """Production receipt door: exact raw body/header verification before JSON parsing.""" + raw = await request.body() + signature = request.headers.get("telnyx-signature-ed25519", "") + timestamp = request.headers.get("telnyx-timestamp", "") + try: + if not voice.verify_telnyx_raw_request(raw, signature, timestamp): + raise voice.VoiceRefusal("Telnyx webhook signature verification failed") + payload = json.loads(raw.decode("utf-8")) + from harness import runtime as _rt + matches = [] + for slug in _rt.known_tenants(): + try: + rt = _rt.get_runtime(slug) + row = (voice._store(rt).get("voice_agent") or {}).get(str(voice_id)) + if isinstance(row, dict) and str(row.get("tenant_id") or slug) == str(slug): + matches.append((slug, rt, row)) + except Exception: + continue + if len(matches) != 1: + raise voice.VoiceRefusal("Voice Agent receipt binding is absent or ambiguous") + slug, rt, agent_row = matches[0] + owner = str(agent_row.get("created_by") or "") + service = _voice_owner_session(slug, rt, owner) + engine.authorize_agent_principal(rt, voice_id, service.uname, service.admin) + contract = engine.agent_contract_dto(rt, voice_id) or {} + workflow = next(iter(contract.get("workflows") or []), {}) + ready = (contract.get("semanticState") or {}).get("retrievable") or [] + if not workflow or not ready: + raise voice.VoiceRefusal("Voice Agent has no approved governed Query field") + database, field = ready[0].get("database"), ready[0].get("field") + walls = _allowed_agent_databases(service) + if str(field) not in walls.get(str(database), set()): + raise voice.VoiceRefusal("Voice Agent field is outside the current owner wall") + return voice.whatsapp_query_dry_run( + rt, voice_id, payload, + {"telnyx-signature-ed25519": signature, "telnyx-timestamp": timestamp}, + lambda *_: True, + lambda prompt, context: engine.governed_agent_query_dry_run( + rt, voice_id, workflow.get("id"), database, field, prompt, + _governed_query_tool(service), + actor_databases=walls.keys(), actor_fields=walls.get(str(database), set()))["query"], + verification="received_raw_callback") + except (ValueError, PermissionError, voice.VoiceRefusal, voice_adapters.ProviderAdapterRefusal) as exc: + raise err(400, "whatsapp_receipt_refused", str(exc)) + + +@router.post("/voice-agents/{voice_id}/feedback/consent") +def feedback_consent(voice_id: str, body: dict = Body(default=None), + session: Session = Depends(_GATE)): + body = body or {} + try: + return {"consent": voice.set_feedback_consent( + session.runtime, voice_id, body.get("contactRef"), body.get("state"), body.get("purpose"))} + except voice.VoiceRefusal as exc: + raise err(400, "feedback_refused", str(exc)) + + +@router.post("/voice-agents/{voice_id}/feedback/schedules") +def create_feedback_schedule(voice_id: str, body: dict = Body(default=None), + session: Session = Depends(_GATE)): + body = body or {} + try: + row = voice.create_feedback_schedule( + session.runtime, voice_id, body.get("contactRefs"), body.get("localTime") or "09:00", + body.get("timezone") or "UTC") + return {"schedule": voice.feedback_schedule_dto(session.runtime, row["id"]), + "adapter": voice.future_integration_adapter_contract()} + except voice.VoiceRefusal as exc: + raise err(400, "feedback_refused", str(exc)) + + +@router.post("/voice-agents/{voice_id}/feedback/schedules/{schedule_id}/dry-run") +def feedback_schedule_dry_run(voice_id: str, schedule_id: str, + body: dict = Body(default=None), + session: Session = Depends(_GATE)): + body = body or {} + try: + return voice.feedback_dry_run( + session.runtime, voice_id, schedule_id, body.get("day")) + except voice.VoiceRefusal as exc: + raise err(400, "feedback_refused", str(exc)) + + def _sync_schedule_or_503(tenant, defn): """Apply the exact native schedules after a definition write, never silently drift.""" try: @@ -254,6 +395,7 @@ def _wire(defn, tenant, rt=None): if live: status = {**status, "state": "running", "startedAt": live.get("startedAt"), "step": live.get("step")} + agent_contract = engine.agent_contract_dto(rt, defn.get("id")) if rt else None return { "id": defn.get("id"), "name": defn.get("name"), "kind": defn.get("kind"), "config": defn.get("config") or {}, "schedule": sched, "status": status, @@ -342,6 +484,18 @@ def _wire(defn, tenant, rt=None): # an absent prop the client silently reads as undefined # [[flag-shipped-without-its-writer]]. "agentActions": (_agent_marks(rt).get(str(defn.get("id") or "")) or {}) if rt else {}, + **({ + "privateEventStore": agent_contract["eventStore"], + "workflows": agent_contract["workflows"], + "sharing": agent_contract["sharing"], + "capabilities": agent_contract["capabilities"], + "semanticState": agent_contract["semanticState"], + "setupInterview": agent_contract["setupInterview"], + "providerStatus": {"configured": False, "referencesMasked": True}, + } if agent_contract else { + "agentAvailability": {"state": "disabled", "reason": + "This system Agent is configured at its source."} + }), } @@ -1837,6 +1991,8 @@ def create_automation(body: dict = Body(default=None), session: Session = Depend notes=_notes) if error: raise err(400, "invalid_automation", error) + engine.ensure_agent_contract(session.runtime, defn.get("id"), session.uname, + defn.get("name") or "") _sync_schedule_or_503(session.tenant, defn) return {"automation": _wire(defn, session.tenant, rt=session.runtime), "notes": _notes} @@ -1883,6 +2039,8 @@ def patch_automation(auto_id: str, body: dict = Body(default=None), raise err(400 if error != "no such automation" else 404, "invalid_automation" if error != "no such automation" else "unknown_automation", error) + engine.ensure_agent_contract(session.runtime, defn.get("id"), session.uname, + defn.get("name") or "") # An action the user removed takes its annotation with it, or the mark outlives its subject # and refuses an id nobody can see. _prune_marks(session, auto_id) @@ -1890,6 +2048,142 @@ def patch_automation(auto_id: str, body: dict = Body(default=None), return {"automation": _wire(defn, session.tenant, rt=session.runtime), "notes": _notes} +def _stored_agent_or_404(session, agent_id): + defn = engine.all_definitions(session.runtime).get(str(agent_id)) + if not isinstance(defn, dict): + voice_row = (voice._store(session.runtime).get("voice_agent") or {}).get(str(agent_id)) + if not isinstance(voice_row, dict): + raise err(404, "unknown_agent", "there is no stored Agent with that id") + defn = {"id": str(agent_id), "createdBy": voice_row.get("created_by") or session.uname, + "name": voice_row.get("display_name") or "Voice Agent"} + engine.ensure_agent_contract(session.runtime, agent_id, defn.get("createdBy") or session.uname, + defn.get("name") or "") + contract = engine.agent_contract_dto(session.runtime, agent_id) or {} + sharing = contract.get("sharing") or {} + permitted = (session.admin or contract.get("agentId") and + (defn.get("createdBy") == session.uname or + session.uname in set(sharing.get("sharedWith") or []))) + if not permitted: + raise err(403, "agent_principal_refused", "this Agent is private or not shared with you") + return defn + + +def _allowed_agent_databases(session): + allowed = {} + for row in automation_tables(session).get("tables") or []: + key = str(row.get("key") or "") + if key: + allowed[key] = {str(field.get("key") or "") for field in row.get("fields") or [] + if isinstance(field, dict) and field.get("key")} + try: + session.require("customer_data") + from routes_customers import grid_assembly + customer = grid_assembly(session, consume_corrections=False) + allowed["customer_data"] = {str(field.get("key") or "") + for field in customer.get("fields") or [] + if isinstance(field, dict) and field.get("key")} + except Exception: # noqa: BLE001 + pass + return allowed + + +def _voice_owner_session(tenant, rt, owner): + import core.users as user_registry + rows = user_registry.registry() or {} + raw = rows.get(str(owner or "")) if isinstance(rows, dict) else None + if not isinstance(raw, dict) or raw.get("active", True) is False: + raise voice.VoiceRefusal("Voice Agent owner is inactive or unavailable") + user = user_registry._public(str(owner), raw) + if str(user.get("tenant") or "").strip().lower() != str(tenant).strip().lower(): + raise voice.VoiceRefusal("Voice Agent owner is not in this tenant") + return Session(str(tenant), user, {}, rt) + + +@router.get("/agents/{agent_id}/contract") +def get_agent_contract(agent_id: str, session: Session = Depends(_GATE)): + _stored_agent_or_404(session, agent_id) + return {"agent": engine.agent_contract_dto(session.runtime, agent_id)} + + +@router.post("/agents/{agent_id}/workflows") +def add_agent_workflow(agent_id: str, body: dict = Body(default=None), + session: Session = Depends(_GATE)): + _stored_agent_or_404(session, agent_id) + try: + row = engine.create_agent_workflow(session.runtime, agent_id, + (body or {}).get("name"), + (body or {}).get("purpose")) + except ValueError as exc: + raise err(400, "workflow_refused", str(exc)) + return {"workflow": row, "agent": engine.agent_contract_dto(session.runtime, agent_id)} + + +@router.patch("/agents/{agent_id}/governance") +def patch_agent_governance(agent_id: str, body: dict = Body(default=None), + session: Session = Depends(_GATE)): + _stored_agent_or_404(session, agent_id) + try: + import core.users as user_registry + engine.configure_agent_governance(session.runtime, agent_id, body or {}, + _allowed_agent_databases(session), + [row.get("username") for row in + user_registry.assignable_identities(session.tenant)]) + except PermissionError as exc: + raise err(403, "agent_scope_refused", str(exc)) + except ValueError as exc: + raise err(400, "agent_governance_refused", str(exc)) + return {"agent": engine.agent_contract_dto(session.runtime, agent_id)} + + +@router.post("/agents/{agent_id}/setup/answer") +def answer_agent_setup(agent_id: str, body: dict = Body(default=None), + session: Session = Depends(_GATE)): + _stored_agent_or_404(session, agent_id) + try: + interview = engine.answer_setup_interview( + session.runtime, agent_id, (body or {}).get("key"), + (body or {}).get("answer"), bool((body or {}).get("disable"))) + except ValueError as exc: + raise err(400, "setup_interview_refused", str(exc)) + return {"setupInterview": interview, + "agent": engine.agent_contract_dto(session.runtime, agent_id)} + + +@router.post("/agents/{agent_id}/governed-query/dry-run") +def agent_governed_query_dry_run(agent_id: str, request: Request, body: dict = Body(default=None), + session: Session = Depends(_GATE)): + """C4's production door: all scope/readiness checks run before the injected C2 Query seam.""" + _stored_agent_or_404(session, agent_id) + body = body or {} + walls = _allowed_agent_databases(session) + table, field = body.get("table"), body.get("field") + query_tool = _governed_query_tool(session) + try: + return engine.governed_agent_query_dry_run( + session.runtime, agent_id, body.get("workflowId"), table, + field, body.get("prompt"), query_tool, actor_databases=walls.keys(), + actor_fields=walls.get(str(table), set())) + except PermissionError as exc: + raise err(403, "agent_scope_refused", str(exc)) + except ValueError as exc: + raise err(400, "agent_query_refused", str(exc)) + + +@router.put("/agents/{agent_id}/semantic-readiness") +def set_agent_semantic_readiness(agent_id: str, body: dict = Body(default=None), + session: Session = Depends(_GATE)): + _stored_agent_or_404(session, agent_id) + walls = _allowed_agent_databases(session) + rows = (body or {}).get("fields") + if not isinstance(rows, list) or any( + str(row.get("database")) not in walls or + str(row.get("field")) not in walls.get(str(row.get("database")), set()) + for row in rows if isinstance(row, dict)): + raise err(403, "agent_scope_refused", "semantic readiness exceeds the acting user's field wall") + return {"agent": engine.agent_contract_dto( + session.runtime, agent_id) if engine.set_semantic_readiness(session.runtime, agent_id, rows) else None} + + @router.delete("/automations/{auto_id}") def delete_automation(auto_id: str, session: Session = Depends(_GATE)): """Delete the DEFINITION. ⚠ The database it filled is NOT touched — an automation is the diff --git a/api/routes_connectors.py b/api/routes_connectors.py index 25bafcd1158e54d207444bff1a56c86d08e7870b..b02517e9f09177856bc1b590271247216e2e1c83 100644 --- a/api/routes_connectors.py +++ b/api/routes_connectors.py @@ -22,10 +22,10 @@ from __future__ import annotations import os -from fastapi import APIRouter, Depends +from fastapi import APIRouter, Body, Depends import oauth_connect -from deps import Session, require_session +from deps import Session, err, require_session #: ⛔ W32-T10 — IMPORTED, NEVER RE-DERIVED. "Does the environment serve THIS tenant's Odoo?" is #: one question, and this module used to answer it with an unguarded `os.environ.get("ODOO_URL")` #: while `routes_keychain` answered it correctly. Importing the correct one is the fix; writing a @@ -614,3 +614,87 @@ def connectors_directory(session: Session = Depends(require_session)): "counts": {"connected": sum(1 for r in rows if r["state"] == "connected"), "available": sum(1 for r in rows if r["state"] == "available"), "planned": sum(1 for r in rows if r["state"] == "planned")}} + + +_ODOO_SCHEMA_ENABLEMENTS = "odoo_schema_enablements" +_ODOO_SYNC_STATUS = "odoo_sync_status" + + +def _odoo_admin(session): + if not session.admin: + raise err(403, "not_admin", "only an administrator can inspect the Odoo schema") + + +def _schema_catalog(session): + """Rows-free Odoo field catalogue; source fields remain discoverable when every value is empty.""" + import core.user_tables as user_tables + enabled = session.runtime.get(_ODOO_SCHEMA_ENABLEMENTS) or {} + snapshot = user_tables.odoo_schema_snapshot(session.runtime) + raw_models = snapshot.get("models") or {} + if raw_models: + rows = [] + for model, raw_fields in raw_models.items(): + for raw_name, meta in (raw_fields or {}).items(): + key = f"{model}.{raw_name}" + rows.append({"key": key, "model": model, "rawName": raw_name, + "label": str(meta.get("string") or raw_name), + "type": str(meta.get("type") or "text"), "source": "Odoo", + "emptyDiscoverable": True, "defaultEnabled": False, + "enabled": enabled.get(key) is True}) + return sorted(rows, key=lambda row: row["label"].casefold()) + # Never present compiled mirror declarations as an authoritative fields_get catalogue. + # Until a sync captures metadata, the schema is unavailable and enablement stays refused. + return [] + + +@router.get("/connectors/odoo/status") +def odoo_status(session: Session = Depends(require_session)): + _odoo_admin(session) + import odoo_relational + cfg = odoo_relational.read_config(session.runtime) + stored = session.runtime.get(_ODOO_SYNC_STATUS) or {} + errors = stored.get("errors") if isinstance(stored.get("errors"), list) else [] + phase = str(stored.get("phase") or "unknown") + return {"schedule": cfg.get("syncEvery"), + "health": "paused" if cfg.get("frozen") else "error" if errors else phase, + "lastRun": stored.get("lastRun"), "errors": errors[:20], + "frozen": cfg.get("frozen") is True} + + +@router.get("/connectors/odoo/schema") +def odoo_schema(session: Session = Depends(require_session)): + _odoo_admin(session) + fields = _schema_catalog(session) + import core.user_tables as user_tables + snapshot = user_tables.odoo_schema_snapshot(session.runtime) + return {"fields": fields, "default": "off", "adminOnly": True, + "complete": snapshot.get("complete") is True, + "capturedAt": snapshot.get("capturedAt"), + "errors": list(snapshot.get("errors") or ()), "source": "Odoo"} + + +@router.patch("/connectors/odoo/schema/{field_key}") +def odoo_schema_enable(field_key: str, body: dict = Body(default=None), + session: Session = Depends(require_session)): + _odoo_admin(session) + import core.user_tables as user_tables + _snapshot = user_tables.odoo_schema_snapshot(session.runtime) + if _snapshot.get("complete") is not True: + raise err(409, "odoo_schema_unavailable", + "run a successful schema sync before changing raw-field enablement") + known = {field["key"] for field in _schema_catalog(session)} + key = str(field_key or "").strip() + if key not in known: + raise err(404, "unknown_odoo_field", "that field is not in the Odoo schema catalogue") + enabled = (body or {}).get("enabled") + if not isinstance(enabled, bool): + raise err(400, "bad_enabled", "enabled must be true or false") + session.runtime.update( + _ODOO_SCHEMA_ENABLEMENTS, + lambda current: {**(current if isinstance(current, dict) else {}), key: enabled}, + flush="sync") + for cache_key in list(getattr(session.runtime, "pool_cache", {}) or {}): + if isinstance(cache_key, tuple) and cache_key[:1] == ("product_pool",): + session.runtime.pool_cache.pop(cache_key, None) + return {"key": key, "enabled": enabled, "source": "Odoo", + "permissionWall": "normal field permissions"} diff --git a/api/routes_customers.py b/api/routes_customers.py index 1db214db9671a852ac675554fd4c3beaa8757938..0554e2f40ec77740c762b8938ee06c1c5ff65f17 100644 --- a/api/routes_customers.py +++ b/api/routes_customers.py @@ -21,6 +21,7 @@ SOURCE OF TRUTH; it does not make the two runtimes coherent. The fix is X4/Postg owner-blocked on B-3), and no test here may claim read-your-writes ACROSS runtimes — a TestClient proof is single-process and would report green on exactly the thing that is still broken. """ +import datetime as _dt import hashlib import json import math @@ -519,6 +520,7 @@ def grid_assembly(session: Session, scope: str = "customer", storage_key: str = for pid, cells in _ov.items()} today = time.strftime("%Y-%m-%d") + fields = aios_grid.project_date_recurrences(fields, today=today) stamp = _pool_stamp(rt, team_id, agent) # ⭐⭐ W38-T19 — THE METRICS CAPABILITY, AND THIS GRAIN NEEDS **THREE** GUARDS WHERE THE # OTHER TWO NEED ONE. `routes_products` and `routes_tables` funnel their cells and their @@ -857,12 +859,18 @@ def _route_dto(route, allowed, active, session, share_mark=None): mark = share_mark if isinstance(share_mark, dict) else _route_share_mark(session, route) owns_route = bool(session.admin or ( mark.get("owner") and mark.get("owner") == str(session.uname or "").strip().lower())) + source_edit = _route_view_can_edit(session, route.get("sourceViewId")) + can_edit = bool(not compatibility and owns_route and source_edit) dto = {"id": route["id"], "name": route["name"], "color": route["color"], "number": route["number"], "visibleStopIds": visible, "outsideFilter": bool(active) and not bool(active.intersection(visible)), - "permission": {"canEdit": (False if compatibility else owns_route and - _route_view_can_edit(session, route.get("sourceViewId"))), + "permission": {"canEdit": can_edit, "sourceViewId": route.get("sourceViewId")}, + "capabilities": route_system.route_capabilities( + route, can_edit=can_edit, can_duplicate=source_edit, + can_delete=can_edit), + "assignee": route.get("assignee"), "date": route.get("date"), + "starred": route.get("starred") is True, "source": {"viewId": route.get("sourceViewId"), "revision": route.get("sourceViewRevision")}, "owner": mark["owner"], "shared": mark["shared"], @@ -876,16 +884,6 @@ def _route_dto(route, allowed, active, session, share_mark=None): return dto -def _query_pids(raw): - values = [] - for value in str(raw or "").split(","): - try: - values.append(int(value)) - except (TypeError, ValueError): - continue - return values - - def _route_or_404(session, route_id): route = next((item for item, _mark in _visible_normalized_routes(session) if item.get("id") == str(route_id)), None) @@ -930,21 +928,114 @@ def _route_nav_mutation_context(session): return routes, folders, placements +def _route_assignee_people(session, directory=None): + """Return the active same-tenant accounts that may open Customer data. + + This is deliberately narrower than the account registry: a people picker may not enumerate + another tenant, inactive users, or coworkers who cannot open the Route's own module. + ``directory`` is a focused test seam; production always reads the canonical registry. + """ + from core import perm_scope + import core.users as users + + tenant = str(getattr(session, "tenant", None) or "").strip().lower() + if not tenant: + return [] + injected = directory is not None + source = users.registry if directory is None else directory + try: + records = source() if callable(source) else source + except Exception: + return [] + if not isinstance(records, dict): + return [] + + people = {} + for raw_username, record in records.items(): + if not isinstance(record, dict) or record.get("active") is False: + continue + username = str(raw_username or "").strip().lower() + if not username: + continue + default_tenant = tenant if injected else "royal-imports" + account_tenant = str(record.get("tenant") or default_tenant).strip().lower() + if account_tenant != tenant: + continue + try: + principal = users._public(username, record) + if not perm_scope.nav_may_open(principal, MODULE, st=session.runtime): + continue + except Exception: + continue + people.setdefault(username, { + "username": username, + "name": str(record.get("name") or username).strip()[:120] or username, + }) + return sorted(people.values(), key=lambda item: (item["name"].casefold(), item["username"])) + + +def _route_assignee_or_400(session, raw, *, directory=None, + code="unknown_route_assignee", + message="choose a permitted person for this Route"): + username = str(raw or "").strip().lower() + people = _route_assignee_people(session, directory=directory) + if not username or username not in {person["username"] for person in people}: + # Non-enumerating by design: the refusal does not reveal whether an account is inactive, + # belongs to another tenant, or merely lacks Customer data access. + raise err(400, code, message) + return username + + +def _route_schedule_patch_or_400(session, raw, *, directory=None): + """Normalize an all-or-nothing assignee/date pair in-place.""" + has_assignee, has_date = "assignee" in raw, "date" in raw + if has_assignee != has_date: + raise err(400, "route_schedule_pair", "assignee and date must be saved together") + if not has_assignee: + return raw + assignee = str(raw.get("assignee") or "").strip() + route_date = str(raw.get("date") or "").strip() + if bool(assignee) != bool(route_date): + raise err(400, "route_schedule_pair", "assignee and date must be saved together") + if not assignee: + raw["assignee"], raw["date"] = None, None + return raw + try: + canonical_date = _dt.date.fromisoformat(route_date).isoformat() + except ValueError: + raise err(400, "invalid_route_date", "choose a valid Route date") + if canonical_date != route_date: + raise err(400, "invalid_route_date", "choose a valid Route date") + raw["assignee"] = _route_assignee_or_400(session, assignee, directory=directory) + raw["date"] = canonical_date + return raw + + @router.get("/customers/routes") -def routes(visiblePids: str = "", currentPids: str = "", - session: Session = Depends(module_gate(MODULE))): - """Route picker DTO consumed by C1/C3; only permitted stops ever leave the server.""" +def routes(session: Session = Depends(module_gate(MODULE))): + """Constant-request Route catalogue; only permitted Route stops ever leave the server. + + Filter membership is deliberately not encoded as a comma-delimited pid query. A grid with + 50,000 visible records sends exactly the same request as an empty grid; a future annotation + door may accept a compact saved-View/filter reference without reopening that row-sized wire. + """ allowed = allowed_pids(session) - active = _query_pids(currentPids or visiblePids) records = _route_share_records(session) navigation = _route_navigation(session, _compatible_routes(session, allowed, records)) return {"routes": [_route_dto( - route, allowed, active, session, + route, allowed, (), session, _route_share_mark(session, route, records)) for route in navigation["routes"]], "folders": navigation["folders"]} +@router.get("/customers/routes/assignees") +def route_assignees(session: Session = Depends(module_gate(MODULE))): + from core import route_system + return {"today": route_system.tenant_day_for_runtime(session.runtime).isoformat(), + "people": _route_assignee_people(session)} + + @router.post("/customers/routes", status_code=201) def create_route(body: dict = Body(default=None), session: Session = Depends(module_gate(MODULE))): from core import route_system @@ -955,6 +1046,7 @@ def create_route(body: dict = Body(default=None), session: Session = Depends(mod permitted = allowed_pids(session) if any(pid not in permitted for pid in _route_input_pids(raw)): raise err(403, "out_of_scope", "a Route stop must be in your customer book") + _route_schedule_patch_or_400(session, raw) route = route_system.put_route(session.runtime, raw, tenant_id=session.tenant, created_by=session.uname) if route is None: raise err(400, "invalid_route", "a Route needs a name and an optional HTTPS Google Maps URL") @@ -996,6 +1088,7 @@ def update_route(route_id: str, body: dict = Body(default=None), raw["id"] = prior["id"] if "stops" in raw and any(pid not in allowed_pids(session) for pid in _route_input_pids(raw)): raise err(403, "out_of_scope", "a Route stop must be in your customer book") + _route_schedule_patch_or_400(session, raw) route = route_system.put_route(session.runtime, raw, tenant_id=session.tenant, created_by=session.uname) if route is None: raise err(400, "invalid_route", "a Route needs a name and an optional HTTPS Google Maps URL") @@ -1320,6 +1413,9 @@ def create_zone(body: dict = Body(default=None), session: Session = Depends(modu from core import route_system raw = dict(body or {}) _reject_client_create_id(raw, "Zone") + if "memberships" in raw or "memberPids" in raw: + raise err(400, "zone_membership_read_only", + "Zone membership is worked out from current record coordinates") if not _zone_can_edit(session, raw.get("sourceViewId")): raise err(403, "zone_view_read_only", "a Zone requires an editable source View") zone = route_system.put_zone(session.runtime, raw, rows=_pool_rows(session), @@ -1338,6 +1434,9 @@ def update_zone(zone_id: str, body: dict = Body(default=None), if prior is None: raise err(404, "zone_not_found", "that Zone does not exist") raw = dict(body or {}) + if "memberships" in raw or "memberPids" in raw: + raise err(400, "zone_membership_read_only", + "Zone membership is worked out from current record coordinates") # A Zone keeps the View it was authored from. Editing its colour or boundary is not a # second door for silently rebasing its permission wall onto another View. source_view_id = prior.get("sourceViewId") or raw.get("sourceViewId") @@ -1503,6 +1602,9 @@ def patch_customer(pid: int, body: dict = Body(default=None), if not updates: raise err(400, "empty_patch", "no fields to update") from core import route_system + if route_system.ZONE_FIELD_KEY in updates or "zone" in updates: + raise err(403, "zone_membership_read_only", + "Zone membership is worked out from the record's current location") retired = {str(key) for key, proof in (route_system.read(session.runtime).get("legacyMigrations") or {}).items() if isinstance(proof, dict) and proof.get("verified")} if retired.intersection(updates): @@ -2354,18 +2456,9 @@ def _route_delivery_recipient_or_400(session, raw, directory=None): consume it yet. Staging-only identities can extend this one helper without leaking fixture logic into Route persistence or falling through to a different account plane. """ - recipient = str(raw or "").strip() - try: - if directory is None: - import core.users as users - directory = users.registry - user = (directory() or {}).get(recipient) - except Exception: # registry unavailable is not permission to queue a phantom Inbox delivery - user = None - if not recipient or not isinstance(user, dict) or not user.get("active", True): - raise err(400, "unknown_route_recipient", - "choose an active account to receive this route in Inbox") - return recipient + return _route_assignee_or_400( + session, raw, directory=directory, code="unknown_route_recipient", + message="choose an active permitted account to receive this route in Inbox") def _route_view_revision(view_id, config): diff --git a/api/routes_geo.py b/api/routes_geo.py index c3b7ee46865f303dec8d71d84005d3b2c409690f..5776cd81a78c56a807fa3310a5c7d01a4d1a1975 100644 --- a/api/routes_geo.py +++ b/api/routes_geo.py @@ -244,12 +244,12 @@ def _clean_zone(raw, previous=None, *, zone_id=""): def _zone_wire(zone): wire = {key: zone.get(key) for key in ("id", "name", "color", "geometryGeoJson", "visible", - "geographicField", "zipCodes", "memberships", + "geographicField", "zipCodes", "createdBy", "createdAt")} # Storage keeps ids as strings because table records may originate in several backends; Map # selection is numeric `pid`s. The wire performs that one boundary conversion explicitly. members = [] - for raw in zone.get("memberships") or (): + for raw in zone.get("memberPids", zone.get("memberships")) or (): try: pid = int(raw) except (TypeError, ValueError): @@ -314,7 +314,7 @@ def _zone_context(session, raw_table_key): grid = product_assembly(session, consume_corrections=False) elif table_key.startswith("ut_"): from routes_tables import ut_assembly - grid = ut_assembly(session, table_key, consume_corrections=False, with_rows=False) + grid = ut_assembly(session, table_key, consume_corrections=False, with_rows=True) else: raise err(400, "invalid_zone_table", "choose a database before working with Zones") fields = {str(field.get("key") or "") for field in (grid.get("fields") or []) @@ -323,8 +323,8 @@ def _zone_context(session, raw_table_key): def _zone_subject(session, table_key, zone_id): - from core import user_tables - zone = next((item for item in user_tables.zones(table_key, st=session.runtime) + from core import route_system + zone = next((item for item in route_system.list_zones(session.runtime) if str(item.get("id") or "") == str(zone_id)), None) if not isinstance(zone, dict): raise err(404, "unknown_zone", "no Zone with that id") @@ -505,14 +505,14 @@ def geo_providers(session: Session = Depends(require_session)): @router.get("/geo/zones") def geo_zones(tableKey: str = Query(default=""), session: Session = Depends(require_session)): """The canonical table-scoped Zone DTOs a Map may draw or filter by.""" - from core import user_tables + from core import route_system table_key, _fields, pids, rows = _zone_context(session, tableKey) - zones = user_tables.zones(table_key, st=session.runtime) + zones = route_system.list_zones(session.runtime, rows) # A hidden Zone remains recoverable to its creator/admin, but cannot become visible through a # second table simply because row ids happen to collide there. visible = [zone for zone in zones if zone.get("visible", True) or session.admin or str(zone.get("createdBy") or "") == session.uname] - projected = [_zone_membership_union(zone, rows, pids) for zone in visible] + projected = [zone for zone in visible] return {"zones": [_zone_wire(zone) for zone in sorted(projected, key=lambda zone: (str(zone.get("name") or "").casefold(), str(zone.get("id") or "")))]} @@ -520,46 +520,50 @@ def geo_zones(tableKey: str = Query(default=""), session: Session = Depends(requ @router.post("/geo/zones") def geo_zone_create(body: dict = Body(default=None), session: Session = Depends(require_session)): """Persist one Zone in the existing table-scoped Zone store and field/filter contract.""" - from core import user_tables + from core import route_system body = body if isinstance(body, dict) else {} table_key, field_keys, pids, rows = _zone_context(session, body.get("tableKey")) try: - zone = _clean_zone(body) - zone = _zone_membership_union(zone, rows, pids, refuse_overflow=True) + if any(key in body for key in ("memberships", "memberPids", "member_pids")): + raise ValueError("Zone membership is derived from current permitted rows") + zone = route_system.put_zone(session.runtime, { + "id": body.get("id"), "name": body.get("name"), "color": body.get("color"), + "geometryGeoJson": body.get("geometryGeoJson"), "zipCodes": body.get("zipCodes"), + "visible": body.get("visible"), "geographicField": body.get("geographicField") + }, rows=rows, tenant_id=session.tenant, created_by=session.uname) + if zone is None: + raise ValueError("the Zone does not match this database's fields and records") except ValueError as exc: raise err(400, "invalid_zone", str(exc)) if not zone["id"]: raise err(400, "invalid_zone", "name this Zone") - if any(str(item.get("id") or "") == zone["id"] - for item in user_tables.zones(table_key, st=session.runtime)): - raise err(409, "zone_exists", "a Zone with that name already exists") - saved = user_tables.upsert_zone(table_key, zone, field_keys=field_keys, - allowed_row_ids=pids, st=session.runtime, - created_by=session.uname) - if saved is None: - raise err(400, "invalid_zone", "the Zone does not match this database's fields and records") - return {"zone": _zone_wire(saved)} + return {"zone": _zone_wire(zone)} @router.patch("/geo/zones/{zone_id}") def geo_zone_update(zone_id: str, body: dict = Body(default=None), session: Session = Depends(require_session)): """Edit a table-scoped Zone while retaining creator, geometry and membership invariants.""" - from core import user_tables + from core import route_system body = body if isinstance(body, dict) else {} table_key, field_keys, pids, rows = _zone_context(session, body.get("tableKey")) previous = _zone_subject(session, table_key, zone_id) try: - zone = _clean_zone(body, previous, zone_id=str(zone_id)) - zone = _zone_membership_union(zone, rows, pids, refuse_overflow=True) + if any(key in body for key in ("memberships", "memberPids", "member_pids")): + raise ValueError("Zone membership is derived from current permitted rows") + zone = route_system.put_zone(session.runtime, { + "id": zone_id, "name": body.get("name", previous.get("name")), + "color": body.get("color", previous.get("color")), + "geometryGeoJson": body.get("geometryGeoJson", previous.get("geometryGeoJson")), + "zipCodes": body.get("zipCodes", previous.get("zipCodes")), + "visible": body.get("visible", previous.get("visible")), + "geographicField": body.get("geographicField", previous.get("geographicField")) + }, rows=rows, tenant_id=session.tenant, created_by=session.uname) + if zone is None: + raise ValueError("the Zone does not match this database's fields and records") except ValueError as exc: raise err(400, "invalid_zone", str(exc)) - saved = user_tables.upsert_zone(table_key, zone, field_keys=field_keys, - allowed_row_ids=pids, st=session.runtime, - created_by=session.uname) - if saved is None: - raise err(400, "invalid_zone", "the Zone does not match this database's fields and records") - return {"zone": _zone_wire(saved)} + return {"zone": _zone_wire(zone)} @router.delete("/geo/zones/{zone_id}") @@ -569,8 +573,8 @@ def geo_zone_delete(zone_id: str, body: dict = Body(default=None), body = body if isinstance(body, dict) else {} table_key, _fields, _pids, _rows = _zone_context(session, body.get("tableKey")) _zone_subject(session, table_key, zone_id) - from core import user_tables - if not user_tables.delete_zone(table_key, zone_id, st=session.runtime): + from core import route_system + if not route_system.delete_zone(session.runtime, zone_id): raise err(404, "unknown_zone", "no Zone with that id") return {"deleted": str(zone_id)} diff --git a/api/routes_grid.py b/api/routes_grid.py index 91fd673a8bef28b3ac26e6b8056a845a8e3c8353..40c4f26616a13043801b3c1300b1acdc8c9aa303 100644 --- a/api/routes_grid.py +++ b/api/routes_grid.py @@ -24,7 +24,7 @@ back. A 200 over a write that evaporated is the failure this rule exists to prev import datetime as dt import time -from fastapi import APIRouter, Body, Depends, HTTPException +from fastapi import APIRouter, Body, Depends, HTTPException, Query from deps import Session, err, module_gate, perms, require_session @@ -332,6 +332,174 @@ def workspace(scope: str = "customer", return {"workspace": workspace} +def _shared_note_subject(session, scope, view_id): + """Resolve a note subject only after the ordinary workspace wall admits it.""" + scope = _scope_or_400(scope) + payload = workspace(scope=scope, session=session)["workspace"] + if view_id: + view = next((item for item in (payload.get("views") or []) + if str(item.get("id") or "") == str(view_id)), None) + if view is None: + raise err(404, "view_not_found", "that View is not visible in this database") + return "view", f"{scope}:{view_id}", view + return "database", scope, None + + +@router.get("/grid/shared-note") +def get_shared_note(scope: str = "customer", viewId: str = "", + session: Session = Depends(require_session)): + """Return the one shared note attached to the open Database or View.""" + from core import user_tables + + kind, subject_id, _view = _shared_note_subject(session, scope, viewId) + return {"note": user_tables.shared_note(kind, subject_id, st=session.runtime)} + + +@router.put("/grid/shared-note") +def update_shared_note(scope: str = "customer", viewId: str = "", + body: dict = Body(default=None), + session: Session = Depends(require_session)): + """Edit a shared note through the same database/View permission wall as the grid.""" + from core import table_store, user_tables + + kind, subject_id, view = _shared_note_subject(session, scope, viewId) + may_edit = bool(session.admin) + if view is not None: + may_edit = table_store._may_edit(view, session.uname, session.admin) + elif scope.startswith("ut_"): + may_edit = user_tables.may_edit_database( + scope, session.uname, is_admin=session.admin, st=session.runtime) + if not may_edit: + raise err(403, "note_read_only", "your Database or View role does not allow note edits") + note = user_tables.put_shared_note( + kind, subject_id, (body or {}).get("body"), session.uname, st=session.runtime) + return {"note": note} + + +_RECURRENCE_CONDITION_OPS = frozenset({"eq", "neq", "isNotEmpty", "between"}) +_RECURRENCE_ORDERED_TYPES = frozenset({ + "date", "datetime", "int", "integer", "number", "currency", "pct", "rating", +}) +_RECURRENCE_NUMERIC_TYPES = frozenset({ + "int", "integer", "number", "currency", "pct", "rating", +}) + + +def _recurrence_target_fields(session, table_key): + """Permission-projected fields for a linked recurrence target.""" + target = str(table_key or "").strip() + mapped = {"customer_data": "customer", "product_data": "product"}.get(target, target) + if not (mapped in _SCOPES or mapped.startswith("ut_")): + raise ValueError("the linked recurrence database is not a supported visible database") + payload = workspace(scope=mapped, session=session)["workspace"] + return [field for field in (payload.get("fields") or []) + if isinstance(field, dict) and field.get("key")] + + +def _govern_recurrence_conditions(raw_conditions, current_fields): + """Validate each recurrence condition against the caller's visible field contract.""" + if raw_conditions in (None, []): + return [] + if not isinstance(raw_conditions, list) or len(raw_conditions) > 12: + raise ValueError("recurrence conditions must be a list of at most 12 field comparisons") + field_map = {str(field.get("key")): field for field in current_fields + if isinstance(field, dict) and field.get("key")} + normalized = [] + for raw in raw_conditions: + if not isinstance(raw, dict) or "children" in raw: + raise ValueError("recurrence conditions must be individual visible-field comparisons") + col_id = str(raw.get("colId") or "").strip() + field = field_map.get(col_id) + if field is None: + raise ValueError("a recurrence condition names a field this account cannot see") + op = "isNotEmpty" if raw.get("op") == "not_empty" else str(raw.get("op") or "") + if op not in _RECURRENCE_CONDITION_OPS: + raise ValueError("a recurrence condition uses an unsupported comparison") + field_type = str(field.get("type") or "text").lower() + if op == "between" and field_type not in _RECURRENCE_ORDERED_TYPES: + raise ValueError("between is available only for Date, Date-time, or numeric fields") + value = "" if raw.get("value") is None else str(raw.get("value"))[:500] + value2 = "" if raw.get("value2") is None else str(raw.get("value2"))[:500] + if op != "isNotEmpty" and not value.strip(): + raise ValueError("a recurrence condition needs a comparison value") + if op == "between" and not value2.strip(): + raise ValueError("a between recurrence condition needs both boundary values") + if field_type in _RECURRENCE_NUMERIC_TYPES and op in {"eq", "neq", "between"}: + try: + float(value) + if op == "between": + float(value2) + except ValueError: + raise ValueError("a numeric recurrence condition needs numeric values") + if field_type in {"date", "datetime"} and op in {"eq", "neq", "between"}: + values = (value, value2) if op == "between" else (value,) + for item in values: + try: + dt.date.fromisoformat(item[:10]) + except ValueError: + raise ValueError("a Date recurrence condition needs ISO date values") + normalized.append({"colId": col_id, "op": op, "value": value, + "value2": value2 if op == "between" else ""}) + return normalized + + +def govern_date_recurrence(session, scope, current_fields, raw): + """One authority for preview and every externally reachable recurrence write door.""" + import aios_grid + + if raw is None: + return None + if not isinstance(raw, dict): + raise ValueError("recurrence must be an object") + raw_rules = raw.get("rules") + if not isinstance(raw_rules, list) or not raw_rules or len(raw_rules) > 12: + raise ValueError("recurrence needs between 1 and 12 repeat rules") + fields = [field for field in (current_fields or []) + if isinstance(field, dict) and field.get("key")] + current = {str(field.get("key")): field for field in fields} + anchor = raw.get("anchorField") + if not isinstance(anchor, dict): + raise ValueError("recurrence needs a same or linked Date/Date-time anchor") + table = str(anchor.get("table") or "same").strip() + field_key = str(anchor.get("field") or "").strip() + linked_via = str(anchor.get("linkedVia") or "").strip() + if not field_key: + raise ValueError("recurrence needs a Date or Date-time anchor field") + anchor_fields = current + if table == "same": + if linked_via: + raise ValueError("a same-database recurrence cannot name a Link field") + else: + link_field = current.get(linked_via) or {} + link = link_field.get("link") if isinstance(link_field.get("link"), dict) else {} + if not linked_via or str(link_field.get("type") or "") != "link" \ + or str(link.get("table") or "") != table: + raise ValueError("the linked Date anchor must use a visible Link field to that database") + anchor_fields = {str(field.get("key")): field + for field in _recurrence_target_fields(session, table)} + if str((anchor_fields.get(field_key) or {}).get("type") or "").lower() \ + not in {"date", "datetime"}: + raise ValueError("a recurrence anchor must be a visible Date or Date-time field") + + conditions = _govern_recurrence_conditions(raw.get("conditions"), fields) + candidate = dict(raw) + candidate["anchorField"] = {"table": table, "field": field_key, + **({"linkedVia": linked_via} if table != "same" else {})} + candidate["conditions"] = conditions + # Server-derived only. The generic finite-shape cleaner consumes this admission list but + # never persists it, so a browser-authored `conditionFields` member has no authority. + candidate["conditionFields"] = list(current) + clean = aios_grid.clean_date_recurrence(candidate) + if clean is None or clean.get("anchorField") != candidate["anchorField"]: + raise ValueError("recurrence needs an anchor and at least one supported repeat rule") + if len(clean.get("rules") or []) != len(raw_rules): + raise ValueError("one or more recurrence rules are malformed or unsupported") + if conditions: + if clean.get("conditions") != conditions: + raise ValueError("one or more recurrence conditions are malformed") + return clean + + # ─────────────────────────────────────────────── the TIME-SERIES channel (C-TS, 2026-08-02) _TS_BUCKETS = ("week", "month", "quarter", "year") _TS_MAX_BUCKETS = 120 @@ -740,6 +908,110 @@ def grid_calendar_metrics(body: dict = Body(default=None), scope: str = "custome return out +def _builtin_history_context(session, scope, *, restoring=False): + """Resolve history through the same assembly and write context as grid events.""" + from routes_customers import grid_assembly + scope = _scope_or_400(scope) + if scope == "product": + from routes_products import MODULE as product_module, product_assembly + session.require(product_module) + g = product_assembly(session, consume_corrections=False) + else: + session.require(MODULE) + g = grid_assembly(session, scope=scope, consume_corrections=False) + ctx = _ctx(session, g["fields"], g["pids"], scope_key=scope, + history_kind="restore" if restoring else "edit") + return scope, g, ctx + + +@router.get("/grid/history") +def builtin_history(scope: str = "customer", rowId: str = Query(default=""), + retentionDays: int = Query(default=30), + session: Session = Depends(require_session)): + from core import user_tables + import core.platform_admin as platform_admin + scope, g, ctx = _builtin_history_context(session, scope) + if rowId and (not rowId.isdigit() or int(rowId) not in g["pids"]): + raise err(403, "out_of_scope", "that row is not in this database") + try: + history = user_tables.value_history( + scope, rowId or None, st=session.runtime, retention_days=retentionDays, + extended_entitled=platform_admin.is_platform_admin(session.user)) + except PermissionError as exc: + raise err(403, "history_entitlement_required", str(exc)) + except ValueError as exc: + raise err(400, "bad_history_retention", str(exc)) + return user_tables.project_value_history(history, g["pids"], ctx.hidden_keys) + + +@router.post("/grid/history/{event_id}/restore") +def restore_builtin_history(event_id: str, scope: str = "customer", + body: dict = Body(default=None), + session: Session = Depends(require_session)): + from core import grid_events, user_tables + import core.platform_admin as platform_admin + body = body if isinstance(body, dict) else {} + scope, g, ctx = _builtin_history_context(session, scope, restoring=True) + row_id = str(body.get("rowId") or "").strip() + if not row_id: + raise err(400, "row_required", "choose a record to restore") + if not row_id.isdigit() or int(row_id) not in g["pids"]: + raise err(403, "out_of_scope", "that row is not in this database") + field_key = str(body.get("field") or "").strip() or None + if field_key and field_key in ctx.hidden_keys: + raise err(403, "field_hidden", "that field is not available to your account") + try: + days = int(body.get("retentionDays") or user_tables.HISTORY_RETENTION_DAYS) + except (TypeError, ValueError): + raise err(400, "bad_history_retention", "history retention must be 30 or 90 days") + entitled = platform_admin.is_platform_admin(session.user) + try: + updates = user_tables.history_restore_values( + scope, row_id, event_id, field_key=field_key, st=session.runtime, + retention_days=days, extended_entitled=entitled) + except PermissionError as exc: + raise err(403, "history_entitlement_required", str(exc)) + except ValueError as exc: + raise err(400, "bad_history_retention", str(exc)) + if updates is None: + raise err(404, "history_event_not_found", "that retained value version is unavailable") + if any(key in ctx.hidden_keys for key in updates): + raise err(403, "field_hidden", "that retained version includes a hidden field") + prior_history_ids = { + str(item.get("id")) for item in user_tables.value_history( + scope, row_id, st=session.runtime, retention_days=days, + extended_entitled=entitled).get("events", []) + } + stored_before = dict((((g.get("ws") or {}).get("overlays") or {}).get(row_id)) or {}) + try: + grid_events.handle_one( + {"id": f"restore:{event_id}:{time.time_ns()}", "type": "overlay_patch", + "pid": int(row_id), "updates": updates}, ctx) + except grid_events.StoreUnavailable: + raise err(503, "store_unavailable", "the retained version was not restored") + stored = dict(((grid_events.table_workspace( + ctx, allowed_pids=g["pids"], consume_corrections=False).get("overlays") or {}) + .get(row_id)) or {}) + landed = all((key not in stored if value is None else stored.get(key) == str(value)) + for key, value in updates.items()) + newest = user_tables.project_value_history(user_tables.value_history( + scope, row_id, st=session.runtime, retention_days=days, + extended_entitled=entitled), g["pids"], ctx.hidden_keys) + restore_event = next((item for item in newest["events"] + if item.get("kind") == "restore" + and str(item.get("id")) not in prior_history_ids), None) + changed_keys = { + key for key, value in updates.items() + if (key in stored_before if value is None else stored_before.get(key) != str(value)) + } + audited = (not changed_keys or (restore_event is not None + and changed_keys.issubset(set(restore_event.get("changes") or {})))) + if not landed or not audited: + raise err(403, "restore_refused", "the retained version is outside your write authority") + return {"ok": True, "rowId": int(row_id), "restoredFields": sorted(updates), + "event": restore_event} + + #: ⭐⭐ THE BULK CELL DOOR'S OWN CEILING, and it is REPORTED rather than silently enforced. The #: standing no-cap rule governs data read from a connected source; this is a WRITE of team-typed #: values, so a bound is right — but R6's second sentence still binds, which is why exceeding it @@ -856,13 +1128,19 @@ def recurrence_preview(body: dict = Body(default=None), """ import aios_grid - raw = (body or {}).get("recurrence") - clean = aios_grid.clean_date_recurrence(raw) - if clean is None: - raise err(400, "bad_recurrence", - "recurrence needs a start date and at least one supported repeat rule") + body = body if isinstance(body, dict) else {} + raw = body.get("recurrence") + scope = _scope_or_400(body.get("scopeKey")) + current = workspace(scope=scope, session=session)["workspace"] + try: + clean = govern_date_recurrence(session, scope, current.get("fields") or [], raw) + except ValueError as exc: + raise err(400, "bad_recurrence", str(exc)) return { - "recurrence": aios_grid.recurrence_payload(clean, today=time.strftime("%Y-%m-%d")), + "recurrence": aios_grid.recurrence_payload( + clean, today=time.strftime("%Y-%m-%d"), + condition_fields={str(field.get("key")) for field in current.get("fields") or [] + if isinstance(field, dict) and field.get("key")}), "calendars": list(aios_grid.PUBLIC_HOLIDAY_CALENDARS), } @@ -955,7 +1233,9 @@ def grid_events_route(body: dict = Body(default=None), resolved_ids=frozenset(g["measure_sets"]), cohort_ids=frozenset(c["id"] for c in g["lists"]), measure_offer=tuple(g["measures"]), - visible_views=tuple(g["views"])) + visible_views=tuple(g["views"]), + recurrence_validator=lambda raw: govern_date_recurrence( + session, scope, g["fields"], raw)) # Per-event results so the client can tell which of a batch landed — the component's own # bridge has no response channel at all, so this is strictly more than the embed gets. diff --git a/api/routes_nav.py b/api/routes_nav.py index 65ffa54eccfe4a22d1d5d6d86b60826db392b2de..94b5091a1faedf5b4a155a859d38d05a74bc4032 100644 --- a/api/routes_nav.py +++ b/api/routes_nav.py @@ -528,7 +528,8 @@ def nav(session: Session = Depends(require_session)): if not _perm_scope.nav_may_open(session.user, e["key"], st=_lent): continue pages.append({"key": e["key"], "label": e["label"], - "source": e.get("source") or "Blank", "chrome": "main"}) + "source": e.get("source") or "Blank", + "sourceGroup": e.get("sourceGroup") or "Custom", "chrome": "main"}) except Exception: # ⭐⭐ W31-T11 (owner item 6b) — STILL SWALLOWED, NO LONGER SILENT. # @@ -563,6 +564,14 @@ def nav(session: Session = Depends(require_session)): # leak with no symptom until two tenants rename the same registry key. Copy the rows before # merging on the day that invariant changes. meta = _read_nav_meta(session.runtime) + for page in pages: + source = str(page.get("source") or "") + key = str(page.get("key") or "") + if key in {"customer_data", "product_data"} or source.lower() == "odoo" \ + or key.startswith("ut_odoo_"): + page["sourceGroup"] = "Odoo" + elif page.get("user_table") or source in {"", "Blank", "Custom"}: + page.setdefault("sourceGroup", "Custom") # ⛔⛔ W35-T43 (rulings R4/R7) — THE MARK-IMPORTANT COUNTS ARE NO LONGER COMPUTED HERE. # # W34-T10 put them on this payload with a 2.5 s budget and a `degraded` entry, spent in RECENTS diff --git a/api/routes_platform_admin.py b/api/routes_platform_admin.py index d6556459e9883895c87807f39a9e2356ec6ae6f4..df1ea25ec8f74523f39adbee62c674041d5addfa 100644 --- a/api/routes_platform_admin.py +++ b/api/routes_platform_admin.py @@ -38,10 +38,12 @@ import sys import time from pathlib import Path -from fastapi import APIRouter, Depends +from fastapi import APIRouter, Body, Depends, Header, Request import core.platform_admin as platform_admin import core.store as store +import operator_control +import voice_control_plane as voice from deps import Session, err, require_session, users router = APIRouter(prefix="/api/v1/platform-admin") @@ -67,6 +69,131 @@ def padmin_gate(session: Session = Depends(require_session)) -> Session: return session +def _operator_origin(request: Request) -> str: + return str(request.headers.get("origin") or request.base_url).rstrip("/") + + +@router.post("/operator/grants") +def operator_grant(request: Request, body: dict = Body(default=None), + session: Session = Depends(padmin_gate)): + body = body or {} + try: + return operator_control.mint_grant( + session.runtime, actor=session.uname, tenant=body.get("tenant"), + capabilities=body.get("capabilities") or [], origin=_operator_origin(request), + audience=body.get("audience") or operator_control.AUDIENCE, + ttl_seconds=body.get("ttlSeconds") or 900) + except operator_control.OperatorRefusal as exc: + raise err(400, "operator_grant_refused", str(exc)) + + +@router.delete("/operator/grants/{grant_id}") +def operator_revoke(grant_id: str, session: Session = Depends(padmin_gate)): + try: + return {"grant": operator_control.revoke_grant(session.runtime, grant_id, session.uname)} + except operator_control.OperatorRefusal as exc: + raise err(404, "operator_grant_refused", str(exc)) + + +@router.get("/operator/audit") +def operator_audit(limit: int = 200, session: Session = Depends(padmin_gate)): + return {"audit": operator_control.audit_rows(session.runtime, limit)} + + +def _operator_authorize(request, session, token, tenant, capability, action): + try: + return operator_control.authorize( + session.runtime, token=token, actor=session.uname, tenant=tenant, + capability=capability, origin=_operator_origin(request), action=action) + except operator_control.OperatorRefusal as exc: + raise err(403, "operator_capability_refused", str(exc)) + + +def _operator_final(session, tenant, capability, action, outcome): + operator_control.audit_attempt(session.runtime, actor=session.uname, tenant=tenant, + capability=capability, action=action, outcome=outcome) + + +@router.get("/operator/tenants/{tenant}/setup") +def operator_read_setup(request: Request, tenant: str, + x_loopable_operator_grant: str = Header(default=""), + session: Session = Depends(padmin_gate)): + try: + _operator_authorize(request, session, x_loopable_operator_grant, tenant, + "tenant.read", "read tenant setup") + rt, problem = _runtime_for(tenant) + if rt is None: + raise err(404, "operator_tenant_unavailable", problem or "tenant unavailable") + out = operator_control.setup_dto(rt, tenant) + except Exception: + _operator_final(session, tenant, "tenant.read", "read tenant setup", "refused") + raise + _operator_final(session, tenant, "tenant.read", "read tenant setup", "allowed") + return out + + +@router.patch("/operator/tenants/{tenant}/setup") +def operator_change_setup(request: Request, tenant: str, body: dict = Body(default=None), + x_loopable_operator_grant: str = Header(default=""), + session: Session = Depends(padmin_gate)): + try: + _operator_authorize(request, session, x_loopable_operator_grant, tenant, + "tenant.configure", "change tenant setup") + rt, problem = _runtime_for(tenant) + if rt is None: + raise err(404, "operator_tenant_unavailable", problem or "tenant unavailable") + out = operator_control.update_setup(rt, tenant, body or {}, session.uname) + except operator_control.OperatorRefusal as exc: + _operator_final(session, tenant, "tenant.configure", "change tenant setup", "refused") + raise err(400, "operator_setup_refused", str(exc)) + except Exception: + _operator_final(session, tenant, "tenant.configure", "change tenant setup", "refused") + raise + _operator_final(session, tenant, "tenant.configure", "change tenant setup", "allowed") + return out + + +@router.get("/operator/tenants/{tenant}/voice-setup") +def operator_read_voice_setup(request: Request, tenant: str, + x_loopable_operator_grant: str = Header(default=""), + session: Session = Depends(padmin_gate)): + try: + _operator_authorize(request, session, x_loopable_operator_grant, tenant, + "voice.read", "read voice setup") + rt, problem = _runtime_for(tenant) + if rt is None: + raise err(404, "operator_tenant_unavailable", problem or "tenant unavailable") + out = {"tenant": tenant, "voice": voice.browser_options(rt), + "secretsPresent": False, "rawProviderBindings": False} + except Exception: + _operator_final(session, tenant, "voice.read", "read voice setup", "refused") + raise + _operator_final(session, tenant, "voice.read", "read voice setup", "allowed") + return out + + +@router.patch("/operator/tenants/{tenant}/voice-setup") +def operator_change_voice_setup(request: Request, tenant: str, body: dict = Body(default=None), + x_loopable_operator_grant: str = Header(default=""), + session: Session = Depends(padmin_gate)): + try: + _operator_authorize(request, session, x_loopable_operator_grant, tenant, + "voice.configure", "change voice setup") + rt, problem = _runtime_for(tenant) + if rt is None: + raise err(404, "operator_tenant_unavailable", problem or "tenant unavailable") + saved = operator_control.update_setup(rt, tenant, body or {}, session.uname) + except operator_control.OperatorRefusal as exc: + _operator_final(session, tenant, "voice.configure", "change voice setup", "refused") + raise err(400, "operator_setup_refused", str(exc)) + except Exception: + _operator_final(session, tenant, "voice.configure", "change voice setup", "refused") + raise + _operator_final(session, tenant, "voice.configure", "change voice setup", "allowed") + return {**saved, "voiceConfigurationAccepted": True, + "secretsPresent": False, "rawProviderBindings": False} + + # ══════════════════════════════════════════════════════════════════════════ tenants diff --git a/api/routes_products.py b/api/routes_products.py index b831da983ea2d64e83ee1c08f8484ae623f97ee1..0e06befacb291d78fafb13fa1eb39347f41ea2ab 100644 --- a/api/routes_products.py +++ b/api/routes_products.py @@ -19,6 +19,9 @@ the product ops). ⛔ The separate bucket is load-bearing, not tidy: product pid hashes and customer pids are Odoo partner ids — one shared overlay bucket and a hash collision silently writes a product note onto somebody's customer. """ +import html +import json +import re import time from fastapi import Body, Depends @@ -26,6 +29,7 @@ from fastapi import Body, Depends from fastapi import APIRouter import scope_cache +from core import grid_events, user_tables from deps import Session, err, module_gate router = APIRouter(prefix="/api/v1") @@ -51,6 +55,8 @@ def _pool_for(rt, team_id): return [] import modules.product_data as pd + from core import user_tables + enabled_raw = user_tables.enabled_odoo_schema_fields(rt, model="product.product") # ⚠ Signature is (cache, key, ttl, build) — the wave-15 version passed `ttl=` as a keyword # after the build lambda and DIED on every call ("multiple values for 'ttl'"). Latent until @@ -91,8 +97,16 @@ def _pool_for(rt, team_id): "this data source is paused and nothing has been read since — " "an admin can resume it under Settings → Connectors") - return scope_cache.get(rt.pool_cache, key, _CACHE_TTL, - lambda: pd.pool(team_id=team_id)) + def build(): + # Preserve the long-standing pool call shape when no raw field is enabled. Besides + # keeping existing adapters valid, this avoids paying for an unnecessary `extra_fields` + # branch on the overwhelmingly common/default-off path. + rows = (pd.pool(team_id=team_id, + raw_fields=[field.get("rawName") for field in enabled_raw]) + if enabled_raw else pd.pool(team_id=team_id)) + return _project_enabled_raw_rows(rows, enabled_raw, drop_source=True) + + return scope_cache.get(rt.pool_cache, key, _CACHE_TTL, build) def scoped_pool(session: Session): @@ -129,6 +143,12 @@ def scoped_pool(session: Session): # so such a leaf denied every product row while the admin was told the rule saved. See # `perm_scope._enrich_for_wall`. This route is BOTH walls at once (it returns the pids the # write door uses), so there is no second call here to keep in step. + enabled_raw = user_tables.enabled_odoo_schema_fields(session.runtime, model="product.product") + field_keys = {field.get("key") for field in fields_base if isinstance(field, dict)} + fields_base = list(fields_base) + [ + field for field in enabled_raw if field.get("key") not in field_keys + ] + rows_src = _project_enabled_raw_rows(rows_src, enabled_raw, drop_source=True) rows_src = perm_scope.apply_row_scope(rows_src, session.user, MODULE, fields_base, st=session.runtime) pids = frozenset(r["pid"] for r in rows_src if r.get("pid") is not None) @@ -332,6 +352,50 @@ def _hydrate_effective_choice_options(fields, rows_src, overlays, derived): fields, effective_rows, ignored_by_field={"supplier": {"(none)"}}) +def _project_enabled_raw_rows(rows, enabled, *, drop_source=False): + """Namespace enabled Odoo values as bounded grid scalars before the permission wall.""" + def scalar(value, raw_type): + if raw_type == "boolean": + return bool(value) + if value is None or value is False: + return "" + if raw_type in {"many2one", "reference"}: + if isinstance(value, (list, tuple)): + value = value[1] if len(value) > 1 else (value[0] if value else "") + elif isinstance(value, dict): + value = value.get("display_name") or value.get("name") or value.get("id") or "" + elif raw_type in {"one2many", "many2many"}: + if isinstance(value, (list, tuple, set)): + value = ", ".join(str(item) for item in value) + elif raw_type == "binary": + # A binary field is discoverable and enableable, but a grid row never transports + # its base64 body. Presence is the only safe scalar representation at this seam. + return "[binary value]" if value else "" + elif raw_type == "html": + value = re.sub(r"<[^>]*>", " ", html.unescape(str(value))) + value = re.sub(r"\s+", " ", value).strip() + if isinstance(value, (dict, list, tuple, set)): + value = json.dumps(value, ensure_ascii=False, sort_keys=True, default=str) + if isinstance(value, (str, bytes, bytearray)): + if isinstance(value, (bytes, bytearray)): + return "[binary value]" if value else "" + return value[:32000] + return value + + out = [] + for row in rows or (): + item = dict(row or {}) + for field in enabled or (): + key, name, model = field.get("key"), field.get("rawName"), field.get("model") + if key and name: + value = item.get(str(name), item.get(f"{model}.{name}")) + item[str(key)] = scalar(value, str(field.get("rawType") or "text")) + if drop_source and str(name) != str(key): + item.pop(str(name), None) + out.append(item) + return out + + def product_assembly(session: Session, scope: str = "product", storage_key: str = "", consume_corrections: bool = True): """The product topic's mirror of `routes_customers.grid_assembly` — SAME g-dict keys, so @@ -363,8 +427,6 @@ def product_assembly(session: Session, scope: str = "product", storage_key: str import aios_grid import core.perm_scope as perm_scope import modules.product_data as pd - from core import grid_events - pids, team_id, rows_src, fields_base = scoped_pool(session) shared_defs = pd.shared_fields(st=session.runtime) @@ -438,6 +500,7 @@ def product_assembly(session: Session, scope: str = "product", storage_key: str } today = time.strftime("%Y-%m-%d") + fields = aios_grid.project_date_recurrences(fields, today=today) stamp = _measure_stamp(session.runtime, team_id) measures = product_measures(session) diff --git a/api/routes_query.py b/api/routes_query.py index 4074e75e5689175ade2f4c5283126e081e6a18d9..8234862824e96e82afeea65680df4fb928b70885 100644 --- a/api/routes_query.py +++ b/api/routes_query.py @@ -7,12 +7,16 @@ citations and virtual views; it never opens or mutates a source workspace. import copy import csv import datetime as _dt +import base64 import hashlib +import hmac import io import json import os import re +import secrets import sys +import time import uuid from fastapi import APIRouter, Body, Depends, Response @@ -45,6 +49,9 @@ MAX_HISTORY_CHARS = 6000 MAX_RATING_REASON = 200 QUERY_RATINGS = ("up", "down") MODEL_AUTO = "auto" +QUERY_RESUME_TTL_SECONDS = 300 +_resume_env = os.environ.get("AIOS_QUERY_RESUME_SECRET") or os.environ.get("AIOS_SESSION_SECRET") or "" +_QUERY_RESUME_SECRET = (_resume_env.encode("utf-8") if _resume_env else secrets.token_bytes(32)) # ⭐⭐ WAVE 36 · W36-T35 (R4 / contract C5) — THE LADDER IS DECLARED IN ONE PLACE NOW. # # ⛔ THIS BLOCK USED TO BE A SECOND COPY OF THE LADDER, and it said so: *"A SECOND DECLARATION IS A @@ -60,6 +67,8 @@ MODEL_AUTO = "auto" # reconciles the two WITHOUT forcing the import, so a divergence is still reported and nobody pays # the eight seconds to hear about it. import providers as prov # noqa: E402 +import formula_runtime as formula_rt # noqa: E402 +import semantic_registry as semantic_rt # noqa: E402 #: The Messages API version header. ⚠ A DATE, NOT A MODEL: it pins the wire format, not the model, #: and Anthropic requires it on every request. @@ -192,7 +201,8 @@ def model_status(): rows_by_name = {row["provider"]: row for row in prov.llm_status()} live = [name for name, row in rows_by_name.items() if row["configured"] and row["capable"] and not row["outOfCredit"]] - out = [{"model": MODEL_AUTO, "available": bool(live), + out = [{"model": MODEL_AUTO, "provider": "automatic", "modelId": "automatic", + "available": bool(live), "reason": "" if live else "no model can be reached from this deployment right now", "toolCalling": True, "jsonMode": True, "outOfCredit": False}] for name, row in rows_by_name.items(): @@ -204,11 +214,13 @@ def model_status(): reason = f"{row['label']} cannot carry this kind of request" else: reason = "" - out.append({"model": name, "available": name in live, "reason": reason, + out.append({"model": name, "provider": name, "modelId": row.get("model") or name, + "available": name in live, "reason": reason, "toolCalling": row["toolCalling"], "jsonMode": row["jsonMode"], "outOfCredit": row["outOfCredit"]}) # Anything the ANALYST's ladder carries that Query does not offer, once a chat has taught us. - out.extend({"model": name, "available": False, "toolCalling": False, "jsonMode": False, + out.extend({"model": name, "provider": name, "modelId": name, + "available": False, "toolCalling": False, "jsonMode": False, "outOfCredit": False, "reason": "the platform can reach this model but Query does not offer it yet"} for name in _ladder_drift()) @@ -231,6 +243,11 @@ def _providers(model=MODEL_AUTO): "model": p.model, "wire": p.wire} for p in chain] +def _model_identity(provider): + row = next((item for item in prov.llm_status() if item.get("provider") == provider), None) + return {"provider": provider, "modelId": (row or {}).get("model") or provider} + + def _history(state, thread_id): """The prior turns of THIS thread, oldest first, in the transport's own message shape. @@ -795,7 +812,7 @@ def _validate(spec, fields): return None, "the assistant gave an unsupported aggregation", "bad_aggregation" if op == "count": field = None - elif field not in keys or by_key.get(field) not in {"int", "currency", "pct"}: + elif field not in keys or by_key.get(field) not in {"int", "currency", "pct", "formula"}: return None, "that aggregation needs one visible numeric field", "bad_aggregation" display = {"mode": kind} @@ -968,6 +985,8 @@ def _public_view(view): "explain": view["explain"], "threadId": view["threadId"], "createdAt": view["createdAt"], "virtual": True, "source": _safe(source), "view": _safe(view["view"]), "citationIds": list(view["citationIds"]), "numeric": _safe(view["numeric"]), + "provider": view.get("provider") or view.get("model"), + "modelId": view.get("modelId") or view.get("model"), # ⚠ AN ARTEFACT MADE BEFORE THIS WAVE HAS NO ORIGINAL, so it reports `edited: false` and # sends no `original_spec` — and the client shows neither the badge nor Revert. C4 is # explicit that this is the right answer: a Revert with nothing to revert to is worse @@ -1303,12 +1322,14 @@ def _submit(body, session, chat=None): # ⚠ `_safe(view)` twice, not the same object twice: `view` is mutable and a # shared reference would let an edit rewrite the original through the alias. "original_spec": _safe(view), - "model": provider, "requestedModel": selected_model} + "model": provider, **_model_identity(provider), + "requestedModel": selected_model} assistant_message = { "id": assistant_message_id, "threadId": thread_id, "role": "assistant", "createdAt": now, "content": (said or _said_fallback(artifact)) if artifact else (refusal or "the assistant could not answer"), "targetDatabase": target, "requestedModel": selected_model, "model": provider, + **_model_identity(provider), "reason": refusal_code, "viewId": artifact["id"] if artifact else None, "citationIds": artifact["citationIds"] if artifact else [], "numeric": artifact["numeric"] if artifact else None, } @@ -1350,6 +1371,485 @@ def submit_chat(body: dict = Body(default=None), session: Session = Depends(requ return _submit(body, session) +# Governed Query contract (Wave 45 C2). The existing chat workspace remains the presentation +# surface; this provider-neutral door is the shared execution seam for Assistant, Agents, voice, +# Slack and future adapters. +def _semantic_baseline(database): + from harness import semantic as sem + topic = next((row for row in sem.topics().values() + if isinstance(row, dict) and row.get("grid") == database), None) + if not topic: + raise err(404, "semantic_source_unknown", + "that database does not have a semantic contract") + return semantic_rt.topic_records(database, topic) + + +def _semantic_records(session, database): + return semantic_rt.load(session.runtime, _semantic_baseline(database)) + + +def _semantic_authorize(session, database, *, require_answerable=False): + status = assistant_source_status(session, [database]).get(database) or {} + if not status.get("permitted"): + raise err(403, "assistant_forbidden", + "your current permissions do not allow this database") + if require_answerable and not status.get("answerable"): + raise err(409, "query_source_unavailable", + str(status.get("reason") or "this database cannot answer Query yet")) + return status + + +@router.get("/query/semantics/{database}") +def semantic_catalog(database: str, session: Session = Depends(require_session)): + # Permission is resolved by the same source boundary Query uses. An empty field selection is + # a schema-only authorization check and never widens to records. + _semantic_authorize(session, database) + records = _semantic_records(session, database) + rows = [copy.deepcopy(row) for key, row in sorted(records.items()) + if key.startswith(f"{database}::")] + for row in rows: + row["markdown"] = semantic_rt.generated_markdown(row) + return {"database": database, "fields": rows, + "router": semantic_rt.router_projection(records, database)} + + +@router.patch("/query/semantics/{database}/{field}") +def update_semantic_field(database: str, field: str, body: dict = Body(default=None), + session: Session = Depends(require_session)): + _semantic_authorize(session, database) + try: + row = semantic_rt.update(session.runtime, _semantic_baseline(database), database, field, + body, actor=session.uname, is_admin=session.admin) + except semantic_rt.SemanticError as exc: + code = "forbidden" if "only a tenant admin" in str(exc) else "bad_semantic_contract" + raise err(403 if code == "forbidden" else 400, code, str(exc)) + row["markdown"] = semantic_rt.generated_markdown(row) + return {"field": row} + + +@router.post("/query/semantics/{database}/{field}/profile") +def profile_semantic_field(database: str, field: str, + session: Session = Depends(require_session)): + """Profile values through Query's permissioned snapshot and persist only a bounded summary.""" + _semantic_authorize(session, database, require_answerable=True) + baseline = _semantic_baseline(database) + records = semantic_rt.load(session.runtime, baseline) + try: + current = records[semantic_rt.field_key(database, field)] + except KeyError: + raise err(404, "semantic_field_unknown", "that field is not in the database contract") + if not session.admin: + raise err(403, "forbidden", "only a tenant admin can profile semantic values") + # Sensitive fields are redacted without reading one value. This order is part of the + # contract: changing sensitivity cannot turn a profiling request into a disclosure. + values = [] + if current.get("sensitivity") not in ("sensitive", "restricted"): + snapshot = assistant_read_scope(session, database, fields=[field], filters=None) + values = [row.get(field) for row in (snapshot.get("records") or ())] + try: + row = semantic_rt.set_value_profile( + session.runtime, baseline, database, field, values, + actor=session.uname, is_admin=session.admin) + except semantic_rt.SemanticError as exc: + raise err(400, "bad_semantic_contract", str(exc)) + row["markdown"] = semantic_rt.generated_markdown(row) + return {"field": row} + + +def _resume_b64(raw): + return base64.urlsafe_b64encode(raw).decode("ascii").rstrip("=") + + +def _resume_unb64(value): + encoded = str(value) + decoded = base64.urlsafe_b64decode(encoded + "=" * (-len(encoded) % 4)) + if _resume_b64(decoded) != encoded: + raise ValueError("resume token encoding is not canonical") + return decoded + + +def _resume_choices(approved): + return [{"field": row["field"], "label": row["label"]} + for row in list(approved.values())[:8]] + + +def _mint_resume_token(session, database, question, choices, now=None): + stamp = int(time.time() if now is None else now) + claims = {"v": 1, "tenant": str(session.tenant), "user": str(session.uname), + "database": str(database), "prompt": str(question), + "choices": copy.deepcopy(choices), "issuedAt": stamp, + "expiresAt": stamp + QUERY_RESUME_TTL_SECONDS} + payload = json.dumps(claims, sort_keys=True, separators=(",", ":")).encode("utf-8") + signature = hmac.new(_QUERY_RESUME_SECRET, payload, hashlib.sha256).digest() + return f"{_resume_b64(payload)}.{_resume_b64(signature)}" + + +def _verify_resume_token(token, session, database, question, approved, requested, now=None): + """Authenticate one bounded clarification continuation before Query reads anything.""" + refused = lambda: err(403, "query_resume_refused", + "that clarification cannot be resumed; ask Query again") + if not isinstance(token, str) or token.count(".") != 1: + raise refused() + encoded, supplied = token.split(".", 1) + try: + payload = _resume_unb64(encoded) + signature = _resume_unb64(supplied) + except Exception: + raise refused() from None + expected = hmac.new(_QUERY_RESUME_SECRET, payload, hashlib.sha256).digest() + if not hmac.compare_digest(signature, expected): + raise refused() + try: + claims = json.loads(payload.decode("utf-8")) + issued, expires = int(claims["issuedAt"]), int(claims["expiresAt"]) + except (KeyError, TypeError, ValueError, UnicodeDecodeError, json.JSONDecodeError): + raise refused() from None + stamp = int(time.time() if now is None else now) + choices = _resume_choices(approved) + expected_claims = {"v": 1, "tenant": str(session.tenant), "user": str(session.uname), + "database": str(database), "prompt": str(question), "choices": choices} + if (any(claims.get(key) != value for key, value in expected_claims.items()) + or expires - issued != QUERY_RESUME_TTL_SECONDS + or stamp < issued or stamp >= expires + or not requested + or not set(requested).issubset({row["field"] for row in choices})): + raise refused() + return claims + + +def _clarification(session, database, question, approved, now=None): + choices = _resume_choices(approved) + return {"status": "clarification", "alignmentQuestion": { + "id": "align_fields", "question": "Which approved fields should this answer use?", + "choices": choices, "maxSelections": 8}, + "resumeToken": _mint_resume_token(session, database, question, choices, now=now)} + + +def _metric_dtos(offer, catalog): + """Project compatible registered Metrics; channel variants are not registered Metric keys.""" + out, seen = [], set() + for row in offer or (): + if not isinstance(row, dict): + continue + key = str(row.get("key") or "") + base = str(row.get("base_key") or key) + metric = catalog.get(base) if isinstance(catalog, dict) else None + if not key or base in seen or not isinstance(metric, dict): + continue + source = str(row.get("topic") or metric.get("topic") or "") + if metric.get("topic") != source: + continue + seen.add(base) + out.append({"key": base, "label": str(metric.get("label") or base), + "description": str(metric.get("description") or ""), + "format": str(metric.get("format") or ""), "source": source, + "approved": True}) + return out + + +def _compatible_metric_dtos(session, database): + from core import perm_scope + if not perm_scope.may_metrics(session.user, database): + return [] + from harness import semantic as sem + topic = sem.topic_for_grid(database) + if not topic: + return [] + return _metric_dtos(sem.entity_measures(topic), sem.metrics()) + + +@router.get("/query/metrics") +def query_metrics(database: str = "", session: Session = Depends(require_session)): + database = str(database or "").strip() + if not database: + raise err(400, "bad_request", "a database is required") + _semantic_authorize(session, database) + _semantic_records(session, database) # The same governed semantic source wall as Query. + return {"metrics": _compatible_metric_dtos(session, database)} + + +def _tool_spec(body, requested): + supplied = body.get("view") + if isinstance(supplied, dict): + return copy.deepcopy(supplied) + hidden = {str(value) for value in (body.get("hiddenFields") or ())} + group = body.get("group") + group_by = (str(group.get("field") or group.get("by") or "") + if isinstance(group, dict) else str(group or "")) + sorts = [] + for row in body.get("sorts") or (): + if not isinstance(row, dict): + continue + sorts.append({"colId": str(row.get("colId") or row.get("field") or ""), + "dir": "desc" if row.get("dir") == "desc" + or row.get("direction") == "desc" else "asc"}) + return {"kind": str(body.get("kind") or "grid"), "name": "Query", + "visible": [key for key in requested if key not in hidden], + "filters": copy.deepcopy(body.get("filters") or []), + "filterConj": "or" if body.get("filterConj") == "or" else "and", + "sorts": sorts, "groupBy": group_by or None, + "aggregation": copy.deepcopy(body.get("aggregation") or {"op": "count"})} + + +def _tool_sort(rows, sorts): + out = list(rows) + for rule in reversed(sorts or []): + key = rule.get("colId") + if not key: + continue + present = [row for row in out if row.get(key) is not None] + missing = [row for row in out if row.get(key) is None] + present.sort(key=lambda row: ((0, float(row[key])) if isinstance(row.get(key), (int, float)) + and not isinstance(row.get(key), bool) + else (1, str(row.get(key)).casefold())), + reverse=rule.get("dir") == "desc") + out = present + missing + return out + + +def _tool_groups(rows, view): + key = view.get("groupBy") + if not key: + return [] + grouped = {} + for row in rows: + value = row.get(key) + grouped.setdefault(str(value) if value is not None else "(blank)", []).append(row) + return [{"value": value, "numeric": _numeric_result(view, members)} + for value, members in sorted(grouped.items(), key=lambda item: item[0].casefold())] + + +def _tool_wording(question, numeric, count, group_count=0): + value = numeric.get("value") + if isinstance(value, float) and value.is_integer(): + value = int(value) + shown = "blank" if value is None else (f"{value:,}" if isinstance(value, (int, float)) else str(value)) + grouped = f" in {group_count} groups" if group_count else "" + return (f"For {question}: {numeric.get('label') or 'Result'} is {shown} across " + f"{count:,} matching record" + ("" if count == 1 else "s") + grouped + ".") + + +def _tool_formula_snapshot(body, snapshot, requested): + definitions = body.get("formulas") + if definitions in (None, {}): + return snapshot, {} + if not isinstance(definitions, dict): + raise err(400, "bad_formula", "formulas must be an object keyed by approved field") + cleaned = {} + for field, definition in definitions.items(): + field = str(field or "") + if field not in requested or not isinstance(definition, dict): + raise err(400, "bad_formula", "every formula must target a requested approved field") + refs = list(definition.get("refs") or ()) + if not set(refs).issubset(set(requested)): + raise err(403, "semantic_field_unapproved", + "a formula references a field outside the approved request") + try: + formula_rt.evaluate(definition, {}, strict=True) + except formula_rt.FormulaError as exc: + raise err(400, "bad_formula", str(exc)) + cleaned[field] = copy.deepcopy(definition) + projected = copy.deepcopy(snapshot) + projected_rows = [] + for row in snapshot.get("records") or (): + result = formula_rt.evaluate_fields(cleaned, row) + if any(reason == "cycle" for reason in result["errors"].values()): + raise err(400, "bad_formula", "formula dependencies contain a cycle") + projected_rows.append({**row, **result["values"]}) + projected["records"] = projected_rows + return projected, cleaned + + +def _tool_plan(question, body, snapshot, session): + """Use the existing governed planner only when a channel did not supply typed artifacts.""" + artifact_keys = {"view", "filters", "sorts", "group", "hiddenFields", "aggregation", "kind"} + if any(key in body for key in artifact_keys): + return _tool_spec(body, [field["key"] for field in snapshot.get("fields") or ()]), None, None + selected_model = str(body.get("model") or MODEL_AUTO).strip().lower() + if selected_model not in model_choices(): + raise err(400, "unknown_model", "that model is not available in Query") + spec, said, provider, reason = _call_model( + question, snapshot, model=selected_model, chat=None, history=None, session=session) + if spec is None: + return None, (said or "Query could not build an answer"), provider + return spec, said, provider + + +@router.post("/query/tool") +def governed_query(body: dict = Body(default=None), + session: Session = Depends(require_session)): + """Deterministic Query substrate; it never knows or calls a model/provider.""" + body = body if isinstance(body, dict) else {} + database = str(body.get("database") or "").strip() + question = " ".join(str(body.get("prompt") or body.get("question") or "").split()) + if not database or not question: + raise err(400, "bad_request", "a database and prompt are required") + if len(question) > MAX_QUESTION: + raise err(400, "question_too_long", f"a prompt must be under {MAX_QUESTION} characters") + _semantic_authorize(session, database, require_answerable=True) + records = _semantic_records(session, database) + approved = semantic_rt.approved_fields(records, database) + if not approved: + raise err(409, "query_semantics_unconfigured", + "this database has no approved semantic fields yet") + raw_requested = body.get("fieldKeys") + if raw_requested is not None and not isinstance(raw_requested, list): + raise err(400, "bad_request", "fieldKeys must be a list") + requested = list(dict.fromkeys(str(x) for x in (raw_requested or ()) if str(x))) + if not requested: + return _clarification(session, database, question, approved) + _verify_resume_token(body.get("resumeToken"), session, database, question, + approved, requested) + try: + selected = [semantic_rt.require_approved(records, database, key) for key in requested] + except semantic_rt.SemanticError as exc: + raise err(403, "semantic_field_unapproved", str(exc)) + snapshot = assistant_read_scope(session, database, fields=requested, filters=None) + snapshot, formulas = _tool_formula_snapshot(body, snapshot, requested) + spec, planned_wording, provider = _tool_plan(question, body, snapshot, session) + if spec is None: + return {"status": "refused", "wording": planned_wording, + "speakable": planned_wording, "artifacts": None, + "model": _model_identity(provider)} + view, refusal, refusal_code = _validate(spec, snapshot.get("fields") or selected) + if view is None: + raise err(400, refusal_code or "bad_query_artifact", refusal or "that Query is invalid") + filtered = _view_records(snapshot, view) + ordered = _tool_sort(filtered, view.get("sorts")) + rows = ordered[:50] + row_ids = [str(row.get("pid") or row.get("id") or index) + for index, row in enumerate(rows)] + hidden = [key for key in requested if key not in set(view.get("visible") or ())] + groups = _tool_groups(ordered, view) + numeric = _numeric_result(view, ordered) + artifacts = {"filters": _safe(view.get("filters") or []), + "sorts": _safe(view.get("sorts") or []), + "group": _safe(view.get("groupBy")), + "hiddenFields": hidden, + "formulas": _safe(formulas), + "source": {"database": database, "fields": requested, + "permissionScopeApplied": True}} + wording = _tool_wording(question, numeric, len(ordered), len(groups)) + return {"status": "answered", "wording": wording, "speakable": wording, + "citations": [{"database": database, "rowId": value} for value in row_ids], + "rows": [{key: row.get(key) for key in view["visible"]} for row in rows], + "groups": groups, "numeric": numeric, + "artifacts": artifacts, "semanticRouter": semantic_rt.router_projection(records, database), + "model": _model_identity(provider)} + + +@router.post("/query/formulas/parse") +def parse_formula(body: dict = Body(default=None), session: Session = Depends(require_session)): + try: + definition = formula_rt.parse_formula((body or {}).get("source")) + return {"contractVersion": formula_rt.CONTRACT_VERSION, + "astVersion": formula_rt.AST_VERSION, + "authority": formula_rt.AUTHORITY, + "definition": definition} + except formula_rt.FormulaError as exc: + raise err(400, "bad_formula", str(exc)) + + +@router.get("/query/formulas/fixtures") +def formula_fixtures(session: Session = Depends(require_session)): + return {**formula_rt.contract_dto(), + "transport": { + "parse": {"method": "POST", "path": "/api/v1/query/formulas/parse"}, + "evaluate": {"method": "POST", "path": "/api/v1/query/formulas/evaluate"}, + }} + + +@router.post("/query/formulas/evaluate") +def evaluate_formula(body: dict = Body(default=None), + session: Session = Depends(require_session)): + body = body if isinstance(body, dict) else {} + try: + if "definitions" in body or "rows" in body: + if "definition" in body or "row" in body: + raise formula_rt.FormulaError( + "single and batch formula payloads cannot be mixed") + return formula_rt.evaluate_batch( + body.get("definitions"), body.get("rows"), + consumer=str(body.get("consumer") or ""), today=body.get("today"), + field_revisions=body.get("fieldRevisions") or {}) + definition = body.get("definition") + if not isinstance(definition, dict): + raise formula_rt.FormulaError("a parsed formula definition is required") + if not isinstance(body.get("row", {}), dict): + raise formula_rt.FormulaError("formula row must be an object") + result = formula_rt.evaluate_result( + definition, body.get("row") or {}, today=body.get("today"), strict=True) + except formula_rt.FormulaError as exc: + raise err(400, "bad_formula", str(exc)) + return {"contractVersion": formula_rt.CONTRACT_VERSION, + "astVersion": formula_rt.AST_VERSION, + "authority": formula_rt.AUTHORITY, + **result, + "recalculation": formula_rt.explain_recalculation( + definition, body.get("fieldRevisions") or {})} + + +def validate_timeseries_contract(body, records=None, approved_metric_keys=None): + """Metric-first Time Series contract, pure so frontend fixtures can consume it.""" + body = body if isinstance(body, dict) else {} + series = body.get("series") if isinstance(body.get("series"), list) else [] + if not series: + raise ValueError("at least one series is required") + metric_keys = set(approved_metric_keys or ()) + first = series[0] if isinstance(series[0], dict) else {} + if first.get("kind") != "metric" or first.get("key") not in metric_keys: + raise ValueError("the first Time Series series must be an approved compatible Metric") + normalized = [{"kind": "metric", "key": first["key"]}] + for index, item in enumerate(series[1:], 2): + item = item if isinstance(item, dict) else {} + if item.get("kind") == "metric" and item.get("key") in metric_keys: + normalized.append({"kind": "metric", "key": item["key"]}) + continue + if item.get("kind") != "field" or not item.get("key") or item.get("aggregation") not in ( + "sum", "avg", "min", "max", "count", "count_distinct"): + raise ValueError(f"series {index} needs a Metric or a valid field aggregation") + if not item.get("dateField"): + raise ValueError(f"series {index} needs an explicit Date field") + if records is not None: + value_row = semantic_rt.require_approved( + records, body.get("database"), item.get("key")) + date_row = semantic_rt.require_approved(records, body.get("database"), item.get("dateField")) + if date_row.get("dataType") not in ("date", "datetime"): + raise ValueError(f"series {index} dateField is not a Date or Date-time") + if item.get("aggregation") in ("sum", "avg", "min", "max") and ( + value_row.get("dataType") not in ("int", "currency", "pct", "formula")): + raise ValueError(f"series {index} aggregation needs a numeric Field") + normalized.append({"kind": "field", "key": item["key"], + "aggregation": item["aggregation"], + "dateField": item["dateField"]}) + return {"version": 1, "series": normalized, + "formulaEvaluation": { + "contractVersion": formula_rt.CONTRACT_VERSION, + "astVersion": formula_rt.AST_VERSION, + "authority": formula_rt.AUTHORITY, + "consumer": "timeseries", + "method": "POST", + "path": "/api/v1/query/formulas/evaluate", + "mode": "batch", + }} + + +@router.post("/query/timeseries/validate") +def validate_timeseries(body: dict = Body(default=None), + session: Session = Depends(require_session)): + body = body if isinstance(body, dict) else {} + database = str(body.get("database") or "").strip() + if not database: + raise err(400, "bad_timeseries", "a database is required") + _semantic_authorize(session, database) + records = _semantic_records(session, database) + approved_metrics = {row["key"] for row in _compatible_metric_dtos(session, database)} + try: + return validate_timeseries_contract(body, records, approved_metrics) + except (ValueError, semantic_rt.SemanticError) as exc: + raise err(400, "bad_timeseries", str(exc)) + + @router.delete("/query/threads/{tid}") def delete_thread(tid: str, session: Session = Depends(require_session)): """Remove one of the caller's own chats, and only the chat. diff --git a/api/routes_starred.py b/api/routes_starred.py index c017e94c6788da13f98e01f1246786ea172a9c88..a86a16da0a4c4c1de9318a45012525ad90195b1c 100644 --- a/api/routes_starred.py +++ b/api/routes_starred.py @@ -44,7 +44,7 @@ router = APIRouter(prefix="/api/v1") _STARRED_KEY = "starred" #: The four kinds C2 declares. `view` is the odd one out — see the header. -KINDS = ("database", "agent", "query", "view") +KINDS = ("database", "agent", "query", "view", "application") #: The three kinds this file's own bucket holds, and the payload key each answers under. _LIST_KINDS = {"database": "databases", "agent": "agents", "query": "queries"} @@ -289,9 +289,20 @@ def _payload(session, keys=None, cache=None): if qresolved: lists["queries"] = [q for q in lists["queries"] if q in qnames] views, unread = _starred_views(session, keys, cache=cache) + try: + import application_registry + import routes_assets + application_databases = routes_assets._application_databases(session.runtime) + allowed_databases, _ = routes_assets.application_access_context( + session, application_databases) + application_rows = application_registry.list_apps( + session.runtime, session.uname, session.admin, starred=True, + allowed_databases=allowed_databases) + except Exception: + application_rows = [] out = {"databases": lists["databases"], "agents": lists["agents"], "queries": lists["queries"], "queryNames": qnames, - "views": views, "degraded": unread} + "views": views, "applications": application_rows, "degraded": unread} if unread: # R6's second sentence, in the payload: the CAUSE and what it means, not a bare list. out["note"] = (f"{len(unread)} database(s) could not be read in time, so a view you " @@ -497,6 +508,24 @@ def set_starred(body: dict = Body(default=None), cache = _refresh_one(session, cache, db_key) return _payload(session, cache=cache) + if kind == "application": + try: + import application_registry + import routes_assets + application_databases = routes_assets._application_databases(session.runtime, oid) + allowed_databases, _ = routes_assets.application_access_context( + session, application_databases) + if not allowed_databases: + raise application_registry.ApplicationError("application does not exist") + application_registry.mutate(session.runtime, oid, session.uname, session.admin, + {"starred": on}, allowed_databases=allowed_databases) + except PermissionError as exc: + raise err(403, "forbidden", str(exc)) + except Exception: + # Same 404 for absent and unreadable, so an id guess is not an oracle. + raise err(404, "unknown_application", "that Application is not available") + return _payload(session) + field = _LIST_KINDS[kind] uname = session.uname # ⛔ THE CAP IS ENFORCED INSIDE THE TRANSACTION, NOT BY A READ BEFORE IT, AND THAT IS NOT diff --git a/api/routes_tables.py b/api/routes_tables.py index 8ffa880e88c4e4aebb463c054203d8f1878bce9d..925ffb69eb61b674bff3b7e8e05e13b0ce7a4d59 100644 --- a/api/routes_tables.py +++ b/api/routes_tables.py @@ -1315,11 +1315,15 @@ def ut_assembly(session: Session, table_key: str, storage_key: str = "", # W43C C2: store compact recurrence configuration, then derive next/preview under the # request's server day only after every field stratum is merged. _recurrence_today = time.strftime("%Y-%m-%d") + _recurrence_keys = {str(field.get("key")) for field in fields + if isinstance(field, dict) and field.get("key")} def _wire_recurrence(field): if not isinstance(field, dict) or field.get("type") != "date": return field - payload = aios_grid.recurrence_payload(field.get("recurrence"), today=_recurrence_today) + payload = aios_grid.recurrence_payload( + field.get("recurrence"), today=_recurrence_today, + condition_fields=_recurrence_keys) return dict(field, recurrence=payload) if payload else field fields = [_wire_recurrence(field) for field in fields] @@ -1407,6 +1411,7 @@ def ut_assembly(session: Session, table_key: str, storage_key: str = "", # so the closure covers this user's `custom_` and `measure_` columns too. fields = _ut_contract_fields(table_key, fields, st=st) fields, rows_src, hidden = _ut_field_wall(session, table_key, fields, rows_src, st=st) + fields = aios_grid.project_date_recurrences(fields, today=_recurrence_today) if hidden and shared_cells: # ⛔ BOTH WIRES, AND THE SHARED STRATUM IS A THIRD ONE. `strip_row` above cleaned the # DEFINITION rows; these cells arrived by a different door and one narrowing cannot speak @@ -1923,9 +1928,22 @@ def create_table(body: dict = Body(default=None), if option_refusal: raise err(400, "refused", option_refusal) source = body.get("source") + raw_fields = list(body.get("fields") or []) + # Normalise the initial schema once without recurrence. Anchors and conditions are then + # checked against the fields that would actually survive this create, never the raw list. + base_fields = ut.clean_fields([ + {key: value for key, value in field.items() if key != "recurrence"} + for field in raw_fields if isinstance(field, dict) + ]) or [] + from routes_grid import govern_date_recurrence + recurrence_validator = lambda raw: govern_date_recurrence( + session, "same", base_fields, raw) try: - key = ut.create(label, session.uname, fields=body.get("fields"), - source=source, st=session.runtime) + key = ut.create(label, session.uname, fields=raw_fields, + source=source, st=session.runtime, + recurrence_validator=recurrence_validator) + except ValueError as exc: + raise err(400, "bad_recurrence", str(exc)) except Exception: raise err(503, "store_unavailable", "the tenant store refused the write. Nothing was created") @@ -2320,6 +2338,9 @@ def add_row(table_key: str, body: dict = Body(default=None), values = (body or {}).get("values") or {} if not isinstance(values, dict): raise err(400, "bad_values", "values must be an object of {fieldKey: value}") + if ut.ZONE_FIELD_KEY in values or "zone" in values: + raise err(400, "zone_membership_read_only", + "Zone membership is worked out from the record's current location") try: rid = ut.add_row(table_key, values, session.uname, st=session.runtime, rid=(body or {}).get("rid")) @@ -2373,6 +2394,9 @@ def import_rows(table_key: str, body: dict = Body(default=None), defn = _defn_or_refuse(session, table_key) by_key = {f["key"]: f for f in (defn.get("fields") or [])} asked = {k for r in rows_in for k in r} + if ut.ZONE_FIELD_KEY in asked or "zone" in asked: + raise err(400, "zone_membership_read_only", + "Zone membership is worked out from current record coordinates") unknown = sorted(k for k in asked if k not in by_key) if unknown: raise err(400, "unknown_field", @@ -2469,7 +2493,15 @@ def add_field(table_key: str, body: dict = Body(default=None), _managed_or_refuse(session, table_key, mutation=True) _field_or_refuse(session, table_key) ut = _ut() - field = ut.add_field(table_key, body or {}, st=session.runtime) + from routes_grid import govern_date_recurrence + visible_fields = ut_write_ctx(session, table_key)["fields"] + try: + field = ut.add_field( + table_key, body or {}, st=session.runtime, + recurrence_validator=lambda raw: govern_date_recurrence( + session, table_key, visible_fields, raw)) + except ValueError as exc: + raise err(400, "bad_recurrence", str(exc)) if not field: # ⭐ D-46 CLOSED (wave 23) — the C8 flow law gets its OWN sentence. `add_field` answers # None for every refusal, so this route said "check the name and type" to somebody whose @@ -2592,6 +2624,8 @@ def patch_field(table_key: str, fkey: str, body: dict = Body(default=None), _field_or_refuse(session, table_key, fkey) ut = _ut() body = body or {} + from routes_grid import govern_date_recurrence + visible_fields = ut_write_ctx(session, table_key)["fields"] if 'deletionLocked' in body and not session.admin: raise err(403, "owner_required", "only an Owner can change a field deletion lock") migrated = None @@ -2613,7 +2647,13 @@ def patch_field(table_key: str, fkey: str, body: dict = Body(default=None), migrated = dict(migrated or {}, workspace=True) except Exception: # noqa: BLE001 migrated = dict(migrated or {}, workspace=False) - field = ut.patch_field(table_key, fkey, body, st=session.runtime) + try: + field = ut.patch_field( + table_key, fkey, body, st=session.runtime, + recurrence_validator=lambda raw: govern_date_recurrence( + session, table_key, visible_fields, raw)) + except ValueError as exc: + raise err(400, "bad_recurrence", str(exc)) if not field: # C3: the same narrator as `add_field`. Flagging a SECOND column is the same act as # adding one, so it must get the same sentence naming the column that already holds the @@ -2878,6 +2918,9 @@ def patch_row(table_key: str, pid: int, body: dict = Body(default=None), updates = dict(body or {}) if not updates: raise err(400, "empty_patch", "no fields to update") + if _ut().ZONE_FIELD_KEY in updates or "zone" in updates: + raise err(403, "zone_membership_read_only", + "Zone membership is worked out from the record's current location") _managed_or_refuse(session, table_key, mutation=True) _records_or_refuse(session, table_key) g = ut_assembly(session, table_key, consume_corrections=False) @@ -2893,7 +2936,7 @@ def patch_row(table_key: str, pid: int, body: dict = Body(default=None), st=session.runtime, # R6b (D-16) scope_key=table_key, table=_ops(session, table_key)) try: - grid_events.handle_one( + refused = grid_events.handle_one( {"id": f"patch:{table_key}:{pid}:{time.time_ns()}", "type": "overlay_patch", "pid": pid, "updates": updates}, ctx) except grid_events.StoreUnavailable: @@ -2994,6 +3037,100 @@ def patch_row(table_key: str, pid: int, body: dict = Body(default=None), return out +@router.get("/tables/{table_key}/history") +def table_history(table_key: str, rowId: str = Query(default=""), + retentionDays: int = Query(default=30), + session: Session = Depends(require_session)): + g = ut_write_ctx(session, table_key) + if rowId and (not str(rowId).isdigit() or int(rowId) not in g["pids"]): + raise err(403, "out_of_scope", "that row is not in this database") + import core.platform_admin as platform_admin + try: + history = _ut().value_history( + table_key, rowId or None, st=session.runtime, retention_days=retentionDays, + extended_entitled=platform_admin.is_platform_admin(session.user)) + return _ut().project_value_history(history, g["pids"], g.get("hidden") or ()) + except PermissionError as exc: + raise err(403, "history_entitlement_required", str(exc)) + except ValueError as exc: + raise err(400, "bad_history_retention", str(exc)) + + +@router.post("/tables/{table_key}/history/{event_id}/restore") +def restore_history(table_key: str, event_id: str, body: dict = Body(default=None), + session: Session = Depends(require_session)): + from core import grid_events + + _managed_or_refuse(session, table_key, mutation=True) + _records_or_refuse(session, table_key) + body = body if isinstance(body, dict) else {} + row_id = str(body.get("rowId") or "").strip() + if not row_id: + raise err(400, "row_required", "choose a record to restore") + g = ut_write_ctx(session, table_key) + if not row_id.isdigit() or int(row_id) not in g["pids"]: + raise err(403, "out_of_scope", "that row is not in this database") + field_key = str(body.get("field") or "").strip() or None + if field_key and field_key in (g.get("hidden") or ()): + raise err(403, "field_hidden", "that field is not available to your account") + import core.platform_admin as platform_admin + try: + days = int(body.get("retentionDays") or _ut().HISTORY_RETENTION_DAYS) + except (TypeError, ValueError): + raise err(400, "bad_history_retention", "history retention must be 30 or 90 days") + entitled = platform_admin.is_platform_admin(session.user) + try: + updates = _ut().history_restore_values( + table_key, row_id, event_id, field_key=field_key, + st=session.runtime, retention_days=days, + extended_entitled=entitled) + except PermissionError as exc: + raise err(403, "history_entitlement_required", str(exc)) + except ValueError as exc: + raise err(400, "bad_history_retention", str(exc)) + if updates is None: + raise err(404, "history_event_not_found", "that retained value version is unavailable") + if any(key in (g.get("hidden") or ()) for key in updates): + raise err(403, "field_hidden", "that retained version includes a hidden field") + prior_history_ids = { + str(item.get("id")) for item in _ut().value_history( + table_key, row_id, st=session.runtime, retention_days=days, + extended_entitled=entitled).get("events", []) + } + stored_before = dict((((_ut().get(table_key, st=session.runtime) or {}).get("rows") or {}) + .get(row_id)) or {}) + ctx = grid_events.EventCtx( + uname=session.uname, allowed_pids=g["pids"], fields=g["fields"], + admin=session.admin, fallback_ws=None, seen_ids={}, + hidden_keys=g.get("hidden") or frozenset(), history_kind="restore", + st=session.runtime, scope_key=table_key, table=_ops(session, table_key)) + try: + grid_events.handle_one( + {"id": f"restore:{event_id}:{time.time_ns()}", "type": "overlay_patch", + "pid": int(row_id), "updates": updates}, ctx) + except grid_events.StoreUnavailable: + raise err(503, "store_unavailable", "the retained version was not restored") + stored = dict((((_ut().get(table_key, st=session.runtime) or {}).get("rows") or {}) + .get(row_id)) or {}) + landed = all((key not in stored if value is None else stored.get(key) == str(value)) + for key, value in updates.items()) + newest = _ut().project_value_history(_ut().value_history( + table_key, row_id, st=session.runtime, retention_days=days, + extended_entitled=entitled), g["pids"], g.get("hidden") or ()) + restore_event = next((item for item in newest["events"] + if item.get("kind") == "restore" + and str(item.get("id")) not in prior_history_ids), None) + changed_keys = { + key for key, value in updates.items() + if (key in stored_before if value is None else stored_before.get(key) != str(value)) + } + audited = (not changed_keys or (restore_event is not None + and changed_keys.issubset(set(restore_event.get("changes") or {})))) + if not landed or not audited: + raise err(403, "restore_refused", "the retained version is outside your write authority") + return {"row": stored, "event": restore_event} + + # ── CONTRACT C1 (W36-T20): THE MIRROR READER ────────────────────────────────────────────────── # ⭐⭐ R6. `core.perm_scope.scoped_table` is the ONE door to any database's rows and it answers for # a MATERIALISED `ut_*` table entirely on its own — deliberately, so a cold process (E's sandbox diff --git a/api/routes_templates.py b/api/routes_templates.py index ce0413eec0e0352b881be77c543dd00bd85e1fa7..a0672f727930df2de281d08d87da1aa437fa9393 100644 --- a/api/routes_templates.py +++ b/api/routes_templates.py @@ -84,6 +84,23 @@ def list_templates(table: str = "", session: Session = Depends(require_session)) return {"table": table, "templates": vt.offer(fields, source or None)} +@router.get("/templates/application-defaults") +def application_defaults(table: str = "", session: Session = Depends(require_session)): + """Deterministic starting Application for a new View, by type. + + The View creator consumes only the returned reference. It never clones the Application's + config into the View, so two Views remain attached to one reusable asset. + """ + _target_or_refuse(session, table) + import application_registry + return {"table": table, "defaults": { + kind: application_registry.resolve_default( + session.runtime, table, kind, session.uname, session.admin, + allowed_databases={table}) + for kind in application_registry.APPLICATION_TYPES + }} + + @router.post("/templates/{key}/apply") def apply_template(key: str, body: dict = Body(default=None), session: Session = Depends(require_session)): @@ -112,7 +129,7 @@ def apply_template(key: str, body: dict = Body(default=None), raise err(404, "unknown_table", "that database does not exist") if not session.runtime.available(): raise err(503, "store_unavailable", - "the tenant store is unavailable — nothing was applied") + "the tenant store is unavailable. Nothing was applied") import core.table_store as table_store ops = table_store.make(ws_key, st=session.runtime) applied = [] diff --git a/api/semantic_registry.py b/api/semantic_registry.py new file mode 100644 index 0000000000000000000000000000000000000000..73d8ba247edaeb82107dcfc9fbdbcaae4d6092e8 --- /dev/null +++ b/api/semantic_registry.py @@ -0,0 +1,287 @@ +"""Governed, tenant-scoped semantic metadata. + +Structured records are authoritative. Markdown and value summaries are projections of those +records and can never add information that was not admitted by the governance door here. +""" +from __future__ import annotations + +import copy +import hashlib +import json +from collections import Counter + + +SEMANTIC_STATES = ("draft", "approved", "disabled") +SENSITIVITY = ("public", "internal", "sensitive", "restricted") +VALUE_PROFILE_CAP = 50 +VALUE_SAMPLE_CAP = 12 +_KEY = "semantic_registry_v1" + + +class SemanticError(ValueError): + """A semantic contract was refused before persistence or retrieval.""" + + +def _text(value, limit=500): + return " ".join(str(value or "").split())[:limit] + + +def _strings(values, *, limit=20): + out = [] + for value in values if isinstance(values, (list, tuple)) else (): + item = _text(value, 80) + if item and item.casefold() not in {x.casefold() for x in out}: + out.append(item) + if len(out) >= limit: + break + return out + + +def field_key(database, field): + return f"{_text(database, 120)}::{_text(field, 120)}" + + +def model_record(database, topic, field): + """Build the governed baseline from a topic field declaration. + + Existing topic files predate approval. They therefore enter as ``draft`` unless the + declaration explicitly says otherwise; merely having a label never grants retrieval. + """ + field = field if isinstance(field, dict) else {} + semantic = field.get("semantic") if isinstance(field.get("semantic"), dict) else {} + state = str(semantic.get("state") or "draft").lower() + sensitivity = str(semantic.get("sensitivity") or "internal").lower() + if state not in SEMANTIC_STATES: + raise SemanticError(f"unknown semantic state {state!r}") + if sensitivity not in SENSITIVITY: + raise SemanticError(f"unknown sensitivity {sensitivity!r}") + return { + "database": _text(database, 120), + "topic": _text((topic or {}).get("key"), 120), + "field": _text(field.get("key"), 120), + "label": _text(field.get("label") or field.get("key"), 120), + "purpose": _text(semantic.get("purpose") or field.get("means"), 500), + "dataType": _text(field.get("type") or "text", 40), + "role": _text(semantic.get("role") or "attribute", 60), + "synonyms": _strings(semantic.get("synonyms")), + "sensitivity": sensitivity, + "state": state, + "valueProfile": None, + "revision": int(semantic.get("revision") or 1), + "approvedBy": _text(semantic.get("approvedBy"), 120) or None, + "disabledBy": _text(semantic.get("disabledBy"), 120) or None, + "profiledBy": _text(semantic.get("profiledBy"), 120) or None, + } + + +def topic_records(database, topic): + return { + field_key(database, row.get("key")): model_record(database, topic, row) + for row in (topic or {}).get("fields") or () + if isinstance(row, dict) and row.get("key") + } + + +def _clean_override(raw, baseline, *, actor, is_admin): + if not is_admin: + raise SemanticError("only a tenant admin can change semantic approval") + raw = raw if isinstance(raw, dict) else {} + out = copy.deepcopy(baseline) + for key in ("purpose", "role"): + if key in raw: + out[key] = _text(raw[key], 500 if key == "purpose" else 60) + if "synonyms" in raw: + out["synonyms"] = _strings(raw["synonyms"]) + if "sensitivity" in raw: + sensitivity = str(raw["sensitivity"] or "").lower() + if sensitivity not in SENSITIVITY: + raise SemanticError("sensitivity is not recognized") + previous = out.get("sensitivity") + out["sensitivity"] = sensitivity + # A profile admitted while a field was internal must not survive the transaction that + # makes it sensitive. The reverse transition clears the redacted placeholder too: only a + # fresh, permission-scoped profile may put values back. + if sensitivity in ("sensitive", "restricted"): + out["valueProfile"] = redacted_profile() + elif previous in ("sensitive", "restricted"): + out["valueProfile"] = None + if "state" in raw: + state = str(raw["state"] or "").lower() + if state not in SEMANTIC_STATES: + raise SemanticError("semantic state is not recognized") + out["state"] = state + out["approvedBy"] = _text(actor, 120) if state == "approved" else None + out["disabledBy"] = _text(actor, 120) if state == "disabled" else None + out["revision"] = int(baseline.get("revision") or 0) + 1 + return out + + +def merge_records(baseline, persisted): + """Merge only records still present in the source contract. + + A stale persisted field cannot resurrect a removed source field. + """ + out = copy.deepcopy(baseline) + stored = (persisted or {}).get("fields") if isinstance(persisted, dict) else None + for key, raw in (stored or {}).items(): + if key not in out or not isinstance(raw, dict): + continue + for name in ("purpose", "role", "synonyms", "sensitivity", "state", "valueProfile", + "revision", "approvedBy", "disabledBy", "profiledBy"): + if name in raw: + out[key][name] = copy.deepcopy(raw[name]) + out[key] = enforce_redaction(out[key]) + return out + + +def load(runtime, baseline): + try: + raw = runtime.get(_KEY) if runtime is not None else {} + except Exception: + raw = {} + return merge_records(baseline, raw) + + +def update(runtime, baseline, database, field, patch, *, actor, is_admin): + key = field_key(database, field) + if key not in baseline: + raise SemanticError("that field is not in the database contract") + current = load(runtime, baseline)[key] + changed = _clean_override(patch, current, actor=actor, is_admin=is_admin) + + def apply(raw): + doc = copy.deepcopy(raw) if isinstance(raw, dict) else {} + fields = doc.setdefault("fields", {}) + fields[key] = changed + return doc + + runtime.update(_KEY, apply) + return enforce_redaction(changed) + + +def redacted_profile(): + return {"redacted": True, "distinctCount": None, "values": [], "sample": []} + + +def enforce_redaction(record): + """Sanitize one record at every read/projection boundary, including legacy stored rows.""" + row = copy.deepcopy(record) if isinstance(record, dict) else {} + if str(row.get("sensitivity") or "internal").lower() in ("sensitive", "restricted"): + row["valueProfile"] = redacted_profile() + return row + + +def set_value_profile(runtime, baseline, database, field, values, *, actor, is_admin): + """Persist one bounded profile without trusting a caller-supplied summary.""" + if not is_admin: + raise SemanticError("only a tenant admin can profile semantic values") + key = field_key(database, field) + if key not in baseline: + raise SemanticError("that field is not in the database contract") + landed = {} + + def apply(raw): + doc = copy.deepcopy(raw) if isinstance(raw, dict) else {} + current = merge_records(baseline, doc)[key] + current["valueProfile"] = value_profile( + values, sensitivity=current.get("sensitivity") or "internal") + current["revision"] = int(current.get("revision") or 0) + 1 + current["profiledBy"] = _text(actor, 120) or None + doc.setdefault("fields", {})[key] = enforce_redaction(current) + landed.update(doc["fields"][key]) + return doc + + runtime.update(_KEY, apply) + return enforce_redaction(landed) + + +def is_approved(record): + return isinstance(record, dict) and record.get("state") == "approved" + + +def approved_fields(records, database): + prefix = f"{_text(database, 120)}::" + return {row["field"]: copy.deepcopy(row) for key, row in (records or {}).items() + if key.startswith(prefix) and is_approved(row)} + + +def require_approved(records, database, field): + row = (records or {}).get(field_key(database, field)) + if not is_approved(row): + raise SemanticError(f"field {field!r} is not approved for Loopable retrieval") + return copy.deepcopy(row) + + +def value_profile(values, *, sensitivity="internal", distinct_cap=VALUE_PROFILE_CAP, + sample_cap=VALUE_SAMPLE_CAP): + """Return a bounded profile without ever serializing a sensitive value.""" + sensitivity = str(sensitivity or "internal").lower() + if sensitivity in ("sensitive", "restricted"): + return redacted_profile() + distinct_cap = max(1, min(int(distinct_cap), VALUE_PROFILE_CAP)) + sample_cap = max(1, min(int(sample_cap), VALUE_SAMPLE_CAP)) + normalized, seen = [], set() + for value in values if isinstance(values, (list, tuple)) else (): + if value is None: + continue + if isinstance(value, (dict, list, tuple, set)): + text = json.dumps(value, sort_keys=True, default=str) + else: + text = str(value) + text = text[:240] + if text not in seen: + seen.add(text) + normalized.append(text) + count = len(normalized) + if count <= distinct_cap: + return {"redacted": False, "distinctCount": count, "values": normalized, "sample": []} + # Stable hash order prevents database order from turning a profile into a covert channel. + sample = sorted(normalized, key=lambda x: hashlib.sha256(x.encode("utf-8")).hexdigest())[:sample_cap] + return {"redacted": False, "distinctCount": count, "values": [], "sample": sample} + + +def generated_markdown(record): + """A deterministic projection. Sensitive profiles carry no values by construction.""" + row = enforce_redaction(record) + profile = row.get("valueProfile") if isinstance(row.get("valueProfile"), dict) else None + lines = [f"# {row.get('label') or row.get('field') or 'Field'}", + f"- Field: `{row.get('field') or ''}`", + f"- State: {row.get('state') or 'draft'}", + f"- Type: {row.get('dataType') or 'text'}", + f"- Role: {row.get('role') or 'attribute'}", + f"- Sensitivity: {row.get('sensitivity') or 'internal'}", + f"- Purpose: {row.get('purpose') or 'Not configured'}"] + synonyms = _strings(row.get("synonyms")) + if synonyms: + lines.append("- Synonyms: " + ", ".join(synonyms)) + if profile: + if profile.get("redacted"): + lines.append("- Values: Redacted") + elif profile.get("values"): + lines.append("- Values: " + ", ".join(profile["values"])) + else: + lines.append(f"- Distinct values: {profile.get('distinctCount', 0)}") + if profile.get("sample"): + lines.append("- Sample: " + ", ".join(profile["sample"])) + return "\n".join(lines) + "\n" + + +def router_projection(records, database): + """A provider-neutral table of contents over approved semantic sources.""" + approved = approved_fields(records, database) + sections = [] + for role, rows in sorted(_group_by_role(approved).items()): + sections.append({"role": role, "fields": [ + {"key": row["field"], "label": row["label"], "purpose": row["purpose"]} + for row in rows + ]}) + return {"database": database, "sections": sections, "fieldCount": len(approved)} + + +def _group_by_role(records): + grouped = {} + for row in records.values(): + grouped.setdefault(row.get("role") or "attribute", []).append(row) + for rows in grouped.values(): + rows.sort(key=lambda row: (row.get("label") or "").casefold()) + return grouped diff --git a/api/voice_control_plane.py b/api/voice_control_plane.py index c0ae2310ad73a3fd82cb57a14f8dcd6708549f40..0df539286ffd8fa3776bd080c549ca50d0d513ef 100644 --- a/api/voice_control_plane.py +++ b/api/voice_control_plane.py @@ -9,7 +9,9 @@ records a normalized local event, and always reports ``noExternalDispatch: true` from __future__ import annotations import copy +import hmac import hashlib +import os import re import secrets import time @@ -19,6 +21,7 @@ import voice_provider_adapters as adapters STORE_KEY = "voice_control_plane" +FEEDBACK_KEY = "voice_feedback_control" SCHEMA_VERSION = 1 RECORD_TYPES = ( "voice_provider_connection", @@ -48,6 +51,29 @@ class VoiceRefusal(ValueError): """A request was structurally unsafe; its message is fit for the API client.""" +def verify_telnyx_raw_request(raw_body: bytes, signature: str, timestamp: str) -> bool: + """Server-configured receipt verifier over exact received bytes; never browser input.""" + public_key = os.environ.get("TELNYX_WEBHOOK_PUBLIC_KEY") + if not public_key: + raise VoiceRefusal("Telnyx receipt verifier is not configured") + try: + stamp = int(timestamp) + except (TypeError, ValueError): + raise VoiceRefusal("Telnyx webhook timestamp is invalid") from None + if abs(_now() - stamp) > 300: + raise VoiceRefusal("Telnyx webhook timestamp is outside the replay window") + try: + import base64 + from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PublicKey + key = Ed25519PublicKey.from_public_bytes(base64.b64decode(public_key)) + key.verify(base64.b64decode(str(signature or "")), str(timestamp).encode("utf-8") + b"|" + raw_body) + return True + except ImportError as exc: + raise VoiceRefusal("Telnyx Ed25519 verifier is unavailable") from exc + except Exception: + return False + + def _text(value: Any, limit: int = 160) -> str: return " ".join(str(value or "").split())[:limit] @@ -60,7 +86,8 @@ def _now(value: Any = None) -> int: def _mint(prefix: str) -> str: - return f"{prefix}_{secrets.token_urlsafe(9)}" + # Browser/API ids must satisfy the same lowercase reference grammar as supplied ids. + return f"{prefix}_{secrets.token_hex(12)}" def _empty_document() -> dict[str, Any]: @@ -131,7 +158,7 @@ def _reject_sensitive(value: Any) -> None: elif isinstance(value, list): for item in value: _reject_sensitive(item) - elif isinstance(value, str) and _PHONE.search(value): + elif isinstance(value, str) and _PHONE.search(value) and not _ID.fullmatch(value): raise VoiceRefusal("raw phone numbers are not accepted by the browser Voice API") @@ -254,7 +281,7 @@ def _provider_for(doc: dict[str, Any], connection_id: str, extension: Any) -> st def _agent_record(doc: dict[str, Any], raw: Any, previous: Any = None, - actor: str = "") -> dict[str, Any]: + actor: str = "", tenant_id: str = "") -> dict[str, Any]: raw = raw if isinstance(raw, dict) else {} previous = previous if isinstance(previous, dict) else {} _reject_sensitive(raw) @@ -302,7 +329,7 @@ def _agent_record(doc: dict[str, Any], raw: Any, previous: Any = None, stamp = _now() return { "id": str(previous.get("id") or _mint("va")), - "tenant_id": str(previous.get("tenant_id") or ""), + "tenant_id": str(previous.get("tenant_id") or tenant_id or ""), "display_name": name, "lifecycle": lifecycle, "provider_connection_id": connection_id, @@ -319,14 +346,17 @@ def _agent_record(doc: dict[str, Any], raw: Any, previous: Any = None, } -def save_draft(rt: Any, raw: Any, actor: str = "") -> dict[str, Any]: +def save_draft(rt: Any, raw: Any, actor: str = "", tenant_id: str = "") -> dict[str, Any]: result: dict[str, Any] = {} def up(doc: dict[str, Any]) -> dict[str, Any]: - rec = _agent_record(doc, raw, actor=actor) + rec = _agent_record(doc, raw, actor=actor, tenant_id=tenant_id) doc["voice_agent"][rec["id"]] = rec result.update(rec) return doc _update(rt, up) + import automation_engine as _agent_engine + _agent_engine.ensure_agent_contract(rt, result.get("id"), actor, + result.get("display_name") or "") return result @@ -356,7 +386,7 @@ def server_register_provider_connection(rt: Any, provider: str, secret_ref: str, rec = { "id": _safe_id(connection_id, "provider connection") or _mint("vpc"), "tenant_id": str(tenant_id or "")[:120], - "provider": key, "secret_ref": ref, "status": "unconfigured", + "provider": key, "secret_ref": ref, "visibility": "private", "status": "unconfigured", "last_probe_at": 0, "created_at": _now(), "updated_at": _now(), } def up(doc: dict[str, Any]) -> dict[str, Any]: @@ -631,8 +661,19 @@ def browser_dto(record: Any, document: Any = None) -> dict[str, Any]: def list_browser_dtos(rt: Any) -> list[dict[str, Any]]: doc = _store(rt) - return [browser_dto(rec, doc) for _, rec in sorted(doc["voice_agent"].items()) - if isinstance(rec, dict)] + import automation_engine as _agent_engine + rows = [] + for _, rec in sorted(doc["voice_agent"].items()): + if not isinstance(rec, dict): + continue + row = browser_dto(rec, doc) + contract = _agent_engine.agent_contract_dto(rt, rec.get("id")) + if contract: + row.update({"privateEventStore": contract["eventStore"], + "workflows": contract["workflows"], + "setupInterview": contract["setupInterview"]}) + rows.append(row) + return rows def get_browser_dto(rt: Any, voice_id: str) -> dict[str, Any]: @@ -640,7 +681,14 @@ def get_browser_dto(rt: Any, voice_id: str) -> dict[str, Any]: rec = doc["voice_agent"].get(str(voice_id)) if not isinstance(rec, dict): raise VoiceRefusal("unknown Voice agent") - return browser_dto(rec, doc) + row = browser_dto(rec, doc) + import automation_engine as _agent_engine + contract = _agent_engine.agent_contract_dto(rt, voice_id) + if contract: + row.update({"privateEventStore": contract["eventStore"], + "workflows": contract["workflows"], + "setupInterview": contract["setupInterview"]}) + return row def browser_options(rt: Any) -> dict[str, Any]: @@ -658,8 +706,8 @@ def browser_options(rt: Any) -> dict[str, Any]: "maxRecipientsCeiling": MAX_RECIPIENTS} -def _event_key(provider: str, event_ref: str) -> str: - return hashlib.sha256(f"{provider}:{event_ref}".encode("utf-8")).hexdigest()[:32] +def _event_key(provider: str, event_ref: str, tenant_id: str = "", voice_id: str = "") -> str: + return hashlib.sha256(f"{tenant_id}:{voice_id}:{provider}:{event_ref}".encode("utf-8")).hexdigest()[:32] def record_provider_event(rt: Any, provider: str, event_ref: str, voice_id: str, @@ -675,8 +723,8 @@ def record_provider_event(rt: Any, provider: str, event_ref: str, voice_id: str, if not isinstance(normalized_outcome, dict): raise VoiceRefusal("provider outcome must be a normalized object") _reject_sensitive(normalized_outcome) - eid = "vpe_" + _event_key(key, ref) - rec = {"id": eid, "provider": key, "event_ref_hash": _event_key(key, ref), + eid = "vpe_" + _event_key(key, ref, tenant_id, voice_id) + rec = {"id": eid, "provider": key, "event_ref_hash": _event_key(key, ref, tenant_id, voice_id), "tenant_id": str(tenant_id or "")[:120], "voice_agent_id": str(voice_id or ""), "received_at": _now(received_at), "verification_state": verification_state, "normalized_outcome": copy.deepcopy(normalized_outcome), "raw_payload_stored": False} @@ -687,6 +735,39 @@ def record_provider_event(rt: Any, provider: str, event_ref: str, voice_id: str, return rec +def _reserve_provider_event(rt: Any, provider: str, event_ref: str, voice_id: str, + tenant_id: str, received_at: Any = None) -> tuple[bool, dict[str, Any]]: + """One runtime update reserves a full tenant/Agent/provider replay identity before Query.""" + eid = "vpe_" + _event_key(provider, event_ref, tenant_id, voice_id) + result: dict[str, Any] = {"_new": False} + def up(doc): + existing = doc["voice_provider_event"].get(eid) + if isinstance(existing, dict) and existing.get("deliveryState") in {"pending", "completed"}: + result.update(existing) + return doc + rec = {"id": eid, "provider": str(provider), + "event_ref_hash": _event_key(provider, event_ref, tenant_id, voice_id), + "tenant_id": tenant_id, "voice_agent_id": voice_id, "received_at": _now(received_at), + "verification_state": "verified", "deliveryState": "pending", + "normalized_outcome": {}, "raw_payload_stored": False} + doc["voice_provider_event"][eid] = rec + result.update(rec); result["_new"] = True + return doc + _update(rt, up) + return bool(result.pop("_new", False)), result + + +def _finish_provider_event(rt: Any, event_id: str, state: str, outcome: Any = None) -> None: + def up(doc): + rec = dict(doc["voice_provider_event"].get(event_id) or {}) + if rec: + rec["deliveryState"] = state + rec["normalized_outcome"] = copy.deepcopy(outcome or {}) + doc["voice_provider_event"][event_id] = rec + return doc + _update(rt, up) + + def dry_run(rt: Any, voice_id: str, audience: Any, resolve_view: Callable[[dict[str, Any]], dict[str, Any]]) -> dict[str, Any]: """Resolve one allowed saved View and record a redacted local event; never dispatch.""" @@ -743,3 +824,156 @@ def refuse_external_dispatch(source: str = "automation") -> None: """The explicit ordinary-run/webhook/schedule wall until production activation is built.""" raise VoiceRefusal( f"Voice {str(source or 'automation')} dispatch is not activated; use the no-dial harness") + + +def whatsapp_query_dry_run(rt: Any, voice_id: str, payload: Any, headers: Any, + verify_signature: Callable, query_tool: Callable, + now: Any = None, verification: str = "recorded_fixture") -> dict[str, Any]: + """Recorded Telnyx-shaped inbound proof over the provider-neutral Query seam.""" + doc = _store(rt) + rec = doc["voice_agent"].get(str(voice_id)) + if not isinstance(rec, dict): + raise VoiceRefusal("unknown Voice agent") + event_ref = str((payload.get("data") or {}).get("payload", {}).get("id") or "") + tenant_id = str(rec.get("tenant_id") or "") + # Authenticate the received body before considering a replay record. + adapters.verify_telnyx_inbound(payload, headers, verify_signature, + now=_now(now) if now is not None else None) + created, reservation = _reserve_provider_event(rt, "telnyx", event_ref, str(voice_id), tenant_id, now) + if not created: + return {"verifiedWebhook": True, "replayed": True, "deliveryState": reservation.get("deliveryState"), + "eventId": reservation["id"], "query": dict((reservation.get("normalized_outcome") or {}).get("query") or {}), + "noExternalDispatch": True, "externalDispatches": 0} + try: + result = adapters.telnyx_whatsapp_dry_run(payload, headers, verify_signature, query_tool, + now=_now(now) if now is not None else None) + except Exception: + _finish_provider_event(rt, reservation["id"], "failed") + raise + outcome = {"kind": "whatsapp_query_dry_run", "query": result["query"], + "noExternalDispatch": True, "externalDispatches": 0} + _finish_provider_event(rt, reservation["id"], "completed", outcome) + return {**result, "eventId": reservation["id"], "verification": verification} + + +def _feedback_document(rt: Any) -> dict[str, Any]: + try: + raw = rt.get(FEEDBACK_KEY) or {} + except Exception: + raw = {} + raw = raw if isinstance(raw, dict) else {} + return {"schedules": copy.deepcopy(raw.get("schedules") or {}), + "consents": copy.deepcopy(raw.get("consents") or {}), + "dryRuns": copy.deepcopy(raw.get("dryRuns") or {})} + + +def _feedback_update(rt: Any, fn): + return rt.update(FEEDBACK_KEY, lambda raw: fn({ + "schedules": copy.deepcopy((raw or {}).get("schedules") or {}), + "consents": copy.deepcopy((raw or {}).get("consents") or {}), + "dryRuns": copy.deepcopy((raw or {}).get("dryRuns") or {}), + }), flush="sync") + + +def set_feedback_consent(rt: Any, voice_id: str, contact_ref: str, state: str, purpose: str) -> dict[str, Any]: + if str(voice_id or "") not in _store(rt).get("voice_agent", {}): + raise VoiceRefusal("unknown voice Agent") + _reject_sensitive({"contactRef": contact_ref, "purpose": purpose}) + ref = _safe_id(contact_ref, "opaque contact", allow_empty=False) + value = str(state or "") + if value not in {"consented", "withdrawn"}: + raise VoiceRefusal("feedback consent is consented or withdrawn") + rec = {"contactRef": ref, "voiceAgentId": str(voice_id), "state": value, + "purpose": _text(purpose, 160), "updatedAt": _now()} + def up(doc): + doc["consents"][ref] = rec + return doc + _feedback_update(rt, up) + return copy.deepcopy(rec) + + +def create_feedback_schedule(rt: Any, voice_id: str, contact_refs: Any, + local_time: str = "09:00", timezone_name: str = "UTC") -> dict[str, Any]: + voice_key = str(voice_id or "") + if voice_key not in _store(rt).get("voice_agent", {}): + raise VoiceRefusal("unknown voice Agent") + if not isinstance(contact_refs, list) or not contact_refs: + raise VoiceRefusal("feedback schedule needs consented opaque contacts") + _reject_sensitive(contact_refs) + refs = [_safe_id(value, "opaque contact", allow_empty=False) for value in contact_refs] + _reject_sensitive(refs) + if not re.fullmatch(r"(?:[01]\d|2[0-3]):[0-5]\d", str(local_time or "")): + raise VoiceRefusal("feedback schedule time is invalid") + consents = _feedback_document(rt)["consents"] + if any((consents.get(ref) or {}).get("state") != "consented" or + (consents.get(ref) or {}).get("voiceAgentId") != voice_key for ref in refs): + raise VoiceRefusal("every feedback contact needs current consent for this Voice Agent") + sid = "vfs_" + hashlib.sha256( + f"{voice_key}:{','.join(sorted(refs))}".encode("utf-8")).hexdigest()[:18] + rec = {"id": sid, "voiceAgentId": voice_key, "contactRefs": sorted(set(refs)), + "localTime": str(local_time), "timezone": _text(timezone_name, 80), + "enabled": False, "activationState": "disabled", + "eventSchema": {"transcript": "agent_event/transcript", + "consent": "agent_event/consent", "run": "agent_event/run"}, + "activationRequirements": ["all_contacts_consented", "agent_configured", + "private_provider_adapter", "explicit_activation_release"], + "createdAt": _now()} + def up(doc): + doc["schedules"][sid] = rec + return doc + _feedback_update(rt, up) + return copy.deepcopy(rec) + + +def feedback_schedule_dto(rt: Any, schedule_id: str) -> dict[str, Any]: + rec = _feedback_document(rt)["schedules"].get(str(schedule_id)) + if not isinstance(rec, dict): + raise VoiceRefusal("unknown feedback schedule") + return {**copy.deepcopy(rec), "contactRefs": ["opaque" for _ in rec.get("contactRefs") or []], + "noExternalDispatch": True, "externalDispatches": 0} + + +def feedback_dry_run(rt: Any, voice_id: str, schedule_id: str, day: str) -> dict[str, Any]: + doc = _feedback_document(rt) + rec = doc["schedules"].get(str(schedule_id)) + if not isinstance(rec, dict): + raise VoiceRefusal("unknown feedback schedule") + if rec.get("voiceAgentId") != str(voice_id or ""): + raise VoiceRefusal("feedback schedule does not belong to this Voice Agent") + if rec.get("enabled") or rec.get("activationState") != "disabled": + raise VoiceRefusal("feedback activation is unavailable in this release") + voice_rec = _store(rt).get("voice_agent", {}).get(str(voice_id)) or {} + connection = _store(rt).get("voice_provider_connection", {}).get( + str(voice_rec.get("provider_connection_id") or "")) + if not isinstance(connection, dict) or connection.get("visibility") != "private": + raise VoiceRefusal("feedback requires a private server-owned provider connection") + if not re.fullmatch(r"\d{4}-\d{2}-\d{2}", str(day or "")): + raise VoiceRefusal("feedback dry-run day is invalid") + import automation_engine as _agent_engine + configured = ((_agent_engine.agent_contract_dto(rt, voice_id) or {}) + .get("setupInterview", {}).get("state") == "configured") + if not configured: + raise VoiceRefusal("configure the Agent before feedback dry-run") + missing = [ref for ref in rec.get("contactRefs") or [] + if (doc["consents"].get(ref) or {}).get("state") != "consented" or + (doc["consents"].get(ref) or {}).get("voiceAgentId") != str(voice_id)] + if missing: + raise VoiceRefusal("every feedback contact needs current consent") + key = f"{schedule_id}:{day}" + if key in doc["dryRuns"]: + raise VoiceRefusal("feedback schedule was already dry-run for that day") + result = {"scheduleId": str(schedule_id), "day": str(day), + "dueCount": len(rec.get("contactRefs") or []), "state": "dry_run_complete", + "noExternalDispatch": True, "externalDispatches": 0, + "activationState": "disabled", "adapterVisibility": "private"} + def up(current): + current["dryRuns"].setdefault(key, result) + return current + _feedback_update(rt, up) + return copy.deepcopy(result) + + +def future_integration_adapter_contract() -> dict[str, Any]: + return {"version": 1, "interface": ["normalizeInbound", "renderQueryResult", "dryRun"], + "providers": [], "makeImplemented": False, "nangoImplemented": False, + "externalDispatch": False} diff --git a/api/voice_provider_adapters.py b/api/voice_provider_adapters.py index a251d11252bdbbb32d6b77e9324b098e06acb758..1cb1708501f113c76ffd91b779c19c23aea8db98 100644 --- a/api/voice_provider_adapters.py +++ b/api/voice_provider_adapters.py @@ -8,6 +8,9 @@ Results contain counts and readiness classes only, never provider identifiers or from __future__ import annotations from dataclasses import dataclass +import copy +import json +import time from typing import Any, Callable @@ -55,9 +58,24 @@ _EXTENSIONS: dict[str, dict[str, tuple[type, ...]]] = { "stability": (int, float), "similarityBoost": (int, float), }, + "telnyx": { + "messagingProfileRef": (str,), + }, } -_LABELS = {"retell": "Retell", "elevenlabs": "ElevenLabs"} +_LABELS = {"retell": "Retell", "elevenlabs": "ElevenLabs", "telnyx": "Telnyx"} + +_CAPABILITIES = { + "retell": {"inboundVoice": "supported", "outboundVoice": "supported", + "whatsAppMessaging": "unsupported", "whatsAppCalling": "unproven", + "governedQuery": "supported"}, + "elevenlabs": {"inboundVoice": "supported", "outboundVoice": "supported", + "whatsAppMessaging": "supported", "whatsAppCalling": "unproven", + "governedQuery": "supported"}, + "telnyx": {"inboundVoice": "carrier_dependent", "outboundVoice": "carrier_dependent", + "whatsAppMessaging": "supported", "whatsAppCalling": "unproven", + "governedQuery": "supported"}, +} def providers() -> list[dict[str, Any]]: @@ -71,6 +89,7 @@ def providers() -> list[dict[str, Any]]: {"key": name, "type": "number" if float in types else "text"} for name, types in fields.items() ], + "capabilities": copy.deepcopy(_CAPABILITIES[key]), } for key, fields in _EXTENSIONS.items() ] @@ -85,7 +104,7 @@ def clean_extension(provider: str, raw: Any) -> dict[str, Any]: key = str(provider or "").strip().lower() allowed = _EXTENSIONS.get(key) if allowed is None: - raise ProviderAdapterRefusal("select Retell or ElevenLabs") + raise ProviderAdapterRefusal("select Retell, ElevenLabs, or Telnyx") if raw in (None, ""): return {"provider": key, "version": 1, "config": {}} if not isinstance(raw, dict): @@ -187,3 +206,59 @@ def assert_read_only_request(method: str) -> None: """Public negative control for transports that want an explicit verb guard.""" if str(method or "").upper() != "GET": raise ProviderAdapterRefusal("provider reconciliation accepts GET inventory only") + + +def verify_telnyx_inbound(payload: Any, headers: Any, verify_signature: Callable, + now: int | None = None) -> tuple[str, str]: + """Verify one recorded Telnyx-shaped inbound event and simulate a Query reply. + + There is intentionally no transport argument. A verified event can reach governed Query and a + local formatter only; sending, dialing, and provider mutation are structurally unavailable. + """ + if not isinstance(payload, dict) or not isinstance(headers, dict): + raise ProviderAdapterRefusal("Telnyx webhook fixture is invalid") + signature = str(headers.get("telnyx-signature-ed25519") or "") + timestamp = str(headers.get("telnyx-timestamp") or "") + try: + stamp = int(timestamp) + except (TypeError, ValueError): + raise ProviderAdapterRefusal("Telnyx webhook timestamp is invalid") from None + clock = int(now if now is not None else time.time()) + if abs(clock - stamp) > 300: + raise ProviderAdapterRefusal("Telnyx webhook timestamp is outside the replay window") + canonical = json.dumps(payload, sort_keys=True, separators=(",", ":")) + if not callable(verify_signature) or not verify_signature(signature, f"{timestamp}|{canonical}"): + raise ProviderAdapterRefusal("Telnyx webhook signature verification failed") + data = payload.get("data") if isinstance(payload.get("data"), dict) else {} + event = data.get("payload") if isinstance(data.get("payload"), dict) else {} + if str(data.get("event_type") or "") != "message.received": + raise ProviderAdapterRefusal("Telnyx fixture is not an inbound message") + message_id = str(event.get("id") or "").strip() + text = str((event.get("text") or event.get("body") or "")).strip()[:2000] + if not message_id or not text: + raise ProviderAdapterRefusal("Telnyx inbound message is incomplete") + return message_id, text + + +def telnyx_whatsapp_dry_run(payload: Any, headers: Any, verify_signature: Callable, + query_tool: Callable, now: int | None = None) -> dict[str, Any]: + """Verify one recorded Telnyx-shaped inbound event and simulate a Query reply.""" + message_id, text = verify_telnyx_inbound(payload, headers, verify_signature, now) + if not callable(query_tool): + raise ProviderAdapterRefusal("governed Query is unavailable") + query = query_tool(text, {"channel": "whatsapp", "provider": "telnyx"}) + if not isinstance(query, dict): + raise ProviderAdapterRefusal("governed Query returned no typed result") + state = str(query.get("state") or "complete") + if state == "clarification": + reply = str(query.get("question") or "Please clarify your request.")[:1000] + else: + reply = str(query.get("speakable") or "")[:2000] + if not reply: + raise ProviderAdapterRefusal("governed Query returned no speakable answer") + return {"provider": "telnyx", "channel": "whatsapp", "verifiedWebhook": True, + "eventRefHash": __import__("hashlib").sha256(message_id.encode()).hexdigest()[:24], + "query": {"state": state, "clarification": state == "clarification"}, + "simulatedReply": {"text": reply, "status": "simulated"}, + "rawPhoneReturned": False, "rawMessageReturned": False, + "noExternalDispatch": True, "externalDispatches": 0} diff --git a/platform/aios_grid.py b/platform/aios_grid.py index 9c01a1f1b44c0c7f4161460ccb9e0a0843698036..12d8fbc9c347c3fb97a8d511d94b1906f8f661e4 100644 --- a/platform/aios_grid.py +++ b/platform/aios_grid.py @@ -24,6 +24,7 @@ Design notes: injected-HTML path remains a read/local-write fallback when component assets are absent. """ import datetime as _dt +import copy import json import math import re @@ -804,7 +805,7 @@ def public_holidays(calendar, year): return frozenset(days) -def clean_date_recurrence(raw): +def clean_date_recurrence(raw, condition_fields=None): """One date field's recurrence config, or ``None`` for absent/malformed input. ``rules`` are an intentional union: selecting weekday + public holiday means either makes @@ -814,7 +815,17 @@ def clean_date_recurrence(raw): if not isinstance(raw, dict): return None start = _iso_day(raw.get('startDate', raw.get('anchor'))) - if start is None: + anchor_raw = raw.get('anchorField') + anchor = None + if isinstance(anchor_raw, dict): + table = str(anchor_raw.get('table') or 'same').strip()[:80] + field = str(anchor_raw.get('field') or '').strip()[:80] + via = str(anchor_raw.get('linkedVia') or '').strip()[:80] + if field and (table == 'same' or via): + anchor = {'table': table, 'field': field} + if via: + anchor['linkedVia'] = via + if start is None and anchor is None: return None out_rules = [] for raw_rule in list(raw.get('rules') or [])[:12]: @@ -831,7 +842,9 @@ def clean_date_recurrence(raw): weekdays = sorted({day for day in (raw_rule.get('weekdays') or []) if isinstance(day, int) and not isinstance(day, bool) and 0 <= day <= 6}) - rule['weekdays'] = weekdays or [start.weekday()] + rule['weekdays'] = weekdays or ([start.weekday()] if start else []) + if not rule['weekdays']: + continue elif kind == 'monthly': # ``day: 1`` means the first of every N months. ``ordinal`` + ``weekday`` adds # Google-Calendar-style first Monday / last Friday without a competing model. @@ -844,12 +857,13 @@ def clean_date_recurrence(raw): continue rule.update({'ordinal': ordinal, 'weekday': weekday}) else: - day = raw_rule.get('day', start.day) + day = raw_rule.get('day', start.day if start else None) if not isinstance(day, int) or isinstance(day, bool) or not 1 <= day <= 31: continue rule['day'] = day elif kind == 'yearly': - month, day = raw_rule.get('month', start.month), raw_rule.get('day', start.day) + month = raw_rule.get('month', start.month if start else None) + day = raw_rule.get('day', start.day if start else None) if (not isinstance(month, int) or isinstance(month, bool) or not 1 <= month <= 12 or not isinstance(day, int) or isinstance(day, bool) or not 1 <= day <= 31): continue @@ -876,8 +890,18 @@ def clean_date_recurrence(raw): preview_count = raw.get('previewCount', raw.get('preview', 10)) if not isinstance(preview_count, int) or isinstance(preview_count, bool): preview_count = 10 - return {'startDate': start.isoformat(), 'rules': out_rules, - 'previewCount': max(1, min(preview_count, 50))} + out = {'rules': out_rules, 'previewCount': max(1, min(preview_count, 50))} + if start: + out['startDate'] = start.isoformat() + if anchor: + out['anchorField'] = anchor + admitted_conditions = (condition_fields if condition_fields is not None + else raw.get('conditionFields')) + conditions = clean_filter_tree(raw.get('conditions') or [], valid_keys={ + str(key) for key in admitted_conditions or () if str(key)}) + if conditions: + out['conditions'] = conditions + return out def _recurrence_matches(day, start, rule): @@ -903,13 +927,17 @@ def _recurrence_matches(day, start, rule): return False -def recurrence_preview(raw, today=None, count=None): +def recurrence_preview(raw, today=None, count=None, condition_fields=None): """The next distinct ISO dates, anchored to the server's supplied/current day.""" - config = clean_date_recurrence(raw) + config = clean_date_recurrence(raw, condition_fields=condition_fields) if config is None: return [] now = _iso_day(today) or _dt.date.today() - start = _iso_day(config['startDate']) + start = _iso_day(config.get('startDate')) + if start is None and isinstance(raw, dict): + start = _iso_day(raw.get('resolvedAnchorDate')) + if start is None: + return [] want = config['previewCount'] if count is None else max(1, min(int(count), 50)) lower = max(start, now) # A preview of at most fifty values needs no unbounded search. Ten years also makes an @@ -935,16 +963,37 @@ def recurrence_preview(raw, today=None, count=None): return [item.isoformat() for item in sorted(found)[:want]] -def recurrence_payload(raw, today=None): +def recurrence_payload(raw, today=None, condition_fields=None): """Durable config plus server-authoritative ``nextDate`` and finite ``preview``.""" - config = clean_date_recurrence(raw) + config = clean_date_recurrence(raw, condition_fields=condition_fields) if config is None: return None - preview = recurrence_preview(config, today=today) + preview = recurrence_preview(config, today=today, condition_fields=condition_fields) return {**config, 'nextDate': preview[0] if preview else None, 'preview': preview} -def _field_extras(saved, ftype): +def project_date_recurrences(fields, today=None): + """Re-project recurrence metadata against the final visible field contract.""" + valid_keys = {str(field.get('key')) for field in fields or () + if isinstance(field, dict) and field.get('key')} + out = [] + for field in fields or (): + if not isinstance(field, dict) or field.get('type') != 'date' \ + or not isinstance(field.get('recurrence'), dict): + out.append(field) + continue + item = dict(field) + recurrence = recurrence_payload( + field.get('recurrence'), today=today, condition_fields=valid_keys) + if recurrence: + item['recurrence'] = recurrence + else: + item.pop('recurrence', None) + out.append(item) + return out + + +def _field_extras(saved, ftype, condition_fields=None): """createdBy / permissions / format / scope — the validated passthrough the created strata share (wave 5). `createdBy` is only ever WRITTEN host-side (the handler stamps it); here it is carried so the client can gate its menus and the handler can enforce against it. @@ -987,7 +1036,9 @@ def _field_extras(saved, ftype): # W43C C2 — the ISO date remains scalar in each row; recurrence is field metadata. Serve # the compact persisted config together with a server-derived next date and preview so the # client never substitutes its browser clock or holiday library for the tenant contract. - recurrence = recurrence_payload(saved.get("recurrence")) if ftype == "date" else None + recurrence = (recurrence_payload(saved.get("recurrence"), + condition_fields=condition_fields) + if ftype == "date" else None) if recurrence: out["recurrence"] = recurrence corrected_from = saved.get("labelCorrectedFrom") @@ -1153,6 +1204,7 @@ def fields_from_workspace(workspace=None, cohorts=False, scope_key=None, fields_ out = [] base_fields = fields_base if fields_base is not None else FIELDS base_keys = {field["key"] for field in base_fields} + recurrence_keys = base_keys | {str(key) for key in saved} for base in base_fields: meta = saved.get(base["key"]) or {} field = dict(base) @@ -1194,7 +1246,8 @@ def fields_from_workspace(workspace=None, cohorts=False, scope_key=None, fields_ # fields need this explicit read half because their override is otherwise rebuilt from # the immutable contract on every workspace projection. if base.get("type") == "date" and "recurrence" in meta: - recurrence = recurrence_payload(meta.get("recurrence")) + recurrence = recurrence_payload( + meta.get("recurrence"), condition_fields=recurrence_keys) if recurrence: field["recurrence"] = recurrence out.append(field) @@ -1229,7 +1282,7 @@ def fields_from_workspace(workspace=None, cohorts=False, scope_key=None, fields_ else "sum" if mtype in ("currency", "int") else None), "note": str(field.get("note") or "")[:2000], "measure": {"key": mkey[:80], "window": window}, - **_field_extras(field, mtype), + **_field_extras(field, mtype, condition_fields=recurrence_keys), }) continue if not str(key).startswith("custom_"): @@ -1255,7 +1308,7 @@ def fields_from_workspace(workspace=None, cohorts=False, scope_key=None, fields_ "default": bool(field.get("default", True)), "custom": True, "note": str(field.get("note") or "")[:2000], - **_field_extras(field, ftype), + **_field_extras(field, ftype, condition_fields=recurrence_keys), } if ftype == "formula": formula = _clean_formula(field.get("formula")) @@ -1297,7 +1350,7 @@ def fields_from_workspace(workspace=None, cohorts=False, scope_key=None, fields_ # string. Absent = `plain`, which is what an unconfigured code column renders as. **({"code": _clean_code(field.get("code"))} if ftype == "code" and _clean_code(field.get("code")) else {}), - **_field_extras(field, ftype), + **_field_extras(field, ftype, condition_fields=recurrence_keys), }) if cohorts and not any(f.get('key') == COHORT_COLUMN for f in out): # ⚠ The emptiness check is WAVE 19's, and it is about the topics R9 opened this column to. @@ -1831,10 +1884,37 @@ _HEX6 = re.compile(r'^#[0-9A-Fa-f]{6}$') TS_DELTA_KINDS = ('abs', 'pct', 'yoy', 'ytd') TS_MAX_CUSTOM_ROWS = 12 TS_MAX_STYLES = 200 -#: R2 — a formula row's `expr` is stored VERBATIM and NEVER parsed here (evaluation is client -#: law; the client refuses unknown refs/cycles/div-zero itself). The charset wall is the whole -#: server-side contract: row refs `[...]`, arithmetic, numbers — no markup, no control chars. -_TS_EXPR_OK = re.compile(r'^[A-Za-z0-9_ .+\-*/()\[\]]+$') +TS_MAX_EXPR = 500 + + +def _ts_formula_source_ok(expr): + """Transport wall for a Time Series formula governed by the shared server parser. + + This function deliberately does not duplicate the formula grammar. It preserves any bounded + source, including quotes and regex punctuation, for the API parser to accept or explain. The + sole syntax-shaped refusal is an unquoted square bracket: the shared contract declares that + legacy ``[field]`` references cannot enter the braces-only grammar. Brackets inside a quoted + regex/text literal remain ordinary source and round-trip byte-for-byte. + """ + if not isinstance(expr, str) or len(expr) > TS_MAX_EXPR: + return False + quote, i = None, 0 + while i < len(expr): + char = expr[i] + if quote: + if char == quote: + if i + 1 < len(expr) and expr[i + 1] == quote: + i += 2 + continue + quote = None + i += 1 + continue + if char in ("'", '"'): + quote = char + elif char in '[]': + return False + i += 1 + return True #: Old wire value -> the value we store today. Applied AFTER the membership test so an unknown #: mode is still rejected rather than accidentally aliased. @@ -2256,12 +2336,11 @@ def _clean_display(raw, valid_keys): one = {'id': rid, 'kind': r_kind, 'label': label.strip()[:120]} if r_kind == 'formula': expr = r.get('expr') - if (isinstance(expr, str) and expr.strip() - and len(expr) <= 200 and _TS_EXPR_OK.match(expr)): + if _ts_formula_source_ok(expr): one['expr'] = expr - # else: keep the ROW, drop the EXPR — it renders "—". Vanishing the row - # would delete the user's label to punish their arithmetic (GRID's dated - # asymmetry amendment; the calendarMetrics per-entry-drop precedent). + # Else keep the ROW and drop only the source. Legacy square references and + # over-bound transport cannot enter the shared formula parser; deleting the row + # would also delete the user's label and its saved order. seen_r.add(rid) clean_r.append(one) if clean_r: @@ -2726,14 +2805,21 @@ def clean_filter_tree(raw, valid_keys, depth=1, budget=None, cohort_ids=None, leaf['locked'] = True out.append(leaf) continue - if node.get('colId') in valid_keys and node.get('op') in FILTER_OPS: + # Wave 45 C1: ``zone`` was the pre-canonical projection key. A saved View is migrated + # at its ordinary validated save door, rather than kept as a second readable field or + # dropped as an unknown condition. The migration is admissible only when this database + # actually exposes ``__zone__``; on every other database an unknown ``zone`` leaf keeps + # the longstanding fail-closed behaviour. + column = ('__zone__' if node.get('colId') == 'zone' and '__zone__' in valid_keys + else node.get('colId')) + if column in valid_keys and node.get('op') in FILTER_OPS: budget[0] -= 1 # `or ''` would be wrong here: it maps every FALSY value to '', and '' is the # signal for "inactive". A numeric 0 (or 0.0, or False) is a real value the client # treats as active — `0 === ""` is false in TS — so `revenue = 0` would silently # stop filtering and show every row instead of the zero-revenue ones. val, val2 = node.get('value'), node.get('value2') - leaf = {'colId': node['colId'], 'op': node['op'], + leaf = {'colId': column, 'op': node['op'], 'value': ('' if val is None else str(val))[:500], 'value2': ('' if val2 is None else str(val2))[:500]} # CG-8. A MEASURE condition ("Sales, in the last 90 days, > 5,000") carries two @@ -2789,6 +2875,51 @@ def clean_filter_tree(raw, valid_keys, depth=1, budget=None, cohort_ids=None, return out +def migrate_zone_projection_config(raw): + """Copy one saved View config while renaming every legacy ``zone`` field reference. + + Filter leaves are validated later by ``clean_filter_tree``. The other members are field-key + projections and can be migrated structurally here, before their normal allow-list checks. + This retains old View intent without re-publishing ``zone`` as a writable or readable field. + """ + config = copy.deepcopy(raw if isinstance(raw, dict) else {}) + + def key(value): + return '__zone__' if value == 'zone' else value + + def filters(nodes): + for node in nodes or (): + if not isinstance(node, dict): + continue + if isinstance(node.get('children'), list): + filters(node['children']) + else: + if node.get('colId') == 'zone': + node['colId'] = '__zone__' + rank_scope = node.get('rankScope') + if isinstance(rank_scope, dict): + for member in ('field', 'groupBy'): + if member in rank_scope: + rank_scope[member] = key(rank_scope.get(member)) + + raw_filters = config.get('filters') + filters((raw_filters.get('nodes') or raw_filters.get('children') or ()) + if isinstance(raw_filters, dict) else raw_filters) + for member in ('sorts',): + for item in config.get(member) or (): + if isinstance(item, dict) and item.get('colId') == 'zone': + item['colId'] = '__zone__' + for member in ('order', 'visible'): + if isinstance(config.get(member), list): + config[member] = list(dict.fromkeys(key(value) for value in config[member])) + if isinstance(config.get('widths'), dict) and 'zone' in config['widths']: + config['widths']['__zone__'] = config['widths'].pop('zone') + for member in ('groupBy', 'colorBy', 'stackField'): + if config.get(member) == 'zone': + config[member] = '__zone__' + return config + + def preserve_locked_view_records(prior_config, filters, sorts): """Return authoritative filter/sort records when an in-place lock was touched. diff --git a/platform/core/assets.py b/platform/core/assets.py index 0b8b950a1fee8296ce08e4625cbcaec75d7f1bd6..3c86dd321f1ff5b32dfc1bd5362e05b371f3b716 100644 --- a/platform/core/assets.py +++ b/platform/core/assets.py @@ -23,6 +23,7 @@ import time from pathlib import Path DEFAULT_REPO = 'royal-imports/product-assets' +APPLICATION_TYPES = ('chart', 'route', 'map', 'timeseries') _MANIFEST_TTL = 300 #: (fetched_at, {"CODE": {...}}, {"slug": {...}}) — one manifest read per process per TTL. #: DEBT-5 (2026-08-04): the same manifest.json now carries a second, NON-SKU half under @@ -216,3 +217,19 @@ def put_record(tenant, asset_id, orig_bytes, ext, web_png): def reset_cache(): """Drop the memoized manifest — for gates and for a post-ingest refresh.""" _MANIFEST.update(at=0.0, codes=None, editorial=None, records=None) + + +def application_open_target(application): + """Stable typed target shared by Home, Starred and the Application selector. + + Application assets are tenant-store records, not image-manifest entries. This helper lives + here only as the common asset-address contract and performs no I/O. + """ + row = application if isinstance(application, dict) else {} + kind = str(row.get('type') or '').lower() + app_id = str(row.get('id') or '').strip() + if kind not in APPLICATION_TYPES or not app_id: + return None + databases = row.get('compatibleDatabases') if isinstance(row.get('compatibleDatabases'), list) else [] + return {'kind': 'application', 'id': app_id, 'type': kind, + 'database': str(databases[0]) if databases else None} diff --git a/platform/core/grid_events.py b/platform/core/grid_events.py index 51682bc926695e42e07da5fb9b7acb66344ba801..d847eb027535a029bd7915b98ec1160e4b276dd8 100644 --- a/platform/core/grid_events.py +++ b/platform/core/grid_events.py @@ -45,7 +45,7 @@ from dataclasses import dataclass, field # `field(default_factory=list)` written inside one of those functions would resolve to the local # dict and raise. The alias is unshadowable, so D-291's channel cannot be broken by a rename. from dataclasses import field as dataclass_field -from typing import List, MutableMapping, Optional +from typing import Callable, List, MutableMapping, Optional import core.store as store import core.table_store as _tstore # wave-9 I17: the shared-view authorisation predicates @@ -152,6 +152,14 @@ class EventCtx: #: naming the key, because `overlay_keys` only asks whether the column exists. Empty means #: "nothing hidden", which is every legacy record and therefore every caller today. hidden_keys: frozenset = frozenset() + # Wave 45 R15: the transport adapter supplies the authority-aware recurrence validator. + # Core can normalise a recurrence's finite shape, but it cannot decide whether a named + # same/linked field is visible to this caller without reaching upward into the API layer. + recurrence_validator: Optional[Callable[[object], Optional[dict]]] = None + # Value-history restore reuses the ordinary cell wall. This stamp changes only the audit + # event kind and admits ``None`` as "remove the prior cell" after every normal field/row + # permission check has passed; browser-authored edits never set it. + history_kind: str = 'edit' scope_key: str = 'customer' #: Wave 16 C-TOPIC — the TABLE OPS this ctx reads and writes (a `core.table_store.make` #: object). None = the CUSTOMER table (`modules.customer_data.TABLE_OPS`), which keeps @@ -1253,6 +1261,16 @@ def handle_one(event, ctx): for stale in list(seen)[:200]: seen.pop(stale, None) kind = event.get('type') + from core import user_tables as _zone_contract + _zone_key = _zone_contract.ZONE_FIELD_KEY + _offered_keys = set((event.get('updates') or {}).keys()) if isinstance( + event.get('updates'), dict) else set() + _raw_field = event.get('field') if isinstance(event.get('field'), dict) else {} + if _offered_keys.intersection({_zone_key, 'zone'}) \ + or str(_raw_field.get('key') or '') in {_zone_key, 'zone'}: + return _refuse(ctx, 'zone_membership_read_only', + "Zone membership is worked out from the record's current location", + key=_zone_key, event=str(kind or 'grid_event')) field_by_key = {f['key']: f for f in fields} # A Product shared-select option can have been admitted by an earlier event while this # EventCtx remains alive. The renderer gets the persisted choice projection on refresh, @@ -1339,12 +1357,12 @@ def handle_one(event, ctx): requested_name = name if not view_id or not name: return False - cfg = raw['config'] # The filter TREE (leaf conditions and/or nested condition groups) is # validated by the module-agnostic grid contract helper — imported lazily # here for the same reason the page does it: a missing embed build must # never break page import. import aios_grid as _ag + cfg = _ag.migrate_zone_projection_config(raw['config']) # A MEASURE key is valid in a FILTER and nowhere else (CG-8): it may not be ordered, # shown, grouped or coloured by, because it is not a column of this table. Hence the # widened key set here and the plain `valid_keys` everywhere below. @@ -2015,7 +2033,18 @@ def handle_one(event, ctx): if 'recurrence' not in raw and recurrence_raw is None and not shared_field: previous = (ws.get('fields') or {}).get(key) recurrence_raw = previous.get('recurrence') if isinstance(previous, dict) else None - recurrence = _ag2.clean_date_recurrence(recurrence_raw) + if 'recurrence' in raw: + if recurrence_raw is not None and ctx.recurrence_validator is None: + return _refuse( + ctx, 'bad_recurrence', + 'recurrence needs the server field-authority validator', key) + try: + recurrence = (ctx.recurrence_validator(recurrence_raw) + if ctx.recurrence_validator is not None else None) + except (TypeError, ValueError) as exc: + return _refuse(ctx, 'bad_recurrence', str(exc)[:240], key) + else: + recurrence = _ag2.clean_date_recurrence(recurrence_raw) if recurrence: field['recurrence'] = recurrence else: @@ -2969,6 +2998,13 @@ def handle_one(event, ctx): pid = event.get('pid') if not isinstance(pid, int) or pid not in allowed_pids: return False + _history_table = str(ctx.scope_key or 'customer') + if _history_table.startswith('ut_') and ctx.table is not None: + import core.user_tables as _ut_history + _history_before = dict((((_ut_history.get( + _history_table, st=ctx.table.st) or {}).get('rows') or {}).get(str(pid)) or {})) + else: + _history_before = dict(((ws.get('overlays') or {}).get(str(pid)) or {})) updates = {} refused = False stage_moved = False @@ -2980,7 +3016,9 @@ def handle_one(event, ctx): choice_additions = {} raw_updates = event.get('updates') if isinstance(event.get('updates'), dict) else {} for key, value in raw_updates.items(): - if key not in overlay_keys or not isinstance(value, (str, int, float)): + restoring_absence = value is None and ctx.history_kind == 'restore' + if key not in overlay_keys or ( + not restoring_absence and not isinstance(value, (str, int, float))): refused = True continue # C-PERM: a field this caller's permissions HIDE cannot be written, only refused. @@ -3110,6 +3148,12 @@ def handle_one(event, ctx): if isinstance(_auto, dict) and (_auto.get('flowId') or _auto.get('preset')): refused = True continue + # A restore of an event whose before-value was absent removes the cell. It reaches + # this branch only after the same hidden, edit-role, automation and computed-field + # checks as an ordinary edit. A browser-authored null remains refused above. + if restoring_absence: + updates[key] = None + continue # ⭐ WAVE 23 (C7) — the json wall, BEFORE the generic 10 000-char truncation, which # would otherwise cut a document in half and store the halves as valid text. A json # column promises its readers a parseable document; a write that is not one is @@ -3221,6 +3265,20 @@ def handle_one(event, ctx): _ut_ev.emit_row_event({'type': 'event_field', 'table': str(ctx.scope_key), 'rowId': str(pid), 'field': _k, 'after': _v, 'st': ctx.table.st, 'user': uname}) + if updates or row_events: + import core.user_tables as _ut_history + if _history_table.startswith('ut_') and ctx.table is not None: + _history_after = dict((((_ut_history.get( + _history_table, st=ctx.table.st) or {}).get('rows') or {}).get(str(pid)) or {})) + _history_st = ctx.table.st + else: + _history_after = dict(_history_before) + _history_after.update(updates) + _history_after.update(dict(row_events)) + _history_st = ctx.st + _ut_history.append_value_history( + _history_table, pid, _history_before, _history_after, uname, st=_history_st, + kind=ctx.history_kind) # Wave-6 item 3: the client's cell already shows the typed value — rerun ONLY when # some of what it shows was refused or clamped and must be repainted to the truth. # A stage move reruns too: its `arrive` side-effects (the tracked flag) changed cells diff --git a/platform/core/route_system.py b/platform/core/route_system.py index c3581ab9a1b7337862107b98ddac666cf0ae616b..76394b6c8ad6172c9b8491b318ce7ecb0a6c3b65 100644 --- a/platform/core/route_system.py +++ b/platform/core/route_system.py @@ -12,12 +12,15 @@ import hashlib import math import re import uuid +from zoneinfo import ZoneInfo, ZoneInfoNotFoundError MANAGED_BY = "loopable" LOOPABLE_FOLDER_ID = "loopable" LOOPABLE_FOLDER = {"id": LOOPABLE_FOLDER_ID, "name": "Loopable", "managedBy": MANAGED_BY} ROUTES_KEY, ZONES_KEY = "ut_odoo_customers_routes", "ut_odoo_customers_zones" SYSTEM_TABLES = {ROUTES_KEY: "Odoo Customers - Routes", ZONES_KEY: "Odoo Customers - Zones"} +ZONE_FIELD_KEY = "__zone__" +CLIENT_ZONE_MEMBERSHIP_KEYS = frozenset(("memberPids", "member_pids", "memberships")) # TenantRuntime namespaces this bucket. User-table rows are scalar user data; route stops are a # protected relation (route id + customer id + order), so they cannot be a generic Link field. @@ -174,6 +177,18 @@ def _route(raw, *, ordinal=0, created_by="", tenant_id="", prior=None): source_view = str(source_view or "").strip()[:120] or None revision = raw.get("sourceViewRevision", prior.get("sourceViewRevision")) revision = str(revision or "").strip()[:128] or None + assignee = str(raw.get("assignee", prior.get("assignee")) or "").strip().lower()[:120] or None + route_date = str(raw.get("date", prior.get("date")) or "").strip() or None + if route_date: + try: + route_date = _dt.date.fromisoformat(route_date).isoformat() + except ValueError: + return None + if (assignee is None) != (route_date is None): + # A Route assignment is one fact. Persisting either half would make the Inbox task + # appear at an unknowable time or for nobody, and a later PATCH could accidentally + # complete a schedule the user never submitted together. + return None stops_raw = raw.get("stops") if "stops" in raw else prior.get("stops") maps_url = str(raw.get("mapsUrl", prior.get("mapsUrl")) or "").strip()[:2048] if maps_url and not _GOOGLE_MAPS_URL_RE.match(maps_url): @@ -186,6 +201,8 @@ def _route(raw, *, ordinal=0, created_by="", tenant_id="", prior=None): "sourceViewRevision": revision, "stops": _stops(stops_raw), "managedBy": MANAGED_BY, "createdBy": str(prior.get("createdBy") or created_by or "")[:120], "shareKey": share_key, + "assignee": assignee, "date": route_date, + "starred": raw.get("starred", prior.get("starred", False)) is True, "createdAt": prior.get("createdAt") or stamp, "updatedAt": stamp} if maps_url: out["mapsUrl"] = maps_url @@ -226,11 +243,78 @@ def _polygon_contains(geometry, lon, lat): return False +def clean_zone_geometry(raw): + """Return the one bounded polygon/circle geometry accepted by every Zone writer.""" + if not isinstance(raw, dict): + return None + kind = raw.get("type") + if kind == "Circle": + center = raw.get("center") + try: + lon, lat = float(center[0]), float(center[1]) + radius = float(raw.get("radiusMeters")) + except (TypeError, ValueError, IndexError): + return None + if not (math.isfinite(lon) and math.isfinite(lat) and math.isfinite(radius)): + return None + if not (-180 <= lon <= 180 and -90 <= lat <= 90 and 0 < radius <= 2_000_000): + return None + return {"type": "Circle", "center": [lon, lat], "radiusMeters": radius} + coords = raw.get("coordinates") + polygons = [coords] if kind == "Polygon" else coords if kind == "MultiPolygon" else None + if not isinstance(polygons, list) or not polygons: + return None + clean_polygons = [] + for polygon in polygons: + if not isinstance(polygon, list) or not polygon: + return None + clean_rings = [] + for ring in polygon: + if not isinstance(ring, list) or len(ring) < 4: + return None + clean_ring = [] + for point in ring: + try: + lon, lat = float(point[0]), float(point[1]) + except (TypeError, ValueError, IndexError): + return None + if not (math.isfinite(lon) and math.isfinite(lat)) or not ( + -180 <= lon <= 180 and -90 <= lat <= 90): + return None + clean_ring.append([lon, lat]) + if clean_ring[0] != clean_ring[-1]: + return None + clean_rings.append(clean_ring) + clean_polygons.append(clean_rings) + return {"type": kind, + "coordinates": clean_polygons[0] if kind == "Polygon" else clean_polygons} + + +def point_in_zone(geometry, lon, lat): + """The single backend point-in-shape implementation for polygon and circle Zones.""" + try: + lon, lat = float(lon), float(lat) + except (TypeError, ValueError): + return False + if not (math.isfinite(lon) and math.isfinite(lat)): + return False + clean = clean_zone_geometry(geometry) + if clean is None: + return False + if clean["type"] != "Circle": + return _polygon_contains(clean, lon, lat) + center_lon, center_lat = clean["center"] + mean_lat = math.radians((center_lat + lat) / 2) + dx = math.radians(lon - center_lon) * math.cos(mean_lat) + dy = math.radians(lat - center_lat) + return math.hypot(dx, dy) * 6_371_008.8 <= clean["radiusMeters"] + + def _zone(raw, *, ordinal=0, created_by="", tenant_id="", prior=None): raw = raw if isinstance(raw, dict) else {} prior = prior if isinstance(prior, dict) else {} name = _name(raw.get("name", prior.get("name"))) - geometry = raw.get("geometryGeoJson", prior.get("geometryGeoJson")) + geometry = clean_zone_geometry(raw.get("geometryGeoJson", prior.get("geometryGeoJson"))) zips_raw = raw.get("zipCodes", prior.get("zipCodes", ())) zips, seen = [], set() for value in zips_raw or (): @@ -238,7 +322,7 @@ def _zone(raw, *, ordinal=0, created_by="", tenant_id="", prior=None): if text and text not in seen and _ZIP_RE.fullmatch(text): seen.add(text) zips.append(text) - if not name or (geometry is not None and not isinstance(geometry, dict)) or (geometry is None and not zips): + if not name or geometry is None: return None zid = str(raw.get("id") or prior.get("id") or "zone_" + uuid.uuid4().hex)[:96] source_view = raw.get("sourceViewId", prior.get("sourceViewId")) @@ -248,11 +332,14 @@ def _zone(raw, *, ordinal=0, created_by="", tenant_id="", prior=None): stamp = _now() return {"id": zid, "tenantId": str(tenant_id or prior.get("tenantId") or ""), "name": name, "color": _color(raw.get("color", prior.get("color")), ordinal), - "geometryGeoJson": copy.deepcopy(geometry) if isinstance(geometry, dict) else None, + "geometryGeoJson": geometry, "zipCodes": zips, "visible": raw.get("visible", prior.get("visible", True)) is not False, "sourceViewId": source_view, "geographicField": geographic_field, "membershipRevision": int(prior.get("membershipRevision") or 0), "memberPids": [int(pid) for pid in prior.get("memberPids") or [] if _pid(pid) is not None], + "legacyMembershipEvidence": [int(pid) for pid in ( + prior.get("legacyMembershipEvidence") or prior.get("memberships") or ()) + if _pid(pid) is not None], "managedBy": MANAGED_BY, "createdBy": str(prior.get("createdBy") or created_by or "")[:120], "createdAt": prior.get("createdAt") or stamp, "updatedAt": stamp} @@ -267,21 +354,18 @@ def _customer_zip(row): def zone_members(zone, rows): - """Polygon-or-ZIP UNION, recalculated from the current customer address projection.""" - zips = set(zone.get("zipCodes") or ()) + """Recalculate membership only from current exact coordinates and visible geometry.""" out = [] for row in rows or (): pid = _pid((row or {}).get("pid")) if pid is None: continue - zip_match = _customer_zip(row) in zips try: lon, lat = float(row.get("lon")), float(row.get("lat")) - polygon_match = math.isfinite(lon) and math.isfinite(lat) and _polygon_contains( - zone.get("geometryGeoJson"), lon, lat) + geometry_match = point_in_zone(zone.get("geometryGeoJson"), lon, lat) except (TypeError, ValueError): - polygon_match = False - if zip_match or polygon_match: + geometry_match = False + if geometry_match: out.append(pid) return out @@ -313,6 +397,7 @@ def _persist(st, mutate): state.setdefault("zones", []) state.setdefault("legacyMigrations", {}) state.setdefault("routeNavigation", {}) + state.setdefault("inboxTasks", []) next_state, value = mutate(state) result["value"] = copy.deepcopy(value) return next_state @@ -327,6 +412,7 @@ def read(st, rows=None): state.setdefault("zones", []) state.setdefault("legacyMigrations", {}) state.setdefault("routeNavigation", {}) + state.setdefault("inboxTasks", []) return refresh_zone_memberships(state, rows or ()) if rows is not None else state @@ -359,6 +445,129 @@ def duplicate_route(st, route_id, *, tenant_id, created_by, name=None): return put_route(st, raw, tenant_id=tenant_id, created_by=created_by) +def route_capabilities(route, *, can_edit=False, can_duplicate=False, can_delete=False): + """Explicit asset-menu capabilities; clients never infer rights from button visibility.""" + compatibility = (route or {}).get("compatibility") == "legacy-route-order" + editable = bool(can_edit and not compatibility) + return {"rename": editable, "duplicate": bool(can_duplicate and not compatibility), "star": editable, + "share": editable, "delete": bool(can_delete and not compatibility)} + + +def tenant_day(timezone_name="UTC", *, now=None): + """Return the tenant-local calendar day from an explicit IANA timezone.""" + stamp = now or _dt.datetime.now(_dt.timezone.utc) + if stamp.tzinfo is None: + stamp = stamp.replace(tzinfo=_dt.timezone.utc) + try: + zone = ZoneInfo(str(timezone_name or "UTC")) + except (ZoneInfoNotFoundError, ValueError): + zone = ZoneInfo("UTC") + return stamp.astimezone(zone).date() + + +def tenant_day_for_runtime(st, *, now=None): + """Resolve the calendar day from one tenant runtime, failing safely to UTC.""" + config = getattr(getattr(st, "tenant", None), "config", None) or {} + return tenant_day(config.get("timezone") or config.get("timeZone") or "UTC", now=now) + + +def _inbox_day(value, *, st=None): + if isinstance(value, _dt.datetime): + return value.date() + if isinstance(value, _dt.date): + return value + fallback = tenant_day_for_runtime(st) if st is not None else tenant_day() + try: + return _dt.date.fromisoformat(str(value or fallback.isoformat())[:10]) + except (TypeError, ValueError): + return fallback + + +def _reconcile_inbox_state(state, day): + """Project exactly one canonical task for every due Route assignment. + + The semantic identity is Route + date + normalized assignee. Old duplicate ids are folded + into the deterministic id while preserving a read receipt from any copy. Rebuilt rows also + make rename/reschedule/delete changes visible in the same atomic state mutation. + """ + routes = [item for item in state.get("routes") or () if isinstance(item, dict)] + existing = {} + for item in state.get("inboxTasks") or (): + if not isinstance(item, dict): + continue + key = (str(item.get("routeId") or ""), str(item.get("routeDate") or ""), + str(item.get("assignee") or "").strip().lower()) + if all(key): + existing[key] = bool(existing.get(key)) or item.get("read") is True + + tasks = [] + seen = set() + for route in routes: + route_id = str(route.get("id") or "") + route_date = str(route.get("date") or "") + assignee = str(route.get("assignee") or "").strip().lower() + key = (route_id, route_date, assignee) + if not all(key) or key in seen: + continue + try: + due = _dt.date.fromisoformat(route_date) + except ValueError: + continue + if due > day: + continue + seen.add(key) + task_id = "route-task:" + hashlib.sha256( + f"{route_id}|{route_date}|{assignee}".encode("utf-8")).hexdigest()[:24] + tasks.append({ + "id": task_id, "kind": "route_task", "subject": route.get("name") or "Route", + "routeId": route_id, "routeDate": route_date, "assignee": assignee, + "source": "Loopable Routes", + "target": {"module": "customer_data", "application": "map", "routeId": route_id}, + "read": bool(existing.get(key)), "at": route_date, + }) + state["inboxTasks"] = sorted( + tasks, key=lambda item: (item.get("at", ""), item["id"]), reverse=True) + return state["inboxTasks"] + + +def reconcile_inbox_tasks(st, *, today=None): + """Materialize one persistent task for each due Route assignee/day, idempotently.""" + day = _inbox_day(today, st=st) + + def mutate(state): + _reconcile_inbox_state(state, day) + return state, copy.deepcopy(state["inboxTasks"]) + + return _persist(st, mutate) or [] + + +def inbox_tasks(st, assignee, *, today=None): + return [task for task in reconcile_inbox_tasks(st, today=today) + if str(task.get("assignee") or "").lower() == str(assignee or "").lower()] + + +def mark_inbox_tasks(st, assignee, ids=None, *, read=True): + """Set read state only on this assignee's persistent Route tasks.""" + who = str(assignee or "").lower() + wanted = None if ids is None else {str(value) for value in ids} + + def mutate(state): + tasks = [] + for raw in state.get("inboxTasks") or (): + item = dict(raw) if isinstance(raw, dict) else None + if item is None: + continue + if (str(item.get("assignee") or "").lower() == who + and (wanted is None or str(item.get("id")) in wanted)): + item["read"] = bool(read) + tasks.append(item) + state["inboxTasks"] = tasks + return state, [dict(item) for item in tasks + if str(item.get("assignee") or "").lower() == who] + + return _persist(st, mutate) or [] + + def _nav_user(username): return str(username or "").strip().lower()[:120] @@ -562,7 +771,7 @@ def list_zones(st, rows=None): return [copy.deepcopy(zone) for zone in read(st, rows).get("zones") or [] if isinstance(zone, dict)] -def put_route(st, raw, *, tenant_id, created_by): +def put_route(st, raw, *, tenant_id, created_by, today=None): """Dedicated normalized Route writer. Generic table endpoints cannot call this.""" wanted = str((raw or {}).get("id") or "").strip() @@ -580,6 +789,7 @@ def put_route(st, raw, *, tenant_id, created_by): else: routes[prior_index] = clean state["routes"] = routes + _reconcile_inbox_state(state, _inbox_day(today, st=st)) return state, clean return _persist(st, mutate) @@ -642,7 +852,7 @@ def set_stop_membership(st, route_id, customer_pid, present, *, tenant_id, creat return _persist(st, mutate) -def delete_route(st, route_id): +def delete_route(st, route_id, *, today=None): route_id = str(route_id or "").strip() def mutate(state): @@ -664,12 +874,17 @@ def delete_route(st, route_id): nav["order"] = [value for value in nav.get("order") or () if str(value) != route_id] navigation[username] = nav state["routeNavigation"] = navigation + _reconcile_inbox_state(state, _inbox_day(today, st=st)) return state, len(prior) != len(state["routes"]) return bool(_persist(st, mutate)) def put_zone(st, raw, *, rows, tenant_id, created_by): + # Membership is always derived from the permission-scoped current rows supplied by the + # mounted route. Refuse forged membership keys instead of silently lending them authority. + if not isinstance(raw, dict) or any(key in raw for key in CLIENT_ZONE_MEMBERSHIP_KEYS): + return None wanted = str((raw or {}).get("id") or "").strip() def mutate(state): @@ -727,8 +942,9 @@ def customer_projection_fields(routes, zones): return [field_axes({"key": "routes", "label": "Routes", "type": "multiselect", "source": "odoo", "derived": True, "readOnly": True, "options": route_names}), - field_axes({"key": "zone", "label": "Zone", "type": "multiselect", + field_axes({"key": ZONE_FIELD_KEY, "label": "Zone", "type": "multiselect", "source": "odoo", "derived": True, "readOnly": True, + "multi": True, "system": True, "protected": True, "options": zone_names})] @@ -740,7 +956,10 @@ def apply_customer_projections(rows, routes, zones): item = dict(row) values = projection.get(item.get("pid"), projection.get(str(item.get("pid")), {})) or {} item["routes"] = ", ".join(str(name) for name in values.get("Routes", ()) if name) - item["zone"] = ", ".join(str(name) for name in values.get("Zone", ()) if name) + item[ZONE_FIELD_KEY] = ", ".join(str(name) for name in values.get("Zone", ()) if name) + # Read-only migration evidence for saved filters created before ``__zone__`` became + # canonical. New contracts never publish this alias as a field. + item.pop("zone", None) out.append(item) return out diff --git a/platform/core/user_tables.py b/platform/core/user_tables.py index 7d43ae991b03b93665e7b7efbeb876e9dc16fbef..06e5794c3e7981b894e59a747033b2b07d3d1bed 100644 --- a/platform/core/user_tables.py +++ b/platform/core/user_tables.py @@ -56,11 +56,18 @@ from core.route_system import MAX_OPTIONS STORE_KEY = 'user_tables' ZONE_STORE_KEY = 'table_zones' +VALUE_HISTORY_KEY = 'value_history' +SHARED_NOTES_KEY = 'shared_notes' +ODOO_SCHEMA_SNAPSHOT_KEY = 'odoo_schema_snapshot' +ODOO_SCHEMA_ENABLEMENTS_KEY = 'odoo_schema_enablements' +ODOO_SYNC_STATUS_KEY = 'odoo_sync_status' # A reserved, derived multi-select field. It is table data backed by durable Zone membership, # not a user-editable column and therefore cannot collide with a field a user creates. ZONE_FIELD_KEY = '__zone__' MAX_TABLES = 40 MAX_LABEL = 60 +HISTORY_RETENTION_DAYS = 30 +HISTORY_EXTENDED_DAYS = 90 #: ⭐ THE PER-TABLE ROW CEILING — 5,000 until 2026-08-09, and it was never a property of the #: substrate. Owner: *"Why do we keep having this MAX_ROWS problem, 5000 is too little. We need to @@ -944,7 +951,7 @@ def _clean_format(raw): return out or None -def clean_fields(raw): +def clean_fields(raw, recurrence_validator=None): """Validate a caller-supplied field list into the stored shape — refuse junk, never store it. Returns a list of `{key,label,type,source}` dicts (source is always 'overlay': every base column of a user table is user-editable by construction), or None when nothing @@ -1089,7 +1096,10 @@ def clean_fields(raw): # type: the row still stores ``YYYY-MM-DD`` while this durable bag lets the server project # the next date and preview consistently for every reader. if entry is not None and ftype == 'date' and f.get('recurrence') is not None: - recurrence = _ag_recurrence(f.get('recurrence')) + if recurrence_validator is None: + raise ValueError('recurrence needs the server field-authority validator') + recurrence = (recurrence_validator(f.get('recurrence')) + if recurrence_validator is not None else None) if recurrence: entry['recurrence'] = recurrence if entry is None: @@ -1174,7 +1184,7 @@ _ZONE_ID_RE = re.compile(r'^[A-Za-z0-9_-]{1,96}$') _ZONE_FIELD_RE = re.compile(r'^[A-Za-z][A-Za-z0-9_]{0,79}$') _ZONE_COLOR_RE = re.compile(r'^#[0-9A-Fa-f]{6}$') _ZONE_ZIP_RE = re.compile(r'^\d{5}(?:-\d{4})?$') -_ZONE_GEOMETRIES = frozenset({'Polygon', 'MultiPolygon'}) +_ZONE_GEOMETRIES = frozenset({'Polygon', 'MultiPolygon', 'Circle'}) MAX_ZONE_ZIP_CODES = 500 @@ -1205,25 +1215,9 @@ def _zone_ring(raw): def _clean_zone_geometry(raw): - """A bounded GeoJSON Polygon/MultiPolygon geometry, with no executable/foreign members.""" - if not isinstance(raw, dict) or raw.get('type') not in _ZONE_GEOMETRIES: - return None - kind, coords = raw['type'], raw.get('coordinates') - if kind == 'Polygon': - rings = [_zone_ring(ring) for ring in (coords if isinstance(coords, list) else [])] - if not rings or any(ring is None for ring in rings): - return None - clean = rings - else: - clean = [] - for polygon in (coords if isinstance(coords, list) else []): - rings = [_zone_ring(ring) for ring in (polygon if isinstance(polygon, list) else [])] - if not rings or any(ring is None for ring in rings): - return None - clean.append(rings) - if not clean: - return None - return {'type': kind, 'coordinates': clean} + """Delegate to the one Route/Zone geometry authority.""" + from core import route_system + return route_system.clean_zone_geometry(raw) def clean_zone(raw, field_keys=None, allowed_row_ids=None): @@ -1245,9 +1239,9 @@ def clean_zone(raw, field_keys=None, allowed_row_ids=None): if zip_code not in seen_zips: seen_zips.add(zip_code) zip_codes.append(zip_code) - # Existing polygon Zones remain valid byte-for-byte. Wave 44 also permits a ZIP-only Zone; - # the two shapes share this record rather than inventing a second persistence model. - if not name or (geometry is None and not zip_codes): + # Wave 45 makes visible geometry authoritative. ZIP codes remain migration metadata but may + # not create a membership-bearing Zone without a polygon or circle. + if not name or geometry is None: return None out = {'name': name, 'color': (str(raw.get('color') or '#1F4E78').upper()), @@ -1276,7 +1270,12 @@ def clean_zone(raw, field_keys=None, allowed_row_ids=None): continue seen.add(pid) memberships.append(pid) - out['memberships'] = memberships + # Historical hand-selected membership is retained only as migration evidence. Membership + # returned to Grid is always recomputed from current exact coordinates in ``zone_cells``. + evidence = raw.get('legacyMembershipEvidence') or memberships + out['memberships'] = [] + if evidence: + out['legacyMembershipEvidence'] = list(dict.fromkeys(str(pid) for pid in evidence if str(pid))) creator = str(raw.get('createdBy') or '').strip()[:120] created_at = str(raw.get('createdAt') or '').strip()[:64] if creator: @@ -1353,7 +1352,8 @@ def zone_field(table_key, st=None): names.append(name) seen.add(name) return {'key': ZONE_FIELD_KEY, 'label': 'Zone', 'type': 'multiselect', 'source': 'odoo', - 'derived': True, 'multi': True, 'default': True, 'options': names, + 'derived': True, 'readOnly': True, 'system': True, 'protected': True, + 'multi': True, 'default': True, 'options': names, 'colorCodeOptions': True} @@ -1367,12 +1367,13 @@ def with_zone_field(table_key, fields, st=None): def zone_cells(table_key, rows, allowed_row_ids=None, st=None): """Layer canonical Zone memberships into ordinary row cells for FilterBuilder/evaluator use.""" allowed = ({str(pid) for pid in allowed_row_ids} if allowed_row_ids is not None else None) + from core import route_system labels_by_pid = {} for zone in zones(table_key, st=st): name = str(zone.get('name') or '').strip() if not name: continue - for pid in zone.get('memberships') or (): + for pid in route_system.zone_members(zone, rows): key = str(pid) if allowed is not None and key not in allowed: continue @@ -1420,7 +1421,8 @@ def is_managed_system_table(table_key, st=None, definition=None): return route_system.is_managed(definition if definition is not None else (get(table_key, st) or table_key)) -def create(label, username, fields=None, source=None, st=None, *, managed_by=None, capability=None): +def create(label, username, fields=None, source=None, st=None, *, managed_by=None, capability=None, + recurrence_validator=None): """Create a blank table. Returns its key, or None if refused. Refuses rather than raises on the ordinary conditions (no name, duplicate, cap reached) — @@ -1453,8 +1455,9 @@ def create(label, username, fields=None, source=None, st=None, *, managed_by=Non 'source': (source if source in (BLANK_SOURCE, AUTOMATION_SOURCE) else BLANK_SOURCE), 'createdBy': username, 'created': _dt.datetime.now().strftime('%Y-%m-%dT%H:%M:%S'), - 'fields': clean_fields(fields) or [{'key': 'name', 'label': 'Name', 'type': 'text', - 'source': 'overlay', 'default': True}], + 'fields': clean_fields(fields, recurrence_validator=recurrence_validator) or [ + {'key': 'name', 'label': 'Name', 'type': 'text', + 'source': 'overlay', 'default': True}], 'rows': {}, } if managed_by: @@ -2634,7 +2637,7 @@ def note_human_edit(table_key, values, row_id, st=None): return sorted(touched) -def _clean_field(raw, previous=None): +def _clean_field(raw, previous=None, recurrence_validator=None): """One field dict → the stored shape, or None. The single validator for create AND patch, so a column cannot be typed one way on the way in and another on the way back.""" prev = previous or {} @@ -2958,7 +2961,9 @@ def _clean_field(raw, previous=None): recurrence_raw = (raw.get('recurrence') if 'recurrence' in raw else (prev.get('recurrence') if ftype == 'date' else None)) if recurrence_raw is not None: - recurrence = _ag_recurrence(recurrence_raw) + recurrence = (recurrence_validator(recurrence_raw) + if recurrence_validator is not None + else _ag_recurrence(recurrence_raw)) if recurrence and ftype == 'date': out['recurrence'] = recurrence return out @@ -3511,15 +3516,18 @@ def flow_bound(bag, st=None): return False -def add_field(table_key, raw, st=None): +def add_field(table_key, raw, st=None, recurrence_validator=None): """Append one column to the shared schema. Returns the stored field, or None if refused.""" + if isinstance(raw, dict) and raw.get('recurrence') is not None \ + and recurrence_validator is None: + raise ValueError('recurrence needs the server field-authority validator') if not is_user_table(table_key, st): return None defn = get(table_key, st) or {} have = [f for f in (defn.get('fields') or [])] if len(have) >= MAX_FIELDS: return None - field = _clean_field(raw) + field = _clean_field(raw, recurrence_validator=recurrence_validator) if not field or any(f.get('key') == field['key'] for f in have): return None if not dynamic_link_filter_valid(field, have): @@ -3694,13 +3702,16 @@ def sync_reciprocal_link(table_key, field_key, st=None): return result -def patch_field(table_key, fkey, raw, st=None): +def patch_field(table_key, fkey, raw, st=None, recurrence_validator=None): """Edit one column's definition IN PLACE. Returns the stored field, or None if refused. ⚠ THE KEY NEVER MOVES. A field's key is what every stored cell is filed under, so accepting a new one here would orphan every value in the column while looking like a rename. The LABEL is the renameable thing; the key is identity. """ + if isinstance(raw, dict) and raw.get('recurrence') is not None \ + and recurrence_validator is None: + raise ValueError('recurrence needs the server field-authority validator') if not is_user_table(table_key, st): return None fields = [dict(f) for f in ((get(table_key, st) or {}).get('fields') or [])] @@ -3724,7 +3735,7 @@ def patch_field(table_key, fkey, raw, st=None): if 'description' in merged and isinstance(fields[idx].get('automation'), dict) \ and fields[idx]['automation'].get('preset') is True: merged.pop('description') - field = _clean_field(merged, fields[idx]) + field = _clean_field(merged, fields[idx], recurrence_validator=recurrence_validator) if not field: return None if not dynamic_link_filter_valid(field, fields): @@ -3917,9 +3928,17 @@ def patch_cells(table_key, row_id, values, st=None): def _set(cur): t = cur.get(str(table_key)) if t is not None: - clean = {str(k): str(v) for k, v in (values or {}).items()} - t.setdefault('rows', {}).setdefault(str(row_id), {}).update(clean) - _hydrate_choice_options(t.get('fields') or [], [clean]) + clean = {str(k): (None if v is None else str(v)) + for k, v in (values or {}).items()} + row = t.setdefault('rows', {}).setdefault(str(row_id), {}) + for key, value in clean.items(): + if value is None: + row.pop(key, None) + else: + row[key] = value + _hydrate_choice_options(t.get('fields') or [], [ + {key: value for key, value in clean.items() if value is not None} + ]) return cur _st(st).update(STORE_KEY, _set, flush='sync') @@ -4163,6 +4182,21 @@ def may_open(table_key, viewer, is_admin=False, st=None): return False +def may_edit_database(table_key, viewer, is_admin=False, st=None): + """Canonical Database-note/editor authority: owner/admin or an editor grant.""" + table = get(table_key, st) + if not table: + return False + if bool(is_admin) or table.get('createdBy') == viewer: + return True + try: + import core.shares as shares + return shares.may_edit('database', table_key, viewer, + is_admin=is_admin, st=st) + except Exception: + return False + + #: ⭐⭐ W31-T10 / CONTRACT C1 — THE TWO BUCKETS `may_open` READS, LENT FOR THE LENGTH OF ONE PASS. #: #: `core/store.py:Store.get` deep-copies the WHOLE tenant document on every call, hit or miss, @@ -4261,6 +4295,229 @@ def lend_defs(st=None): return lend(st, **{STORE_KEY: all_defs(st)}) +def _utc_now(): + return _dt.datetime.now(_dt.timezone.utc).replace(microsecond=0) + + +def record_odoo_schema_snapshot(models, *, complete, errors=(), st=None): + """Store an authoritative fields_get snapshot supplied by the sync lifecycle.""" + clean_models = {} + for model, fields in (models or {}).items(): + if not isinstance(fields, dict): + continue + clean_fields = {} + for name, meta in fields.items(): + if not str(name).strip() or not isinstance(meta, dict): + continue + clean_fields[str(name)] = { + key: meta.get(key) for key in + ('string', 'type', 'required', 'readonly', 'relation') if key in meta + } + clean_models[str(model)] = clean_fields + row = {'models': clean_models, 'complete': bool(complete), + 'errors': [str(error)[:500] for error in errors][:50], + 'capturedAt': _utc_now().isoformat()} + _st(st).update(ODOO_SCHEMA_SNAPSHOT_KEY, lambda _current: row, flush='sync') + return row + + +def odoo_schema_snapshot(st=None): + row = _st(st).get(ODOO_SCHEMA_SNAPSHOT_KEY) or {} + return dict(row) if isinstance(row, dict) else {} + + +def enabled_odoo_schema_fields(st=None, model=None): + """Product assembly seam: enabled raw metadata, still subject to its normal field wall.""" + enabled = _st(st).get(ODOO_SCHEMA_ENABLEMENTS_KEY) or {} + snapshot = odoo_schema_snapshot(st) + rows = [] + for model_name, fields in (snapshot.get('models') or {}).items(): + if model and str(model_name) != str(model): + continue + for name, meta in (fields or {}).items(): + key = f'{model_name}.{name}' + if enabled.get(key) is True: + raw_type = str(meta.get('type') or 'text') + # The browser's FieldType vocabulary is closed. Odoo's vocabulary is not, so + # an unknown/add-on type must degrade to text rather than reaching the client as + # an impossible union member. Float is text because the grid has no neutral + # decimal type (``currency`` would fabricate a dollar sign and ``int`` rounds). + grid_type = { + 'boolean': 'checkbox', + 'integer': 'int', + 'monetary': 'currency', + 'date': 'date', + 'datetime': 'date', + 'char': 'text', + 'text': 'text', + 'html': 'text', + 'float': 'text', + 'selection': 'text', + 'reference': 'text', + 'many2one': 'text', + 'one2many': 'text', + 'many2many': 'text', + 'binary': 'text', + }.get(raw_type, 'text') + rows.append({'key': key, 'model': model_name, 'rawName': name, + 'label': str(meta.get('string') or name), + 'type': grid_type, 'rawType': raw_type, 'source': 'odoo'}) + return rows + + +def record_odoo_sync_status(*, phase, errors=(), st=None): + """Lifecycle writer seam for the out-of-fence sync runner.""" + clean_phase = str(phase or 'unknown').strip().lower() + if clean_phase not in {'syncing', 'ok', 'error', 'unknown'}: + clean_phase = 'unknown' + row = {'phase': clean_phase, + 'lastRun': _utc_now().isoformat(), + 'errors': [str(error)[:500] for error in errors][:50]} + _st(st).update(ODOO_SYNC_STATUS_KEY, lambda _current: row, flush='sync') + return row + + +def append_value_history(table_key, row_id, before, after, actor, *, st=None, + retention_days=HISTORY_RETENTION_DAYS, extended_entitled=False, + kind='edit'): + """Append one compact before/after event and prune by the entitled retention window.""" + days = int(retention_days or HISTORY_RETENTION_DAYS) + if days == HISTORY_EXTENDED_DAYS and not extended_entitled: + raise PermissionError('90-day history requires a Loopable-admin entitlement') + if days not in (HISTORY_RETENTION_DAYS, HISTORY_EXTENDED_DAYS): + raise ValueError('history retention must be 30 or 90 days') + before = dict(before or {}) + after = dict(after or {}) + keys = sorted(key for key in set(before) | set(after) if before.get(key) != after.get(key)) + if not keys: + return None + now = _utc_now() + event = {'id': 'vh_' + _uuid.uuid4().hex, 'table': str(table_key), 'rowId': str(row_id), + 'kind': str(kind), 'actor': str(actor or '')[:120], 'at': now.isoformat(), + 'changes': {key: {'before': before.get(key), 'after': after.get(key)} for key in keys}} + # Never let a 30-day reader/edit destroy history that the tenant may be + # entitled to inspect for 90 days. Visibility is applied by value_history. + cutoff = now - _dt.timedelta(days=HISTORY_EXTENDED_DAYS) + + def _append(current): + rows = [item for item in (current or []) if isinstance(item, dict)] + kept = [] + for item in rows: + try: + stamp = _dt.datetime.fromisoformat(str(item.get('at') or '').replace('Z', '+00:00')) + except ValueError: + continue + if stamp.tzinfo is None: + stamp = stamp.replace(tzinfo=_dt.timezone.utc) + if stamp >= cutoff: + kept.append(item) + return (kept + [event])[-10000:] + + _st(st).update(VALUE_HISTORY_KEY, _append, flush='sync') + return event + + +def value_history(table_key, row_id=None, *, st=None, + retention_days=HISTORY_RETENTION_DAYS, extended_entitled=False): + days = int(retention_days or HISTORY_RETENTION_DAYS) + if days == HISTORY_EXTENDED_DAYS and not extended_entitled: + raise PermissionError('90-day history requires a Loopable-admin entitlement') + if days not in (HISTORY_RETENTION_DAYS, HISTORY_EXTENDED_DAYS): + raise ValueError('history retention must be 30 or 90 days') + cutoff = _utc_now() - _dt.timedelta(days=days) + rows = _st(st).get(VALUE_HISTORY_KEY) or [] + out = [] + for item in rows: + if (not isinstance(item, dict) or str(item.get('table')) != str(table_key) + or (row_id is not None and str(item.get('rowId')) != str(row_id))): + continue + try: + stamp = _dt.datetime.fromisoformat(str(item.get('at') or '').replace('Z', '+00:00')) + except ValueError: + continue + if stamp.tzinfo is None: + stamp = stamp.replace(tzinfo=_dt.timezone.utc) + if stamp >= cutoff: + out.append(dict(item)) + out.sort(key=lambda item: str(item.get('at') or ''), reverse=True) + encoded = _json.dumps(out, separators=(',', ':'), ensure_ascii=False).encode('utf-8') + return {'events': out, 'metering': {'events': len(out), 'bytes': len(encoded)}} + + +def project_value_history(history, allowed_row_ids, hidden_keys=()): + """Apply the normal row/field visibility closure to already-retained events.""" + allowed = {str(row_id) for row_id in (allowed_row_ids or ())} + hidden = {str(key) for key in (hidden_keys or ())} + events = [] + for item in (history or {}).get('events') or []: + if str(item.get('rowId')) not in allowed: + continue + changes = {str(key): dict(value) for key, value in + (item.get('changes') or {}).items() + if str(key) not in hidden and isinstance(value, dict)} + if not changes: + continue + projected = dict(item) + projected['changes'] = changes + events.append(projected) + encoded = _json.dumps(events, separators=(',', ':'), ensure_ascii=False).encode('utf-8') + return {'events': events, 'metering': {'events': len(events), 'bytes': len(encoded)}} + + +def history_restore_values(table_key, row_id, event_id, *, field_key=None, st=None, + retention_days=HISTORY_RETENTION_DAYS, + extended_entitled=False): + """Resolve retained prior values only; the caller must use its ordinary write door.""" + event = next((item for item in value_history( + table_key, row_id, st=st, retention_days=retention_days, + extended_entitled=extended_entitled)['events'] + if str(item.get('id')) == str(event_id)), None) + if event is None: + return None + changes = event.get('changes') or {} + wanted = [str(field_key)] if field_key else list(changes) + if not wanted or any(key not in changes for key in wanted): + return None + return {key: changes[key].get('before') for key in wanted} + + +def restore_row_values(table_key, row_id, event_id, actor, *, field_key=None, st=None, + retention_days=HISTORY_RETENTION_DAYS, extended_entitled=False): + """Retired unsafe raw-store restore door; routes must resolve then use overlay_patch.""" + raise RuntimeError('restore_row_values is retired; restore through the ordinary patch authority') + + +def shared_note(subject_kind, subject_id, *, st=None): + key = f"{subject_kind}:{subject_id}" + row = (_st(st).get(SHARED_NOTES_KEY) or {}).get(key) or {} + return dict(row) if isinstance(row, dict) else {} + + +def put_shared_note(subject_kind, subject_id, body, actor, *, st=None): + """One shared rich note per Database/View, with append-only revisions.""" + if subject_kind not in {'database', 'view'}: + return None + key, text = f"{subject_kind}:{subject_id}", str(body or '')[:100000] + result = {} + + def _put(current): + current = dict(current or {}) + prior = current.get(key) if isinstance(current.get(key), dict) else {} + revisions = list(prior.get('revisions') or [])[-99:] + revision = {'version': int(prior.get('version') or 0) + 1, 'body': text, + 'author': str(actor or '')[:120], 'at': _utc_now().isoformat()} + row = {'subjectKind': subject_kind, 'subjectId': str(subject_id), + 'version': revision['version'], 'body': text, + 'author': revision['author'], 'updated': revision['at'], + 'revisions': revisions + [revision]} + current[key] = row + result.update(row) + return current + + _st(st).update(SHARED_NOTES_KEY, _put, flush='sync') + return result + + def nav_entries(viewer=None, is_admin=False, st=None): """Registry-SHAPED dicts so the nav can render user tables beside real modules with no special-casing: the same keys the flat nav reads (`key`, `label`, `source`). @@ -4286,7 +4543,13 @@ def nav_entries(viewer=None, is_admin=False, st=None): key=lambda kv: (kv[1].get('label') or '').lower()): if viewer is not None and not may_open(key, viewer, is_admin, st=lent): continue + from core import route_system + if route_system.is_managed(t) or route_system.is_managed(key): + continue + source = str(t.get('source') or BLANK_SOURCE) + source_group = ('Odoo' if source.lower() == 'odoo' or key.startswith('ut_odoo_') + else 'Custom' if source == BLANK_SOURCE else source) out.append({'key': key, 'label': t.get('label') or key, - 'source': t.get('source') or BLANK_SOURCE, + 'source': source, 'sourceGroup': source_group, 'user_table': True, 'nav': True, 'validate': False}) return out diff --git a/platform/core/view_templates.py b/platform/core/view_templates.py index 32f60256518e59c15d04ba62518e8f54e10eef68..cb63dcc1cc3e9dca694a2e9aeee4c93eba610080 100644 --- a/platform/core/view_templates.py +++ b/platform/core/view_templates.py @@ -144,7 +144,7 @@ TEMPLATES = [ # The owner's own example (C12). One view, one job, and it is a DATA-QUALITY view: the # brand a customer buys from decides which BU-scoped account can see them at all, so a # blank DBA is a customer nobody's book contains. - 'desc': 'Customers with no brand recorded — they fall outside every scoped book.', + 'desc': 'Customers with no brand recorded; they fall outside every scoped book.', 'source': 'odoo_customer', 'alert': True, 'views': [ @@ -171,7 +171,7 @@ TEMPLATES = [ { 'key': 'buy_list_companion', 'label': 'Buy-list companion', - 'desc': 'What is running out, and what it costs to bring in — beside the buy list.', + 'desc': 'What is running out and what it costs to bring in, beside the buy list.', 'source': 'odoo_product', 'views': [ _view( @@ -209,7 +209,7 @@ TEMPLATES = [ _view( 'tpl_ig_candidates_board', 'Review board', 'Discovered profiles grouped by category. Move a card to say what you ' - 'decided about it — the automation never decides for you.', + 'decided about it. The automation never decides for you.', # ⚠ WAVE 26 (R6): `tracked` was here and the column is DELETED. The owner # retired it because the per-automation stage field already tracks a record's # progress, and two progress columns that can disagree is worse than one. A @@ -299,3 +299,27 @@ def offer(field_keys, source=None): 'source': t['source'], 'views': len(t.get('views') or ()), 'alert': bool(t.get('alert'))}) return out + + +def application_ref(view): + """Return a View's reusable Application reference, never an embedded config copy.""" + config = (view or {}).get('config') if isinstance(view, dict) else None + ref = (config or {}).get('application') if isinstance(config, dict) else None + if not isinstance(ref, dict) or not ref.get('id') or not ref.get('type'): + return None + return {'id': str(ref['id']), 'type': str(ref['type']), + 'revision': int(ref.get('revision') or 0)} + + +def bind_application(view, application): + """Copy a View shell and bind identity/revision only; Application config remains single-source.""" + import copy + import core.assets as assets + target = assets.application_open_target(application) + if not target: + raise ValueError('that Application has no stable typed target') + out = copy.deepcopy(view) if isinstance(view, dict) else {} + config = out.setdefault('config', {}) + config['application'] = {'id': target['id'], 'type': target['type'], + 'revision': int((application or {}).get('revision') or 0)} + return out diff --git a/platform/harness/semantic.py b/platform/harness/semantic.py index f9b6e5e77c53eabf826e3e3a82ecd2edf0f6b40f..f1b6c4aafadc584e71b348f74897aa21c5b1333d 100644 --- a/platform/harness/semantic.py +++ b/platform/harness/semantic.py @@ -160,6 +160,44 @@ def metrics(): return dict(_model()["metrics"]) +def semantic_fields(topic_key): + """Structured semantic suggestions for one topic, before tenant governance overrides. + + ``means`` remains accepted as the historical purpose source, but this projection is the one + consumers use: every field has all governance axes and enters as draft unless the model + explicitly disables it. Presence in a topic is never approval for retrieval. + """ + topic = _model()["topics"].get(str(topic_key)) + if not topic: + raise ModelError(f"unknown topic {topic_key!r}") + out = {} + for field in topic.get("fields") or (): + if not isinstance(field, dict) or not field.get("key"): + continue + meta = field.get("semantic") if isinstance(field.get("semantic"), dict) else {} + state = str(meta.get("state") or "draft").lower() + sensitivity = str(meta.get("sensitivity") or "internal").lower() + if state not in ("draft", "approved", "disabled"): + raise ModelError(f"topic {topic_key!r} field {field['key']!r}: bad semantic state") + if sensitivity not in ("public", "internal", "sensitive", "restricted"): + raise ModelError(f"topic {topic_key!r} field {field['key']!r}: bad sensitivity") + out[field["key"]] = { + "field": field["key"], "label": field.get("label") or field["key"], + "purpose": meta.get("purpose") or field.get("means") or "", + "dataType": field.get("type") or "text", + "role": meta.get("role") or "attribute", + "synonyms": list(meta.get("synonyms") or ()), + "sensitivity": sensitivity, "state": state, + } + return out + + +def metric_catalog(topic_key=None): + """Existing validated Metrics only; no customer measure is inferred from a numeric field.""" + return {key: dict(row) for key, row in _model()["metrics"].items() + if topic_key is None or row.get("topic") == topic_key} + + def describe(key): """Full metadata for a metric (the dictionary page / AI describe_topic feed).""" m = _model()["metrics"].get(key) diff --git a/platform/model/topics/odoo_customers.yml b/platform/model/topics/odoo_customers.yml index ae5759b10e026a190c11365ec0493457c8e69f60..845d4beec526bad83575a4fff19a33c0003c4191 100644 --- a/platform/model/topics/odoo_customers.yml +++ b/platform/model/topics/odoo_customers.yml @@ -35,11 +35,21 @@ fields: label: "Customer" type: text kind: data + semantic: + state: draft + role: entity_name + synonyms: [customer name, account name] + sensitivity: internal means: "The customer's name in Odoo. One row per customer who ordered in the last 24 months." - key: partner_id label: "Odoo ID" type: int kind: data + semantic: + state: draft + role: entity_id + synonyms: [customer id, Odoo partner id] + sensitivity: restricted means: "The Odoo res.partner id — this row's identity, and the key every Odoo document joins on. W33-T43 / owner item 12: 'One unique ID per database always.' It is DERIVED rather than read off the pool row because a customer row's pid IS the partner id, so the value is already on every row and a second copy in the pool would be a second source for one fact." - key: odoo_status label: "Odoo record" @@ -65,6 +75,11 @@ fields: label: "Street" type: text kind: data + semantic: + state: draft + role: address + synonyms: [address, street address] + sensitivity: sensitive means: "First address line, from res.partner directly - not the geocoder, so a customer the map cannot place still shows its address." - key: street2 label: "Street 2" @@ -215,6 +230,11 @@ fields: label: "Notes" type: text kind: data + semantic: + state: disabled + role: private_note + synonyms: [] + sensitivity: restricted means: "Your notes on this customer. Saved in this app only, visible only to you." ai_context: > diff --git a/platform/modules/product_data.py b/platform/modules/product_data.py index 4bd08b69a794c4174d4ad1909597df5b558a7e77..1f30b937fcb58544b351d192d1be768276c7205e 100644 --- a/platform/modules/product_data.py +++ b/platform/modules/product_data.py @@ -584,7 +584,7 @@ def _catalogue_by_code(): return products.catalogue() -def pool(team_id=None, t=None): +def pool(team_id=None, t=None, raw_fields=()): """One row per ACTIVE product — the PRODUCT analogue of `customer_data.pool`. CATALOGUE-FIRST, REVENUE LEFT-JOINED (R12 — see decision 4 in the module header). The row set @@ -609,7 +609,7 @@ def pool(team_id=None, t=None): # both degrade to `{}` on a read failure rather than taking the grid down: they are columns, # not the row set. try: - _prods = products.active_products() # ONE read, shared by both (see its docstring) + _prods = products.active_products(extra_fields=raw_fields) # ONE read, shared by both except Exception: # noqa: BLE001 _prods = None tiers, _tier_report = products.tier_prices_by_code(_prods) @@ -628,6 +628,8 @@ def pool(team_id=None, t=None): # out. `validate()` asserts that of Odoo rather than assuming it, and reports the revenue # that therefore sits outside the grid (MEASURED 2026-08-11: 3 codes, $0.00 YTD / $294.50 LY). rev = {r["code"]: r for r in products.directory(t=t, team_id=team_id)} + raw_by_code = {(p.get("default_code") or f"pid:{p.get('id')}"): p + for p in (_prods or []) if p.get("default_code") or p.get("id")} rows = [] for code, meta in cat.items(): @@ -687,6 +689,11 @@ def pool(team_id=None, t=None): "qty_ytd": r.get("qty_ytd", 0.0) if r else None, "orders_ytd": r.get("orders_ytd", 0) if r else None, } + raw = raw_by_code.get(code) or {} + if raw_fields and raw: + for field in raw_fields: + if field in raw: + row[field] = raw.get(field) # ⭐⭐ W37-T14 / T15 — THE HONEST SETS. These answer "how many prices/units does this SKU # actually have" across every list. The numbered per-list cells below select Fisch, # Royal 1, and Royal 2 without rebuilding a legacy base-price payload. diff --git a/platform/modules/products.py b/platform/modules/products.py index a1b821e5ee91075ffe1c42ee5e35e9e7e8183571..b03553451912b72a9f2d9d72ace84022e4d39829 100644 --- a/platform/modules/products.py +++ b/platform/modules/products.py @@ -554,7 +554,7 @@ def tier_cap_overflow(tiers): if name in declared and n > PRICELIST_TIER_DEPTH} -def active_products(limit=50000): +def active_products(limit=50000, extra_fields=()): """`[{id, default_code, product_tmpl_id}]` for every ACTIVE product — read ONCE and shared. ⛔ RAISES ON A SHORT PULL, like `catalogue()` and for the same reason: a truncated product read @@ -564,7 +564,8 @@ def active_products(limit=50000): on this connection, on a path that is somebody's first page load. """ dom = [('active', '=', True)] - prods = O.search_read('product.product', dom, ['id', 'default_code', 'product_tmpl_id'], + fields = ['id', 'default_code', 'product_tmpl_id'] + [str(f) for f in extra_fields if str(f) not in {'id', 'default_code', 'product_tmpl_id'}] + prods = O.search_read('product.product', dom, fields, limit=limit) n = O.get_odoo().search_count('product.product', dom) if len(prods) != n: diff --git a/web/src/assistant/AssistantPage.tsx b/web/src/assistant/AssistantPage.tsx index e4e3020dceb1a2788bd46203b6bbaab3183e5288..62e2e1d09e0e911289fd0fef4e2664866417a7c1 100644 --- a/web/src/assistant/AssistantPage.tsx +++ b/web/src/assistant/AssistantPage.tsx @@ -8,11 +8,13 @@ import { queryGroups, QueryRail } from "../query/queryParts"; import { DbIcon } from "../shell/dbFrame"; import { ASSISTANT_ROUTE, databaseEntries, QUERY_ROUTE, routeKeyOf } from "../shell/nav"; import type { NavEntry } from "../shell/nav"; +import { getQueryModelCatalog, persistQueryArtifacts, runQueryTool, type QueryModelIdentity, type QueryToolArtifacts } from "../automation/automationApi"; import { deleteQuery, deleteThread, fetchQueries, rateMessage, submitChat, type QueryCitation, type QueryIndex, type QueryMessage, type SavedQuery, } from "../query/queryApi"; import "./assistant.css"; +import "../styles/wave45/agent-operator.css"; /** * ⭐ LAZY ON PURPOSE. `QueryPage` imports `CustomerGrid`, the largest module in the client; a @@ -165,14 +167,27 @@ function ThumbIcon({ down = false }: { down?: boolean }) { ); } -export function AssistantMessage({ message, view, citations, onPreview, onRate }: { +export function AssistantMessage({ + message, + view, + citations, + onPreview, + onRate, + identity, + governedArtifacts, +}: { message: QueryMessage; view?: SavedQuery; citations: QueryCitation[]; onPreview: (id: string) => void; /** Optional so the message can be rendered outside a live conversation without a rating door. */ onRate?: (id: string, rating: "up" | "down" | null, reason?: string) => void; + identity?: QueryModelIdentity; + governedArtifacts?: QueryToolArtifacts; }) { const [openSources, setOpenSources] = useState(false); const [askWhy, setAskWhy] = useState(false); const [why, setWhy] = useState(""); + const [slackOpen, setSlackOpen] = useState(false); + const [artifacts, setArtifacts] = useState(governedArtifacts || null); + const [artifactStatus, setArtifactStatus] = useState(""); const matching = citations.filter((citation) => (message.citationIds || []).includes(citation.id)); if (message.role === "user") { return ( @@ -185,6 +200,61 @@ export function AssistantMessage({ message, view, citations, onPreview, onRate } return (

{message.content}

+

+ Provider {identity?.provider || message.model || "Not reported"} + Model {identity?.modelId || "Not reported"} +

+ {view && artifacts ? ( +
+

Query setup

+
+ {(["filters", "sorts", "group", "hiddenFields", "source"] as const).map((key) => ( + + ))} +
+
+ + + +
+ {artifactStatus ?

{artifactStatus}

: null} + {slackOpen ? ( +
+

Slack correction

+

Slack can correct Filter, Sort, and Group after the Query capability check. Complex edits open the exact server-returned Query in Loopable.

+ {(["filters", "sorts", "group"] as const).map((key) => )} +

Draft only. No Slack message was sent.

+ + +
+ ) : null} +
+ ) : null} {numeric && numeric.value !== null && numeric.value !== undefined ? (

{numeric.label} @@ -299,6 +369,13 @@ export default function AssistantPage({ granted }: AssistantPageProps) { const [activeQuery, setActiveQuery] = useState(""); /** Bumped on delete so the hosted workspace's own copy of the list cannot go stale behind us. */ const [queryRefresh, setQueryRefresh] = useState(0); + const [modelCatalog, setModelCatalog] = useState([]); + const [artifactByView, setArtifactByView] = useState>({}); + const [alignment, setAlignment] = useState; maxSelections: number; + }>(null); + const [alignmentFields, setAlignmentFields] = useState([]); const openCancel = useRef<(() => void) | null>(null); const foot = useRef(null); /** A counter, not a random id: the optimistic message only has to be unique within this page. */ @@ -347,12 +424,25 @@ export default function AssistantPage({ granted }: AssistantPageProps) { }, []); useEffect(() => { void reload(); }, [reload]); + useEffect(() => { + const controller = new AbortController(); + void getQueryModelCatalog(controller.signal).then(setModelCatalog).catch(() => setModelCatalog([])); + return () => controller.abort(); + }, []); useEffect(() => () => openCancel.current?.(), []); useEffect(() => { const permitted = databases.map((entry) => entry.key); setSelected((current) => current.filter((key) => permitted.includes(key))); setTarget((current) => permitted.includes(current) ? current : ""); }, [databases]); + // A resume token is bound to the database/source wall that produced it. Changing either + // invalidates the clarification instead of replaying the token against a different target. + useEffect(() => { + if (alignment && (alignment.database !== target || !selected.includes(alignment.database))) { + setAlignment(null); + setAlignmentFields([]); + } + }, [alignment, selected, target]); // The conversation reads bottom-up, like every chat the owner named. useEffect(() => { foot.current?.scrollIntoView({ block: "end" }); }, [messages.length, busy]); @@ -464,19 +554,35 @@ export default function AssistantPage({ granted }: AssistantPageProps) { * the model as if it had [[undo-capture-before-the-write]]. The typed text goes back in the * composer too: losing somebody's sentence to a failed request is the second injury. */ - const ask = useCallback(async () => { - const text = question.trim(); - if (!text || !target || !selected.includes(target) || busy) return; + const ask = useCallback(async (approvedFields?: string[]) => { + const text = approvedFields && alignment ? alignment.prompt : question.trim(); + const database = approvedFields && alignment ? alignment.database : target; + if (!text || !database || !selected.includes(database) || busy) return; + setBusy(true); setProblem(""); + let governed: Awaited>; + try { + governed = await runQueryTool({ database, prompt: text, + ...(approvedFields ? { fieldKeys: approvedFields, resumeToken: alignment?.resumeToken } : {}) }); + } catch (error) { + setBusy(false); setProblem(error instanceof Error ? error.message : "Query capability check failed."); return; + } + if (governed.status === "clarification") { + setBusy(false); setAlignmentFields([]); setAlignment({ prompt: text, database, + resumeToken: governed.resumeToken, question: governed.alignmentQuestion.question, + choices: governed.alignmentQuestion.choices, maxSelections: governed.alignmentQuestion.maxSelections }); + return; + } + setAlignment(null); const pendingId = `pending_${(pending.current += 1)}`; const optimistic: QueryMessage = { id: pendingId, threadId: activeThread, role: "user", content: text, - createdAt: new Date().toISOString(), targetDatabase: target, + createdAt: new Date().toISOString(), targetDatabase: database, sources: selected, requestedModel: model, }; const withoutPending = (rows: QueryMessage[]) => rows.filter((row) => row.id !== pendingId); - setBusy(true); setProblem(""); setQuestion(""); + setQuestion(""); setIndex((current) => ({ ...current, messages: [...current.messages, optimistic] })); - const result = await submitChat({ question: text, database: target, sources: selected, + const result = await submitChat({ question: text, database, sources: selected, ...(activeThread ? { threadId: activeThread } : {}), model }); setBusy(false); if (!result.ok) { @@ -486,6 +592,7 @@ export default function AssistantPage({ granted }: AssistantPageProps) { return; } setActiveThread(result.value.thread.id); + if (result.value.view) setArtifactByView((current) => ({ ...current, [result.value.view!.id]: governed.artifacts })); setIndex((current) => ({ ...current, threads: [result.value.thread, ...current.threads.filter((row) => row.id !== result.value.thread.id)], @@ -495,7 +602,7 @@ export default function AssistantPage({ granted }: AssistantPageProps) { views: result.value.view ? [result.value.view, ...current.views.filter((row) => row.id !== result.value.view?.id)] : current.views, citations: [...current.citations, ...result.value.citations], })); - }, [activeThread, busy, model, question, selected, target]); + }, [activeThread, alignment, busy, model, question, selected, target]); /** ⛔ The chat goes; the views it built STAY (they live in Query). Two-click, in the row. */ const removeThread = useCallback(async (id: string) => { @@ -533,6 +640,18 @@ export default function AssistantPage({ granted }: AssistantPageProps) { const empty = messages.length === 0; const composer = (

+ {alignment ?
+

Clarify

{alignment.question}

+
{alignment.choices.map((choice) => { + const on = alignmentFields.includes(choice.field); + return ; + })}
+ +
: null}