diff --git a/RELEASES.json b/RELEASES.json
index c016502df01c4233937531ee3bacb28410b8ed8c..8c4219301f5dfce4c2524459466c82266bc692d4 100644
--- a/RELEASES.json
+++ b/RELEASES.json
@@ -1,5 +1,5 @@
{
- "current": "v75 (2c51469)",
+ "current": "df68553 (df68553)",
"releases": [
{
"version": "v75",
diff --git a/VERSION b/VERSION
index 1b32611de31cb8a356b660771150e69fb2ed4a22..a26d6ccb396bf303c5ec200f29beb929cae28d05 100644
--- a/VERSION
+++ b/VERSION
@@ -1 +1 @@
-v75 (2c51469)
+df68553 (df68553)
diff --git a/api/application_registry.py b/api/application_registry.py
index 91b6a24cb7cafadad6adfc04366c8ae4608aad5b..56f880fa04026c7da219319fb4b4a816234b9492 100644
--- a/api/application_registry.py
+++ b/api/application_registry.py
@@ -2,12 +2,20 @@
from __future__ import annotations
import copy
-import datetime as dt
-import uuid
+from core import write_execution
+
+from modules.finance_apps import (finance_application as _finance_application,
+ finance_applications as _finance_applications,
+ finance_group as _finance_group)
+from modules.finance_spine import fact_specs as _finance_fact_specs
APPLICATION_TYPES = ("chart", "route", "map", "timeseries")
-APPLICATION_TEMPLATE_KINDS = ("route", "catalog")
+# The wizard binds a database and fields. Finance templates use the catalogue POST instead;
+# validation accepts both creation doors without offering Finance to the five-step wizard.
+APPLICATION_WIZARD_TEMPLATE_KINDS = ("route", "catalog")
+FINANCE_TEMPLATE_KIND = "finance"
+APPLICATION_TEMPLATE_KINDS = APPLICATION_WIZARD_TEMPLATE_KINDS + (FINANCE_TEMPLATE_KIND,)
_KEY = "applications_v1"
_MAX = 500
@@ -48,6 +56,15 @@ _TEMPLATE_CONTRACTS = {
],
"defaultPlacement": {"section": "app", "order": 0},
},
+ "finance": {
+ "kind": "finance",
+ "label": "Finance",
+ # Like Catalog, templateKind selects a trusted surface without growing APPLICATION_TYPES.
+ "applicationType": "chart",
+ "displayMode": "chart",
+ "fieldRoles": [],
+ "defaultPlacement": {"section": "app", "order": 0},
+ },
}
@@ -57,7 +74,130 @@ class ApplicationError(ValueError):
def template_contracts():
"""The trusted renderer contracts offered by the Application creation flow."""
- return [copy.deepcopy(_TEMPLATE_CONTRACTS[kind]) for kind in APPLICATION_TEMPLATE_KINDS]
+ return [copy.deepcopy(_TEMPLATE_CONTRACTS[kind]) for kind in APPLICATION_WIZARD_TEMPLATE_KINDS]
+
+
+def finance_group():
+ """The one immutable Finance group consumed by navigation and direct-link routes."""
+ return _finance_group()
+
+
+def finance_applications(may_open=None):
+ """Six separately addressable Finance application descriptors."""
+ return _finance_applications(may_open=may_open)
+
+
+# R6: Finance records are read-only projections of finance_apps.py, never validated, created,
+# mutated, or persisted. Adding "finance" to APPLICATION_TYPES buys nothing and weakens
+# validate_application's governed type vocabulary.
+# This remains true for the SEEDED six. Users may now persist their own Finance applications
+# through templateKind, pointing to the same trusted surface without modifying those projections.
+def finance_application_records(may_open=None):
+ """Project the grant-filtered Finance catalogue as seeded Application records."""
+ return [
+ {"id": f"finance_{app['key']}",
+ "key": app["key"],
+ "name": app["navLabel"],
+ "type": "finance",
+ "templateKind": None,
+ "origin": "finance",
+ "permissionKey": app["permissionKey"],
+ "mode": app["mode"],
+ "question": app["question"],
+ "deepLink": app["deepLink"],
+ "compatibleDatabases": [],
+ "navPlacement": {"section": "finance", "order": index},
+ "openTarget": {"kind": "finance", "id": f"finance_{app['key']}", "type": "finance",
+ "database": None},
+ "capabilities": {"read": True, "edit": False, "share": False, "delete": False,
+ "duplicate": False, "star": False, "default": False},
+ "starred": False,
+ "isDefault": False}
+ for index, app in enumerate(finance_applications(may_open=may_open))
+ ]
+
+
+_FINANCE_SCOPES = {"business_unit", "hq_consolidated", "all_channel_inventory"}
+
+
+def _require_unavailable(value, context):
+ if not isinstance(value, dict) or value.get("state") != "unavailable":
+ raise RuntimeError(f"{context} must be explicitly unavailable")
+ if "value" in value or not value.get("missingSource"):
+ raise RuntimeError(f"{context} cannot fabricate a missing value")
+
+
+def validate_finance_application_contract(app):
+ """Fail closed when an app bypasses its declared workflow/dashboard or fact boundary."""
+ key = str(app.get("key") or "unknown")
+ mode = app.get("mode")
+ worklist = app.get("worklist")
+ dashboard = app.get("dashboard")
+ fact_keys = set(app.get("factKeys") or ())
+ if mode not in {"workflow", "dashboard"}:
+ raise RuntimeError(f"invalid Finance application mode for {key}")
+ if mode == "workflow":
+ if not isinstance(worklist, dict) or dashboard:
+ raise RuntimeError(f"invalid Finance worklist contract for {key}")
+ source_fact = worklist.get("sourceFact")
+ if source_fact:
+ if source_fact not in fact_keys:
+ raise RuntimeError(f"invalid Finance worklist source for {key}")
+ required = ("ownerField", "statusField", "dueDateField", "nextAction")
+ if not all(worklist.get(name) for name in required):
+ raise RuntimeError(f"incomplete Finance worklist fields for {key}")
+ elif worklist.get("kind") == "unavailable_requirements":
+ signals = worklist.get("signals")
+ if not isinstance(signals, list) or not signals:
+ raise RuntimeError(f"Finance requirements worklist {key} has no signals")
+ for signal in signals:
+ _require_unavailable(signal, f"Finance work signal in {key}")
+ if signal.get("scope") not in _FINANCE_SCOPES:
+ raise RuntimeError(f"Finance work signal in {key} has no governed scope")
+ if not all(signal.get(name) for name in (
+ "key", "label", "basisLabel", "sourceModule", "validationMethod",
+ "ownerRole", "status", "nextAction")):
+ raise RuntimeError(f"Finance work signal in {key} is ungrounded")
+ if signal.get("status") != "unavailable":
+ raise RuntimeError(f"Finance work signal in {key} claims executable state")
+ else:
+ raise RuntimeError(f"Finance worklist {key} has no governed source")
+ elif worklist or not isinstance(dashboard, dict):
+ raise RuntimeError(f"invalid Finance dashboard contract for {key}")
+
+ for section in (dashboard or {}).get("sections", []):
+ for metric in section.get("metrics", []):
+ if metric.get("sourceFact") and metric["sourceFact"] not in fact_keys:
+ raise RuntimeError(f"dashboard {key} references an ungoverned source fact")
+ if metric.get("state") == "unavailable":
+ _require_unavailable(metric, f"dashboard metric in {key}")
+ if metric.get("scope") not in _FINANCE_SCOPES:
+ raise RuntimeError(f"dashboard metric in {key} has no governed scope")
+ if not metric.get("basisLabel") or not metric.get("validationMethod"):
+ raise RuntimeError(f"dashboard metric in {key} is ungrounded")
+ for row in section.get("rows", []):
+ for column in ("actual", "budget", "prior"):
+ cell = row.get(column) or {}
+ if cell.get("state") == "fact" and cell.get("factKey") not in fact_keys:
+ raise RuntimeError(f"dashboard {key} references an ungoverned fact")
+ for row in section.get("planningRows", []):
+ if row.get("scope") not in _FINANCE_SCOPES or not row.get("basisLabel"):
+ raise RuntimeError(f"planning row in {key} is ungrounded")
+ actual = row.get("actual") or {}
+ if actual.get("state") != "fact" or actual.get("factKey") not in fact_keys:
+ raise RuntimeError(f"planning row in {key} references an ungoverned actual")
+ for column in ("plan", "variance", "driver"):
+ _require_unavailable(row.get(column), f"planning {column} in {key}")
+ return True
+
+
+def finance_application(key, may_open=None):
+ """Resolve one Finance application without constructing a Finance-home payload."""
+ app = _finance_application(key, may_open=may_open)
+ validate_finance_application_contract(app)
+ app["facts"] = _finance_fact_specs(app["factKeys"])
+ app["kpis"] = _finance_fact_specs(app["kpiKeys"])
+ return app
def blank_doc():
@@ -205,6 +345,24 @@ def _template_definition(kind, payload, *, semantic_records=None, field_contract
template_kind = str(payload.get("templateKind") or "").strip().lower()
if template_kind != kind:
raise ApplicationError(f"{kind} applications require templateKind {kind!r}")
+ if kind == FINANCE_TEMPLATE_KIND:
+ finance_key = str(payload.get("financeKey") or "").strip()
+ try:
+ app = _finance_application(finance_key)
+ except KeyError:
+ raise ApplicationError(f"Finance application {finance_key!r} does not exist.")
+ if payload.get("sourceDatabase") or payload.get("fieldMap") or payload.get("compatibleDatabases"):
+ raise ApplicationError("Finance applications do not bind a database or fields.")
+ raw_placement = payload.get("navPlacement")
+ if not isinstance(raw_placement, dict) or set(raw_placement) != {"section", "order"}:
+ raise ApplicationError("navPlacement must contain section and order")
+ if raw_placement.get("section") != "app":
+ raise ApplicationError("Finance applications must be placed in App.")
+ order = raw_placement.get("order")
+ if type(order) is not int or not 0 <= order <= 999:
+ raise ApplicationError("navPlacement order must be an integer from 0 to 999")
+ return {"templateKind": kind, "financeKey": finance_key, "deepLink": app["deepLink"],
+ "navPlacement": {"section": "app", "order": order}}
source = str(payload.get("sourceDatabase") or "").strip()[:120]
if not source:
raise ApplicationError("sourceDatabase is required")
@@ -305,6 +463,13 @@ def _public(app, user, is_admin=False, *, allowed_databases=None, target_databas
"default":bool(is_admin)}
row["starred"] = user in (app.get("starredBy") or ())
row["isDefault"] = bool(is_default)
+ if row.get("templateKind") == FINANCE_TEMPLATE_KIND:
+ # The database-bound mutation doors cannot edit Finance definitions. Honour that in
+ # every card rather than displaying controls that can only refuse.
+ row["capabilities"].update({key: False for key in (
+ "edit", "share", "delete", "duplicate", "star", "default")})
+ row["starred"] = False
+ row["isDefault"] = False
target_database = target_database if target_database in compatible else (compatible[0] if compatible else None)
row["openTarget"]={"kind":"application","id":row["id"],"type":row["type"],
"database":target_database}
@@ -318,7 +483,8 @@ def list_apps(runtime, user, is_admin=False, *, database=None, kind=None, starre
for app in doc["applications"].values():
if not may_read(app,user,is_admin): continue
if database and database not in (app.get("compatibleDatabases") or ()): continue
- if not _allowed(app.get("compatibleDatabases"), allowed_databases): continue
+ if (app.get("templateKind") != FINANCE_TEMPLATE_KIND
+ and not _allowed(app.get("compatibleDatabases"), allowed_databases)): continue
if kind and app.get("type") != kind: continue
if starred is not None and bool(user in (app.get("starredBy") or ())) != bool(starred): continue
default_key = f"{database}::{app.get('type')}" if database else ""
@@ -328,6 +494,22 @@ def list_apps(runtime, user, is_admin=False, *, database=None, kind=None, starre
return sorted(rows,key=lambda x:(str(x.get("updatedAt") or ""),str(x.get("id"))),reverse=True)
+def nav_applications(runtime, user, is_admin=False, *, may_open=None, allowed_databases=None):
+ """The ONE generic navigation read for stored and seeded Application records (R6)."""
+ rows = [{"id": app["id"], "name": app.get("name"),
+ "type": app.get("type"), "templateKind": app.get("templateKind"),
+ "deepLink": app.get("deepLink"),
+ "origin": "tenant",
+ "compatibleDatabases": app.get("compatibleDatabases") or [],
+ "openTarget": app.get("openTarget"),
+ "navPlacement": app.get("navPlacement") or {"section": "app", "order": index},
+ "capabilities": app.get("capabilities"), "starred": app.get("starred"),
+ "isDefault": app.get("isDefault")}
+ for index, app in enumerate(list_apps(runtime, user, is_admin,
+ allowed_databases=allowed_databases))]
+ return rows + finance_application_records(may_open=may_open)
+
+
def create(runtime, tenant, user, payload, *, semantic_records=None, field_contracts=None,
allowed_databases=None):
payload=payload if isinstance(payload,dict) else {}
@@ -339,8 +521,9 @@ def create(runtime, tenant, user, payload, *, semantic_records=None, field_contr
allowed_databases=allowed_databases)
name=" ".join(str(payload.get("name") or "").split())[:120]
if not name: raise ApplicationError("application name is required")
- app_id=f"app_{uuid.uuid4().hex[:16]}"; now=dt.datetime.now(dt.timezone.utc).isoformat()
- compatible = ([definition["sourceDatabase"]] if definition
+ app_id=f"app_{write_execution.new_id()[:16]}"; now=write_execution.utcnow().isoformat()
+ compatible = ([] if template_kind == FINANCE_TEMPLATE_KIND
+ else [definition["sourceDatabase"]] if definition
else _compat(payload.get("compatibleDatabases")))
if _allowed(compatible, allowed_databases) != set(compatible):
raise PermissionError("an Application cannot bind a database you cannot open")
@@ -368,7 +551,8 @@ def mutate(runtime, app_id, user, is_admin, patch, *, semantic_records=None,
allowed_databases=None, target_database=None):
patch=patch if isinstance(patch,dict) else {}; landed={}
immutable = sorted(set(patch) & {
- "id", "tenant", "type", "templateKind", "sourceDatabase", "fieldMap", "navPlacement"
+ "id", "tenant", "type", "templateKind", "sourceDatabase", "fieldMap", "navPlacement",
+ "financeKey", "deepLink"
})
if immutable:
raise ApplicationError("Application identity and template definition are immutable: "
@@ -409,7 +593,7 @@ def mutate(runtime, app_id, user, is_admin, patch, *, semantic_records=None,
stars=set(app.get("starredBy") or ())
(stars.add if patch["starred"] else stars.discard)(user); app["starredBy"]=sorted(stars)
app["revision"]=int(app.get("revision") or 0)+1
- app["updatedAt"]=dt.datetime.now(dt.timezone.utc).isoformat(); landed.update(app)
+ app["updatedAt"]=write_execution.utcnow().isoformat(); landed.update(app)
return doc
runtime.update(_KEY,apply)
return _public(landed,user,is_admin,allowed_databases=allowed_databases,
diff --git a/api/automation_engine.py b/api/automation_engine.py
index 7d7e986f3580900519a7523825e5a3e77d946fd8..2b5306fdb428c3bfbb4d96ea11c682fe7a261735 100644
--- a/api/automation_engine.py
+++ b/api/automation_engine.py
@@ -34,7 +34,7 @@ import re
import socket
import threading
import time
-from urllib.parse import urljoin, urlparse
+from urllib.parse import quote, urljoin, urlparse
from zoneinfo import ZoneInfo, ZoneInfoNotFoundError
import requests
@@ -61,6 +61,25 @@ STORE_KEY = "automations"
AGENT_CONTRACT_KEY = "agent_contracts"
AGENT_EVENT_TYPES = ("transcript", "message", "gathered_data", "run", "consent")
AGENT_ACTIONS = ("read", "create", "update", "overwrite", "delete")
+
+#: MIGRATION NOTE (SDLC 64 / R3, 2026-09-21) — the only surviving occurrence of the
+#: removed preset name. D-295 records why this must remain: a person's stored automation
+#: must be dropped with a named reason, not refused on every save forever. Every read
+#: drops these definitions; `retire_preset_automations` also sweeps them from the store.
+RETIRED_PRESETS = {
+ "customer_intelligence": (
+ "Customer intelligence has been removed from Loopable. This automation's trigger and "
+ "its insight action no longer exist, so it can no longer run and has been removed."),
+}
+
+
+def retired_preset_reason(definition):
+ """The sentence to show for a stored definition whose preset is gone, else ``""``."""
+ if not isinstance(definition, dict):
+ return ""
+ return RETIRED_PRESETS.get(str(definition.get("preset") or "").strip(), "")
+
+
#: ⭐⭐ W47-T21 / R3 — THE THREE NAMED REASONS AN AGENT'S ROW FILTER REFUSES WITH.
#:
#: R3, verbatim: an agent's row filter is *"its whole world: it may neither read nor write a
@@ -2218,13 +2237,18 @@ def clean_definition(raw, previous=None, username="", notes=None, rt=None):
"""Whole-definition validation. Returns `(defn, error)`."""
raw = raw if isinstance(raw, dict) else {}
prev = previous or {}
+ preset = _s(raw.get("preset") if "preset" in raw else prev.get("preset"), 60).strip()
+ if preset in RETIRED_PRESETS:
+ return None, RETIRED_PRESETS[preset]
+ if preset:
+ return None, f"unknown automation preset {preset!r}"
# ⭐ WAVE 24 · C-TRIG — THE TRIGGER IS RESOLVED FIRST NOW, because law 1 makes it the thing
# that CHOOSES the kind. Reading `raw["trigger"]["key"]` here instead would be a second
# reader of the trigger vocabulary, free to disagree with `clean_trigger` about which keys
# exist and which are refused. Gate-visible consequence, recorded in AMENDMENT A1: a payload
# invalid in BOTH its trigger and its config now answers with the trigger's sentence.
- trigger, terr = clean_trigger(raw.get("trigger") if "trigger" in raw
- else prev.get("trigger"), prev.get("trigger"))
+ trigger_raw = raw.get("trigger") if "trigger" in raw else prev.get("trigger")
+ trigger, terr = clean_trigger(trigger_raw, prev.get("trigger"))
if terr:
return None, terr
# C-TRIG law 2 (wave 24): a definition that names no kind is a `plain` one. Before R6 this
@@ -2619,6 +2643,56 @@ def retire_automation_board_state(rt, tables=None):
return {"definitions": len(plan), **fields}
+def _no_preset_retirement():
+ """The receipt when nothing was retired: the SAME keys the success path answers with.
+
+ A FUNCTION rather than a module constant, because the receipt is handed to a caller and a
+ shared literal would let one caller's edit reach the next one.
+ """
+ return {"definitions": 0, "ids": [], "reasons": {}, "removed": []}
+
+
+def retire_preset_automations(rt):
+ """Persist retired-definition removal and name each removed automation in the log.
+
+ ⭐⭐ THE RECEIPT CARRIES `removed` SO THE CALLER NEEDS NO SECOND READ. `list_automations` has
+ to tell the reader WHICH automation went and why, and the obvious way to get the names — read
+ the bucket again beside the sweep — is the defect the WAVE 30 · T12 comment block above that
+ call site is the tombstone of: a whole-tenant document re-serialised under the store lock on
+ every single request, to report something that is empty on all but one of them. This function
+ has already read every row it removed, so it hands them back.
+ """
+ try:
+ stored = dict(rt.get(STORE_KEY) or {})
+ except Exception:
+ return _no_preset_retirement()
+ plan = {aid for aid, definition in stored.items() if retired_preset_reason(definition)}
+ if not plan:
+ return _no_preset_retirement()
+ removed = {}
+
+ def _up(cur):
+ cur = cur if isinstance(cur, dict) else {}
+ # A store retry replays this callback. Report only the successful attempt, and
+ # recheck each row so an intervening edit to a live definition is preserved.
+ removed.clear()
+ for aid in sorted(plan):
+ definition = cur.get(aid)
+ reason = retired_preset_reason(definition)
+ if reason:
+ removed[str(aid)] = (str(definition.get("name") or "Unnamed automation"), reason)
+ cur.pop(aid)
+ return cur
+
+ _store_update(rt, _up, flush="sync")
+ for aid, (name, reason) in removed.items():
+ print(f"[aios-auto] removed automation {aid!r} ({name!r}): {reason}")
+ return {"definitions": len(removed), "ids": list(removed),
+ "reasons": {aid: reason for aid, (_name, reason) in removed.items()},
+ "removed": [{"id": aid, "name": name, "reason": reason}
+ for aid, (name, reason) in removed.items()]}
+
+
#: ⭐⭐ WAVE 31 · T35 (D-134) — the definitions memo, and the two things that make it safe.
#: `tenant -> (monotonic_at, defs)`. Read ONLY through `all_definitions(..., cached=True)`, which
#: exactly one caller uses (`grid_hook`).
@@ -2696,7 +2770,8 @@ def all_definitions(rt, cached=False):
raw = dict(rt.get(STORE_KEY) or {})
except Exception:
return {}
- out = {str(aid): _without_retired_board(defn)[0] for aid, defn in raw.items()}
+ out = {str(aid): _without_retired_board(defn)[0] for aid, defn in raw.items()
+ if not retired_preset_reason(defn)}
if cached:
_DEFS_MEMO[str(getattr(rt, "key", "") or "")] = (time.monotonic(), out)
return out
@@ -2813,7 +2888,11 @@ def agent_contract_dto(rt, agent_id):
def authorize_agent_principal(rt, agent_id, actor, is_admin=False):
- rec = _agent_contracts(rt).get(str(agent_id or "")) or {}
+ rec = _agent_contracts(rt).get(str(agent_id or ""))
+ # An admin may read a live private Agent; that role cannot resurrect a deleted Agent
+ # or replace the user identity required for authorization.
+ if not str(actor or "").strip() or not isinstance(rec, dict) or not rec:
+ raise PermissionError("this Agent or its principal is no longer available")
sharing = rec.get("sharing") or {}
if is_admin or rec.get("createdBy") == str(actor or "") or str(actor or "") in set(sharing.get("sharedWith") or []):
return True
@@ -3353,7 +3432,7 @@ def patch(rt, auto_id, raw, username="", notes=None):
# DISCARDED — the client shows a saved flow that the store never received, and nothing goes
# red. `flow` joined it in the same edit that created `flow` (wave 23), which is the only
# ordering that never has a window where the bug exists.
- for k in ("name", "config", "schedule", "kind", "trigger", "flow"):
+ for k in ("name", "config", "schedule", "kind", "trigger", "flow", "preset"):
if k in (raw or {}):
merged[k] = raw[k]
# 2026-08-10 — the PATCH door needs it too, and this is the one that actually bit: picking the
@@ -9958,8 +10037,6 @@ ACTION_CATALOG = [
"ready": True, "menu": False,
"detail": "Fill an AI column for the records this flow walks. Configured on the column, not "
"here"},
- {"kind": "slack", "label": "Send Slack message", "group": "Connected", "ready": False,
- "detail": "Needs the Slack connector"},
# ── 4. ADVANCED LOGIC ────────────────────────────────────────────────────────────────────
# `group` is relabelled "If / then" (C-ACT): "Conditional logic" described the mechanism,
# and R8 made it a FORK with lettered branches, which is a thing people already have a name
@@ -15492,7 +15569,10 @@ def _row_gate(rt, defn, trg):
"""
key = trg.get("key")
if key == "enters_view":
- tree, fields, problem = view_filter(rt, trg.get("table"), trg.get("viewId"))
+ # Native View intelligence keeps the C2 trigger exact (`kind` + `viewId`); its table is
+ # the definition's source binding rather than a second, client-editable trigger field.
+ tree, fields, problem = view_filter(
+ rt, trg.get("table") or _flow_table(defn), trg.get("viewId"))
if problem:
return None, problem
import harness.filter_eval as filter_eval
@@ -15527,7 +15607,7 @@ def _settle_eval(rt, tenant, auto_id, log=print):
return
fired = []
if gate is not None:
- rows = (ut_get(rt, trg.get("table") or "") or {}).get("rows") or {}
+ rows = (ut_get(rt, trg.get("table") or _flow_table(d)) or {}).get("rows") or {}
disarmed = set((d.get("state") or {}).get("eventDisarmed") or [])
nxt = set(disarmed)
for rid in touched:
@@ -15572,7 +15652,7 @@ def _seed_event_state(rt, defn):
gate, problem = _row_gate(rt, defn, trg)
if gate is None or problem:
return # no match gate ⇒ nothing to arm; a broken gate seeds nothing
- rows = (ut_get(rt, trg.get("table") or "") or {}).get("rows") or {}
+ rows = (ut_get(rt, trg.get("table") or _flow_table(defn)) or {}).get("rows") or {}
matching = sorted(str(rid) for rid, row in rows.items() if gate(row or {}))
set_state(rt, defn.get("id"), {"eventDisarmed": matching[:5000]})
@@ -15600,10 +15680,11 @@ def grid_hook(evt):
for aid, d in defs.items():
trg = (d or {}).get("trigger") or {}
key = str(trg.get("key") or "")
+ source_table = str(trg.get("table") or _flow_table(d) or "")
if trg.get("paused") or not trg.get("enabled", True) \
or not trg.get("configured", True) \
or key not in TRIGGER_ROW_KEYS \
- or str(trg.get("table") or "") != table:
+ or source_table != table:
continue
if kind == "record_created" and key == "record_created":
hw = _ig_int((d.get("state") or {}).get("rcHighwater")) or 0
@@ -16060,7 +16141,9 @@ def run_now(rt, tenant, auto_id, username="automation", log=print, rows=None):
# inside `dispatch_authorized_slack_automation` before its reply sentence is composed;
# notification trouble must never rewrite the truthful outcome of work that already ran.
try:
- dispatch_authorized_slack_automation(rt, defn, {"state": state})
+ dispatch_authorized_slack_automation(
+ rt, defn, {"state": state, "summary": summary,
+ "rowIds": affected or []})
except Exception as e: # noqa: BLE001
log(f"[aios-auto] Slack notification for {auto_id} failed: {type(e).__name__}: {e}")
return _commit_run(rt, auto_id, state, summary, counts, state != "error", affected,
diff --git a/api/connectors_ig.py b/api/connectors_ig.py
index adf95fd225694aa975c32b745a8e91995d818519..eaeaa04ac4c61514e79c5bfc6566b330364eecfb 100644
--- a/api/connectors_ig.py
+++ b/api/connectors_ig.py
@@ -1652,3 +1652,33 @@ def pull_profile(url, max_posts=None, log=print,
"no provider could read this profile "
f"({'; '.join(attempts) or 'blocked'})")}
+
+def pull_ig_recipe(seed, limit, post_type="video", log=print):
+ """Run one saved Instagram database recipe through the established paid runner.
+
+ This deliberately stays a very small adapter over :func:`pull_profile`. The database route
+ owns persistence and the explicit ``Run now`` boundary; this module owns the vendor's native
+ last-N/type vocabulary. Passing ``post_groups`` is load-bearing: it reaches the existing
+ reels-only specialization (Apify when configured, otherwise Bright Data's native
+ ``post_type: reel`` request) instead of buying a mixed window and filtering it afterwards.
+ """
+ try:
+ n = int(limit)
+ except (TypeError, ValueError):
+ return {"state": "error", "profile": {}, "posts": [], "comments": [],
+ "via": "", "note": "the post limit must be a whole number"}
+ kind = str(post_type or "").strip().lower()
+ if kind not in ("video", "image", "carousel"):
+ return {"state": "error", "profile": {}, "posts": [], "comments": [],
+ "via": "", "note": "choose video, image, or carousel posts"}
+ if n < 1 or n > MAX_POST_WINDOW:
+ return {"state": "error", "profile": {}, "posts": [], "comments": [],
+ "via": "", "note": f"the post limit must be between 1 and {MAX_POST_WINDOW}"}
+ return pull_profile(
+ seed,
+ max_posts=n,
+ post_metrics=True,
+ comment_metrics=False,
+ post_groups=[{"type": kind, "limit": n}],
+ log=log,
+ )
diff --git a/api/connectors_tt.py b/api/connectors_tt.py
index 3a692a7f2505cee8db8e0f3cf6950c4d7045e52b..dec3adfdb703a9c86552bdeac5879076f34333db 100644
--- a/api/connectors_tt.py
+++ b/api/connectors_tt.py
@@ -337,7 +337,7 @@ def tt_handle(url):
return s if s and all(c.isalnum() or c in "._" for c in s) else ""
-def tt_post_urls(node, limit=0):
+def tt_post_urls(node, limit=0, post_types=None):
"""⭐ WAVE 30 · T10 — the profile row's own post permalinks, newest-first as the vendor sends.
⛔ THIS IS WHY TIKTOK POST CAPTURE COSTS NO EXTRA DISCOVERY. `top_videos` rides the PROFILE row
@@ -363,6 +363,7 @@ def tt_post_urls(node, limit=0):
not a number this function may invent.
"""
raw = (node or {}).get("top_videos")
+ wanted = {str(value or "").strip().lower() for value in (post_types or ()) if str(value or "").strip()}
out = []
for item in raw if isinstance(raw, list) else []:
# ⛔⛔ MEASURED ON A REAL ROW 2026-08-12, AND IT IS NOT WHAT THE SCHEMA SAID.
@@ -389,8 +390,17 @@ def tt_post_urls(node, limit=0):
# whichever array happened to be longer is how one creator's window silently differs from
# another's, and that reasoning is unchanged by this correction.
if isinstance(item, dict):
+ # ``top_videos`` is the vendor's native video window. Keep the type selection at
+ # this boundary so an excluded type never reaches the paid Posts dataset. A future
+ # vendor row that starts declaring photo-mode entries can use its own ``post_type``;
+ # today's measured rows have no such key and are videos by construction.
+ delivered_type = str(item.get("post_type") or "video").strip().lower()
+ if wanted and delivered_type not in wanted:
+ continue
u = str(item.get("video_url") or item.get("url") or "").strip()
else:
+ if wanted and "video" not in wanted:
+ continue
u = str(item or "").strip()
if u.lower().startswith("http") and u not in out:
out.append(u)
@@ -458,7 +468,8 @@ def _media_deferrals(deferred):
def pull_profile_tt(url, log=print, pending_profile=None, prefetch=None,
- max_posts=0, post_metrics=False, comment_metrics=False):
+ max_posts=0, post_metrics=False, comment_metrics=False,
+ post_types=None):
"""ONE TikTok profile from the vendor. Same return contract as `pull_profile`.
`{state, profile, posts, comments, via, note}` with `state ∈ ok | partial | blocked | error`,
@@ -529,7 +540,7 @@ def pull_profile_tt(url, log=print, pending_profile=None, prefetch=None,
return {"state": "partial", "profile": profile, "posts": [], "comments": [],
"via": "brightdata",
"note": note or "profile read; post capture is off for this step"}
- urls = tt_post_urls(node, limit=max_posts)
+ urls = tt_post_urls(node, limit=max_posts, post_types=post_types)
if not urls:
# ⚠ NOT AN ERROR AND NOT A RETRY. A creator with no `top_videos` has nothing to buy, and
# saying so is what stops the next run paying to be told the same thing.
@@ -566,3 +577,32 @@ def pull_profile_tt(url, log=print, pending_profile=None, prefetch=None,
return {"state": "ok", "profile": profile, "posts": posts, "comments": comments,
"via": "brightdata", "deferredMedia": deferred_media,
"note": c_note or note or ""}
+
+
+def pull_tt_recipe(seed, limit, post_type="video", log=print):
+ """Run one saved TikTok database recipe through the established paid runner.
+
+ TikTok's measured profile payload exposes a native ``top_videos`` window, so ``limit`` and
+ ``post_types`` are applied before the paid Posts scrape. Keeping this adapter beside the
+ vendor runner makes the route unable to silently regress to an after-the-fact UI filter.
+ """
+ try:
+ n = int(limit)
+ except (TypeError, ValueError):
+ return {"state": "error", "profile": {}, "posts": [], "comments": [],
+ "via": "", "note": "the post limit must be a whole number"}
+ kind = str(post_type or "").strip().lower()
+ if kind != "video":
+ return {"state": "error", "profile": {}, "posts": [], "comments": [],
+ "via": "", "note": "TikTok's native profile window supports video posts"}
+ if n < 1 or n > 30:
+ return {"state": "error", "profile": {}, "posts": [], "comments": [],
+ "via": "", "note": "the post limit must be between 1 and 30"}
+ return pull_profile_tt(
+ seed,
+ max_posts=n,
+ post_metrics=True,
+ comment_metrics=False,
+ post_types=(kind,),
+ log=log,
+ )
diff --git a/api/deps.py b/api/deps.py
index e72935107ebf55076734ad9fe7c6cd8a32d1f7b8..19f4fc91afbbb27014eee560230ac435957cf04d 100644
--- a/api/deps.py
+++ b/api/deps.py
@@ -37,6 +37,35 @@ def err(status, code, message):
return HTTPException(status_code=status, detail={"error": {"code": code, "message": message}})
+
+def running_main():
+ """The `main` module THIS PROCESS IS ACTUALLY RUNNING, or None. Never an `import`.
+
+ ⛔⛔ D-781 — `import main` IN A ROUTE MODULE STARTED A SECOND COPY OF EVERY RESIDENT DAEMON.
+ The Space runs `uvicorn api.main:app`, so the module executing is `api.main`. Python keys
+ `sys.modules` by the imported NAME, so a plain `import main` from a route file is a DIFFERENT
+ module object and **re-executes main.py's whole body** — including its module-level
+ `threading.Thread(target=_prewarm, …).start()` and `threading.Thread(target=_store_resync_loop,
+ …).start()`. Two resync loops, two prewarms, in one single-worker container.
+
+ ⛔ AND IT COST REAL MONEY. Staging's own log showed the tell twice over: two `prewarm done`
+ lines from ONE boot (46.3s and 110.3s, 2026-09-21T11:35:24) and two
+ `odoo relational rebuild required`/`done` pairs per resync pass, every pass — double the
+ whole-document Neon reads, on a bill whose entire overage was read egress.
+
+ ⛔⛔ AND THE STATUS ROUTE WAS READING THE SHADOW. `_resync_state` pulled `RESYNC_STATE` from the
+ duplicate module — a second, separately-populated dict belonging to the second loop — so the
+ operator surface answering *"is the mirror advancing"* was describing the copy nobody meant to
+ start. That is the worse half: the cost was invisible and the disclosure was wrong.
+
+ ⚠ IT RESOLVES, IT NEVER IMPORTS, and that restraint is the whole fix: falling back to
+ `import main` when neither name is loaded would reintroduce exactly the defect. `api.main`
+ first (uvicorn), then `main` (the gates, which import it under that name) — and None when this
+ process is running neither, which each caller already handles.
+ """
+ return sys.modules.get("api.main") or sys.modules.get("main")
+
+
@dataclass
class Session:
"""A verified request identity. `user` is the PUBLIC record (never a hash or a salt)."""
diff --git a/api/formula_runtime.py b/api/formula_runtime.py
index 84ba9960a8fb91b0d9dc32414b3e2c17833eff12..a306d38c44d0bca2e737fefcd5024c2515ff22e0 100644
--- a/api/formula_runtime.py
+++ b/api/formula_runtime.py
@@ -84,7 +84,10 @@ FN_ARITY = {
"OFFSET": (3, 5), "INDIRECT": (1, 2), "RAND": (0, 0),
"FILTER": (2, 3), "SORT": (1, 4),
}
-ALIASES = {"CONCAT": "CONCATENATE"}
+# Excel workbooks in the Daily Sales lineage use both spellings. Keep the authored formula text
+# untouched in the workbook; normalization maps AVG to the same AST/function the catalogue's
+# Average action emits, so quoted-sheet references and cached formula text still round-trip.
+ALIASES = {"CONCAT": "CONCATENATE", "AVG": "AVERAGE"}
#: ⭐ THE TRANSPORT SPELLING OF A DOTTED EXCEL NAME, AND IT IS NOT AN ALIAS.
#:
@@ -1755,6 +1758,26 @@ def _eval_value(node, row, today, diagnostics=None):
if isinstance(row, _SheetInputs):
left, right = 0 if left is None else left, 0 if right is None else right
left, right = int(left) if isinstance(left, bool) else left, int(right) if isinstance(right, bool) else right
+ # ISO is the one date value shape shared by import, rendering and database filters. Give
+ # that same value Excel's day arithmetic here instead of forcing a caller to replace it
+ # with a serial number that the date filter cannot read. Date-date subtraction answers a
+ # day count; date +/- number answers another ISO date. Other operators remain numeric.
+ if node["op"] in ("+", "-"):
+ left_day, right_day = _date(left), _date(right)
+ left_num, right_num = _num(left), _num(right)
+ try:
+ if left_day is not None and right_day is not None and node["op"] == "-":
+ return (left_day - right_day).days
+ if left_day is not None and right_num is not None:
+ shift = right_num if node["op"] == "+" else -right_num
+ if int(shift) == shift:
+ return (left_day + dt.timedelta(days=int(shift))).isoformat()
+ if right_day is not None and left_num is not None and node["op"] == "+":
+ if int(left_num) == left_num:
+ return (right_day + dt.timedelta(days=int(left_num))).isoformat()
+ except (OverflowError, ValueError):
+ _diagnose(diagnostics, "bad_value")
+ return None
a, b = _num(left), _num(right)
if a is None or b is None: return None
if node["op"] == "/" and b == 0:
@@ -3257,9 +3280,17 @@ class SheetCalculation:
token = id(node)
if token not in self._volatile_values:
now = dt.datetime.fromisoformat(str(self.today)) if self.today else self._clock.replace(tzinfo=None)
- epoch = dt.datetime(1904, 1, 1) if self.date_system == "1904" else dt.datetime(1899, 12, 30)
- self._volatile_values[token] = (random.random() if fn == "RAND" else
- (now - epoch).total_seconds() / 86400 if fn == "NOW" else (now - epoch).days)
+ # Formula dates have one representation in this runtime: ISO, the same representation
+ # imported date literals, DATE/DATEVALUE, filters and the cell display path consume.
+ # The old document graph special-cased volatile dates as Excel serials while the
+ # compatibility evaluator returned ISO, so TODAY() could render as 46284 while
+ # DATE(...) beside it rendered 2026-09-19. dateSystem still governs imported Excel
+ # serial conversion at the input boundary; it must not create a second stored value
+ # type after that conversion.
+ self._volatile_values[token] = (
+ random.random() if fn == "RAND" else
+ now.strftime("%Y-%m-%d %H:%M:%S") if fn == "NOW" else
+ now.date().isoformat())
return self._volatile_values[token]
def _connect(self, key):
@@ -3612,7 +3643,7 @@ def _sheet_error_message(code):
}.get(code, "This formula could not be calculated.")
-def recompute_document(book, *, cache_key=None, changes=None):
+def recompute_document(book, *, cache_key=None, changes=None, today=None):
"""Recalculate a C7 document in place; return C8 values/errors/provenance/work counters.
Only a trusted (tenant, user, workbook) tuple can opt into this process-local bounded LRU.
@@ -3622,12 +3653,18 @@ def recompute_document(book, *, cache_key=None, changes=None):
formula values; it never scans a sheet's cell map on a valid delta.
"""
with _document_calculations_lock:
+ explicit_today = today is not None
+ calculation_day = str(today or dt.date.today().isoformat())[:10]
+ try:
+ calculation_day = dt.date.fromisoformat(calculation_day).isoformat()
+ except ValueError as exc:
+ raise ValueError("document calculation today must be an ISO date") from exc
tabs = [tab for tab in book.get("sheets", ()) if isinstance(tab, dict)]
names = _document_scopes(tabs)
tab_by_scope = {name.upper(): tab for tab, name in zip(tabs, names)}
signature = json.dumps([[(tab.get("id"), name) for tab, name in zip(tabs, names)],
book.get("namedRanges"), book.get("dateSystem"),
- dt.date.today().isoformat()], sort_keys=True)
+ calculation_day], sort_keys=True)
revision = book.get("rev")
valid_revision = type(revision) is int and revision >= 0
trusted = (isinstance(cache_key, tuple) and len(cache_key) == 3
@@ -3650,7 +3687,8 @@ def recompute_document(book, *, cache_key=None, changes=None):
else:
formula_keys.discard(key)
if incremental:
- engine.update(delta, revision=revision)
+ engine.update(delta, revision=revision,
+ today=calculation_day if explicit_today else None)
if not incremental:
specs, formula_keys = {}, set()
for tab, name in zip(tabs, names):
@@ -3661,8 +3699,12 @@ def recompute_document(book, *, cache_key=None, changes=None):
specs[key] = _document_spec(cell, name)
if cell.get("kind") == "formula":
formula_keys.add(key)
- engine = SheetCalculation(specs, revision=revision if valid_revision else 0, sheets=names,
- named_ranges=book.get("namedRanges"), date_system=book.get("dateSystem", "1900"))
+ engine = SheetCalculation(
+ specs, revision=revision if valid_revision else 0, sheets=names,
+ named_ranges=book.get("namedRanges"), date_system=book.get("dateSystem", "1900"),
+ # A supplied day makes an exceptional repair deterministic. Ordinary workbook
+ # recalculation keeps the live clock, so NOW() does not silently become midnight.
+ today=calculation_day if explicit_today else None)
projected = 0
for key in formula_keys:
scope, _, addr = key.rpartition("!")
diff --git a/api/main.py b/api/main.py
index 9d16bc42d09383f9b5d6cb869d91433de969fc85..28a68ad482b5fef803fe2289ce7ea0aab2c28eb8 100644
--- a/api/main.py
+++ b/api/main.py
@@ -13,7 +13,8 @@ WHAT CHANGED IN EXIT WAVE 1 (2026-07-30) — three things, all of them load-bear
user RECORD. `/api/health` stays unauthenticated (liveness only).
* **The overlay fork is deleted.** `aios-web/api/data/overlay.json` was a second writable home
for user-owned fields the Streamlit app keeps in the tenant store. ONE store now (C1c).
- * **The SSL shim is env-gated.** It used to run at import, unconditionally.
+ * **Outbound TLS always uses the verified default context.** A certificate failure is a source
+ failure, never permission to accept an untrusted connection.
⚠ TENANT RESOLUTION IS PER REQUEST (X7). Nothing tenant-shaped is held at module level; the
session's tenant claim resolves through `harness.runtime.get_runtime`, which is LRU-bounded. This
@@ -25,18 +26,6 @@ import os
import sys
from pathlib import Path
-# --- the local Odoo SSL quirk, now BEHIND A GATE -------------------------------------------------
-# The Windows trust store reports the (valid) Odoo cert as expired, so local runs need an
-# unverified default context; the HF Space and the container are unaffected. This used to run
-# unconditionally at import — i.e. the shipped container disabled TLS verification for every
-# outbound HTTPS call it ever made, to Odoo and to everything else, forever. That is a
-# man-in-the-middle away from being someone else's data. It is now opt-in, must be set
-# deliberately, and is never on by default.
-# ⛔ NEVER set AIOS_INSECURE_SSL in a deployed environment. It exists for one developer laptop.
-if os.environ.get("AIOS_INSECURE_SSL") == "1":
- import ssl
- ssl._create_default_https_context = ssl._create_unverified_context # noqa: S323
-
# --- reuse the tenant #0 data layer verbatim. RI_DIR overrides the location in the container
# (where the layout differs from the local sibling-dir default). ---
_HERE = Path(__file__).resolve()
@@ -93,6 +82,7 @@ import routes_feedback # noqa: E402 (wave 35 R8/C6 — feedback to the operato
import routes_agent_harness # noqa: E402 (wave 36 R8/C4 — the agent harness store; E's router)
import routes_geo # noqa: E402 (wave 37 R14/A11 — the keyless map provider seam; D's router)
from core import grid_events # noqa: E402
+
# D-315 / D-305 (2026-08-18) — the two store REFUSALS get their own app-level handlers below.
# ⚠ Neither is a `StoreUnavailable` subclass, on purpose: routes that degrade a store outage into a
# session fallback must NOT degrade a refusal, or the user is told a change was saved that was not.
@@ -324,6 +314,8 @@ app.include_router(routes_nav.router)
app.include_router(routes_customers.router)
app.include_router(routes_products.router)
app.include_router(routes_assets.router)
+import routes_applications
+app.include_router(routes_applications.router)
app.include_router(routes_tables.router)
app.include_router(routes_grid.router)
app.include_router(routes_records.router)
@@ -541,9 +533,6 @@ def _startup_notes():
print("[aios-api] AIOS_SESSION_SECRET is not set. Signing with a random per-process "
"key. Sessions will not survive a restart and will not work across workers. "
"Set it in production.")
- if os.environ.get("AIOS_INSECURE_SSL") == "1":
- print("[aios-api] AIOS_INSECURE_SSL=1. TLS verification is DISABLED for outbound "
- "requests. Local development only; never in a deployed environment.")
_assert_contract_floor()
@@ -782,49 +771,93 @@ class _SyncNowRefused(Exception):
"""An application-authored, credential-free sentence for a Sync now receipt."""
-def _sync_now_failure(cause):
+def _sync_now_cause(cause):
"""Classify actual provider failures without returning vendor text or exception reprs."""
import re
- text = str(cause).lower()
+ raw = str(cause)
+ text = raw.lower()
causes = (
- (("reduce the amount of data", "response too large"),
+ ("response_too_large", ("reduce the amount of data", "response too large"),
"The provider refused the pull because the response contains too much data."),
- (("no ad accounts",), "The Meta token cannot access any ad accounts."),
- (("max_pages", "page cap"), "The pull stopped at the provider page limit."),
- (("sandbox",), "Amazon sync requires a production credential; this credential is for the sandbox."),
- (("deletion", "retention"), "Amazon sync is blocked by an outstanding data deletion requirement."),
- (("no stored", "no odoo credentials", "access_token is not set"),
+ ("meta_no_ad_accounts", ("no ad accounts",), "The Meta token cannot access any ad accounts."),
+ ("page_cap", ("max_pages", "page cap"), "The pull stopped at the provider page limit."),
+ ("sandbox_credential", ("sandbox",),
+ "Amazon sync requires a production credential; this credential is for the sandbox."),
+ ("deletion_outstanding", ("deletion", "retention"),
+ "Amazon sync is blocked by an outstanding data deletion requirement."),
+ ("no_credential", ("no stored", "no odoo credentials", "access_token is not set"),
"This tenant has no usable credential for the connector."),
- (("enablement document", "capability unavailable"),
+ ("enablement_unreadable", ("enablement document", "capability unavailable"),
"The connector database settings could not be read."),
- (("throttl", "rate limit", "too many requests"),
+ ("rate_limited", ("throttl", "rate limit", "too many requests"),
"The provider rate limit stopped the sync. Retry after the provider delay."),
- (("timed out", "timeout"), "The sync failed because the provider request timed out."),
- (("permission", "access denied", "unauthorized", "invalid oauth", "invalid token",
+ ("timeout", ("timed out", "timeout"), "The sync failed because the provider request timed out."),
+ ("credential_refused", ("permission", "access denied", "unauthorized", "invalid oauth", "invalid token",
"expired", "authentication"), "The provider refused the connector credential or its permissions."),
- (("duplicate key", "primary key", "constraint"),
+ ("mirror_constraint", ("duplicate key", "primary key", "constraint"),
"The mirror rejected rows because they violate a database constraint."),
- (("connection", "network", "name resolution"),
+ ("connect_failed", ("connection", "network", "name resolution"),
"The sync could not connect to the provider or tenant database."),
# A-META / D-201: a Meta pull on a container whose mirror never held `meta_*` raises
# DuckDB's CatalogException. Standing rule 1 wants the CAUSE, not "unexpected error":
# "the loader is broken" and "this server has never held these tables" are different
# operator actions, and only the second one is true here.
- (("does not exist", "no such table", "catalog error"),
+ ("mirror_missing_tables", ("does not exist", "no such table", "catalog error"),
"This server's mirror does not hold this connector's tables yet, so there was nothing "
"to update. Run the connector's first load on the server that owns this tenant."),
)
- for needles, sentence in causes:
+ for classifier, needles, sentence in causes:
if any(needle in text for needle in needles):
- return sentence
+ return classifier, sentence
status = re.search(r"\bhttp\s+(\d{3})\b", text)
if status:
- return f"The provider refused the sync request with HTTP status {status.group(1)}."
+ return "http_status", f"The provider refused the sync request with HTTP status {status.group(1)}."
if isinstance(cause, TypeError):
- return "The connector failed because its provider call used incompatible arguments."
+ return "bad_arguments", "The connector failed because its provider call used incompatible arguments."
if isinstance(cause, (KeyError, ValueError)):
- return "The connector could not read an expected value in the sync response."
- return "The connector encountered an unexpected error while processing the sync response."
+ return "unreadable_value", "The connector could not read an expected value in the sync response."
+ if isinstance(cause, str):
+ # amazon_store._sync_problem appends the application-authored fix LAST. The
+ # exception message may itself contain delimiters; none of it is display text.
+ problem, separator, fix = raw.rpartition("; fix: ")
+ header = re.match(
+ r"(?P
[A-Za-z_][A-Za-z0-9_.-]*): "
+ r"(?P[A-Za-z_][A-Za-z0-9_.-]*) (?P[A-Z]+): "
+ r"(?P[A-Za-z_][A-Za-z0-9_]*): ", problem)
+ if separator and header and fix.strip():
+ # Keep the authored action, with sentence punctuation suitable for the UI.
+ action = fix.strip().replace("\u2014", ". ").replace("\u2013", " to ")
+ sentence = (
+ f"The Amazon table {header['table']} could not be read. "
+ f"The {header['operation']} call was {header['state'].lower()} "
+ f"with {header['exception']}. What to do: {action}"
+ )
+ return "connector_reported", sentence if sentence.endswith(".") else sentence + "."
+ return "unclassified", (
+ "The connector reported an unclassified plain-text failure. "
+ "Ask an administrator to inspect the connector's sync report and retry.")
+ return "unclassified", (
+ f"The connector failed with an unclassified {type(cause).__name__}. "
+ "Ask an administrator to inspect the connector's sync report and retry.")
+
+
+def _sync_now_failure(cause):
+ """Classify actual provider failures without returning vendor text or exception reprs."""
+ return _sync_now_cause(cause)[1]
+
+
+def _sync_now_detail(provider, tenant, *, ok, detail="", cause=None):
+ """Record one classified failure; leave successful receipt text untouched."""
+ if ok:
+ return detail
+ classifier, sentence = _sync_now_cause(cause)
+ print(f"[aios-api] sync FAILED provider={provider} tenant={tenant} classifier={classifier}")
+ suffix = f" (cause code: {classifier})"
+ # finish_sync_request stores only 300 characters. Reserve the suffix before it clips.
+ budget = 300 - len(suffix)
+ if len(sentence) > budget:
+ sentence = sentence[:budget - 3].rstrip() + "..."
+ return sentence + suffix
def _run_sync_requests():
@@ -854,28 +887,47 @@ def _run_sync_requests():
if provider == "odoo":
import core.odoo as odoo
import odoo_relational as rel
+ import routes_connectors as _sync_routes
if rt.odoo_paused() or rel.frozen(rt):
raise _SyncNowRefused("Odoo is paused or disconnected. Resume the connection before syncing.")
- source = rt.odoo_source()
- if source is None:
- raise _SyncNowRefused("This tenant has no Odoo credential. Connect Odoo before syncing.")
- odoo_requested = True
- previous_client = getattr(odoo._tlocal, "client", None)
- previous_context = getattr(odoo._tlocal, "tenant_ctx", None)
- try:
- # Raw mirror pulls use the resolved tenant credential and a dedicated
- # transport. Do not borrow a request's cached XML-RPC client.
- odoo._tlocal.client = odoo.OdooClient(creds=source._creds)
- odoo._tlocal.tenant_ctx = None
- report = datastore.sync_all(log=lambda *a, **k: None)
- finally:
- odoo._tlocal.client = previous_client
- odoo._tlocal.tenant_ctx = previous_context
- if not report:
- raise _SyncNowRefused("Odoo did not run a sync. Check whether the connection is paused.")
- if any(row.get("phase") != "live" for row in report.values()):
- raise _SyncNowRefused("Odoo backfill is incomplete. Run Sync now again to continue it.")
- problems = []
+ requested_cid = (_sync_routes.sync_request_connection(provider, tenant)
+ or _sync_routes._odoo_connection_slot(rt, None))
+ default_cid = _sync_routes._odoo_connection_slot(rt, None)
+ if requested_cid != default_cid:
+ # Named connections have their own provider reader and must never be pulled
+ # through the process-global default mirror under another connection's id.
+ from types import SimpleNamespace
+ report = _sync_routes._odoo_sync_runner(
+ SimpleNamespace(runtime=rt, tenant=tenant,
+ uname=receipt.get("requestedBy") or "automation"),
+ requested_cid)
+ problems = list(report.get("errors") or ())
+ if report.get("state") in ("error", "partial") and not problems:
+ problems.append("The named Odoo connection did not finish cleanly.")
+ else:
+ source = rt.odoo_source()
+ if source is None:
+ raise _SyncNowRefused(
+ "This tenant has no Odoo credential. Connect Odoo before syncing.")
+ odoo_requested = True
+ previous_client = getattr(odoo._tlocal, "client", None)
+ previous_context = getattr(odoo._tlocal, "tenant_ctx", None)
+ try:
+ # Raw mirror pulls use the resolved tenant credential and a dedicated
+ # transport. Do not borrow a request's cached XML-RPC client.
+ odoo._tlocal.client = odoo.OdooClient(creds=source._creds)
+ odoo._tlocal.tenant_ctx = None
+ report = datastore.sync_all(log=lambda *a, **k: None)
+ finally:
+ odoo._tlocal.client = previous_client
+ odoo._tlocal.tenant_ctx = previous_context
+ if not report:
+ raise _SyncNowRefused(
+ "Odoo did not run a sync. Check whether the connection is paused.")
+ if any(row.get("phase") != "live" for row in report.values()):
+ raise _SyncNowRefused(
+ "Odoo backfill is incomplete. Run Sync now again to continue it.")
+ problems = []
elif provider == "amazon_sp":
from harness import amazon_store as amazon
if not amazon.credential(tenant, rt=rt):
@@ -911,13 +963,13 @@ def _run_sync_requests():
if problems:
detail += " " + _sync_now_failure(problems[0])
elif problems:
- detail = _sync_now_failure(problems[0])
+ detail = _sync_now_detail(provider, tenant, ok=False, cause=problems[0])
else:
ok, detail = True, "Sync finished."
except _SyncNowRefused as exc:
detail = str(exc)
except Exception as exc: # noqa: BLE001
- detail = _sync_now_failure(exc)
+ detail = _sync_now_detail(provider, tenant, ok=False, cause=exc)
if first_load_progress is not None:
connector_sync.finish_sync_request(provider, tenant, ok=ok, detail=detail,
first_load=first_load_progress)
@@ -1117,7 +1169,8 @@ def _sweep_automation_schemas(why):
print(f"[aios-api] schema sweep ({why}) done over {len(tenants)} tenant(s)")
-def _rebuild_odoo_relational(why, previous_meta_fingerprint=None, meta_state_out=None):
+def _rebuild_odoo_relational(why, previous_meta_fingerprint=None, meta_state_out=None, *,
+ tenant=None, cadence_seconds=None):
"""Spawn/refresh the four locked Odoo databases, then compute their cells. `why` is 'boot' or
'resync' and rides every log line, because "it failed" and "it failed at boot, before anyone
could have asked" are different diagnoses.
@@ -1127,21 +1180,38 @@ def _rebuild_odoo_relational(why, previous_meta_fingerprint=None, meta_state_out
and was fine" from "the rebuild was never reached". Three failure markers and no success
marker means silence is ambiguous; now it is not.
- ⚠ SCOPED TO ROYAL-IMPORTS, deliberately, and it is NOT the D-29 shortcut it resembles. The
- caller has just advanced whichever DuckDB file this process holds open — tenant #0's — and
- royal is the only tenant with an Odoo mirror to derive from (R1). Iterating tenants here walks
- straight into D-29's documented hazard: `harness/datastore` is a ONE-FILE-AT-A-TIME
- process-global, and rebinding it in a daemon thread under live readers can serve one tenant's
- rows to another with nothing raised. `is_royal` stays the authority on which slugs qualify.
+ The tenant is the owner of the process-global mirror resolved by `_resync_tenant`; this
+ function never loops or rebinds datastore paths. That preserves D-29's isolation rule while
+ removing the old hard-coded tenant/status path.
"""
try:
+ import time as _t
import odoo_relational as _rel
from harness import runtime as _runtime
- if not _rel.is_royal("royal-imports"):
- return False
- _rt = _runtime.get_runtime("royal-imports")
+ tenant_key = str(tenant or _resync_tenant()[0])
+ _rt = _runtime.get_runtime(tenant_key)
from core import user_tables as _ut
- _ut.record_odoo_sync_status(phase="syncing", st=_rt)
+ default_cid = routes_connectors._odoo_connection_slot(_rt, None)
+ connection_ids = [default_cid]
+ try:
+ import core.keychain as _keychain
+ connection_ids.extend(str(row.get("connection_id") or row.get("id"))
+ for row in _keychain.list_entries(_rt)
+ if row.get("type") == "odoo")
+ except Exception: # noqa: BLE001
+ pass
+ connection_ids = list(dict.fromkeys(cid for cid in connection_ids if cid))
+
+ def _health(cid, stage, state, **kwargs):
+ try:
+ routes_connectors.record_odoo_stage(
+ _rt, cid, stage, state, cadence_seconds=cadence_seconds, **kwargs)
+ except Exception: # noqa: BLE001
+ pass # The established lifecycle receipt below remains independently visible.
+
+ for cid in connection_ids:
+ _ut.record_odoo_sync_status(phase="syncing", st=_rt, connection_id=cid)
+ _health(cid, "materialization", "running")
# ⭐⭐ W54-T46 / R4 / R7 — EVERY Odoo CONNECTION, not just the first. `all_connections`
# delegates to `odoo_relational.refresh_all`, which walks this tenant's Odoo
# connections in slot order and records a failure against the connection it belongs
@@ -1156,7 +1226,7 @@ def _rebuild_odoo_relational(why, previous_meta_fingerprint=None, meta_state_out
# because a gate that counts RAW occurrences of a call would count the one in this
# comment too. That gate happens to strip comments first; the next one may not.
# See `odoo_relational.refresh` and its `all_connections` arm.
- counts = _rel.refresh(_rt, "royal-imports", all_connections=True)
+ counts = _rel.refresh(_rt, tenant_key, all_connections=True)
raw_models, schema_errors = _rel.authoritative_schema_snapshot()
_ut.record_odoo_schema_snapshot(raw_models, complete=not schema_errors,
errors=schema_errors, st=_rt)
@@ -1183,19 +1253,134 @@ def _rebuild_odoo_relational(why, previous_meta_fingerprint=None, meta_state_out
except Exception as e: # noqa: BLE001
rebuild_errors.append(f"relations: {type(e).__name__}: {e}")
print(f"[aios-api] odoo relation cells failed ({why}): {type(e).__name__}: {e}")
+ # ⭐⭐ 2026-09-21 — ONE DOCUMENT READ FOR ALL EIGHT TABLES, NOT EIGHT. This loop used to
+ # call `compute(_rt, _key)` with no `tables=`, and that argument is the whole cost of the
+ # pass. Without it `rollup_sql.compute` does `rt.get(ut.STORE_KEY)` ITSELF, once per table
+ # — and under `STORE_BACKEND=pg` the store is key -> whole-JSONB-document, so each of
+ # those is the ENTIRE tenant workspace (~28 MB for tenant #0) coming back through Neon's
+ # public proxy. Eight tables meant EIGHT full reads per rebuild before a single cell was
+ # computed, and up to sixteen in the worst case, because each table that HAS source
+ # rollups and finds a changed cell re-reads the document in its `persist` write.
+ # ⚠ EIGHT IS THE MEASURED FLOOR AND SIXTEEN IS A CEILING, NOT AN OBSERVATION: `compute()`
+ # takes its read BEFORE it checks `source_fields(tdef)`, so every table pays the read even
+ # when it returns immediately — but only a table WITH source rollups can reach the write.
+ # Two of the eight (`order_lines`, `gl_lines`) are read-through grains with no stored rows.
+ #
+ # ⛔ AND EGRESS IS THE HALF THAT IS BILLED. Neon's public network transfer is defined as
+ # *"data sent from your databases through the Neon proxy to clients"* — outbound only.
+ # Writes are free; these READS are the 823.47 GB that took September past a 500 GB
+ # allowance. Skipping redundant WRITES would have saved nothing, which is why this is the
+ # change that was made and that one was not.
+ #
+ # ⭐ `tables=`/`persist=` ARE NOT NEW AND THIS IS NOT A NEW PATTERN — it is the calling
+ # convention `automation_engine._refresh_source_rollups` has always used
+ # (`rollup_sql.compute(rt, table_key, tables=snapshot, persist=True)`). The `owned` branch
+ # applies every table's plan against the snapshot in MEMORY and only opens a write when a
+ # cell actually changed, so a quiet pass now costs ONE read instead of eight.
+ # ⚠ THE SHARED SNAPSHOT IS DELIBERATE, not an accident of reuse: table N sees the cells
+ # table N-1 just computed, which is one consistent view of the workspace rather than eight
+ # reads of a document that other writers can move underneath the loop.
+ # ⚠ `persist=True` IS LOAD-BEARING — the `owned` branch returns WITHOUT writing when it is
+ # false, so dropping it would turn this into an expensive no-op that still reports counts.
+ # ⛔ THE SCOPE STAYS `_rel.TABLES`, not `snapshot.items()` like the automation helper: this
+ # is the Odoo rebuild and it must not start computing rollups for unrelated tables.
try:
import rollup_sql as _rollup
+ _rollup_blob = _rt.get(_ut.STORE_KEY) or {}
for _b, _key, _l, _f in _rel.TABLES:
- _rollup.compute(_rt, _key)
+ _rollup.compute(_rt, _key, tables=_rollup_blob, persist=True)
except Exception as e: # noqa: BLE001
rebuild_errors.append(f"rollups: {type(e).__name__}: {e}")
print(f"[aios-api] odoo source rollups failed ({why}): {type(e).__name__}: {e}")
print(f"[aios-api] odoo relational rebuild done ({why}): "
+ ", ".join(f"{k}={v}" for k, v in sorted((counts or {}).items())))
- if rebuild_errors:
- _ut.record_odoo_sync_status(phase="error", errors=rebuild_errors, st=_rt)
- else:
- _ut.record_odoo_sync_status(phase="ok", st=_rt)
+ reports = ((counts or {}).get("connections")
+ if isinstance((counts or {}).get("connections"), dict) else {})
+ if not reports:
+ reports = {default_cid: counts or {}}
+ any_connection_failed = False
+ from datetime import datetime as _datetime, timezone as _timezone
+ refreshed_at = _datetime.now(_timezone.utc).isoformat()
+ for cid, report in reports.items():
+ cid = str(cid or default_cid)
+ report = report if isinstance(report, dict) else {}
+ failed = report.get("failed")
+ database_failures, unmatched_failures = (
+ routes_connectors._odoo_result_database_failures(_rt, cid, report))
+ failure = (routes_connectors._odoo_failure(failed, "materialization")
+ if failed else None)
+ if not failure and (database_failures or unmatched_failures):
+ failure = routes_connectors._odoo_failure(
+ "one or more databases failed", "materialization")
+ tables = report.get("tables") if isinstance(report.get("tables"), dict) else {}
+ database_receipts = report.get("databases")
+ database_receipts = database_receipts if isinstance(database_receipts, dict) else {}
+ canonical_receipts = report.get("canonicalDatabases")
+ canonical_receipts = canonical_receipts if isinstance(canonical_receipts, dict) else {}
+ all_receipts = {**database_receipts, **canonical_receipts}
+ if failure:
+ any_connection_failed = True
+ _health(cid, "materialization", "failed", failure=failure)
+ _health(cid, "freshness", "failed",
+ failure={**failure, "stage": "freshness"})
+ else:
+ _health(cid, "source", "ready")
+ _health(cid, "materialization", "ready")
+ # A receipt is the authoritative per-database result. ``tables`` alone omits
+ # read-through grains and the two compiled canonical databases, which previously
+ # made their rich reconciliation evidence disappear at this scheduler boundary.
+ database_keys = list(dict.fromkeys([*all_receipts, *tables]))
+ for key in database_keys:
+ table_counts = tables.get(key)
+ if key in database_failures:
+ continue
+ receipt = all_receipts.get(key)
+ receipt = receipt if isinstance(receipt, dict) else None
+ nrows = (receipt.get("servingCount") if receipt is not None
+ else table_counts.get("rows") if isinstance(table_counts, dict)
+ else None)
+ _health(cid, "materialization", "ready", database_key=key,
+ row_count=nrows, receipt=receipt)
+ if rebuild_errors:
+ recon_failure = routes_connectors._odoo_failure(
+ "relation or rollup pass failed", "reconciliation")
+ _health(cid, "reconciliation", "failed", database_key=key,
+ row_count=nrows, failure=recon_failure, receipt=receipt)
+ _health(cid, "freshness", "failed", database_key=key,
+ row_count=nrows, failure={**recon_failure, "stage": "freshness"},
+ receipt=receipt)
+ else:
+ _health(cid, "reconciliation", "ready", database_key=key,
+ row_count=nrows, receipt=receipt)
+ _health(cid, "freshness", "ready", database_key=key,
+ row_count=nrows, refreshed_at=refreshed_at, receipt=receipt)
+ for key, db_failure in database_failures.items():
+ receipt = all_receipts.get(key)
+ receipt = receipt if isinstance(receipt, dict) else None
+ failed_stage = str(db_failure.get("stage") or "materialization")
+ failed_stage = (failed_stage if failed_stage in
+ routes_connectors._ODOO_HEALTH_STAGES else "materialization")
+ _health(cid, failed_stage, "failed", database_key=key,
+ failure=db_failure, receipt=receipt)
+ _health(cid, "freshness", "failed", database_key=key,
+ failure={**db_failure, "stage": "freshness"}, receipt=receipt)
+ if rebuild_errors:
+ recon_failure = routes_connectors._odoo_failure(
+ "relation or rollup pass failed", "reconciliation")
+ _health(cid, "reconciliation", "failed", failure=recon_failure)
+ _health(cid, "freshness", "failed",
+ failure={**recon_failure, "stage": "freshness"})
+ elif not failure:
+ _health(cid, "reconciliation", "ready")
+ _health(cid, "freshness", "ready")
+ safe_errors = []
+ if failure:
+ safe_errors.append(f"{failure['code']}: {failure['message']}")
+ if rebuild_errors:
+ safe_errors.append("reconciliation_failed: Odoo reconciliation did not complete.")
+ _ut.record_odoo_sync_status(
+ phase="error" if safe_errors else "ok", errors=safe_errors,
+ st=_rt, connection_id=cid)
meta_state = _rebuild_meta_relational(
why, _rt, previous_fingerprint=previous_meta_fingerprint, force_relations=True)
if isinstance(meta_state_out, dict):
@@ -1207,7 +1392,7 @@ def _rebuild_odoo_relational(why, previous_meta_fingerprint=None, meta_state_out
# The isolated cells/rollups/meta handlers above are deliberately allowed to report and
# continue. This return value answers the narrower resync-policy question: did the Odoo
# table refresh itself complete, so a confirmed zero-change pass may be skipped?
- return True
+ return not any_connection_failed and not rebuild_errors
except Exception as e: # noqa: BLE001
# ⚠ The TYPE is named here as it is in the two inner handlers. The original printed only
# `{e}`, and a bare message is exactly what made D-107 unreadable from the outside: a
@@ -1217,7 +1402,14 @@ def _rebuild_odoo_relational(why, previous_meta_fingerprint=None, meta_state_out
try:
if '_rt' in locals():
from core import user_tables as _ut
- _ut.record_odoo_sync_status(phase="error", errors=[f"{type(e).__name__}: {e}"], st=_rt)
+ failure = routes_connectors._odoo_failure(e, "materialization")
+ cid = locals().get("default_cid") or "default"
+ routes_connectors.record_odoo_stage(
+ _rt, cid, failure["stage"], "failed", failure=failure,
+ cadence_seconds=cadence_seconds)
+ _ut.record_odoo_sync_status(
+ phase="error", errors=[f"{failure['code']}: {failure['message']}"],
+ st=_rt, connection_id=cid)
except Exception:
pass
return False
@@ -1297,8 +1489,10 @@ RESYNC_STATE = {
"disagreement": None,
"passes": 0,
"lastPassAt": None,
+ "lastSuccessAt": None,
"lastPassOk": None,
"lastError": None,
+ "connectionId": "",
"consecutiveFailures": 0,
"bootSyncError": None,
}
@@ -1467,15 +1661,29 @@ def _classify_resync_error(exc):
⚠ NOT the whole of D-314: naming the offending TABLE and ROW ID belongs to the DuckDB writer
that raised, which is not this loop.
"""
+ import ssl as _ssl
text = f"{type(exc).__name__}: {exc}"
low = text.lower()
+ tls = (isinstance(exc, _ssl.SSLCertVerificationError)
+ or "certificate_verify_failed" in low
+ or "certificate verify failed" in low
+ or "certificate has expired" in low
+ or "self signed certificate" in low)
persistent = any(t in low for t in
("duplicate key", "primary key", "unique constraint", "constraint error",
"binder", "conversion error"))
- return {"type": type(exc).__name__, "message": str(exc)[:400],
- "kind": "persistent" if persistent else "transient",
+ code = ("tls_certificate_invalid" if tls else
+ "mirror_integrity_failure" if persistent else "mirror_pull_failed")
+ message = ("Odoo TLS certificate verification failed because the certificate is expired or "
+ "untrusted. No source data was accepted." if tls else
+ "The analytical mirror failed an integrity check and was not marked fresh."
+ if persistent else
+ "The Odoo mirror pull failed and freshness was not advanced.")
+ return {"stage": "source" if tls else "mirrorPull", "code": code,
+ "type": type(exc).__name__, "message": message,
+ "kind": "persistent" if persistent or tls else "transient",
"meaning": ("this failure repeats until someone repairs the mirror; retrying the same "
- "pass will not clear it" if persistent else
+ "pass will not clear it" if persistent or tls else
"this failure usually clears on the next pass")}
@@ -1543,6 +1751,14 @@ def _store_resync_loop():
"tenantSeconds": _cad["tenantSeconds"], "envSeconds": _cad["envSeconds"],
"resolvedSeconds": _every, "manual": _manual,
"disagreement": _cad["disagreement"]})
+ try:
+ from harness import runtime as _health_runtime
+ import odoo_relational as _health_rel
+ RESYNC_STATE["connectionId"] = str(
+ _health_rel.default_connection_id(
+ _health_runtime.get_runtime(_cad["tenant"])) or "default")
+ except Exception: # noqa: BLE001
+ RESYNC_STATE["connectionId"] = "default"
# ⚠ ON CHANGE, NOT EVERY CYCLE. An hourly warning is a skimmed warning.
if _cad["disagreement"] != _RESYNC_NOTICE[0]:
_RESYNC_NOTICE[0] = _cad["disagreement"]
@@ -1566,6 +1782,8 @@ def _store_resync_loop():
RESYNC_STATE["lastError"] = _err
if _err is None:
RESYNC_STATE["consecutiveFailures"] = 0
+ if not _manual:
+ RESYNC_STATE["lastSuccessAt"] = RESYNC_STATE["lastPassAt"]
else:
RESYNC_STATE["consecutiveFailures"] = int(RESYNC_STATE["consecutiveFailures"]) + 1
# D-314: the TYPE and the persistent/transient reading, both in the line and in the
@@ -1619,7 +1837,17 @@ def _store_resync_loop():
_meta_state = {}
if _reason is not None:
print(f"[aios-api] odoo relational rebuild required ({_reason})")
- if _rebuild_odoo_relational("resync", _meta_previous, _meta_state):
+ # Test/recovery harnesses replace the rebuild seam with its historical three-argument
+ # callable. Bind tenant/cadence whenever the active implementation accepts them,
+ # without turning an internal TypeError into a second materialization attempt.
+ import inspect as _inspect
+ _rebuild_params = _inspect.signature(_rebuild_odoo_relational).parameters
+ _rebuild_variadic = any(
+ p.kind == _inspect.Parameter.VAR_KEYWORD for p in _rebuild_params.values())
+ _rebuild_scope = ({"tenant": _cad["tenant"], "cadence_seconds": _every}
+ if _rebuild_variadic or "tenant" in _rebuild_params else {})
+ if _rebuild_odoo_relational(
+ "resync", _meta_previous, _meta_state, **_rebuild_scope):
last_relational_success_at = _t.time()
else:
# A prior successful projection no longer proves the changed mirror was applied.
@@ -1797,7 +2025,7 @@ if os.environ.get("AIOS_PREWARM") == "1":
# ⭐⭐ W50-T29 / R8 — AND THE DEPLOYMENT NOW OPTS IN, ON STAGING AND ON LIVE. The owner's ruling is
# that the Space resyncs its own mirror hourly rather than waiting for someone to visit; without
# it a Space serves whatever the seed was cut with, forever, and August reads zero while Odoo has
-# 723 orders. `deploy_web.py` pushes `AIOS_STORE_RESYNC=1` plus `AIOS_STORE_RESYNC_SECONDS=3600`.
+# 723 orders. `deploy_web.py` pushes `AIOS_STORE_RESYNC=1` plus `AIOS_STORE_RESYNC_SECONDS=14400`.
#
# ⛔ THE OLD REASON FOR KEEPING IT OFF WAS "so a Pg-backed Live Space can scale its Neon compute to
# zero between real requests", AND THAT REASON IS ONLY HALF SATISFIED. `_resync_mirror_pass` is
@@ -1806,6 +2034,24 @@ if os.environ.get("AIOS_PREWARM") == "1":
# `_rebuild_meta_relational` write through `core.user_tables`, which is the tenant product store,
# and under `STORE_BACKEND=pg` that is Neon. Named here rather than left for the next reader to
# discover from a bill, and carried as a register row by this ticket's handoff.
+#
+# ⛔⛔ AND IT WAS LEFT FOR THE NEXT READER TO DISCOVER FROM A BILL — 2026-09-21, THIRTY-ONE DAYS
+# LATER. The register row never became work, and the paragraph above was the only thing in the
+# codebase that knew. September's Neon transfer reached 823.47 GB against a 500 GB allowance by
+# day 21 (compute 165 of 300 CU-hours, storage 96 MB of 10 GiB — both fine), which is an
+# over-$50 bill on a $19 plan, pacing to ~$87. The two Spaces' own run logs showed 61 of 61
+# hourly passes on live and two loops an hour on staging: 72 full rebuilds a day at ~545 MB each,
+# because `core.store_pg.update()` is a whole-JSONB-document read-modify-write and one pass makes
+# about ten of them against tenant #0's ~28.6 MB document.
+#
+# ⭐ THE HOTFIX MOVED THE CADENCE ONLY, 3600 -> 14400 (R11's existing four-hourly preset), which
+# is safe at any value because the wait below is `sync_wake_event().wait(_every)` and a Sync now
+# SETS it. TWO STRUCTURAL CAUSES REMAIN OPEN AS REGISTER ROWS, and neither is a hotfix:
+# * `_resync_rebuild_reason` treats `pulled > 0` as a change, but `pulled` counts rows FETCHED,
+# not rows CHANGED — live rebuilt on passes its own report scored `added/updated/removed: 0`.
+# Narrowing it is what would make an hourly cadence nearly free.
+# * the whole-document store write itself. A partial-JSONB write is a wave.
+# ⚠ SO DO NOT PUT THIS BACK TO 3600 to "fix staleness" without closing one of those two first.
if os.environ.get("AIOS_STORE_RESYNC") == "1":
import threading as _threading_resync
_threading_resync.Thread(target=_store_resync_loop, daemon=True, name="store-resync").start()
diff --git a/api/odoo_relational.py b/api/odoo_relational.py
index 45ea700983de24e92d12bcdef4c0681448e9c936..70d33e4de66e0baf16f8e1a8746a1fc2fb1c170d 100644
--- a/api/odoo_relational.py
+++ b/api/odoo_relational.py
@@ -1546,13 +1546,15 @@ def read_customers(cur, excluded=None):
The two DOCUMENT legs are the original pair: Amazon books as direct invoices with no sale
order (the `odoo-api` gotcha), so a sale-order leg alone would silently drop a real customer.
- ⭐⭐ THE THIRD IS `customer_rank > 0 AND active` (wave 29, item 22 / R12 via finding F2 —
+ ⭐⭐ THE THIRD IS `customer_rank > 0` across both active states (wave 29, item 22 / R12
+ via finding F2, widened by S63-T14 —
the owner's *"never an arbitrary limit… applies to ALL connected database"*). The old
docstring said partners with no document are *"left out on purpose — a row that can never
appear in any topic has nothing to roll up"*; that reasoning is RETIRED. It is the same
join-drop class as the Product grid's 2,717, and it dropped **~1,149 real customer records**
(MEASURED 2026-08-11: `rank>0 active` = 3,617 against a document union of ~2,000). A customer
- a salesperson has not sold to yet is exactly the row a prospecting view needs.
+ a salesperson has not sold to yet is exactly the row a prospecting view needs. Archived is a
+ projected boolean; the default View hides it with an ordinary removable condition.
⛔ IT IS A UNION AND NOT A REPLACEMENT, AND THAT IS MEASURED, NOT TIDINESS. Swapping the
document legs for the rank leg would drop **16 partners that hold posted documents** (7
@@ -1585,18 +1587,18 @@ def read_customers(cur, excluded=None):
agent = ("ag.name" if "agent_id" in have else "NULL")
agent_id_col = ("p.agent_id" if "agent_id" in have else "NULL")
join = ("LEFT JOIN res_partner ag ON ag.id = p.agent_id " if "agent_id" in have else "")
- # ⚠ BOTH columns must be present, not just `customer_rank`: `active` is what keeps an
- # archived prospect out, and a rank test without it would re-admit the 47 archived partners
- # the mirror carries. Absent ⇒ the leg is dropped WHOLE, exactly like the agent join above.
- rank_leg = (" OR (p.customer_rank > 0 AND p.active) "
- if {"customer_rank", "active"} <= have else "")
+ # The rank leg intentionally includes active and archived accounts. An absent rank column
+ # still drops the leg whole; an absent active column keeps row arity with a conservative live
+ # default while the health gate reports the schema gap separately.
+ rank_leg = (" OR (p.customer_rank > 0) " if "customer_rank" in have else "")
# ⭐⭐ W37-T11 / D-165 — the POSTAL columns join city/state/country here. They ride `_col` like
# every other optional column, so a mirror predating the 2026-08-15 backfill degrades the CELL
# to blank rather than failing the spawn with a Binder error (see `columns()`'s header for the
# live 500 that rule was written after).
sql = (f"SELECT p.id, p.name, {_col(have, 'p.city')}, {_col(have, 'p.state_name')}, "
f" {_col(have, 'p.country_name')}, {agent}, {agent_id_col}, "
- f" {_col(have, 'p.street')}, {_col(have, 'p.street2')}, {_col(have, 'p.zip')} "
+ f" {_col(have, 'p.street')}, {_col(have, 'p.street2')}, {_col(have, 'p.zip')}, "
+ f" {_col(have, 'p.active', 'TRUE')} "
"FROM res_partner p "
f"{join}"
"WHERE p.id IN ("
@@ -1607,7 +1609,7 @@ def read_customers(cur, excluded=None):
f"{rank_leg}")
out = []
for r in cur.execute(sql).fetchall():
- (pid, name, city, state, country, agent, agent_id, street, street2, zipc) = r
+ (pid, name, city, state, country, agent, agent_id, street, street2, zipc, active) = r
out.append({
"_id": str(pid),
"customer": str(name or ""),
@@ -1620,6 +1622,7 @@ def read_customers(cur, excluded=None):
"country": str(country or ""),
"agent": str(agent or ""),
AGENT_JOIN_KEY: int(agent_id) if agent_id else "",
+ "archived": not bool(active),
"wholesale_scope": _in_scope(pid, excluded),
})
return out
@@ -2771,6 +2774,292 @@ TABLE_SUBJECTS = {
GL_LINES_KEY: "odoo:account.move.line",
}
+# One receipt identity per curated database. The model is not enough: invoices/bills share
+# ``account.move`` and agents/vendors/customers share ``res.partner`` while presenting different
+# governed populations. These keys are public metadata only; no source row enters a receipt.
+TABLE_TOPICS = {
+ INVOICES_KEY: "odoo_invoices",
+ ORDERS_KEY: "odoo_orders",
+ AGENTS_KEY: "odoo_agents",
+ ACCOUNTS_KEY: "odoo_accounts",
+ VENDORS_KEY: "odoo_vendors",
+ BILLS_KEY: "odoo_bills",
+ ORDER_LINES_KEY: "sales_lines",
+ GL_LINES_KEY: "gl_lines",
+}
+
+_SOURCE_COUNT_SPECS = {
+ "invoices": {"model": "account.move", "mode": "count", "domain": [
+ ("state", "=", "posted"), ("move_type", "in", ["out_invoice", "out_refund"]),
+ ("partner_id", "!=", False)]},
+ "orders": {"model": "sale.order", "mode": "count", "domain": [
+ ("state", "in", ["sale", "done"]), ("partner_id", "!=", False)]},
+ "agents": {"model": "res.partner", "mode": "agent_union", "domain": [
+ ("agent", "=", True)]},
+ "accounts": {"model": "account.account", "mode": "count", "domain": []},
+ "vendors": {"model": "account.move", "mode": "distinct_partner", "domain": [
+ ("state", "=", "posted"), ("move_type", "in", ["in_invoice", "in_refund"]),
+ ("partner_id", "!=", False)]},
+ "bills": {"model": "account.move", "mode": "count", "domain": [
+ ("state", "=", "posted"), ("move_type", "in", ["in_invoice", "in_refund"]),
+ ("partner_id", "!=", False)]},
+ "order_lines": {"model": "sale.order.line", "mode": "count", "domain": [
+ ("order_id.state", "in", ["sale", "done"])]},
+ "gl_lines": {"model": "account.move.line", "mode": "count", "domain": []},
+}
+
+# The two canonical registry databases replaced the retired ``ut_odoo_customers`` and
+# ``ut_odoo_products`` twins. They still need an identity receipt after every full default-
+# connection sync; otherwise the eight curated-table receipts can all be green while either of
+# the two databases people actually use is short. These are public schema/domain facts only.
+CANONICAL_RECEIPT_SPECS = {
+ "customer_data": {
+ "topicKey": "odoo_customers", "domainKey": "odoo:res.partner",
+ "model": "res.partner", "selection": "customer eligibility union",
+ },
+ "product_data": {
+ "topicKey": "odoo_products", "domainKey": "odoo:product.product",
+ "model": "product.product", "selection": "active catalogue by business key",
+ },
+}
+
+
+def _relation_id(value):
+ """Return the id from an Odoo many2one/group value without retaining its label."""
+ if isinstance(value, (list, tuple)) and value:
+ value = value[0]
+ try:
+ return int(value) if value not in (None, False, "") else None
+ except (TypeError, ValueError):
+ return None
+
+
+def _source_domain_count(client, bucket):
+ """Exact source population for one curated database domain, plus its counting method."""
+ spec = _SOURCE_COUNT_SPECS[bucket]
+ if spec["mode"] == "count":
+ return int(client.search_count(spec["model"], spec["domain"])), "search_count"
+ if spec["mode"] == "distinct_partner":
+ groups = client.read_group(spec["model"], spec["domain"], ["partner_id"],
+ ["partner_id"], lazy=False)
+ ids = {_relation_id(row.get("partner_id")) for row in groups}
+ ids.discard(None)
+ return len(ids), "distinct_partner"
+ if spec["mode"] == "agent_union":
+ flagged = client.search_read("res.partner", spec["domain"], ["id"], limit=100000)
+ commissioned = client.read_group(
+ "account.invoice.line.agent", [("agent_id", "!=", False)], ["agent_id"],
+ ["agent_id"], lazy=False)
+ ids = {_relation_id(row.get("id")) for row in flagged}
+ ids.update(_relation_id(row.get("agent_id")) for row in commissioned)
+ ids.discard(None)
+ return len(ids), "union_distinct_agent"
+ raise Refused("the database source-count mode is not supported")
+
+
+def _group_identity_ids(client, model, domain, field):
+ """Return a read-only grouped identity set without retaining relation labels."""
+ groups = client.read_group(model, domain, [field], [field], lazy=False)
+ ids = {_relation_id(row.get(field)) for row in groups}
+ ids.discard(None)
+ return ids
+
+
+def _canonical_source_population(client, database_key):
+ """Return the exact source identities used by a canonical registry database.
+
+ Customer eligibility is deliberately a union: rank-positive partners in either active state,
+ confirmed-order partners, and posted customer-document partners. Products are the active
+ catalogue. Every operation is a verified read method; a capped read is refused rather than
+ reported as a complete population.
+ """
+ if database_key == "customer_data":
+ rank_domain = [("customer_rank", ">", 0), ("active", "in", [True, False])]
+ ranked = client.search_read("res.partner", rank_domain, ["id"], limit=200000)
+ ranked_count = int(client.search_count("res.partner", rank_domain))
+ if len(ranked) != ranked_count:
+ raise Refused("the customer identity read was incomplete")
+ ids = {_relation_id(row.get("id")) for row in ranked}
+ ids.update(_group_identity_ids(
+ client, "sale.order",
+ [("state", "in", ["sale", "done"]), ("partner_id", "!=", False)],
+ "partner_id"))
+ ids.update(_group_identity_ids(
+ client, "account.move",
+ [("state", "=", "posted"),
+ ("move_type", "in", ["out_invoice", "out_refund"]),
+ ("partner_id", "!=", False)], "partner_id"))
+ ids.discard(None)
+ return [{"id": value} for value in sorted(ids)]
+ if database_key == "product_data":
+ domain = [("active", "=", True)]
+ rows = client.search_read(
+ "product.product", domain, ["id", "default_code"], limit=100000)
+ count = int(client.search_count("product.product", domain))
+ if len(rows) != count:
+ raise Refused("the product identity read was incomplete")
+ return [{"id": _relation_id(row.get("id")),
+ "code": str(row.get("default_code") or "")}
+ for row in rows]
+ raise Refused("the canonical database identity is not supported")
+
+
+def _identity_counts(values):
+ """Return a normalized identity list, set, and duplicate count."""
+ normalized = [value for value in values if value not in (None, False, "")]
+ return normalized, set(normalized), len(normalized) - len(set(normalized))
+
+
+def canonical_reconciliation_receipts(cur, source_client=None, captured_at=None):
+ """Data-free source -> mirror -> canonical receipts for Customers and Products.
+
+ The canonical grids are compiled registry databases, not ``ut_*`` tables. Customer rows keep
+ the Odoo partner id. Product rows are SKU-grained, so multiple active product records sharing
+ one business key are an explained merge rather than an unexplained missing row. Only counts
+ and public schema/domain metadata leave this function; source ids and SKU values never do.
+ """
+ captured = str(captured_at or _dt.datetime.now(_dt.timezone.utc).isoformat())
+ receipts = {}
+ for database_key, spec in CANONICAL_RECEIPT_SPECS.items():
+ identity = {
+ "databaseKey": database_key, "schemaKey": database_key,
+ "topicKey": spec["topicKey"], "domainKey": spec["domainKey"],
+ "model": spec["model"],
+ "selection": {"rule": spec["selection"], "enabled": True},
+ }
+ source_rows, source_failure = None, None
+ if source_client is not None:
+ try:
+ source_rows = _canonical_source_population(source_client, database_key)
+ except Exception as exc: # noqa: BLE001
+ source_failure = {
+ "code": "source_reconciliation_failed", "type": type(exc).__name__,
+ "message": "This canonical Odoo population could not be reconciled.",
+ }
+ try:
+ if database_key == "customer_data":
+ mirror_rows = read_customers(cur)
+ mirror_values = [row.get(JOIN_KEY) for row in mirror_rows]
+ mirror_canonical_values = list(mirror_values)
+ if source_client is not None:
+ import modules.customer_data as customer_data
+ # This is the same identity evaluator consumed by ``_pool_build``. Reading
+ # the mirror again here would let a dropped canonical row certify itself.
+ canonical_values = list(customer_data.identity_pids())
+ else:
+ canonical_values = None
+ source_values = ([row.get("id") for row in source_rows]
+ if source_rows is not None else None)
+ source_canonical_values = source_values
+ explained_merges = 0
+ else:
+ mirror_rows = [row for row in read_products(cur) if row.get("active") == "1"]
+ mirror_values = [row.get(PRODUCT_JOIN_KEY) for row in mirror_rows]
+ # product_data is one row per business key. An uncoded product has a stable,
+ # per-record fallback, matching modules.products.catalogue().
+ mirror_canonical_values = [
+ (str(row.get("code") or "").strip()
+ or f"pid:{row.get(PRODUCT_JOIN_KEY)}") for row in mirror_rows]
+ if source_client is not None:
+ import modules.product_data as product_data
+ # ``identity_rows`` uses the catalogue seam and collision guard that the
+ # shipped product pool consumes. It intentionally omits expensive cells.
+ canonical_values = [row.get("code") for row in product_data.identity_rows()]
+ else:
+ canonical_values = None
+ source_values = ([row.get("id") for row in source_rows]
+ if source_rows is not None else None)
+ source_canonical_values = ([
+ (str(row.get("code") or "").strip() or f"pid:{row.get('id')}")
+ for row in source_rows] if source_rows is not None else None)
+ explained_merges = (len(mirror_canonical_values)
+ - len(set(mirror_canonical_values)))
+
+ mirror_list, mirror_set, mirror_duplicates = _identity_counts(mirror_values)
+ _mirror_canonical_list, mirror_canonical_set, mirror_key_duplicates = (
+ _identity_counts(mirror_canonical_values))
+ if canonical_values is None:
+ canonical_list, canonical_set, canonical_duplicates = None, None, None
+ else:
+ canonical_list, canonical_set, canonical_duplicates = _identity_counts(
+ canonical_values)
+ if source_values is None:
+ source_list, source_set, source_duplicates = None, None, None
+ source_to_mirror = "unconfirmed"
+ missing, extra = None, None
+ canonical_missing, canonical_extra = None, None
+ else:
+ source_list, source_set, source_duplicates = _identity_counts(source_values)
+ missing = len(source_set - mirror_set)
+ extra = len(mirror_set - source_set)
+ _source_canonical_list, source_canonical_set, _source_canonical_duplicates = (
+ _identity_counts(source_canonical_values))
+ canonical_missing = (len(source_canonical_set - canonical_set)
+ if canonical_set is not None else None)
+ canonical_extra = (len(canonical_set - source_canonical_set)
+ if canonical_set is not None else None)
+ source_to_mirror = (
+ "matched" if not missing and not extra and not source_duplicates
+ and not mirror_duplicates else "mismatch")
+ if canonical_set is None:
+ mirror_to_canonical = "unconfirmed"
+ elif (mirror_canonical_set == canonical_set and not mirror_duplicates
+ and not canonical_duplicates):
+ mirror_to_canonical = (
+ "matched_with_explained_merges"
+ if database_key == "product_data" and explained_merges else "matched")
+ else:
+ mirror_to_canonical = "mismatch"
+ # ID and canonical-key gaps are two views of the same records, so use the larger
+ # layer gap rather than double-counting one absent row twice.
+ unexplained = (max((missing or 0) + (extra or 0),
+ (canonical_missing or 0) + (canonical_extra or 0))
+ + (source_duplicates or 0)
+ + mirror_duplicates
+ + (canonical_duplicates or 0)
+ + (0 if database_key == "product_data"
+ else mirror_key_duplicates))
+ partial = (source_to_mirror != "matched" or mirror_to_canonical not in {
+ "matched", "matched_with_explained_merges"}
+ or unexplained > 0 or source_failure is not None)
+ receipts[database_key] = {
+ **identity, "state": "partial" if partial else "succeeded",
+ "sourceCount": len(source_list) if source_list is not None else None,
+ "sourceCountMode": "identity_set" if source_list is not None else "unavailable",
+ "sourceReconciliation": source_to_mirror,
+ "mirrorCount": len(mirror_list), "materializedCount": None,
+ "canonicalCount": (len(canonical_set) if canonical_set is not None else None),
+ "servingCount": (len(canonical_set) if canonical_set is not None else None),
+ "readThrough": True,
+ "duplicateCount": canonical_duplicates,
+ "explainedMergeCount": explained_merges,
+ "unexplainedMissingCount": missing,
+ "unexplainedExtraCount": extra,
+ "unexplainedCanonicalMissingCount": canonical_missing,
+ "unexplainedCanonicalExtraCount": canonical_extra,
+ "unexplainedCount": unexplained,
+ "reconciliation": {"sourceToMirror": source_to_mirror,
+ "mirrorToCanonical": mirror_to_canonical},
+ "freshness": {"state": "partial" if partial else "current",
+ "capturedAt": captured},
+ **({"sourceError": source_failure} if source_failure else {}),
+ }
+ except Exception as exc: # noqa: BLE001
+ failure = {"code": "canonical_reconciliation_failed", "type": type(exc).__name__,
+ "message": "This canonical Odoo database could not be reconciled."}
+ receipts[database_key] = {
+ **identity, "state": "failed", "sourceCount": None,
+ "sourceCountMode": "unavailable", "sourceReconciliation": "unconfirmed",
+ "mirrorCount": None, "materializedCount": None, "canonicalCount": None,
+ "servingCount": None, "readThrough": True, "duplicateCount": None,
+ "explainedMergeCount": None, "unexplainedMissingCount": None,
+ "unexplainedExtraCount": None, "unexplainedCanonicalMissingCount": None,
+ "unexplainedCanonicalExtraCount": None, "unexplainedCount": None,
+ "freshness": {"state": "failed", "capturedAt": captured}, "error": failure,
+ **({"sourceError": source_failure} if source_failure else {}),
+ }
+ return receipts
+
#: ⛔⛔ THE TWO RETIRED KEYS — W33-T44 / AMENDMENT A2. The rows a tenant ALREADY HAS.
#:
#: Dropping the `TABLES` rows above stops these being planned or re-created; it does NOT remove the
@@ -3999,6 +4288,119 @@ def available(rt=None, tenant_key=None, connection_id=None):
return False
+def refresh_curated(rt, cur, buckets, *, connection_id=None, username="automation",
+ today=None, retire=True, on_apply=None, source_client=None):
+ """Refresh each enabled database independently; retain failed databases unchanged.
+
+ Counts describe the database's own domain, never its shared Odoo model. When a read-only
+ source client is available, each receipt asks Odoo independently for that exact population;
+ a failed source/count or mirror reader marks only that database and cannot stale siblings.
+ """
+ from copy import deepcopy
+ built, written, errors, receipts, report = {}, {}, {}, {}, {}
+ captured_at = _dt.datetime.now(_dt.timezone.utc).isoformat()
+ for bucket, base_key, _label, _fields in TABLES:
+ if bucket not in buckets:
+ continue
+ key = table_key_for(base_key, connection_id)
+ source_count = None
+ source_mode = "unavailable"
+ source_failure = None
+ if source_client is not None:
+ try:
+ source_count, source_mode = _source_domain_count(source_client, bucket)
+ except Exception as exc: # noqa: BLE001
+ source_failure = {
+ "code": "source_reconciliation_failed", "type": type(exc).__name__,
+ "message": "This Odoo database source population could not be counted.",
+ }
+ identity = {
+ "databaseKey": key,
+ "schemaKey": key,
+ "topicKey": TABLE_TOPICS[base_key],
+ "domainKey": TABLE_SUBJECTS[base_key],
+ "model": ODOO_TABLE_MODELS.get(base_key),
+ "countModel": _SOURCE_COUNT_SPECS[bucket]["model"],
+ "domain": deepcopy(_SOURCE_COUNT_SPECS[bucket]["domain"]),
+ "selection": {"bucket": bucket, "enabled": True},
+ }
+ try:
+ one = plan(cur, rt=rt, connection_id=connection_id, buckets={bucket})
+ rows = one[bucket]
+ ids = [row.get("_id") for row in rows]
+ if len(ids) != len(set(ids)) or any(value is None for value in ids):
+ raise Refused("the database reader returned duplicate or missing record identities")
+ if one.get("problems"):
+ raise Refused("; ".join(one["problems"]))
+ read_through = is_read_through_key(key) or _ut().row_limit(key, st=rt) == 0
+ count = len(rows)
+ if read_through and cur is not None:
+ from routes_odoo_tables import population
+ count = population(key, cur=cur, rt=rt)
+ if count is None:
+ raise Refused("the database has no readable mirror binding")
+ if on_apply is not None:
+ on_apply()
+ # Each callback receives a detached document so even a store double or an
+ # in-memory backend cannot leak a partially failed database update.
+ class DetachedRuntime:
+ def __getattr__(self, name):
+ return getattr(rt, name)
+
+ def update(self, name, fn, **kwargs):
+ return rt.update(name, lambda value: fn(deepcopy(value)), **kwargs)
+
+ result = apply_plan(DetachedRuntime(), one, username=username, today=today,
+ connection_id=connection_id, retire=False)
+ written.update(result)
+ built[bucket] = rows
+ if one.get("limits"):
+ built.setdefault("limits", {}).update(one["limits"])
+ materialized_count = 0 if read_through else result[key]["rows"]
+ source_reconciliation = (
+ "unconfirmed" if source_count is None else
+ "matched" if source_count == count else "mismatch")
+ mirror_reconciliation = (
+ "served_read_through" if read_through else
+ "matched" if count == materialized_count else "mismatch")
+ partial = bool(one.get("limits") or source_failure
+ or source_reconciliation == "mismatch"
+ or mirror_reconciliation == "mismatch")
+ receipts[key] = {
+ **identity, "state": "partial" if partial else "succeeded",
+ "sourceCount": source_count, "sourceCountMode": source_mode,
+ "sourceReconciliation": source_reconciliation,
+ "mirrorCount": count, "materializedCount": materialized_count,
+ "servingCount": count, "readThrough": read_through, "duplicateCount": 0,
+ "reconciliation": {"sourceToMirror": source_reconciliation,
+ "mirrorToMaterialized": mirror_reconciliation},
+ "freshness": {"state": "partial" if partial else "current",
+ "capturedAt": captured_at},
+ **({"sourceError": source_failure} if source_failure else {}),
+ }
+ if source_failure:
+ errors[key] = source_failure
+ except Exception as exc:
+ # Exception prose may contain connection or record values. Keep only the
+ # structural class and a fixed product message in returned/persisted evidence.
+ failure = {"code": "materialization_failed", "type": type(exc).__name__,
+ "message": "This Odoo database could not be materialized."}
+ errors[key] = failure
+ receipts[key] = {
+ **identity, "state": "failed", "sourceCount": source_count,
+ "sourceCountMode": source_mode, "sourceReconciliation": "unconfirmed",
+ "mirrorCount": None, "materializedCount": None, "servingCount": None,
+ "readThrough": is_read_through_key(key), "duplicateCount": None,
+ "freshness": {"state": "failed", "capturedAt": captured_at},
+ "error": failure,
+ **({"sourceError": source_failure} if source_failure else {}),
+ }
+ if retire:
+ apply_plan(rt, {}, username=username, today=today, report=report,
+ connection_id=connection_id, retire=True)
+ return built, written, errors, receipts, report
+
+
def refresh(rt, tenant, username="automation", cur=None, today=None, connection_id=None,
all_connections=False, table_keys=None, on_apply=None):
"""THE entry point — the store-resync path and the route both call this.
@@ -4105,18 +4507,36 @@ def refresh(rt, tenant, username="automation", cur=None, today=None, connection_
if table_key_for(key, ns) in requested}
skipped = sorted(table_key_for(key, ns)
for bucket, key, _l, _f in TABLES if bucket not in keep)
- built = plan(cur, rt=rt, connection_id=ns, buckets=keep)
+ source = None
+ try:
+ source = (rt.odoo_source() if connection_id is None else rt.odoo_source(connection_id))
+ except Exception: # noqa: BLE001
+ source = None
+ canonical_receipts = {}
+ if source is None:
+ built, written, database_errors, database_receipts, plan_report = refresh_curated(
+ rt, cur, keep, connection_id=ns, username=username, today=today,
+ retire=requested is None, on_apply=on_apply)
+ if ns is None and requested is None and cur is not None:
+ canonical_receipts = canonical_reconciliation_receipts(cur)
+ else:
+ # The binding selects this connection's verified read-only client. Authentication failure
+ # leaves source counts unconfirmed but does not discard a usable mirror materialization.
+ with source._bound(): # tenant credential + scope
+ try:
+ from core.odoo import get_odoo
+ source_client = get_odoo()
+ except Exception: # noqa: BLE001
+ source_client = None
+ built, written, database_errors, database_receipts, plan_report = refresh_curated(
+ rt, cur, keep, connection_id=ns, username=username, today=today,
+ retire=requested is None, on_apply=on_apply, source_client=source_client)
+ if ns is None and requested is None and cur is not None:
+ canonical_receipts = canonical_reconciliation_receipts(
+ cur, source_client=source_client)
# ⭐ W33-T44: a table this pass REMOVED is reported, not inferred from a grid going missing.
# Same rule as `skipped` below — R6's second sentence: what was deliberately not built (or no
# longer built) SAYS SO, with the keys.
- plan_report = {}
- if on_apply is not None:
- try:
- on_apply()
- except Exception: # noqa: BLE001
- pass # Optional status reporting cannot fail a valid plan's publication.
- written = apply_plan(rt, built, username=username, today=today, report=plan_report,
- connection_id=ns, retire=requested is None)
# The fixed mirror roster above and the discovered long tail deliberately use separate
# materialisers. Only an explicit long-tail enablement reaches this read-only path; disabled
# databases are not passed an empty set and therefore cannot be erased by reconciliation.
@@ -4131,7 +4551,9 @@ def refresh(rt, tenant, username="automation", cur=None, today=None, connection_
# FROZEN workspace returned before reaching here, so a disconnected connector is not
# re-recorded. See `_refresh_declared_schema` for why it writes the declared half only.
declared_report = _refresh_declared_schema(rt) if requested is None else None
- out = {"tables": written,
+ out = {"tables": written, "databases": database_receipts,
+ **({"canonicalDatabases": canonical_receipts} if canonical_receipts else {}),
+ **({"databaseErrors": database_errors} if database_errors else {}),
**({"schemaDeclared": declared_report} if declared_report else {}),
**({"discovered": discovered_report}
if discovered_report and (discovered_report.get("tables")
@@ -4143,6 +4565,8 @@ def refresh(rt, tenant, username="automation", cur=None, today=None, connection_
# ⚠ THE DEFAULT CONNECTION'S RETURN SHAPE DOES NOT MOVE BY ONE KEY. The two additions below
# are reachable only on the namespaced path, because `verify_odoo_relational` asserts on this
# dict as it stands for the default.
+ if built.get("limits"):
+ out["limits"] = built["limits"]
if ns is not None:
out["connection"] = cid
if built.get("limits"):
@@ -4191,7 +4615,7 @@ def refresh_all(rt, tenant, username="automation", cur=None, today=None):
cur=(cur if named is None else None),
today=today, connection_id=named)
except Exception as exc: # noqa: BLE001
- one = {"tables": {}, "failed": f"{type(exc).__name__}: {exc}"}
+ one = {"tables": {}, "failed": f"{type(exc).__name__}: Odoo connection refresh failed"}
reports[str(entry_cid or default or DEFAULT_CONNECTION)] = one
if named is None and primary is None:
primary = one
diff --git a/api/providers.py b/api/providers.py
index 587061579818fd20c626e2af424b219a3d5bb8a5..9e131d2e3d2b44fe0c755adf3deea61ab992114f 100644
--- a/api/providers.py
+++ b/api/providers.py
@@ -37,6 +37,7 @@ import json
import os
import time
from dataclasses import dataclass, field
+from datetime import datetime, timezone
from typing import Callable
# ⚠ TOP-LEVEL AND SAFE, unlike the `anthropic` import in `anthropic_sdk()`: `requests` is pinned in
@@ -58,6 +59,32 @@ import requests
#: reads a live price — a modelled figure that says so is honest; one that pretends is not.
_COST_BRIGHTDATA = float(os.environ.get("AIOS_COST_BRIGHTDATA") or 0.0015)
_COST_APIFY = float(os.environ.get("AIOS_COST_APIFY") or 0.0027)
+#: Google Maps store discovery is a different Apify actor and therefore has its own modelled
+#: list rate. This is a planning/receipt rate, not an invoice; deployments may replace it without
+#: a release when the actor's published price changes. Keeping it separate from Instagram avoids
+#: making a Maps estimate silently inherit another actor's commercial terms.
+_COST_APIFY_MAPS = float(os.environ.get("AIOS_COST_APIFY_MAPS") or 0.004)
+
+#: One actor run produces one dataset. Pagination reads that dataset; it never launches a second
+#: paid run. The result cap remains the hard provider-record and modelled-spend ceiling.
+APIFY_BASE = "https://api.apify.com/v2"
+APIFY_MAPS_ACTOR = os.environ.get("AIOS_APIFY_MAPS_ACTOR") or "compass~crawler-google-places"
+APIFY_MAPS_WAIT = float(os.environ.get("AIOS_APIFY_MAPS_WAIT") or 240)
+APIFY_MAPS_PAGE_SIZE = 50
+APIFY_MAPS_MAX_RECORDS = 500
+
+
+class ProviderRefusal(RuntimeError):
+ """A safe, named provider refusal whose message may cross the API boundary."""
+
+ def __init__(self, code: str, public_message: str):
+ super().__init__(public_message)
+ self.code = str(code)
+ self.public_message = str(public_message)
+
+
+class MapsProviderRefusal(ProviderRefusal):
+ """Named Google Maps refusal; never contains a credential, URL, or vendor response body."""
@dataclass(frozen=True)
@@ -135,6 +162,9 @@ PROVIDERS: dict[str, Provider] = {
"apify": Provider(
key="apify", label="Apify", key_env="AIOS_APIFY_KEY",
caps={
+ "google_maps_stores": Capability(
+ True, _COST_APIFY_MAPS,
+ "Google Maps places by category/search and geography; modelled per place"),
# ⭐ MEASURED 2026-08-08 against the public Reels grid: `videoPlayCount` 137,684 and
# 299,493 vs a browser-read ground truth of 134K-137K and 299K. Exact.
"ig_post_views": Capability(True, _COST_APIFY,
@@ -146,6 +176,7 @@ PROVIDERS: dict[str, Provider] = {
#: The DEFAULT chain per capability, cheapest-capable-first. Overridable — see `chain()`.
DEFAULT_CHAINS: dict[str, tuple] = {
+ "google_maps_stores": ("apify",),
"ig_profile": ("brightdata", "apify"),
"ig_post_metrics": ("brightdata", "apify"),
# Only one entry, and that is the point: Bright Data is declared incapable, so listing it here
@@ -239,6 +270,11 @@ def run(capability, work, satisfied=None, log=None):
started = time.time()
try:
result, note = work(provider)
+ except ProviderRefusal:
+ # A named refusal is part of the product contract, not a fallback signal. In
+ # particular, retrying after Maps schema drift can buy the same unusable dataset
+ # twice and replace the actionable error code with a generic provider failure.
+ raise
except Exception as e: # noqa: BLE001
# ⚠ TYPE NAME ONLY. A vendor client's str() can carry the URL, and the credential is
# one refactor away from being a query param; this line must not be what leaks it.
@@ -264,6 +300,8 @@ def _count(result):
if isinstance(result, (list, tuple, set)):
return len(result)
if isinstance(result, dict):
+ if isinstance(result.get("rows"), list):
+ return len(result["rows"])
return len(result)
return 1 if result else 0
@@ -888,6 +926,312 @@ def normalize_profile_apify(row):
return {k: v for k, v in out.items() if v is not None and v != ""}
+def _maps_json(raw, *, subject: str):
+ """Decode one Apify Maps envelope without echoing its body into a refusal."""
+ try:
+ return json.loads(bytes(raw or b"").decode("utf-8", "replace"))
+ except Exception as exc:
+ raise MapsProviderRefusal(
+ "maps_schema_drift",
+ f"Apify's Google Maps {subject} was not readable JSON. No result was written.") from exc
+
+
+def _apify_maps_page_source(provider, search, geography, cap):
+ """Start one bounded actor run and return a reader for its one resulting dataset.
+
+ The actor is launched once. The returned closure pages the resulting dataset by offset, so
+ walking a 500-place result never re-buys the search. Credentials remain in request URLs and
+ are consequently never copied into an exception or receipt.
+ """
+ from automation_engine import Refused, fetch, fetch_json # lazy: only explicit Run now pays
+
+ key = (os.environ.get(provider.key_env) or "").strip()
+ if not key:
+ raise MapsProviderRefusal(
+ "maps_credential_missing",
+ "Add the Apify credential in Keychain before choosing Run now.")
+ token = requests.utils.quote(key, safe="")
+ actor = requests.utils.quote(APIFY_MAPS_ACTOR, safe="~")
+ start_url = (f"{APIFY_BASE}/acts/{actor}/runs?token={token}"
+ f"&waitForFinish={max(1, min(60, int(APIFY_MAPS_WAIT)))}")
+ body = {
+ "searchStringsArray": [str(search)],
+ "locationQuery": str(geography),
+ "maxCrawledPlacesPerSearch": int(cap),
+ "maxCrawledPlaces": int(cap),
+ # Details required by the stable schema, without unrelated review/image payloads.
+ "maxReviews": 0,
+ "maxImages": 0,
+ "scrapePlaceDetailPage": True,
+ }
+ try:
+ status, raw = fetch_json(start_url, body, timeout=min(APIFY_MAPS_WAIT, 60) + 30,
+ max_kb=1024)
+ except Refused as exc:
+ raise MapsProviderRefusal(
+ "maps_vendor_error",
+ "The Apify Google Maps address was refused by the provider safety rail. "
+ "No result was written.") from exc
+ except Exception as exc: # noqa: BLE001
+ raise MapsProviderRefusal(
+ "maps_vendor_error",
+ f"Apify Google Maps did not answer ({type(exc).__name__}). No result was written.") from exc
+ if not 200 <= int(status or 0) < 300:
+ raise MapsProviderRefusal(
+ "maps_vendor_error",
+ "Apify Google Maps refused the bounded run. No result was written.")
+ envelope = _maps_json(raw, subject="run envelope")
+ data = envelope.get("data") if isinstance(envelope, dict) else None
+ if not isinstance(data, dict):
+ raise MapsProviderRefusal(
+ "maps_schema_drift",
+ "Apify's Google Maps run envelope changed. No result was written.")
+
+ deadline = time.monotonic() + max(1.0, float(APIFY_MAPS_WAIT))
+ terminal = {"SUCCEEDED", "FAILED", "ABORTED", "TIMED-OUT"}
+ run_id = str(data.get("id") or "").strip()
+ run_status = str(data.get("status") or "").strip().upper()
+ while run_status not in terminal:
+ remaining = deadline - time.monotonic()
+ if remaining <= 0 or not run_id:
+ raise MapsProviderRefusal(
+ "maps_vendor_error",
+ "Apify Google Maps did not finish within the bounded wait. No result was written.")
+ wait_seconds = max(1, min(60, int(remaining)))
+ run_url = (f"{APIFY_BASE}/actor-runs/{requests.utils.quote(run_id, safe='')}"
+ f"?token={token}&waitForFinish={wait_seconds}")
+ try:
+ poll_status, _final, poll_raw = fetch(
+ run_url, timeout=wait_seconds + 30, max_kb=1024)
+ except Exception as exc: # noqa: BLE001
+ raise MapsProviderRefusal(
+ "maps_vendor_error",
+ f"Apify Google Maps status did not answer ({type(exc).__name__}). "
+ "No result was written.") from exc
+ if not 200 <= int(poll_status or 0) < 300:
+ raise MapsProviderRefusal(
+ "maps_vendor_error",
+ "Apify Google Maps status could not be read. No result was written.")
+ polled = _maps_json(poll_raw, subject="status envelope")
+ data = polled.get("data") if isinstance(polled, dict) else None
+ if not isinstance(data, dict):
+ raise MapsProviderRefusal(
+ "maps_schema_drift",
+ "Apify's Google Maps status envelope changed. No result was written.")
+ run_status = str(data.get("status") or "").strip().upper()
+
+ if run_status != "SUCCEEDED":
+ raise MapsProviderRefusal(
+ "maps_vendor_error",
+ f"Apify Google Maps ended in {run_status.lower() or 'an error'}. No result was written.")
+ dataset_id = str(data.get("defaultDatasetId") or "").strip()
+ if not dataset_id:
+ raise MapsProviderRefusal(
+ "maps_schema_drift",
+ "Apify's Google Maps result did not name its dataset. No result was written.")
+
+ def _page(offset, limit):
+ dataset = requests.utils.quote(dataset_id, safe="")
+ page_url = (f"{APIFY_BASE}/datasets/{dataset}/items?token={token}&clean=true&format=json"
+ f"&offset={int(offset)}&limit={int(limit)}")
+ try:
+ page_status, _final, page_raw = fetch(page_url, timeout=60, max_kb=16384)
+ except Exception as exc: # noqa: BLE001
+ raise MapsProviderRefusal(
+ "maps_vendor_error",
+ f"Apify Google Maps results did not answer ({type(exc).__name__}). "
+ "No result was written.") from exc
+ if not 200 <= int(page_status or 0) < 300:
+ raise MapsProviderRefusal(
+ "maps_vendor_error",
+ "Apify Google Maps results could not be read. No result was written.")
+ page = _maps_json(page_raw, subject="dataset page")
+ if not isinstance(page, list):
+ raise MapsProviderRefusal(
+ "maps_schema_drift",
+ "Apify's Google Maps dataset page changed shape. No result was written.")
+ return page
+
+ return _page
+
+
+def _maps_coordinate(value, name):
+ try:
+ number = float(value)
+ except (TypeError, ValueError) as exc:
+ raise MapsProviderRefusal(
+ "maps_schema_drift",
+ f"Apify's Google Maps result has no usable {name}. No result was written.") from exc
+ if name == "latitude" and not -90 <= number <= 90:
+ raise MapsProviderRefusal(
+ "maps_schema_drift",
+ "Apify's Google Maps result has an invalid latitude. No result was written.")
+ if name == "longitude" and not -180 <= number <= 180:
+ raise MapsProviderRefusal(
+ "maps_schema_drift",
+ "Apify's Google Maps result has an invalid longitude. No result was written.")
+ return number
+
+
+def normalize_google_maps_store(row, *, captured_at=""):
+ """One Apify Google Maps place -> the stable store-discovery database schema.
+
+ Identity, name, physical address, category and coordinates are the structural minimum. Phone
+ and website are optional facts about a real place, not schema failures. The capture timestamp
+ falls back to the run timestamp so every observation says when its source was read.
+ """
+ if not isinstance(row, dict):
+ raise MapsProviderRefusal(
+ "maps_schema_drift",
+ "Apify's Google Maps result is not a place object. No result was written.")
+ location = row.get("location") if isinstance(row.get("location"), dict) else {}
+ place_id = str(row.get("placeId") or row.get("place_id") or "").strip()
+ name = str(row.get("title") or row.get("name") or "").strip()
+ address = str(row.get("address") or row.get("completeAddress") or "").strip()
+ category = str(row.get("categoryName") or row.get("category") or "").strip()
+ if not category and isinstance(row.get("categories"), list):
+ category = str(next((item for item in row["categories"] if item), "")).strip()
+ missing = [label for label, value in (
+ ("placeId", place_id), ("title", name), ("address", address),
+ ("categoryName", category)) if not value]
+ if missing:
+ raise MapsProviderRefusal(
+ "maps_schema_drift",
+ "Apify's Google Maps result schema changed; required fields are missing: "
+ f"{', '.join(missing)}. No result was written.")
+ latitude = _maps_coordinate(location.get("lat", row.get("latitude")), "latitude")
+ longitude = _maps_coordinate(location.get("lng", row.get("longitude")), "longitude")
+ source_timestamp = str(
+ row.get("scrapedAt") or row.get("scraped_at") or row.get("sourceTimestamp")
+ or captured_at or datetime.now(timezone.utc).isoformat()).strip()
+ out = {
+ "place_id": place_id,
+ "name": name,
+ "category": category,
+ "address": address,
+ "city": str(row.get("city") or "").strip(),
+ "region": str(row.get("state") or row.get("region") or "").strip(),
+ "postal_code": str(row.get("postalCode") or row.get("postal_code") or "").strip(),
+ "country": str(row.get("countryCode") or row.get("country") or "").strip(),
+ "phone": str(row.get("phone") or row.get("phoneUnformatted") or "").strip(),
+ "website": str(row.get("website") or row.get("webSite") or "").strip(),
+ "latitude": latitude,
+ "longitude": longitude,
+ "rating": row.get("totalScore", row.get("rating")),
+ "reviews": _int_or_none(row.get("reviewsCount", row.get("reviews"))),
+ "source_timestamp": source_timestamp,
+ "source_payload": json.dumps(row, default=str, separators=(",", ":"))[:32_000],
+ }
+ return {key: value for key, value in out.items() if value is not None and value != ""}
+
+
+def discover_google_maps_stores(search, geography, cap, *, page_source=None,
+ page_size=APIFY_MAPS_PAGE_SIZE, captured_at=""):
+ """Run one bounded Maps discovery through the provider chain and return rows + receipts.
+
+ `page_source(provider, search, geography, cap) -> page(offset, limit)` is the network seam used
+ by the recorded gate fixture. Production uses one Apify actor run and pages its dataset.
+ Provider rows, including duplicates, consume the explicit cap; the runner never buys extra
+ rows merely to replace duplicate place IDs.
+ """
+ query = str(search or "").strip()
+ area = str(geography or "").strip()
+ if not query or not area:
+ raise MapsProviderRefusal(
+ "maps_invalid_search",
+ "Google Maps discovery needs a category or search term and a geography.")
+ try:
+ limit = int(cap)
+ except (TypeError, ValueError) as exc:
+ raise MapsProviderRefusal(
+ "maps_unbounded_request",
+ "Google Maps discovery requires an explicit result cap between 1 and 500.") from exc
+ if limit < 1 or limit > APIFY_MAPS_MAX_RECORDS:
+ raise MapsProviderRefusal(
+ "maps_unbounded_request",
+ "Google Maps discovery requires an explicit result cap between 1 and 500.")
+ try:
+ chunk = max(1, min(int(page_size), APIFY_MAPS_PAGE_SIZE, limit))
+ except (TypeError, ValueError):
+ chunk = min(APIFY_MAPS_PAGE_SIZE, limit)
+ if not chain("google_maps_stores"):
+ raise MapsProviderRefusal(
+ "maps_credential_missing",
+ "Add the Apify credential in Keychain before choosing Run now.")
+ captured = str(captured_at or datetime.now(timezone.utc).isoformat())
+ factory = page_source or _apify_maps_page_source
+
+ def _work(provider):
+ page = factory(provider, query, area, limit)
+ if not callable(page):
+ raise MapsProviderRefusal(
+ "maps_schema_drift",
+ "The Google Maps provider did not supply a page reader. No result was written.")
+ offset = 0
+ pages = 0
+ raw_count = 0
+ duplicates = 0
+ seen = set()
+ rows = []
+ while raw_count < limit:
+ ask = min(chunk, limit - raw_count)
+ batch = page(offset, ask)
+ pages += 1
+ if not isinstance(batch, list):
+ raise MapsProviderRefusal(
+ "maps_schema_drift",
+ "The Google Maps provider returned a non-list dataset page. "
+ "No result was written.")
+ if len(batch) > ask:
+ raise MapsProviderRefusal(
+ "maps_vendor_error",
+ "The Google Maps provider exceeded the requested page bound. "
+ "No result was written.")
+ if not batch:
+ break
+ for vendor_row in batch:
+ normal = normalize_google_maps_store(vendor_row, captured_at=captured)
+ raw_count += 1
+ place_id = normal["place_id"]
+ if place_id in seen:
+ duplicates += 1
+ continue
+ seen.add(place_id)
+ rows.append(normal)
+ offset += len(batch)
+ if len(batch) < ask:
+ break
+ rate = float((provider.cap("google_maps_stores") or Capability()).cost_per_record or 0.0)
+ cost_cap = round(rate * limit, 4)
+ payload = {
+ "state": "ok",
+ "via": provider.key,
+ "rows": rows,
+ "counts": {"cap": limit, "pages": pages, "raw": raw_count,
+ "normalized": raw_count, "duplicates": duplicates,
+ "returned": len(rows)},
+ "cost": {"currency": "USD", "basis": "modelled per provider record",
+ "ratePerRecordUsd": rate, "estimatedUsd": cost_cap,
+ "costCapUsd": cost_cap,
+ "incurredUsd": round(rate * raw_count, 4)},
+ }
+ return payload, ""
+
+ result, attempts = run(
+ "google_maps_stores", _work, satisfied=lambda answer: isinstance(answer, dict))
+ if not isinstance(result, dict):
+ raise MapsProviderRefusal(
+ "maps_vendor_error",
+ "Apify Google Maps could not complete the bounded run. No result was written.")
+ result["attempts"] = [
+ {"provider": attempt.provider, "ok": attempt.ok,
+ "records": (len(result.get("rows") or []) if attempt.ok else 0),
+ "seconds": attempt.seconds}
+ for attempt in attempts
+ ]
+ return result
+
+
def canonical_gaps(row, want=("views",)):
"""Which requested canonical fields this row does NOT carry — the `satisfied` input.
diff --git a/api/routes_activity.py b/api/routes_activity.py
new file mode 100644
index 0000000000000000000000000000000000000000..c605438c0ecf22dd0a485092f48bd583feadf9df
--- /dev/null
+++ b/api/routes_activity.py
@@ -0,0 +1,392 @@
+"""Value-free activity history, persisted only through the tenant store (S65-T16)."""
+import json
+import math
+import re
+import time
+import uuid
+from urllib.parse import urlsplit
+
+from fastapi import APIRouter, Depends, Query, Request
+from starlette.concurrency import run_in_threadpool
+
+from deps import Session, err, require_session
+
+
+RETENTION_DAYS = 30
+MAX_SESSIONS = 200
+MAX_EVENTS_PER_SESSION = 2000
+MAX_MOVEMENT_SAMPLES = 500
+MOVEMENT_WINDOW_MS = 30_000
+MAX_BATCH_EVENTS = 100
+MAX_PAYLOAD_BYTES = 512 * 1024
+MAX_STRING_LENGTH = 256
+MAX_CLASS_LENGTH = 128
+MAX_DATA_ATTRIBUTES = 16
+MAX_PAGE_EVENTS = 100
+BUCKET = "activity_log"
+
+# The entire client field vocabulary, including nested objects, is here. Each
+# shape below selects only its own subset; no arbitrary recursive object survives.
+ACTIVITY_ALLOWED_KEYS = frozenset({
+ "sessionId", "events", "dropped", "kind", "ts", "route", "from", "to", "ms",
+ "target", "method", "url", "status", "message", "source", "line", "name",
+ "movement", "movementDropped", "x", "y", "key", "tagName", "role", "type",
+ "id", "data", "className", "aria-label", "path",
+})
+SECRET_KEYS = frozenset({
+ "value", "defaultvalue", "textcontent", "innertext", "innerhtml", "placeholder",
+ "password", "token", "apikey", "authorization", "cookie", "secret", "credential",
+ "accesstoken", "refreshtoken", "clientsecret", "clipboard", "body", "query",
+ "cellvalue", "celldata", "text", "content", "payload",
+})
+_BEARER = re.compile(r"\bbearer\s+\S+", re.I)
+_JWT = re.compile(r"\beyJ[A-Za-z0-9_-]*\.")
+_CREDENTIAL = re.compile(r"\blp[bcor]1\.[A-Za-z0-9._~-]+")
+_TOKEN = re.compile(r"^[a-zA-Z][a-zA-Z0-9_-]{0,79}$")
+_CLASS = re.compile(r"^(?:set|shell|sheet|cg|pg|activity|lp|is)-[a-z][a-z0-9-]*$")
+_SID = re.compile(r"^[a-f0-9-]{32,36}$")
+_TAGS = set("a button input textarea select option div span svg path rect circle canvas td th tr table body html main section nav header footer label form ul ol li h1 h2 h3 h4 p img dialog details summary video audio".split())
+_ROLES = set("button link textbox checkbox radio switch combobox listbox option grid gridcell cell row columnheader rowheader dialog tab tabpanel menu menuitem navigation region slider spinbutton searchbox".split())
+_TYPES = set("button submit reset text password email number search tel url date time checkbox radio range file hidden color".split())
+_ERROR_NAMES = {"Error", "TypeError", "ReferenceError", "SyntaxError", "RangeError", "URIError", "EvalError", "AggregateError", "NetworkError", "HttpError", "ResourceError"}
+_MESSAGES = {"Application error.", "Unhandled rejection.", "Script error.", "A request failed.", "A network request failed.", "A resource failed to load."}
+_LABELS = {"Open", "Close", "Save", "Cancel", "Delete", "Edit", "Search", "Settings", "Close settings", "Activity log", "Refresh", "Previous", "Next", "Add", "Remove", "Menu", "More", "Retry"}
+# Attribute values may hold tenant data even when their names look harmless.
+# Retain only structural enums; all other safe data-* names record presence.
+_DATA_ENUMS = {
+ "data-section": {"account", "scope", "security", "shortcuts", "activity", "users", "database", "keychains", "connectors", "padmin"},
+ "data-state": {"open", "closed", "active", "inactive", "loading", "ready"},
+ "data-view-kind": {"sheet", "grid", "table", "chart", "timeline"},
+}
+# Routes describe the surface, not record ids, searches, or cell text embedded in
+# a URL. Unknown segments become :item on BOTH sides of the wire.
+_ROUTE_SEGMENTS = set("api v1 auth me login logout refresh nav workspace settings account scope security shortcuts activity sessions users database databases grid sheet sheets view views table tables records rows columns fields cells query sort filter data applications finance facts orders inventory customers products sales collections procurement agents agent automation automations connectors keychains external oauth connections admin operator subscriptions billing health status run runs events logs actions export import search schema preferences folders pages charts catalog reports reconciliation permissions detail assets".split())
+
+# ⛔ NO `prefix=` HERE, AND THAT IS DELIBERATE. This router is included INSIDE
+# `routes_applications.router` (main.py belongs to another session this wave), and a child
+# router that carries its own `prefix=` has that prefix baked into its own `.routes` at
+# DECLARATION time. `verify_api.py::_mounted_paths` then adds `nested.prefix` a SECOND time
+# while walking the nested include, so `/api/v1/activity` is derived as
+# `/api/v1/api/v1/activity` and D-48 reports a client call with no route -- a false RED over a
+# route the app genuinely serves (`app.openapi()` proves it). Spelling the full path on each
+# decorator makes the declared path the served path, which is the honest reading either way.
+router = APIRouter(tags=["activity"])
+
+
+class ActivityRefused(ValueError):
+ """Never put a refused value into an exception, traceback, or response."""
+
+ def __init__(self, code, message):
+ self.code = code
+ self.message = message
+ super().__init__(message)
+
+
+def _text(value, maximum=MAX_STRING_LENGTH):
+ if not isinstance(value, str):
+ return ""
+ if (_BEARER.search(value) or _JWT.search(value) or _CREDENTIAL.search(value)
+ or value.startswith("Atzr|")):
+ return ""
+ return re.sub(r"[\x00-\x1f\x7f\u2013\u2014]", " ", value)[:maximum]
+
+
+def _shape(value):
+ if not isinstance(value, dict):
+ return {}
+ return {key: val for key, val in value.items() if key in ACTIVITY_ALLOWED_KEYS}
+
+
+def _number(value, default=0, maximum=10**15):
+ if type(value) not in (int, float) or (isinstance(value, float) and not math.isfinite(value)):
+ return default
+ return min(maximum, max(0, round(value)))
+
+
+def _path(value):
+ value = _text(value)
+ if not value:
+ return ""
+ try:
+ # Hash routes are paths too, but hash query parameters never survive.
+ raw = value[1:] if value.startswith("#") else urlsplit(value).path
+ raw = raw.split("?", 1)[0].split("#", 1)[0]
+ segments = raw.strip("/").split("/")
+ clean = [part if part in _ROUTE_SEGMENTS or re.fullmatch(r"[a-zA-Z0-9_-]+\.(?:js|css|tsx|ts)", part) else ":item" for part in segments if part]
+ return ("/" + "/".join(clean))[:MAX_STRING_LENGTH]
+ except ValueError:
+ return ""
+
+
+def _classes(value):
+ return " ".join(part for part in _text(value).split() if _CLASS.fullmatch(part))[:MAX_CLASS_LENGTH]
+
+
+def _element(raw):
+ raw = _shape(raw)
+ tag = _text(raw.get("tagName")).lower()
+ role = _text(raw.get("role"))
+ result = {"tagName": tag if tag in _TAGS else "element"}
+ if role in _ROLES:
+ result["role"] = role
+ element_type = _text(raw.get("type"), 32)
+ if element_type in _TYPES:
+ result["type"] = element_type
+ # The collector marks descendants of a grid/editor with role=gridcell/textbox.
+ # Defend direct forged input too; never accept rich metadata on these shapes.
+ protected = (tag in {"input", "textarea", "option", "select", "canvas", "td", "th"}
+ or role in {"grid", "gridcell", "cell", "textbox", "searchbox", "option"}
+ or "sheet-cell" in str(raw.get("className", ""))
+ or "cg-cell" in str(raw.get("className", "")))
+ hops = raw.get("path", [])
+ if isinstance(hops, list):
+ protected = protected or any(
+ _text(hop).split(".")[0] in {"td", "th", "input", "textarea", "option", "canvas"}
+ or re.search(r"\.(?:sheet|cg)-cell(?:\.|$)", _text(hop))
+ for hop in hops[:5])
+ path = []
+ for hop in hops[:5]:
+ parts = _text(hop).split(".")
+ if parts[0] in _TAGS:
+ classes = _classes(" ".join(parts[1:])) if not protected else ""
+ path.append(parts[0] + ("." + classes.replace(" ", ".") if classes else ""))
+ result["path"] = path
+ if protected:
+ return result
+ classes = _classes(raw.get("className"))
+ if classes:
+ result["className"] = classes
+ for key in ("id", "name"):
+ val = _text(raw.get(key), 80)
+ if val == "root" or (_TOKEN.fullmatch(val) and _CLASS.fullmatch(val)):
+ result[key] = val
+ label = _text(raw.get("aria-label"))
+ if label in _LABELS:
+ result["aria-label"] = label
+ data = raw.get("data")
+ if isinstance(data, dict):
+ safe = {}
+ for key, val in list(data.items())[:MAX_DATA_ATTRIBUTES]:
+ if not isinstance(key, str) or not re.fullmatch(r"data-[a-z][a-z-]{0,47}", key):
+ continue
+ normalized = re.sub(r"[^a-z]", "", key[5:])
+ if any(word in normalized for word in SECRET_KEYS):
+ continue
+ val = _text(val)
+ if not val:
+ continue
+ safe[key] = val if val in _DATA_ENUMS.get(key, set()) else "present"
+ if safe:
+ result["data"] = safe
+ return result
+
+
+def _key(value, target):
+ value = _text(value, 64)
+ parts = value.split("+")
+ key = parts[-1]
+ if any(part not in {"Control", "Alt", "Shift", "Meta"} for part in parts[:-1]):
+ return "Key"
+ named = {"Enter", "Escape", "Tab", "Backspace", "Delete", "ArrowUp", "ArrowDown", "ArrowLeft", "ArrowRight", "Home", "End", "PageUp", "PageDown", "Control", "Alt", "Shift", "Meta", "CapsLock", "Space", "Character"}
+ if key not in named and not re.fullmatch(r"(?:[a-zA-Z0-9]|F(?:[1-9]|1[0-2]))", key):
+ return "Key"
+ editable = target.get("tagName") in {"input", "textarea", "option", "select"} or target.get("role") in {"textbox", "searchbox", "gridcell", "cell", "grid"}
+ shortcut = any(part in {"Control", "Meta"} for part in parts[:-1]) and "Alt" not in parts[:-1]
+ if editable and (len(key) == 1 or key == "Space") and not shortcut:
+ return "Character"
+ return value
+
+
+def project_event(raw, now_ms, *, movement=False):
+ raw = _shape(raw)
+ kind = raw.get("kind")
+ allowed = {"hover", "scroll", "focus", "keypress", "pointer"} if movement else {"route", "click", "api", "error"}
+ if not isinstance(kind, str) or kind not in allowed:
+ return None
+ ts = _number(raw.get("ts"), now_ms, now_ms)
+ if ts < now_ms - RETENTION_DAYS * 86400_000:
+ return None
+ event = {"kind": kind, "ts": ts}
+ if movement or kind == "click":
+ event["target"] = _element(raw.get("target"))
+ if movement:
+ if kind in {"pointer", "scroll"}:
+ event.update({key: _number(raw.get(key), maximum=10**7) for key in ("x", "y")})
+ if kind == "keypress":
+ event["key"] = _key(raw.get("key"), event["target"])
+ return event
+ event["route"] = _path(raw.get("route"))
+ if kind == "route":
+ event.update({"from": _path(raw.get("from")), "to": _path(raw.get("to")), "ms": _number(raw.get("ms"))})
+ elif kind == "api":
+ method = _text(raw.get("method"), 16).upper()
+ event.update({"method": method if method in {"GET", "POST", "PUT", "PATCH", "DELETE", "HEAD", "OPTIONS"} else "OTHER",
+ "url": _path(raw.get("url")), "status": _number(raw.get("status"), maximum=599), "ms": _number(raw.get("ms"))})
+ elif kind == "error":
+ name = _text(raw.get("name"), 64)
+ message = _text(raw.get("message"))
+ event.update({"name": name if name in _ERROR_NAMES else "Error",
+ # Raw exception text often interpolates the offending cell.
+ "message": message if message in _MESSAGES else "Application error.",
+ "source": _path(raw.get("source")), "line": _number(raw.get("line"), maximum=10**7)})
+ samples = raw.get("movement")
+ samples = samples if isinstance(samples, list) else []
+ trail = [project_event(sample, now_ms, movement=True) for sample in samples[-MAX_MOVEMENT_SAMPLES:]]
+ event["movement"] = sorted([sample for sample in trail if sample and ts - MOVEMENT_WINDOW_MS <= sample["ts"] <= ts], key=lambda sample: sample["ts"])
+ event["movementDropped"] = _number(raw.get("movementDropped")) + len(samples) - len(event["movement"])
+ return event
+
+
+def _limits():
+ return {"retentionDays": RETENTION_DAYS, "maxSessions": MAX_SESSIONS,
+ "maxEventsPerSession": MAX_EVENTS_PER_SESSION, "maxMovementSamples": MAX_MOVEMENT_SAMPLES,
+ "movementWindowSeconds": MOVEMENT_WINDOW_MS // 1000, "pageSize": MAX_PAGE_EVENTS}
+
+
+def _retained(doc, tenant, now_ms):
+ if not isinstance(doc, dict) or not isinstance(doc.get("sessions"), list):
+ return []
+ cutoff = now_ms - RETENTION_DAYS * 86400_000
+ rows = [row for row in doc["sessions"] if isinstance(row, dict) and row.get("tenant") == tenant
+ and isinstance(row.get("events"), list) and _number(row.get("endedAt")) >= cutoff]
+ retained = []
+ for row in rows:
+ events = [event for event in row["events"] if isinstance(event, dict) and _number(event.get("ts")) >= cutoff]
+ if events:
+ retained.append({**row, "events": events[-MAX_EVENTS_PER_SESSION:]})
+ return sorted(retained, key=lambda row: row["endedAt"])[-MAX_SESSIONS:]
+
+
+def _record(session, session_id, events, dropped, now_ms):
+ from harness import runtime
+
+ def _append(doc):
+ rows = _retained(doc, session.tenant, now_ms)
+ row = next((item for item in rows if item["sessionId"] == session_id and item["user"] == session.uname), None)
+ if row is None:
+ row = {"sessionId": session_id, "user": session.uname, "tenant": session.tenant,
+ "startedAt": min(event["ts"] for event in events), "endedAt": now_ms,
+ "route": next((event.get("to") or event.get("route") for event in events if event.get("to") or event.get("route")), "/"),
+ "events": [], "dropped": 0, "prunedEvents": 0}
+ rows.append(row)
+ merged = sorted(row["events"] + events, key=lambda event: event["ts"])
+ if merged[0]["ts"] < row["startedAt"]:
+ row["route"] = merged[0].get("to") or merged[0].get("route") or "/"
+ row["startedAt"] = min(row["startedAt"], merged[0]["ts"])
+ row["endedAt"] = merged[-1]["ts"]
+ row["prunedEvents"] += max(0, len(merged) - MAX_EVENTS_PER_SESSION)
+ row["dropped"] += dropped
+ row["events"] = merged[-MAX_EVENTS_PER_SESSION:]
+ return {"sessions": sorted(rows, key=lambda item: item["endedAt"])[-MAX_SESSIONS:]}
+
+ # Async coalescing avoids blocking uploads and one HF commit per batch.
+ # A container loss before the worker flushes may lose cached telemetry.
+ runtime.get_runtime(str(session.tenant)).update(BUCKET, _append, flush="async")
+ # platform/ is available after session resolution, not necessarily at import.
+ try:
+ from harness.telemetry import error as telemetry_error
+ except ImportError:
+ return
+ for event in events:
+ if event["kind"] == "error":
+ try:
+ # This is the existing diagnostic sink, not a second activity store.
+ # Never forward exc, the incoming payload, or the movement snapshot.
+ telemetry_error("activity", name=event["name"], message=event["message"],
+ source=event["source"], line=event["line"])
+ except Exception:
+ pass
+
+
+def _batch(body):
+ events = body.get("events")
+ session_id = _text(body.get("sessionId"), 80)
+ if not _SID.fullmatch(session_id) or not isinstance(events, list):
+ raise ActivityRefused("activity_invalid", "Activity requires a tab session and an event list.")
+ if len(events) > MAX_BATCH_EVENTS:
+ raise ActivityRefused("activity_batch_too_large", f"Activity batches must contain at most {MAX_BATCH_EVENTS} events.")
+ return session_id, events
+
+
+@router.post("/api/v1/activity")
+async def ingest_activity(request: Request, session: Session = Depends(require_session)):
+ data = bytearray()
+ async for chunk in request.stream():
+ if len(data) + len(chunk) > MAX_PAYLOAD_BYTES:
+ raise err(400, "activity_payload_too_large", f"Activity batches must be at most {MAX_PAYLOAD_BYTES} bytes.")
+ data.extend(chunk)
+ try:
+ body = _shape(json.loads(data))
+ except (ValueError, UnicodeError, RecursionError):
+ raise err(400, "activity_invalid", "Activity requires a valid JSON object.") from None
+ try:
+ session_id, events = _batch(body)
+ except ActivityRefused as exc:
+ raise err(400, exc.code, exc.message) from None
+ now_ms = round(time.time() * 1000)
+ clean = []
+ for raw in events:
+ event = project_event(raw, now_ms)
+ if event is not None:
+ clean.append({**event, "eventId": uuid.uuid4().hex})
+ dropped = len(events) - len(clean)
+ clean.sort(key=lambda event: event["ts"])
+ if clean:
+ try:
+ await run_in_threadpool(_record, session, session_id, clean,
+ _number(body.get("dropped")) + dropped, now_ms)
+ except Exception:
+ raise err(503, "activity_unavailable", "Activity could not be saved.") from None
+ return {"accepted": len(clean), "dropped": dropped}
+
+
+def _user(session, requested):
+ if requested is not None and requested != session.uname and not session.admin:
+ raise err(403, "activity_forbidden", "You can only view your own activity.")
+ return session.uname if requested is None else requested
+
+
+def _read(session):
+ from harness import runtime
+ try:
+ return _retained(runtime.get_runtime(str(session.tenant)).get(BUCKET), session.tenant, round(time.time() * 1000))
+ except Exception:
+ raise err(503, "activity_unavailable", "Activity could not be loaded.") from None
+
+
+@router.get("/api/v1/activity/sessions")
+def activity_sessions(user: str | None = None, session: Session = Depends(require_session)):
+ owner = _user(session, user)
+ rows = []
+ for row in reversed(_read(session)):
+ if row["user"] == owner:
+ rows.append({key: row[key] for key in ("sessionId", "startedAt", "endedAt", "route", "dropped", "prunedEvents")})
+ rows[-1].update(events=len(row["events"]), errors=sum(event["kind"] == "error" for event in row["events"]))
+ return {"sessions": rows, "limits": _limits()}
+
+
+@router.get("/api/v1/activity/sessions/{session_id}")
+def activity_session(session_id: str, user: str | None = None,
+ limit: int = Query(MAX_PAGE_EVENTS, ge=1, le=MAX_PAGE_EVENTS),
+ offset: int = Query(0, ge=0), session: Session = Depends(require_session)):
+ owner = _user(session, user)
+ row = next((item for item in _read(session) if item["user"] == owner and item["sessionId"] == session_id), None)
+ if row is None:
+ raise err(404, "activity_session_missing", "This activity session is no longer available.")
+ # Re-project the displayed slice as defense against a legacy or corrupt document.
+ now_ms = round(time.time() * 1000)
+ events = []
+ for raw in row["events"][offset:offset + limit]:
+ clean = project_event(raw, now_ms)
+ if clean:
+ events.append({**clean, "eventId": _text(raw.get("eventId"), 36)})
+ return {"sessionId": session_id, "events": events, "total": len(row["events"]),
+ "offset": offset, "limit": limit, "dropped": row["dropped"],
+ "prunedEvents": row["prunedEvents"], "limits": _limits()}
+
+
+@router.get("/api/v1/activity/users")
+def activity_users(session: Session = Depends(require_session)):
+ if not session.admin:
+ raise err(403, "activity_forbidden", "Only administrators can view other users' activity.")
+ return {"users": sorted({row["user"] for row in _read(session)})}
diff --git a/api/routes_admin.py b/api/routes_admin.py
index 3eeeeb6d3cfa051df9c7134f49647a3adb7efcc0..5083129c25661dcbb08c9e7d1737dab6b34dd5ca 100644
--- a/api/routes_admin.py
+++ b/api/routes_admin.py
@@ -2091,8 +2091,13 @@ def _confines_to_one_bu(tree):
return any(kids) if node.get("conj") == "or" else all(kids)
if node.get("colId") != "dba":
return True # every other condition: assume satisfied
- want = str(node.get("value") or "").strip().lower()
op = node.get("op")
+ if op in fe.MULTI_VALUE_OPS:
+ # HOTFIX 2026-09-23: `dba is any of [Royal]` is the shape the condition builder writes
+ # now, and `perm_scope` already reads it. Falling to "cannot judge" here made a save
+ # that KEPT the BU condition read as one that removed it, and refused it as bu_widening.
+ return fe._match_multi_value(op, brand, fe.parse_filter_values(node.get("values")))
+ want = str(node.get("value") or "").strip().lower()
if op == "eq":
return brand.lower() == want
if op == "neq":
@@ -2107,6 +2112,8 @@ def _confines_to_one_bu(tree):
return True
return True # an op that cannot judge a brand does not confine
+ from harness import filter_eval as fe
+
nodes, conj = (tree.get("nodes") or []), tree.get("conj", "and")
root = {"conj": conj, "children": nodes}
admitted = {b for b in ("Fisch", "Royal", "Both") if admits(root, b)}
diff --git a/api/routes_applications.py b/api/routes_applications.py
new file mode 100644
index 0000000000000000000000000000000000000000..5f94659633671e871d3a51ba9efe85fbd68d4e57
--- /dev/null
+++ b/api/routes_applications.py
@@ -0,0 +1,258 @@
+"""Permissioned, directly addressable Finance application contracts (S63-T16)."""
+from fastapi import APIRouter, Body, Depends, Query
+
+import application_registry
+from deps import Session, err, require_session
+from harness import runtime as mirror_runtime
+from modules import finance_spine
+from pages import resolve_bu
+from routes_activity import router as activity_router
+from routes_statements import _gate as statement_gate
+
+
+finance_router = APIRouter(prefix="/api/v1/finance", tags=["finance"])
+nav_router = APIRouter(prefix="/api/v1", tags=["applications"])
+# ⛔ NO `prefix=` HERE, for the same reason S65-T16 spelled full paths on `routes_activity`:
+# this router is included INSIDE this module's own `router`, and a child router carrying
+# `prefix=` has that prefix baked into its `.routes` at DECLARATION time, so
+# `verify_api.py::_mounted_paths` adds it a SECOND time and D-48 reads a route the app really
+# serves as one it does not. The full path on each decorator is also the honest reading.
+collection_router = APIRouter(tags=["collection"], dependencies=[Depends(require_session)])
+
+
+def _may_open(session, permission_key):
+ try:
+ session.require(permission_key)
+ return True
+ except Exception:
+ return False
+
+
+def _require_app(session, key):
+ try:
+ return application_registry.finance_application(
+ key, may_open=lambda permission: _may_open(session, permission))
+ except PermissionError:
+ raise err(403, "finance_forbidden", "your account does not have access to this Finance application")
+ except KeyError:
+ raise err(404, "finance_app_not_found", "Finance application not found")
+
+
+def _require_fact(session, key):
+ candidates = [app for app in application_registry.finance_applications()
+ if key in app["factKeys"]]
+ if not candidates:
+ raise err(404, "finance_fact_not_found", "Finance fact not found")
+ if not any(_may_open(session, app["permissionKey"]) for app in candidates):
+ raise err(403, "finance_forbidden", "your account does not have access to this Finance fact")
+ return finance_spine.fact_spec(key)
+
+
+@finance_router.get("/applications")
+def list_finance_applications(session: Session = Depends(require_session)):
+ """One Finance group containing only the applications this session may open."""
+ apps = application_registry.finance_applications(
+ may_open=lambda permission: _may_open(session, permission))
+ apps = [{**app, "id": app["key"], "name": app["navLabel"],
+ "navPlacement": {"section": "finance", "order": index}}
+ for index, app in enumerate(apps)]
+ return {"group": application_registry.finance_group(), "applications": apps}
+
+
+@finance_router.get("/applications/{app_key}")
+def get_finance_application(app_key: str, bu: str | None = None,
+ date_from: str | None = None, date_to: str | None = None,
+ session: Session = Depends(require_session)):
+ """Render one deep-linked app; this endpoint deliberately has no Finance-home aggregate."""
+ app = _require_app(session, app_key)
+ team_id = resolve_bu(session, bu)
+ try:
+ cursor = mirror_runtime.mirror_cursor(session.runtime)
+ except mirror_runtime.DatastoreMismatch:
+ raise err(503, "finance_store_mismatch", "the Finance mirror belongs to a different workspace")
+ except RuntimeError:
+ facts = [{**finance_spine.fact_spec(key), "state": "warming", "value": None}
+ for key in app["factKeys"]]
+ return {"group": application_registry.finance_group(), "application": app,
+ "businessUnit": team_id, "state": "warming",
+ "facts": facts,
+ "kpis": [fact for fact in facts if fact["key"] in app["kpiKeys"]]}
+ facts = [finance_spine.observe(key, cursor, date_from=date_from, date_to=date_to,
+ team_id=team_id) for key in app["factKeys"]]
+ kpis = [fact for fact in facts if fact["key"] in app["kpiKeys"]]
+ state = "ready" if all(fact["state"] in {"ready", "requires_sku"} for fact in facts) else "partial"
+ return {"group": application_registry.finance_group(), "application": app,
+ "businessUnit": team_id, "state": state, "facts": facts, "kpis": kpis}
+
+
+@finance_router.get("/facts/{fact_key}/records")
+def get_finance_fact_records(fact_key: str, bu: str | None = None,
+ date_from: str | None = None, date_to: str | None = None,
+ limit: int = Query(100, ge=1, le=500), offset: int = Query(0, ge=0),
+ session: Session = Depends(require_session)):
+ """Source-record drill-through under the same permission and tenant mirror walls."""
+ spec = _require_fact(session, fact_key)
+ team_id = resolve_bu(session, bu)
+ try:
+ finance_spine.effective_team_id(spec, team_id)
+ mirror_runtime.mirror_cursor(session.runtime)
+ return finance_spine.source_records(
+ fact_key, date_from=date_from, date_to=date_to, team_id=team_id,
+ limit=limit, offset=offset)
+ except PermissionError:
+ raise err(403, "finance_scope_forbidden", "this fact requires consolidated Finance access")
+ except ValueError as exc:
+ raise err(400, "finance_scope_invalid", str(exc))
+ except mirror_runtime.DatastoreMismatch:
+ raise err(503, "finance_store_mismatch", "the Finance mirror belongs to a different workspace")
+ except RuntimeError:
+ return {"fact": spec, "state": "warming", "rows": [], "row_count": None,
+ "total_count": None, "aggregates": {}}
+
+
+@finance_router.get("/facts/{fact_key}/reconciliation")
+def reconcile_finance_fact(fact_key: str, bu: str | None = None,
+ date_from: str | None = None, date_to: str | None = None,
+ session: Session = Depends(require_session)):
+ """Explicit independent Odoo read; ordinary page loads never issue live source reads."""
+ spec = _require_fact(session, fact_key)
+ team_id = resolve_bu(session, bu)
+ try:
+ finance_spine.effective_team_id(spec, team_id)
+ mirror_runtime.mirror_cursor(session.runtime)
+ except PermissionError:
+ raise err(403, "finance_scope_forbidden", "this fact requires consolidated Finance access")
+ except ValueError as exc:
+ raise err(400, "finance_scope_invalid", str(exc))
+ except mirror_runtime.DatastoreMismatch:
+ raise err(503, "finance_store_mismatch", "the Finance mirror belongs to a different workspace")
+ except RuntimeError:
+ return {"state": "warming", "message": "The tenant mirror is still warming."}
+ return finance_spine.reconcile(
+ fact_key, date_from=date_from, date_to=date_to, team_id=team_id)
+
+
+@nav_router.get("/nav/applications")
+def list_nav_applications(session: Session = Depends(require_session)):
+ """Every Application this session may open, in ONE generic navigation shape (R6).
+
+ Finance's six arrive here as seeded Application records rather than as new template kinds,
+ so the App module reaches them by the same path Routing and Catalog ride.
+ """
+ if not session.runtime.available():
+ raise err(503, "store_unavailable", "the tenant store is unavailable")
+ return {"applications": application_registry.nav_applications(
+ session.runtime, session.uname, session.admin,
+ may_open=lambda permission: _may_open(session, permission))}
+
+
+def _collection_ids(body, *, confirm=False):
+ body = body or {}
+ if confirm and body.get("confirm") is not True:
+ raise err(400, "collection_confirmation_required", "Confirm the whole batch before sending.")
+ allowed = {"partnerIds", "confirm"} if confirm else {"partnerIds"}
+ if set(body) - allowed:
+ raise err(400, "collection_request_invalid", "Only customer identifiers are accepted.")
+ ids = body.get("partnerIds")
+ if not isinstance(ids, list) or not ids:
+ raise err(400, "collection_selection_required", "Select at least one customer.")
+ if any(type(pid) is not int or pid <= 0 for pid in ids):
+ raise err(400, "collection_selection_invalid", "Customer identifiers must be positive integers.")
+ return list(dict.fromkeys(ids))
+
+
+def _collection_source(cs):
+ try:
+ odoo = cs.Odoo()
+ return odoo, cs.load_collection_list(odoo)
+ except Exception:
+ raise err(502, "collection_odoo_unavailable", "The Collection worklist could not be read from Odoo.")
+
+
+def _collection_selection(rows, ids):
+ by_id = {row["partner_id"]: row for row in rows}
+ selected = [by_id[pid] for pid in ids if pid in by_id]
+ refused = [{"customer": f"Customer {pid}", "partnerId": pid,
+ "reason": "This customer is no longer on the Collection worklist."}
+ for pid in ids if pid not in by_id]
+ return selected, refused
+
+
+def _collection_refusal(cs, row):
+ """Preview the existing data-layer guard; the sender still enforces it independently."""
+ if not row.get("Email"):
+ return "No email address is saved for this customer."
+ if not cs.safe_recipient_ok(row["Email"]):
+ return "Safe mode does not allow this recipient address."
+ return None
+
+
+@collection_router.get("/api/v1/apps/collection/worklist")
+def collection_worklist(session: Session = Depends(statement_gate)):
+ import modules.collections_send as cs
+ _, rows = _collection_source(cs)
+ public = []
+ for row in rows:
+ docs = row.get("_docs") or []
+ public.append({**{key: value for key, value in row.items() if key != "_docs"},
+ "_docs": len(docs),
+ "documents": [{"name": doc["name"],
+ "date": doc.get("invoice_date") or None,
+ "dueDate": doc.get("invoice_date_due") or None,
+ "daysOverdue": doc["days_overdue"],
+ "openBalance": doc["open_signed"]} for doc in docs],
+ "sendRefusal": _collection_refusal(cs, row)})
+ return {"rows": public, "safeMode": bool(cs.SAFE_MODE),
+ "safeRecipients": sorted(cs.SAFE_RECIPIENTS),
+ "sender": {"name": cs.SENDER_NAME, "email": cs.SENDER_EMAIL, "replyTo": cs.REPLY_TO}}
+
+
+@collection_router.post("/api/v1/apps/collection/preview")
+def collection_preview(body: dict = Body(default=None), session: Session = Depends(statement_gate)):
+ """Read-only rendered statements, using the same defaults the batch will queue."""
+ import datetime as dt
+ import modules.collections_send as cs
+ ids = _collection_ids(body)
+ _, rows = _collection_source(cs)
+ selected, refused = _collection_selection(rows, ids)
+ statements = []
+ for row in selected:
+ reason = _collection_refusal(cs, row)
+ if reason:
+ refused.append({"customer": row["Customer"], "reason": reason})
+ statements.append({"partnerId": row["partner_id"], "customer": row["Customer"],
+ "to": row["Email"], "refusal": reason,
+ "subject": cs.DEFAULT_SUBJECT.format(
+ customer=row["Customer"], company=cs.COMPANY,
+ month=dt.date.today().strftime("%B %Y")),
+ "html": cs.render_statement_html(row)})
+ return {"statements": statements, "refused": refused, "requested": len(ids),
+ "safeMode": bool(cs.SAFE_MODE), "safeRecipients": sorted(cs.SAFE_RECIPIENTS),
+ "sender": {"name": cs.SENDER_NAME, "email": cs.SENDER_EMAIL, "replyTo": cs.REPLY_TO}}
+
+
+@collection_router.post("/api/v1/apps/collection/send")
+def collection_send(body: dict = Body(default=None), session: Session = Depends(statement_gate)):
+ import modules.collections_send as cs
+ ids = _collection_ids(body, confirm=True)
+ odoo, rows = _collection_source(cs)
+ selected, refused = _collection_selection(rows, ids)
+ try:
+ result = cs.queue_statement_batch(odoo, selected)
+ except Exception:
+ # An interrupted response may follow a successful create. Never invite a blind retry.
+ raise err(502, "collection_send_unconfirmed",
+ "Odoo did not confirm the batch result. Some statements may already be in the "
+ "outbox. Check Odoo before trying again.")
+ return {"created": len(result["created"]), "refused": refused + result["refused"],
+ "safeMode": result["safe_mode"]}
+
+
+# main.py includes exactly one object named router. These sub-routers keep
+# /nav/applications outside the Finance prefix while every Finance URL stays unchanged.
+router = APIRouter()
+router.include_router(finance_router)
+router.include_router(nav_router)
+# main.py belongs to another session this wave; this router is already mounted there.
+router.include_router(activity_router)
+router.include_router(collection_router)
diff --git a/api/routes_automation.py b/api/routes_automation.py
index 37a2fe43754480705da8bcaa46e7ff4bf37428b7..391cbf044e1df4d36a10e2f004bc952d986ec117 100644
--- a/api/routes_automation.py
+++ b/api/routes_automation.py
@@ -433,6 +433,7 @@ def _wire(defn, tenant, rt=None):
"config": defn.get("config") or {}, "schedule": sched, "status": status,
"runs": list(defn.get("runs") or [])[:engine.MAX_RUNS],
"created": defn.get("created"), "createdBy": defn.get("createdBy"),
+ **({"preset": str(defn.get("preset"))} if defn.get("preset") else {}),
"nextRunAt": nxt.strftime("%Y-%m-%d %H:%M") if nxt else "",
"running": bool(live),
# ⭐ WAVE 27 (contract C6) — DEBT D-70: "a paid search is already outstanding at the
@@ -1210,8 +1211,9 @@ def list_automations(session: Session = Depends(_GATE)):
#
# ⛔ WHY SKIPPING THE SCAN ON CALLS 2..N IS SAFE, and it is not "because it is idempotent":
# `all_definitions` STRIPS retired board state on every read (`automation_engine.py`'s
- # `_without_retired_board`), so the persist step is hygiene, not correctness. No client can
- # ever be shown state this skip left behind. Nothing writes new `stage_auto_` cells either —
+ # `_without_retired_board`) and drops retired presets, so persistence is hygiene, not
+ # correctness. No client can see state this skip left behind. Nothing writes new
+ # `stage_auto_` cells either —
# wave 26's R6 deleted the board that made them.
# ⭐⭐ WAVE 30 · T12 (owner items 4/5, the complaint that has now survived TWO waves).
# ⛔ W29-T01 GUARDED THE SCAN AND NOT THE READ, and that is the whole of what was left. The
@@ -1221,6 +1223,13 @@ def list_automations(session: Session = Depends(_GATE)):
# behind each other) — while on calls 2..N the result was DISCARDED: `tables` had exactly two
# consumers, the `_BOARD_RETIRED` scan (skipped after call 1) and a picker DEFAULT STRING.
# A whole-tenant document, per request, to render a rail showing a name and a toggle.
+ # ⛔⛔ AND THE REMOVAL NOTICE OBEYS THE SAME RULE, which is why it is built HERE and not above.
+ # The first draft of this read the whole `automations` document on EVERY request to name the
+ # retired rows, and then a SECOND time to fetch their names — the defect this very comment
+ # block is the tombstone of, re-introduced two lines under it. The sweep has already read
+ # every row it removed, so its receipt carries them: calls 2..N report nothing and read
+ # nothing, which is correct as well as cheap, because after the sweep no retired row exists.
+ retired = []
tables = None
if session.runtime.available() and session.tenant not in _BOARD_RETIRED:
tables = engine.ut_all(session.runtime)
@@ -1228,6 +1237,7 @@ def list_automations(session: Session = Depends(_GATE)):
# Idempotent Board retirement removes only engine-marked stage fields and legacy Board
# state. User-created Status/Stage columns remain intact.
engine.retire_automation_board_state(session.runtime, tables=tables)
+ retired = engine.retire_preset_automations(session.runtime)["removed"]
# ⭐⭐ WAVE 35 · T37 / OWNER RULING R10 — ROYAL IMPORTS' STATEMENTS AGENT EXISTS BEFORE ANYBODY
# ASKS FOR IT. It is minted ONCE, by the container, SWITCHED OFF; every subsequent call finds
# it present and returns immediately. See `engine.seed_statements_agent` for why this one is
@@ -1254,6 +1264,7 @@ def list_automations(session: Session = Depends(_GATE)):
items = sorted(items + _field_agent_rows(session) + _sys_rows,
key=lambda r: str(r.get("name") or "").lower())
return {"automations": items,
+ "retired": retired,
# ⭐ WAVE 24 — DERIVED from the engine's public `PICKER_KINDS`, not a hand-written trio. It was
# three literals that happened to match, which is a second copy of a server
# vocabulary; R6 has just made two of them uncreatable and `plain` has joined, so a
@@ -2114,7 +2125,7 @@ def create_automation(body: dict = Body(default=None), session: Session = Depend
# ⚠ It rides the SAVE's own response, beside `unconfigured`, because that is the moment the
# person is looking — a note in a log they never open is the same silence with extra steps.
_notes = []
- defn, error = engine.create(session.runtime, body or {}, username=session.uname,
+ defn, error = engine.create(session.runtime, body, username=session.uname,
notes=_notes)
if error:
raise err(400, "invalid_automation", error)
@@ -2159,8 +2170,9 @@ def patch_automation(auto_id: str, body: dict = Body(default=None),
# attribute in the builder is a suggestion; this is the refusal. It permits a DELETE of the
# step and permits an ordinary save of a flow that merely contains one.
_agent_owned_guard(session, auto_id, body or {})
+ previous = engine.all_definitions(session.runtime).get(str(auto_id))
_notes = []
- defn, error = engine.patch(session.runtime, auto_id, body or {}, username=session.uname,
+ defn, error = engine.patch(session.runtime, auto_id, body, username=session.uname,
notes=_notes)
if error:
raise err(400 if error != "no such automation" else 404,
diff --git a/api/routes_connected_tables.py b/api/routes_connected_tables.py
index 1cd102b148141f08c4634be179319b9e070a5be6..f51584efca4a275be7c3adc439d61dbbe3bf21bc 100644
--- a/api/routes_connected_tables.py
+++ b/api/routes_connected_tables.py
@@ -62,6 +62,8 @@ def connected_table_rows(table_key: str,
sorts: str = Query(default=None),
search: str = Query(default=None),
related: str = Query(default=None),
+ groupBy: str = Query(default=None),
+ groupKey: str = Query(default=None),
session: Session = Depends(require_session)):
"""ONE WINDOW over a connected grid — `{fields, rows, total, totalUnfiltered, offset, limit,
limits, identity, recordsMutable}`, whichever connector the rows come from.
@@ -89,4 +91,5 @@ def connected_table_rows(table_key: str,
import routes_odoo_tables
return routes_odoo_tables.odoo_table_rows(
table_key, offset=offset, limit=limit, filters=filters, filterConj=filterConj,
- sorts=sorts, search=search, related=related, session=session)
+ sorts=sorts, search=search, related=related, groupBy=groupBy, groupKey=groupKey,
+ session=session)
diff --git a/api/routes_connectors.py b/api/routes_connectors.py
index c9f462aaf85aa84210b2a98a620bf503a07a4165..83148bd4972f378385095a41f75d61b3280777dc 100644
--- a/api/routes_connectors.py
+++ b/api/routes_connectors.py
@@ -21,6 +21,7 @@ that reported another user's Gmail as "connected" would be both wrong and a smal
from __future__ import annotations
import os
+import ssl
import time
import threading
import xmlrpc.client
@@ -30,7 +31,7 @@ from datetime import datetime, timezone
from fastapi import APIRouter, Body, Depends
import oauth_connect
-from deps import Session, err, require_session
+from deps import Session, err, require_session, running_main
#: ⭐⭐ W49-T18 / R27 / C4 — THE TAXONOMY LIVES IN THE DATA LAYER, NOT HERE. `core` is already on
#: the path by the time this module loads (`deps` imports three of its modules), so this is the
#: same import shape the rest of the API uses. It is in `platform/core/` rather than beside this
@@ -82,6 +83,61 @@ from routes_monid import router as monid_router
router = APIRouter(prefix="/api/v1")
CONNECTOR_DETAIL_DEADLINE_SECONDS = 30
+SCRAPER_RECIPE_KINDS = frozenset(("instagram", "tiktok", "google_maps"))
+SCRAPER_RECIPE_MAX_ROWS = 30
+SCRAPER_RUN_HISTORY_LIMIT = 20
+DATABASE_SYNC_STATES = frozenset(
+ ("never_run", "queued", "running", "succeeded", "partial", "failed", "disabled")
+)
+_DATABASE_SAFE_ERRORS = {
+ "tls_certificate_invalid": ("Odoo TLS certificate verification failed because the certificate "
+ "is expired or untrusted. No source data was accepted."),
+ "source_read_refused": "Odoo refused the configured account's read access.",
+ "source_auth_failed": "Odoo rejected the configured read-only credential.",
+ "source_unavailable": "Odoo could not be reached for this read-only sync.",
+ "source_read_failed": "Odoo source validation failed.",
+ "mirror_pull_failed": "The Odoo mirror pull failed.",
+ "materialization_failed": "The database could not be materialized.",
+ "reconciliation_failed": "The database reconciliation failed.",
+ "freshness_unconfirmed": "Freshness could not be confirmed for this database.",
+ "sync_status_unavailable": "Sync status is unavailable for this database.",
+ "sync_stalled": "This database sync is stalled.",
+ "sync_state_unreadable": "This database's saved sync state could not be read.",
+ "maps_credential_missing": "Add the scraper credential in Keychain before choosing Run now.",
+ "scraper_credential_missing": "Add the scraper credential in Keychain before choosing Run now.",
+ "maps_unbounded_request": "Google Maps discovery requires an explicit bounded result cap.",
+ "maps_invalid_search": "Google Maps discovery needs a valid category or search.",
+ "maps_vendor_error": "The Maps provider could not complete this run.",
+ "maps_schema_drift": "The Maps provider returned an unreadable result shape.",
+ "scraper_run_failed": "The provider could not complete this run. No result was written.",
+ "provider_result_failed": "The provider returned no usable result for this database.",
+ "scraper_result_not_saved": "The provider answered, but the result could not be saved.",
+}
+
+# T11 established the saved database contract; T12's provider normalises every Apify answer into
+# exactly this schema. Vendor field names never reach a saved View or downstream automation.
+_MAPS_RECIPE_FIELDS = [
+ {"key": "place_id", "label": "Place id", "type": "text", "source": "automation"},
+ {"key": "name", "label": "Name", "type": "text", "source": "automation"},
+ {"key": "category", "label": "Category", "type": "text", "source": "automation"},
+ {"key": "address", "label": "Address", "type": "text", "source": "automation"},
+ {"key": "city", "label": "City", "type": "text", "source": "automation"},
+ {"key": "region", "label": "Region", "type": "text", "source": "automation"},
+ {"key": "postal_code", "label": "Postal code", "type": "text", "source": "automation"},
+ {"key": "country", "label": "Country", "type": "text", "source": "automation"},
+ {"key": "phone", "label": "Phone", "type": "phone", "source": "automation"},
+ {"key": "website", "label": "Website", "type": "url", "source": "automation"},
+ # Coordinates remain text because `number` is not a user-table field kind; the Map binding
+ # parses the canonical scalar value. Declaring an unsupported kind would silently drop both
+ # columns at database creation.
+ {"key": "latitude", "label": "Latitude", "type": "text", "source": "automation"},
+ {"key": "longitude", "label": "Longitude", "type": "text", "source": "automation"},
+ {"key": "rating", "label": "Rating", "type": "rating", "source": "automation"},
+ {"key": "reviews", "label": "Reviews", "type": "int", "source": "automation"},
+ {"key": "source_timestamp", "label": "Source timestamp", "type": "date",
+ "source": "automation"},
+ {"key": "source_payload", "label": "Source data", "type": "json", "source": "automation"},
+]
# Measured reads take 1.34 s cold, 0.01 s warm, and above 8 s under boot contention.
@@ -150,13 +206,16 @@ PLANNED = [
#: 'Scraper' … would I see the option to choose either Instagram OR TikTok."* So the directory
#: mirrors the automation menus exactly: ONE card for the capability, and the platforms are behind
#: it rather than beside it.
-#: ⚠ These are not connectors and must never become rows: they have no state, no credential and
-#: no manage door of their own — the Scraper's key is what is configured, and it configures both.
+#: ⚠ These are not connectors and must never become rows: they have no state or manage door of
+#: their own. Instagram and TikTok share the Scraper credential; Maps uses the Apify credential
+#: through the same governed provider registry.
SCRAPER_PLATFORMS = [
{"key": "instagram", "label": "Instagram",
"desc": "Profiles, posts and comments, by search or by handle."},
{"key": "tiktok", "label": "TikTok",
"desc": "The same three, on TikTok's own schema."},
+ {"key": "google_maps", "label": "Google Maps",
+ "desc": "Places by category or search and geography, through a bounded Apify run."},
]
#: Token connectors an admin can wire up TODAY by pasting a key — no OAuth client, no app review.
@@ -1063,13 +1122,16 @@ def _scraper_row():
# ⭐ WAVE 30 · R4 — the sentence names the CAPABILITY; the platforms are the drill below.
# It used to say "Instagram profile and post capture", which stopped being true the day
# TikTok discovery shipped and would have to be re-edited for every platform after it.
- "desc": "Profile, post and comment capture for the discovery automations.",
+ "desc": "Create saved Instagram, TikTok and Google Maps database recipes.",
"state": "connected" if ready else "unconfigured",
"needs": "" if ready else "AIOS_BRIGHTDATA_KEY is not set on this deployment",
"manage": "",
- # ⛔ ONE CREDENTIAL, TWO PLATFORMS — which is why this is a field on the row and not two
- # rows. `bd_ready()` above answers for both; a per-platform state would be a second
- # answer to a question that has one.
+ # The directory taxonomy normally derives this flag from connected table prefixes.
+ # Scraper recipes are user databases, so the route stamps it again after normalisation.
+ "spawns_tables": True,
+ # Platforms remain a drill behind one recipe/database capability, never directory rows.
+ # `bd_ready()` describes today's social runners only; Maps' own sentence names that its
+ # execution provider is intentionally absent until T12.
"platforms": [dict(p) for p in SCRAPER_PLATFORMS],
}
@@ -1129,6 +1191,8 @@ def connectors_directory(session: Session = Depends(require_session)):
# trap; `chip_counts` carries the argument).
served = connector_directory.to_client_rows(rows)
for row in served:
+ if str(row.get("key") or "") == "scraper":
+ row["spawns_tables"] = True
_safe_connector_status(row.get("key"), row)
_safe_connector_status("amazon_sp", {"incomplete": incomplete})
return {"connectors": served,
@@ -1162,6 +1226,357 @@ def connectors_directory(session: Session = Depends(require_session)):
_ODOO_SCHEMA_ENABLEMENTS = "odoo_schema_enablements"
_ODOO_SYNC_STATUS = "odoo_sync_status"
+_ODOO_HEALTH_STATUS = "odoo_health_status_v1"
+_ODOO_HEALTH_STAGES = ("source", "mirrorPull", "materialization", "reconciliation", "freshness")
+_SYNC_REQUEST_CONNECTIONS = {}
+_SYNC_REQUEST_CONNECTION_LOCK = threading.Lock()
+
+
+def _odoo_connection_slot(rt, connection_id=None):
+ """Return the durable Odoo connection id used by jobs, status, and health receipts."""
+ requested = str(connection_id or "").strip()
+ if requested and requested != "default":
+ return requested
+ try:
+ import odoo_relational
+ return str(odoo_relational.default_connection_id(rt) or "default")
+ except Exception: # noqa: BLE001
+ return "default"
+
+
+def sync_request_connection(provider, tenant):
+ """Return the connection bound to a process-local Sync now intent."""
+ with _SYNC_REQUEST_CONNECTION_LOCK:
+ return _SYNC_REQUEST_CONNECTIONS.get((str(provider), str(tenant)))
+
+
+def _odoo_failure(exc, stage="source"):
+ """Classify a source/database exception without persisting its potentially sensitive text."""
+ stage = stage if stage in _ODOO_HEALTH_STAGES else "source"
+ text = str(exc or "").casefold()
+ exc_type = type(exc).__name__ if isinstance(exc, BaseException) else "SourceError"
+ if (isinstance(exc, ssl.SSLCertVerificationError)
+ or "certificate_verify_failed" in text
+ or "certificate verify failed" in text
+ or "certificate has expired" in text
+ or "self signed certificate" in text):
+ return {"stage": "source", "code": "tls_certificate_invalid",
+ "type": exc_type,
+ "message": ("Odoo TLS certificate verification failed because the certificate "
+ "is expired or untrusted. No source data was accepted.")}
+ if isinstance(exc, xmlrpc.client.Fault) and any(
+ token in text for token in ("accesserror", "access denied", "security restrictions")):
+ return {"stage": "source", "code": "source_read_refused", "type": exc_type,
+ "message": "Odoo refused the configured account's read access."}
+ if any(token in text for token in ("authentication", "access denied", "invalid api key")):
+ return {"stage": "source", "code": "source_auth_failed", "type": exc_type,
+ "message": "Odoo rejected the configured read-only credential."}
+ if isinstance(exc, (TimeoutError, ConnectionError)) or any(
+ token in text for token in ("timed out", "timeout", "name resolution", "connection refused")):
+ return {"stage": "source", "code": "source_unavailable", "type": exc_type,
+ "message": "Odoo could not be reached for this read-only sync."}
+ labels = {
+ "source": ("source_read_failed", "Odoo source validation failed."),
+ "mirrorPull": ("mirror_pull_failed", "The Odoo mirror pull failed."),
+ "materialization": ("materialization_failed", "The Odoo database could not be materialized."),
+ "reconciliation": ("reconciliation_failed", "The Odoo database reconciliation failed."),
+ "freshness": ("freshness_unconfirmed", "Freshness could not be confirmed for this database."),
+ }
+ code, message = labels[stage]
+ return {"stage": stage, "code": code, "type": exc_type, "message": message}
+
+
+def _clean_odoo_failure(failure, stage):
+ if failure is None:
+ return None
+ if not isinstance(failure, dict):
+ return _odoo_failure(failure, stage)
+ return {"stage": str(failure.get("stage") or stage),
+ "code": str(failure.get("code") or f"{stage}_failed"),
+ "type": str(failure.get("type") or "SourceError")[:80],
+ "message": str(failure.get("message") or "Odoo sync failed.")[:300]}
+
+
+def _odoo_stage_failure(stages):
+ """Return the earliest unresolved lifecycle failure, preserving its named cause."""
+ return next((stages.get(name, {}).get("failure") for name in _ODOO_HEALTH_STAGES
+ if stages.get(name, {}).get("state") == "failed"
+ and stages.get(name, {}).get("failure")), None)
+
+
+def _clean_odoo_database_receipt(receipt, database_key):
+ """Allow-list one data-free reconciliation receipt for durable health storage."""
+ if not isinstance(receipt, dict):
+ return None
+ key = str(database_key or "").strip()
+ if str(receipt.get("databaseKey") or "").strip() != key:
+ raise ValueError("the reconciliation receipt belongs to a different database")
+ out = {"databaseKey": key}
+ for name in ("schemaKey", "topicKey", "domainKey", "model", "countModel", "state",
+ "sourceCountMode", "sourceReconciliation"):
+ value = receipt.get(name)
+ if value is not None:
+ out[name] = str(value)[:160]
+ for name in ("sourceCount", "mirrorCount", "materializedCount", "canonicalCount",
+ "servingCount", "duplicateCount", "explainedMergeCount",
+ "unexplainedMissingCount", "unexplainedExtraCount",
+ "unexplainedCanonicalMissingCount", "unexplainedCanonicalExtraCount",
+ "unexplainedCount"):
+ value = receipt.get(name)
+ out[name] = None if value is None else max(0, int(value))
+ if "readThrough" in receipt:
+ out["readThrough"] = bool(receipt.get("readThrough"))
+ selection = receipt.get("selection")
+ if isinstance(selection, dict):
+ out["selection"] = {
+ name: (bool(selection.get(name)) if name == "enabled"
+ else str(selection.get(name) or "")[:160])
+ for name in ("bucket", "rule", "enabled") if name in selection}
+ domain = receipt.get("domain")
+ if isinstance(domain, list):
+ # Curated domains contain public field/operator/value predicates. Refuse nested objects,
+ # which could otherwise turn this metadata seam into durable source-row storage.
+ clean_domain = []
+ for predicate in domain:
+ if not isinstance(predicate, (list, tuple)) or len(predicate) != 3:
+ continue
+ field, operator, value = predicate
+ if isinstance(value, (list, tuple)):
+ clean_value = [item for item in value
+ if isinstance(item, (str, int, float, bool))][:50]
+ elif isinstance(value, (str, int, float, bool)) or value is None:
+ clean_value = value
+ else:
+ continue
+ clean_domain.append([str(field)[:100], str(operator)[:40], clean_value])
+ out["domain"] = clean_domain
+ reconciliation = receipt.get("reconciliation")
+ if isinstance(reconciliation, dict):
+ out["reconciliation"] = {
+ name: str(reconciliation.get(name) or "")[:80]
+ for name in ("sourceToMirror", "mirrorToMaterialized", "mirrorToCanonical")
+ if name in reconciliation}
+ freshness = receipt.get("freshness")
+ if isinstance(freshness, dict):
+ out["freshness"] = {
+ name: str(freshness.get(name) or "")[:100]
+ for name in ("state", "capturedAt") if name in freshness}
+ for name in ("sourceError", "error"):
+ if receipt.get(name) is not None:
+ out[name] = _clean_odoo_failure(receipt.get(name), "reconciliation")
+ return out
+
+
+def record_odoo_stage(rt, connection_id, stage, state, *, database_key=None, row_count=None,
+ refreshed_at=None, cadence_seconds=None, failure=None, receipt=None,
+ at=None):
+ """Persist one data-free Odoo stage transition for one connection/database.
+
+ This is additive observability over the established ``connector_sync`` and materialization
+ job documents. It stores identifiers, counts, timestamps, and classified failures only;
+ credentials and source rows never cross this seam.
+ """
+ if stage not in _ODOO_HEALTH_STAGES:
+ raise ValueError(f"unknown Odoo health stage: {stage}")
+ if state not in ("unknown", "queued", "running", "ready", "failed", "paused"):
+ raise ValueError(f"unknown Odoo health state: {state}")
+ cid = _odoo_connection_slot(rt, connection_id)
+ stamp = str(at or datetime.now(timezone.utc).isoformat())
+ failed = _clean_odoo_failure(failure, stage) if state == "failed" else None
+ db_key = str(database_key or "").strip()
+ safe_receipt = _clean_odoo_database_receipt(receipt, db_key) if receipt is not None else None
+
+ def _update(current):
+ document = dict(current or {})
+ connections = dict(document.get("connections") or {})
+ connection = dict(connections.get(cid) or {})
+ stages = dict(connection.get("stages") or {})
+ target_stages = stages
+ database = None
+ databases = None
+ if db_key:
+ databases = dict(connection.get("databases") or {})
+ database = dict(databases.get(db_key) or {"databaseKey": db_key})
+ target_stages = dict(database.get("stages") or {})
+ previous = dict(target_stages.get(stage) or {})
+ stage_row = {**previous, "state": state, "at": stamp, "failure": failed}
+ if state == "running":
+ stage_row["startedAt"] = stamp
+ stage_row["finishedAt"] = None
+ elif state in ("ready", "failed", "paused"):
+ stage_row["finishedAt"] = stamp
+ if state == "ready":
+ stage_row["lastSuccess"] = stamp
+ target_stages[stage] = stage_row
+ connection.update({"connectionId": cid, "stages": stages, "lastAttempt": stamp})
+ if cadence_seconds is not None:
+ connection["cadenceSeconds"] = int(cadence_seconds)
+
+ if db_key:
+ database.update({"databaseKey": db_key, "stages": target_stages,
+ "lastAttempt": stamp})
+ if row_count is not None:
+ database["rowCount"] = max(0, int(row_count))
+ if refreshed_at:
+ database["recordRefreshed"] = str(refreshed_at)
+ if safe_receipt is not None:
+ database["reconciliationReceipt"] = safe_receipt
+ db_failure = _odoo_stage_failure(target_stages)
+ if db_failure:
+ database["failure"] = db_failure
+ else:
+ database.pop("failure", None)
+ if state == "ready" and stage == "freshness":
+ database["lastSuccess"] = stamp
+ databases[db_key] = database
+ connection["databases"] = databases
+ else:
+ stages[stage] = stage_row
+ connection["stages"] = stages
+ connection_failure = _odoo_stage_failure(stages)
+ if connection_failure:
+ connection["failure"] = connection_failure
+ else:
+ connection.pop("failure", None)
+ if state == "ready" and stage == "freshness":
+ connection["lastSuccess"] = stamp
+ connections[cid] = connection
+ document["connections"] = connections
+ return document
+
+ rt.update(_ODOO_HEALTH_STATUS, _update, flush="sync")
+ return odoo_health_receipt(rt, cid)
+
+
+def _lag_seconds(stamp, now=None):
+ if not stamp:
+ return None
+ try:
+ parsed = datetime.fromisoformat(str(stamp).replace("Z", "+00:00"))
+ if parsed.tzinfo is None:
+ parsed = parsed.replace(tzinfo=timezone.utc)
+ current = now or datetime.now(timezone.utc)
+ return max(0, int((current - parsed.astimezone(timezone.utc)).total_seconds()))
+ except (TypeError, ValueError):
+ return None
+
+
+def _empty_odoo_stages():
+ return {stage: {"state": "unknown", "at": None, "startedAt": None,
+ "finishedAt": None, "lastSuccess": None, "failure": None}
+ for stage in _ODOO_HEALTH_STAGES}
+
+
+def odoo_health_receipt(rt, connection_id=None, *, now=None):
+ """Project the truthful Odoo receipt for one connection and each of its databases."""
+ cid = _odoo_connection_slot(rt, connection_id)
+ try:
+ document = rt.get(_ODOO_HEALTH_STATUS, fresh=True) or {}
+ except TypeError:
+ document = rt.get(_ODOO_HEALTH_STATUS) or {}
+ except Exception: # noqa: BLE001
+ document = {}
+ stored = dict(((document.get("connections") or {}).get(cid)) or {})
+ stages = _empty_odoo_stages()
+ stages.update({key: dict(value) for key, value in (stored.get("stages") or {}).items()
+ if key in stages and isinstance(value, dict)})
+ databases = {str(key): dict(value) for key, value in (stored.get("databases") or {}).items()
+ if isinstance(value, dict)}
+
+ # Durable materialization receipts are the authority for database work and survive restarts.
+ for job in connector_sync.jobs(rt, "odoo", cid):
+ key = str(job.get("databaseKey") or "")
+ if not key:
+ continue
+ database = dict(databases.get(key) or {"databaseKey": key})
+ db_stages = _empty_odoo_stages()
+ db_stages.update({name: dict(value) for name, value in (database.get("stages") or {}).items()
+ if name in db_stages and isinstance(value, dict)})
+ job_state = str(job.get("state") or "unknown")
+ material_state = {"configured": "queued", "catalogue-ready": "queued",
+ "queued": "queued", "syncing": "running", "ready": "ready",
+ "failed": "failed"}.get(job_state, "unknown")
+ job_at = None
+ if job.get("at") is not None:
+ try:
+ job_at = datetime.fromtimestamp(float(job["at"]), timezone.utc).isoformat()
+ except (TypeError, ValueError, OSError):
+ job_at = None
+ job_failure = (_odoo_failure(job.get("error"), "materialization")
+ if material_state == "failed" else None)
+ db_stages["materialization"] = {
+ **db_stages["materialization"], "state": material_state,
+ "at": job_at, "finishedAt": job_at if material_state in ("ready", "failed") else None,
+ "lastSuccess": (job_at if material_state == "ready" else
+ db_stages["materialization"].get("lastSuccess")),
+ "failure": job_failure,
+ }
+ progress = job.get("progress") if isinstance(job.get("progress"), dict) else {}
+ if progress.get("rows") is not None:
+ database["rowCount"] = max(0, int(progress["rows"]))
+ database.update({"databaseKey": key, "model": str(job.get("model") or ""),
+ "state": material_state, "stages": db_stages})
+ persisted_failure = database.get("failure") or _odoo_stage_failure(db_stages)
+ if persisted_failure or job_failure:
+ database["failure"] = persisted_failure or job_failure
+ elif material_state == "ready" and not database.get("failure"):
+ database["lastSuccess"] = database.get("lastSuccess") or job_at
+ databases[key] = database
+
+ # The resident mirror state is process-local by design; overlay it only on the connection
+ # that owns this tenant's mirror. A named sibling must never inherit default's fresh badge.
+ cadence = stored.get("cadenceSeconds")
+ try:
+ _main = running_main() # D-781: resolve, never import
+ resident = dict(_main.RESYNC_STATE)
+ resident_cid = str(resident.get("connectionId") or "")
+ if resident_cid == cid:
+ cadence = resident.get("resolvedSeconds") or cadence
+ mirror_failure = resident.get("lastError")
+ mirror_state = ("paused" if resident.get("manual") else
+ "failed" if resident.get("lastPassOk") is False else
+ "ready" if resident.get("lastPassOk") is True else "unknown")
+ stages["mirrorPull"] = {**stages["mirrorPull"], "state": mirror_state,
+ "at": resident.get("lastPassAt"),
+ "finishedAt": resident.get("lastPassAt"),
+ "lastSuccess": resident.get("lastSuccessAt"),
+ "failure": (_clean_odoo_failure(mirror_failure, "mirrorPull")
+ if mirror_failure else None)}
+ except Exception: # noqa: BLE001
+ pass
+
+ for database in databases.values():
+ db_stages = _empty_odoo_stages()
+ db_stages.update({name: dict(value) for name, value in (database.get("stages") or {}).items()
+ if name in db_stages and isinstance(value, dict)})
+ # Connection source/mirror/reconciliation facts apply to each database, while its own
+ # materialization/freshness facts remain independent.
+ for stage in ("source", "mirrorPull", "reconciliation"):
+ if db_stages[stage].get("state") == "unknown":
+ db_stages[stage] = dict(stages[stage])
+ database["stages"] = db_stages
+ success = database.get("lastSuccess") or db_stages["freshness"].get("lastSuccess")
+ database["lastSuccess"] = success
+ database["currentLagSeconds"] = _lag_seconds(success, now)
+ database.setdefault("rowCount", None)
+ database.setdefault("recordRefreshed", None)
+ stage_failure = _odoo_stage_failure(db_stages)
+ if stage_failure:
+ database["failure"] = stage_failure
+ elif not database.get("failure"):
+ database["failure"] = None
+
+ successes = [row.get("lastSuccess") for row in databases.values() if row.get("lastSuccess")]
+ last_success = max(successes, default=stored.get("lastSuccess"))
+ row_counts = [row.get("rowCount") for row in databases.values()
+ if row.get("rowCount") is not None]
+ failure = (_odoo_stage_failure(stages) or stored.get("failure") or next(
+ (row.get("failure") for row in databases.values() if row.get("failure")), None))
+ return {"connectionId": cid, "cadenceSeconds": cadence,
+ "lastSuccess": last_success, "currentLagSeconds": _lag_seconds(last_success, now),
+ "rowCount": sum(row_counts) if row_counts else None,
+ "failure": failure, "stages": stages,
+ "databases": [databases[key] for key in sorted(databases)]}
def _odoo_admin(session):
@@ -1178,8 +1593,7 @@ def _connection_jobs(rt, provider, connection_id):
"""A status endpoint selects one connection; the jobs inventory may enumerate them all."""
cid = str(connection_id or "").strip() or "default"
if provider == "odoo" and cid == "default":
- import odoo_relational
- cid = odoo_relational.default_connection_id(rt) or "default"
+ cid = _odoo_connection_slot(rt, connection_id)
return connector_sync.jobs(rt, provider, cid)
@@ -1200,17 +1614,86 @@ def _sync_payload(connector_key, connection_id, row=None):
return _safe_connector_status(connector_key, answer)
+def _odoo_result_database_failures(rt, connection_id, result):
+ """Map provider-reported model/table failures to their durable database keys."""
+ import odoo_relational as rel
+ try:
+ choices = rel.grid_choices(rt, connection_id)
+ except Exception: # noqa: BLE001
+ choices = []
+ by_model = {str(row.get("model") or ""): str(row.get("key") or "")
+ for row in choices if row.get("model") and row.get("key")}
+ failures = {str(key): _clean_odoo_failure(value, "materialization")
+ for key, value in (result.get("databaseErrors") or {}).items()}
+ receipt_groups = []
+ for name in ("databases", "canonicalDatabases"):
+ group = result.get(name) if isinstance(result, dict) else None
+ if isinstance(group, dict):
+ receipt_groups.append(group)
+ for group in receipt_groups:
+ for raw_key, receipt in group.items():
+ if not isinstance(receipt, dict) or receipt.get("state") == "succeeded":
+ continue
+ key = str(raw_key or "")
+ if not key or key in failures:
+ continue
+ raw_failure = receipt.get("error") or receipt.get("sourceError") or {
+ "stage": "reconciliation", "code": "reconciliation_incomplete",
+ "type": "ReconciliationError",
+ "message": "The Odoo database reconciliation is incomplete.",
+ }
+ failures[key] = _clean_odoo_failure(raw_failure, "reconciliation")
+ unmatched = []
+ discovered = result.get("discovered") if isinstance(result, dict) else {}
+ for raw in ((discovered or {}).get("errors") or ()):
+ model = str(raw).split(":", 1)[0].strip()
+ key = by_model.get(model)
+ if key:
+ failures[key] = _odoo_failure(raw, "materialization")
+ else:
+ unmatched.append(str(raw))
+ limits = result.get("limits") if isinstance(result, dict) else None
+ if isinstance(limits, dict):
+ for raw_key, raw in limits.items():
+ key = by_model.get(str(raw_key or ""), str(raw_key or ""))
+ if key:
+ failures[key] = _odoo_failure(raw, "materialization")
+ elif limits:
+ unmatched.append(str(limits))
+ return failures, unmatched
+
+
def _odoo_sync_runner(session, connection_id):
"""Run Odoo through C6 without moving provider imports into the data layer."""
- connector_sync.record(session.runtime, "odoo", connection_id, "syncing")
+ cid = _odoo_connection_slot(session.runtime, connection_id)
+ connector_sync.record(session.runtime, "odoo", cid, "syncing")
+ record_odoo_stage(session.runtime, cid, "source", "running")
+ record_odoo_stage(session.runtime, cid, "materialization", "queued")
import odoo_relational
- result = odoo_relational.refresh(session.runtime, session.tenant,
- username=session.uname or "automation",
- connection_id=connection_id,
- on_apply=lambda: connector_sync.record(
- session.runtime, "odoo", connection_id, "materializing"))
+ def _materializing():
+ connector_sync.record(session.runtime, "odoo", cid, "materializing")
+ record_odoo_stage(session.runtime, cid, "materialization", "running")
+ try:
+ result = odoo_relational.refresh(session.runtime, session.tenant,
+ username=session.uname or "automation",
+ connection_id=connection_id,
+ on_apply=_materializing)
+ except Exception as exc:
+ failure = _odoo_failure(exc, "source")
+ record_odoo_stage(session.runtime, cid, failure["stage"], "failed", failure=failure)
+ record_odoo_stage(session.runtime, cid, "freshness", "failed",
+ failure={**failure, "stage": "freshness"})
+ connector_sync.record(session.runtime, "odoo", cid, "error",
+ errors=[f"{failure['code']}: {failure['message']}"])
+ raise
tables = result.get("tables") if isinstance(result, dict) else {}
- databases = sorted(str(key) for key in (tables or {}).keys())
+ published = dict(tables or {})
+ database_receipts = result.get("databases") if isinstance(result, dict) else {}
+ database_receipts = database_receipts if isinstance(database_receipts, dict) else {}
+ canonical_receipts = result.get("canonicalDatabases") if isinstance(result, dict) else {}
+ canonical_receipts = canonical_receipts if isinstance(canonical_receipts, dict) else {}
+ reconciliation_receipts = {**database_receipts, **canonical_receipts}
+ databases = sorted(str(key) for key in published)
sync_errors = []
if isinstance(result, dict):
if result.get("failed"):
@@ -1220,12 +1703,57 @@ def _odoo_sync_runner(session, connection_id):
sync_errors.extend(str(value) for value in (discovered.get("errors") or ()))
discovered_tables = discovered.get("tables")
if isinstance(discovered_tables, dict):
+ published.update(discovered_tables)
databases.extend(str(key) for key in discovered_tables)
if result.get("frozen"):
sync_errors.append(str(result.get("note") or "the connector is frozen"))
- state = ("partial" if databases else "error") if sync_errors else "ready"
+ database_failures, unmatched_errors = _odoo_result_database_failures(
+ session.runtime, connection_id, result if isinstance(result, dict) else {})
+ databases.extend(reconciliation_receipts)
+ databases.extend(database_failures)
+ sync_errors.extend(f"{key}: {failure['code']}" for key, failure in database_failures.items())
+ sync_errors.extend(raw for raw in unmatched_errors if raw not in sync_errors)
+ state = ("partial" if published else "error") if sync_errors else "ready"
+ record_odoo_stage(session.runtime, cid, "source", "ready")
+ refreshed_at = datetime.now(timezone.utc).isoformat()
+ database_keys = list(dict.fromkeys([*reconciliation_receipts, *published]))
+ for key in database_keys:
+ if key in database_failures:
+ continue
+ counts = published.get(key)
+ receipt = reconciliation_receipts.get(key)
+ receipt = receipt if isinstance(receipt, dict) else None
+ nrows = (receipt.get("servingCount") if receipt is not None
+ else counts.get("rows") if isinstance(counts, dict) else None)
+ record_odoo_stage(session.runtime, cid, "source", "ready", database_key=key,
+ receipt=receipt)
+ record_odoo_stage(session.runtime, cid, "materialization", "ready",
+ database_key=key, row_count=nrows, receipt=receipt)
+ record_odoo_stage(session.runtime, cid, "reconciliation", "ready",
+ database_key=key, row_count=nrows, receipt=receipt)
+ record_odoo_stage(session.runtime, cid, "freshness", "ready",
+ database_key=key, row_count=nrows, refreshed_at=refreshed_at,
+ receipt=receipt)
+ for key, failure in database_failures.items():
+ receipt = reconciliation_receipts.get(key)
+ receipt = receipt if isinstance(receipt, dict) else None
+ failed_stage = str(failure.get("stage") or "materialization")
+ failed_stage = failed_stage if failed_stage in _ODOO_HEALTH_STAGES else "materialization"
+ record_odoo_stage(session.runtime, cid, failed_stage, "failed",
+ database_key=key, failure=failure, receipt=receipt)
+ record_odoo_stage(session.runtime, cid, "freshness", "failed", database_key=key,
+ failure={**failure, "stage": "freshness"}, receipt=receipt)
+ if sync_errors:
+ failure = _odoo_failure("provider result contained errors", "materialization")
+ record_odoo_stage(session.runtime, cid, "materialization", "failed", failure=failure)
+ record_odoo_stage(session.runtime, cid, "freshness", "failed",
+ failure={**failure, "stage": "freshness"})
+ else:
+ record_odoo_stage(session.runtime, cid, "materialization", "ready")
+ record_odoo_stage(session.runtime, cid, "reconciliation", "ready")
+ record_odoo_stage(session.runtime, cid, "freshness", "ready")
return connector_sync.record(
- session.runtime, "odoo", connection_id, state,
+ session.runtime, "odoo", cid, state,
detail=str(result.get("note") or "") if isinstance(result, dict) else "",
errors=sync_errors, databases=sorted(set(databases)),
)
@@ -1339,6 +1867,9 @@ def _run_database_job(rt, candidate):
return
fenced = _MaterializationRuntime(rt, job)
key = job["databaseKey"]
+ cid = job["connectionId"]
+ record_odoo_stage(rt, cid, "source", "running", database_key=key)
+ record_odoo_stage(rt, cid, "materialization", "running", database_key=key)
def finish(success=False, cause=""):
# The pre-claim check cannot cover a disconnect during source I/O or publication.
# Only the current lease may requeue; disable and a newer Save still win.
@@ -1398,22 +1929,64 @@ def _run_database_job(rt, candidate):
fenced, job, rows=rows, total=total))}
discovered = result.get("discovered") or {}
published = {**(result.get("tables") or {}), **(discovered.get("tables") or {})}
+ receipt = (result.get("databases") or {}).get(key)
+ receipt = receipt if isinstance(receipt, dict) else None
+ database_errors = result.get("databaseErrors") or {}
success = (key in published and not result.get("failed") and not result.get("frozen")
- and not result.get("limits") and not discovered.get("errors"))
- finish(success=success, cause=result.get("note") or discovered.get("note") or "")
+ and not result.get("limits") and not discovered.get("errors")
+ and key not in database_errors
+ and (receipt is None or receipt.get("state") == "succeeded"))
+ counts = published.get(key) if isinstance(published.get(key), dict) else {}
+ nrows = (receipt.get("servingCount") if receipt is not None
+ else counts.get("rows") if isinstance(counts, dict) else None)
+ if success:
+ refreshed_at = datetime.now(timezone.utc).isoformat()
+ record_odoo_stage(rt, cid, "source", "ready", database_key=key,
+ receipt=receipt)
+ record_odoo_stage(rt, cid, "materialization", "ready",
+ database_key=key, row_count=nrows, receipt=receipt)
+ record_odoo_stage(rt, cid, "reconciliation", "ready",
+ database_key=key, row_count=nrows, receipt=receipt)
+ record_odoo_stage(rt, cid, "freshness", "ready", database_key=key,
+ row_count=nrows, refreshed_at=refreshed_at, receipt=receipt)
+ finish(success=True)
+ else:
+ raw_failure = ((database_errors.get(key) if isinstance(database_errors, dict) else None)
+ or (receipt or {}).get("error") or (receipt or {}).get("sourceError"))
+ failure = (_clean_odoo_failure(raw_failure, "reconciliation") if raw_failure
+ else _odoo_failure("provider result was incomplete", "materialization"))
+ failed_stage = str(failure.get("stage") or "materialization")
+ failed_stage = (failed_stage if failed_stage in _ODOO_HEALTH_STAGES
+ else "materialization")
+ record_odoo_stage(rt, cid, failed_stage, "failed",
+ database_key=key, failure=failure, receipt=receipt)
+ record_odoo_stage(rt, cid, "freshness", "failed", database_key=key,
+ failure={**failure, "stage": "freshness"}, receipt=receipt)
+ finish(cause=f"{failure['code']}: {failure['message']}")
except _SupersededMaterialization:
# Disable/new Save already invalidated the table marker; never restore it here.
finish()
except rel.Refused as exc:
- finish(cause=str(exc))
+ failure = _odoo_failure(exc, "materialization")
+ record_odoo_stage(rt, cid, failure["stage"], "failed",
+ database_key=key, failure=failure)
+ record_odoo_stage(rt, cid, "freshness", "failed", database_key=key,
+ failure={**failure, "stage": "freshness"})
+ finish(cause=f"{failure['code']}: {failure['message']}")
except Exception as exc:
- # Raw provider exceptions may carry credentials or source values. Store only safe copy.
- cause = ""
+ # Raw provider exceptions may carry credentials or source values. Store only a named,
+ # data-free classification, including the fail-closed TLS case.
+ failure = _odoo_failure(exc, "source")
if _odoo_read_access_refused(exc):
- cause = (f"Odoo refused to read {job['model']} because the connected Odoo user has no "
- "read access, so retrying cannot load rows. "
- "Give that user read access in Odoo, or switch this database off.")
- finish(cause=cause)
+ failure = {"stage": "source", "code": "source_read_refused",
+ "type": type(exc).__name__,
+ "message": (f"Odoo refused read access to {job['model']}. Give the "
+ "connected account read access, or switch this database off.")}
+ record_odoo_stage(rt, cid, failure["stage"], "failed",
+ database_key=key, failure=failure)
+ record_odoo_stage(rt, cid, "freshness", "failed", database_key=key,
+ failure={**failure, "stage": "freshness"})
+ finish(cause=f"{failure['code']}: {failure['message']}")
finally:
stopped.set()
if heartbeat_thread:
@@ -1644,17 +2217,20 @@ def _odoo_field_key(rt, database_key, model, raw_name, connection_id=...):
def odoo_status(connection_id: str = None, session: Session = Depends(require_session)):
_odoo_admin(session)
import odoo_relational
+ resolved_connection = _odoo_connection_slot(session.runtime, connection_id)
cfg = odoo_relational.read_config(session.runtime)
stored = session.runtime.get(_ODOO_SYNC_STATUS) or {}
errors = stored.get("errors") if isinstance(stored.get("errors"), list) else []
phase = str(stored.get("phase") or "unknown")
- lifecycle = connector_sync.read(session.runtime, "odoo", connection_id)
+ lifecycle = connector_sync.read(session.runtime, "odoo", resolved_connection)
durable = connector_sync.job_summary(_connection_jobs(session.runtime, "odoo", connection_id))
+ receipt = odoo_health_receipt(session.runtime, resolved_connection)
if durable:
return {"schedule": cfg.get("syncEvery"),
"health": "paused" if cfg.get("frozen") else durable["state"],
"lastRun": durable["at"], "frozen": cfg.get("frozen") is True,
- **durable, "stateAt": durable["at"], "connectionId": connection_id or "default"}
+ **durable, "stateAt": durable["at"], "connectionId": resolved_connection,
+ "receipt": receipt}
# The old keys stay the old default-connection answer. A caller that names a connection gets
# that slot's lifecycle instead of the tenant-wide legacy document that caused the C6 defect.
named = connection_id is not None
@@ -1678,7 +2254,8 @@ def odoo_status(connection_id: str = None, session: Session = Depends(require_se
"retryable": state in ("error", "partial"),
"errorsTotal": int(errors_total or 0),
"connectionId": (lifecycle or {}).get("connectionId")
- or str(connection_id or "").strip() or "default"}
+ or resolved_connection,
+ "receipt": receipt}
@router.get("/connectors/{connector_key}/sync")
@@ -1694,13 +2271,25 @@ def connector_sync_status(connector_key: str, connection_id: str = None,
@router.post("/connectors/{connector_key}/sync/now")
-def connector_sync_now(connector_key: str, session: Session = Depends(require_session)):
+def connector_sync_now(connector_key: str, connection_id: str = None,
+ session: Session = Depends(require_session)):
_sync_admin(session)
if connector_key not in connector_sync.providers():
raise err(404, "sync_provider_unknown", "This connector is not registered for sync.")
try:
- return connector_sync.request_sync(connector_key, session.runtime.key,
- requested_by=session.uname)
+ requested_connection = (_odoo_connection_slot(session.runtime, connection_id)
+ if connector_key == "odoo" else
+ str(connection_id or "default").strip() or "default")
+ key = (connector_key, session.runtime.key)
+ with _SYNC_REQUEST_CONNECTION_LOCK:
+ # Bind under one API-side lock so a duplicate request cannot race ahead of the
+ # accepted request and attach its different connection id to the queued intent.
+ receipt = connector_sync.request_sync(connector_key, session.runtime.key,
+ requested_by=session.uname)
+ if receipt.get("accepted") or key not in _SYNC_REQUEST_CONNECTIONS:
+ _SYNC_REQUEST_CONNECTIONS[key] = requested_connection
+ receipt["connectionId"] = _SYNC_REQUEST_CONNECTIONS[key]
+ return receipt
except ValueError:
raise err(404, "sync_provider_unknown", "This connector is not registered for sync.") from None
@@ -1710,7 +2299,11 @@ def connector_sync_now_status(connector_key: str, session: Session = Depends(req
_sync_admin(session)
if connector_key not in connector_sync.providers():
raise err(404, "sync_provider_unknown", "This connector is not registered for sync.")
- return connector_sync.sync_request(connector_key, session.runtime.key) or {
+ receipt = connector_sync.sync_request(connector_key, session.runtime.key)
+ if receipt is not None:
+ receipt["connectionId"] = sync_request_connection(connector_key, session.runtime.key)
+ return receipt
+ return {
"provider": connector_key, "tenant": session.runtime.key, "state": "idle",
"requestedAt": None, "startedAt": None, "finishedAt": None,
"requestedBy": "", "detail": "", "accepted": False,
@@ -1760,8 +2353,11 @@ def connector_database_jobs(connector_key: str, session: Session = Depends(requi
_sync_admin(session)
if connector_key not in connector_sync.providers():
raise err(404, "sync_provider_unknown", "This connector has no database sync runner.")
- return _safe_connector_status(connector_key, {"provider": connector_key, "jobs": connector_sync.jobs(
- session.runtime, connector_key, connection_id)})
+ payload = {"provider": connector_key, "jobs": connector_sync.jobs(
+ session.runtime, connector_key, connection_id)}
+ if connector_key == "odoo":
+ payload["receipt"] = odoo_health_receipt(session.runtime, connection_id)
+ return _safe_connector_status(connector_key, payload)
@router.get("/connectors/odoo/schema")
@@ -2241,6 +2837,114 @@ def _safe_connector_status(connector_key, payload):
return payload
+def _sync_stamp(value):
+ """Return a parseable public timestamp, never a creation-time freshness substitute."""
+ text = str(value or "").strip()
+ if not text:
+ return None
+ try:
+ parsed = datetime.fromisoformat(text.replace("Z", "+00:00"))
+ if parsed.tzinfo is None:
+ parsed = parsed.replace(tzinfo=timezone.utc)
+ return parsed.astimezone(timezone.utc).isoformat()
+ except (TypeError, ValueError, OverflowError):
+ return None
+
+
+def _database_sync_health(*, enabled=True, state="never_run", last_success=None,
+ source_timestamp=None, materialized_timestamp=None, row_count=None,
+ error=None, now=None):
+ """Normalize one database's durable facts into S63-T34's shared public vocabulary."""
+ clean_state = str(state or "never_run")
+ if clean_state not in DATABASE_SYNC_STATES:
+ clean_state = "failed"
+ error = {"code": "sync_state_unreadable",
+ "message": "This database's saved sync state could not be read."}
+ if not enabled:
+ clean_state = "disabled"
+ count = row_count if type(row_count) is int and row_count >= 0 else None
+ success = _sync_stamp(last_success)
+ source = _sync_stamp(source_timestamp)
+ materialized = _sync_stamp(materialized_timestamp)
+ safe_error = None
+ if error is not None:
+ raw = error if isinstance(error, dict) else {}
+ requested_code = str(raw.get("code") or "")
+ code = requested_code if requested_code in _DATABASE_SAFE_ERRORS else "connector_sync_failed"
+ safe_error = {
+ "code": code,
+ # Raw provider and exception text never crosses this boundary.
+ "message": _DATABASE_SAFE_ERRORS.get(code, "This database could not be synced."),
+ }
+ # A current failure may retain the prior success timestamp, but it can never look current.
+ lag = _lag_seconds(success, now) if success else None
+ return {"state": clean_state, "lastSuccess": success,
+ "sourceTimestamp": source, "materializedTimestamp": materialized,
+ "rowCount": count, "lagSeconds": lag, "error": safe_error}
+
+
+def _odoo_database_sync_health(raw, enabled=True, *, now=None):
+ row = raw if isinstance(raw, dict) else {}
+ stages = row.get("stages") if isinstance(row.get("stages"), dict) else {}
+ failure = row.get("failure") if isinstance(row.get("failure"), dict) else None
+ raw_state = str(row.get("state") or "unknown")
+ stage_states = {str((value or {}).get("state") or "unknown")
+ for value in stages.values() if isinstance(value, dict)}
+ last_success = row.get("lastSuccess")
+ if failure or "failed" in stage_states or raw_state == "failed":
+ state = "partial" if last_success else "failed"
+ elif raw_state == "running" or "running" in stage_states:
+ state = "running"
+ elif raw_state == "queued" or "queued" in stage_states:
+ state = "queued"
+ elif last_success or raw_state == "ready":
+ state = "succeeded"
+ else:
+ state = "never_run"
+ source_stage = stages.get("source") if isinstance(stages.get("source"), dict) else {}
+ material_stage = (stages.get("materialization")
+ if isinstance(stages.get("materialization"), dict) else {})
+ return _database_sync_health(
+ enabled=enabled, state=state, last_success=last_success,
+ source_timestamp=source_stage.get("lastSuccess"),
+ materialized_timestamp=material_stage.get("lastSuccess"),
+ row_count=row.get("rowCount"), error=failure, now=now)
+
+
+def _amazon_database_sync_health(raw, enabled=True, *, now=None):
+ if not enabled:
+ return _database_sync_health(enabled=False)
+ if not isinstance(raw, dict):
+ return _database_sync_health(
+ state="failed", error={"code": "sync_status_unavailable",
+ "message": "Sync status is unavailable for this database."})
+ success = raw.get("lastSuccess")
+ if raw.get("stuck") is True:
+ state = "partial" if success else "failed"
+ error = {"code": "sync_stalled",
+ "message": "This database sync is stalled."}
+ elif raw.get("inProgress") is True:
+ state, error = "running", None
+ elif success:
+ state, error = "succeeded", None
+ elif raw.get("deferred"):
+ state, error = "queued", None
+ else:
+ state, error = "never_run", None
+ return _database_sync_health(
+ state=state, last_success=success, materialized_timestamp=success,
+ row_count=raw.get("rowsLoaded"), error=error, now=now)
+
+
+def _stored_database_sync_health(raw, enabled=True, *, now=None):
+ row = raw if isinstance(raw, dict) else {}
+ return _database_sync_health(
+ enabled=enabled, state=row.get("state") or "never_run",
+ last_success=row.get("lastSuccess"), source_timestamp=row.get("sourceTimestamp"),
+ materialized_timestamp=row.get("materializedTimestamp"), row_count=row.get("rowCount"),
+ error=row.get("error"), now=now)
+
+
def _detail_connect(entry_type, is_admin):
"""C4's `keychain.connect` half: what the Connect form must ask for, and what it may store.
@@ -2624,6 +3328,173 @@ def _named_databases(rt, is_admin, uname, rows):
return named
+def _clean_scraper_recipe(raw):
+ """Return T11's persisted recipe shape and explicit structural validation errors.
+
+ This function is intentionally pure. Preview and creation both call it, so opening a saved
+ recipe cannot reinterpret it differently, and validation cannot accidentally become a paid
+ provider probe.
+ """
+ value = raw if isinstance(raw, dict) else {}
+ kind = str(value.get("kind") or "").strip().lower()
+ errors = []
+ if kind not in SCRAPER_RECIPE_KINDS:
+ errors.append("Choose Instagram, TikTok, or Google Maps.")
+ return {}, errors
+
+ limits = value.get("limits") if isinstance(value.get("limits"), dict) else {}
+ schedule = value.get("schedule") if isinstance(value.get("schedule"), dict) else {}
+ schedule_mode = str(schedule.get("mode") or "manual").strip().lower()
+ if schedule_mode != "manual":
+ errors.append("This recipe must use the manual Run now schedule.")
+
+ if kind == "google_maps":
+ seed = value.get("seed") if isinstance(value.get("seed"), dict) else {}
+ search = str(seed.get("search") or value.get("search") or "").strip()
+ geography = str(seed.get("geography") or value.get("geography") or "").strip()
+ raw_cap = limits.get("records", value.get("limit"))
+ if not search:
+ errors.append("Enter a category or search term.")
+ if not geography:
+ errors.append("Enter a geography.")
+ if raw_cap in (None, ""):
+ errors.append("Enter a result cap.")
+ cap = 0
+ else:
+ try:
+ cap = int(raw_cap)
+ except (TypeError, ValueError):
+ cap = 0
+ errors.append("The result cap must be a whole number.")
+ if raw_cap not in (None, "") and (cap < 1 or cap > 500):
+ errors.append("The result cap must be between 1 and 500.")
+ return ({"kind": kind, "source": "places",
+ "seed": {"search": search, "geography": geography},
+ "limits": {"records": cap}, "schedule": {"mode": "manual"}}, errors)
+
+ seed = str(value.get("seed") or "").strip()
+ raw_limit = limits.get("posts", value.get("limit"))
+ post_type = str(limits.get("postType") or value.get("postType") or "video").strip().lower()
+ if not seed:
+ errors.append(f"Enter an {kind.title()} profile seed.")
+ if raw_limit in (None, ""):
+ errors.append("Enter a last-N post limit.")
+ limit = 0
+ else:
+ try:
+ limit = int(raw_limit)
+ except (TypeError, ValueError):
+ limit = 0
+ errors.append("The post limit must be a whole number.")
+ if raw_limit not in (None, "") and (limit < 1 or limit > SCRAPER_RECIPE_MAX_ROWS):
+ errors.append(f"The post limit must be between 1 and {SCRAPER_RECIPE_MAX_ROWS}.")
+ allowed_types = ("video", "image", "carousel") if kind == "instagram" else ("video",)
+ if post_type not in allowed_types:
+ errors.append("Choose video, image, or carousel posts."
+ if kind == "instagram" else
+ "TikTok's native profile window supports video posts only.")
+ return ({"kind": kind, "source": "profile", "seed": seed,
+ "limits": {"posts": limit, "postType": post_type},
+ "schedule": {"mode": "manual"}}, errors)
+
+
+def _scraper_recipe_fields(kind):
+ """Stable user-table schema for one recipe, derived from the shipped normalisers."""
+ if kind == "google_maps":
+ return [dict(field) for field in _MAPS_RECIPE_FIELDS]
+ import automation_engine as engine # noqa: PLC0415 - existing schema owner
+ source = engine.POST_FIELDS if kind == "instagram" else engine.TT_POST_FIELDS
+ # Links and rollups target the shared automation tables. A recipe database is independent,
+ # so copying those references would create a plausible but broken relationship graph.
+ return [dict(field) for field in source
+ if isinstance(field, dict) and field.get("type") not in ("link", "rollup")]
+
+
+def _scraper_plan(recipe):
+ """Network-free setup preview: declared provider, modelled spend, cap, and credential state."""
+ kind = str((recipe or {}).get("kind") or "")
+ limit_key = "records" if kind == "google_maps" else "posts"
+ limit = int(((recipe or {}).get("limits") or {}).get(limit_key) or 0)
+ if kind == "google_maps":
+ import providers # noqa: PLC0415 - registry owns cost/capability
+ apify = providers.PROVIDERS["apify"]
+ capability = apify.cap("google_maps_stores")
+ per_record = float(capability.cost_per_record or 0.0) if capability else 0.0
+ amount = round(per_record * limit, 4)
+ return {"provider": apify.label, "credentialReady": apify.configured(),
+ "estimatedUsd": amount, "costCapUsd": amount, "recordsCap": limit,
+ "canRun": True,
+ "note": "Modelled from the declared per-place rate; the provider receipt "
+ "reconciles the bounded result count."}
+
+ import providers # noqa: PLC0415 - registry is the authority
+ bright = providers.PROVIDERS["brightdata"]
+ apify = providers.PROVIDERS["apify"]
+ if kind == "tiktok":
+ per_record = float(bright.cap("tt_post_metrics").cost_per_record or 0.0)
+ profile_cost = float(bright.cap("tt_profile").cost_per_record or 0.0)
+ estimate_usd = profile_cost + (per_record * limit)
+ provider_label = bright.label
+ credential_ready = bright.configured()
+ else:
+ # The established Instagram runner can use Bright Data and its Apify fallback. The cap
+ # includes both post metrics and view enrichment, so it never hides the worst declared
+ # cost from the person authorising Run now.
+ bd_cost = float(bright.cap("ig_post_metrics").cost_per_record or 0.0)
+ profile_cost = float(bright.cap("ig_profile").cost_per_record or 0.0)
+ view_cost = float(apify.cap("ig_post_views").cost_per_record or 0.0)
+ estimate_usd = profile_cost + ((bd_cost + view_cost) * limit)
+ provider_label = (bright.label if bright.configured() or not apify.configured()
+ else apify.label)
+ credential_ready = bright.configured() or apify.configured()
+ amount = round(estimate_usd, 4)
+ return {"provider": provider_label, "credentialReady": bool(credential_ready),
+ "estimatedUsd": amount, "costCapUsd": amount, "recordsCap": limit,
+ "canRun": True, "note": "Modelled from declared per-record list rates."}
+
+
+def _scraper_databases(rt, is_admin, uname):
+ """Saved recipe databases this reader may open, with no row payloads or secrets."""
+ try:
+ import core.user_tables as user_tables
+ definitions = user_tables.all_defs(rt)
+ lent = user_tables.lend(rt, **{user_tables.STORE_KEY: definitions})
+ except Exception: # noqa: BLE001
+ return []
+ rows = []
+ for key, definition in (definitions or {}).items():
+ recipe = definition.get("scraperRecipe") if isinstance(definition, dict) else None
+ if not isinstance(recipe, dict):
+ continue
+ try:
+ if not user_tables.may_open(key, uname, bool(is_admin), st=lent):
+ continue
+ except Exception: # noqa: BLE001
+ continue
+ cleaned, errors = _clean_scraper_recipe(recipe)
+ if errors:
+ continue
+ plan = _scraper_plan(cleaned)
+ # Recipe existence alone is not enablement. The creation door stamps both ownership
+ # markers; a legacy/malformed row without them is disclosed as disabled and cannot run.
+ enabled = (definition.get("connectorKey") == "scraper"
+ and definition.get("recordMode") == "automation")
+ rows.append({"key": str(key), "label": str(definition.get("label") or key),
+ "defaultLabel": str(definition.get("label") or key),
+ "native": cleaned["kind"], "enabled": enabled, "exists": True,
+ "canRename": True, "sourceRows": None, "populated": None,
+ "fields": [dict(field) for field in (definition.get("fields") or ())
+ if isinstance(field, dict)],
+ "excluded": [], "sharesFieldsWith": [], "scraperRecipe": cleaned,
+ "syncHealth": _stored_database_sync_health(
+ definition.get(user_tables.CONNECTOR_HEALTH_FIELD), enabled),
+ "scraperRuns": [dict(item) for item in (definition.get("scraperRuns") or ())
+ if isinstance(item, dict)],
+ "runPath": f"/connectors/scraper/databases/{key}/run",
+ "canRun": enabled and plan["canRun"] is True})
+ return _named_databases(rt, is_admin, uname, rows)
+
+
def _detail_databases(rt, is_admin, connector_key, uname="", connection_id=None):
"""C4's `databases` half: what this connector materialises, and the fields behind each one.
@@ -2652,6 +3523,8 @@ def _detail_databases(rt, is_admin, connector_key, uname="", connection_id=None)
reason: what a database is CALLED is a fact about the reader's own workspace. Only the WRITE
is walled, per row, by `canRename`.
"""
+ if str(connector_key) == "scraper":
+ return _scraper_databases(rt, is_admin, uname)
if not connector_directory.is_introspectable(connector_key):
return []
spec = _connector_spec_tables(connector_key)
@@ -2668,6 +3541,9 @@ def _detail_databases(rt, is_admin, connector_key, uname="", connection_id=None)
census = odoo_relational.population_census(rt, namespace)
except Exception: # noqa: BLE001
return []
+ health = odoo_health_receipt(rt, connection_id)
+ health_by_key = {str(item.get("databaseKey") or ""): item
+ for item in (health.get("databases") or ())}
census_at = str(census.get("capturedAt") or "")
counts = census.get("counts") if isinstance(census.get("counts"), dict) else {}
catalog = {}
@@ -2712,6 +3588,12 @@ def _detail_databases(rt, is_admin, connector_key, uname="", connection_id=None)
# Counts are model-level facts. Databases backed by the same model deliberately show the
# same sourceRows number; inventing a per-database domain here would make the number lie.
source_rows = counts.get(model) if census_at and model in counts else None
+ database_health = health_by_key.get(key) or {
+ # Source rows is a model census. It is not this governed database's materialized
+ # row count, especially when two curated databases share one source model.
+ "databaseKey": key, "state": "unknown", "rowCount": None,
+ "lastSuccess": None, "currentLagSeconds": None, "recordRefreshed": None,
+ "failure": None, "stages": _empty_odoo_stages()}
out.append({"key": key,
"identity": connector_sync.database_identity(
"odoo", resolved_connection_id, key, model),
@@ -2721,6 +3603,12 @@ def _detail_databases(rt, is_admin, connector_key, uname="", connection_id=None)
"enabled": bool(choice.get("enabled")),
"sourceRows": source_rows,
"populated": None if source_rows is None else source_rows > 0,
+ # Data-free client receipt: source/mirror/materialization/reconciliation/
+ # freshness are distinct, and one database's failure cannot make a sibling
+ # look newly successful.
+ "sync": database_health,
+ "syncHealth": _odoo_database_sync_health(
+ database_health, bool(choice.get("enabled"))),
"fields": scoped_fields,
"excluded": [dict(item) for item in excluded],
"sharesFieldsWith": []})
@@ -2811,6 +3699,8 @@ def _amazon_databases(rt, is_admin, spec=None):
# what `native` carries: the vendor's own name for the thing, never one we invented. It is
# empty for a table whose report type is unknown, and the tab says so rather than guessing.
declared = (spec or {}).get(key) or {}
+ raw_sync = (states or {}).get(name) if is_admin else None
+ is_enabled = enabled.get(key) is True
out.append({"key": key,
"label": str(declared.get("label") or LABELS_AMAZON.get(key) or key),
"defaultLabel": str(declared.get("label")
@@ -2819,8 +3709,9 @@ def _amazon_databases(rt, is_admin, spec=None):
# W55-T19 / R7 — AMAZON'S OWN tenant-local answer. This is deliberately not
# read from Odoo's grid config, and disabling it leaves every stored row and
# definition untouched: only this provider config document changes.
- "enabled": enabled.get(key) is True,
- **({"sync": (states or {}).get(name)} if is_admin else {}),
+ "enabled": is_enabled,
+ **({"sync": raw_sync} if is_admin else {}),
+ "syncHealth": _amazon_database_sync_health(raw_sync, is_enabled),
"fields": fields,
"excluded": excluded,
"sharesFieldsWith": [k for k in by_report.get(rep, ()) if k != key]})
@@ -2888,6 +3779,14 @@ def _detail_introspection(rt, is_admin, row):
stops "administrator only" being rendered as "this database has no fields".
"""
key = str(row.get("key") or "")
+ if key == "scraper":
+ # Recipe databases publish their saved, stable schema to every reader who may open the
+ # database. They have no generic schema-toggle route, so no write capability is implied.
+ return {"introspectable": True, "reason": "", "fieldsVisible": True,
+ "canToggleFields": False, "canToggleTables": False,
+ "catalogueComplete": True, "capturedAt": "", "excludedBasis": "",
+ "censusAt": "", "censusComplete": False, "censusDurationMs": 0,
+ "censusModelsCounted": 0, "censusModelsWithRows": 0}
if not connector_directory.is_introspectable(key):
reason = ("Nothing is connected behind this listing yet, so there is no schema to read."
if str(row.get("state") or "") == connector_directory.CATALOGUE_STATE
@@ -3008,6 +3907,229 @@ def _detail_enablement(rt, is_admin, row):
"updatePath": "", "preservesData": True, "available": False}
+def _scraper_preview(body):
+ request = body if isinstance(body, dict) else {}
+ raw_recipe = request.get("recipe") if isinstance(request.get("recipe"), dict) else request
+ recipe, errors = _clean_scraper_recipe(raw_recipe)
+ if errors:
+ raise err(400, "invalid_scraper_recipe", " ".join(errors))
+ plan = _scraper_plan(recipe)
+ missing = [] if plan["credentialReady"] else ["credential"]
+ return {"recipe": recipe, "inputs": recipe["seed"], "provider": plan["provider"],
+ "credentialReady": plan["credentialReady"], "estimatedUsd": plan["estimatedUsd"],
+ "costCapUsd": plan["costCapUsd"], "recordsCap": plan["recordsCap"],
+ "canRun": plan["canRun"], "missing": missing, "note": plan["note"],
+ # This is a contract field, not a measurement. A preview and a create never call a
+ # provider, so both truthfully report zero incurred spend.
+ "incurredUsd": 0.0}
+
+
+def _create_scraper_database(rt, uname, body):
+ """Persist one recipe database. Contains no provider import or runner call by construction."""
+ if not rt.available():
+ raise err(503, "store_unavailable", "the tenant store is unavailable")
+ request = body if isinstance(body, dict) else {}
+ label = str(request.get("label") or "").strip()
+ if not label:
+ raise err(400, "missing_database_name", "Enter a database name.")
+ preview = _scraper_preview(request)
+ recipe = preview["recipe"]
+ import core.user_tables as user_tables # noqa: PLC0415 - existing create door
+ key = user_tables.create(label, uname, fields=_scraper_recipe_fields(recipe["kind"]),
+ source=user_tables.AUTOMATION_SOURCE, st=rt)
+ if not key:
+ raise err(409, "database_not_created",
+ "The database name is unavailable or this workspace reached its database limit.")
+
+ def _attach(cur):
+ current = dict(cur or {})
+ definition = dict(current.get(key) or {})
+ if not definition:
+ raise RuntimeError("the new database disappeared before its recipe was attached")
+ definition.update({"connectorKey": "scraper", "recordMode": "automation",
+ "scraperRecipe": recipe, "scraperRuns": [],
+ user_tables.CONNECTOR_HEALTH_FIELD:
+ user_tables.connector_health_receipt("never_run")})
+ current[key] = definition
+ return current
+
+ try:
+ rt.update(user_tables.STORE_KEY, _attach, flush="sync")
+ except Exception: # noqa: BLE001
+ user_tables.delete(key, st=rt)
+ raise err(503, "recipe_not_saved",
+ "The recipe could not be saved, so the new database was removed.")
+ return {"key": key, "label": label, "recipe": recipe, "preview": preview,
+ "createdWithoutProviderCall": True, "incurredUsd": 0.0}
+
+
+def _scraper_database_run(rt, uname, is_admin, database_key):
+ """The one explicit paid-work boundary for any saved scraper recipe."""
+ if not rt.available():
+ raise err(503, "store_unavailable", "the tenant store is unavailable")
+ import core.user_tables as user_tables
+ key = str(database_key or "").strip()
+ definitions = user_tables.lend_defs(rt)
+ definition = user_tables.get(key, st=definitions) or {}
+ if not definition:
+ raise err(404, "unknown_scraper_database", "This saved scraper database was not found.")
+ raw_recipe = definition.get("scraperRecipe")
+ recipe, errors = _clean_scraper_recipe(raw_recipe)
+ if errors:
+ raw_kind = str((raw_recipe or {}).get("kind") or "") if isinstance(raw_recipe, dict) else ""
+ raw_limits = ((raw_recipe or {}).get("limits") or {}) if isinstance(raw_recipe, dict) else {}
+ raw_cap = raw_limits.get("records") if isinstance(raw_limits, dict) else None
+ try:
+ bounded = 1 <= int(raw_cap) <= 500
+ except (TypeError, ValueError):
+ bounded = False
+ if raw_kind == "google_maps" and not bounded:
+ raise err(400, "maps_unbounded_request",
+ "Google Maps discovery requires an explicit result cap between 1 and 500.")
+ raise err(409, "invalid_saved_scraper_recipe",
+ "This saved scraper recipe is no longer valid. Review its settings before running it.")
+ # Run now both spends against a provider account and mutates this database. A viewer grant
+ # may expose the rows without granting either power, so the ordinary open wall is too broad.
+ # Reuse the database editor authority that every other mutation surface already honours.
+ if not user_tables.may_edit_database(key, uname, bool(is_admin), st=definitions):
+ raise err(403, "database_forbidden", "You cannot run this database recipe.")
+ plan = _scraper_plan(recipe)
+ previous_health = definition.get(user_tables.CONNECTOR_HEALTH_FIELD)
+ previous_health = previous_health if isinstance(previous_health, dict) else {}
+
+ def record_attempt(state, error=None):
+ try:
+ return user_tables.record_connector_health(
+ key, state, st=rt,
+ last_success=previous_health.get("lastSuccess"),
+ source_timestamp=previous_health.get("sourceTimestamp"),
+ materialized_timestamp=previous_health.get("materializedTimestamp"),
+ row_count=previous_health.get("rowCount"), error=error)
+ except Exception: # noqa: BLE001
+ raise err(503, "sync_status_not_saved",
+ "The run did not start because its durable status could not be saved.") from None
+
+ if not plan["credentialReady"]:
+ code = "maps_credential_missing" if recipe["kind"] == "google_maps" else \
+ "scraper_credential_missing"
+ message = "Add the scraper credential in Keychain before choosing Run now."
+ record_attempt("failed", {"code": code, "message": message})
+ raise err(409, code, "Add the scraper credential in Keychain before choosing Run now.")
+
+ # Imports and provider calls occur only below the explicit Run-now endpoint. Preview, create,
+ # detail, and page load never reach either runner.
+ started_at = datetime.now(timezone.utc).isoformat()
+ record_attempt("running")
+ key_field = "shortcode"
+ if recipe["kind"] == "google_maps":
+ import providers
+ try:
+ result = providers.discover_google_maps_stores(
+ recipe["seed"]["search"], recipe["seed"]["geography"],
+ recipe["limits"]["records"], captured_at=started_at)
+ except providers.MapsProviderRefusal as exc:
+ record_attempt("failed", {"code": exc.code, "message": exc.public_message})
+ status = 409 if exc.code == "maps_credential_missing" else \
+ (400 if exc.code in ("maps_unbounded_request", "maps_invalid_search") else 502)
+ raise err(status, exc.code, exc.public_message)
+ incoming_key = "rows"
+ key_field = "place_id"
+ else:
+ limit = recipe["limits"]["posts"]
+ post_type = recipe["limits"]["postType"]
+ try:
+ if recipe["kind"] == "instagram":
+ import connectors_ig as runner
+ run_recipe = runner.pull_ig_recipe
+ else:
+ import connectors_tt as runner
+ run_recipe = runner.pull_tt_recipe
+ result = run_recipe(recipe["seed"], limit, post_type=post_type,
+ log=lambda *_args: None)
+ except Exception: # noqa: BLE001
+ record_attempt("failed", {
+ "code": "scraper_run_failed",
+ "message": "The provider could not complete this run. No result was written.",
+ })
+ raise err(502, "scraper_run_failed",
+ "The provider could not complete this run. No result was written.")
+ incoming_key = "posts"
+ result = result if isinstance(result, dict) else {}
+ incoming = [dict(row) for row in (result.get(incoming_key) or ()) if isinstance(row, dict)]
+ state = str(result.get("state") or "error")
+ if state not in ("ok", "partial"):
+ incoming = []
+ finished_at = datetime.now(timezone.utc).isoformat()
+ source_stamps = [_sync_stamp(row.get("source_timestamp")) for row in incoming]
+ source_timestamp = max((stamp for stamp in source_stamps if stamp), default=None)
+ provider_counts = dict(result.get("counts") or {})
+ provider_counts.setdefault("returned", len(incoming))
+ provider_cost = dict(result.get("cost") or {})
+ provider_cost.setdefault("currency", "USD")
+ provider_cost.setdefault("estimatedUsd", plan["estimatedUsd"])
+ provider_cost.setdefault("costCapUsd", plan["costCapUsd"])
+ receipt = {"id": finished_at, "state": state,
+ "provider": str(result.get("via") or plan["provider"]),
+ "rows": len(incoming), "startedAt": started_at, "finishedAt": finished_at,
+ "estimatedUsd": plan["estimatedUsd"], "costCapUsd": plan["costCapUsd"],
+ "cost": provider_cost, "providerCounts": provider_counts,
+ "attempts": [dict(item) for item in (result.get("attempts") or ())
+ if isinstance(item, dict)],
+ # Provider notes can echo a submitted handle. Run history stores operational
+ # metadata only; the saved recipe already owns the input and no customer value is
+ # duplicated into an audit-like receipt.
+ "note": f"Provider returned {state}."}
+ counts = {"inserted": 0, "updated": 0, "unchanged": 0, "skipped": 0,
+ "duplicates": 0, "orphans": 0, "capped": 0}
+
+ def _write(cur):
+ nonlocal counts
+ current = dict(cur or {})
+ stored = dict(current.get(key) or {})
+ if not stored:
+ raise RuntimeError("the database disappeared before the run could be written")
+ if incoming:
+ import automation_engine as engine
+ rows, counts = engine.upsert_rows(stored.get("rows") or {}, incoming, key_field)
+ stored["rows"] = rows
+ processed = sum(int(counts.get(name) or 0)
+ for name in ("inserted", "updated", "unchanged", "skipped", "capped"))
+ receipt["writeCounts"] = dict(counts)
+ receipt["countReconciled"] = (
+ int(provider_counts.get("returned") or 0) == len(incoming) == processed)
+ history = [dict(item) for item in (stored.get("scraperRuns") or ())
+ if isinstance(item, dict)]
+ stored["scraperRuns"] = [receipt, *history][:SCRAPER_RUN_HISTORY_LIMIT]
+ final_state = "succeeded" if state == "ok" else "partial" if state == "partial" else "failed"
+ final_error = None if final_state != "failed" else {
+ "code": "provider_result_failed",
+ "message": "The provider returned no usable result for this database.",
+ }
+ stored[user_tables.CONNECTOR_HEALTH_FIELD] = user_tables.connector_health_receipt(
+ final_state,
+ last_success=finished_at if final_state in ("succeeded", "partial")
+ else previous_health.get("lastSuccess"),
+ source_timestamp=source_timestamp,
+ materialized_timestamp=finished_at if final_state in ("succeeded", "partial") else None,
+ row_count=len(stored.get("rows") or {}), error=final_error)
+ current[key] = stored
+ return current
+
+ try:
+ rt.update(user_tables.STORE_KEY, _write, flush="sync")
+ except Exception: # noqa: BLE001
+ try:
+ record_attempt("failed", {
+ "code": "scraper_result_not_saved",
+ "message": "The provider answered, but the result could not be saved.",
+ })
+ except Exception: # noqa: BLE001
+ pass
+ raise err(503, "scraper_result_not_saved",
+ "The provider answered, but the result could not be saved. Reload before retrying.")
+ return {"key": key, "state": state, "counts": counts, "receipt": receipt}
+
+
def connector_detail(rt, user, uname, is_admin, connector_key):
"""The whole C4 detail DTO for one connector, or `None` when this reader has no such card.
@@ -3106,6 +4228,26 @@ def amazon_database_enablement(database_key: str, body: dict = Body(default=None
return _set_amazon_database_enablement(database_key, body, session)
+@router.post("/connectors/scraper/databases/preview")
+def scraper_database_preview(body: dict = Body(default=None),
+ session: Session = Depends(require_session)):
+ """Validate and price a recipe locally. This endpoint makes zero provider calls."""
+ return _scraper_preview(body)
+
+
+@router.post("/connectors/scraper/databases")
+def scraper_database_create(body: dict = Body(default=None),
+ session: Session = Depends(require_session)):
+ """Create a saved recipe database for $0; paid work begins only at its Run-now door."""
+ return _create_scraper_database(session.runtime, session.uname, body)
+
+
+@router.post("/connectors/scraper/databases/{database_key}/run")
+def scraper_database_run(database_key: str, session: Session = Depends(require_session)):
+ """Explicitly execute one bounded saved scraper recipe through its provider seam."""
+ return _scraper_database_run(session.runtime, session.uname, bool(session.admin), database_key)
+
+
# ⚠ DECLARED AFTER THE LITERAL `/connectors/odoo/*` ROUTES. FastAPI matches in declaration order,
# and while none of those ends in `/detail` — so no shadowing is possible today — the ordering is
# the habit that keeps it impossible tomorrow. `/connectors/directory` is two segments and cannot
diff --git a/api/routes_customers.py b/api/routes_customers.py
index 0fca2b2ed7e4cd951fecfa34d28437157f9e2dd6..192607e719b0a7bfff118e71b400d5c9a1a2836d 100644
--- a/api/routes_customers.py
+++ b/api/routes_customers.py
@@ -22,8 +22,10 @@ owner-blocked on B-3), and no test here may claim read-your-writes ACROSS runtim
proof is single-process and would report green on exactly the thing that is still broken.
"""
import datetime as _dt
+from copy import deepcopy
import hashlib
import json
+import threading
import time
from fastapi import APIRouter, Body, Depends
@@ -39,6 +41,14 @@ MODULE = "customer_data"
_CACHE_TTL = 900 # the pool build is slow (Odoo + reconciliation); 15 min, as before
+# A slow producer must not indefinitely hold another threadpool request; timeout recomputes.
+_GRID_ASSEMBLY_WAIT_SECONDS = 10.0
+_grid_assembly_lock = threading.Lock()
+_grid_assembly_inflight = {}
+_grid_assembly_enabled = True
+grid_assembly_computed = 0
+grid_assembly_shared = 0
+
def _pool_rows(session: Session):
"""The reconciled Odoo pool for this session's SCOPE — the slow part, and the only part that
@@ -304,6 +314,65 @@ def _mark_offerability(fields):
def grid_assembly(session: Session, scope: str = "customer", storage_key: str = "",
consume_corrections: bool = True):
+ """Share only overlapping assemblies of the same user's grid, never completed results."""
+ global grid_assembly_shared
+ rt = session.runtime
+ team_id, agent = _team_agent(session)
+ key = (session.tenant, session.uname, session.admin, team_id, agent, scope,
+ consume_corrections, _pool_stamp(rt, team_id, agent),
+ # A changed permission record must not borrow an assembly from the old record.
+ # ⛔ `default=str` IS NOT TIDINESS. This runs on every `/customers` and `/workspace`,
+ # the product's main grid; a `TypeError` from an unserialisable value in a stored user
+ # record would 500 BOTH doors, which is a far worse failure than the duplicated work
+ # this key exists to remove. The record round-trips JSON by construction today, so the
+ # fallback should never fire -- and a key that cannot raise is the only kind worth
+ # putting in front of a hot path.
+ json.dumps(session.user, sort_keys=True, separators=(",", ":"), default=str))
+
+ def own_copy(result):
+ # The pool is already shared by scope_cache and rows_from_pool never mutates it.
+ # Copy the rest together to preserve internal aliases without lending mutable state.
+ out = deepcopy({k: v for k, v in result.items() if k != "rows_src"})
+ out["rows_src"] = result["rows_src"]
+ out["workspace"]["storageKey"] = storage_key
+ return out
+
+ if not _grid_assembly_enabled:
+ return own_copy(_compute_grid_assembly(session, scope, storage_key, consume_corrections))
+
+ with _grid_assembly_lock:
+ entry = _grid_assembly_inflight.get(key)
+ producer = entry is None
+ if producer:
+ entry = {"event": threading.Event(), "result": None, "error": None}
+ _grid_assembly_inflight[key] = entry
+
+ if not producer:
+ if entry["event"].wait(_GRID_ASSEMBLY_WAIT_SECONDS):
+ if entry["error"] is not None:
+ raise entry["error"]
+ result = own_copy(entry["result"])
+ with _grid_assembly_lock:
+ grid_assembly_shared += 1
+ return result
+ # A timeout changes neither the caller's answer nor the producer's ownership.
+ return own_copy(_compute_grid_assembly(session, scope, storage_key, consume_corrections))
+
+ try:
+ entry["result"] = _compute_grid_assembly(session, scope, storage_key, consume_corrections)
+ except BaseException as exc:
+ entry["error"] = exc
+ raise
+ finally:
+ with _grid_assembly_lock:
+ if _grid_assembly_inflight.get(key) is entry:
+ del _grid_assembly_inflight[key]
+ entry["event"].set()
+ return own_copy(entry["result"])
+
+
+def _compute_grid_assembly(session: Session, scope: str = "customer", storage_key: str = "",
+ consume_corrections: bool = True):
"""ONE assembly of this session's grid state, shared by `/customers`, `/workspace` and the
events route (2026-07-31 — the standalone measure gap, owner item 1).
@@ -330,6 +399,10 @@ def grid_assembly(session: Session, scope: str = "customer", storage_key: str =
import core.perm_scope as perm_scope
+ global grid_assembly_computed
+ with _grid_assembly_lock:
+ grid_assembly_computed += 1
+
rt = session.runtime
team_id, agent = _team_agent(session)
rows_src = _pool_for(rt, team_id, agent)
diff --git a/api/routes_external.py b/api/routes_external.py
index 7eb5fe13b6c922f46fe9de3cb2b230f6a69048df..92cd93ce4cfbffa81febbbf7ca609ed02fc2444b 100644
--- a/api/routes_external.py
+++ b/api/routes_external.py
@@ -1,21 +1,508 @@
"""Per-user credentials, governed reads and explicitly allowed bearer build tools."""
+import base64
import copy
+import hashlib
+import hmac
+import importlib
+import inspect
import json
+import time
import uuid
-from urllib.parse import quote
+from urllib.parse import parse_qs, quote, urlencode, urlsplit, urlunsplit
from fastapi import APIRouter, Body, Depends, Header, HTTPException, Request, Response
+from fastapi.responses import JSONResponse, RedirectResponse
from starlette.concurrency import run_in_threadpool
import deps
+import aios_session
from deps import Session, err, require_session
-from core import external_access, external_tools, sheets
+from core import audit, external_access, external_oauth, external_tools, sheets
+from harness import runtime
router = APIRouter(prefix="/api/v1/external", tags=["external"])
-_CREDENTIAL_PREFIX = "lpb1."
+_CREDENTIAL_PREFIXES = ("lpb1.", "lpc1.", "lpo1.", "lpr1.")
+_QUERY_DEFAULT_LIMIT = 100
+_QUERY_MAX_LIMIT = 500
+_QUERY_MAX_SORTS = 8
+_QUERY_MAX_AGGREGATES = 16
+_QUERY_MAX_CURSOR = 768
+_QUERY_MAX_FILTER_DEPTH = 3
+_QUERY_MAX_FILTER_NODES = 100
+_QUERY_MAX_FILTER_SIBLINGS = 50
+_QUERY_KEYS = frozenset({"fields", "filters", "sorts", "limit", "cursor", "aggregates"})
+_VIEW_DRAFT_CONFIG_KEYS = frozenset({
+ "filters", "filterConj", "sorts", "groupBy", "visible", "order", "display",
+ "memberPids",
+})
+_CONFIRMATION_BUCKET = "external_confirmations"
+_CONFIRMATION_TTL_SECONDS = 5 * 60
+_CONFIRMATION_MAX_BYTES = 512 * 1024
+_CONFIRMATION_PHRASE = "CONFIRM SCHEMA OR DESTRUCTIVE CHANGE"
+_CONFIRMATION_PREFIX = "cfm1"
+
+
+def _contains_credential(value) -> bool:
+ text = str(value or "")
+ return any(prefix in text for prefix in _CREDENTIAL_PREFIXES)
+
+
+def _json_digest(value) -> str:
+ try:
+ encoded = json.dumps(value, sort_keys=True, separators=(",", ":"),
+ ensure_ascii=True, allow_nan=False, default=str)
+ except (TypeError, ValueError, OverflowError) as exc:
+ raise err(400, "query_arguments_invalid",
+ "The query contains a value that cannot be read safely.") from exc
+ return hashlib.sha256(encoded.encode("utf-8")).hexdigest()
+
+
+def _canonical_json(value) -> bytes:
+ try:
+ encoded = json.dumps(value, sort_keys=True, separators=(",", ":"),
+ ensure_ascii=True, allow_nan=False)
+ except (TypeError, ValueError, OverflowError) as exc:
+ raise err(400, "tool_arguments_invalid",
+ "The tool arguments contain a value that cannot be represented safely.") from exc
+ raw = encoded.encode("utf-8")
+ if len(raw) > _CONFIRMATION_MAX_BYTES:
+ raise err(413, "confirmation_too_large",
+ "This change is too large for one preview. Split it into smaller changes.")
+ if _contains_credential(encoded):
+ raise err(400, "credential_in_arguments",
+ "Tool arguments must not contain credential values.")
+ return raw
+
+
+def _b64(raw: bytes) -> str:
+ return base64.urlsafe_b64encode(raw).rstrip(b"=").decode("ascii")
+
+
+def _unb64(value: str) -> bytes:
+ return base64.urlsafe_b64decode(value + "=" * (-len(value) % 4))
+
+
+class _PreviewRuntime:
+ """Lazy copy-on-write view of one tenant runtime; no method may reach a real writer.
+
+ Existing Sheet/View/App writers therefore remain the validators and diff producers. Reads are
+ copied once, while every update/put is applied only to this private graph. Database CRUD uses
+ the separate pure planner below because those routes deliberately emit automation callbacks.
+ """
+
+ def __init__(self, real):
+ self._real = real
+ self._initial = {}
+ self._docs = {}
+ self._base_revisions = {}
+ self._local_revisions = {}
+ self.update_calls = 0
+
+ @property
+ def tenant(self):
+ return getattr(self._real, "tenant", None)
+
+ def _load(self, name):
+ key = str(name)
+ if key not in self._docs:
+ try:
+ value = self._real.get(key, fresh=True)
+ except TypeError:
+ value = self._real.get(key)
+ value = copy.deepcopy(value)
+ self._initial[key] = copy.deepcopy(value)
+ self._docs[key] = value
+ try:
+ revision = self._real.revision(key)
+ except Exception:
+ revision = None
+ self._base_revisions[key] = copy.deepcopy(revision)
+ self._local_revisions[key] = 0
+ return self._docs[key]
+
+ def get(self, name, *args, **kwargs):
+ return copy.deepcopy(self._load(name))
+
+ def get_projection(self, name, drop=()):
+ value = self.get(name)
+ if isinstance(value, dict):
+ for key in drop or ():
+ value.pop(key, None)
+ return value
+
+ def get_many(self, names):
+ return {str(name): self.get(name) for name in names or ()}
+
+ def get_projections(self, names, drop=(), timeout_s=None):
+ return {str(name): self.get_projection(name, drop=drop) for name in names or ()}
+
+ def update(self, name, callback, flush="sync", expect_rev=None, audit_row=None):
+ key = str(name)
+ current = copy.deepcopy(self._load(key))
+ if expect_rev is not None:
+ base = self.revision(key)
+ if int((base or {}).get("rev") or 0) != int(expect_rev):
+ from core import view_audit
+ raise view_audit.ViewRevisionStale(
+ rev_current=int((base or {}).get("rev") or 0),
+ rev_sent=int(expect_rev), message="The preview revision is stale.")
+ next_value = callback(current)
+ self._docs[key] = copy.deepcopy(next_value)
+ self._local_revisions[key] += 1
+ self.update_calls += 1
+ return copy.deepcopy(next_value)
+
+ def put(self, name, value):
+ key = str(name)
+ self._load(key)
+ self._docs[key] = copy.deepcopy(value)
+ self._local_revisions[key] += 1
+ self.update_calls += 1
+ return copy.deepcopy(value)
+
+ def revision(self, name):
+ key = str(name)
+ self._load(key)
+ base = self._base_revisions.get(key)
+ rev = int((base or {}).get("rev") or 0) + self._local_revisions.get(key, 0)
+ return {"rev": rev, "token": _json_digest({"key": key, "rev": rev,
+ "base": base})}
+
+ def available(self):
+ return bool(self._real.available()) if callable(getattr(self._real, "available", None)) else True
+
+ def store_key(self, name):
+ fn = getattr(self._real, "store_key", None)
+ return fn(name) if callable(fn) else str(name)
+
+ def changed(self):
+ return {key: {"before": copy.deepcopy(self._initial.get(key)),
+ "after": copy.deepcopy(value)}
+ for key, value in self._docs.items()
+ if self._initial.get(key) != value}
+
+ def revision_binding(self):
+ return _json_digest({key: self._base_revisions.get(key)
+ for key in sorted(self._docs)})
+
+ def base_revisions(self):
+ return copy.deepcopy(self._base_revisions)
+
+ def __getattr__(self, name):
+ # Read-only runtime capabilities (tenant config, mirror readers) remain available. Store
+ # writers are all named above and never fall through this proxy.
+ if name in {"upload_bytes", "delete_path", "flush"}:
+ raise AttributeError(name)
+ return getattr(self._real, name)
+
+
+class _ConfirmationRuntime:
+ """Revision-guard every bucket the confirmed product writer may mutate."""
+
+ def __init__(self, real, expected):
+ self._real = real
+ self._expected = copy.deepcopy(expected if isinstance(expected, dict) else {})
+
+ @property
+ def tenant(self):
+ return getattr(self._real, "tenant", None)
+
+ def update(self, name, callback, flush="sync", expect_rev=None, audit_row=None):
+ from core import view_audit
+ key = str(name)
+ revision = self._expected.get(key)
+ rev = revision.get("rev") if isinstance(revision, dict) else None
+ if type(rev) is not int:
+ raise view_audit.RevisionUnverifiable(
+ "The confirmed writer addressed a bucket outside its preview.")
+ if expect_rev is not None and int(expect_rev) != rev:
+ raise view_audit.ViewRevisionStale(
+ rev_current=rev, rev_sent=expect_rev,
+ message="The writer revision no longer matches its preview.")
+ answer = self._real.update(
+ key, callback, flush=flush, expect_rev=rev, audit_row=audit_row)
+ try:
+ self._expected[key] = copy.deepcopy(self._real.revision(key))
+ except Exception:
+ self._expected[key] = {"rev": rev + 1}
+ return answer
+
+ def put(self, name, value):
+ from core import view_audit
+ raise view_audit.RevisionUnverifiable(
+ "An unguarded document replacement cannot execute a confirmed change.")
+
+ def __getattr__(self, name):
+ # Object-capability dispatchers receive this runtime too. They may read optional
+ # capability data, but no byte-store writer can join a document confirmation: it has
+ # neither the preview diff nor a revision guard. Refuse it rather than letting a future
+ # C5 operation turn this proxy into a side door around the guarded update above.
+ if name in {"upload_bytes", "delete_path", "flush"}:
+ raise AttributeError(name)
+ return getattr(self._real, name)
+
+
+def _diff_documents(documents, *, limit=250):
+ changes = []
+
+ def walk(before, after, path):
+ if len(changes) >= limit or before == after:
+ return
+ if isinstance(before, dict) and isinstance(after, dict):
+ for key in sorted(before.keys() | after.keys(), key=str):
+ walk(before.get(key), after.get(key), [*path, str(key)])
+ return
+ if isinstance(before, list) and isinstance(after, list):
+ for index in range(max(len(before), len(after))):
+ walk(before[index] if index < len(before) else None,
+ after[index] if index < len(after) else None,
+ [*path, str(index)])
+ return
+ changes.append({"path": "/" + "/".join(path),
+ "before": copy.deepcopy(before), "after": copy.deepcopy(after)})
+
+ for bucket, pair in sorted(documents.items()):
+ walk(pair.get("before"), pair.get("after"), [bucket])
+ return changes, len(changes) >= limit
+
+
+def _credential_binding(session: Session) -> str:
+ claims = session.claims or {}
+ return str(claims.get("external_credential_id") or claims.get("external_key_id") or "")
+
+
+def _confirmation_token(payload) -> str:
+ raw = _canonical_json(payload)
+ signature = aios_session._sign(b"external-confirm:v1:" + raw)
+ return f"{_CONFIRMATION_PREFIX}.{_b64(raw)}.{signature}"
+
+
+def _read_confirmation_token(token) -> dict:
+ if not isinstance(token, str) or len(token) > _CONFIRMATION_MAX_BYTES * 2:
+ raise err(400, "confirmation_token_invalid", "The confirmation token is invalid.")
+ try:
+ prefix, body, supplied = token.split(".", 2)
+ raw = _unb64(body)
+ expected = aios_session._sign(b"external-confirm:v1:" + raw)
+ if prefix != _CONFIRMATION_PREFIX or not hmac.compare_digest(supplied, expected):
+ raise ValueError("signature")
+ payload = json.loads(raw.decode("utf-8"))
+ except (ValueError, TypeError, UnicodeError, json.JSONDecodeError) as exc:
+ raise err(400, "confirmation_token_invalid", "The confirmation token is invalid.") from exc
+ if not isinstance(payload, dict) or payload.get("v") != 1:
+ raise err(400, "confirmation_token_invalid", "The confirmation token is invalid.")
+ return payload
+
+
+def _assert_finite_query_value(value) -> None:
+ """Keep non-finite source values from reaching sorting, aggregation, or a JSON response."""
+ try:
+ json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=True,
+ allow_nan=False, default=str)
+ except (TypeError, ValueError, OverflowError) as exc:
+ raise err(422, "query_value_invalid",
+ "A permitted record contains a value that cannot be represented safely.") from exc
+
+
+def _cursor_proof(session: Session, query_digest: str, snapshot_digest: str, offset: int) -> str:
+ claims = session.claims or {}
+ binding = {
+ "tenant": session.tenant,
+ "user": session.uname,
+ "credential": claims.get("external_credential_id") or "browser",
+ "query": query_digest,
+ "snapshot": snapshot_digest,
+ "offset": offset,
+ "version": 1,
+ }
+ raw = json.dumps(binding, sort_keys=True, separators=(",", ":")).encode("utf-8")
+ return aios_session._sign(b"external-query:v1:" + raw)
+
+
+def _encode_query_cursor(session: Session, query_digest: str,
+ snapshot_digest: str, offset: int) -> str:
+ payload = {
+ "v": 1, "q": query_digest, "s": snapshot_digest, "o": offset,
+ "p": _cursor_proof(session, query_digest, snapshot_digest, offset),
+ }
+ raw = json.dumps(payload, sort_keys=True, separators=(",", ":")).encode("ascii")
+ return base64.urlsafe_b64encode(raw).rstrip(b"=").decode("ascii")
+
+
+def _decode_query_cursor(session: Session, raw, query_digest: str,
+ snapshot_digest: str) -> int:
+ if not isinstance(raw, str) or not raw or len(raw) > _QUERY_MAX_CURSOR:
+ raise err(400, "query_cursor_invalid", "The query cursor is invalid.")
+ try:
+ padded = raw + "=" * (-len(raw) % 4)
+ payload = json.loads(base64.b64decode(
+ padded.encode("ascii"), altchars=b"-_", validate=True).decode("utf-8"))
+ except (ValueError, TypeError, UnicodeError, json.JSONDecodeError) as exc:
+ raise err(400, "query_cursor_invalid", "The query cursor is invalid.") from exc
+ if (not isinstance(payload, dict)
+ or set(payload) != {"v", "q", "s", "o", "p"}
+ or payload.get("v") != 1
+ or type(payload.get("o")) is not int
+ or payload["o"] < 0
+ or not isinstance(payload.get("q"), str)
+ or not isinstance(payload.get("s"), str)
+ or not isinstance(payload.get("p"), str)):
+ raise err(400, "query_cursor_invalid", "The query cursor is invalid.")
+ expected = _cursor_proof(session, payload["q"], payload["s"], payload["o"])
+ if not hmac.compare_digest(payload["p"], expected):
+ raise err(400, "query_cursor_invalid", "The query cursor is invalid.")
+ if payload["q"] != query_digest or payload["s"] != snapshot_digest:
+ raise err(409, "query_cursor_stale",
+ "The query or its permitted result set changed. Start again without a cursor.")
+ return payload["o"]
+
+
+def _sort_atom(value):
+ if isinstance(value, bool):
+ return "bool", 1 if value else 0
+ if isinstance(value, (int, float)):
+ return "number", float(value)
+ if isinstance(value, str):
+ return "text", value.casefold(), value
+ return type(value).__name__, json.dumps(value, sort_keys=True, default=str, ensure_ascii=True)
+
+
+def _sort_records(records, sorts):
+ ordered = list(records)
+ # Python's stable sort lets the first requested key remain the primary key when the rules
+ # are applied in reverse. Nulls stay last in both directions rather than floating to the top.
+ for rule in reversed(sorts):
+ field = rule["field"]
+ present = [row for row in ordered if row.get(field) is not None]
+ missing = [row for row in ordered if row.get(field) is None]
+ present.sort(key=lambda row: _sort_atom(row.get(field)),
+ reverse=rule["direction"] == "desc")
+ ordered = present + missing
+ return ordered
+
+
+def _normalise_aggregates(raw, visible_keys):
+ if raw is None:
+ return []
+ if not isinstance(raw, list) or len(raw) > _QUERY_MAX_AGGREGATES:
+ raise err(400, "query_aggregates_invalid",
+ f"aggregates must contain at most {_QUERY_MAX_AGGREGATES} operations.")
+ allowed = {"count", "count_distinct", "sum", "avg", "min", "max"}
+ out = []
+ for item in raw:
+ if not isinstance(item, dict) or set(item) - {"fn", "field"}:
+ raise err(400, "query_aggregates_invalid",
+ "Every aggregate must contain only fn and an optional readable field.")
+ fn, field = item.get("fn"), item.get("field")
+ has_field = "field" in item
+ if (fn not in allowed
+ or (has_field and (not isinstance(field, str) or not field))
+ or (fn != "count" and not has_field)):
+ raise err(400, "query_aggregates_invalid",
+ "Choose count, count_distinct, sum, avg, min, or max.")
+ if field is not None and field not in visible_keys:
+ raise err(400, "assistant_unknown_field",
+ "Every requested field must be readable in this database.")
+ out.append({"fn": fn, **({"field": field} if field is not None else {})})
+ return out
+
+
+def _assert_query_filter_bounds(raw) -> None:
+ """Reject an oversized tree before the shared strict filter validator recurses through it."""
+ if raw is None:
+ return
+ if isinstance(raw, list):
+ roots = raw
+ elif isinstance(raw, dict):
+ roots = raw.get("nodes")
+ if not isinstance(roots, list):
+ return # The shared validator supplies the named shape refusal.
+ else:
+ return
+ if len(roots) > _QUERY_MAX_FILTER_SIBLINGS:
+ raise err(400, "query_filter_limit",
+ f"A filter group may contain at most {_QUERY_MAX_FILTER_SIBLINGS} conditions.")
+ stack = [(node, 1) for node in roots]
+ seen = 0
+ while stack:
+ node, depth = stack.pop()
+ seen += 1
+ if seen > _QUERY_MAX_FILTER_NODES:
+ raise err(400, "query_filter_limit",
+ f"A query may contain at most {_QUERY_MAX_FILTER_NODES} filter nodes.")
+ if not isinstance(node, dict) or "children" not in node:
+ continue
+ children = node.get("children")
+ if not isinstance(children, list):
+ continue
+ if len(children) > _QUERY_MAX_FILTER_SIBLINGS:
+ raise err(400, "query_filter_limit",
+ f"A filter group may contain at most {_QUERY_MAX_FILTER_SIBLINGS} conditions.")
+ if children and depth >= _QUERY_MAX_FILTER_DEPTH:
+ raise err(400, "query_filter_limit",
+ f"A filter may nest at most {_QUERY_MAX_FILTER_DEPTH} levels.")
+ stack.extend((child, depth + 1) for child in children)
+
+
+def _aggregate_records(records, specs):
+ out = []
+ for spec in specs:
+ fn, field = spec["fn"], spec.get("field")
+ values = ([row.get(field) for row in records if row.get(field) is not None]
+ if field is not None else [])
+ if fn == "count":
+ value = len(values) if field is not None else len(records)
+ elif fn == "count_distinct":
+ value = len({_json_digest(item) for item in values})
+ elif fn in {"sum", "avg"}:
+ if any(isinstance(item, bool) or not isinstance(item, (int, float))
+ for item in values):
+ raise err(400, "query_aggregate_type",
+ "sum and avg require a readable numeric field.")
+ value = sum(values) if fn == "sum" else (sum(values) / len(values) if values else None)
+ elif fn in {"min", "max"}:
+ value = (None if not values else
+ sorted(values, key=_sort_atom, reverse=fn == "max")[0])
+ else: # Kept fail-closed even if the normalizer is changed independently.
+ raise err(400, "query_aggregates_invalid", "That aggregate is not available.")
+ out.append({**spec, "value": copy.deepcopy(value)})
+ return out
+
+
+def _view_referenced_fields(config) -> list[str]:
+ keys, seen = [], set()
+
+ def add(value):
+ if isinstance(value, str) and value and value not in seen:
+ seen.add(value)
+ keys.append(value)
+
+ def walk_filter(node):
+ if isinstance(node, list):
+ for child in node:
+ walk_filter(child)
+ elif isinstance(node, dict):
+ add(node.get("colId"))
+ rhs = node.get("rhs")
+ if isinstance(rhs, dict) and rhs.get("kind") != "stat":
+ add(rhs.get("colId"))
+ walk_filter(node.get("nodes"))
+ walk_filter(node.get("children"))
+
+ if not isinstance(config, dict):
+ return keys
+ for name in ("visible", "order"):
+ for value in config.get(name) or ():
+ add(value)
+ add(config.get("groupBy"))
+ for rule in config.get("sorts") or ():
+ if isinstance(rule, dict):
+ add(rule.get("colId"))
+ walk_filter(config.get("filters"))
+ return keys
def _key_scope(session):
@@ -65,23 +552,52 @@ def _databases_in_arguments(value):
return found
-def _auth_refusal():
- raise err(401, external_access.AUTH_ERROR, "The credential is invalid or revoked.")
+def _auth_refusal(request: Request | None = None):
+ challenge = "Bearer"
+ if request is not None:
+ metadata = _oauth_urls(request)["resource_metadata"]
+ challenge += f' resource_metadata="{metadata}"'
+ raise HTTPException(
+ status_code=401,
+ detail={"error": {"code": external_access.AUTH_ERROR,
+ "message": "The credential is invalid, expired, or revoked."}},
+ headers={"WWW-Authenticate": challenge},
+ )
def require_external_session(
authorization: str | None = Header(default=None, alias="Authorization"),
+ request: Request = None,
) -> Session:
"""Resolve a bearer credential without consulting or extending the browser session cookie."""
if not isinstance(authorization, str):
- _auth_refusal()
+ _auth_refusal(request)
scheme, separator, token = authorization.strip().partition(" ")
if not separator or scheme.lower() != "bearer" or not token.strip():
- _auth_refusal()
+ _auth_refusal(request)
+ raw_token = token.strip()
try:
- verified = external_access.verify(token.strip())
- except external_access.ExternalAuthError:
- _auth_refusal()
+ if raw_token.startswith("lpo1."):
+ verified_oauth = external_oauth.verify_access(
+ raw_token, resource=_oauth_urls(request)["protected_resource"],
+ runtime_resolver=runtime.get_runtime)
+ claims = {
+ "t": verified_oauth.tenant,
+ "u": verified_oauth.user["username"],
+ "external": True,
+ "external_auth_type": "oauth",
+ "external_credential_id": verified_oauth.grant_id,
+ "external_client_id": verified_oauth.client_id,
+ "external_access": verified_oauth.access,
+ "external_databases": (external_access.ALL_DATABASES
+ if verified_oauth.databases == external_access.ALL_DATABASES
+ else list(verified_oauth.databases)),
+ }
+ return Session(verified_oauth.tenant, verified_oauth.user, claims,
+ verified_oauth.runtime)
+ verified = external_access.verify(raw_token)
+ except (external_access.ExternalAuthError, external_oauth.ExternalOAuthAuthError):
+ _auth_refusal(request)
return Session(
tenant=verified.tenant,
user=verified.user,
@@ -89,6 +605,8 @@ def require_external_session(
"t": verified.tenant,
"u": verified.user["username"],
"external": True,
+ "external_auth_type": "api_key",
+ "external_credential_id": verified.key_id,
"external_key_id": verified.key_id,
"external_access": verified.access,
"external_databases": (external_access.ALL_DATABASES
@@ -124,6 +642,10 @@ def _management_refusal(exc: external_access.ExternalAccessError):
raise err(400, exc.code, "Choose all databases, or name at least one database.")
if exc.code == "external_label_invalid":
raise err(400, exc.code, "Enter a label between 1 and 80 characters.")
+ if exc.code == "external_expiry_invalid":
+ raise err(400, exc.code, "Choose a future UTC expiry, or no expiry.")
+ if exc.code == "external_edit_empty":
+ raise err(400, exc.code, "Change at least one credential setting.")
if exc.code == "external_key_not_found":
raise err(404, exc.code, "That credential was not found.")
if exc.code in {"external_identity_invalid", "external_account_unavailable"}:
@@ -131,6 +653,34 @@ def _management_refusal(exc: external_access.ExternalAccessError):
raise err(503, "external_access_unavailable", "The AI API is unavailable right now.")
+def _oauth_management_refusal(exc: external_access.ExternalAccessError):
+ messages = {
+ "oauth_redirect_uri_invalid": "Use an exact HTTPS callback, or an HTTP loopback callback.",
+ "oauth_state_invalid": "The OAuth state must be a non-empty, high-entropy value.",
+ "oauth_state_replayed": "This OAuth state was already used. Start a new connection.",
+ "oauth_pkce_required": "This connection requires PKCE with S256.",
+ "oauth_scope_invalid": "The requested OAuth scope is not available for this connection.",
+ "oauth_response_type_invalid": "This server supports the authorization-code flow only.",
+ "oauth_client_invalid": "That OAuth client is unavailable.",
+ "oauth_client_not_found": "That OAuth client was not found.",
+ "oauth_connection_not_found": "That OAuth connection was not found.",
+ "oauth_edit_empty": "Change at least one OAuth client setting.",
+ "oauth_resource_invalid": "Use this MCP server's exact resource URL.",
+ }
+ if exc.code in messages:
+ status = 404 if exc.code in {"oauth_client_not_found", "oauth_connection_not_found"} else 400
+ raise err(status, exc.code, messages[exc.code])
+ if exc.code in {"external_access_scope_invalid", "external_databases_invalid",
+ "external_label_invalid"}:
+ _management_refusal(exc)
+ raise err(503, "oauth_unavailable", "OAuth is unavailable right now.")
+
+
+def _credential_audit(kind: str, session: Session, credential_kind: str, **detail) -> None:
+ safe = {"surface": "ai_api", "credential_kind": credential_kind, **detail}
+ audit.record(kind, session.tenant, session.uname, safe)
+
+
def _resolve_external_scope(
session: Session,
databases=None,
@@ -197,8 +747,13 @@ def _markdown_inline(value) -> str:
return " ".join(text.replace("`", "'").split())
-def _portable_skill_document(session: Session, base_url: str) -> str:
- scoped = [
+def _portable_skill_document(session: Session | None, base_url: str) -> str:
+ """The skill.md an agent reads. With a session it carries that user's own read scope; with
+ None it is the PUBLIC document served at `/skill.md` (owner hotfix 2026-09-23, Monid's
+ `set up /SKILL.md`), which names no tenant, database or field and sends the agent to
+ discover its scope after it authenticates."""
+ public = session is None
+ scoped = [] if public else [
row for row in _resolve_external_scope(session, include_schema=True)
if row["permitted"]
]
@@ -207,6 +762,35 @@ def _portable_skill_document(session: Session, base_url: str) -> str:
example = scoped[0]
surface_url = f"{base_url.rstrip('/')}/api/v1/external"
+ connect = [
+ "## Set up",
+ "",
+ "Connect once, with the Loopable sign-in (remote MCP over OAuth) or with an API key.",
+ "",
+ f"MCP server URL: `{surface_url}/mcp`",
+ "",
+ "Claude Code:",
+ "",
+ "```sh",
+ f"claude mcp add --transport http loopable {surface_url}/mcp",
+ "```",
+ "",
+ "Then run `/mcp` in Claude Code and finish the Loopable sign-in in the browser.",
+ "",
+ "Codex:",
+ "",
+ "```sh",
+ f"codex mcp add loopable --url {surface_url}/mcp",
+ "codex mcp login loopable",
+ "```",
+ "",
+ ("Cursor: add `{\"mcpServers\": {\"loopable\": {\"url\": \"" + surface_url + "/mcp\"}}}` "
+ "to `~/.cursor/mcp.json`, keeping any existing servers, then finish the Loopable sign-in."),
+ "",
+ ("For code or an SDK, ask the user to create an API key in Loopable under AI API and to "
+ "store it in an environment variable. Never ask for the key in chat and never print it."),
+ "",
+ ] if public else []
lines = [
"---",
"name: loopable-external-data",
@@ -230,6 +814,7 @@ def _portable_skill_document(session: Session, base_url: str) -> str:
"",
f"Base URL: `{surface_url}`",
"",
+ *connect,
"## First call: list databases",
"",
"Endpoint: `GET /api/v1/external/databases`",
@@ -286,6 +871,27 @@ def _portable_skill_document(session: Session, base_url: str) -> str:
"answerable right now. The reason appears in the scope below."),
"",
])
+ elif public:
+ lines.extend([
+ "```sh",
+ "curl --request GET \\",
+ " --header 'Authorization: Bearer ' \\",
+ f" '{surface_url}/databases/{{database}}/schema'",
+ "```",
+ "",
+ "## Query a database",
+ "",
+ "Endpoint: `POST /api/v1/external/databases/{database}/query`",
+ "",
+ "```sh",
+ "curl --request POST \\",
+ " --header 'Authorization: Bearer ' \\",
+ " --header 'Content-Type: application/json' \\",
+ " --data '{\"fields\":[\"\"],\"filters\":null}' \\",
+ f" '{surface_url}/databases/{{database}}/query'",
+ "```",
+ "",
+ ])
else:
lines.extend([
"No schema call is available because this user's current scope has no databases.",
@@ -298,8 +904,18 @@ def _portable_skill_document(session: Session, base_url: str) -> str:
"",
])
- lines.extend(["## This user's current read scope", ""])
- if not scoped:
+ if public:
+ lines.extend([
+ "## Your read scope",
+ "",
+ ("Loopable answers only with what the signed-in user may already read in the app. "
+ "After connecting, call `GET /databases` for the databases, then "
+ "`GET /databases/{database}/schema` for the fields each one exposes, before any query."),
+ "",
+ ])
+ else:
+ lines.extend(["## This user's current read scope", ""])
+ if not scoped and not public:
lines.extend(["No databases are currently permitted.", ""])
for row in scoped:
database = _markdown_inline(row["database"])
@@ -359,7 +975,7 @@ def _portable_skill_document(session: Session, base_url: str) -> str:
"",
("Sheet tools require analyst and sheets grants. Geometry patches carry the revision "
"read by the server; pass rev to also protect the revision you read. Cell writes "
- "return accepted and refused addresses, retaining the app's partial-write behavior."),
+ "are previewed before confirmation. A partially refused patch changes no cells."),
"",
("View tools use the grid's native config: filters, sorts, groupBy, visible and display. "
"Fields omitted from visible are hidden. The grid validates the config. App tools use "
@@ -393,7 +1009,7 @@ def _portable_skill_document(session: Session, base_url: str) -> str:
"",
])
document = "\n".join(lines)
- if _CREDENTIAL_PREFIX in document:
+ if _contains_credential(document):
raise RuntimeError("portable skill document contained a credential value")
return document
@@ -405,11 +1021,16 @@ def create_key(body=Body(default=None), session: Session = Depends(require_sessi
label = body.get("label") if isinstance(body, dict) else None
access = body.get("access") if isinstance(body, dict) else None
databases = body.get("databases") if isinstance(body, dict) else None
+ expires_at = body.get("expires_at") if isinstance(body, dict) else None
try:
key, token = external_access.mint(session.runtime, session.uname, session.tenant, label,
- access=access, databases=databases)
+ access=access, databases=databases,
+ expires_at=expires_at)
except external_access.ExternalAccessError as exc:
_management_refusal(exc)
+ _credential_audit("credential.added", session, "api_key", access=key["access"],
+ resource_mode="all" if key["databases"] == "all" else "allowlist",
+ resource_count=(0 if key["databases"] == "all" else len(key["databases"])))
return {"key": key, "secret": token}
@@ -423,6 +1044,36 @@ def list_keys(session: Session = Depends(require_session)):
return {"keys": keys, "can_mint": _can_use_analyst(session)}
+@router.patch("/keys/{key_id}")
+def edit_key(key_id: str, body=Body(default=None), session: Session = Depends(require_session)):
+ """Edit metadata and narrowing scope without ever reading or replacing the secret."""
+ if not isinstance(body, dict):
+ raise err(400, "external_edit_empty", "Change at least one credential setting.")
+ allowed = {"label", "access", "databases", "expires_at"}
+ if not body or set(body) - allowed:
+ raise err(400, "external_edit_invalid", "Only label, access, databases, and expiry can be changed.")
+ kwargs = {key: body[key] for key in allowed if key in body}
+ try:
+ key = external_access.edit(session.runtime, session.uname, session.tenant, key_id, **kwargs)
+ except external_access.ExternalAccessError as exc:
+ _management_refusal(exc)
+ _credential_audit("config.changed", session, "api_key", setting="credential_metadata",
+ access=key["access"],
+ resource_mode="all" if key["databases"] == "all" else "allowlist")
+ return {"key": key}
+
+
+@router.post("/keys/{key_id}/rotate")
+def rotate_key(key_id: str, session: Session = Depends(require_session)):
+ """Atomically invalidate the old secret and reveal its replacement once."""
+ try:
+ key, token = external_access.rotate(session.runtime, session.uname, session.tenant, key_id)
+ except external_access.ExternalAccessError as exc:
+ _management_refusal(exc)
+ _credential_audit("credential.rotated", session, "api_key", reason="user_rotation")
+ return {"key": key, "secret": token}
+
+
@router.post("/keys/{key_id}/revoke")
def revoke_key(key_id: str, session: Session = Depends(require_session)):
"""Tombstone one key owned by the signed-in human. Bearer auth is not accepted here."""
@@ -430,9 +1081,300 @@ def revoke_key(key_id: str, session: Session = Depends(require_session)):
key = external_access.revoke(session.runtime, session.uname, session.tenant, key_id)
except external_access.ExternalAccessError as exc:
_management_refusal(exc)
+ _credential_audit("credential.revoked", session, "api_key", reason="user_revoke")
return {"key": key}
+def _oauth_urls(request: Request) -> dict:
+ origin = str(request.base_url).rstrip("/")
+ root = origin + "/api/v1/external"
+ return {
+ "issuer": root,
+ "authorization_endpoint": root + "/oauth/authorize",
+ "token_endpoint": root + "/oauth/token",
+ "registration_endpoint": root + "/oauth/clients",
+ "revocation_endpoint": root + "/oauth/disconnect",
+ "protected_resource": root + "/mcp",
+ "metadata": origin + "/.well-known/oauth-authorization-server/api/v1/external",
+ "resource_metadata": (origin
+ + "/.well-known/oauth-protected-resource/api/v1/external/mcp"),
+ }
+
+
+def _require_oauth_resource(request: Request, value) -> str:
+ """Bind every grant and token request to this exact remote MCP audience."""
+ supplied = external_oauth.normalise_resource(value)
+ expected = external_oauth.normalise_resource(_oauth_urls(request)["protected_resource"])
+ if supplied != expected:
+ raise external_oauth.ExternalOAuthError("oauth_resource_invalid")
+ return expected
+
+
+@router.get("/.well-known/oauth-authorization-server")
+def oauth_server_metadata(request: Request):
+ """Public authorization-server discovery for the path-scoped remote MCP issuer."""
+ urls = _oauth_urls(request)
+ return {
+ "issuer": urls["issuer"],
+ "authorization_endpoint": urls["authorization_endpoint"],
+ "token_endpoint": urls["token_endpoint"],
+ "revocation_endpoint": urls["revocation_endpoint"],
+ "response_types_supported": ["code"],
+ "grant_types_supported": ["authorization_code", "refresh_token"],
+ "token_endpoint_auth_methods_supported": ["none"],
+ "code_challenge_methods_supported": ["S256"],
+ "scopes_supported": list(external_oauth.SCOPES),
+ }
+
+
+@router.get("/.well-known/oauth-protected-resource")
+def oauth_resource_metadata(request: Request):
+ """Public protected-resource metadata; it contains endpoints, never a client secret."""
+ urls = _oauth_urls(request)
+ return {
+ "resource": urls["protected_resource"],
+ "authorization_servers": [urls["issuer"]],
+ "scopes_supported": list(external_oauth.SCOPES),
+ "bearer_methods_supported": ["header"],
+ }
+
+
+@router.get("/oauth/metadata")
+def oauth_metadata(request: Request):
+ return {"authorization_server": oauth_server_metadata(request),
+ "protected_resource": oauth_resource_metadata(request)}
+
+
+@router.post("/oauth/clients", status_code=201)
+def create_oauth_client(body=Body(default=None), session: Session = Depends(require_session)):
+ """Register one user-owned public client; PKCE replaces a shared client secret."""
+ session.require("analyst")
+ body = body if isinstance(body, dict) else {}
+ try:
+ client = external_oauth.register_client(
+ session.runtime, session.uname, session.tenant, body.get("label"),
+ body.get("redirect_uris"), access=body.get("access"),
+ databases=body.get("databases"),
+ )
+ except external_access.ExternalAccessError as exc:
+ _oauth_management_refusal(exc)
+ _credential_audit("config.changed", session, "oauth_client", setting="client_registered",
+ access=client["access"])
+ return {"client": client}
+
+
+@router.get("/oauth/clients")
+def list_oauth_clients(session: Session = Depends(require_session)):
+ try:
+ clients = external_oauth.list_clients(session.runtime, session.uname, session.tenant)
+ except external_access.ExternalAccessError as exc:
+ _oauth_management_refusal(exc)
+ return {"clients": clients, "can_connect": _can_use_analyst(session)}
+
+
+@router.patch("/oauth/clients/{client_id}")
+def edit_oauth_client(client_id: str, body=Body(default=None),
+ session: Session = Depends(require_session)):
+ if not isinstance(body, dict):
+ raise err(400, "oauth_edit_empty", "Change at least one OAuth client setting.")
+ allowed = {"label", "redirect_uris", "access", "databases"}
+ if not body or set(body) - allowed:
+ raise err(400, "oauth_edit_invalid", "Only label, callbacks, access, and resources can be changed.")
+ try:
+ client = external_oauth.edit_client(
+ session.runtime, session.uname, session.tenant, client_id,
+ **{key: body[key] for key in allowed if key in body},
+ )
+ except external_access.ExternalAccessError as exc:
+ _oauth_management_refusal(exc)
+ _credential_audit("config.changed", session, "oauth_client", setting="client_metadata",
+ access=client["access"])
+ return {"client": client}
+
+
+def _oauth_redirect(uri: str, **params):
+ parts = urlsplit(uri)
+ query = parse_qs(parts.query, keep_blank_values=True)
+ for key, value in params.items():
+ query[key] = [str(value)]
+ location = urlunsplit((parts.scheme, parts.netloc, parts.path,
+ urlencode(query, doseq=True), parts.fragment))
+ return RedirectResponse(location, status_code=302)
+
+
+@router.get("/oauth/callback")
+@router.get("/oauth/authorize")
+def oauth_authorize(
+ request: Request,
+ response_type: str,
+ client_id: str,
+ redirect_uri: str,
+ state: str,
+ code_challenge: str,
+ code_challenge_method: str,
+ resource: str,
+ scope: str | None = None,
+ session: Session = Depends(require_session),
+):
+ """Authorize the signed-in user and redirect an exact registered callback with one code."""
+ session.require("analyst")
+ try:
+ audience = _require_oauth_resource(request, resource)
+ result = external_oauth.authorize(
+ session.runtime, session.uname, session.tenant, client_id=client_id,
+ redirect_uri=redirect_uri, state=state, code_challenge=code_challenge,
+ code_challenge_method=code_challenge_method, resource=audience, scope=scope,
+ response_type=response_type,
+ )
+ except external_access.ExternalAccessError as exc:
+ _oauth_management_refusal(exc)
+ return _oauth_redirect(result["redirect_uri"], code=result["code"], state=result["state"])
+
+
+async def _oauth_token_body(request: Request) -> dict:
+ raw = await request.body()
+ if len(raw) > 16_384:
+ raise external_oauth.ExternalOAuthError("oauth_request_invalid")
+ content_type = request.headers.get("content-type", "").split(";", 1)[0].strip().lower()
+ if content_type == "application/json":
+ try:
+ value = json.loads(raw or b"{}")
+ except (TypeError, ValueError, UnicodeError) as exc:
+ raise external_oauth.ExternalOAuthError("oauth_request_invalid") from exc
+ if not isinstance(value, dict):
+ raise external_oauth.ExternalOAuthError("oauth_request_invalid")
+ return value
+ try:
+ parsed = parse_qs(raw.decode("utf-8"), keep_blank_values=True, strict_parsing=True)
+ except (UnicodeError, ValueError) as exc:
+ raise external_oauth.ExternalOAuthError("oauth_request_invalid") from exc
+ return {key: values[-1] for key, values in parsed.items()}
+
+
+def _oauth_token_error(code: str):
+ public = "invalid_grant" if code in {
+ "oauth_invalid_grant", "oauth_refresh_replayed", "oauth_refresh_exhausted",
+ external_access.AUTH_ERROR,
+ } else "invalid_target" if code == "oauth_resource_invalid" else "invalid_request"
+ description = ("The authorization grant is invalid, expired, replayed, or revoked."
+ if public == "invalid_grant" else "The OAuth token request is invalid.")
+ return JSONResponse({"error": public, "error_description": description}, status_code=400,
+ headers={"Cache-Control": "no-store", "Pragma": "no-cache"})
+
+
+@router.post("/oauth/token")
+async def oauth_token(request: Request):
+ """Cookie-free authorization-code exchange and rotating refresh-token endpoint."""
+ try:
+ body = await _oauth_token_body(request)
+ grant_type = body.get("grant_type")
+ audience = _require_oauth_resource(request, body.get("resource"))
+ if grant_type == "authorization_code":
+ result = external_oauth.exchange_code(
+ body.get("code"), client_id=body.get("client_id"),
+ redirect_uri=body.get("redirect_uri"), code_verifier=body.get("code_verifier"),
+ resource=audience, runtime_resolver=runtime.get_runtime,
+ )
+ verified = external_oauth.verify_access(
+ result["access_token"], resource=audience, runtime_resolver=runtime.get_runtime)
+ audit.record("credential.added", verified.tenant, verified.user["username"],
+ {"surface": "ai_api", "credential_kind": "oauth"})
+ elif grant_type == "refresh_token":
+ result = external_oauth.refresh(
+ body.get("refresh_token"), client_id=body.get("client_id"),
+ resource=audience, runtime_resolver=runtime.get_runtime)
+ verified = external_oauth.verify_access(
+ result["access_token"], resource=audience, runtime_resolver=runtime.get_runtime)
+ audit.record("credential.rotated", verified.tenant, verified.user["username"],
+ {"surface": "ai_api", "credential_kind": "oauth",
+ "reason": "refresh_rotation"})
+ else:
+ raise external_oauth.ExternalOAuthError("oauth_request_invalid")
+ except external_access.ExternalAccessError as exc:
+ return _oauth_token_error(exc.code)
+ response = JSONResponse(result)
+ response.headers.update({"Cache-Control": "no-store", "Pragma": "no-cache"})
+ return response
+
+
+@router.post("/oauth/disconnect")
+def oauth_disconnect(
+ authorization: str | None = Header(default=None, alias="Authorization"),
+):
+ """Cookie-free immediate revocation using the access or refresh token itself."""
+ if not isinstance(authorization, str):
+ _auth_refusal()
+ scheme, separator, token = authorization.strip().partition(" ")
+ if not separator or scheme.lower() != "bearer" or not token.strip():
+ _auth_refusal()
+ try:
+ stopped = external_oauth.disconnect(token.strip(), runtime_resolver=runtime.get_runtime)
+ except external_oauth.ExternalOAuthAuthError:
+ _auth_refusal()
+ audit.record("credential.revoked", stopped.tenant, stopped.username,
+ {"surface": "ai_api", "credential_kind": "oauth",
+ "reason": "disconnect"})
+ return {"connection": stopped.connection}
+
+
+@router.get("/oauth/connections")
+def oauth_connections(session: Session = Depends(require_session)):
+ try:
+ connections = external_oauth.list_connections(
+ session.runtime, session.uname, session.tenant)
+ except external_access.ExternalAccessError as exc:
+ _oauth_management_refusal(exc)
+ return {"connections": connections}
+
+
+@router.post("/oauth/connections/{connection_id}/disconnect")
+def oauth_disconnect_human(connection_id: str, session: Session = Depends(require_session)):
+ try:
+ connection = external_oauth.disconnect_for_user(
+ session.runtime, session.uname, session.tenant, connection_id)
+ except external_access.ExternalAccessError as exc:
+ _oauth_management_refusal(exc)
+ _credential_audit("credential.revoked", session, "oauth", reason="user_disconnect")
+ return {"connection": connection}
+
+
+@router.get("/setup")
+def external_setup(request: Request):
+ """Secret-free client-specific setup metadata for the AI API page."""
+ urls = _oauth_urls(request)
+ starter = ("List the Loopable databases I can access, describe the fields in the one I choose, "
+ "and ask before querying any records. Do not make changes.")
+ caveat = ("Client availability, plan eligibility, and workspace approval depend on the AI "
+ "client. Check with that client's workspace administrator.")
+ mcp = urls["protected_resource"]
+ return {
+ "mcp_url": mcp,
+ "oauth": {"metadata_url": urls["metadata"], "client_registration": urls["registration_endpoint"],
+ "steps": ["Register this client's exact callback in Loopable.",
+ "Open the authorization URL and approve as the signed-in Loopable user.",
+ "Send the MCP URL as resource on authorization and token requests.",
+ "Exchange the code with PKCE S256; never paste a token into chat."]},
+ "code_sdk": {"authentication": "Bearer API key from an environment variable",
+ "header": "Authorization: Bearer ${LOOPABLE_API_KEY}",
+ "base_url": urls["issuer"],
+ "warning": "The key is revealed only on create or rotate. Store it in a secret manager."},
+ "clients": {
+ "claude": {"mcp_url": mcp,
+ "command_or_config": f"claude mcp add --transport http loopable {mcp}",
+ "plan_admin_caveat": caveat, "starter_prompt": starter},
+ "chatgpt": {"mcp_url": mcp,
+ "command_or_config": "Add a custom remote MCP connector and enter the MCP URL; complete OAuth in the opened browser.",
+ "plan_admin_caveat": caveat, "starter_prompt": starter},
+ "claude_code": {"mcp_url": mcp,
+ "command_or_config": f"claude mcp add --transport http loopable {mcp}",
+ "plan_admin_caveat": caveat, "starter_prompt": starter},
+ "codex": {"mcp_url": mcp,
+ "command_or_config": f"codex mcp add loopable --url {mcp}",
+ "plan_admin_caveat": caveat, "starter_prompt": starter},
+ },
+ }
+
+
@router.get("/skill.md")
def portable_skill(request: Request, session: Session = Depends(require_session)):
"""Download current governed external read and build instructions."""
@@ -453,7 +1395,10 @@ def whoami(session: Session = Depends(require_external_analyst)):
"username": session.uname,
"name": session.user.get("name") or session.uname,
"tenant": session.tenant,
- "key_id": session.claims["external_key_id"],
+ "auth_type": session.claims.get("external_auth_type", "api_key"),
+ "credential_id": session.claims.get("external_credential_id"),
+ "key_id": session.claims.get("external_key_id"),
+ "client_id": session.claims.get("external_client_id"),
"access": access,
"databases": (external_access.ALL_DATABASES
if databases == external_access.ALL_DATABASES else list(databases)),
@@ -468,19 +1413,19 @@ def scope_databases(session: Session = Depends(require_session)):
return {"databases": [
{key: row[key] for key in ("database", "label", "permitted", "answerable")}
for row in _resolve_external_scope(session)
- if row.get("permitted") or row.get("visible")
+ if row.get("permitted")
]}
@router.get("/databases")
def list_databases(session: Session = Depends(require_external_analyst)):
- """List only sources this user may read or already sees elsewhere in the product."""
+ """List only sources this user may currently read through the ordinary permission wall."""
databases = []
key_databases = _key_scope(session)[1]
for row in _resolve_external_scope(session):
if not external_access.database_allowed(key_databases, row["database"]):
continue
- if not (row.get("permitted") or row.get("visible")):
+ if not row.get("permitted"):
continue
public = {
"database": row["database"],
@@ -510,15 +1455,124 @@ def query_database(
body: dict | None = Body(default=None),
session: Session = Depends(require_external_analyst),
):
- """Read one detached Assistant snapshot; FastAPI rejects non-object request bodies."""
+ """Read one freshly permissioned, bounded and cursor-stable Assistant snapshot."""
+ return _query_database(session, database, body or {})
+
+
+def _query_database(session: Session, database: str, payload: dict):
+ if not isinstance(payload, dict) or set(payload) - _QUERY_KEYS:
+ raise err(400, "query_arguments_invalid",
+ "Use only fields, filters, sorts, limit, cursor, and aggregates.")
_assert_database_scope(session, database)
- payload = body or {}
- return deps.assistant_read_scope(
- session,
- database,
- fields=payload.get("fields"),
- filters=payload.get("filters"),
- )
+ limit = payload.get("limit", _QUERY_DEFAULT_LIMIT)
+ if type(limit) is not int or not 1 <= limit <= _QUERY_MAX_LIMIT:
+ raise err(400, "query_limit_invalid",
+ f"limit must be between 1 and {_QUERY_MAX_LIMIT}.")
+ raw_sorts = payload.get("sorts") or []
+ if not isinstance(raw_sorts, list) or len(raw_sorts) > _QUERY_MAX_SORTS:
+ raise err(400, "query_sorts_invalid",
+ f"sorts must contain at most {_QUERY_MAX_SORTS} rules.")
+ _assert_query_filter_bounds(payload.get("filters"))
+
+ # This is the same permissioned record reader the in-app Assistant uses. It resolves the
+ # database, field and row walls on every call before this adapter sorts, caps, or aggregates.
+ scoped = deps.assistant_read_scope(
+ session, database, fields=None, filters=payload.get("filters"))
+ visible_fields = list(scoped.get("fields") or [])
+ visible_keys = {
+ field.get("key") for field in visible_fields
+ if isinstance(field, dict) and isinstance(field.get("key"), str)
+ }
+ requested = payload.get("fields")
+ if requested is None:
+ selected_keys = [field["key"] for field in visible_fields if field.get("key") in visible_keys]
+ else:
+ if (not isinstance(requested, list) or len(requested) > 200
+ or any(not isinstance(key, str) or not key or key not in visible_keys
+ for key in requested)):
+ raise err(400, "assistant_unknown_field",
+ "Every requested field must be readable in this database.")
+ selected_keys = list(dict.fromkeys(requested))
+
+ sorts = []
+ for rule in raw_sorts:
+ if (not isinstance(rule, dict) or set(rule) != {"field", "direction"}
+ or rule.get("field") not in visible_keys
+ or rule.get("direction") not in {"asc", "desc"}):
+ # Hidden, unknown and malformed sort fields deliberately share one refusal. A guessed
+ # field must never become an existence oracle at this external boundary.
+ raise err(400, "assistant_unknown_field",
+ "Every requested field must be readable in this database.")
+ sorts.append({"field": rule["field"], "direction": rule["direction"]})
+ aggregates = _normalise_aggregates(payload.get("aggregates"), visible_keys)
+
+ rows = list(scoped.get("records") or [])
+ inspected_keys = (set(selected_keys)
+ | {rule["field"] for rule in sorts}
+ | {spec["field"] for spec in aggregates if spec.get("field") is not None})
+ for row in rows:
+ _assert_finite_query_value({
+ "pid": row.get("pid"),
+ **{key: row.get(key) for key in inspected_keys},
+ })
+ rows = _sort_records(rows, sorts)
+ query_digest = _json_digest({
+ "database": database, "fields": selected_keys, "filters": scoped.get("filters"),
+ "sorts": sorts, "aggregates": aggregates, "limit": limit,
+ })
+ # Bind a cursor to the live permission projection as well as the source revision. A field or
+ # row-scope revocation changes this hash on the next call, so an old page cannot skip into a
+ # newly narrowed result set. Only the digest enters the cursor; no record id or value does.
+ snapshot_digest = _json_digest({
+ "source_version": scoped.get("source_version"),
+ "visible_fields": [field["key"] for field in visible_fields
+ if isinstance(field, dict) and field.get("key") in visible_keys],
+ "permitted_pids": [row.get("pid") for row in rows],
+ })
+ offset = (0 if payload.get("cursor") is None else
+ _decode_query_cursor(session, payload.get("cursor"),
+ query_digest, snapshot_digest))
+ if offset > len(rows):
+ raise err(409, "query_cursor_stale",
+ "The permitted result set changed. Start again without a cursor.")
+ page_rows = rows[offset:offset + limit]
+ records = [
+ {"pid": copy.deepcopy(row.get("pid")),
+ **{key: copy.deepcopy(row.get(key)) for key in selected_keys}}
+ for row in page_rows
+ ]
+ next_offset = offset + len(page_rows)
+ next_cursor = (_encode_query_cursor(session, query_digest, snapshot_digest, next_offset)
+ if next_offset < len(rows) else None)
+ result = {
+ "database": scoped.get("database"),
+ "label": scoped.get("label"),
+ "source_kind": scoped.get("source_kind"),
+ "source_version": copy.deepcopy(scoped.get("source_version")),
+ "retrieved_at": scoped.get("retrieved_at"),
+ "permission_scope_applied": True,
+ "fields": [copy.deepcopy(field) for field in visible_fields
+ if field.get("key") in selected_keys],
+ "filters": copy.deepcopy(scoped.get("filters")),
+ "sorts": copy.deepcopy(sorts),
+ "records": records,
+ "aggregates": _aggregate_records(rows, aggregates),
+ "page": {"limit": limit, "returned": len(records), "offset": offset,
+ "has_more": next_cursor is not None, "next_cursor": next_cursor},
+ "caps": {"max_page_size": _QUERY_MAX_LIMIT, "max_sorts": _QUERY_MAX_SORTS,
+ "max_aggregates": _QUERY_MAX_AGGREGATES,
+ "max_filter_depth": _QUERY_MAX_FILTER_DEPTH,
+ "max_filter_nodes": _QUERY_MAX_FILTER_NODES,
+ "max_filter_siblings": _QUERY_MAX_FILTER_SIBLINGS},
+ }
+ _assert_finite_query_value(result)
+ audit.record("external.query", session.tenant, session.uname, {
+ "surface": "ai_api", "authentication": session.claims.get("external_auth_type", "api_key"),
+ "field_count": len(selected_keys), "filter_applied": scoped.get("filters") is not None,
+ "sort_count": len(sorts), "aggregate_count": len(aggregates),
+ "page_size": limit, "returned_count": len(records), "cursor_used": offset > 0,
+ })
+ return result
def _view_context(session, database, event):
@@ -583,15 +1637,148 @@ def _application_access(session, databases, *, strict=False):
return allowed, records
+def _validate_personal_view_draft(session: Session, database: str, raw: dict) -> None:
+ if (not isinstance(raw, dict) or set(raw) - {"id", "name", "config", "note"}
+ or not isinstance(raw.get("config"), dict)
+ or set(raw["config"]) - _VIEW_DRAFT_CONFIG_KEYS):
+ raise err(400, "tool_arguments_invalid",
+ "A personal draft View may contain only id, name, note, and the published config fields.")
+ if (not isinstance(raw.get("id"), str) or not 1 <= len(raw["id"]) <= 120
+ or raw["id"] != raw["id"].strip()
+ or not isinstance(raw.get("name"), str) or not 1 <= len(raw["name"]) <= 120
+ or raw["name"] != raw["name"].strip()
+ or ("note" in raw and (not isinstance(raw["note"], str)
+ or len(raw["note"]) > 2_000))):
+ raise err(400, "tool_arguments_invalid",
+ "A personal draft View needs bounded id, name, and optional note text.")
+ config = raw["config"]
+ for name in ("visible", "order"):
+ value = config.get(name, [])
+ if not isinstance(value, list) or len(value) > 400 or any(not isinstance(key, str) for key in value):
+ raise err(400, "tool_arguments_invalid",
+ f"View {name} must be a bounded list of readable field keys.")
+ sorts = config.get("sorts", [])
+ if (not isinstance(sorts, list) or len(sorts) > 20
+ or any(not isinstance(rule, dict)
+ or set(rule) - {"colId", "dir", "locked"}
+ or not isinstance(rule.get("colId"), str)
+ or rule.get("dir") not in {"asc", "desc"}
+ or ("locked" in rule and type(rule["locked"]) is not bool)
+ for rule in sorts)):
+ raise err(400, "tool_arguments_invalid",
+ "View sorts must use readable fields and asc or desc direction.")
+ if config.get("filterConj") not in (None, "and", "or"):
+ raise err(400, "tool_arguments_invalid", "View filterConj must be and or or.")
+ if config.get("groupBy") is not None and not isinstance(config.get("groupBy"), str):
+ raise err(400, "tool_arguments_invalid", "View groupBy must be a readable field key.")
+ if config.get("display") is not None and not isinstance(config.get("display"), dict):
+ raise err(400, "tool_arguments_invalid", "View display must be an object.")
+ member_pids = config.get("memberPids", [])
+ if (not isinstance(member_pids, list) or len(member_pids) > 10_000
+ or any(type(pid) not in (int, str) or not str(pid).strip() for pid in member_pids)):
+ raise err(400, "tool_arguments_invalid",
+ "View memberPids must be a bounded list of record ids.")
+
+ _assert_query_filter_bounds(config.get("filters"))
+ schema = database_schema(database, session)
+ visible = {
+ field.get("key") for field in schema.get("fields") or ()
+ if isinstance(field, dict) and isinstance(field.get("key"), str)
+ }
+ if any(key not in visible for key in _view_referenced_fields(config)):
+ raise err(400, "assistant_unknown_field",
+ "Every requested field must be readable in this database.")
+ # The ordinary Assistant validator owns filter syntax and operands. fields=[] keeps values
+ # out of this validation result while still applying the live database/field/row walls.
+ filters = config.get("filters")
+ deps.assistant_read_scope(
+ session, database, fields=[], filters=None if filters == [] else filters)
+
+
+def _view_share_preview(session: Session, args: dict):
+ """Build C8's audience/field-inheritance plan without changing a View or any grant."""
+ import routes_shares
+
+ database, view_id = args["database"], str(args["view_id"] or "").strip()
+ entries = args["entries"]
+ _assert_database_scope(session, database)
+ if not view_id or len(view_id) > 120 or view_id != args["view_id"]:
+ raise err(400, "tool_arguments_invalid",
+ "A View id must have 1 to 120 characters without surrounding spaces.")
+ if (not isinstance(entries, list) or len(entries) > 200
+ or any(not isinstance(row, dict) or set(row) != {"user", "role"}
+ or not isinstance(row.get("user"), str)
+ or row.get("role") not in {"view", "edit"} for row in entries)):
+ raise err(400, "bad_entries", "entries must be a bounded list of {user, role}.")
+ routes_shares._entries_or_400(session, entries)
+
+ ctx = _view_context(session, database, {})
+ found = ctx.table.find_view(view_id, prefer_owner=session.uname)
+ if not found or str(found[0]).strip().lower() != str(session.uname).strip().lower():
+ raise err(404, "view_unavailable",
+ "That personal draft View is unavailable.")
+ view = found[1]
+ current = routes_shares.get_share("view", view_id, session)
+ if current.get("owner") or current.get("entries"):
+ raise err(409, "view_is_shared",
+ "This View is already shared. Use a separately confirmed View audience change.")
+ schema = database_schema(database, session)
+ readable = {
+ field.get("key") for field in schema.get("fields") or ()
+ if isinstance(field, dict) and isinstance(field.get("key"), str)
+ }
+ included = routes_shares._view_field_keys(view)
+ if any(key not in readable for key in included):
+ raise err(409, "view_permission_changed",
+ "This View includes a field you can no longer read. Update the draft before sharing.")
+
+ # Use the same C8 preparation as the share writer. Preview validates private/shared
+ # definitions and the caller's field wall, but never promotes definitions or writes grants.
+ try:
+ topic, _view, _grant_topic, included, _count = routes_shares._prepare_view_share(
+ session, view_id, preview=True)
+ except routes_shares._ViewSharePreparationError as exc:
+ raise err(409, "view_permission_changed",
+ "This View includes a field you can no longer share. Update the draft first.") from exc
+ if f"{topic}_table_workspace" != ctx.table.table_key or _view != view:
+ raise err(409, "view_permission_changed",
+ "This View changed while its sharing preview was prepared. Open the draft again.")
+ definitions = routes_shares._view_field_definitions(
+ session, f"{topic}_table_workspace", included)
+ database_owned = [key for key in included
+ if not isinstance(definitions[key][0], dict)
+ or definitions[key][1] == "catalogue"]
+ return {
+ "preview": True,
+ "mutated": False,
+ "database": database,
+ "view_id": view_id,
+ "draft": "personal",
+ "current_entries": copy.deepcopy(current.get("entries") or []),
+ "proposed_entries": copy.deepcopy(entries),
+ "field_inheritance": {
+ "included": included,
+ "database_owned": database_owned,
+ "grantable": included,
+ "refused": [],
+ "scope": "view_audience",
+ "retains_recipient_permission_walls": True,
+ },
+ "requires_confirmation": True,
+ "next_tool": "confirm_change",
+ }
+
+
def _view_event(session, kind, args):
from core import grid_events
database = args["database"]
_assert_database_scope(session, database)
- event = {"id": uuid.uuid4().hex, "type": kind}
+ from core import write_execution
+ event = {"id": write_execution.new_id(), "type": kind}
if kind == "view_upsert":
raw = args["view"]
- if (not isinstance(raw.get("config"), dict)
+ if (not isinstance(raw, dict) or not isinstance(raw.get("config"), dict)
or not isinstance(raw.get("id"), str) or not raw["id"].strip()
or not isinstance(raw.get("name"), str) or not raw["name"].strip()):
raise err(400, "tool_arguments_invalid", "A View needs an id, a name and an object config.")
@@ -606,6 +1793,21 @@ def _view_event(session, kind, args):
if not view_id.strip() or len(view_id) > 120 or view_id != view_id.strip():
raise err(400, "tool_arguments_invalid", "A View id must have 1 to 120 characters without surrounding spaces.")
ctx = _view_context(session, database, event)
+ if kind == "view_upsert":
+ # Resolve the ordinary View/database wall before the additional T31 draft validation so
+ # the established refusal vocabulary and connector record-scope guard stay intact.
+ _validate_personal_view_draft(session, database, raw)
+ existing = ctx.table.find_view(view_id, prefer_owner=session.uname)
+ if (existing and str(existing[0]).strip().lower()
+ != str(session.uname).strip().lower()):
+ raise err(404, "view_unavailable",
+ f"View {view_id}: another user's personal draft is read-only.")
+ from core import shares
+ share_record = shares.grants("view", view_id, st=session.runtime)
+ if share_record.get("owner") or share_record.get("entries"):
+ raise err(409, "view_is_shared",
+ "This View has entered the sharing workflow and cannot be changed by the "
+ "personal-draft tool. Use a separately confirmed View change.")
try:
before = ctx.table.find_view(view_id, prefer_owner=session.uname)
rerender = grid_events.handle_one(event, ctx)
@@ -633,7 +1835,12 @@ def _view_event(session, kind, args):
# after is the only thing that can tell that apart from a save that landed
# ([[lost-write-looks-like-failed-read]]). `rerender` rides out as information, never as the
# verdict.
- return {"database": database, "view_id": view_id, "rerender": bool(rerender)}
+ if (kind == "view_upsert" and
+ (not after or str(after[0]).strip().lower()
+ != str(session.uname).strip().lower())):
+ raise err(409, "view_not_personal", f"View {view_id}: the personal draft was not saved.")
+ return {"database": database, "view_id": view_id, "rerender": bool(rerender),
+ "draft": "personal", "shared": False}
def _application_upsert(session, args):
@@ -674,7 +1881,435 @@ def _application_upsert(session, args):
routes_assets._application_error(exc)
-def _tool_context(session: Session) -> external_tools.ToolContext:
+def _bucket_revision(st, key, fallback):
+ try:
+ revision = st.revision(key)
+ if isinstance(revision, dict):
+ return copy.deepcopy(revision)
+ except Exception:
+ pass
+ return {"rev": None, "token": _json_digest(fallback)}
+
+
+def _database_value_plan(session, database, record_id, values, fields):
+ """Apply the grid's field/value vocabulary without calling its event side effects."""
+ from core import grid_events, user_tables
+ import aios_grid
+
+ if not isinstance(values, dict) or not values:
+ raise err(400, "tool_arguments_invalid", "Record values must be a non-empty object.")
+ field_by_key = {str(field.get("key")): field for field in fields if isinstance(field, dict)}
+ unknown = sorted(str(key) for key in values if str(key) not in field_by_key)
+ if unknown:
+ raise err(403, "field_unavailable",
+ "Every changed field must be readable and editable in this database.")
+ ctx = _view_context(session, database, {"pid": record_id})
+ planned = {}
+ for key, raw in values.items():
+ key = str(key)
+ definition = field_by_key[key]
+ if key in (ctx.hidden_keys or frozenset()) or not grid_events._may_edit_field_value(
+ ctx, key, definition):
+ raise err(403, "field_unavailable",
+ "Every changed field must be readable and editable in this database.")
+ if user_tables.is_computed_cell(definition) or definition.get("type") == "automation":
+ raise err(403, "computed_field", "A computed or automation field is read only.")
+ why = user_tables.cell_type_refusal(definition, raw)
+ if why:
+ raise err(400, "bad_value", why)
+ if definition.get("type") == "link":
+ bag = definition.get("link") if isinstance(definition.get("link"), dict) else {}
+ if bag.get("on") or bag.get("inverse"):
+ raise err(403, "computed_field", "A derived link field is read only.")
+ target = user_tables.get(str(bag.get("table") or ""), st=session.runtime) or {}
+ available = set((target.get("rows") or {}).keys())
+ raw_ids = raw if isinstance(raw, (list, tuple, set)) else str(raw or "").split(",")
+ selected = []
+ for item in raw_ids:
+ candidate = str(item).strip()
+ if not candidate or candidate in selected:
+ continue
+ if candidate not in available or len(selected) >= user_tables.LINK_MAX_IDS:
+ raise err(400, "bad_value", "A linked record is unavailable.")
+ selected.append(candidate)
+ if bag.get("single"):
+ break
+ planned[key] = ",".join(selected)
+ continue
+ if isinstance(definition.get("profile"), dict):
+ handle, ok = user_tables.normalize_profile(raw, definition["profile"].get("source"))
+ if not ok:
+ raise err(400, "bad_value", "That value is not a valid profile handle.")
+ planned[key] = handle
+ continue
+ if definition.get("type") == "json":
+ text = raw if isinstance(raw, str) else json.dumps(raw, separators=(",", ":"))
+ if text.strip():
+ try:
+ json.loads(text)
+ except (ValueError, TypeError) as exc:
+ raise err(400, "bad_value", "A JSON field requires a valid JSON document.") from exc
+ planned[key] = text
+ continue
+ clean = str(raw)[:10000]
+ if isinstance(raw, str) and clean != raw:
+ raise err(400, "bad_value", "A field value exceeds the supported length.")
+ if definition.get("type") in ("select", "multiselect") and clean:
+ options = aios_grid.merge_choice_options(definition.get("options"))
+ canonical = {value.casefold(): value for value in options}
+ parts = [clean] if definition.get("type") == "select" else clean.split(",")
+ selected = []
+ for part in parts:
+ candidate = aios_grid._clean_options([part])
+ if len(candidate) != 1:
+ raise err(400, "bad_value", "A choice value is invalid.")
+ value = canonical.get(candidate[0].casefold(), candidate[0])
+ if value not in selected:
+ selected.append(value)
+ clean = selected[0] if definition.get("type") == "select" else ",".join(selected)
+ planned[key] = clean
+ return planned
+
+
+def _plan_database_write(session, args):
+ import routes_tables
+ from core import user_tables
+
+ action = args.get("action")
+ database = str(args.get("database") or "").strip()
+ if action not in {"record_create", "record_update", "record_delete",
+ "field_create", "field_update", "field_delete"} or not database:
+ raise err(400, "tool_arguments_invalid", "Name a database and a published CRUD action.")
+ _assert_database_scope(session, database)
+ _assert_write_scope(session, "database_object")
+ routes_tables._managed_or_refuse(session, database, mutation=True)
+ bucket_before = session.runtime.get(user_tables.STORE_KEY)
+ if not isinstance(bucket_before, dict):
+ raise err(503, "store_unavailable", "The database store is unavailable. Nothing was saved.")
+ table = copy.deepcopy(bucket_before.get(database) or {})
+ if not table or not user_tables.is_user_table(database, st=session.runtime):
+ raise err(403, "connected_source_read_only",
+ "Connected and Odoo-backed database records are read only.")
+ revision = _bucket_revision(session.runtime, user_tables.STORE_KEY, bucket_before)
+ if type(revision.get("rev")) is not int:
+ raise err(503, "confirmation_revision_unavailable",
+ "This deployment cannot revision-guard a database change.")
+ warnings = []
+ bucket_after = copy.deepcopy(bucket_before)
+
+ if action.startswith("record_"):
+ routes_tables._records_or_refuse(session, database)
+ g = routes_tables.ut_write_ctx(session, database)
+ rows = table.get("rows") or {}
+ if action == "record_create":
+ values = _database_value_plan(session, database, None, args.get("values"), g["fields"])
+ record_id = str(max((int(key) for key in rows if str(key).isdigit()), default=0) + 1)
+ before, after = None, values
+ else:
+ record_id = str(args.get("record_id") or "").strip()
+ if not record_id.isdigit() or int(record_id) not in g["pids"]:
+ raise err(403, "record_unavailable", "That record is not available to this account.")
+ before = copy.deepcopy(rows.get(record_id))
+ if before is None:
+ raise err(403, "record_unavailable", "That record is not available to this account.")
+ if action == "record_update":
+ values = _database_value_plan(
+ session, database, int(record_id), args.get("values"), g["fields"])
+ after = {**before, **values}
+ for key, value in values.items():
+ if isinstance(next((field for field in g["fields"]
+ if field.get("key") == key), {}).get("profile"), dict) and not value:
+ for field in table.get("fields") or ():
+ if (field.get("key") in user_tables.PROFILE_PRESET_KEYS
+ and isinstance(field.get("automation"), dict)):
+ after[field["key"]] = ""
+ else:
+ after = None
+ target = bucket_after[database]
+ if action == "record_delete":
+ (target.get("rows") or {}).pop(record_id, None)
+ else:
+ target.setdefault("rows", {})[record_id] = copy.deepcopy(after)
+ # This is part of the UI row writer's durable result, not an optional callback.
+ # Include newly encountered select values in the preview and the one guarded commit.
+ user_tables._hydrate_choice_options(target.get("fields") or [], [after])
+ changed, truncated = _diff_documents({user_tables.STORE_KEY: {
+ "before": bucket_before, "after": bucket_after}})
+ if truncated:
+ warnings.append("The preview diff reached its display cap; split this change.")
+ return {"diff": changed, "affected": {"ids": [record_id], "count": 1},
+ "warnings": warnings, "revision": revision,
+ "bucket_before_digest": _json_digest(bucket_before),
+ "bucket_after": bucket_after,
+ "planned": {"database": database, "action": action, "record_id": record_id}}
+
+ field_key = str(args.get("field_key") or "").strip()
+ visible = routes_tables.ut_write_ctx(session, database)["fields"]
+ recurrence_validator = lambda raw: __import__("routes_grid").govern_date_recurrence(
+ session, database, visible, raw)
+ if action == "field_create":
+ routes_tables._field_or_refuse(session, database)
+ raw = args.get("field")
+ if not isinstance(raw, dict):
+ raise err(400, "tool_arguments_invalid", "field must be an object.")
+ have = list(table.get("fields") or [])
+ field = user_tables._clean_field(
+ raw, recurrence_validator=recurrence_validator,
+ valid_keys={str(item.get("key")) for item in have if item.get("key")})
+ if (not field or any(item.get("key") == field.get("key") for item in have)
+ or len(have) >= user_tables.MAX_FIELDS
+ or not user_tables.dynamic_link_filter_valid(field, have)
+ or not user_tables.flow_bound(field.get("automation"), session.runtime)):
+ raise err(400, "field_refused", "The field does not pass the database's schema rules.")
+ field_key, before, after = field["key"], None, field
+ else:
+ if not field_key:
+ raise err(400, "tool_arguments_invalid", "Name the field to change.")
+ routes_tables._field_or_refuse(session, database, field_key)
+ before = next((copy.deepcopy(field) for field in table.get("fields") or ()
+ if field.get("key") == field_key), None)
+ if before is None:
+ raise err(403, "field_unavailable", "That field is not available to this account.")
+ if action == "field_update":
+ raw = args.get("field")
+ if not isinstance(raw, dict) or "renames" in raw:
+ raise err(400, "field_refused",
+ "External schema updates do not combine choice-value migrations.")
+ merged = {**raw, "key": field_key}
+ after = user_tables._clean_field(
+ merged, before, recurrence_validator=recurrence_validator,
+ valid_keys=({str(item.get("key")) for item in table.get("fields") or ()
+ if item.get("key") != field_key}
+ if "formula" in raw else None))
+ if (not after or not user_tables.dynamic_link_filter_valid(
+ after, table.get("fields") or ())):
+ raise err(400, "field_refused", "The field does not pass the database's schema rules.")
+ else:
+ if (len(table.get("fields") or ()) <= 1 or before.get("deletionLocked") is True
+ or user_tables.delete_field_refusal(
+ database, field_key, session.uname, session.admin, session.runtime,
+ field=before, grant_topic=database) is not None):
+ raise err(403, "field_refused", "That field cannot be removed.")
+ after = None
+ # Build the exact schema document through the core schema mechanics on an isolated runtime.
+ # This includes reciprocal-link cleanup/creation while keeping preview free of relation
+ # refreshes, audit calls, automation events and every other out-of-store side effect.
+ preview = _PreviewRuntime(session.runtime)
+ preview.update(user_tables.STORE_KEY, lambda _cur: copy.deepcopy(bucket_before))
+ if action == "field_create":
+ stored = user_tables.add_field(
+ database, after, st=preview, recurrence_validator=recurrence_validator)
+ if not stored:
+ raise err(400, "field_refused", "The field does not pass the database's schema rules.")
+ synced = user_tables.sync_reciprocal_link(database, field_key, st=preview)
+ if synced.get("refused"):
+ warnings.append(str(synced.get("reason") or "A reciprocal field was not created."))
+ elif action == "field_update":
+ stored = user_tables.patch_field(
+ database, field_key, after, st=preview, recurrence_validator=recurrence_validator)
+ if not stored:
+ raise err(400, "field_refused", "The field does not pass the database's schema rules.")
+ synced = user_tables.sync_reciprocal_link(database, field_key, st=preview)
+ if synced.get("refused"):
+ warnings.append(str(synced.get("reason") or "A reciprocal field was not created."))
+ elif not user_tables.delete_field(database, field_key, st=preview):
+ raise err(403, "field_refused", "That field cannot be removed.")
+ bucket_after = preview.get(user_tables.STORE_KEY)
+ changed, truncated = _diff_documents({user_tables.STORE_KEY: {
+ "before": bucket_before, "after": bucket_after}})
+ if truncated:
+ warnings.append("The preview diff reached its display cap; split this change.")
+ affected_ids = [field_key]
+ for item in changed:
+ for side in (item.get("before"), item.get("after")):
+ candidate = side.get("key") if isinstance(side, dict) else None
+ if candidate and str(candidate) not in affected_ids:
+ affected_ids.append(str(candidate))
+ return {"diff": changed,
+ "affected": {"ids": affected_ids, "count": len(affected_ids)},
+ "warnings": warnings, "revision": revision,
+ "bucket_before_digest": _json_digest(bucket_before),
+ "bucket_after": bucket_after,
+ "planned": {"database": database, "action": action, "field_key": field_key}}
+
+
+def _execute_database_object(session, args, *, plan=None):
+ from core import user_tables, view_audit
+
+ action = args.get("action")
+ database = str(args.get("database") or "").strip()
+ if action == "list":
+ return list_databases(session)
+ if action == "describe":
+ return database_schema(database, session)
+ if action == "query":
+ return _query_database(session, database, args.get("query") or {})
+ plan = plan or _plan_database_write(session, args)
+ expected_rev = plan.get("revision", {}).get("rev")
+ expected_digest = plan.get("bucket_before_digest")
+ replacement = copy.deepcopy(plan.get("bucket_after"))
+ if type(expected_rev) is not int or not isinstance(replacement, dict):
+ raise err(503, "confirmation_revision_unavailable",
+ "This deployment cannot revision-guard a database change.")
+
+ def apply_exact(current):
+ if _json_digest(current) != expected_digest:
+ raise view_audit.ViewRevisionStale(
+ rev_sent=expected_rev, rev_current=expected_rev + 1,
+ message="The database changed after preview.")
+ return copy.deepcopy(replacement)
+
+ try:
+ # One customer bucket, one guarded read-modify-write. The expected revision is checked
+ # inside the PG lock where available; the closure's digest check also protects a backend
+ # that can only provide the runtime's documented guarded fallback.
+ session.runtime.update(
+ user_tables.STORE_KEY, apply_exact, flush="sync", expect_rev=expected_rev)
+ except (view_audit.ViewRevisionStale, view_audit.RevisionUnverifiable) as exc:
+ raise err(409, "confirmation_stale",
+ "The database changed after preview. Preview the change again.") from exc
+ return {"database": database, **copy.deepcopy(plan.get("planned") or {}),
+ "affected": copy.deepcopy(plan.get("affected") or {})}
+
+
+def _sheet_compat_contract():
+ """Load C5's neighboring manifest/dispatcher without defining a second inventory here."""
+ try:
+ module = importlib.import_module("core.sheet_compat")
+ except (ImportError, ModuleNotFoundError):
+ raise sheets.SheetRefusal(
+ "tool_capability_unsupported",
+ "This deployment has not published the Sheet compatibility manifest.")
+ manifest = None
+ for name in ("capability_manifest", "get_capability_manifest", "CAPABILITY_MANIFEST",
+ "CAPABILITIES"):
+ candidate = getattr(module, name, None)
+ if callable(candidate):
+ try:
+ candidate = candidate()
+ except TypeError:
+ continue
+ if isinstance(candidate, (dict, list, tuple)):
+ manifest = copy.deepcopy(candidate)
+ break
+ if manifest is None:
+ raise sheets.SheetRefusal(
+ "tool_capability_unsupported",
+ "This deployment has not published a readable Sheet compatibility manifest.")
+ return module, manifest
+
+
+def _manifest_operation(manifest, object_name, action):
+ """Return an explicitly supported C5 entry; absence and ambiguity both fail closed."""
+ object_name, action = str(object_name).casefold(), str(action).casefold()
+ wanted = {f"{object_name}.{action}", f"{object_name}s.{action}",
+ f"{object_name}/{action}"}
+ matches = []
+
+ def walk(node, path=()):
+ if isinstance(node, dict):
+ explicit_name = node.get("operation") or node.get("name") or node.get("id")
+ if explicit_name is not None:
+ full = str(explicit_name).casefold()
+ if full in wanted or (object_name in full and action in full):
+ matches.append(node)
+ for key, value in node.items():
+ key_text = str(key).casefold()
+ full_path = ".".join((*path, key_text))
+ if (key_text in wanted or full_path in wanted
+ or (object_name in full_path and action in full_path)):
+ if isinstance(value, dict):
+ matches.append(value)
+ elif isinstance(value, bool):
+ matches.append({"supported": value})
+ if key not in {"examples", "fixtures", "notes"}:
+ walk(value, (*path, key_text))
+ elif isinstance(node, (list, tuple)):
+ for item in node:
+ walk(item, path)
+
+ walk(manifest)
+ for entry in matches:
+ if entry.get("supported") is True or str(entry.get("status") or "").casefold() in {
+ "supported", "implemented", "available"}:
+ return entry
+ raise sheets.SheetRefusal(
+ "tool_capability_unsupported",
+ f"The C5 capability manifest does not support {object_name}.{action}.")
+
+
+def _invoke_compat(dispatcher, *, session, object_name, action, arguments, operations=None):
+ supplied = {
+ "session": session, "runtime": session.runtime, "actor": session.uname,
+ "username": session.uname, "object": object_name, "object_name": object_name,
+ "action": action, "operation": f"{object_name}.{action}" if object_name else None,
+ "arguments": copy.deepcopy(arguments), "operations": copy.deepcopy(operations),
+ }
+ try:
+ signature = inspect.signature(dispatcher)
+ if any(param.kind == inspect.Parameter.VAR_KEYWORD
+ for param in signature.parameters.values()):
+ return dispatcher(**supplied)
+ kwargs = {key: value for key, value in supplied.items()
+ if key in signature.parameters}
+ if not kwargs and len(signature.parameters) == 1:
+ return dispatcher(copy.deepcopy(arguments if operations is None else operations))
+ return dispatcher(**kwargs)
+ except sheets.SheetRefusal:
+ raise
+ except (TypeError, ValueError, KeyError, AttributeError) as exc:
+ raise sheets.SheetRefusal(
+ "tool_capability_unsupported",
+ "The published Sheet compatibility dispatcher does not implement this operation.") from exc
+
+
+def _sheet_object(session, args):
+ object_name = str(args.get("object") or "").strip().casefold()
+ action = str(args.get("action") or "").strip()
+ module, manifest = _sheet_compat_contract()
+ _manifest_operation(manifest, object_name, action)
+ dispatcher = next((getattr(module, name, None) for name in (
+ "dispatch_object_operation", "dispatch_operation", "execute_operation", "dispatch")
+ if callable(getattr(module, name, None))), None)
+ if dispatcher is None:
+ raise sheets.SheetRefusal(
+ "tool_capability_unsupported",
+ "The published Sheet capability has no object-operation dispatcher.")
+ return _invoke_compat(
+ dispatcher, session=session, object_name=object_name, action=action,
+ arguments={key: copy.deepcopy(value) for key, value in args.items() if key != "object"})
+
+
+def _validated_batch(session, args):
+ operations = args.get("operations")
+ if not isinstance(operations, list) or not operations:
+ raise err(400, "tool_arguments_invalid", "A validated batch needs at least one operation.")
+ module, manifest = _sheet_compat_contract()
+ normalised = []
+ for item in operations:
+ if not isinstance(item, dict) or set(item) != {"name", "arguments"}:
+ raise err(400, "tool_arguments_invalid", "Every batch item needs name and arguments.")
+ name, arguments = item.get("name"), item.get("arguments")
+ if name not in external_tools._SHEET_OBJECT_ACTIONS or not isinstance(arguments, dict):
+ raise sheets.SheetRefusal(
+ "tool_capability_unsupported", "Validated batches contain only C5 object tools.")
+ object_name = name.removesuffix("_object")
+ action = str(arguments.get("action") or "")
+ _manifest_operation(manifest, object_name, action)
+ normalised.append({"name": name, "arguments": copy.deepcopy(arguments)})
+ dispatcher = next((getattr(module, name, None) for name in (
+ "dispatch_validated_batch", "execute_validated_batch", "dispatch_batch")
+ if callable(getattr(module, name, None))), None)
+ if dispatcher is None:
+ raise sheets.SheetRefusal(
+ "tool_capability_unsupported",
+ "The C5 manifest does not publish an atomic validated-batch dispatcher.")
+ return _invoke_compat(
+ dispatcher, session=session, object_name="", action="", arguments=args,
+ operations=normalised)
+
+
+def _tool_context(session: Session, *, preview=False) -> external_tools.ToolContext:
# Import within the API layer. Core receives callbacks, never an upward dependency.
import routes_sheet
import formula_runtime
@@ -714,10 +2349,16 @@ def _tool_context(session: Session) -> external_tools.ToolContext:
except Exception:
pass
- label = external_access.label_for(session.runtime, session.claims["external_key_id"])
+ if session.claims.get("external_auth_type") == "oauth":
+ label = external_oauth.label_for(
+ session.runtime, session.claims["external_credential_id"])
+ attribution_kind = "OAuth app"
+ else:
+ label = external_access.label_for(session.runtime, session.claims["external_key_id"])
+ attribution_kind = "key"
maker = session.user.get("name") or session.uname
- actor_name = f"{maker} via key {label}"
- if _CREDENTIAL_PREFIX in actor_name:
+ actor_name = f"{maker} via {attribution_kind} {label}"
+ if _contains_credential(actor_name):
raise sheets.SheetRefusal("credential_in_attribution", "Maker names and key labels must not contain credential values.")
return external_tools.ToolContext(
session=session, actor_name=actor_name,
@@ -728,27 +2369,302 @@ def _tool_context(session: Session) -> external_tools.ToolContext:
cache_binding=cache_binding,
list_databases=lambda: list_databases(session),
describe_database=lambda database: database_schema(database, session),
+ query_database=lambda database, payload: _query_database(session, database, payload),
view_event=lambda kind, args: _view_event(session, kind, args),
+ view_share_preview=lambda args: _view_share_preview(session, args),
app_upsert=lambda args: _application_upsert(session, args),
# Reuse the UI's atomic NameConfirmedDelete wrapper as well as the registry delete.
app_delete=lambda args: routes_assets.delete_application(
args["app_id"], {"confirmName": args["confirmName"]}, session),
+ database_object=lambda args: (_plan_database_write(session, args)
+ if preview and external_tools.is_write(
+ "database_object", args)
+ else _execute_database_object(session, args)),
+ sheet_object=lambda args: _sheet_object(session, args),
+ validated_batch=lambda args: _validated_batch(session, args),
+ confirm_change=lambda args: _confirm_change(session, args),
)
+def _strong_confirmation_required(name, args):
+ if name == "database_object":
+ return args.get("action") in {"record_delete", "field_create", "field_update", "field_delete"}
+ if name in {"view_delete", "app_delete"}:
+ return True
+ if name == "sheet_write_cells":
+ return any(isinstance(cell, dict) and cell.get("kind") == "empty"
+ for cell in (args.get("cells") or {}).values())
+ if name == "view_object":
+ return args.get("action") == "delete"
+ if name == "sheet_structure":
+ return int(((args.get("op") or {}).get("delta") or 0)) < 0
+ if name == "worksheet_object":
+ return args.get("action") == "delete"
+ if name == "range_object":
+ return args.get("action") == "clear"
+ if name == "validated_batch":
+ return any(_strong_confirmation_required(
+ item.get("name"), item.get("arguments") or {})
+ for item in args.get("operations") or () if isinstance(item, dict))
+ return False
+
+
+def _affected_for(name, args, diff, result=None):
+ values = []
+ for key in ("database", "record_id", "field_key", "view_id", "app_id",
+ "workbook_id", "worksheet", "tab", "range", "addr"):
+ value = args.get(key)
+ if value is not None and str(value) not in values:
+ values.append(str(value))
+ if isinstance(result, dict):
+ for key in ("rid", "pid", "id", "view_id", "app_id", "chart_id"):
+ value = result.get(key)
+ if value is not None and str(value) not in values:
+ values.append(str(value))
+ return {"ids": values[:100], "count": max(len(values), len(diff))}
+
+
+def _preview_external_write(session, name, arguments, *, issue_token=True, execution=None):
+ from core import write_execution
+ if not external_tools.is_write(name, arguments):
+ raise err(400, "confirmation_not_write", "Only a write operation can be previewed.")
+ _assert_write_scope(session, name)
+ normalised = json.loads(_canonical_json(arguments).decode("utf-8"))
+ for database in _databases_in_arguments(normalised):
+ _assert_database_scope(session, database)
+
+ preview_runtime = _PreviewRuntime(session.runtime)
+ preview_session = Session(
+ tenant=session.tenant, user=copy.deepcopy(session.user),
+ claims=copy.deepcopy(session.claims), runtime=preview_runtime)
+ execution = execution or {'seed': uuid.uuid4().hex, 'timestamp': time.time()}
+ with write_execution.replay(execution):
+ result = external_tools.dispatch(_tool_context(preview_session, preview=True), name, normalised)
+ if isinstance(result, dict) and result.get("refused"):
+ raise err(409, "write_partial_refused",
+ "The writer refused part of this change; nothing was previewed.")
+
+ plan = result if name == "database_object" else None
+ if plan is not None:
+ diff = copy.deepcopy(plan.get("diff") or [])
+ warnings = list(plan.get("warnings") or [])
+ affected = copy.deepcopy(plan.get("affected") or {})
+ current_revision = copy.deepcopy(plan.get("revision") or {})
+ else:
+ documents = preview_runtime.changed()
+ diff, truncated = _diff_documents(documents)
+ warnings = (["The preview diff reached its display cap; split this change."]
+ if truncated else [])
+ affected = _affected_for(name, normalised, diff, result)
+ current_revision = {
+ key: revision for key, revision in preview_runtime.base_revisions().items()
+ if key in documents
+ }
+ if not diff:
+ raise err(409, "write_not_changed", "The requested write would not change anything.")
+ if any("display cap" in warning for warning in warnings):
+ raise err(413, "confirmation_too_large",
+ "The complete diff does not fit in one confirmation. Split the change.")
+
+ diff_digest = _json_digest(diff)
+ revision_digest = _json_digest(current_revision)
+ expires_at = int(time.time()) + _CONFIRMATION_TTL_SECONDS
+ payload = {
+ "v": 1, "tenant": session.tenant, "actor": session.uname,
+ "credential": _credential_binding(session),
+ "authentication": session.claims.get("external_auth_type", "api_key"),
+ "tool": name, "arguments": normalised,
+ "arguments_digest": _json_digest(normalised),
+ "revision_digest": revision_digest, "diff_digest": diff_digest,
+ "expiry": expires_at, "nonce": uuid.uuid4().hex,
+ "strong": _strong_confirmation_required(name, normalised),
+ "affected_count": int(affected.get("count") or 0),
+ "execution": execution,
+ }
+ response = {
+ "preview": True, "mutated": False, "tool": name,
+ "diff": diff, "diff_digest": diff_digest,
+ "affected": affected, "warnings": warnings,
+ "current_revision": current_revision,
+ "expires_at": expires_at, "requires_confirmation": True,
+ "strong_confirmation_required": payload["strong"],
+ }
+ if payload["strong"]:
+ response["confirmation_phrase"] = _CONFIRMATION_PHRASE
+ if issue_token:
+ response["confirmation_token"] = _confirmation_token(payload)
+ response["_confirmation"] = payload
+ response["_plan"] = plan
+ return response
+
+
+def _confirmation_claim(session, payload):
+ nonce = str(payload.get("nonce") or "")
+ if not nonce:
+ raise err(400, "confirmation_token_invalid", "The confirmation token is invalid.")
+ nonce_key = hashlib.sha256(nonce.encode("utf-8")).hexdigest()
+
+ def claim(raw):
+ doc = raw if isinstance(raw, dict) else {}
+ claims = doc.get("claims") if isinstance(doc.get("claims"), dict) else {}
+ now = int(time.time())
+ claims = {key: value for key, value in claims.items()
+ if isinstance(value, dict) and int(value.get("expiry") or 0) >= now - 3600}
+ if nonce_key in claims:
+ raise err(409, "confirmation_replayed",
+ "This confirmation token has already been used.")
+ claims[nonce_key] = {
+ "status": "claimed", "expiry": int(payload["expiry"]),
+ "tool": str(payload["tool"]), "actor_digest": _json_digest(payload["actor"]),
+ "diff_digest": str(payload["diff_digest"]),
+ }
+ doc["claims"] = claims
+ return doc
+
+ session.runtime.update(_CONFIRMATION_BUCKET, claim, flush="sync")
+ return nonce_key
+
+
+def _confirmation_already_used(session, payload):
+ nonce = str(payload.get("nonce") or "")
+ if not nonce:
+ raise err(400, "confirmation_token_invalid", "The confirmation token is invalid.")
+ nonce_key = hashlib.sha256(nonce.encode("utf-8")).hexdigest()
+ doc = session.runtime.get(_CONFIRMATION_BUCKET)
+ claims = doc.get("claims") if isinstance(doc, dict) and isinstance(doc.get("claims"), dict) else {}
+ if nonce_key in claims:
+ raise err(409, "confirmation_replayed", "This confirmation token has already been used.")
+
+
+def _confirmation_status(session, nonce_key, status):
+ def mark(raw):
+ doc = raw if isinstance(raw, dict) else {}
+ claims = doc.get("claims") if isinstance(doc.get("claims"), dict) else {}
+ item = claims.get(nonce_key)
+ if isinstance(item, dict):
+ claims[nonce_key] = {**item, "status": status}
+ doc["claims"] = claims
+ return doc
+ try:
+ session.runtime.update(_CONFIRMATION_BUCKET, mark, flush="sync")
+ except Exception:
+ pass
+
+
+def _confirm_change(session, args):
+ from core import view_audit, write_execution
+
+ payload = _read_confirmation_token(args.get("confirmation_token"))
+ now = int(time.time())
+ if int(payload.get("expiry") or 0) <= now:
+ raise err(409, "confirmation_expired",
+ "This confirmation token expired. Preview the change again.")
+ bindings = {
+ "tenant": session.tenant, "actor": session.uname,
+ "credential": _credential_binding(session),
+ "authentication": session.claims.get("external_auth_type", "api_key"),
+ }
+ if any(payload.get(key) != value for key, value in bindings.items()):
+ raise err(403, "confirmation_principal_changed",
+ "This confirmation belongs to another principal or credential.")
+ if payload.get("strong") and args.get("confirmation_phrase") != _CONFIRMATION_PHRASE:
+ raise err(400, "confirmation_phrase_required",
+ "Enter the exact stronger confirmation phrase shown in the preview.")
+ _confirmation_already_used(session, payload)
+
+ # Freshly resolve the account, Analyst grant, credential revocation/label, credential scope,
+ # database walls and writer semantics before claiming the nonce or touching customer state.
+ fresh_user = external_access.resolve_active_user(session.uname, session.tenant)
+ fresh_session = Session(session.tenant, fresh_user, copy.deepcopy(session.claims), session.runtime)
+ require_external_analyst(fresh_session)
+ _tool_context(fresh_session) # credential label lookup is the live revocation check
+ tool, arguments = payload.get("tool"), payload.get("arguments")
+ if not isinstance(tool, str) or tool == "confirm_change" or not isinstance(arguments, dict):
+ raise err(400, "confirmation_token_invalid", "The confirmation token is invalid.")
+ execution = payload.get('execution')
+ if (not isinstance(execution, dict) or not isinstance(execution.get('seed'), str)
+ or not execution['seed'] or type(execution.get('timestamp')) not in (int, float)):
+ raise err(409, 'confirmation_stale',
+ 'This preview predates the current writer. Preview the change again.')
+ recomputed = _preview_external_write(
+ fresh_session, tool, arguments, issue_token=False, execution=execution)
+ current = recomputed["_confirmation"]
+ if (payload.get("arguments_digest") != current.get("arguments_digest")
+ or payload.get("revision_digest") != current.get("revision_digest")
+ or payload.get("diff_digest") != current.get("diff_digest")):
+ raise err(409, "confirmation_stale",
+ "The data, permissions, or resulting diff changed. Preview the change again.")
+
+ nonce_key = _confirmation_claim(fresh_session, payload)
+ try:
+ if tool == "database_object":
+ result = _execute_database_object(
+ fresh_session, arguments, plan=recomputed.get("_plan"))
+ else:
+ guarded_session = Session(
+ fresh_session.tenant, fresh_session.user, fresh_session.claims,
+ _ConfirmationRuntime(
+ fresh_session.runtime, recomputed.get("current_revision")))
+ with write_execution.replay(execution):
+ result = external_tools.dispatch(_tool_context(guarded_session), tool, arguments)
+ if isinstance(result, dict) and result.get("refused"):
+ raise err(409, "write_partial_refused",
+ "The writer refused part of the confirmed change.")
+ except (view_audit.ViewRevisionStale, view_audit.RevisionUnverifiable) as exc:
+ _confirmation_status(fresh_session, nonce_key, "failed")
+ raise err(409, "confirmation_stale",
+ "The data changed while confirmation was committing. Preview it again.") from exc
+ except Exception:
+ _confirmation_status(fresh_session, nonce_key, "failed")
+ raise
+ _confirmation_status(fresh_session, nonce_key, "committed")
+ before_revision = copy.deepcopy(recomputed.get("current_revision") or {})
+ if tool == "database_object":
+ from core import user_tables
+ after_revision = _bucket_revision(
+ fresh_session.runtime, user_tables.STORE_KEY,
+ fresh_session.runtime.get(user_tables.STORE_KEY))
+ else:
+ after_revision = {}
+ for key in before_revision if isinstance(before_revision, dict) else ():
+ try:
+ after_revision[key] = copy.deepcopy(fresh_session.runtime.revision(key))
+ except Exception:
+ after_revision[key] = None
+ after_revision_digest = _json_digest(after_revision)
+ receipt = audit.external_write_receipt(
+ fresh_session.tenant, fresh_session.uname, tool=tool,
+ auth_type=payload.get("authentication"),
+ affected_count=payload.get("affected_count"),
+ diff_digest=payload.get("diff_digest"),
+ revision_before=payload.get("revision_digest"), revision_after=after_revision_digest,
+ confirmation_id=nonce_key)
+ return {
+ "confirmed": True, "applied": True, "tool": tool,
+ "affected": copy.deepcopy(recomputed.get("affected") or {}),
+ "revision_before": before_revision,
+ "revision_after": after_revision, "audit_receipt": receipt,
+ }
+
+
def _tool_refusal(name, code, message, status=400, **extra):
# Never echo arbitrary method/tool names or raw exception text across the bearer seam.
tool_name = name if isinstance(name, str) and name in external_tools.TOOLS else "tools/call"
sentence = f"{tool_name}: {_markdown_inline(message)}"
- if _CREDENTIAL_PREFIX in sentence:
+ if _contains_credential(sentence):
sentence = f"{tool_name}: The result contained a credential value and was withheld."
return {"ok": False, "error": {"tool": tool_name, "code": code,
"message": sentence, "status": status, **extra}}
def _mcp_content(rid, out):
- encoded = json.dumps(out, default=str, ensure_ascii=True, allow_nan=False)
- if _CREDENTIAL_PREFIX in encoded:
+ try:
+ encoded = json.dumps(out, default=str, ensure_ascii=True, allow_nan=False)
+ except (TypeError, ValueError, OverflowError):
+ out = _tool_refusal(None, "tool_output_invalid",
+ "The result could not be represented safely.", 422)
+ encoded = json.dumps(out)
+ if _contains_credential(encoded):
out = _tool_refusal(None, "credential_in_output", "The result contained a credential value and was withheld.")
encoded = json.dumps(out)
return {"jsonrpc": "2.0", "id": rid, "result": {
@@ -763,7 +2679,7 @@ def _handle_mcp(req, session: Session):
if not isinstance(req, dict):
return protocol_error(None, -32600, "An MCP request must be a JSON-RPC object.")
rid, method = req.get("id"), req.get("method")
- if (rid is not None and (type(rid) not in (str, int) or _CREDENTIAL_PREFIX in str(rid))):
+ if (rid is not None and (type(rid) not in (str, int) or _contains_credential(rid))):
return protocol_error(None, -32600, "The request id must be a number or a safe string.")
if rid is None:
# Notifications never dispatch writes and never return a JSON-RPC body.
@@ -780,7 +2696,7 @@ def _handle_mcp(req, session: Session):
require_external_analyst(session)
if method == "initialize":
protocol = params.get("protocolVersion") or "2025-06-18"
- if not isinstance(protocol, str) or _CREDENTIAL_PREFIX in protocol:
+ if not isinstance(protocol, str) or _contains_credential(protocol):
return protocol_error(rid, -32602, "The protocol version must be a safe string.")
return {"jsonrpc": "2.0", "id": rid, "result": {
"protocolVersion": protocol, "capabilities": {"tools": {}},
@@ -793,11 +2709,18 @@ def _handle_mcp(req, session: Session):
for key, tool in external_tools.TOOLS.items()]}}
if method != "tools/call":
return protocol_error(rid, -32601, "The requested MCP method was not found.")
- if isinstance(name, str) and name in _WRITE_TOOLS:
- _assert_write_scope(session, name)
- for database in _databases_in_arguments(params.get("arguments", {})):
+ arguments = params.get("arguments", {})
+ for database in _databases_in_arguments(arguments):
_assert_database_scope(session, database)
- data = external_tools.dispatch(_tool_context(session), name, params.get("arguments", {}))
+ if name == "confirm_change":
+ data = external_tools.dispatch(_tool_context(session), name, arguments)
+ elif external_tools.is_write(name, arguments):
+ _assert_write_scope(session, name)
+ data = _preview_external_write(session, name, arguments)
+ data.pop("_confirmation", None)
+ data.pop("_plan", None)
+ else:
+ data = external_tools.dispatch(_tool_context(session), name, arguments)
if isinstance(data, dict) and data.get("refused"):
first = data["refused"][0]
out = _tool_refusal(name, first["code"], first["sentence"])
@@ -847,6 +2770,8 @@ _WRITE_TOOLS = frozenset({
"sheet_add_rule", "sheet_set_chart", "sheet_structure",
"sheet_save_template",
"view_upsert", "view_delete", "app_upsert", "app_delete",
+ "database_object", "view_object", "workbook_object", "worksheet_object",
+ "range_object", "calculation_object", "validated_batch",
})
@@ -860,3 +2785,37 @@ def _assert_write_allow_list() -> None:
_assert_write_allow_list()
+
+
+# RFC 8414 and RFC 9728 insert `.well-known` before a path-bearing issuer/resource. Keep the
+# path-local aliases above for compatibility, while the router mounted by `main.py` also serves
+# the standards-derived locations followed by generic OAuth and MCP clients. The auxiliary
+# router's already-built routes are appended directly so the established external router prefix
+# remains unchanged for every existing caller and invariant.
+_discovery_router = APIRouter(tags=["external"])
+
+
+@_discovery_router.get("/.well-known/oauth-authorization-server/api/v1/external")
+def oauth_server_metadata_rfc(request: Request):
+ return oauth_server_metadata(request)
+
+
+@_discovery_router.get("/.well-known/oauth-protected-resource/api/v1/external/mcp")
+def oauth_resource_metadata_rfc(request: Request):
+ return oauth_resource_metadata(request)
+
+
+# The PUBLIC skill.md at the site root, like Monid's `monid.ai/SKILL.md`: an agent told
+# "set up /skill.md" reads it with no session. It is the session-free document, so it
+# carries no tenant, database or field name; `/api/v1/external/skill.md` keeps the per-user one.
+@_discovery_router.get("/skill.md")
+@_discovery_router.get("/SKILL.md")
+def public_skill(request: Request):
+ return Response(
+ content=_portable_skill_document(None, str(request.base_url)),
+ media_type="text/markdown; charset=utf-8",
+ headers={"Cache-Control": "public, max-age=300"},
+ )
+
+
+router.routes.extend(_discovery_router.routes)
diff --git a/api/routes_geo.py b/api/routes_geo.py
index 44f57ffd26f3a8a16425dea353ca7b40814f42d9..a0f00c7ac995e12dce4209931f72e8db0317a38b 100644
--- a/api/routes_geo.py
+++ b/api/routes_geo.py
@@ -998,6 +998,53 @@ def _route_endpoint_index(pids, raw, label):
return matches[0]
+def _route_location_partition(body, located_pids):
+ """Validate C1's explicit client partition without trusting labels as coordinates.
+
+ Older map callers submit only ``stops`` and keep their established response shape. RouteNav
+ submits both keys together; once either is present the pair is atomic and every pid must occur
+ exactly once across the two halves.
+ """
+ has_located = "locatedPids" in body
+ has_unlocated = "unlocated" in body
+ if not has_located and not has_unlocated:
+ return list(located_pids), []
+ if not has_located or not has_unlocated:
+ raise err(400, "invalid_route_partition",
+ "Route locations must identify both located and unlocated stops.")
+ raw_located, raw_unlocated = body.get("locatedPids"), body.get("unlocated")
+ if not isinstance(raw_located, list) or not isinstance(raw_unlocated, list):
+ raise err(400, "invalid_route_partition",
+ "Route locations must be lists of located and unlocated stops.")
+ try:
+ claimed_located = [int(pid) for pid in raw_located]
+ except (TypeError, ValueError):
+ raise err(400, "invalid_route_partition", "Every located Route stop needs a record id.")
+ if claimed_located != list(located_pids) or len(set(claimed_located)) != len(claimed_located):
+ raise err(400, "invalid_route_partition",
+ "The located Route stop ids must match the submitted coordinates in order.")
+ cleaned, unlocated_ids = [], set()
+ for item in raw_unlocated:
+ if not isinstance(item, dict):
+ raise err(400, "invalid_route_partition",
+ "Every unlocated Route stop needs a record id, label, and reason.")
+ try:
+ pid = int(item.get("pid"))
+ except (TypeError, ValueError):
+ raise err(400, "invalid_route_partition", "Every unlocated Route stop needs a record id.")
+ label = " ".join(str(item.get("label") or "").split())[:160]
+ reason = " ".join(str(item.get("reason") or "").split())[:240]
+ if pid in unlocated_ids or pid in claimed_located or not label or not reason:
+ raise err(400, "invalid_route_partition",
+ "Every Route stop must occur once with a non-empty label and location reason.")
+ unlocated_ids.add(pid)
+ cleaned.append({"pid": pid, "label": label, "reason": reason})
+ if len(cleaned) > MAX_ROUTE_STOPS:
+ raise err(400, "too_many_stops",
+ f"Identify at most {MAX_ROUTE_STOPS} unlocated Route stops at once.")
+ return claimed_located, cleaned
+
+
@router.post("/geo/route")
def geo_route(body: dict = Body(...), session: Session = Depends(require_session)):
"""Road distance, duration and the drawn line, for the order given AND for a better one.
@@ -1015,6 +1062,8 @@ def geo_route(body: dict = Body(...), session: Session = Depends(require_session
Request body:
`stops` [{lat, lon, pid?}] the order to measure, at least two.
+ `locatedPids` [pid] matching ``stops`` in order when RouteNav submits C1's partition.
+ `unlocated` [{pid,label,reason}] named fixed anchors excluded from both measurements.
`roundTrip` bool. The server closes the measured route at its selected start. A legacy
closing duplicate is accepted too and never becomes a visitable stop.
`startPid` optional selected start among the submitted `pid` values.
@@ -1067,6 +1116,7 @@ def geo_route(body: dict = Body(...), session: Session = Depends(require_session
round_trip = bool(body.get("roundTrip")) or has_closing_stop
core = points[:-1] if has_closing_stop else points
core_pids = pids[:-1] if has_closing_stop else pids
+ _claimed_located, unlocated = _route_location_partition(body, core_pids)
start_index = _route_endpoint_index(core_pids, body.get("startPid"), "start")
end_index = _route_endpoint_index(core_pids, body.get("endPid"), "end")
if round_trip and end_index is not None:
@@ -1139,6 +1189,8 @@ def geo_route(body: dict = Body(...), session: Session = Depends(require_session
"attribution": credit,
# ---- C2, for the order as submitted
"ordered": core_pids,
+ "locatedPids": core_pids,
+ "unlocated": unlocated,
"meters": current["meters"],
"seconds": current["seconds"],
"source": current["source"],
diff --git a/api/routes_module_index.py b/api/routes_module_index.py
index fe9ea335582f43f40d172eba42c92c634275fd64..5618005eceb358f2e1e20dbf0b58140afd87e7f3 100644
--- a/api/routes_module_index.py
+++ b/api/routes_module_index.py
@@ -65,10 +65,28 @@ def _readable_ids(module, session):
voice._store(session.runtime)["voice_agent"].values() if isinstance(row, dict)})
+def _collapsed_groups(raw):
+ """The module-index document owns per-user App chrome disclosure preferences.
+
+ Finance is the only registry group mounted in chrome today. Refuse unknown identifiers rather
+ than turning this preference bucket into unbounded client storage.
+ """
+ if not isinstance(raw, list) or any(not isinstance(value, str) for value in raw):
+ raise SheetFolderRefusal("module_index_invalid", "collapsedGroups must be a list of supported group names.")
+ if any(value != "finance" for value in raw):
+ raise SheetFolderRefusal("module_index_invalid", "collapsedGroups names an unsupported App group.")
+ return list(dict.fromkeys(raw))
+
+
def _project(bucket, ids):
folders, placed, _ = validate_folder_tree(bucket.get("folders"), bucket.get("itemFolders"))
- return {"folders": folders, "itemFolders": {k: v for k, v in placed.items() if k in ids},
- "rev": bucket.get("rev", 0)}
+ projected = {"folders": folders, "itemFolders": {k: v for k, v in placed.items() if k in ids},
+ "rev": bucket.get("rev", 0)}
+ # Additive and presence-preserving: old buckets retain their byte-for-byte response shape,
+ # while a user who explicitly opens Finance can persist an empty list as a real preference.
+ if "collapsedGroups" in bucket:
+ projected["collapsedGroups"] = _collapsed_groups(bucket.get("collapsedGroups"))
+ return projected
def _available(session):
@@ -111,6 +129,7 @@ def write_module_index(module: str, body: dict = Body(default=None), user: str =
raise SheetFolderRefusal("module_index_item_forbidden",
"A folder placement names an item you cannot read in this module.")
folders, placed, _ = validate_folder_tree(raw["folders"], placements)
+ collapsed = _collapsed_groups(raw.get("collapsedGroups")) if "collapsedGroups" in raw else None
saved = {}
def apply(current):
@@ -122,6 +141,10 @@ def write_module_index(module: str, body: dict = Body(default=None), user: str =
raise SheetFolderRefusal("index_revision_stale",
"Folders changed somewhere else. Reload folders and try again.")
saved.update(folders=folders, itemFolders=placed, rev=rev + 1)
+ if collapsed is not None:
+ saved["collapsedGroups"] = collapsed
+ elif "collapsedGroups" in (users.get(owner) or {}):
+ saved["collapsedGroups"] = _collapsed_groups(users[owner].get("collapsedGroups"))
users[owner] = dict(saved)
state["schema"] = 1
return state
diff --git a/api/routes_odoo_tables.py b/api/routes_odoo_tables.py
index b5bd776e3ced4bff1200670c8dea7385c352c018..b6a96f94af25e558df0c2e79ef91ad9ad06a841e 100644
--- a/api/routes_odoo_tables.py
+++ b/api/routes_odoo_tables.py
@@ -22,13 +22,15 @@ import base64
import hashlib
import hmac
import json
+import math
import time
from contextlib import contextmanager
from fastapi import Body, Depends, HTTPException
from fastapi import APIRouter, Query
-from deps import (Session, err, module_cache_rows, module_cache_source, require_session)
+from deps import (Session, err, module_cache_rows, module_cache_source,
+ require_session, running_main)
router = APIRouter(prefix="/api/v1")
@@ -120,13 +122,14 @@ def refresh_odoo_tables(session: Session = Depends(require_session)):
# section and its negative control.
if not session.admin:
raise err(403, "forbidden", "Refreshing the Odoo databases is an admin action.")
- if not rel.is_royal(session.tenant):
+ if not rel.available(session.runtime) and not rel.frozen(session.runtime):
# A plain 400 with the reason: this tenant has no Odoo mirror, and spawning two empty
# locked databases it can never fill would be worse than refusing.
raise err(400, "not_applicable",
- "These databases are built from the Royal Imports Odoo mirror (R1).")
+ "Connect an Odoo source before refreshing its databases.")
try:
- out = rel.refresh(session.runtime, session.tenant, username=session.user)
+ out = rel.refresh(session.runtime, session.tenant, username=session.uname,
+ all_connections=True)
# ⭐⭐ W30-T31 — AND THEN THE ROWS THAT SHOULD NOT BE HERE LEAVE AGAIN.
#
# The spawn writes every bucket's rows straight into the tenant document by mutating it
@@ -209,7 +212,19 @@ def odoo_tables_status(brief: bool = Query(False),
# while those were the only two; the 2026-08-09 widening added products and orders, and a
# hard-coded list here would have reported "everything is fine" over two databases it had
# stopped knowing about ([[gate-answers-the-wrong-question]] — the hard-coded count).
- for _bucket, key, _label, _fields in rel.TABLES:
+ declared = list(rel.TABLES)
+ declared_keys = {key for _bucket, key, _label, _fields in declared}
+ # Include every namespaced/discovered Odoo database already present in this tenant. The old
+ # compiled-only loop made connection two invisible and therefore falsely healthy by omission.
+ for key, table in tables.items():
+ connector = table.get("connector") if isinstance(table, dict) else None
+ if (key not in declared_keys and isinstance(connector, dict)
+ and connector.get("key") == "odoo"):
+ declared.append(("stored", key, str(table.get("label") or key), lambda: ()))
+ declared_keys.add(key)
+
+ health_cache = {}
+ for _bucket, key, _label, _fields in declared:
table = tables.get(key) or {}
rows = (table.get("rows") or {})
stamps = [str((r or {}).get("refreshed") or "") for r in rows.values()]
@@ -220,6 +235,8 @@ def odoo_tables_status(brief: bool = Query(False),
# says a worklist that quietly stopped updating must be legible without a refresh. So a
# read-through table is counted from the MIRROR and says where its count came from.
materialised = user_tables.materialises(key, defn=table)
+ spec = None
+ mirror_refreshed = ""
if not materialised:
try:
rows = {}
@@ -233,11 +250,36 @@ def odoo_tables_status(brief: bool = Query(False),
order_by=spec["id"], limit=1, cur=cur, **frm)["total"]
else:
n = -1 # unknown, and never reported as zero
+ mirror_refreshed = mirror_stamp(
+ spec or {}, cur=cur, table_key=key, rt=session.runtime,
+ defn=(table if isinstance(table, dict) and table else None))
except Exception: # noqa: BLE001
n = -1
+ connection_id = str(((table.get("connector") or {}).get("connectionId")
+ if isinstance(table, dict) else "") or "")
+ if not connection_id:
+ try:
+ connection_id = str(rel.connection_for_table_key(key, session.runtime) or "default")
+ except Exception: # noqa: BLE001
+ connection_id = "default"
+ if connection_id not in health_cache:
+ try:
+ import routes_connectors as _connectors
+ receipt = _connectors.odoo_health_receipt(session.runtime, connection_id)
+ health_cache[connection_id] = {
+ str(item.get("databaseKey") or ""): item
+ for item in (receipt.get("databases") or ())}
+ except Exception: # noqa: BLE001
+ health_cache[connection_id] = {}
+ db_health = health_cache[connection_id].get(key) or {
+ "databaseKey": key, "state": "unknown", "lastSuccess": None,
+ "currentLagSeconds": None, "failure": None, "stages": {}}
+ refreshed = (max([s for s in stamps if s], default="") if materialised
+ else mirror_refreshed)
out[key] = {
"exists": bool(table),
"rows": (len(rows) if materialised else n),
+ "rowCount": (len(rows) if materialised else n),
# ⭐ W31-T45: a refused mirror says `refused`, never `mirror` — a size that came from
# nowhere must not be labelled with the source it did not come from.
"rowsFrom": ("document" if materialised
@@ -245,8 +287,14 @@ def odoo_tables_status(brief: bool = Query(False),
"materialised": materialised,
# why this grid still keeps its rows in the tenant document, when it does not have to
"materialisedBecause": (why_not.get(key, "") if materialised else ""),
- "refreshed": (max([s for s in stamps if s], default="") if materialised
- else "live, read through the mirror"),
+ "refreshed": refreshed,
+ # The row-level receipt carries an exact instant; mirror metadata may be date-only.
+ "recordRefreshed": db_health.get("recordRefreshed") or refreshed,
+ "lastSuccess": db_health.get("lastSuccess"),
+ "currentLagSeconds": db_health.get("currentLagSeconds"),
+ "failure": db_health.get("failure"),
+ "stages": db_health.get("stages") or {},
+ "connectionId": connection_id,
"locked": table.get("recordMode") == user_tables.AUTOMATION_RECORD_MODE,
# W30/R7: whether this database is served THROUGH the mirror rather than from the copy
# above. Reported per table, because they convert one at a time and an operator
@@ -316,8 +364,8 @@ def _seed_state():
from harness import datastore as _ds
block["present"] = bool(_ds.DB_PATH.exists())
if not block["present"]:
- import main as _main
- state = dict(_main.MIRROR_SEED)
+ _main = running_main() # D-781: resolve, never import
+ state = dict(getattr(_main, "MIRROR_SEED", None) or {})
block["manifest"] = "absent"
block["cause"] = state.get("cause") or "this container has no analytical mirror yet"
block["recommendation"] = state.get("recommendation") or (
@@ -456,10 +504,11 @@ def _resync_state(cur):
block = {"armed": False, "tenant": "", "tenantSource": "", "tenantSeconds": None,
"envSeconds": None, "resolvedSeconds": None, "manual": False,
"disagreement": None, "passes": 0, "lastPassAt": None, "lastPassOk": None,
- "lastError": None, "bootSyncError": None, "entities": None,
+ "lastSuccessAt": None, "lastError": None, "bootSyncError": None,
+ "connectionId": "", "entities": None,
"entitiesCause": "", "stuck": []}
try:
- import main as _main
+ _main = running_main() # D-781: resolve, never import
block.update({k: v for k, v in dict(_main.RESYNC_STATE).items() if k in block})
cadence = block.get("resolvedSeconds") or _main._RESYNC_DEFAULT_SECONDS
except Exception: # noqa: BLE001
@@ -545,6 +594,7 @@ GRID_SOURCES = {}
#: ⚠ A provider registers a CALLABLE, not a dict, so its module is not imported until the first
#: read — the same lazy rule `_sources` already follows for `odoo_relational`.
_SOURCE_PROVIDERS = []
+_SOURCE_ANALYSES = {}
def register_source_provider(fn):
@@ -558,6 +608,25 @@ def register_source_provider(fn):
return len(_SOURCE_PROVIDERS)
+def register_source_analysis(source_key, *, folds, spawned):
+ """Register lazy `() -> {table_key: reason}` and `() -> set[table_key]` analyses.
+
+ Source ids come from core.registry's policies. Re-registering replaces that source's pair
+ without adding another entry. Returns the number of sources whose analysis is now known.
+ """
+ if callable(folds) and callable(spawned):
+ _SOURCE_ANALYSES[source_key] = {"folds": folds, "spawned": spawned}
+ return len(_SOURCE_ANALYSES)
+
+
+def _source_analysis(source_key):
+ """Resolve one source's analysis lazily; an unregistered source has two empty results."""
+ analysis = _SOURCE_ANALYSES.get(source_key)
+ if analysis is None:
+ return {}, set()
+ return analysis["folds"](), analysis["spawned"]()
+
+
def _sources():
"""Build `GRID_SOURCES` lazily by asking EVERY registered provider — Odoo is one of them.
@@ -929,10 +998,24 @@ def _odoo_sources():
return GRID_SOURCES
+def _odoo_spawned():
+ """The Odoo grains that have a materialised row builder."""
+ rel = _rel()
+ # ⛔ "WAS THIS PART OF THE MATERIALISED SPAWN?", and being in `TABLES` STOPPED ANSWERING IT.
+ # W30-T35 declared the two line grains, which have no python row builder anywhere — their
+ # absence from `_READERS` IS that statement — so a table the spawn could never materialise
+ # started reading as spawned, and on a mirror-less deployment fell through to "its size could
+ # not be read" and came back INELIGIBLE. That inverts eligibility's first law (size
+ # forces read-through; it never blocks it). Ask the question through the reader map, which is
+ # what actually decides whether a row could ever have been built.
+ return {key for _b, key, _l, _f in rel.TABLES if _b in getattr(rel, "_READERS", {})}
+
+
#: Odoo registers itself, at import, exactly as any other connector does. ⚠ The order providers
#: are registered in is the order their specs land; keys are namespaced by connector (`ut_odoo_`,
#: `ut_meta_`), so a later provider cannot shadow an earlier one's grid.
register_source_provider(_odoo_sources)
+register_source_analysis("odoo", folds=lambda: _fold_reasons(_rel()), spawned=_odoo_spawned)
def sync_read_through(cur=None):
@@ -942,7 +1025,7 @@ def sync_read_through(cur=None):
registration is additive and `strip_materialised` is a no-op once a table is empty.
⛔ IT IS ALSO THE ONLY PLACE THAT MAY CALL `register_read_through`, because the eligibility
- question needs BOTH halves — `odoo_relational`'s field declarations (for the fold matrix) and
+ question needs BOTH halves — the source's registered analysis (for the fold matrix) and
the mirror (for the size) — and `core` may import neither.
"""
eligible, reasons = _unmaterialisable(cur)
@@ -1034,7 +1117,7 @@ def _unmaterialisable(cur=None):
⛔ SIZE FORCES READ-THROUGH; IT NEVER BLOCKS IT — and getting that backwards was a real bug in
the first cut of this function. A table too big for the tenant document has NO materialised
- option at all, so making it ineligible would have handed `odoo_relational.plan` a `row_limit`
+ option at all, so making it ineligible would have handed the materialisation plan a `row_limit`
of None and invited it to build 963,783 python dicts. The window ceiling is a different, much
softer thing: it only limits what the whole-table CLIENT door can serve today.
@@ -1046,19 +1129,42 @@ def _unmaterialisable(cur=None):
⚠ Without a mirror cursor the size questions cannot be asked, so only a grid that was never
part of the materialised spawn is eligible — a table is never freed by a question we skipped.
"""
- rel = _rel()
- folds, eligible, reasons = _fold_reasons(rel), {}, {}
- # ⛔ "WAS THIS PART OF THE MATERIALISED SPAWN?", and being in `TABLES` STOPPED ANSWERING IT.
- # W30-T35 declared the two line grains, which have no python row builder anywhere — their
- # absence from `_READERS` IS that statement — so a table the spawn could never materialise
- # started reading as spawned, and on a mirror-less deployment fell through to "its size could
- # not be read" and came back INELIGIBLE. That inverts this function's own first law (size
- # forces read-through; it never blocks it). Ask the question through the reader map, which is
- # what actually decides whether a row could ever have been built.
- spawned = {key for _b, key, _l, _f in rel.TABLES if _b in getattr(rel, "_READERS", {})}
+ from core import registry
from harness import datastore
ut = _ut()
- for key, spec in _sources().items():
+ analyses, eligible, reasons = {}, {}, {}
+ candidates = dict(_sources())
+ if any(policy["candidates"] == "prefix" for policy in registry.source_policies()):
+ # ⚠ Every declared policy is "declared" today, so this walk does not run. It is here so a
+ # source CAN opt in without editing this branch, which is the ticket. The store read is
+ # guarded because a policy is a declaration and a missing store is not its problem.
+ try:
+ owned = list(ut.all_defs() or {})
+ except Exception: # noqa: BLE001
+ owned = []
+ for key in owned:
+ if registry.source_policy(key)["candidates"] == "prefix":
+ candidates.setdefault(key, {})
+ for key, spec in candidates.items():
+ policy = registry.source_policy(key)
+ max_rows = policy["max_rows"] if policy["max_rows"] is not None else ut.MAX_ROWS
+ window_max = (policy["window_max"] if policy["window_max"] is not None
+ else datastore.WINDOW_MAX)
+ source_key = policy["key"]
+ if source_key not in analyses:
+ analyses[source_key] = _source_analysis(source_key)
+ folds, spawned = analyses[source_key]
+ # ⛔ A CANDIDATE WITH NO SPEC CANNOT BE JUDGED, AND MUST THEREFORE BE REFUSED, NOT FREED.
+ # A `candidates: "prefix"` source contributes keys its provider never described, so the
+ # size question below has nothing to ask. Falling through would reach the `total is None
+ # and key not in spawned` arm and hand back ELIGIBLE, un-materialising a database whose
+ # size was never read - the exact inversion of this function's first law, arrived at by
+ # an absence rather than a decision. [S65-T13]
+ if not (spec.get("table") or spec.get("from_sql")):
+ eligible[key] = False
+ reasons[key] = ("its source declares no read-through binding, so its size cannot be "
+ "read and un-materialising it would be a guess")
+ continue
total = None
if cur is not None:
try:
@@ -1068,7 +1174,7 @@ def _unmaterialisable(cur=None):
order_by=spec["id"], limit=1, cur=cur, **frm)["total"]
except Exception: # noqa: BLE001
total = None
- if total is not None and total > ut.MAX_ROWS:
+ if total is not None and total > max_rows:
eligible[key] = True # no other home exists; this is not a choice
continue
if total is None and key not in spawned:
@@ -1079,8 +1185,8 @@ def _unmaterialisable(cur=None):
why.append("its size could not be read from the mirror on this deployment")
if key in folds:
why.append(folds[key])
- if total is not None and total > datastore.WINDOW_MAX:
- why.append(f"its {total:,} rows exceed the {datastore.WINDOW_MAX:,}-row window and "
+ if total is not None and total > window_max:
+ why.append(f"its {total:,} rows exceed the {window_max:,}-row window and "
f"the grid client still asks for whole tables, so un-materialising it "
f"today could only truncate")
eligible[key] = not why
@@ -2197,6 +2303,155 @@ def _views_as_cohorts(tree, table_key, session, spec, cols, base_where, wall_par
return tree, sets
+def _ranks_as_cohorts(tree, conj, spec, cols, base_where, wall_params, cur, cohort_sets,
+ had_cohort, today, limits=None):
+ """Resolve every ACTIVE rank leaf (Top N, quartile, above average ...) into a synthetic
+ `__cohort__` leaf over the ids it selects. `(tree, cohort_sets)`.
+
+ A rank is a property of the SET, so `filter_sql` refuses it at row grain. The set is answered
+ here the way the client answers it: the residue (every non-rank condition) under the wall is
+ the domain, and `filter_eval.resolve_rank_sets` (the Python port of the client's set pass,
+ ties by ascending pid) picks the members. Only the id and the ranked columns are read, uncapped.
+ """
+ fe = _fe()
+ leaves = fe._rank_leaves(tree or [], cols)
+ if not leaves:
+ return tree, cohort_sets
+ if had_cohort:
+ raise err(400, "filter_unsupported",
+ "a condition on cohort membership cannot be evaluated on this database's live "
+ "connection, because a cohort is resolved from this tenant's own records rather "
+ "than from the connection. Filter on a saved view instead, or open this "
+ "database's stored copy")
+ residue = fe._strip_rank_leaves(tree)
+ try:
+ pred = _fs().compile_filter_tree(residue, conj=conj, columns=cols, id_sql=spec["id"],
+ today=today, cohort_sets=cohort_sets)
+ except ValueError as exc:
+ raise err(400, "filter_unsupported", str(exc))
+ if pred is not None and pred.uses_aggregate:
+ raise err(400, "filter_unsupported",
+ "This condition cannot be evaluated for individual rows.")
+ where = base_where
+ params = list(wall_params)
+ if pred is not None:
+ where = f"({where}) AND {pred.sql}" if where else pred.sql
+ params.extend(pred.params)
+ frm_body = spec["from_sql"] if spec.get("from_sql") else spec["table"]
+ answers = {}
+ python_leaves = []
+ for n in leaves:
+ got = _rank_ids_sql(n, cols, spec["id"], frm_body, where, params, cur)
+ if got is None:
+ python_leaves.append(n)
+ else:
+ answers[id(n)] = got
+ if python_leaves:
+ # A GROUP-scoped rank (rare on this door) keeps the reference resolver: read the id and
+ # the ranked columns once, uncapped, and let `resolve_rank_sets` answer it.
+ ranked = sorted({str(n.get("colId")) for n in python_leaves if n.get("colId") in cols})
+ select = ", ".join([spec["id"]] + [cols[c]["sql"] for c in ranked])
+ sql = f"SELECT {select} FROM {frm_body}" + (f" WHERE {where}" if where else "")
+ rows = []
+ for r in cur.execute(sql, params).fetchall():
+ if not r or r[0] is None:
+ continue
+ row = {"pid": int(r[0])}
+ for i, c in enumerate(ranked, start=1):
+ row[c] = r[i]
+ rows.append(row)
+ resolved = fe.resolve_rank_sets({"nodes": python_leaves, "conj": "and"}, rows, cols)
+ for n in python_leaves:
+ answers[id(n)] = sorted(resolved.sets.get(fe._rank_key(n)) or ())
+ sets = dict(cohort_sets or {})
+ counter = [0]
+ wanted = {id(n) for n in leaves}
+
+ def rewrite(nodes):
+ out = []
+ for n in (nodes or []):
+ if isinstance(n, dict) and isinstance(n.get("children"), list):
+ out.append(dict(n, children=rewrite(n["children"])))
+ elif isinstance(n, dict) and id(n) in wanted:
+ counter[0] += 1
+ key = "__rank_%d__" % counter[0]
+ got = answers.get(id(n))
+ sets[key] = got if got is not None else []
+ if (limits is not None and isinstance(got, list)
+ and len(got) >= _PIN_REPORT_AT):
+ limits.append({
+ "subject": str(n.get("colId") or ""), "effect": "slow",
+ "cause": f"this ranking condition selects {len(got):,} rows, and "
+ f"each page of this database names every one of them in the "
+ f"query that narrows it",
+ "recommendation": "rank a smaller slice (a lower Top N or percentage), "
+ "or add a condition beside it"})
+ out.append({"colId": "__cohort__", "op": "anyOf", "value": key})
+ else:
+ out.append(n)
+ return out
+
+ return rewrite(tree), sets
+
+
+def _rank_ids_sql(leaf, cols, id_sql, frm_body, where, params, cur):
+ """One GLOBAL rank leaf as SQL: a `SqlMembership` selecting its ids, or None for the resolver.
+
+ Measured 2026-09-23: `resolve_rank_sets` over 256,542 order lines took 3.6 s in Python, and a
+ paged grid pays it on every page; as a SQL window the same answer is tens of milliseconds,
+ and as a subquery it never becomes a six-figure IN list. Semantics are the resolver's, line
+ for line (parity-tested over all eight ops with ties and blanks): blank or non-finite values
+ are ABSENT, ties break by ascending id in both directions, a bound outside the builder's
+ whole-number range is unanswerable (empty), and a quartile or decile of fewer rows than
+ buckets is empty. The domain's count and mean are read once here and inlined as literals.
+ """
+ fe = _fe()
+ Sql = _fs().SqlMembership
+ empty = Sql("SELECT NULL WHERE FALSE", [])
+ scope, _field = fe._rank_scope(leaf, None)
+ if scope != "global":
+ return None
+ col = cols.get(leaf.get("colId")) or {}
+ if col.get("type") not in fe.NUMERIC_TYPES:
+ return empty
+ v = f"TRY_CAST({col['sql']} AS DOUBLE)"
+ dom = (f"SELECT {id_sql} AS pid, {v} AS v FROM {frm_body} WHERE "
+ + (f"({where}) AND " if where else "") + f"{v} IS NOT NULL AND isfinite({v})")
+ params = list(params)
+ op = leaf.get("op")
+ if op in ("topN", "bottomN"):
+ bound = fe._rank_bound(leaf.get("value"), 1, 10000)
+ if bound is None:
+ return empty
+ direction = "DESC" if op == "topN" else "ASC"
+ return Sql(f"SELECT pid FROM ({dom}) d ORDER BY v {direction}, pid ASC LIMIT {bound}",
+ params)
+ count, total = cur.execute(f"SELECT count(*), sum(v) FROM ({dom}) d", params).fetchone()
+ count = int(count or 0)
+ if not count:
+ return empty
+ if op in ("inTopPct", "inBottomPct"):
+ pct = fe._rank_bound(leaf.get("value"), 1, 100)
+ if pct is None:
+ return empty
+ take = min(count, max(1, math.ceil(count * pct / 100)))
+ direction = "DESC" if op == "inTopPct" else "ASC"
+ return Sql(f"SELECT pid FROM ({dom}) d ORDER BY v {direction}, pid ASC LIMIT {take}",
+ params)
+ if op in ("aboveAvg", "belowAvg"):
+ mean = float(total) / count
+ cmp = ">" if op == "aboveAvg" else "<"
+ return Sql(f"SELECT pid FROM ({dom}) d WHERE v {cmp} {mean!r}", params)
+ if op in ("inQuartile", "inDecile"):
+ buckets = 4 if op == "inQuartile" else 10
+ wanted = fe._rank_bound(leaf.get("value"), 1, buckets)
+ if wanted is None or count < buckets:
+ return empty
+ return Sql(f"SELECT pid FROM (SELECT pid, row_number() OVER (ORDER BY v ASC, pid ASC) - 1"
+ f" AS idx FROM ({dom}) d) r WHERE least({buckets}, (idx * {buckets}) // {count}"
+ f" + 1) = {int(wanted)}", params)
+ return None
+
def _may_see_view(session, vid, store):
"""May this caller open that saved view? The same predicate the view rail asks.
@@ -2437,8 +2692,12 @@ def _compiled_read(table_key, session, *, filters=None, filterConj="and", sorts=
# the reader is held to, exactly as `viewSets` resolves inside the rows the client holds) and
# before `_leaf_cols` below, so the leaf never reaches the sentence that explains a column with
# no mirror expression. One place learns it; `/rows` and `/aggregates` share this compiler.
+ _had_cohort = bool(_active_cohort_leaves(tree))
tree, cohort_sets = _views_as_cohorts(tree, table_key, session, spec, cols,
base_where, wall_params, cur, limits)
+ tree, cohort_sets = _ranks_as_cohorts(
+ tree, (filterConj if filterConj in ("and", "or") else "and"), spec, cols, base_where,
+ wall_params, cur, cohort_sets, _had_cohort, _today(), limits)
# ── the predicate, pushed down ────────────────────────────────────────────────────────────
# ⛔ `_PSEUDO_COLS` is subtracted BEFORE the missing-column test. `__cohort__` and `__view__`
@@ -2563,8 +2822,12 @@ def _materialised_read(table_key, session, query, related):
"cols": bindings, "where": None}
cols = _filter_columns(spec, fields)
limits = list(payload.get("limits") or [])
+ _had_cohort = bool(_active_cohort_leaves(query["filters"]))
tree, cohorts = _views_as_cohorts(query["filters"], table_key, session, spec, cols,
None, [], cur, limits)
+ tree, cohorts = _ranks_as_cohorts(tree, query["filterConj"], spec, cols, None, [], cur,
+ cohorts, _had_cohort,
+ payload.get("today") or _today(), limits)
missing = _leaf_cols(tree) - _PSEUDO_COLS - set(cols)
if missing:
raise err(400, "filter_unsupported", "A filter names a column unavailable to this account.")
@@ -2585,7 +2848,42 @@ def _materialised_read(table_key, session, query, related):
"frm": {"table": "pinned_rows"}, "limits": limits, "payload": payload}
-def _materialised_window(table_key, session, offset, limit, filters, filterConj, sorts, search, related):
+def _group_pin(cols, group_by, group_key):
+ """`(sql, params)` narrowing a window to ONE group of `/aggregates`' grouping, or `(None, [])`.
+
+ HOTFIX 2026-09-23 (owner: "whatever is GROUPED we can add pagination per group"). A group's
+ records page through this door, so the narrowing is the SAME expression `_aggregate_over`
+ groups by (`filter_sql.group_sql`), never a client-built `eq` leaf: `_value_sql` lowercases
+ text and rounds numbers, which would make a group's pages disagree with its count.
+ `group_key` is the JSON of the key `/aggregates` put on the wire (`null` = the blank bucket).
+ """
+ # A plain-Python caller that omits the argument hands us FastAPI's `Query` sentinel, not
+ # None (the `related` scar in `routes_connected_tables`), so anything but a string is "none".
+ gcol = group_by.strip() if isinstance(group_by, str) else ""
+ if not gcol:
+ return None, []
+ group_key = group_key if isinstance(group_key, str) else None
+ gspec = cols.get(gcol)
+ if gspec is None:
+ raise err(400, "bad_group", "this column cannot be grouped on this database's live "
+ "connection, or it is not visible to your account")
+ try:
+ key = json.loads(group_key) if group_key not in (None, "") else None
+ except ValueError:
+ raise err(400, "bad_group", "groupKey must be the JSON of a group key")
+ gexpr = _fs().group_sql(gspec)
+ if key is None:
+ return f"({gexpr}) IS NULL", []
+ if gspec["type"] in _fs().NUMERIC_TYPES:
+ try:
+ return f"({gexpr}) = ?", [float(key)]
+ except (TypeError, ValueError):
+ raise err(400, "bad_group", "groupKey must be a number for a number column")
+ return f"({gexpr}) = ?", [str(key)]
+
+
+def _materialised_window(table_key, session, offset, limit, filters, filterConj, sorts, search, related,
+ groupBy=None, groupKey=None):
from harness import datastore
query = _selection_query({"filters": _json_arg(filters, "filters"), "filterConj": filterConj,
@@ -2593,8 +2891,13 @@ def _materialised_window(table_key, session, offset, limit, filters, filterConj,
with _materialised_read(table_key, session, query, related) as read:
spec = read["spec"]
order = _fs().compile_order_by(query["sorts"], read["cols"], tiebreak_sql=spec["id"])
- win = datastore.window(select=f'{spec["id"]} AS pid', where=read["where"],
- params=tuple(read["params"]), order_by=order or spec["id"],
+ where, params = read["where"], list(read["params"])
+ gsql, gparams = _group_pin(read["cols"], groupBy, groupKey)
+ if gsql:
+ where = f"({where}) AND {gsql}" if where else gsql
+ params.extend(gparams)
+ win = datastore.window(select=f'{spec["id"]} AS pid', where=where,
+ params=tuple(params), order_by=order or spec["id"],
offset=offset, limit=limit or None, cur=read["cur"], **read["frm"])
payload = read["payload"]
by_id = {int(row["pid"]): row for row in payload["rows"]}
@@ -2617,6 +2920,8 @@ def odoo_table_rows(table_key: str,
sorts: str = Query(default=None),
search: str = Query(default=None),
related: str = Query(default=None),
+ groupBy: str = Query(default=None),
+ groupKey: str = Query(default=None),
session: Session = Depends(require_session)):
"""ONE WINDOW over a connected grid, read straight from the mirror (contract C2).
@@ -2647,7 +2952,7 @@ def odoo_table_rows(table_key: str,
if _materialised_table(table_key):
return _materialised_window(table_key, session, offset, limit, filters, filterConj,
- sorts, search, related)
+ sorts, search, related, groupBy, groupKey)
# ⛔⛔ ONE COMPILER, AND THE AGGREGATES DOOR CALLS THE SAME ONE. The permission wall, the
# hidden-field closure, the user predicate and the search all live in `_compiled_read` above,
@@ -2661,6 +2966,10 @@ def odoo_table_rows(table_key: str,
cols, limits, frm = read["cols"], read["limits"], read["frm"]
base_where, wall_params = read["base_where"], read["wall_params"]
where, params, sort_spec = read["where"], read["params"], read["sort_spec"]
+ _gsql, _gparams = _group_pin(cols, groupBy, groupKey)
+ if _gsql:
+ where = f"({where}) AND {_gsql}" if where else _gsql
+ params = list(params) + list(_gparams)
# ── the order, made TOTAL ─────────────────────────────────────────────────────────────────
# ⛔ `tiebreak_sql` is not optional here: without a total order, LIMIT/OFFSET may return one
@@ -2699,7 +3008,7 @@ def odoo_table_rows(table_key: str,
# ⭐ D-155 — the freshness stamp rides the SAME builder, so both read-through doors answer the
# `refreshed` column identically instead of one of them leaving it blank.
rows_src = _pool_from_window(spec, win,
- stamp=mirror_stamp(spec, cur, table_key,
+ stamp=mirror_stamp(spec, cur, table_key, defn=defn, rt=
session.runtime))
overlays = {str(r["pid"]): {k: v for k, v in r.items() if k != "pid"} for r in rows_src}
@@ -3063,7 +3372,7 @@ def odoo_table_aggregates(table_key: str, body: dict = Body(default=None),
return body_out
-def mirror_stamp(spec, cur=None, table_key="", rt=None):
+def mirror_stamp(spec, cur=None, table_key="", rt=None, defn=None):
"""⭐ D-155 — WHEN THIS READ-THROUGH GRID'S DATA WAS LAST RECONCILED AGAINST ODOO, as a date.
⛔ THE COLUMN EXISTED AND THE BINDING DID NOT, which is the whole of D-155. Every Odoo table
@@ -3108,9 +3417,12 @@ def mirror_stamp(spec, cur=None, table_key="", rt=None):
# the materialised carve-out above, made structural. ⚠ Fail-QUIET to `""`: when we cannot
# tell whether this grid stores rows, the safe answer is the behaviour that shipped (blank),
# never a stamp that might overwrite a per-row one.
+ # HOTFIX 2026-09-23: `defn` is LENT by a caller already holding the definition. Without it
+ # `materialises` falls through to a whole-document `user_tables` read, measured on staging
+ # at ~3 s of every 50-row page of order lines (the `timing` phase `pool_and_stamp`).
if table_key:
try:
- if _ut().materialises(str(table_key), st=rt):
+ if _ut().materialises(str(table_key), st=rt, defn=defn):
return ""
except Exception: # noqa: BLE001
return ""
@@ -3227,7 +3539,11 @@ def whole_pool(table_key, cur=None, rt=None):
f"{table_key}: {win['total']:,} rows is more than one {datastore.WINDOW_MAX:,}-row "
f"window, and this database no longer stores rows in the tenant document; it can "
f"only be read a page at a time (`/odoo-tables/{table_key}/rows`)")
- return _pool_from_window(spec, win, stamp=mirror_stamp(spec, cur, table_key, rt))
+ # HOTFIX 2026-09-23: this door serves only a table whose rows have LEFT the tenant document
+ # (its docstring), so that fact is lent rather than re-derived: without `defn`,
+ # `materialises` re-reads the whole `user_tables` document, ~3 s on the pg store.
+ return _pool_from_window(spec, win, stamp=mirror_stamp(spec, cur, table_key, rt,
+ defn={"readThrough": True}))
def _fe():
diff --git a/api/routes_shares.py b/api/routes_shares.py
index d99219fd2adb0c2d8a6f9478f5653727ed48bf0c..f52ea879af478c2caf2132a2197afa4f30a5b706 100644
--- a/api/routes_shares.py
+++ b/api/routes_shares.py
@@ -90,10 +90,13 @@ wall that does not exist for this kind. Scoping user tables is booked, not done
*"Shared with me"*, who may open a source View, and who may re-share. Saved Views carry no
view-level permissions member or secondary access model.
"""
+import secrets
+
from fastapi import APIRouter, Body, Depends
import core.shares as shares
import core.users as users
+from core import field_permissions as _field_permissions
from deps import Session, err, require_session
# ⭐ W32-T28 (C3) — the SHARE notification's topic word, imported from the module that CLASSIFIES
# it (`routes_alerts.notification_view`) rather than typed again here. The producer and the
@@ -103,6 +106,11 @@ from routes_alerts import SHARE_TOPIC as _SHARE_TOPIC
router = APIRouter(prefix="/api/v1")
+# C8's derived field leg belongs to the field-permission seam, while every existing read/write
+# wall already calls ``shares.role_for``. Install it explicitly at application-router startup;
+# no caller gets a second permission predicate and non-field kinds are byte-for-byte delegated.
+_field_permissions.install_view_field_resolver()
+
def _kind_or_400(raw):
try:
@@ -249,11 +257,10 @@ def _field_resolve(session, table_key, field_key, strata=_FIELD_STRATA):
none, because residency IS authorship in a per-account document.
4. **the connector catalogue** — the columns the database itself declares.
- ⚠ `strata` NARROWS THE WALK, AND IT IS A COST CONTROL AS WELL AS A CORRECTNESS ONE.
- `_share_view_fields` asks per VIEW KEY, and `_catalogue_fields` re-parses the canonical
- contract file on every built-in call — so an unfiltered walk would `json.loads`
- `aios_grid_fields.json` once per column per view share, where before this ticket it was zero.
- That arm passes `("shared", "mine")`, which is exactly the pair it could see before W50-T23.
+ ⚠ `strata` NARROWS THE WALK, AND IT IS A COST CONTROL AS WELL AS A CORRECTNESS ONE. A caller
+ that already knows which definition strata are relevant can avoid the remaining reads; the
+ View share preflight deliberately checks all four because unreadable references must refuse
+ the whole transaction rather than be misclassified as database-owned.
⛔ EACH STRATUM RESOLVES IN ITS OWN `try`, AND THE BARE `except Exception: pass` THAT WRAPPED
ALL OF THEM IS GONE. It conflated *"the store threw"* with *"no such field"*, so one
@@ -765,54 +772,12 @@ def _unremovable(held_rows, entries, uname):
stuck.add(user)
return stuck
-# ── ⭐⭐ WAVE 42 · T25 — SHARING A VIEW SHARES THE FIELDS IT NAMES ──────────────────────────────
-#
-# A saved view is a SELECTION expressed in columns: an order, a visible set, a group, a colour,
-# a sort and a filter tree, every one of which names a field key. Granting the view alone hands
-# the receiver a rail row whose columns resolve to nothing they may read, so the feature that
-# looked shipped puts an empty grid on their screen. This block closes that, and it refuses
-# LOUDLY in the three places where a grant would otherwise be a lie.
-#
-# ⛔⛔ TRAP 1 — A FIELD GRANT IS TOPIC-WIDE, NOT VIEW-SCOPED. It is keyed
-# `shares.field_oid(topic, key)`, so it hands the receiver that column on EVERY view of that
-# database, for as long as the grant stands. That is the existing grant primitive and this
-# ticket does not narrow it; what it does is refuse to let the response PRETEND otherwise, which
-# is why `sharedFieldScope` rides back beside the list.
-#
-# ⛔⛔ TRAP 2 — THE SHARER CANNOT GRANT A COLUMN THEY DO NOT OWN. `field` is absent from
-# `shares.RESHARE_KINDS` and `_owns_object`'s field branch demands `createdBy == session.uname`,
-# so a view built over a colleague's column simply cannot carry that column with it. Silently
-# skipping it ships an empty column; silently granting it is a permission hole. It is REFUSED
-# BY NAME, inside the 200 response, with the sentence that says who to ask.
-#
-# ⛔⛔ TRAP 3 — A GRANT CANNOT UN-HIDE AN ADMIN-WALLED COLUMN. `perm_scope.hidden_keys` UNIONS
-# the administrator's `hiddenFields` with the field-grant source, so granting a walled column
-# answers 200, writes the record, and the receiver still sees nothing: granted with no effect,
-# which is the failure this ticket exists to end. Detected BEFORE the grant, refused by name,
-# and the receiver it is walled from is named too.
-#
-# ⚠ THE WALL IS READ FROM THE ADMIN SOURCE DIRECTLY AND NEVER THROUGH `hidden_keys`, AND THAT IS
-# NOT AN OPTIMISATION. `hidden_keys` unions `field_grant_hidden` — "hidden because you were not
-# granted it" — which is true of every column this route is ABOUT to grant. Asking it here would
-# refuse the whole set on the grounds that the grant has not happened yet.
-
-
-def _view_configs(view):
- """The dicts a saved view keeps its column state in.
-
- `SavedView.config` is the contract (`types.ts::ViewConfig`) and is where every key below
- actually lives. The record ITSELF is read as well because a legacy row that stored the same
- members flat costs one dict lookup to cover and would otherwise hand back an empty key set,
- which is the shape that grants nothing and reports nothing wrong.
- """
- out = []
- if not isinstance(view, dict):
- return out
- cfg = view.get("config")
- if isinstance(cfg, dict):
- out.append(cfg)
- out.append(view)
- return out
+# ── SDLC63 / C8 — ONE VIEW AUDIENCE, ONE DERIVED FIELD BOUNDARY ───────────────────────────────
+# The pre-SDLC63 route copied a View audience into every named field grant and returned every
+# field/refusal to the browser. The current path resolves the deliberate field set once,
+# preflights the editor's own readability, and commits only the View audience plus its compact
+# field boundary. Field detail leaves the server only as a count on success; failures retain it
+# under a support reference rather than turning a large View into a large error message.
def _view_field_keys(view):
@@ -826,45 +791,57 @@ def _view_field_keys(view):
⚠ AND `filters` IS A TREE, NOT A LIST. Condition groups nest under `children`, so the walk
recurses rather than iterating once: a column named only inside a group is still named.
"""
- keys, seen = [], set()
+ return _field_permissions.view_field_keys(view)
+
+
+def _view_field_definitions(session, table_key, keys):
+ """Resolve a View's field definitions with a constant number of source reads."""
+ workspace_key, shared_key, grant_topic = _field_storage_keys(table_key)
+ shared, mine, document, catalogue = {}, {}, {}, {}
+ try:
+ from core import shared_overlay
+ shared = shared_overlay.fields(shared_key, st=session.runtime) or {}
+ except Exception: # noqa: BLE001
+ pass
+ try:
+ import core.table_store as table_store
+ mine = (table_store.make(workspace_key, st=session.runtime)
+ .workspace(session.uname).get("fields") or {})
+ except Exception: # noqa: BLE001
+ pass
+ try:
+ document = session.runtime.get(workspace_key) or {}
+ except Exception: # noqa: BLE001
+ pass
+ try:
+ catalogue = _catalogue_fields(session, table_key, workspace_key, grant_topic)
+ except Exception: # noqa: BLE001
+ pass
- def add(raw):
+ me = str(session.uname or "").strip().lower()
+ out = {}
+ for raw in keys or ():
key = str(raw or "").strip()
- if key and key not in seen:
- seen.add(key)
- keys.append(key)
-
- def walk(node):
- if isinstance(node, list):
- for child in node:
- walk(child)
- return
- if not isinstance(node, dict):
- return
- kids = node.get("children")
- if isinstance(kids, list):
- walk(kids)
- add(node.get("colId"))
- rhs = node.get("rhs")
- if isinstance(rhs, dict) and rhs.get("kind") != "stat":
- add(rhs.get("colId"))
-
- for cfg in _view_configs(view):
- for key in (cfg.get("order") or ()):
- add(key)
- for key in (cfg.get("visible") or ()):
- add(key)
- walk(cfg.get("filters"))
- add(cfg.get("groupBy"))
- colour = cfg.get("colorBy")
- add(colour.get("colId") if isinstance(colour, dict) else colour)
- sorts = cfg.get("sorts")
- if isinstance(sorts, dict):
- sorts = [sorts]
- for spec in (sorts or ()):
- if isinstance(spec, dict):
- add(spec.get("colId"))
- return keys
+ defn, stratum = None, None
+ if isinstance(shared, dict) and isinstance(shared.get(key), dict):
+ defn, stratum = shared[key], "shared"
+ elif isinstance(mine, dict) and isinstance(mine.get(key), dict):
+ defn, stratum = mine[key], "mine"
+ elif isinstance(document, dict):
+ for member, workspace in document.items():
+ if str(member).strip().lower() == me or not isinstance(workspace, dict):
+ continue
+ theirs = (workspace.get("fields") or {}).get(key)
+ if isinstance(theirs, dict):
+ defn = dict(theirs)
+ if not str(defn.get("createdBy") or "").strip():
+ defn["createdBy"] = str(member).strip().lower()
+ stratum = "colleague"
+ break
+ if defn is None and isinstance(catalogue, dict) and isinstance(catalogue.get(key), dict):
+ defn, stratum = catalogue[key], "catalogue"
+ out[key] = (defn, stratum, workspace_key, shared_key, grant_topic)
+ return out
def _find_view_topic(session, oid):
@@ -891,7 +868,7 @@ def _find_view_topic(session, oid):
return None, None
-def _promote_field_for_grant(session, table_key, field_key):
+def _promote_field_for_grant(session, table_key, field_key, resolved=None):
"""Promote a private column exactly once, stamp it shared, and answer its GRANT OID.
⛔⛔ ONE SEQUENCE, TWO ARMS, AND THAT IS THE WHOLE REASON THIS IS A FUNCTION. `put_share`'s
@@ -921,8 +898,8 @@ def _promote_field_for_grant(session, table_key, field_key):
person who did not make it. It was a 404 before this ticket and it stays one.
"""
from core import field_permissions, shared_overlay
- defn, stratum, workspace_key, shared_key, grant_topic = _field_resolve(
- session, table_key, field_key)
+ defn, stratum, workspace_key, shared_key, grant_topic = (
+ resolved if resolved is not None else _field_resolve(session, table_key, field_key))
if not isinstance(defn, dict) or stratum == "colleague":
return None
if stratum == "mine":
@@ -936,38 +913,6 @@ def _promote_field_for_grant(session, table_key, field_key):
return shares.field_oid(grant_topic, field_key)
-def _grant_receivers(session, entries):
- """`{username: display name}` for everybody a grant list actually reaches.
-
- `shares.EVERYONE` is a real audience, not a placeholder, so it expands to the tenant roster
- `_people` already computes for the picker. Reading it from a second place would be a second
- answer to "who is in this workspace".
- """
- people = None
- out = {}
- for entry in shares._clean_entries(entries):
- who = str(entry.get("user") or "").strip().lower()
- if not who:
- continue
- if who == shares.EVERYONE:
- if people is None:
- people = _people(session.tenant)
- for person in people:
- out.setdefault(str(person["username"]).strip().lower(), person["name"])
- continue
- out.setdefault(who, who)
- if out:
- try:
- reg = users.registry() or {}
- except Exception: # noqa: BLE001
- reg = {}
- for uname in list(out):
- rec = reg.get(uname)
- if isinstance(rec, dict) and rec.get("name"):
- out[uname] = str(rec["name"])
- return out
-
-
def _admin_walled(session, grant_topic, defs, receivers):
"""`{field key: [display names]}` for receivers an ADMINISTRATOR has already walled off.
@@ -1020,187 +965,74 @@ def _admin_walled(session, grant_topic, defs, receivers):
return walled
-#: ⭐ THE THREE REASONS A VIEW'S COLUMN IS NOT GRANTED, EACH ITS OWN CODE. One fused
-#: "could not share" would make the three walls unprovable apart, which is the
-#: [[a-declared-gate-is-an-unchecked-claim]] failure `verify_field_doors` section 8 exists to
-#: prevent: every cause fails by its own name, or it is not gated at all.
-#: ⛔ STANDING RULE 2 — no em dash and no en dash in any sentence below; these reach a screen.
-FIELD_REFUSAL_CODES = ("not_a_shared_column", "not_owned", "admin_hidden")
-
-#: ⭐ TRAP 1, SAID OUT LOUD. A column grant is keyed by DATABASE, so it is not scoped to the view
-#: that occasioned it. The response carries this sentence rather than letting a caller infer a
-#: narrower promise than the store actually makes.
-FIELD_GRANT_SCOPE = ("A column shared this way is shared on the whole database, so the people "
- "you named can see it on every view of it, not only on this one.")
-
-
-def _refusal(key, code, names=()):
- """One refused column, with the sentence a person reads."""
- if code == "not_a_shared_column":
- why = ('"{k}" belongs to the database itself rather than to a person, so there was '
- 'nothing to grant. Anybody who can open the database already sees it.')
- elif code == "not_owned":
- why = ('"{k}" is somebody else\'s column, and only its owner (or an administrator) can '
- 'share it. Ask them to share it, then share this view again.')
- else:
- why = ('"{k}" is hidden from {who} by an administrator, so sharing it would change '
- 'nothing. Ask an administrator to unhide it first.')
- return {"field": key, "reason": code,
- "message": why.format(k=key, who=_named(names) if names else "them")}
-
-
-def _classify_view_fields(keys, owned, defined, walled):
- """`(sharedFieldKeys, refusedFieldKeys)` for the columns a view names. PURE.
-
- ⛔ EVERY KEY IS DECIDED BEFORE ANY OF THEM IS WRITTEN, which is the same property the ceiling
- block above claims for the three role walls: a refusal must not be able to leave part of the
- set promoted and part of it untouched.
-
- ⚠ THE ORDER OF THE CAUSES IS FIXED AND IT DECIDES WHICH SENTENCE A PERSON READS. A built-in
- column is owned by nobody, so "there was nothing to grant" has to be asked before "you do not
- own it", or every database column in `visible` would be reported as somebody else's.
- """
- granted, refused = [], []
- for key in keys:
- if key not in defined:
- refused.append(_refusal(key, "not_a_shared_column"))
- elif key not in owned:
- refused.append(_refusal(key, "not_owned"))
- elif walled.get(key):
- refused.append(_refusal(key, "admin_hidden", walled.get(key)))
- else:
- granted.append(key)
- return granted, refused
-
-
-def _merge_field_entries(existing, incoming):
- """A column's grant list with `incoming` ADDED to it. Nobody is removed, nobody is lowered.
-
- ⛔⛔ THE SHARE PUT REPLACES, AND A COLUMN'S AUDIENCE IS NOT THE VIEW'S AUDIENCE. This route's
- contract is a REPLACING set (`PUT ... entries` at the top of the file), which is right for the
- object the caller is looking at and catastrophic for the columns underneath it: a column grant
- is TOPIC-WIDE, so handing the view's list straight to `set_grants` would rewrite the column's
- audience to whoever this view happens to be shared with. A colleague granted that column
- through the `field` door would lose it, on every view of the database, silently, inside a 200.
- That is exactly the harm D-474 closed at the view door, arriving one door over.
-
- ⚠ AND A LENGTH CHECK DOES NOT SEE IT. Replacing a one-person list with a different one-person
- list keeps the count identical, so the gate asserts the HELD USER IS STILL THERE AT THEIR OWN
- ROLE rather than counting rows.
-
- ⭐ RAISED, NEVER LOWERED. A receiver already held at `view` who arrives on an `edit` share is
- raised, because only the column's owner (or an administrator) reaches this path at all and
- that is the access they just asked to hand over. An `edit` holder meeting a `view` share keeps
- `edit`: a share is an addition, and nothing here is a revocation.
-
- ⚠ `by` RIDES ON EXISTING ROWS AND IS NEVER TAKEN FROM AN INCOMING ONE. Provenance decides who
- may revoke a grant (`_unremovable`), so an incoming payload must not be able to claim it;
- `set_grants` stamps a genuinely new row from `granter`.
- """
- rank = {"view": 0, "edit": 1}
- order, merged = [], {}
- for row in (existing or ()):
- if not isinstance(row, dict):
- continue
- who = str(row.get("user") or "").strip().lower()
- role = str(row.get("role") or "").strip().lower()
- if not who or role not in rank or who in merged:
- continue
- kept = {"user": who, "role": role}
- stamp = str(row.get("by") or "").strip().lower()
- if stamp:
- kept["by"] = stamp
- merged[who] = kept
- order.append(who)
- for row in (incoming or ()):
- if not isinstance(row, dict):
- continue
- who = str(row.get("user") or "").strip().lower()
- role = str(row.get("role") or "").strip().lower()
- if not who or role not in rank:
- continue
- if who not in merged:
- merged[who] = {"user": who, "role": role}
- order.append(who)
- elif rank[role] > rank[merged[who]["role"]]:
- merged[who]["role"] = role
- return [merged[who] for who in order]
+class _ViewSharePreparationError(RuntimeError):
+ """Server-only field details for a refused all-or-none View share."""
+ def __init__(self, details):
+ self.details = [str(value) for value in (details or ()) if str(value)]
+ super().__init__("one or more View fields could not be prepared")
-def _share_view_fields(session, oid, entries):
- """The `view` arm's field half: classify every column the view names, then grant the ones
- that survive. `(sharedFieldKeys, refusedFieldKeys)`.
- ⛔ CLASSIFY EVERYTHING, THEN WRITE. `_classify_view_fields` sees the whole key set before
- `_promote_field_for_grant` touches a single definition, so a refusal cannot leave half the
- view's columns promoted.
+def _prepare_view_share(session, oid, *, preview=False):
+ """Resolve one View and make its personal definitions tenant-readable before the CAS.
- ⚠ THE VIEW'S OWN GRANT HAS ALREADY BEEN WRITTEN when this runs, and that order is the safe
- one: a `409 grants_changed` on the view must not be preceded by a set of column promotions
- for a share that then did not happen.
+ Database-owned fields need no field write and some connector columns have no overlay
+ definition at all. A personal definition is promoted while it
+ still has no derived audience; if a later transaction fails, it remains an owner-controlled
+ shared definition rather than leaking a value. Resolved unreadable references refuse the
+ whole operation and ride only in server diagnostics; keys absent from definition strata are
+ database-owned fields already admitted by the validated View-save door.
+ Preview performs the same admission checks without promoting personal definitions.
"""
topic, view = _find_view_topic(session, oid)
if not topic or not isinstance(view, dict):
- return [], []
+ raise _ViewSharePreparationError(["view:not_found"])
keys = _view_field_keys(view)
- if not keys:
- return [], []
table_key = f"{topic}_table_workspace"
grant_topic = _field_storage_keys(table_key)[2]
- defs, defined, owned, owners = {}, set(), set(), {}
+ details = []
+ definitions = {}
+ database_owned = 0
+ resolved = _view_field_definitions(session, table_key, keys)
+ shared_roles = _field_permissions.field_roles_for_keys(
+ grant_topic,
+ [key for key, row in resolved.items() if row[1] == "shared"],
+ session.uname, is_admin=session.admin, st=session.runtime)
for key in keys:
- defn, stratum = _field_resolve(session, table_key, key, strata=("shared", "mine"))[:2]
- # ⛔⛔ W50-T23 — THE VIEW ARM SEES EXACTLY THE TWO STRATA IT SAW BEFORE THIS TICKET.
- # `_field_resolve` added `catalogue` and `colleague`, and BOTH are regressions here:
- # * `catalogue` on a built-in topic is the WHOLE canonical contract, so an admin
- # sharing one ordinary view would `defined` every contract column, promote each into
- # `__shared` and stamp `granted: True` — the exact mark
- # `perm_scope.field_grant_hidden` walls on. One view share would turn the product
- # contract into grant-governed columns and hide them from everybody without a grant.
- # W42-T25 states the intent in as many words: *"Throwing instead would make the
- # ordinary case (a view over one built-in column) unshareable."*
- # * `colleague` would put a refusal row per colleague-private column into the dialog's
- # refused list, for columns the sharer never named.
- # The `field` arm keeps both, because there the column IS the request.
- if not isinstance(defn, dict) or stratum in ("catalogue", "colleague"):
+ row = resolved.get(key) or (None, None, None, None, grant_topic)
+ defn, stratum = row[:2]
+ if not isinstance(defn, dict):
+ # The validated View-save door already intersected references with the editor's
+ # served schema. Absence from the overlay/catalogue resolver therefore means a
+ # database-owned column, not a private definition to publish. It belongs in the
+ # compact boundary and needs no per-field ownership or grant mutation.
+ database_owned += 1
continue
- defs[key] = defn
- defined.add(key)
- # ⚠ THE COLUMN'S OWNER, NOT THIS SESSION. `set_grants`' owner is sticky, so stamping the
- # SHARER here would hand an administrator sharing somebody else's column the ownership
- # of it as a side effect of sharing a view. Same resolver the claim wall uses.
- owners[key] = _field_owner(session, defn, stratum == "shared", stratum=stratum)
- # ⛔ ONE OWNERSHIP PREDICATE. `_owns_object` is the wall the `field` arm claims through
- # and the DELETE door reads; a second "does this person own this column" here is how the
- # two answers come apart. It short circuits True for an administrator, which is right:
- # an administrator may already share any column through the `field` door.
- if _owns_object(session, "field", shares.field_oid(grant_topic, key)):
- owned.add(key)
- walled = _admin_walled(session, grant_topic, defs, _grant_receivers(session, entries))
- granted, refused = _classify_view_fields(keys, owned, defined, walled)
- written = []
- for key in granted:
- field_oid = _promote_field_for_grant(session, table_key, key)
- if not field_oid:
- refused.append(_refusal(key, "not_a_shared_column"))
+ definitions[key] = defn
+ if stratum == "catalogue":
+ database_owned += 1
+ if stratum == "colleague":
+ details.append(f"{key}:not_readable")
continue
- # ⛔⛔ ADDED TO, NEVER REPLACED. `set_grants` writes the list it is given, and this route
- # PUTs a REPLACING set, so handing it the view's `entries` would make the view's audience
- # the COLUMN's audience: a colleague granted this column through the `field` door would
- # lose it, on every view of the database, inside a 200. `_merge_field_entries` says what
- # that sentence means once, and it is the only writer of a column list on this path.
- held = shares.grants("field", field_oid, st=session.runtime)
- # ⚠ NO `expect` HERE, AND THAT IS NOT AN OVERSIGHT. The compare-and-set on the view's own
- # grant guards a record TWO PEOPLE ARE EDITING through the same dialog. A column's record
- # was just read (and possibly created) inside this call, nobody is looking at it in a
- # dialog, and a `409` raised at this point would fail a PUT whose view grant has ALREADY
- # been written, which is the half-applied state the refusals above exist to avoid.
- shares.set_grants("field", field_oid,
- _merge_field_entries(held.get("entries"), entries),
- owner=held.get("owner") or owners.get(key) or session.uname,
- granter=session.uname, st=session.runtime)
- written.append(key)
- return written, refused
+ if stratum == "shared":
+ if (shared_roles.get(key) or {}).get("role") is None:
+ details.append(f"{key}:not_readable")
+ continue
+ if stratum == "mine" and not preview:
+ field_oid = _promote_field_for_grant(session, table_key, key, resolved=row)
+ if not field_oid:
+ details.append(f"{key}:promotion_failed")
+
+ # The account's permanent hidden-field wall wins even for catalogue fields. The View save
+ # path already removes these references; repeating that answer here keeps a stale or imported
+ # View from becoming a permission bypass at its first share.
+ walled = _admin_walled(
+ session, grant_topic, definitions,
+ {session.uname: _display_name(session.uname)})
+ details.extend(f"{key}:not_readable" for key in sorted(walled))
+ if details:
+ raise _ViewSharePreparationError(details)
+ return topic, view, grant_topic, keys, database_owned
@router.get("/share/mine")
@@ -1253,7 +1085,16 @@ def get_share(kind: str, oid: str, session: Session = Depends(require_session)):
# administrator-shaped Farhan session must not turn Shantal's absent grant into a record.
# Other share kinds retain their established administrator behaviour.
_admin_share = bool(session.admin) and kind != "cohort"
- role = shares.role_for(kind, oid, session.uname, is_admin=_admin_share, st=session.runtime)
+ if kind == "field":
+ table_key, field_key = shares.split_field_oid(oid)
+ # C8's inherited path grants the data, not the column's Manage Access roster. Use the
+ # direct leg here so a View audience cannot enumerate a field owner's separate grants.
+ role = _field_permissions.field_role_sources(
+ table_key, field_key, session.uname,
+ is_admin=_admin_share, st=session.runtime)["direct"]
+ else:
+ role = shares.role_for(kind, oid, session.uname,
+ is_admin=_admin_share, st=session.runtime)
may_admin = shares.may_administer(kind, oid, session.uname, is_admin=_admin_share,
st=session.runtime)
# W39-T29 — the dialog opens with GET before its first PUT. Until that PUT exists the grant
@@ -1561,6 +1402,44 @@ def put_share(kind: str, oid: str, body: dict = Body(default=None),
# write rather than at the moment of the read: a concurrent save lands between the two, and
# a wave-40 adversarial probe drove a re-sharer's PUT being ACCEPTED while the grant the
# owner had just added disappeared. See `shares.set_grants`' own note.
+ if kind == "view":
+ fields = []
+ stage = "prepare"
+ try:
+ _topic, view, grant_topic, fields, database_owned = _prepare_view_share(session, oid)
+ stage = "transaction"
+ out = _field_permissions.set_view_grants_with_boundary(
+ view, grant_topic, entries,
+ owner=session.uname if claiming else rec["owner"],
+ granter=session.uname, expect=rec["entries"], st=session.runtime)
+ except shares.GrantsChanged as exc:
+ raise err(409, "grants_changed", str(exc))
+ except Exception as exc: # noqa: BLE001
+ # The browser gets one short, actionable sentence. Field names and the internal
+ # cause remain in the tenant-scoped diagnostic ring under the same support reference.
+ reference = f"VS-{secrets.token_hex(4).upper()}"
+ details = getattr(exc, "details", None) or fields
+ try:
+ _field_permissions.record_view_share_diagnostic(
+ reference, oid, session.uname, stage, details, exc,
+ st=session.runtime)
+ except Exception: # noqa: BLE001
+ pass
+ raise err(
+ 500, "view_share_failed",
+ "We could not update this View's access. Try again. If it keeps failing, "
+ f"contact support with reference {reference}.")
+ _notify_new_grantees(
+ session, kind, oid, before=rec["entries"], after=out.get("entries") or [])
+ # One field-free compatibility reason keeps older API callers from mistaking connector
+ # columns for promoted custom definitions. It is never rendered and never scales with
+ # the field count; all actual field detail stays server-side under a support reference.
+ compatibility = ([{"reason": "not_a_shared_column"}] if database_owned else [])
+ return {"owner": out.get("owner"), "entries": out.get("entries") or [],
+ "fieldShare": {"includedCount": int(out.get("fieldCount") or 0),
+ "databaseOwnedCount": database_owned},
+ "refusedFieldKeys": compatibility}
+
try:
out = shares.set_grants(kind, oid, entries,
owner=session.uname if claiming else rec["owner"],
@@ -1570,20 +1449,6 @@ def put_share(kind: str, oid: str, body: dict = Body(default=None),
except shares.GrantsChanged as exc:
raise err(409, "grants_changed", str(exc))
_notify_new_grantees(session, kind, oid, before=rec["entries"], after=out.get("entries") or [])
- # ⭐⭐ W42-T25 — SHARING A VIEW SHARES THE FIELDS IT NAMES. A view is a selection expressed
- # in columns, so a grant that carries the view and none of its columns puts a rail row on
- # the receiver's screen over an empty grid. The three refusals are named rather than thrown:
- # this PUT succeeded, the view IS shared, and what could not follow it is reported by name
- # with the reason. Throwing instead would make the ordinary case (a view over one built-in
- # column) unshareable.
- # ⛔ THE EXISTING KEYS ARE UNTOUCHED. `{owner, entries}` is what every current client reads
- # off this route; the field report is additive, and a caller that ignores it is unaffected.
- if kind == "view":
- shared_fields, refused_fields = _share_view_fields(session, oid, entries)
- return {**out,
- "sharedFieldKeys": shared_fields,
- "refusedFieldKeys": refused_fields,
- "sharedFieldScope": FIELD_GRANT_SCOPE}
return out
diff --git a/api/routes_sheet.py b/api/routes_sheet.py
index 02f11d4cccf6a79ee7da0edba19b0a76f147304b..1e1bcee65dc9d8476d3093b249a360f60b4d73bd 100644
--- a/api/routes_sheet.py
+++ b/api/routes_sheet.py
@@ -44,6 +44,7 @@ import threading
import uuid
from fastapi import APIRouter, Body, Depends, File, HTTPException, UploadFile
+from fastapi.responses import Response
from deps import Session, err, require_session
@@ -974,7 +975,36 @@ def _binding_scope_key(session: Session, database: str, binding: dict) -> str:
import core.perm_scope as perm_scope
wall_tree = ((perm_scope.entry(session.user, database) or {}).get("filter")
if perm_scope.row_scope_applies(session.user, database) else None)
- return sheets.scope_hash(database, wall_tree, (binding or {}).get("scope") or {})
+ from core.sheet_binding import fingerprint
+ # Compare effective access, including field grants. Equivalent callers may share a
+ # snapshot unless a personal workspace makes their database values caller-specific.
+ memo = getattr(session, "_sheet_scope_context", None)
+ if memo is None:
+ memo = session._sheet_scope_context = {}
+ if database not in memo:
+ from core import shared_overlay, shares, user_tables, view_templates
+ st = _store(session)
+ try:
+ if "_definitions" not in memo:
+ memo["_definitions"] = st.get_projection(user_tables.STORE_KEY, drop=("rows",)) or {}
+ memo["_grants"] = fingerprint((st.get(shares.SHARES_KEY, fresh=True) or {}).get("field"))
+ definition = memo["_definitions"].get(database) or {}
+ workspace = view_templates.workspace_key(database)
+ personal = (st.get_projection(workspace, drop=("overlays",)) or {}).get(session.uname) or {}
+ shared_key = database if database.startswith("ut_") else workspace
+ fields = {f["key"]: f for f in definition.get("fields", []) if f.get("key")}
+ fields.update(shared_overlay.fields(shared_key, st=st))
+ fields.update(personal.get("fields") or {})
+ memo[database] = [sorted(perm_scope.hidden_keys(session.user, database,
+ list(fields.values()), st=st)), session.uname if personal else None]
+ except Exception:
+ # An unprovable wall cannot read a previous slot.
+ memo[database] = uuid.uuid4().hex
+ policy = {key: value for key, value in session.user.items()
+ if key not in ("username", "name", "email", "password", "password_hash", "epoch",
+ "created_at", "updated_at", "last_login")}
+ return fingerprint([sheets.scope_hash(database, wall_tree, (binding or {}).get("scope") or {}),
+ binding, policy, memo[database], memo.get("_grants")])
def _has_ending(binding):
@@ -988,128 +1018,131 @@ def _has_cell_inputs(binding):
return True
scope = binding.get("scope") if isinstance(binding, dict) else None
nodes, _ = _fe_parts(scope.get("filter") if isinstance(scope, dict) else None)
+ related = binding.get("related") if isinstance(binding, dict) else None
+ related_nodes, _ = _fe_parts(related.get("filter") if isinstance(related, dict) else None)
+ nodes = [*nodes, *related_nodes]
# Linked values and related-table grants can change independently of the orders' cache key.
if sheet_mod.linked_filter_ids(nodes):
return True
return any(sheet_mod.filter_cell_nodes(nodes))
+def _refresh_context(book, binding, memo=None):
+ """Authored inputs and relative dates invalidate refresh, but never an open's snapshot."""
+ from core.sheet_binding import fingerprint
+ from datetime import date
+ payload = {}
+ if _has_cell_inputs(binding):
+ if memo is None:
+ memo = {}
+ if "inputs" not in memo:
+ memo["inputs"] = fingerprint([[tab.get("id"), tab.get("name"), {
+ addr: {key: cell.get(key) for key in ("kind", "text", "formula", "binding", "value")
+ if key != "value" or cell.get("kind") in ("literal", "empty")}
+ for addr, cell in (tab.get("cells") or {}).items()}]
+ for tab in book.get("sheets") or []])
+ payload["inputs"] = memo["inputs"]
+ # Relative filters and measure windows can change their answer at midnight without a write.
+ encoded = json.dumps(binding, sort_keys=True)
+ if any(word in encoded for word in ('"window"', '"within"', '"ending"', '"view_id"')):
+ payload["today"] = date.today().isoformat()
+ return fingerprint(payload)
+
+
+def _snapshot_cell(cell, snapshot):
+ from copy import deepcopy
+ for key in ("value", "display", "state", "refreshed_at", "refusal", "limits"):
+ cell.pop(key, None)
+ cell.update(deepcopy(snapshot))
+
+
def _reresolved_for_reader(session: Session, book, *, endings_only=False):
- """A COPY of `book` whose every bound cell was recomputed under THIS reader's scope (C5).
-
- ⛔⛔ IN MEMORY, AND IT MUST NEVER PERSIST. `refresh_sheet` runs the same loop and ends with
- `put_workbook`, and a read path that copied that body would write the READER's numbers into
- the OWNER's document: the same leak as serving stale owner values, in the opposite direction,
- caused by a GET, and therefore reachable by anybody the workbook was shared with. So this
- works on a `copy.deepcopy` and returns it without saving the reader's workbook values.
- R10 may create a missing Metric field through the database's ordinary field writer.
-
- ⭐ THE WALL RUNS ONCE PER DISTINCT DATABASE, exactly as `refresh_sheet` does (R16): a reader
- who has lost access to a bound database is refused on every cell of it rather than shown the
- owner's cached number.
-
- ⚠ `refreshedFor` IS STAMPED ONLY WHEN A BOUND CELL WAS ACTUALLY RECOMPUTED. A workbook of
- literals has nothing to re-resolve, and a stamp there would be a claim with nothing behind it.
-
- ⭐⭐ W52-T12 / R3: A CACHE HIT SKIPS `_resolve_binding` ENTIRELY, AND A MISS FILLS ONLY THE
- CALLER'S OWN SLOT. The cache lives in a document `core.sheets` owns and keys by
- `(database, wall, binding scope)` (`_binding_scope_key`), never by user id or role name, so
- two readers who share a row restriction share an answer and a reader whose restriction differs
- never reads the other's slot. Only `refresh_sheet` may ever clear it; a GET here only ever
- fills the ONE slot the caller themselves is entitled to.
- """
- from core import sheets
- import copy as _copy
- fresh = _copy.deepcopy(book)
- stamp = _now_iso()
- seen_db = {}
- # ⭐ The same per-request memo as `refresh_sheet`, for the same reason and with the same
- # lifetime: this loop resolves cell by cell too, and a reader's GET must not rebuild one
- # database's workspace assembly once per bound cell.
- view_memo = {}
- touched = 0
- store = _store(session)
- wid = book.get("id") or ""
- for tab in fresh.get("sheets") or ():
- tab_ref = str(tab.get("id") or tab.get("name") or "")
+ """C2/R4: project last resolved values only. This GET path never resolves a binding."""
+ from copy import deepcopy
+ from core import sheet_binding
+ session._sheet_scope_context = {}
+ fresh = deepcopy(book)
+ owner = _role_on_workbook(session, book) == "owner"
+ try:
+ cache = sheet_binding.read_refresh_cache(_store(session), book["id"])
+ except Exception:
+ cache = {}
+ seen = {}
+ changed_values = False
+ for tab in fresh.get("sheets") or []:
for addr, cell in (tab.get("cells") or {}).items():
- if not isinstance(cell, dict) or cell.get("kind") != "bound":
+ if cell.get("kind") != "bound":
continue
- if endings_only and not _has_cell_inputs(cell.get("binding")):
+ if _is_view_binding(cell):
continue
- touched += 1
+ before = (cell.get("value"), cell.get("state"))
binding = cell.get("binding") or {}
db = str(binding.get("database") or "")
- if db not in seen_db:
+ if db not in seen:
try:
_require_database(session, db)
- seen_db[db] = None
- except Exception as exc: # noqa: BLE001
- seen_db[db] = exc
- failure = seen_db.get(db)
- if failure is not None:
- cell.update(_binding_refusal("database_unreadable", f"You cannot read {db}."))
- cell.pop("limits", None)
- continue
-
- cell_key = tab_ref + "!" + addr
- scope_key = _binding_scope_key(session, db, binding)
- hit = None
- if wid and scope_key and not _has_cell_inputs(binding):
+ seen[db] = True
+ except HTTPException:
+ seen[db] = False
+ slot = _binding_scope_key(session, db, binding)
+ hit = (cache.get("refreshCells", {}).get(str(tab.get("id")) + "!" + addr, {})
+ .get(slot)) if seen[db] else None
+ if not hit and seen[db]:
+ # ⛔⛔ THE BRIDGE S65-T03 LEFT OUT (SDLC 65 combine). It moved this reader onto
+ # `sheet_binding.read_refresh_cache` while `core.sheets`' scope cache kept its two
+ # writers: `compute_cell` fills BOTH, but the MCP gateway
+ # (`routes_external.py::cache_binding`) fills only the legacy one. So a binding
+ # written through MCP was invisible to its own next read and answered
+ # `sheet_refresh_required` with NO VALUE. Measured by `verify_perm_scope.py::An
+ # editor binding write replaces the old scope cache before reading its result`:
+ # got None, want 75.
+ # ⭐ A READ, never a second write: the legacy slot is keyed by the SAME
+ # `_binding_scope_key`, so C5's per-scope isolation is the one this already
+ # enforces, and nothing here resolves a binding (C2/R4 is untouched). Writing the
+ # new cache from the gateway instead was tried and REJECTED: it makes
+ # `refreshCells` non-empty, which retires the pre-C2 passthrough below for every
+ # OTHER cell and blanked a bound cell that had a perfectly good stored value.
try:
- hit = sheets.cached_scope_value(store, wid, cell_key, scope_key)
- except Exception: # noqa: BLE001
- hit = None
- if hit is not None:
- cell["state"] = "fresh"
- cell["value"] = hit.get("value")
- cell["display"] = "" if hit.get("value") is None else str(hit.get("value"))
- cell["refreshed_at"] = hit.get("refreshedAt")
- cell.pop("refusal", None)
- #: ⚠ NOT CLAIMED ON A CACHE HIT: `limits` describes THIS resolve's own truncation,
- #: and a cached slot never recorded one, so a prior (possibly stale) value on the
- #: cell is dropped rather than shown as if it described the answer just served.
- cell.pop("limits", None)
- continue
-
- try:
- out = _resolve_cell_binding(session, binding, fresh, tab_ref, view_memo=view_memo)
- except HTTPException as exc:
- detail = getattr(exc, "detail", None) or {}
- info = detail.get("error") if isinstance(detail, dict) else None
- cell.update(_binding_refusal(
- (info or {}).get("code", "sheet_unresolved"),
- (info or {}).get("message", "This cell could not be refreshed."),
- ))
- cell.pop("limits", None)
- continue
- cell["state"] = "fresh"
- cell["value"] = out["value"]
- cell["display"] = "" if out["value"] is None else str(out["value"])
- cell["refreshed_at"] = stamp
- cell.pop("refusal", None)
- if out["limits"]:
- cell["limits"] = out["limits"]
+ from core import sheets as _sheets
+ legacy = _sheets.cached_scope_value(
+ _store(session), book["id"], str(tab.get("id")) + "!" + addr, slot)
+ except Exception:
+ legacy = None
+ if legacy is not None:
+ value = legacy.get("value")
+ hit = {"cell": {"value": value, "state": "fresh",
+ "display": "" if value is None else str(value),
+ "refreshed_at": legacy.get("refreshedAt")}}
+ if hit:
+ _snapshot_cell(cell, hit["cell"])
+ elif not seen[db]:
+ _snapshot_cell(cell, _binding_refusal("database_unreadable", f"You cannot read {db}."))
+ elif owner and not cache.get("refreshCells") and session.user.get("role") == "admin":
+ # Existing workbooks keep their pre-C2 last result until the first Refresh.
+ pass
else:
- cell.pop("limits", None)
- #: ⚠ FILLED AFTER THE CELL ABOVE IS DONE READING `out`, and best effort: a cache-write
- #: failure only costs a future recompute, never a wrong number, so it never raises past
- #: here.
- if wid and scope_key:
- try:
- sheets.fill_scope_cache(store, wid, cell_key, scope_key, out["value"], stamp,
- out["database"])
- except Exception: # noqa: BLE001
- pass
- if not touched:
- return fresh
- # ⭐ EVERY FORMULA IS RE-EVALUATED TOO, over the numbers this reader is entitled to. Leaving
- # them alone would serve the owner's totals sitting on top of the reader's inputs, which is
- # the leak this whole function exists to close, one layer up.
- from modules import sheet as sheet_mod
- import formula_runtime as fr
- _recompute_formulas(fresh, fr, sheet_mod)
- fresh["refreshedFor"] = str(session.uname or "")
+ _snapshot_cell(cell, _binding_refusal("sheet_refresh_required",
+ "Refresh this workbook to resolve this cell under your current access."))
+ changed_values |= before != (cell.get("value"), cell.get("state"))
+ # ⛔⛔ A STALE FORMULA IS A REASON TO RECOMPUTE TOO, AND MISSING IT MADE THE TWO SERVER READS
+ # DISAGREE (SDLC 65 combine). C2/R4 says this GET path never RESOLVES A BINDING, and it still
+ # does not: `_recompute_formulas` evaluates the formula graph over cell values that are already
+ # present and opens no database. Gating it on `changed_values` alone meant that after a
+ # structure edit -- which stores every dependent formula `stale` with no value and moves no
+ # bound cell -- the UI read returned `{'kind': 'formula', 'state': 'stale'}` with no figure
+ # while `sheet_read_cells` (MCP) returned the computed one. `verify_perm_scope.py::The UI read
+ # and MCP read agree on stored cells, figures and formats` measured exactly that pair: D1 and
+ # F1 fresh 30.0 / 60.0 from MCP against stale-and-blank from the UI. A non-browser consumer has
+ # no client engine to cover for it, so the blank is what it would have shown.
+ stale_formula = any(
+ cell.get("kind") == "formula" and (cell.get("state") != "fresh" or "value" not in cell)
+ for tab in (fresh.get("sheets") or []) for cell in (tab.get("cells") or {}).values())
+ if changed_values or stale_formula:
+ from modules import sheet as sheet_mod
+ import formula_runtime as fr
+ _recompute_formulas(fresh, fr, sheet_mod)
+ if not owner:
+ fresh["refreshedFor"] = str(session.uname or "")
return fresh
@@ -1121,17 +1154,12 @@ def _response_workbook(book):
def _read_for(session: Session, book):
- """The workbook this session is entitled to see: stored for the owner, re-resolved otherwise.
+ """Project saved results under the current access wall without resolving any binding.
W60-T09, 2026-09-14: R28 checks visibility before this projection. Admitted legacy readers
- and grantees still receive their own scoped numbers, never the owner's cached numbers.
+ and grantees receive a proven scoped snapshot or a request to Refresh.
"""
book = _response_workbook(book)
- if _role_on_workbook(session, book) == "owner":
- if any(_has_cell_inputs(cell.get("binding"))
- for tab in book.get("sheets") or [] for cell in (tab.get("cells") or {}).values()):
- return _reresolved_for_reader(session, book, endings_only=True)
- return book
return _reresolved_for_reader(session, book)
@@ -1484,7 +1512,9 @@ async def start_sheet_import(file: UploadFile = File(...),
if not name.lower().endswith(".xlsx"):
raise err(400, "import_not_xlsx",
"Only an .xlsx workbook can be imported. Save your file as .xlsx and try again.")
- data = await file.read()
+ # Read at most one byte past the cap. UploadFile may be backed by a large temporary file; an
+ # unbounded read would allocate hostile input merely to discover that it is too large.
+ data = await file.read(sheet_import.MAX_IMPORT_BYTES + 1)
if len(data) > sheet_import.MAX_IMPORT_BYTES:
raise HTTPException(status_code=413, detail={"error": {
"code": "import_too_large",
@@ -1530,6 +1560,51 @@ def _mb(byte_count):
return "%.1f MB" % (float(byte_count) / (1024.0 * 1024.0))
+# ── C5: Excel-shaped bounded automation (S63-T22) ──────────────────────────────────────────
+
+@router.get("/sheets/automation/manifest")
+def sheet_automation_manifest(session: Session = Depends(require_session)):
+ """The canonical capability bytes shared with the Python SDK.
+
+ The session wall is deliberate: this is product capability metadata, but keeping it beside
+ the authenticated workbook doors avoids creating a second unauthenticated API convention.
+ """
+ del session
+ from core import sheet_compat
+ return Response(content=sheet_compat.capability_manifest_bytes(), media_type="application/json")
+
+
+def _sheet_automation_calculator(book):
+ """Inject the existing formula runtime without making ``platform/core`` import the API."""
+ from modules import sheet as sheet_mod
+ import formula_runtime as fr
+ _recompute_formulas(book, fr, sheet_mod)
+
+
+@router.post("/sheets/{workbook_id}/automation")
+def execute_sheet_automation(workbook_id: str, body: dict = Body(default=None),
+ session: Session = Depends(require_session)):
+ """Execute one manifest operation through the shared core planner and targeted patcher."""
+ from core import sheet_compat, sheets
+ raw = dict(body or {})
+ operation = str(raw.get("operation") or "")
+ arguments = raw.get("arguments")
+ try:
+ # Unsupported and malformed operation-level calls stop here, before any store update
+ # callback can be entered. Authorization then applies to the declared access class.
+ spec = sheet_compat.require_supported(operation)
+ book = sheets.get_workbook(_store(session), workbook_id)
+ _require_reader(session, book)
+ if spec["access"] == "write":
+ _require_editor(session, book)
+ return sheet_compat.execute(
+ _store(session), workbook_id, operation, arguments,
+ actor=session.uname, actor_name=session.user.get("name") or session.uname,
+ calculator=_sheet_automation_calculator)
+ except sheets.SheetRefusal as exc:
+ raise _refuse(exc)
+
+
@router.get("/sheets/{workbook_id}")
def read_sheet(workbook_id: str, tabs: str = "",
session: Session = Depends(require_session)):
@@ -1695,6 +1770,8 @@ def _recomputed_tab_cells(session: Session, workbook_id: str, tab_ref: str, *, c
cache_key=(session.tenant, session.uname, workbook_id), changes=edits)
# Date and filter-cell edits also change database totals. Recompute all readers, then formulas.
ending_cells = set()
+ refreshed_entries = []
+ context_memo = {}
for source in book.get("sheets") or []:
for addr, cell in (source.get("cells") or {}).items():
binding = cell.get("binding") or {}
@@ -1703,6 +1780,11 @@ def _recomputed_tab_cells(session: Session, workbook_id: str, tab_ref: str, *, c
try:
result = _resolve_cell_binding(session, binding, book, source.get("id"))
cell.update(value=result["value"], display="" if result["value"] is None else str(result["value"]), state="fresh")
+ cell["refreshed_at"] = _now_iso()
+ if result.get("limits"):
+ cell["limits"] = result["limits"]
+ else:
+ cell.pop("limits", None)
cell.pop("refusal", None)
except HTTPException as exc:
info = ((exc.detail or {}).get("error") or {})
@@ -1711,7 +1793,16 @@ def _recomputed_tab_cells(session: Session, workbook_id: str, tab_ref: str, *, c
info.get("message", "The Ending date could not be read. Choose a valid date."),
))
ending_cells.add((source.get("id"), addr))
+ refreshed_entries.append((str(source.get("id")) + "!" + addr,
+ _binding_scope_key(session, str(binding.get("database") or ""), binding),
+ {"tokens": {}, "context": _refresh_context(book, binding, context_memo),
+ "cell": {k: cell[k] for k in ("value", "display", "state", "refreshed_at", "limits", "refusal")
+ if k in cell}}))
if ending_cells:
+ from core.sheet_binding import save_refresh_cache
+ save_refresh_cache(_store(session), workbook_id, refreshed_entries,
+ {c.get("binding", {}).get("database") for t in book.get("sheets", [])
+ for c in t.get("cells", {}).values() if c.get("kind") == "bound"})
_recompute_formulas(book, fr, sheet_mod)
found = sheets.find_tab(book, tab_ref)
out = {}
@@ -1887,117 +1978,221 @@ def unshare_sheet(workbook_id: str, who: str, session: Session = Depends(require
@router.post("/sheets/{workbook_id}/refresh")
def refresh_sheet(workbook_id: str, session: Session = Depends(require_session)):
- """R10: recompute every bound cell, then every formula, and restamp `refreshed_at`.
-
- ⭐ THE WALL RUNS PER DISTINCT DATABASE, ON OPEN, NOT ONLY PER CELL (R16). A workbook that binds
- forty cells to one database asks once; a reader who has lost access to that database gets a
- refusal on every one of its cells rather than a stale number quietly repainting.
-
- ⚠ Each distinct `view_id` is resolved ONCE, and as of the wave-53 follow-up to W53-T12 that
- sentence is finally TRUE OF THE CODE. It was written of `seen_db`, which deduplicates the
- DATABASE WALL and nothing else: the resolve underneath it ran per cell, so a forty-cell
- workbook did forty `table_store.find_view` calls and forty legacy migrations with them.
- `view_memo` below is the dedupe this sentence describes.
-
- ⛔⛔ C5 — THIS IS A WRITE DOOR, BECAUSE IT ENDS IN `put_workbook`, AND A VIEWER IS REFUSED
- HERE FOR A REASON THAT IS NOT ABOUT PERMISSION TO SEE. A viewer whose refresh were allowed
- would compute every bound cell under THEIR scope and then SAVE those numbers into the owner's
- document, so a narrowly walled reader pressing Refresh would silently shrink the owner's
- workbook. The reader's own re-resolve happens on the READ path, in memory, and never lands.
- ⚠ An EDITOR still writes their own numbers here, which is what an edit grant means and is
- booked as a residual rather than widened away.
+ """C2/R4: Refresh resolves only databases whose token or binding inputs changed.
+
+ Viewers refresh their own server cache. Only editors save the workbook document.
+ One token is read per distinct database on the unchanged path, regardless of cell count.
"""
- from core import sheets
+ from copy import deepcopy
+ from core import sheets, sheet_binding
from modules import sheet as sheet_mod
import formula_runtime as fr
+ from time import perf_counter
+ store = _store(session)
+ session._sheet_scope_context = {}
try:
- book = sheets.get_workbook(_store(session), workbook_id)
- _require_editor(session, book)
+ book = sheets.get_workbook(store, workbook_id)
+ _require_reader(session, book)
except sheets.SheetRefusal as exc:
raise _refuse(exc)
-
+ editable = _role_on_workbook(session, book) in ("owner", "edit")
+ try:
+ cache = sheet_binding.read_refresh_cache(store, workbook_id)
+ except Exception:
+ cache = {}
stamp = _now_iso()
- seen_db = {}
- # ⭐ ONE MEMO FOR THE WHOLE REFRESH, keyed `(database, view_id)`. `seen_db` deduplicates the
- # WALL and never the scope, so forty cells on one saved view used to resolve that view forty
- # times. See `_view_filter`: the second lookup behind it builds a workspace assembly, which is
- # work no workbook should pay for per cell. Per REQUEST, never on the runtime: a view's filter
- # may be edited between two refreshes and a longer lived cache would serve the old rows.
- view_memo = {}
- for tab in book.get("sheets") or ():
+ seen_db, tokens, entries, status = {}, {}, [], {}
+ view_memo, related_memo = {}, {}
+ context_memo = {}
+ token_reads = 0
+ input_graph = None
+
+ def input_databases(binding, tab_ref):
+ nonlocal input_graph
+ nodes, _ = _fe_parts((binding.get("scope") or {}).get("filter"))
+ related_nodes, _ = _fe_parts((binding.get("related") or {}).get("filter"))
+ refs = [str((node.get("valueRef") or {}).get("ref") or "")
+ for node in sheet_mod.filter_cell_nodes([*nodes, *related_nodes])]
+ period = binding.get("period") or {}
+ if period.get("ref"):
+ refs.append(period["ref"])
+ if not refs:
+ return set()
+ tabs = book.get("sheets") or []
+ names = _tab_scope_names(tabs)
+ if input_graph is None:
+ documents = {fr.a1_key(addr, name): cell for tab, name in zip(tabs, names)
+ for addr, cell in (tab.get("cells") or {}).items()}
+ engine = fr.SheetCalculation({key: fr._document_spec(cell, key.rpartition("!")[0])
+ for key, cell in documents.items()}, sheets=names,
+ named_ranges=book.get("namedRanges"), date_system=book.get("dateSystem", "1900"))
+ input_graph = documents, engine
+ documents, engine = input_graph
+ home = _resolve_home_tab(book, tabs, names, tab_ref)
+ pending, visited, deps = [_asked_key(ref, home, fr) for ref in refs], set(), set()
+ while pending:
+ key = pending.pop()
+ if key in visited:
+ continue
+ visited.add(key)
+ cell = documents.get(key) or {}
+ if cell.get("kind") == "bound":
+ deps.add(str((cell.get("binding") or {}).get("database") or ""))
+ pending.extend(engine.consumed.get(key, ()))
+ return deps
+
+ def token(db):
+ nonlocal token_reads
+ if db not in tokens:
+ token_reads += 1
+ try:
+ # ⭐ Arms `core.sheet_binding.mirror_change_token` (SDLC 65 combine). The mirror
+ # reader lives in `harness` because `core` never imports up, and the API is the
+ # layer allowed to reach both ways, so the wiring belongs exactly here.
+ import harness.mirror_tokens # noqa: F401 (registers the provider on import)
+ tokens[db] = sheet_binding.database_change_token(store, db)
+ except Exception:
+ # Failure to prove unchanged means resolve, never silently reuse a value.
+ tokens[db] = None
+ return tokens[db]
+
+ def dependencies(db, binding, tab_ref):
+ deps = {db}
+ if binding.get("related"):
+ key = (db, binding["related"].get("via"))
+ if key not in related_memo:
+ related_memo[key] = _binding_related_entry(session, db, binding)
+ deps.add(related_memo[key]["target"])
+ nodes, _ = _fe_parts((binding.get("scope") or {}).get("filter"))
+ linked = sheet_mod.linked_filter_ids(nodes)
+ if linked:
+ names = {sheet_mod.linked_field_parts(key)[0] for key in linked}
+ deps.update(b["source"] for b in _linked_customer_bridges(db) if b["key"] in names)
+ deps.update(input_databases(binding, tab_ref))
+ return deps
+
+ bound_inputs = {fr.a1_key(addr, name): (tab.get("id"), addr, cell)
+ for tab, name in zip(book.get("sheets") or [], _tab_scope_names(book.get("sheets") or []))
+ for addr, cell in (tab.get("cells") or {}).items() if cell.get("kind") == "bound"}
+
+ def cached_input(key):
+ item = bound_inputs.get(key)
+ if item is None:
+ return None
+ tab_ref, addr, cell = item
+ binding = cell.get("binding") or {}
+ db = str(binding.get("database") or "")
+ _require_database(session, db)
+ scope_key = _binding_scope_key(session, db, binding)
+ hit = cache.get("refreshCells", {}).get(str(tab_ref) + "!" + addr, {}).get(scope_key)
+ sources = {db: token(db) for db in dependencies(db, binding, tab_ref)}
+ if (hit and all(sources.values()) and hit.get("tokens") == sources
+ and hit.get("context") == _refresh_context(book, binding, context_memo)
+ and hit["cell"].get("state") == "fresh"):
+ return hit["cell"]
+ return None
+
+ for tab in book.get("sheets") or []:
for addr, cell in (tab.get("cells") or {}).items():
- if not isinstance(cell, dict) or cell.get("kind") != "bound":
+ if cell.get("kind") != "bound" or _is_view_binding(cell):
continue
- if _is_view_binding(cell):
- continue # C2: a placed view refreshes through its own door
binding = cell.get("binding") or {}
db = str(binding.get("database") or "")
- if db not in seen_db:
- try:
- _require_database(session, db)
- seen_db[db] = None
- except Exception as exc: # noqa: BLE001
- seen_db[db] = exc
- failure = seen_db.get(db)
- if failure is not None:
- cell.update(_binding_refusal("database_unreadable", f"You cannot read {db}."))
- cell.pop("limits", None)
- continue
+ cell_key = str(tab.get("id")) + "!" + addr
+ scope_key = _binding_scope_key(session, db, binding)
+ context = _refresh_context(book, binding, context_memo)
+ hit = cache.get("refreshCells", {}).get(cell_key, {}).get(scope_key)
+ source_tokens = {}
try:
- out = _resolve_cell_binding(session, binding, book, tab.get("id"), view_memo=view_memo)
+ if db not in seen_db:
+ try:
+ _require_database(session, db)
+ seen_db[db] = None
+ except HTTPException as exc:
+ seen_db[db] = exc
+ if seen_db[db] is not None:
+ raise seen_db[db]
+ source_tokens = {key: token(key) for key in sorted(dependencies(db, binding, tab.get("id")))}
+ if (hit and all(source_tokens.values()) and hit.get("tokens") == source_tokens
+ and hit.get("context") == context
+ and hit["cell"].get("state") == "fresh"):
+ _snapshot_cell(cell, hit["cell"])
+ status.setdefault(db, "skipped")
+ continue
+ status[db] = "resolved"
+ previous = getattr(session, "_sheet_refresh_input", None)
+ session._sheet_refresh_input = cached_input
+ try:
+ out = _resolve_cell_binding(session, binding, book, tab.get("id"), view_memo=view_memo)
+ finally:
+ session._sheet_refresh_input = previous
+ limits = list(out.get("limits") or [])
+ if not all(source_tokens.values()):
+ limits.append({"code": "change_token_unavailable",
+ "cause": "The database change token could not be read. This Refresh resolved the cell.",
+ "recommendation": "Restore the database store and connector sync receipts to enable skipping."})
+ snapshot = {"state": "fresh", "value": out["value"],
+ "display": "" if out["value"] is None else str(out["value"]),
+ "refreshed_at": stamp}
+ if limits:
+ snapshot["limits"] = limits
except HTTPException as exc:
- detail = getattr(exc, "detail", None) or {}
- info = detail.get("error") if isinstance(detail, dict) else None
- cell.update(_binding_refusal(
- (info or {}).get("code", "sheet_unresolved"),
- (info or {}).get("message", "This cell could not be refreshed."),
- ))
- cell.pop("limits", None)
- continue
- cell["state"] = "fresh"
- cell["value"] = out["value"]
- cell["display"] = "" if out["value"] is None else str(out["value"])
- cell["refreshed_at"] = stamp
- cell.pop("refusal", None)
- if out["limits"]:
- cell["limits"] = out["limits"]
- else:
- cell.pop("limits", None)
+ info = ((exc.detail or {}).get("error") or {})
+ snapshot = _binding_refusal(info.get("code", "sheet_unresolved"),
+ info.get("message", "This cell could not be refreshed."))
+ status[db] = "refused"
+ _snapshot_cell(cell, snapshot)
+ entries.append((cell_key, scope_key, {"tokens": source_tokens,
+ "context": context, "cell": deepcopy(snapshot)}))
+
+ # Read again only after resolving. A concurrent write must not certify a mixed result
+ # as reusable. The unchanged path still costs exactly one token read per database.
+ resolved_sources = {db for _, _, entry in entries for db in entry["tokens"]}
+ changed = set()
+ for db in resolved_sources:
+ if tokens[db] is None:
+ continue # Already disclosed as unverified and never eligible for a hit.
+ token_reads += 1
+ try:
+ if sheet_binding.database_change_token(store, db) != tokens[db]:
+ changed.add(db)
+ except Exception:
+ changed.add(db)
+ for cell_key, _, entry in entries:
+ if changed.intersection(entry["tokens"]):
+ entry["tokens"] = {} # no next-refresh hit, even if the writer rolls back
+ if entry["cell"].get("state") == "fresh":
+ entry["cell"].setdefault("limits", []).append({"code": "change_during_refresh",
+ "cause": "This database could not be confirmed unchanged while the cell was being resolved.",
+ "recommendation": "Refresh again to resolve the cell from the updated database."})
+ tab_ref, addr = cell_key.split("!", 1)
+ _snapshot_cell(sheets.find_tab(book, tab_ref)["cells"][addr], entry["cell"])
- # ⭐⭐ R4 / A1: THE RECALC IS TIMED AND THE TIME IS SHOWN. The prototype measured 2.9 s as the
- # FLOOR for the owner's 11,300 formulas (2.9 to 6.2 s observed), so a refresh is a wait a
- # person notices, and a wait nobody explains reads as a hang. `recalc_ms` is what the client
- # renders as "Recalculated in N.N s".
- #
- # ⚠ IT TIMES THE RECALC, NOT THE ROUTE. The bound-cell loop above is database work whose
- # duration says nothing about the formula engine, and the save below is store work; folding
- # either into this number would make the figure on the screen unable to answer the question it
- # exists for, which is whether the ENGINE is inside A1's 5 s target.
- from time import perf_counter
started = perf_counter()
_recompute_formulas(book, fr, sheet_mod)
recalc_ms = round((perf_counter() - started) * 1000.0, 1)
- try:
- # R19: book came from get_workbook; clamping on refresh would destroy imported widths.
- saved = sheets.put_workbook(_store(session), workbook_id,
- {"name": book.get("name"), "sheets": book.get("sheets"),
- "refreshed_at": stamp, "rev": book.get("rev")},
- actor=session.uname,
- actor_name=session.user.get("name") or session.uname,
- lossless=True)
- except sheets.SheetRefusal as exc:
- raise _refuse(exc)
- # ⭐⭐ R3 / W52-T12: REFRESH IS THE ONE EVENT ALLOWED TO INVALIDATE A READER'S CACHED SCOPE
- # VALUE, so it is the one door that clears the whole per-scope cache. Every reader's next GET
- # recomputes once, under their own scope, and fills its own slot again from there.
- sheets.clear_scope_cache(_store(session), workbook_id)
- return {"workbook": _response_workbook(saved), "recalc_ms": recalc_ms}
+ if editable:
+ try:
+ saved = sheets.put_workbook(store, workbook_id,
+ {"name": book.get("name"), "sheets": book.get("sheets"),
+ "refreshed_at": stamp, "rev": book.get("rev")},
+ actor=session.uname, actor_name=session.user.get("name") or session.uname,
+ lossless=True)
+ except sheets.SheetRefusal as exc:
+ raise _refuse(exc)
+ else:
+ saved = book
+ saved["refreshedFor"] = str(session.uname or "")
+ if entries:
+ sheet_binding.save_refresh_cache(store, workbook_id, entries, status)
+ return {"workbook": _response_workbook(saved), "recalc_ms": recalc_ms,
+ "binding_refresh": {"databases": status, "token_reads": token_reads}}
def _now_iso():
- from datetime import datetime, timezone
- return datetime.now(timezone.utc).replace(microsecond=0).isoformat()
+ from core import write_execution
+ return write_execution.utcnow().replace(microsecond=0).isoformat()
#: ⛔ THE NAME A TAB GETS WHEN IT CANNOT KEEP ITS OWN, and it is built to be UNTYPEABLE on two
@@ -2288,8 +2483,11 @@ def _read_binding_cell(session, ref, *, book=None, tab=None, today=None, seen=()
refuse("the cell or its sheet is missing or blank.")
if cell.get("kind") == "bound":
try:
- value = _resolve_binding(session, cell.get("binding"), today=today,
- book=book, tab=key.rpartition("!")[0], ending_seen=(*seen, key))["value"]
+ lookup = getattr(session, "_sheet_refresh_input", None)
+ cached = lookup(key) if lookup is not None else None
+ value = (cached["value"] if cached is not None else _resolve_binding(
+ session, cell.get("binding"), today=today, book=book,
+ tab=key.rpartition("!")[0], ending_seen=(*seen, key))["value"])
except HTTPException as exc:
reason = ((exc.detail or {}).get("error") or {}).get("message", "its source is refused.")
refuse(reason)
@@ -2317,8 +2515,11 @@ def _read_binding_cell(session, ref, *, book=None, tab=None, today=None, seen=()
for dependency in needed:
attempted.add(dependency)
try:
- result = _resolve_binding(session, bound[dependency].get("binding"), today=today,
- book=book, tab=dependency.rpartition("!")[0], ending_seen=(*seen, key, dependency))
+ lookup = getattr(session, "_sheet_refresh_input", None)
+ result = lookup(dependency) if lookup is not None else None
+ if result is None:
+ result = _resolve_binding(session, bound[dependency].get("binding"), today=today,
+ book=book, tab=dependency.rpartition("!")[0], ending_seen=(*seen, key, dependency))
changes[dependency] = {"value": result["value"]}
except HTTPException as exc:
info = ((exc.detail or {}).get("error") or {})
@@ -2389,6 +2590,116 @@ def _resolve_cell_binding(session, binding, book, tab, **kwargs):
return _resolve_binding(session, binding, **kwargs)
+def _binding_related_entry(session, database, binding):
+ """Resolve one authored edge key against the permission-scoped declarations."""
+ if "related" not in binding:
+ return None
+ from core import perm_scope, relations, user_tables
+ from routes_grid import _source_fields
+
+ fields = _source_fields(session, database)
+ defs = {}
+ for key, definition in (user_tables.all_defs(st=session.runtime) or {}).items():
+ if perm_scope.nav_may_open(session.user, key, st=session.runtime):
+ declared = definition.get("fields") or []
+ hidden = perm_scope.hidden_keys(session.user, key, declared, st=session.runtime)
+ defs[key] = {**definition, "fields": [f for f in declared if f.get("key") not in hidden]}
+ entries = relations.related(database, fields=fields, defs=defs)
+ name = binding["related"]["via"]
+ matches = [entry for entry in entries if entry["key"] == name]
+ via = matches[0]["via"] if len(matches) == 1 else {"key": name}
+ entry = relations.find(database, via, fields=fields, defs=defs)
+ if entry is None:
+ candidates = [item["key"] for item in relations.via_candidates(entries, via)
+ if perm_scope.nav_may_open(session.user, item["target"], st=session.runtime)]
+ offered = (" Available relationships: " + ", ".join(candidates) + ".") if candidates else ""
+ # Write our own sentence: the shared pin refusal does not name via.key.
+ raise err(409, "unknown_relation",
+ f"Relationship '{name}' cannot be resolved on '{database}'."
+ + offered + " Choose a declared relationship for this database.")
+ target = _require_database(session, entry["target"])
+ if not entry.get("served"):
+ raise err(409, "relation_target_unavailable",
+ f"Relationship '{name}' cannot read '{target}' because it has no read-through "
+ "binding. Choose a relationship to a database with a live connection.")
+ # A through/manual link cannot be represented by this one declared column pair.
+ declaration = next((f for f in fields if f.get("key") == entry["via"].get("field")), {})
+ if entry["via"]["direction"] == "reverse":
+ declaration = next((f for f in defs.get(target, {}).get("fields", [])
+ if f.get("key") == entry["via"].get("field")), {})
+ link = declaration.get("link") or {}
+ if link.get("through") or link.get("selectionMode") == "manual":
+ raise err(409, "relation_source_unresolvable",
+ f"Relationship '{name}' does not declare a direct column join. "
+ "Choose a relationship with one readable column on each database.")
+ return entry
+
+
+def _binding_related_query(session, binding, entry, cur, spec, columns, *, window=None,
+ today=None, book=None, tab=None, ending_seen=()):
+ """Materialise the far side on the near cursor, then hand metadata to the compiler."""
+ if entry is None:
+ return None
+ from core import perm_scope, relations
+ from modules import sheet as sheet_mod
+
+ database, target = binding["database"], entry["target"]
+ name = binding["related"]["via"]
+ local_key, remote_key = entry["via"].get("from"), entry["via"].get("on")
+
+ def map_join(key, table, source_spec, cols, side):
+ hidden = perm_scope.hidden_keys(session.user, table, source_spec.get("fields") or [],
+ st=session.runtime)
+ if key and key not in hidden:
+ expr = (cols.get(key) or {}).get("sql")
+ if not expr:
+ # Identity and join columns can be absent from the visible field list.
+ # Only the trusted mirror map may supply their expressions.
+ mapped = (source_spec.get("cols") or {}).get(key)
+ expr = (source_spec.get("id") if key == relations.IDENTITY_COL else
+ mapped[0] if mapped else None)
+ if expr and str(expr).strip().upper() != "NULL":
+ cols[key] = {**cols.get(key, {}), "sql": expr}
+ return
+ code = "relation_source_unresolvable" if side == "source" else "relation_target_unavailable"
+ raise err(409, code,
+ f"Relationship '{name}' cannot read its {side} column '{key}' on '{table}'. "
+ "Choose a relationship whose join columns are available on the live connection.")
+
+ map_join(local_key, database, spec, columns, "source")
+ wall = ((perm_scope.entry(session.user, target) or {}).get("filter")
+ if perm_scope.row_scope_applies(session.user, target) else None)
+ wall_nodes, wall_conj = _fe_parts(wall)
+ scope_nodes, scope_conj = _fe_parts(binding["related"]["filter"])
+ scope_nodes = _filter_cell_values(session, scope_nodes, book=book, tab=tab,
+ today=today, seen=ending_seen)
+ needed = (sheet_mod.filter_columns_named(scope_nodes)
+ | sheet_mod.filter_columns_named(wall_nodes) | {remote_key})
+ try:
+ rcur, rspec, rcols = _mirror(session)(target, metric_window=window, needed=needed)
+ except HTTPException as exc:
+ detail = exc.detail if isinstance(exc.detail, dict) else {}
+ if exc.status_code == 404 or (detail.get("error") or {}).get("code") == "not_read_through":
+ raise err(409, "relation_target_unavailable",
+ f"Relationship '{name}' cannot read '{target}' on this live connection. "
+ "Choose a relationship to an available database.") from exc
+ raise
+ if rcur is not cur:
+ raise err(409, "relation_cursor_mismatch",
+ f"Relationship '{name}' cannot be queried on the same live connection as "
+ f"'{database}'. Retry after both databases share a connection.")
+ hidden = perm_scope.hidden_keys(session.user, target, rspec.get("fields") or [],
+ st=session.runtime)
+ rcols = {key: value for key, value in rcols.items() if key not in hidden}
+ map_join(remote_key, target, rspec, rcols, "target")
+ depends = sheet_mod.filter_columns_named(scope_nodes)
+ return {"via": name, "database": target, "local_key": local_key, "remote_key": remote_key,
+ "spec": rspec, "columns": rcols, "where": entry.get("where"),
+ "scope_nodes": scope_nodes, "scope_conj": scope_conj,
+ "wall_nodes": wall_nodes, "wall_conj": wall_conj,
+ "limits": [row for key, row in (rspec.get("unbounded") or {}).items() if key in depends]}
+
+
def _resolve_binding(session: Session, raw_binding, *, today=None, view_memo=None,
book=None, tab=None, ending_seen=()):
"""Evaluate ONE C2 binding and return `{value, limits, sql_columns}`.
@@ -2450,6 +2761,9 @@ def _resolve_binding(session: Session, raw_binding, *, today=None, view_memo=Non
if binding["aggregate"].get("field"):
needed.add(str(binding["aggregate"]["field"]))
+ related_entry = _binding_related_entry(session, database, binding)
+ if related_entry:
+ needed.add(related_entry["via"].get("from"))
cur, spec, cols = _mirror(session)(database, metric_window=window, needed=needed)
cols = dict(cols)
scope_nodes = _linked_filter_scope(session, database, scope_nodes, cur, spec, cols, today=today)
@@ -2491,9 +2805,12 @@ def _resolve_binding(session: Session, raw_binding, *, today=None, view_memo=Non
# spec's column expressions; applying it a second time would double-count the exclusion and
# break every joined spec, whose partner column is `sol.order_partner_id` or `aml.partner_id`.
try:
+ related = _binding_related_query(
+ session, binding, related_entry, cur, spec, cols, window=window, today=today,
+ book=book, tab=tab, ending_seen=ending_seen)
sql, params, limits = sheet_mod.build_aggregate_query(
spec, binding, cols, scope_nodes=scope_nodes, scope_conj=scope_conj,
- wall_nodes=wall_nodes, wall_conj=wall_conj, today=today)
+ wall_nodes=wall_nodes, wall_conj=wall_conj, today=today, related=related)
except sheet_mod.SheetRefusal as exc:
raise err(400, exc.code, exc.message)
except ValueError:
@@ -2534,6 +2851,50 @@ def _resolve_binding(session: Session, raw_binding, *, today=None, view_memo=Non
return {"value": value, "limits": limits, "database": database, "binding": binding}
+def execute_daily_sales_repair(session: Session, request, *, backup_store, workbook_id=None):
+ """Run the explicit T21 repair through the normal scoped binding and formula engines.
+
+ This is intentionally not an HTTP route. A normal workbook request must never acquire an
+ exceptional tenant-data mutation surface. The approved operator calls this helper with the
+ exact marker/revision/source oracle and a backup store, naming the workbook with workbook_id.
+ Omission retains the owner's workbook default. The helper explicitly points the core workflow
+ at that target; the core refuses any workbook it was not pointed at and performs its own marker,
+ snapshot, backup and readback checks.
+ """
+ from core import daily_sales_repair, sheets
+ import formula_runtime
+
+ try:
+ if str(session.tenant or "") != daily_sales_repair.TARGET_TENANT:
+ raise daily_sales_repair.RepairRefusal(
+ "repair_tenant_mismatch",
+ "This operation may run only in the Royal Imports workspace.")
+ target = str(workbook_id or "").strip() or daily_sales_repair.TARGET_WORKBOOK_ID
+ book = sheets.get_workbook(_store(session), target)
+ _require_editor(session, book)
+
+ view_memo = {}
+
+ def resolve(binding, projected, tab):
+ tab_ref = str(tab.get("id") or tab.get("name") or "")
+ return _resolve_cell_binding(
+ session, binding, projected, tab_ref, view_memo=view_memo)
+
+ def formula_check(projected):
+ formula_runtime.recompute_document(
+ projected, today=(request or {}).get("calculation_date"))
+ return {"book": projected,
+ "errors": daily_sales_repair.formula_error_census(projected)}
+
+ return daily_sales_repair.execute(
+ _store(session), backup_store, request,
+ target_workbook_id=target,
+ resolve_binding=resolve, formula_check=formula_check,
+ actor=session.uname, actor_name=session.user.get("name") or session.uname)
+ except sheets.SheetRefusal as exc:
+ raise _refuse(exc)
+
+
def _fe_parts(tree):
"""`(nodes, conj)` for a stored `{conj, nodes}` filter, through the ONE reader of that shape."""
from modules import sheet as sheet_mod
@@ -2716,6 +3077,11 @@ def compute_cell(workbook_id: str, addr: str, body: dict = Body(default=None),
saved["refreshed_at"] = stamp
saved.pop("refusal", None)
+ if limits:
+ saved["limits"] = limits
+ else:
+ saved.pop("limits", None)
+
try:
landed = sheets.patch_cells(_store(session), workbook_id, tab_ref, {key: saved},
actor=session.uname,
@@ -2735,6 +3101,13 @@ def compute_cell(workbook_id: str, addr: str, body: dict = Body(default=None),
except Exception: # noqa: BLE001
pass
+ from core.sheet_binding import save_refresh_cache
+ save_refresh_cache(_store(session), workbook_id, [(tab_ref + "!" + key,
+ _binding_scope_key(session, str(binding.get("database") or ""), binding),
+ {"tokens": {}, "context": _refresh_context(book, binding),
+ "cell": {k: saved[k] for k in ("value", "display", "state", "refreshed_at", "limits", "refusal")
+ if k in saved}})], [binding.get("database")])
+
wire = dict(saved)
#: ⚠ `limits` RIDES THE ANSWER AND IS NOT STORED, because `core.sheets.clean_cell` carries no
#: such key: it describes THIS resolve's truncation, not the cell.
@@ -2916,24 +3289,35 @@ def drill_cell(workbook_id: str, addr: str, session: Session = Depends(require_s
window = _ending_window(session, binding, book=book, tab=tab.get("id"))
except sheet_mod.SheetRefusal as exc:
raise err(400, exc.code, exc.message)
+ related_entry = _binding_related_entry(session, database, binding)
+ needed = (sheet_mod.filter_columns_named(scope_nodes)
+ | sheet_mod.filter_columns_named(wall_nodes))
+ if related_entry:
+ needed.add(related_entry["via"].get("from"))
cur, spec, cols = _mirror(session)(
database, metric_window=window,
- needed=(sheet_mod.filter_columns_named(scope_nodes)
- | sheet_mod.filter_columns_named(wall_nodes)))
+ needed=needed)
cols = dict(cols)
scope_nodes = _linked_filter_scope(session, database, scope_nodes, cur, spec, cols)
try:
+ related = _binding_related_query(
+ session, binding, related_entry, cur, spec, cols, window=window,
+ book=book, tab=tab.get("id"))
count_sql, count_params, limits = sheet_mod.build_aggregate_query(
spec, {"aggregate": {"fn": "count"}}, cols, scope_nodes=scope_nodes,
- scope_conj=scope_conj, wall_nodes=wall_nodes, wall_conj=wall_conj)
+ scope_conj=scope_conj, wall_nodes=wall_nodes, wall_conj=wall_conj, related=related)
+ ids_sql, ids_params, _ = sheet_mod.build_ids_query(
+ spec, cols, scope_nodes=scope_nodes, scope_conj=scope_conj,
+ wall_nodes=wall_nodes, wall_conj=wall_conj, related=related)
except sheet_mod.SheetRefusal as exc:
raise err(400, exc.code, exc.message)
+ except ValueError:
+ raise err(409, "scope_not_applicable",
+ "This cell's scope uses a restriction that cannot be applied on its live "
+ "connection. Edit the scope or ask an administrator to update the restriction.")
total = (cur.execute(count_sql, count_params).fetchone() or [0])[0]
- ids_sql, ids_params, _ = sheet_mod.build_ids_query(
- spec, cols, scope_nodes=scope_nodes, scope_conj=scope_conj,
- wall_nodes=wall_nodes, wall_conj=wall_conj)
record_ids = [int(r[0]) for r in cur.execute(ids_sql, ids_params).fetchall() if r and r[0]]
out = {"database": database, "record_ids": record_ids, "count": int(total or 0),
diff --git a/api/routes_slack.py b/api/routes_slack.py
index cb549f279229b6cc6fa156ee0990d9d7c5e3df92..dbb2832f53e6ee51bca9fa23aef3effe95e8a71d 100644
--- a/api/routes_slack.py
+++ b/api/routes_slack.py
@@ -197,11 +197,11 @@ def agent_may_read(agent, module):
return bool(perm_scope.may_access(agent_principal(agent), module))
-def agent_visible_fields(agent, fields, module):
+def agent_visible_fields(agent, fields, module, st=None):
"""`fields` minus this channel's hidden closure — the SAME transitive closure a person gets,
so a formula over a hidden column cannot leak it back through arithmetic."""
import core.perm_scope as perm_scope
- return perm_scope.visible_fields(fields, agent_principal(agent), module)
+ return perm_scope.visible_fields(fields, agent_principal(agent), module, st=st)
def agent_rows(agent, rows, module, fields, ctx=None, st=None):
@@ -218,15 +218,13 @@ def agent_rows(agent, rows, module, fields, ctx=None, st=None):
default for the same reason `perm_scope.apply_row_scope`'s is: without one this is
byte-identical to the wall that shipped before the argument existed.
- ⚠ `hidden_keys` DELIBERATELY DOES NOT TAKE IT IN THIS CHANGE. Its field-grant leg
- under-hides without a handle (see `perm_scope.visible_overlays`), which is a real and
- separate gap on this surface — but it is the FIELD wall, it moves what a channel receives
- rather than what I16 asked for, and widening two walls in one ticket is how a permission
- change stops being reviewable.
+ The field projection takes the same runtime too. Without it a user-generated field can be
+ absent from the declared grant while its value survives in the row payload — precisely the
+ half-wall a customer-intelligence digest must not cross.
"""
import core.perm_scope as perm_scope
p = agent_principal(agent)
- hide = perm_scope.hidden_keys(p, module, fields)
+ hide = perm_scope.hidden_keys(p, module, fields, st=st)
kept = perm_scope.apply_row_scope(rows, p, module, fields, ctx, st=st)
# Both wires, never one: the field list and the row payload are separate, and stripping only
# the first leaves the value sitting in the second where anyone can read it.
@@ -298,7 +296,8 @@ def channel_answer_context(rt, channel, principal=None, automation_id=None):
if not isinstance(grants, list) or str(automation_id) not in {
str(item) for item in grants if isinstance(item, str)}:
return None
- return {"channel": str(agent.get("channel") or ""), "principal": expected,
+ return {"agentId": str(agent.get("id") or ""),
+ "channel": str(agent.get("channel") or ""), "principal": expected,
"writeApproval": agent.get("writeApproval", True) is not False,
"automationId": str(automation_id or "")}
@@ -336,6 +335,53 @@ def _reply_text(text, provenance):
return f"{body[:max(0, 4000 - len(footer) - 2)]}\n\n{footer}"
+def permission_safe_automation_digest(rt, context, *, module="", fields=None, rows=None,
+ summary="", links=None, limit=12):
+ """Compose a bounded digest only after applying the destination Agent's live read wall.
+
+ The caller supplies trusted object links, never cell/field values in a URL. For a database
+ digest both the schema and rows pass through the exact Slack-agent permission projection;
+ for a Sheet digest the body intentionally carries no cell values at all.
+ """
+ ctx = context if isinstance(context, dict) else {}
+ agent = _agents(rt).get(str(ctx.get("agentId") or ""))
+ if not isinstance(agent, dict) or agent.get("active", True) is False:
+ return None
+ try:
+ cap = max(1, min(int(limit), APPROVAL_MAX_ROWS))
+ except (TypeError, ValueError):
+ cap = APPROVAL_MAX_ROWS
+ visible, safe_rows, readable = [], [], True
+ if module:
+ readable = agent_may_read(agent, module)
+ if readable:
+ visible = agent_visible_fields(agent, fields or [], module, st=rt)
+ safe_rows = agent_rows(agent, rows or [], module, fields or [], st=rt)[:cap]
+ labels = {str(field.get("key") or ""): str(field.get("label") or field.get("key") or "")
+ for field in visible if isinstance(field, dict) and field.get("key")}
+ lines = [" ".join(str(summary or "Customer intelligence found a change.").split())[:500]]
+ if readable:
+ for row in safe_rows:
+ cells = []
+ for key in list(labels)[:8]:
+ value = (row or {}).get(key)
+ if value in (None, ""):
+ continue
+ cells.append(f"{labels[key]}: {' '.join(str(value).split())[:160]}")
+ if cells:
+ lines.append(" • " + "; ".join(cells))
+ trusted_links = [str(value).strip() for value in (links or [])
+ if str(value).strip().startswith("#/")][:cap]
+ base = (os.environ.get("AIOS_PUBLIC_BASE") or os.environ.get("APP_BASE_URL") or "").rstrip("/")
+ if base:
+ trusted_links = [base + "/" + value for value in trusted_links]
+ if trusted_links:
+ lines.append("Open in Loopable: " + " ".join(trusted_links))
+ else:
+ return None
+ return "\n".join(lines)[:3200]
+
+
def deliver_channel_reply(rt, channel, text, *, principal=None, automation_id=None,
provenance=None, post=None):
"""Send only after one channel/principal binding and an optional automation grant validate."""
diff --git a/api/routes_templates.py b/api/routes_templates.py
index 4ed2048f15fa809750663c9e4a0f0d9aa7a1ed0c..278d43922770fa5c6186a809c4c3a9a5ac93d090 100644
--- a/api/routes_templates.py
+++ b/api/routes_templates.py
@@ -239,13 +239,14 @@ def _requirement(database, definitions, have, database_states, connector=""):
else "ready" if switched_on else "off"}
-def _catalog_row(key, name, description, requires):
+def _catalog_row(key, name, description, requires, *, creation_mode="wizard"):
# Prefer the missing connection even when an earlier database is merely off.
blocked = next((row for row in requires if row["state"] == "disconnected"), None)
blocked = blocked or next((row for row in requires if row["state"] != "ready"), None)
return {"key": key, "name": str(name).replace(chr(0x2014), "-").replace(chr(0x2013), "-"),
"description": str(description).replace(chr(0x2014), "-").replace(chr(0x2013), "-"),
"requires": requires, "creatable": all(row["state"] == "ready" for row in requires),
+ "creationMode": creation_mode,
"action": None if blocked is None else {
"kind": "connect" if blocked["state"] == "disconnected" else "switch_on",
"connector": blocked["connector"], "database": blocked["database"]}}
@@ -269,6 +270,8 @@ def _module_templates(session):
from routes_connectors import _entry_types
from routes_keychain import env_odoo_available
from routes_nav import application_templates
+ from routes_applications import _may_open
+ from modules.finance_apps import finance_templates
definitions = user_tables.all_defs(session.runtime)
have, _scopes, _failure = _entry_types(session.runtime, session.uname)
@@ -299,6 +302,9 @@ def _module_templates(session):
# These App contracts map fields in the existing wizard and require no fixed connector.
app = [_catalog_row(row["kind"], row["label"], "Choose a database and map its fields.", [])
for row in application_templates(session=session)["templates"]]
+ app.extend(_catalog_row("finance:" + row["key"], row["label"], row["description"], [],
+ creation_mode="post")
+ for row in finance_templates(may_open=lambda permission: _may_open(session, permission)))
return {"sections": [{"module": "sheet", "templates": sheet},
{"module": "data", "templates": data},
{"module": "app", "templates": app}]}
@@ -406,6 +412,35 @@ def create_module_template(module: str, key: str, body: dict = Body(default=None
from urllib.parse import quote
body = body if isinstance(body, dict) else {}
+ if module == "app" and key.startswith("finance:"):
+ import application_registry
+ from modules.finance_apps import finance_application
+ from routes_applications import _may_open
+
+ finance_key = key.removeprefix("finance:")
+ try:
+ descriptor = finance_application(
+ finance_key, may_open=lambda permission: _may_open(session, permission))
+ except KeyError:
+ raise err(404, "unknown_template", "That template does not exist.")
+ except PermissionError:
+ raise err(403, "finance_forbidden", f"Access to Finance template {finance_key!r} is not granted.")
+ if not session.runtime.available():
+ raise err(503, "store_unavailable", "The tenant store is unavailable. Nothing was saved.")
+ try:
+ application = application_registry.create(
+ session.runtime, session.tenant, session.uname,
+ {"templateKind": application_registry.FINANCE_TEMPLATE_KIND,
+ "financeKey": finance_key, "name": body.get("name") or descriptor["label"],
+ "navPlacement": {"section": "app", "order": 0}},
+ allowed_databases=set())
+ except application_registry.ApplicationError as exc:
+ raise err(400, "bad_application", str(exc))
+ except PermissionError as exc:
+ raise err(403, "forbidden", str(exc))
+ except Exception:
+ raise err(503, "store_unavailable", "The Finance application was not saved.")
+ return {"address": "#" + application["deepLink"], "application": application}
section = next((row for row in _module_templates(session)["sections"] if row["module"] == module), None)
template = next((row for row in section["templates"] if row["key"] == key), None) if section else None
if template is None:
diff --git a/api/test_s63_integration.py b/api/test_s63_integration.py
new file mode 100644
index 0000000000000000000000000000000000000000..5cada201d969dce71e11001aa95e87e652b4cdb4
--- /dev/null
+++ b/api/test_s63_integration.py
@@ -0,0 +1,240 @@
+"""Hermetic regression checks for the SDLC 63 cross-lane repairs."""
+import copy
+import json
+from pathlib import Path
+import subprocess
+import sys
+import unittest
+from unittest.mock import patch
+
+ROOT = Path(__file__).resolve().parents[2]
+sys.path[:0] = [str(ROOT / "platform"), str(ROOT / "aios-web/api")]
+
+import aios_grid
+import odoo_relational as rel
+from core import user_tables
+from harness import filter_eval
+from verify_odoo_relational import FixtureCursor, MemStore
+
+
+class SourceCounts:
+ """Data-free Odoo double whose answers differ by domain, not merely by model."""
+ def __init__(self):
+ self.calls = []
+
+ def search_count(self, model, domain):
+ self.calls.append(("count", model, domain))
+ if model == "account.move":
+ move_types = next(value for field, op, value in domain if field == "move_type")
+ return 2 if "out_invoice" in move_types else 3
+ return {"sale.order": 4, "account.account": 5,
+ "sale.order.line": 6, "account.move.line": 7}[model]
+
+ def search_read(self, model, domain, fields, limit=None):
+ self.calls.append(("read", model, domain))
+ return [{"id": 1}, {"id": 2}]
+
+ def read_group(self, model, domain, fields, groupby, lazy=False):
+ self.calls.append(("group", model, domain))
+ if model == "account.invoice.line.agent":
+ return [{"agent_id": [2, "Two"]}, {"agent_id": [3, "Three"]}]
+ return [{"partner_id": [8, "Eight"]}, {"partner_id": [9, "Nine"]}]
+
+
+class IntegrationTests(unittest.TestCase):
+ def test_archived_defaults_and_removable_filter(self):
+ fields = json.loads((ROOT / "platform/aios_grid_fields.json").read_text())["fields"]
+ config = aios_grid._default_view_config(fields)
+ self.assertEqual(config["filters"], [{"colId": "archived", "op": "neq", "value": "true"}])
+ tree = {"conj": "and", "nodes": config["filters"]}
+ self.assertTrue(filter_eval.evaluate(tree, {"archived": False}, fields))
+ self.assertFalse(filter_eval.evaluate(tree, {"archived": True}, fields))
+ self.assertTrue(filter_eval.matches({"conj": "and", "nodes": []}, {"archived": True}, fields))
+ self.assertEqual(aios_grid._default_view_config([
+ {"key": "archived", "type": "checkbox", "source": "overlay"}])["filters"], [])
+ script = r'''
+const fs=require('fs'),ts=require('typescript'),vm=require('vm'),assert=require('assert/strict');
+const text=fs.readFileSync('src/customer-grid/useGridColumns.ts','utf8');
+const file=ts.createSourceFile('views.ts',text,ts.ScriptTarget.Latest,true,ts.ScriptKind.TS);
+const wanted=new Set(['isDefaultVisible','defaultViewConfig']);
+const declarations=file.statements.filter(node=>ts.isFunctionDeclaration(node)&&node.name&&wanted.has(node.name.text));
+assert.equal(declarations.length,2);
+const js=ts.transpileModule(declarations.map(node=>node.getText(file)).join('\n'),
+ {compilerOptions:{module:ts.ModuleKind.CommonJS,target:ts.ScriptTarget.ES2020}}).outputText;
+const context={exports:{}};vm.runInNewContext(js,context);
+const archived={key:'archived',type:'checkbox',source:'odoo'};
+const ordinary={key:'name',type:'text',source:'odoo'};
+const overlay={key:'archived',type:'checkbox',source:'overlay'};
+console.log(JSON.stringify({odoo:context.exports.defaultViewConfig([ordinary,archived]).filters,
+ overlay:context.exports.defaultViewConfig([ordinary,overlay]).filters}));
+'''
+ result = subprocess.run(["node", "-e", script], cwd=ROOT / "aios-web/web",
+ capture_output=True, text=True, encoding="utf-8", timeout=30)
+ self.assertEqual(result.returncode, 0, result.stdout + result.stderr)
+ client = json.loads(result.stdout)
+ self.assertEqual(client["odoo"], config["filters"])
+ self.assertEqual(client["overlay"], [])
+
+ def test_database_read_failure_preserves_rows_and_successful_sibling(self):
+ rt = MemStore()
+ with patch.object(user_tables, "row_limit", return_value=None), \
+ patch.object(rel, "is_read_through_key", return_value=False):
+ first = rel.refresh_curated(rt, FixtureCursor(), {"invoices", "bills"}, retire=False)
+ self.assertEqual(first[2], {})
+ before = copy.deepcopy(rt.data[user_tables.STORE_KEY][rel.INVOICES_KEY])
+ with patch.dict(rel._READERS, {"invoices": lambda *_: (_ for _ in ()).throw(
+ RuntimeError("private row must never enter receipt"))}):
+ built, written, errors, receipts, _ = rel.refresh_curated(
+ rt, FixtureCursor(), {"invoices", "bills"}, retire=False)
+ self.assertNotIn("invoices", built)
+ self.assertNotIn(rel.INVOICES_KEY, written)
+ self.assertIn(rel.BILLS_KEY, written)
+ self.assertEqual(set(errors), {rel.INVOICES_KEY})
+ self.assertEqual(rt.data[user_tables.STORE_KEY][rel.INVOICES_KEY], before)
+ self.assertNotIn("private row", json.dumps(receipts))
+ self.assertEqual(receipts[rel.BILLS_KEY]["state"], "succeeded")
+ self.assertIsNone(receipts[rel.BILLS_KEY]["sourceCount"])
+ self.assertEqual(receipts[rel.BILLS_KEY]["mirrorCount"], written[rel.BILLS_KEY]["rows"])
+
+ def test_duplicate_reader_ids_refuse_before_mutation(self):
+ rt = MemStore()
+ with patch.object(user_tables, "row_limit", return_value=None), \
+ patch.dict(rel._READERS, {"invoices": lambda *_: [{"_id": "1"}, {"_id": "1"}]}):
+ _, written, errors, _, _ = rel.refresh_curated(rt, FixtureCursor(), {"invoices"}, retire=False)
+ self.assertEqual(written, {})
+ self.assertEqual(set(errors), {rel.INVOICES_KEY})
+ self.assertEqual(rt.data, {})
+
+ def test_all_curated_receipts_are_database_and_domain_specific(self):
+ rt = MemStore()
+ source = SourceCounts()
+ expected = {"invoices": 2, "orders": 4, "agents": 3, "accounts": 5,
+ "vendors": 2, "bills": 3, "order_lines": 6, "gl_lines": 7}
+ by_bucket = {bucket: key for bucket, key, _label, _fields in rel.TABLES}
+
+ def fake_plan(_cur, rt=None, connection_id=None, buckets=None):
+ bucket = next(iter(buckets))
+ rows = ([] if bucket in {"order_lines", "gl_lines"} else
+ [{"_id": f"{bucket}-{index}"} for index in range(expected[bucket])])
+ return {bucket: rows, "problems": [], "limits": {}}
+
+ def fake_apply(_rt, planned, **_kwargs):
+ bucket = next(key for key in expected if key in planned)
+ key = by_bucket[bucket]
+ return {key: {"rows": len(planned[bucket])}}
+
+ def fake_population(key, **_kwargs):
+ bucket = next(name for name, table_key in by_bucket.items() if table_key == key)
+ return expected[bucket]
+
+ with patch.object(rel, "plan", side_effect=fake_plan), \
+ patch.object(rel, "apply_plan", side_effect=fake_apply), \
+ patch.object(user_tables, "row_limit", return_value=None), \
+ patch.object(rel, "is_read_through_key",
+ side_effect=lambda key: key in {rel.ORDER_LINES_KEY, rel.GL_LINES_KEY}), \
+ patch("routes_odoo_tables.population", side_effect=fake_population):
+ built, written, errors, receipts, _ = rel.refresh_curated(
+ rt, FixtureCursor(), set(expected), retire=False, source_client=source)
+
+ self.assertEqual(errors, {})
+ self.assertEqual(set(receipts), set(by_bucket.values()))
+ self.assertEqual(set(written), set(by_bucket.values()))
+ for bucket, key in by_bucket.items():
+ receipt = receipts[key]
+ self.assertEqual(receipt["schemaKey"], key)
+ self.assertEqual(receipt["domainKey"], rel.TABLE_SUBJECTS[key])
+ self.assertEqual(receipt["topicKey"], rel.TABLE_TOPICS[key])
+ self.assertEqual(receipt["selection"], {"bucket": bucket, "enabled": True})
+ self.assertEqual(receipt["sourceCount"], expected[bucket])
+ self.assertEqual(receipt["mirrorCount"], expected[bucket])
+ self.assertEqual(receipt["sourceReconciliation"], "matched")
+ self.assertEqual(receipt["freshness"]["state"], "current")
+ self.assertEqual(receipts[rel.INVOICES_KEY]["model"], "account.move")
+ self.assertEqual(receipts[rel.BILLS_KEY]["model"], "account.move")
+ self.assertNotEqual(receipts[rel.INVOICES_KEY]["domain"],
+ receipts[rel.BILLS_KEY]["domain"])
+ self.assertEqual(receipts[rel.VENDORS_KEY]["model"], "res.partner")
+ self.assertEqual(receipts[rel.VENDORS_KEY]["countModel"], "account.move")
+ self.assertEqual(receipts[rel.ORDER_LINES_KEY]["materializedCount"], 0)
+ self.assertEqual(receipts[rel.ORDER_LINES_KEY]["servingCount"], 6)
+ route = (ROOT / "aios-web/api/routes_odoo_tables.py").read_text(encoding="utf-8")
+ refresh_endpoint = route.split("def refresh_odoo_tables", 1)[1].split(
+ '@router.get("/odoo-tables/status")', 1)[0]
+ self.assertIn("all_connections=True", refresh_endpoint)
+
+ def test_browser_grid_header_and_cell_events_with_both_negative_controls(self):
+ script = r'''
+const fs=require('fs'),ts=require('typescript'),assert=require('assert/strict');
+const text=fs.readFileSync('src/sheet/SheetGrid.tsx','utf8');
+const file=ts.createSourceFile('grid.tsx',text,ts.ScriptTarget.Latest,true,ts.ScriptKind.TSX);
+const handlers=[];
+function visit(node){
+ if(ts.isJsxAttribute(node)&&node.name.getText(file)==='onCellClicked') handlers.push(node.initializer.expression.getText(file));
+ ts.forEachChild(node,visit);
+}
+visit(file); assert.equal(handlers.length,2);
+function compiled(handler){
+ return ts.transpileModule('globalThis.__sheetClick = '+handler,{compilerOptions:{target:ts.ScriptTarget.ES2020}}).outputText;
+}
+const controls=handlers.map(handler=>{
+ const sentinel=handler.indexOf('if (item[0] === -1)');
+ const signalAt=handler.indexOf('signal(NAV_MINIMIZE_EVENT);',sentinel);
+ assert.ok(sentinel>=0&&signalAt>sentinel,'production signal must follow the sentinel');
+ const signal='signal(NAV_MINIMIZE_EVENT);';
+ const moved=handler.slice(0,sentinel)+signal+'\n'+handler.slice(sentinel,signalAt)+
+ handler.slice(signalAt+signal.length);
+ return {positive:compiled(handler),negative:compiled(moved)};
+});
+console.log(JSON.stringify(controls));
+'''
+ result = subprocess.run(["node", "-e", script], cwd=ROOT / "aios-web/web",
+ capture_output=True, text=True, encoding="utf-8", timeout=30)
+ self.assertEqual(result.returncode, 0, result.stdout + result.stderr)
+ controls = json.loads(result.stdout)
+
+ # This leg clicks real browser DOM targets wired to the production TypeScript handlers.
+ # Each pane also gets the inverse build (signal moved before the row-header sentinel), so
+ # the browser proof can demonstrate that it detects the exact regression it guards.
+ from playwright.sync_api import sync_playwright
+ with sync_playwright() as playwright:
+ browser = playwright.chromium.launch(headless=True)
+ page = browser.new_page()
+ page.set_content('')
+ for pane, pair in zip(("frozen", "unfrozen"), controls, strict=True):
+ def run(compiled):
+ page.evaluate("""() => {
+ globalThis.__signals = 0; globalThis.__rows = 0;
+ globalThis.signal = () => globalThis.__signals++;
+ globalThis.NAV_MINIMIZE_EVENT = 'aios:nav-minimize';
+ globalThis.fromBand = value => value;
+ globalThis.selectDocumentAxis = () => globalThis.__rows++;
+ globalThis.painter = false; globalThis.addrAt = () => 'A1';
+ globalThis.openViewHeader = () => false;
+ globalThis.validationOf = () => false;
+ globalThis.cellAt = () => null; globalThis.tab = {};
+ globalThis.followCellLink = () => false; globalThis.setAnchor = () => {};
+ }""")
+ page.evaluate("code => { (0, eval)(code); }", compiled)
+ page.evaluate("""() => {
+ document.querySelector('#header').onclick = () =>
+ globalThis.__sheetClick([-1, 0], {shiftKey: false, bounds: {}});
+ document.querySelector('#cell').onclick = () =>
+ globalThis.__sheetClick([0, 0], {shiftKey: false, bounds: {}});
+ }""")
+ page.click("#header")
+ after_header = page.evaluate("() => ({signals: __signals, rows: __rows})")
+ page.click("#cell")
+ after_cell = page.evaluate("() => ({signals: __signals, rows: __rows})")
+ return after_header, after_cell
+
+ header, cell = run(pair["positive"])
+ self.assertEqual(header, {"signals": 0, "rows": 1}, pane)
+ self.assertEqual(cell, {"signals": 1, "rows": 1}, pane)
+ broken_header, _ = run(pair["negative"])
+ self.assertEqual(broken_header, {"signals": 1, "rows": 1},
+ f"{pane} negative control did not expose the regression")
+ browser.close()
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/api/test_sheet_commit_events.py b/api/test_sheet_commit_events.py
new file mode 100644
index 0000000000000000000000000000000000000000..cfad06be14f7b8d5631f7ce0d02ed019997b0275
--- /dev/null
+++ b/api/test_sheet_commit_events.py
@@ -0,0 +1,162 @@
+"""Exercise value-only workbook commit notifications without crossing cell contents."""
+import copy
+from pathlib import Path
+import sys
+import unittest
+from unittest.mock import patch
+
+ROOT = Path(__file__).resolve().parents[2]
+sys.path[:0] = [str(ROOT / "platform"), str(ROOT / "aios-web/api")]
+from core import sheets, sheet_compat, sheet_events
+
+
+class MemoryStore:
+ key = "sheet-event-fixture"
+
+ def __init__(self):
+ self.data = {}
+ self.locked = False
+ self.fail_commit = False
+
+ def get(self, key, default=None):
+ return copy.deepcopy(self.data.get(key, default))
+
+ def update(self, key, fn, **kwargs):
+ self.locked = True
+ try:
+ value = fn(copy.deepcopy(self.data.get(key, {})))
+ if self.fail_commit:
+ raise RuntimeError("fixture commit failure")
+ self.data[key] = copy.deepcopy(value)
+ finally:
+ self.locked = False
+
+
+class CommitEvents(unittest.TestCase):
+ def setUp(self):
+ self.st = MemoryStore()
+ self.book = sheets.create_workbook(self.st, name="Fixture", created_by="qa")
+ self.tab = self.book["sheets"][0]["id"]
+ self.events = []
+
+ def observe(st, events):
+ self.assertFalse(st.locked, "no Automation call while the workbook lock is held")
+ for event in events:
+ self.assertEqual(st.get(sheets.workbook_key(self.book["id"]))["rev"], event["revision"])
+ self.events.extend(events)
+
+ self.observer = patch.object(sheet_events, "COMMIT_OBSERVER", observe)
+ self.observer.start()
+ self.addCleanup(self.observer.stop)
+
+ def patch_cells(self, cells):
+ return sheets.patch_cells(self.st, self.book["id"], self.tab, cells, actor="qa")
+
+ def test_bulk_patch_is_one_committed_event_without_cell_values(self):
+ self.patch_cells({f"B{i}": {"kind": "literal", "value": i} for i in range(1, 20001)})
+ self.assertEqual(len(self.events), 1)
+ event = self.events[0]
+ self.assertEqual(event["ranges"], ["B1:B20000"])
+ self.assertEqual(event["recordCount"], 20000)
+ self.assertEqual(event["changeKinds"], ["value"])
+ self.assertTrue(event["committed"])
+ self.assertNotIn("cells", event)
+ self.assertNotIn("values", event)
+
+ def test_formatting_identical_values_and_refused_cells_do_not_emit(self):
+ self.patch_cells({"A1": {"kind": "literal", "value": 7}})
+ self.events.clear()
+ self.patch_cells({"A1": {"kind": "literal", "value": 7, "style": {"bold": True}}})
+ self.patch_cells({"A1": {"kind": "literal", "value": 7}, "B2": {"kind": "unknown"}})
+ self.assertEqual(self.events, [])
+
+ def test_failed_commit_and_stale_put_do_not_emit(self):
+ self.st.fail_commit = True
+ with self.assertRaises(RuntimeError):
+ self.patch_cells({"A1": {"kind": "literal", "value": 3}})
+ self.st.fail_commit = False
+ self.assertEqual(self.events, [])
+ self.patch_cells({"A1": {"kind": "literal", "value": 3}})
+ self.events.clear()
+ with self.assertRaises(sheets.SheetRefusal):
+ sheets.put_workbook(self.st, self.book["id"], self.book)
+ self.assertEqual(self.events, [])
+
+ def test_removal_and_sparse_changes_keep_exact_watched_ranges(self):
+ self.patch_cells({"A1": {"kind": "literal", "value": 1},
+ "A3": {"kind": "literal", "value": 3}})
+ self.assertEqual(self.events[-1]["ranges"], ["A1", "A3"])
+ self.patch_cells({"A1": None})
+ self.assertEqual(self.events[-1]["ranges"], ["A1"])
+
+ def test_refresh_and_compatibility_share_the_same_commit_boundary(self):
+ book = sheets.get_workbook(self.st, self.book["id"])
+ book["sheets"][0]["cells"] = {"A1": {"kind": "literal", "value": 4}}
+ saved = sheets.put_workbook(self.st, self.book["id"], book, lossless=True)
+ self.assertEqual(len(self.events), 1)
+ sheet_compat.execute(self.st, self.book["id"], "range.values.write",
+ {"sheet": self.tab, "range": "A1", "values": [[5]],
+ "expected_revision": saved["rev"]}, actor="qa")
+ self.assertEqual(len(self.events), 2)
+ self.assertNotEqual(self.events[0]["batchId"], self.events[1]["batchId"])
+
+ def test_store_retry_publishes_only_the_successful_mutation(self):
+ original = self.st.update
+
+ def retry(key, fn, **kwargs):
+ if key == sheets.workbook_key(self.book["id"]):
+ discarded = self.st.get(key)
+ discarded["rev"] = 900
+ fn(discarded)
+ return original(key, fn, **kwargs)
+
+ with patch.object(self.st, "update", retry):
+ self.patch_cells({"A1": {"kind": "literal", "value": 2}})
+ self.assertEqual(len(self.events), 1)
+ self.assertLess(self.events[0]["revision"], 900)
+
+ def test_new_tab_backfill_is_not_an_existing_watch_change(self):
+ book = sheets.get_workbook(self.st, self.book["id"])
+ book["sheets"].append({"id": "imported", "name": "Imported", "cells": {
+ "A1": {"kind": "literal", "value": 17}}})
+ sheets.put_workbook(self.st, self.book["id"], book)
+ self.assertEqual(self.events, [])
+
+ def test_binding_refresh_publishes_output_change_but_not_refresh_stamp(self):
+ cell = {"kind": "bound", "value": 10, "binding": {
+ "database": "customers", "aggregate": "balance", "id": "current_ytd"}}
+ # Drive the persistence seam with already-canonical stored cells to isolate refresh
+ # notification from the independently tested binding-schema cleaner.
+ key = sheets.workbook_key(self.book["id"])
+ self.st.data[key]["sheets"][0]["cells"]["A1"] = copy.deepcopy(cell)
+
+ def refresh(state):
+ state["rev"] += 1
+ state["sheets"][0]["cells"]["A1"].update(value=11, refreshed_at="fixture-1")
+ return state, state
+
+ sheets._persist(self.st, key, refresh)
+ self.assertEqual(self.events[-1]["changeKinds"], ["value", "binding"])
+ self.assertIn("current_ytd", self.events[-1]["bindings"])
+ self.assertIn("customers.balance", self.events[-1]["bindings"])
+ self.events.clear()
+ sheets._persist(self.st, key, refresh)
+ self.assertEqual(self.events, [])
+
+ def test_notification_failure_does_not_turn_committed_edit_into_a_refusal(self):
+ with patch.object(sheet_events, "COMMIT_OBSERVER", side_effect=RuntimeError("private detail")), \
+ self.assertLogs("core.sheet_events", level="ERROR") as logs:
+ result = self.patch_cells({"A1": {"kind": "literal", "value": 42}})
+ self.assertEqual(sheets.get_workbook(self.st, self.book["id"])["rev"], result["rev"])
+ self.assertNotIn("private detail", str(logs.output))
+
+ def test_negative_control_detects_disconnected_commit_publisher(self):
+ result = unittest.TestResult()
+ with patch.object(sheet_events, "publish", return_value=None):
+ CommitEvents("test_bulk_patch_is_one_committed_event_without_cell_values").run(result)
+ self.assertEqual(len(result.failures), 1)
+ self.assertEqual(result.errors, [])
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/platform/aios_grid.py b/platform/aios_grid.py
index 1b53f612e77d281eb1ebc2129ba19db907934b33..a9cd1528d8cad64648b9a90ba136f52ebfd84247 100644
--- a/platform/aios_grid.py
+++ b/platform/aios_grid.py
@@ -3577,10 +3577,13 @@ def _default_view_config(fields):
# MUST move together, which is the whole subject of the note above.
shown = [field["key"] for field in fields if _is_default_visible(field)]
hidden = [field["key"] for field in fields if field["key"] not in shown]
+ archive_filter = [{"colId": "archived", "op": "neq", "value": "true"}] if any(
+ field.get("key") == "archived" and field.get("type") == "checkbox"
+ and field.get("source") == "odoo" for field in fields) else []
return {
# `filters` is the ROOT of the filter tree: leaf conditions and/or nested
# condition groups ({conj, children}); `filterConj` joins the root level.
- "filters": [], "filterConj": "and",
+ "filters": archive_filter, "filterConj": "and",
"sorts": [], "groupBy": None, "colorBy": None,
"rowHeightMode": "short", "order": shown + hidden, "visible": shown,
"widths": {}, "memberPids": [],
@@ -3702,7 +3705,7 @@ def views_from_defs(defs, saved_views, fields, system_name="All customers", lock
op_map = {">=": "gte", ">": "gt", "<=": "lte", "<": "lt", "=": "eq",
"contains": "contains"}
for name, definition in (defs or {}).items():
- filters = []
+ filters = [dict(rule) for rule in base["filters"]]
for rule in definition.get("rules") or []:
if rule.get("field") not in {field["key"] for field in fields}:
continue
diff --git a/platform/aios_grid_fields.json b/platform/aios_grid_fields.json
index 9073470106cf5678964c36ad62db315246b1f678..68554338bb64e5aa68e3dea8b05d845f959453e0 100644
--- a/platform/aios_grid_fields.json
+++ b/platform/aios_grid_fields.json
@@ -32,6 +32,14 @@
],
"description": "Whether this customer still exists in Odoo. Archived means deleted there."
},
+ {
+ "key": "archived",
+ "label": "Archived",
+ "type": "checkbox",
+ "source": "odoo",
+ "default": false,
+ "description": "Whether this customer is archived in Odoo."
+ },
{
"key": "agent",
"label": "Agent",
diff --git a/platform/core/audit.py b/platform/core/audit.py
index 989948b65624e6ae60020819462a1675701544e1..f156863ddab4447a877e5706f1f9cef326c87c2b 100644
--- a/platform/core/audit.py
+++ b/platform/core/audit.py
@@ -12,6 +12,8 @@ AUDIT_KINDS = frozenset({
"auth.mfa_failed", "auth.logout", "admin.grant", "credential.added",
"credential.rotated", "credential.revoked", "amazon.sync", "amazon.throttled",
"amazon.authorization_lost", "data.purge", "data.restore", "config.changed",
+ "external.query",
+ "external.write",
})
BUCKET = "security_events"
MAX_EVENTS = 2000
@@ -33,6 +35,7 @@ SECRET_KEYS = frozenset(_key_name(key) for key in (
))
_BEARER = re.compile(r"\bbearer\s+\S+", re.IGNORECASE)
_JWT = re.compile(r"\beyJ[A-Za-z0-9_-]*\.")
+_LOOPABLE_CREDENTIAL = re.compile(r"\blp(?:b|o|r|c)1\.[A-Za-z0-9._~-]+")
class AuditRefused(ValueError):
@@ -54,6 +57,8 @@ def _string(key, value):
raise AuditRefused(key, "bearer token shape")
if _JWT.search(value):
raise AuditRefused(key, "JWT token shape")
+ if _LOOPABLE_CREDENTIAL.search(value):
+ raise AuditRefused(key, "Loopable credential shape")
def _attribution(key, value):
@@ -174,3 +179,27 @@ def read(tenant, limit=200, now=None) -> list[dict]:
doc = runtime.get_runtime(str(tenant)).get(BUCKET)
rows = ordered(doc, tenant, limit)
return [row for row in rows if row["ts"] >= now - RETENTION_SECONDS]
+
+
+def external_write_receipt(tenant, principal, *, tool, auth_type, affected_count,
+ diff_digest, revision_before, revision_after,
+ confirmation_id) -> dict:
+ """Emit and return a value-free C7 write receipt.
+
+ Target ids, field names, record values and arguments deliberately do not belong here. The
+ receipt proves who confirmed which normalized diff and how many objects moved; the concrete
+ before/after stays in the short-lived preview response instead of the retained security log.
+ """
+ detail = {
+ "surface": "ai_api",
+ "tool": str(tool or "")[:128],
+ "authentication": str(auth_type or "")[:32],
+ "affected_count": max(0, int(affected_count or 0)),
+ "diff_digest": str(diff_digest or "")[:128],
+ "revision_before": str(revision_before or "")[:128],
+ "revision_after": str(revision_after or "")[:128],
+ "confirmation_id": str(confirmation_id or "")[:128],
+ }
+ item = event("external.write", tenant, principal, detail)
+ stored = record("external.write", tenant, principal, detail)
+ return {"event": item, "persisted": stored is not None}
diff --git a/platform/core/daily_sales_repair.py b/platform/core/daily_sales_repair.py
new file mode 100644
index 0000000000000000000000000000000000000000..7ffeb4a633265c46f4dc0faa45f93e04deb3ce0b
--- /dev/null
+++ b/platform/core/daily_sales_repair.py
@@ -0,0 +1,977 @@
+"""A refusal-first, exact-snapshot repair for a Daily Sales workbook the caller names.
+
+The legacy source-oracle plan and the workbook-derived invoice summary plan share the same
+backup, exact-write and readback guards. Canonical periods use core.windows.normalize; the
+summary uses the workbook's own filter valueRefs instead of freezing its calendar in a period.
+The authorized operator injects the independent invoice validator and the full joined partner set.
+The module validates the complete projected document, writes and verifies a backup, then delegates
+one atomic, preconditioned cell mutation to :mod:`core.sheets`.
+"""
+
+from __future__ import annotations
+
+import copy
+import json
+import math
+import re
+from datetime import date
+
+from core import sheets
+from core import windows as W
+
+
+TARGET_WORKBOOK_ID = "b8edba3d3f47"
+TARGET_TENANT = "royal-imports"
+SUMMARY_TAB = "Summary (All)"
+SUMMARY_TAB_ID = "c2545d26c42c"
+SUMMARY_COLUMNS = tuple(chr(column) for column in range(ord("F"), ord("R") + 1))
+TICKET_FORMULA_ERROR_CODES = frozenset({"#VALUE!", "#DIV/0!", "#REF!", "#N/A", "#NAME?"})
+_EXCEL_ERROR_ALIASES = {
+ "bad_value": "#VALUE!", "div_zero": "#DIV/0!", "invalid_reference": "#REF!",
+ "external_reference": "#REF!", "no_match": "#N/A", "unknown_name": "#NAME?",
+}
+_SUMMARY_RATIO_ROOTS = {
+ "F69": "=(F75/F64) * (365 / (F$7 - DATE(YEAR(F$7),1,1) + 1))",
+ "F70": "=(F76/F67) * (365 / (F$7 - DATE(YEAR(F$7),1,1) + 1))",
+ "R70": "=(R76/R67) * (365 / (R$7 - DATE(YEAR(R$7),1,1) + 1))",
+}
+_SUMMARY_RATIO_CASCADES = {
+ "F71": "=F69-F70", "F72": "=F71/F70", "F73": "=G72-F72",
+ "R71": "=R69-R70", "R72": "=R71/R70", "R73": "=R72-Q72",
+}
+
+
+def operation_for(workbook_id):
+ """Bind a repair operation to the workbook the caller names."""
+ return "repair-daily-sales-" + workbook_id
+
+
+OPERATION = operation_for(TARGET_WORKBOOK_ID)
+YTD_NAMES = {"Current YTD": "ytd", "Previous YTD": "ytd_last_year"}
+if not all(kind in W.WINDOW_KINDS for kind in YTD_NAMES.values()):
+ raise RuntimeError("C2 requires every YTD window kind to exist in core.windows.WINDOW_KINDS.")
+
+_VISUAL_TAB_KEYS = (
+ "colWidths", "rowHeight", "rowHeights", "freeze", "rules", "hiddenRows", "hiddenCols",
+ "charts", "visibility", "view", "print", "excel",
+)
+_VISUAL_CELL_KEYS = ("style", "validation", "link", "note", "sparkline", "excel")
+_FORBIDDEN_BINDING_KEYS = {"fallback", "static", "staticValue", "defaultValue"}
+
+
+class RepairRefusal(sheets.SheetRefusal):
+ """A named refusal from the exceptional repair workflow."""
+
+
+def _canonical(value):
+ return json.loads(json.dumps(value, sort_keys=True, default=str))
+
+
+def _tab(book, name):
+ try:
+ return sheets.find_tab(book, name)
+ except sheets.SheetRefusal as exc:
+ raise RepairRefusal(exc.code, exc.message) from exc
+
+
+def _addr(raw):
+ value = str(raw or "").strip().upper().replace("$", "")
+ match = re.fullmatch(r"([A-Z]{1,3})([1-9][0-9]{0,6})", value)
+ if not match:
+ raise RepairRefusal("repair_address_invalid", "Every repair source needs one A1 address.")
+ column = 0
+ for char in match.group(1):
+ column = column * 26 + ord(char) - ord("A") + 1
+ if column > sheets.DOCUMENT_MAX_COLS or int(match.group(2)) > sheets.DOCUMENT_MAX_ROWS:
+ raise RepairRefusal("repair_address_invalid",
+ "A repair source address is outside the workbook dimensions.")
+ return value
+
+
+def _validate_marker(marker, expected_revision, workbook_id):
+ if not isinstance(marker, dict) or marker.get("direct_tenant_data_live") is not True:
+ raise RepairRefusal(
+ "repair_marker_required",
+ "This repair requires the direct tenant data live marker from the owner instruction.")
+ expected = {
+ "operation": operation_for(workbook_id),
+ "tenant": TARGET_TENANT,
+ "workbook_id": workbook_id,
+ "expected_revision": expected_revision,
+ }
+ if any(marker.get(key) != value for key, value in expected.items()):
+ raise RepairRefusal(
+ "repair_marker_mismatch",
+ "The direct tenant data marker does not exactly match this workbook and revision.")
+ if not str(marker.get("owner_instruction") or "").strip():
+ raise RepairRefusal(
+ "repair_owner_instruction_required",
+ "The direct tenant data marker must carry the owner's exact repair instruction.")
+
+
+def _contains_forbidden_key(value):
+ if isinstance(value, dict):
+ return bool(_FORBIDDEN_BINDING_KEYS.intersection(value)) or any(
+ _contains_forbidden_key(child) for child in value.values())
+ if isinstance(value, list):
+ return any(_contains_forbidden_key(child) for child in value)
+ return False
+
+
+def _validate_binding(name, binding, *, window_kind=None):
+ if not isinstance(binding, dict) or _contains_forbidden_key(binding):
+ raise RepairRefusal(
+ "repair_binding_static",
+ name + " must be a canonical database binding without a static fallback.")
+ if not str(binding.get("database") or "").strip():
+ raise RepairRefusal("repair_binding_invalid", name + " must name its canonical database.")
+ aggregate = binding.get("aggregate")
+ if not isinstance(aggregate, dict) or not (
+ aggregate.get("metric_key") or aggregate.get("fn")):
+ raise RepairRefusal("repair_binding_invalid", name + " must name a real aggregate.")
+ if not aggregate.get("metric_key") and (
+ aggregate.get("fn") not in {"sum", "count", "avg", "min", "max"}
+ or (aggregate["fn"] != "count" and not aggregate.get("field"))):
+ raise RepairRefusal("repair_binding_invalid", name + " must name a real column aggregate.")
+ scope = binding.get("scope")
+ if not isinstance(scope, dict):
+ scope = {}
+ has_view = bool(str(scope.get("view_id") or "").strip())
+ tree = scope.get("filter")
+ # Accept the same explicit, nonempty conjunction used by rows 108-113. Requiring
+ # both referenced date bounds keeps a missing period from becoming an unbounded sum.
+ nodes = tree.get("nodes") if isinstance(tree, dict) else None
+ has_filter = (isinstance(nodes, list) and bool(nodes) and tree.get("conj") == "and"
+ and all(isinstance(n, dict) and n.get("colId") and n.get("op")
+ and ("value" in n or isinstance(n.get("valueRef"), dict))
+ for n in nodes))
+ if not has_view and not has_filter:
+ raise RepairRefusal(
+ "repair_binding_scope_required",
+ name + " must use a saved View or an explicit nonempty filter scope.")
+ if "related" in binding:
+ from core.sheet_binding import BindingMetadataRefusal, clean_related
+ try:
+ clean_related(binding["related"])
+ except BindingMetadataRefusal as exc:
+ raise RepairRefusal(exc.code, exc.message) from exc
+ period = binding.get("period")
+ if period is None and has_filter and not has_view:
+ bounds = {}
+ for node in nodes:
+ slot = node.get("valueRef")
+ if slot is not None:
+ if not isinstance(slot, dict) or set(slot) != {"ref"}:
+ raise RepairRefusal("repair_binding_invalid", name + " has an invalid cell reference.")
+ _addr(slot["ref"])
+ if node["op"] in {"gte", "lte"}:
+ bounds.setdefault(node["colId"], set()).add(node["op"])
+ if any(ops == {"gte", "lte"} for ops in bounds.values()):
+ return
+ window = W.normalize(copy.deepcopy(period.get("window") if isinstance(period, dict) else None))
+ if window is None or (window_kind is not None and window["kind"] != window_kind):
+ expected_window = "the " + window_kind if window_kind is not None else "a valid"
+ raise RepairRefusal(
+ "repair_binding_period_invalid",
+ name + " must use " + expected_window + " canonical date window.")
+
+
+def _preserve_visual(before, desired):
+ """Carry every T20 cell display property across a content-only repair."""
+ held = {key: copy.deepcopy((before or {}).get(key)) for key in _VISUAL_CELL_KEYS
+ if isinstance(before, dict) and key in before}
+ if desired is None:
+ return {"kind": "empty", "state": "typed", **held} if held else None
+ out = copy.deepcopy(desired)
+ for key, value in held.items():
+ out[key] = value
+ return out
+
+
+def visual_metadata(book):
+ """The T20 visual strata, including per-cell display properties."""
+ tabs = []
+ for tab in book.get("sheets") or ():
+ if not isinstance(tab, dict):
+ continue
+ item = {"id": tab.get("id"), "name": tab.get("name")}
+ for key in _VISUAL_TAB_KEYS:
+ if key in tab:
+ item[key] = copy.deepcopy(tab[key])
+ cell_visuals = {}
+ for address, cell in (tab.get("cells") or {}).items():
+ kept = {key: copy.deepcopy(cell[key]) for key in _VISUAL_CELL_KEYS
+ if isinstance(cell, dict) and key in cell}
+ if kept:
+ cell_visuals[address] = kept
+ item["cells"] = cell_visuals
+ tabs.append(item)
+ return _canonical(tabs)
+
+
+def formula_error_census(book):
+ """Return every stored formula that still carries a calculation refusal/error."""
+ errors = []
+ for tab in book.get("sheets") or ():
+ for address, cell in (tab.get("cells") or {}).items():
+ if not isinstance(cell, dict) or cell.get("kind") != "formula":
+ continue
+ error = cell.get("error") or (cell.get("refusal") if cell.get("state") == "refused" else None)
+ if error:
+ errors.append({"sheet": tab.get("name"), "address": address,
+ "code": error.get("code") if isinstance(error, dict) else str(error)})
+ return sorted(errors, key=lambda row: (str(row["sheet"]), row["address"]))
+
+
+def excel_formula_error_census(book, *, sheet=SUMMARY_TAB):
+ """The ticket's five Excel codes, separate from the unchanged full error census."""
+ errors = []
+ for tab in book.get("sheets") or ():
+ if sheet is not None and tab.get("name") != sheet:
+ continue
+ for address, cell in (tab.get("cells") or {}).items():
+ if not isinstance(cell, dict) or cell.get("kind") != "formula":
+ continue
+ error = cell.get("error") or (cell.get("refusal") if cell.get("state") == "refused" else None)
+ if not error:
+ continue
+ code = error.get("code") if isinstance(error, dict) else str(error)
+ excel_code = error.get("excelCode") if isinstance(error, dict) else None
+ excel_code = excel_code or _EXCEL_ERROR_ALIASES.get(code, code)
+ if excel_code in TICKET_FORMULA_ERROR_CODES:
+ errors.append({"sheet": tab.get("name"), "address": address,
+ "code": code, "excelCode": excel_code})
+ return sorted(errors, key=lambda row: (str(row["sheet"]), row["address"]))
+
+
+def check_summary_formula_errors(book):
+ """Count the real tab only after proving this same census finds planted errors.
+
+ The negative control is an isolated in-memory document, never a tenant edit. Exercise
+ both code spellings, excelCode precedence, and the refused-cell fallback on every call.
+ """
+ _tab(book, SUMMARY_TAB)
+ planted, expected = {}, {}
+ for internal, excel in (("bad_value", "#VALUE!"), ("div_zero", "#DIV/0!"),
+ ("invalid_reference", "#REF!"), ("no_match", "#N/A"),
+ ("unknown_name", "#NAME?")):
+ for field in ("error", "refusal"):
+ for payload in ({"code": internal, "excelCode": excel}, {"code": excel},
+ {"code": internal}, {"code": "probe", "excelCode": excel}):
+ address = "A" + str(len(planted) + 1)
+ planted[address] = {"kind": "formula", "state": "refused",
+ "formula": "=1/0", field: payload}
+ expected[address] = excel
+ for index, code in enumerate(("circular_reference", "blank", "#NUM!"), 1):
+ planted["B" + str(index)] = {"kind": "formula", "state": "refused",
+ "formula": "=1/0", "error": {"code": code}}
+ probe = {"sheets": [{"name": SUMMARY_TAB, "cells": planted},
+ {"name": "Summary (SR)", "cells": {"A1": planted["A1"]}}]}
+ found = excel_formula_error_census(probe)
+ if len(found) != len(expected) or {row["address"]: row["excelCode"] for row in found} != expected:
+ raise RepairRefusal("repair_formula_census_blind",
+ "The formula error census failed to find its planted errors.")
+ errors = excel_formula_error_census(book)
+ return {"sheet": SUMMARY_TAB, "codes": sorted(TICKET_FORMULA_ERROR_CODES),
+ "count": len(errors), "errors": errors,
+ "negative_control": {"planted": len(expected), "found": len(found)}}
+
+
+def _formula_error_comparison(before, after, edits, reported_errors=()):
+ """Reject new errors anywhere and every residual error in explicitly planned cells.
+
+ Compare identities, not totals: fixing one cell cannot pay for breaking another. Code
+ aliases denote the same failure; unplanned cell changes still have their separate guard.
+ Opaque engine diagnostics cannot be matched to a pre-existing error and fail closed.
+ """
+ def identity(row):
+ if not isinstance(row, dict) or not row.get("sheet") or not row.get("address"):
+ return None
+ code = row.get("excelCode") or row.get("code")
+ return (str(row["sheet"]).casefold(), str(row["address"]).upper(),
+ _EXCEL_ERROR_ALIASES.get(code, code))
+
+ baseline = {identity(row) for row in formula_error_census(before)}
+ repaired = {(row["sheet"].casefold(), row["address"]) for row in edits}
+ introduced, remaining, unchanged, seen = [], [], [], set()
+ for row in [*formula_error_census(after), *reported_errors]:
+ key = identity(row)
+ if key is not None and key in seen:
+ continue
+ seen.add(key)
+ if key is None or key not in baseline:
+ introduced.append(copy.deepcopy(row))
+ if key is not None and key[:2] in repaired:
+ remaining.append(copy.deepcopy(row))
+ elif key is not None and key in baseline:
+ unchanged.append(copy.deepcopy(row))
+ return {"introduced": introduced, "remaining_in_repair": remaining,
+ "preexisting_elsewhere": unchanged}
+
+
+def _project(book, edits):
+ projected = copy.deepcopy(book)
+ for edit in edits:
+ tab = _tab(projected, edit["sheet"])
+ cells = tab.setdefault("cells", {})
+ if edit["after"] is None:
+ cells.pop(edit["address"], None)
+ else:
+ cells[edit["address"]] = copy.deepcopy(edit["after"])
+ return projected
+
+
+def _cell_map(book):
+ return {(str(tab.get("name") or "").casefold(), address): copy.deepcopy(cell)
+ for tab in book.get("sheets") or () if isinstance(tab, dict)
+ for address, cell in (tab.get("cells") or {}).items()}
+
+
+def _checked_projection(projected, edits, formula_check):
+ """Make formula output part of the plan, never an unpersisted preflight side effect."""
+ if not callable(formula_check):
+ errors = formula_error_census(projected)
+ return projected, errors
+ result = formula_check(copy.deepcopy(projected))
+ if isinstance(result, dict) and isinstance(result.get("book"), dict):
+ checked = result["book"]
+ errors = list(result.get("errors") or ())
+ else:
+ checked = copy.deepcopy(projected)
+ errors = list(result or ())
+ before, after = _cell_map(projected), _cell_map(checked)
+ planned = {(row["sheet"].casefold(), row["address"]): row for row in edits}
+ sheet_names = {str(tab.get("name") or "").casefold(): tab.get("name")
+ for tab in projected.get("sheets") or () if isinstance(tab, dict)}
+ for slot in sorted(before.keys() | after.keys()):
+ if before.get(slot) == after.get(slot):
+ continue
+ row = planned.get(slot)
+ if row is None or row["kind"] == "recalculated":
+ previous, recalculated = before.get(slot), after.get(slot)
+ # Recalculation may refresh a healthy formula's result, but may not repair
+ # an unrelated refused formula or change authored content or presentation.
+ if not all(isinstance(cell, dict) and cell.get("kind") == "formula"
+ and isinstance(cell.get("formula"), str)
+ and not cell.get("error") and not cell.get("refusal")
+ and cell.get("state") != "refused"
+ for cell in (previous, recalculated)) \
+ or previous.get("formula") != recalculated.get("formula") \
+ or {key: previous[key] for key in _VISUAL_CELL_KEYS if key in previous} \
+ != {key: recalculated[key] for key in _VISUAL_CELL_KEYS if key in recalculated}:
+ raise RepairRefusal(
+ "repair_unplanned_formula_change",
+ "Formula validation would change a cell outside the supplied source diff.")
+ desired = sheets.clean_cell(_preserve_visual(previous, recalculated), slot[1])
+ if row is None:
+ sheet = sheet_names[slot[0]]
+ row = {"kind": "recalculated", "name": sheet + "!" + slot[1],
+ "sheet": sheet, "address": slot[1],
+ "before": copy.deepcopy(previous), "after": desired}
+ edits.append(row)
+ planned[slot] = row
+ else:
+ row["after"] = desired
+ continue
+ if row is None or row["kind"] not in {"source", "derived"} \
+ or not isinstance(row.get("after"), dict) \
+ or row["after"].get("kind") != "formula":
+ raise RepairRefusal(
+ "repair_unplanned_formula_change",
+ "Formula validation would change a cell outside the supplied source diff.")
+ recalculated = after.get(slot)
+ if not isinstance(recalculated, dict) \
+ or (row["kind"] == "source"
+ and recalculated.get("value") != row.get("cached_value")):
+ raise RepairRefusal(
+ "repair_formula_cache_mismatch",
+ row["name"] + " does not recalculate to its supplied cached Excel value.")
+ row["after"] = sheets.clean_cell(recalculated, row["address"])
+ return _project(projected, [
+ {**row, "before": before.get((row["sheet"].casefold(), row["address"]))}
+ for row in edits
+ if before.get((row["sheet"].casefold(), row["address"])) != row["after"]
+ ]), errors
+
+
+def summary_block_snapshot(book):
+ """Read-only QI3 pin. The lead saves this as proto/before/QI3-summary-block.json.
+
+ This is the measured F31:R35 extent, not a search for a plausible summary. Labels and
+ units are included in the evidence but are never proposed as edits.
+ """
+ tab = _tab(book, SUMMARY_TAB)
+ cells = tab.get("cells") or {}
+ labels = {28: ("C", "Invoice Data Metrics"), 30: ("D", "Total Sales"),
+ 31: ("D", "Current YTD"), 32: ("D", "Previous YTD"),
+ 33: ("D", "YTD Growth"), 34: ("D", "YTD Growth"),
+ 35: ("D", "YTD improvement")}
+ if book.get("id") != TARGET_WORKBOOK_ID or tab.get("id") != SUMMARY_TAB_ID or any(
+ str((cells.get(col + str(row)) or {}).get("value") or "").strip() != label
+ for row, (col, label) in labels.items()):
+ raise RepairRefusal("repair_summary_extent_mismatch",
+ "The workbook does not match the pinned Invoice Data Metrics block.")
+ addresses = {col + str(row) for col in ("D", "E", *SUMMARY_COLUMNS)
+ for row in range(31, 36)} | {"C28", "D30", "D10"}
+ addresses |= {col + str(row) for col in SUMMARY_COLUMNS for row in (5, 6, 7)}
+ addresses |= {addr for addr in cells if re.fullmatch(r"[A-Z]+2", addr)}
+ return {"workbook_id": book["id"], "revision": book.get("rev"),
+ "workbook_digest": sheets.workbook_digest(book), "sheet": SUMMARY_TAB,
+ "sheet_id": tab["id"], "figure_extent": "F31:R35", "figure_count": 65,
+ "cells": {addr: copy.deepcopy(cells.get(addr)) for addr in sorted(addresses)}}
+
+
+def _summary_value(book, address):
+ cell = (_tab(book, SUMMARY_TAB).get("cells") or {}).get(address) or {}
+ if cell.get("error") or cell.get("state") == "refused":
+ raise RepairRefusal("repair_summary_input_unreadable",
+ "Summary input " + address + " has a calculation error.")
+ return cell.get("value")
+
+
+def _summary_date(value):
+ try:
+ return date.fromisoformat(str(value))
+ except ValueError as exc:
+ raise RepairRefusal("repair_summary_date_invalid",
+ "Every summary date must be a real ISO date from the sheet or mirror.") from exc
+
+
+def _resolved(binding, book, resolve_binding):
+ if not callable(resolve_binding):
+ raise RepairRefusal("repair_binding_resolver_required", "The summary needs its scoped source resolver.")
+ result = resolve_binding(copy.deepcopy(binding), book, _tab(book, SUMMARY_TAB))
+ if not isinstance(result, dict) or "value" not in result or result.get("value") is None:
+ raise RepairRefusal("repair_binding_unreadable", "A summary source did not return a value.")
+ if result.get("limits"):
+ # The route wraps this exception, retaining its message but not custom attributes.
+ # Include each cause and fix there too; upstream clause separators become sentences.
+ details = []
+ for limit in result["limits"]:
+ if isinstance(limit, dict):
+ cause = str(limit.get("cause") or "The source reported an incomplete answer.")
+ fix = str(limit.get("recommendation") or "Repair the source scope and retry.")
+ details.append("Cause: " + cause + " Recommended fix: " + fix)
+ detail = " ".join(details).replace(chr(8212), ". ").replace(chr(8211), " to ")
+ exc = RepairRefusal("repair_binding_limited",
+ "A summary source could not apply its complete scope. " + detail)
+ exc.limits = copy.deepcopy(result["limits"])
+ raise exc
+ return result["value"]
+
+
+def _summary_formula_projection(book, edits, formula_check):
+ if not callable(formula_check):
+ raise RepairRefusal("repair_formula_engine_required",
+ "The summary requires the real document formula engine.")
+ result = formula_check(copy.deepcopy(book))
+ if not isinstance(result, dict) or not isinstance(result.get("book"), dict):
+ raise RepairRefusal("repair_formula_engine_required",
+ "Formula validation must return the calculated document.")
+ # Only adopt planned formulas here. The final whole-document projection records healthy
+ # downstream recalculations as exact edits and refuses every other unplanned change.
+ for row in edits:
+ if row["kind"] == "derived":
+ cell = (_tab(result["book"], row["sheet"]).get("cells") or {}).get(row["address"])
+ if not isinstance(cell, dict) or cell.get("kind") != "formula" or cell.get("error") \
+ or cell.get("state") != "fresh":
+ raise RepairRefusal("repair_formula_errors",
+ "Summary formula " + row["address"] + " could not calculate.")
+ row["after"] = sheets.clean_cell(cell, row["address"])
+ return _project(book, edits)
+
+
+def _append_summary_ratio_plan(current, edits, formula_check):
+ """Plan the measured roots and let the real engine decide cascade blank handling."""
+ if not callable(formula_check):
+ raise RepairRefusal("repair_formula_engine_required",
+ "The ratio repair requires the real document formula engine.")
+ cells = _tab(current, SUMMARY_TAB).get("cells") or {}
+ planned = {(row["sheet"].casefold(), row["address"]) for row in edits}
+ ratios = {}
+ for address, authored in {**_SUMMARY_RATIO_ROOTS, **_SUMMARY_RATIO_CASCADES}.items():
+ before = copy.deepcopy(cells.get(address))
+ guarded = '=IFERROR(' + authored[1:] + ',"")'
+ if not isinstance(before, dict) or before.get("kind") != "formula" \
+ or before.get("formula") not in (authored, guarded):
+ raise RepairRefusal("repair_ratio_formula_mismatch",
+ "The measured ratio formula has changed at " + address + ".")
+ if (SUMMARY_TAB.casefold(), address) in planned:
+ raise RepairRefusal("repair_edit_duplicate", "A repair cell may be named only once.")
+ desired = copy.deepcopy(before)
+ for key in ("value", "display", "error", "refusal"):
+ desired.pop(key, None)
+ desired.update(state="stale", formula=(guarded if address in _SUMMARY_RATIO_ROOTS
+ else before["formula"]))
+ row = {"kind": "derived", "name": SUMMARY_TAB + "!" + address,
+ "sheet": SUMMARY_TAB, "address": address, "before": before,
+ "after": sheets.clean_cell(_preserve_visual(before, desired), address)}
+ edits.append(row)
+ ratios[address] = row
+ result = formula_check(_project(current, edits))
+ if not isinstance(result, dict) or not isinstance(result.get("book"), dict):
+ raise RepairRefusal("repair_formula_engine_required",
+ "Formula validation must return the calculated document.")
+ calculated = _tab(result["book"], SUMMARY_TAB).get("cells") or {}
+ guarded_cascades = []
+ for address, authored in _SUMMARY_RATIO_CASCADES.items():
+ cell = calculated.get(address) or {}
+ error = cell.get("error") or (cell.get("refusal") if cell.get("state") == "refused" else None)
+ # The current engine refuses arithmetic over IFERROR's empty string as `blank`.
+ # Only that observed refusal authorizes wrapping a cascade; other failures remain
+ # visible to the existing projection refusal. Never replace a formula with a value.
+ if isinstance(error, dict) and error.get("code") == "blank":
+ ratios[address]["after"]["formula"] = '=IFERROR(' + authored[1:] + ',"")'
+ guarded_cascades.append(address)
+ return {"roots": list(_SUMMARY_RATIO_ROOTS), "guarded_cascades": guarded_cascades}
+
+
+def _build_summary_plan(current, request, resolve_binding, formula_check):
+ """Build all 65 figures, date/scope helpers and guarded ratios, without tenant writes.
+
+ summary_block carries before=summary_block_snapshot(current), pricelist, the complete
+ partner_ids behind customer_link, and validate_invoice=modules.sheet.validate. The last
+ is a callable injected by the trusted operator (this workflow has no HTTP route).
+ No core-to-modules/API import or second source-query door is needed.
+ """
+ summary = request["summary_block"]
+ if not isinstance(summary, dict) or summary.get("before") != summary_block_snapshot(current):
+ raise RepairRefusal("repair_summary_pin_required",
+ "The summary requires the exact before snapshot recorded by the lane lead.")
+ if request.get("source_cells") or request.get("binding_cells"):
+ raise RepairRefusal("repair_summary_plan_conflict",
+ "Run the workbook-derived summary separately from an Excel source-oracle repair.")
+ if not str(request.get("refreshed_at") or "").strip():
+ raise RepairRefusal("repair_timestamp_required", "The summary needs an explicit refresh timestamp.")
+ cells = _tab(current, SUMMARY_TAB).get("cells") or {}
+ # Allocate four fully empty rows inside the existing sheet dimensions. Never overwrite
+ # authored content, add a tab, resize the sheet, or change hidden/print/visual metadata.
+ tab_rows = int(_tab(current, SUMMARY_TAB).get("rows") or 0)
+ helper_row = next((row for row in range(tab_rows - 3, 35, -1)
+ if all((cells.get(col + str(r)) or {}).get("kind", "empty") == "empty"
+ for r in range(row, row + 4) for col in SUMMARY_COLUMNS)), None)
+ if helper_row is None:
+ raise RepairRefusal("repair_summary_helper_space",
+ "The summary needs four empty rows in its existing sheet for date and scope formulas.")
+ edits = []
+
+ def add(address, desired, kind):
+ before = copy.deepcopy(cells.get(address))
+ edits.append({"kind": kind, "name": SUMMARY_TAB + "!" + address,
+ "sheet": SUMMARY_TAB, "address": address, "before": before,
+ "after": sheets.clean_cell(_preserve_visual(before, desired), address)})
+
+ def formula(address, text):
+ add(address, {"kind": "formula", "state": "stale", "formula": text}, "derived")
+
+ scope_ref = "F" + str(helper_row + 3)
+ formula(scope_ref, '=IF($D$10="RI","Royal 1 (USD)",IF($D$10="FFS","Fisch (USD)",NA()))')
+ windows = {}
+ for col in SUMMARY_COLUMNS:
+ end = _summary_date(_summary_value(current, col + "7"))
+ year_refs = [addr for addr, cell in cells.items() if re.fullmatch(r"[A-Z]+2", addr)
+ and str((cell or {}).get("value")) in {str(end.year), str(float(end.year))}]
+ # The year can be one merged/shared header. Prefer this column, otherwise require
+ # an unambiguous year cell; never guess between two independent calendar headers.
+ year_ref = col + "2" if col + "2" in year_refs else year_refs[0] if len(year_refs) == 1 else None
+ if not year_ref:
+ raise RepairRefusal("repair_summary_year_ambiguous",
+ "The summary needs an unambiguous year on row 2 for column " + col + ".")
+ first, previous_first, previous_end = (col + str(helper_row + offset) for offset in range(3))
+ # DATE/EDATE already return ISO dates. String concatenation preserves that value
+ # without introducing an inferred date style on a previously blank helper cell.
+ formula(first, '=DATE(' + year_ref + ',1,1)&""')
+ formula(previous_first, '=DATE(' + year_ref + '-1,1,1)&""')
+ formula(previous_end, '=EDATE(' + col + '7,-12)&""')
+ windows[col] = ((first, col + "7"), (previous_first, previous_end))
+ projected = _summary_formula_projection(_project(current, edits), edits, formula_check)
+ if _summary_value(projected, scope_ref) != summary.get("pricelist"):
+ raise RepairRefusal("repair_summary_partner_scope",
+ "The supplied partner set must use the pricelist computed from D10.")
+ partners = summary.get("partner_ids")
+ if not isinstance(partners, list) or any(type(pid) is not int or pid <= 0 for pid in partners) \
+ or len(set(partners)) != len(partners):
+ raise RepairRefusal("repair_summary_partner_scope",
+ "Supply every joined partner ID once, with no source row limit.")
+ if not callable(summary.get("validate_invoice")):
+ raise RepairRefusal("repair_summary_validator_required",
+ "Inject modules.sheet.validate for independent invoice reconciliation.")
+ as_of = _summary_date(_resolved({"database": "ut_odoo_invoices",
+ "aggregate": {"kind": "column", "fn": "max", "field": "invoice_date"},
+ "scope": {"filter": {"conj": "and", "nodes": []}}},
+ projected, resolve_binding))
+ for col, ranges in windows.items():
+ for row_number, (first, last) in zip((31, 32), ranges):
+ binding = {"database": "ut_odoo_invoices",
+ "aggregate": {"field": "amount_untaxed", "fn": "sum", "kind": "column"},
+ "scope": {"filter": {"conj": "and", "nodes": [
+ {"colId": "invoice_date", "op": "gte", "valueRef": {"ref": first}},
+ {"colId": "invoice_date", "op": "lte", "valueRef": {"ref": last}},
+ {"colId": "move_type", "op": "eq", "value": "out_invoice"}]}},
+ "related": {"via": "customer_link", "filter": {"conj": "and", "nodes": [
+ {"colId": "pricelist", "op": "eq", "valueRef": {"ref": scope_ref}}]}}}
+ _validate_binding(col + str(row_number), binding)
+ value = _resolved(binding, projected, resolve_binding)
+ if type(value) not in (int, float) or not math.isfinite(value):
+ raise RepairRefusal("repair_binding_unreadable", "An invoice total must be a finite number.")
+ add(col + str(row_number), {"kind": "bound", "state": "fresh", "binding": binding,
+ "value": value, "display": str(value),
+ "refreshed_at": request["refreshed_at"]}, "binding")
+ from openpyxl.formula.translate import Translator
+ for row_number in range(33, 36):
+ origin = "F" + str(row_number)
+ source = cells.get(origin) or {}
+ if source.get("kind") != "formula" or not str(source.get("formula") or "").startswith("="):
+ raise RepairRefusal("repair_summary_formula_missing",
+ "The summary requires its authored formula at " + origin + ".")
+ for col in SUMMARY_COLUMNS:
+ address = col + str(row_number)
+ try:
+ text = Translator(source["formula"], origin=origin).translate_formula(address)
+ except Exception as exc:
+ raise RepairRefusal("repair_summary_formula_translate",
+ "The authored summary formula could not be filled across.") from exc
+ formula(address, text)
+ ratio_evidence = _append_summary_ratio_plan(current, edits, formula_check)
+ projected = _summary_formula_projection(_project(current, edits), edits, formula_check)
+ checks = validate(projected, windows=windows, mirror_as_of=as_of.isoformat(),
+ partner_ids=partners, invoice_validate=summary["validate_invoice"],
+ formula_check=formula_check)
+ return edits, {"before": summary["before"], "helper_rows": list(range(helper_row, helper_row + 4)),
+ "pricelist_cell": scope_ref, "mirror_as_of": as_of.isoformat(),
+ "partner_count": len(partners), "checks": checks,
+ "ratio_repair": ratio_evidence}
+
+
+def validate(book, *, windows, mirror_as_of, partner_ids, invoice_validate, formula_check):
+ """Reconcile 26 invoice figures and 39 formulas against independent Odoo results.
+
+ Windows are read from the planned sheet, with each upper bound capped at the mirror's
+ max(invoice_date). Re-evaluate the authored formulas over the independent amounts too.
+ Invoice-date bounding is not a historical snapshot; preserve validate()'s limitation rows.
+ """
+ proof, checks = copy.deepcopy(book), []
+ as_of = _summary_date(mirror_as_of)
+ for col in SUMMARY_COLUMNS:
+ for row_number, (first_ref, last_ref) in zip((31, 32), windows[col]):
+ first = _summary_date(_summary_value(book, first_ref))
+ last = min(_summary_date(_summary_value(book, last_ref)), as_of)
+ if first > last:
+ raise RepairRefusal("repair_summary_asof_before_window",
+ "The invoice mirror ends before a summary window begins. Refresh it first.")
+ address = col + str(row_number)
+ observed = _summary_value(book, address)
+ rows = invoice_validate(first.isoformat(), last.isoformat(), {"invoiced_sales": observed},
+ team_id=None, partner_ids=list(partner_ids),
+ basis="invoice", include_refunds=False)
+ if not isinstance(rows, list) or len(rows) != 1 or rows[0].get("check") != "invoiced_sales" \
+ or rows[0].get("ok") is not True or rows[0].get("a") != observed \
+ or type(rows[0].get("b")) not in (int, float) or not math.isfinite(rows[0]["b"]) \
+ or abs(observed - rows[0]["b"]) > 0.01:
+ exc = RepairRefusal("repair_summary_reconciliation",
+ "Independent Odoo reconciliation failed for " + address + ".")
+ exc.checks = copy.deepcopy(rows)
+ raise exc
+ checks.append({**copy.deepcopy(rows[0]), "address": address,
+ "date_from": first.isoformat(), "date_to": last.isoformat()})
+ _tab(proof, SUMMARY_TAB)["cells"][address]["value"] = rows[0]["b"]
+ result = formula_check(proof)
+ proof = result["book"]
+ for col in SUMMARY_COLUMNS:
+ for row_number in range(33, 36):
+ address = col + str(row_number)
+ actual, expected = _summary_value(book, address), _summary_value(proof, address)
+ ok = (type(actual) in (int, float) and type(expected) in (int, float)
+ and math.isfinite(actual) and math.isfinite(expected)
+ and abs(actual - expected) <= (0.01 if row_number == 33 else 1e-9))
+ checks.append({"check": "invoice_derived", "address": address, "a": actual,
+ "b": expected, "gap": actual - expected if ok else None, "ok": ok})
+ if not ok:
+ raise RepairRefusal("repair_summary_reconciliation",
+ "The summary formula did not reconcile for " + address + ".")
+ return checks
+
+
+def _build_plan(current, request, resolve_binding):
+ if not isinstance(request.get("source_cells"), list) or not request["source_cells"]:
+ raise RepairRefusal(
+ "repair_source_required",
+ "The repair requires a one-to-one source cell oracle with cached Excel values.")
+ if not isinstance(request.get("binding_cells"), list):
+ raise RepairRefusal("repair_bindings_required", "The repair requires both YTD bindings.")
+
+ edits, names, slots = [], set(), set()
+ for row in request["source_cells"]:
+ if not isinstance(row, dict):
+ raise RepairRefusal("repair_source_invalid", "Every source cell must be an object.")
+ name = str(row.get("name") or "").strip()
+ tab_name, address = str(row.get("sheet") or "").strip(), _addr(row.get("address"))
+ if not name or name in names:
+ raise RepairRefusal("repair_source_name_invalid",
+ "Every source cell needs a unique report-cell name.")
+ names.add(name)
+ slot = (tab_name.casefold(), address)
+ if slot in slots:
+ raise RepairRefusal("repair_edit_duplicate", "A repair cell may be named only once.")
+ slots.add(slot)
+ tab = _tab(current, tab_name)
+ before = copy.deepcopy((tab.get("cells") or {}).get(address))
+ supplied = copy.deepcopy(row.get("cell"))
+ if isinstance(supplied, dict) and "imported_value" in supplied:
+ raise RepairRefusal("repair_cache_invalid",
+ name + " carries a temporary imported value instead of its cache.")
+ desired = _preserve_visual(before, supplied)
+ if desired is not None:
+ desired = sheets.clean_cell(desired, addr=address)
+ if desired.get("value") != row.get("cached_value"):
+ raise RepairRefusal("repair_cache_mismatch",
+ name + " does not match its supplied cached Excel value.")
+ elif row.get("cached_value") is not None:
+ raise RepairRefusal("repair_cache_mismatch", name + " is blank but its cache is not.")
+ edits.append({"kind": "source", "name": name, "sheet": tab_name,
+ "address": address, "before": before, "after": desired,
+ "cached_value": copy.deepcopy(row.get("cached_value"))})
+
+ e4 = [row for row in edits if row["address"] == "E4"]
+ if len(e4) != 1 or (e4[0]["after"] is not None
+ and e4[0]["after"].get("kind") != "empty"):
+ raise RepairRefusal(
+ "repair_e4_not_blank",
+ "The source oracle must name E4 exactly once as blank and non-formula.")
+ if "Beg Date" not in names:
+ raise RepairRefusal("repair_beg_date_missing", "The source oracle must name Beg Date.")
+ beg_date = next(row for row in edits if row["name"] == "Beg Date")
+ try:
+ parsed_beg_date = date.fromisoformat(str((beg_date.get("after") or {}).get("value")))
+ except (TypeError, ValueError):
+ parsed_beg_date = None
+ if parsed_beg_date is None or parsed_beg_date.isoformat() != beg_date["cached_value"]:
+ raise RepairRefusal("repair_beg_date_invalid",
+ "Beg Date must carry one real ISO date matching its Excel cache.")
+ expected_names = request.get("named_report_cells")
+ if not isinstance(expected_names, list) or any(
+ not isinstance(item, str) or not item.strip() for item in expected_names):
+ raise RepairRefusal(
+ "repair_source_manifest_required",
+ "The repair requires the complete named report cell manifest from the source.")
+ normalized_manifest = [item.strip() for item in expected_names]
+ if len(set(normalized_manifest)) != len(normalized_manifest) or set(normalized_manifest) != names:
+ raise RepairRefusal(
+ "repair_source_manifest_mismatch",
+ "The source diff must map one-to-one onto the complete named report cell manifest.")
+
+ binding_names = []
+ refreshed_at = str(request.get("refreshed_at") or "").strip()
+ if not refreshed_at:
+ raise RepairRefusal("repair_timestamp_required",
+ "The deterministic repair plan requires an explicit refresh timestamp.")
+ for row in request["binding_cells"]:
+ if not isinstance(row, dict):
+ raise RepairRefusal("repair_binding_invalid", "Every YTD binding must be an object.")
+ name = str(row.get("name") or "").strip()
+ # S64-T21 widened this loop to ANY named report cell, so a blank or repeated name is now
+ # reachable for an Orders row too. The sentence has to cover both, not just the YTD pair.
+ if not name or name in binding_names:
+ raise RepairRefusal("repair_bindings_required",
+ "Every repair binding needs its own report cell name, and the "
+ "repair requires one Current YTD and one Previous YTD among them.")
+ binding_names.append(name)
+ binding = copy.deepcopy(row.get("binding"))
+ _validate_binding(name, binding, window_kind=YTD_NAMES.get(name))
+ tab_name, address = str(row.get("sheet") or "").strip(), _addr(row.get("address"))
+ slot = (tab_name.casefold(), address)
+ if slot in slots:
+ raise RepairRefusal("repair_edit_duplicate", "A repair cell may be named only once.")
+ slots.add(slot)
+ tab = _tab(current, tab_name)
+ before = copy.deepcopy((tab.get("cells") or {}).get(address))
+ if not callable(resolve_binding):
+ raise RepairRefusal(
+ "repair_binding_resolver_required",
+ "Canonical data must resolve both YTD bindings before this repair can run.")
+ resolved = resolve_binding(binding, current, tab)
+ if not isinstance(resolved, dict) or "value" not in resolved:
+ raise RepairRefusal("repair_binding_unreadable", name + " did not resolve canonical data.")
+ canonical_binding = copy.deepcopy(resolved.get("binding") or binding)
+ _validate_binding(name, canonical_binding, window_kind=YTD_NAMES.get(name))
+ desired = _preserve_visual(before, {
+ "kind": "bound", "state": "fresh", "binding": canonical_binding,
+ "value": copy.deepcopy(resolved["value"]),
+ "display": "" if resolved["value"] is None else str(resolved["value"]),
+ "refreshed_at": refreshed_at,
+ })
+ desired = sheets.clean_cell(desired, addr=address)
+ edits.append({"kind": "binding", "name": name, "sheet": tab_name,
+ "address": address, "before": before, "after": desired})
+ if set(YTD_NAMES) - set(binding_names):
+ raise RepairRefusal("repair_bindings_required",
+ "The repair requires one Current YTD and one Previous YTD binding.")
+
+ edits.sort(key=lambda row: (row["sheet"].casefold(), row["address"]))
+ return edits
+
+
+def _write_backup(backup_store, current, digest, diff_digest, workbook_id):
+ revision = int(current.get("rev") or 0)
+ key = "sheet_repair_backup_" + workbook_id + "_r" + str(revision) + "_" + digest[:16]
+ record = {"operation": operation_for(workbook_id), "workbook_id": workbook_id,
+ "revision": revision, "digest": digest, "diff_digest": diff_digest,
+ "workbook": copy.deepcopy(current)}
+
+ def apply(existing):
+ if existing and existing != record:
+ raise RepairRefusal("repair_backup_conflict",
+ "A different backup already occupies this repair key.")
+ return copy.deepcopy(record)
+
+ backup_store.update(key, apply, flush="sync")
+ readback = backup_store.get(key)
+ if readback != record or sheets.workbook_digest(readback.get("workbook") or {}) != digest:
+ raise RepairRefusal("repair_backup_unverified",
+ "The workbook backup could not be read back byte for byte.")
+ return key
+
+
+def execute(st, backup_store, request, *, target_workbook_id=TARGET_WORKBOOK_ID,
+ resolve_binding, formula_check=None, actor="", actor_name=""):
+ """Validate, back up, repair and read back only the workbook this call is POINTED AT.
+
+ ⭐ `target_workbook_id` is what makes the repair general, and it is the one argument to read
+ twice. It is checked against THREE independent things before a byte moves: the request's own
+ `workbook_id`, the marker's (through `operation_for`, so a marker minted for one workbook
+ cannot authorise another), and the STORED document's identity. A caller pointed at a fixture
+ and handed a request for another workbook is refused, which is the whole point of S64-T21.
+
+ ⚠ THE DEFAULT IS THE OWNER'S WORKBOOK, DELIBERATELY, AND IT IS NOT A HOLE. It is the one
+ workbook this module was written for and the only one an unpointed call could ever have
+ reached; every other target must be NAMED. An EXPLICIT empty or non-string target still
+ refuses `repair_target_required`, so "pointed at nothing" and "pointed at the default" stay
+ different answers. Making the argument mandatory instead would have been a breaking change to
+ `verify_sheet.py`'s five S63-T21 call sites, a gate no lane on this board owns, and it would
+ have bought no safety the route does not already give: `routes_sheet.py` carries the same
+ default for the operator.
+ """
+ if not isinstance(target_workbook_id, str) or not target_workbook_id.strip():
+ raise RepairRefusal("repair_target_required",
+ "The repair requires an explicitly named target workbook.")
+ target_workbook_id = target_workbook_id.strip()
+ if backup_store is None or not callable(getattr(backup_store, "update", None)) \
+ or not callable(getattr(backup_store, "get", None)):
+ raise RepairRefusal("repair_backup_required",
+ "A readable, writable backup store is required before repair.")
+ if not isinstance(request, dict) or request.get("workbook_id") != target_workbook_id:
+ raise RepairRefusal("repair_workbook_mismatch",
+ "This operation may repair only workbook " + target_workbook_id + ".")
+ revision = request.get("expected_revision")
+ if isinstance(revision, bool) or not isinstance(revision, int) or revision < 0:
+ raise RepairRefusal("repair_revision_required",
+ "An exact expected workbook revision is required for repair.")
+ _validate_marker(request.get("marker"), revision, target_workbook_id)
+ try:
+ calculation_date = date.fromisoformat(str(request.get("calculation_date") or ""))
+ except ValueError as exc:
+ raise RepairRefusal("repair_calculation_date_required",
+ "The deterministic repair plan requires one ISO calculation date.") from exc
+ if calculation_date.isoformat() != request["calculation_date"]:
+ raise RepairRefusal("repair_calculation_date_required",
+ "The deterministic repair plan requires one ISO calculation date.")
+
+ current = sheets.workbook_snapshot(st, target_workbook_id)
+ if str(current.get("id") or "") != target_workbook_id:
+ raise RepairRefusal("repair_workbook_mismatch",
+ "The stored workbook identity does not match this repair target.")
+ if int(current.get("rev") or 0) != revision:
+ raise RepairRefusal("repair_revision_mismatch",
+ "The named workbook is not at the expected repair revision.")
+ digest = sheets.workbook_digest(current)
+ summary_evidence = None
+ summary_errors_before = None
+ if "summary_block" in request:
+ summary_errors_before = check_summary_formula_errors(current)
+ edits, summary_evidence = _build_summary_plan(current, request, resolve_binding, formula_check)
+ else:
+ edits = _build_plan(current, request, resolve_binding)
+ projected = _project(current, edits)
+ projected, projected_errors = _checked_projection(projected, edits, formula_check)
+ comparison = _formula_error_comparison(current, projected, edits, projected_errors)
+ summary_errors_after = (check_summary_formula_errors(projected)
+ if summary_evidence is not None else None)
+ if comparison["introduced"] or comparison["remaining_in_repair"] \
+ or (summary_errors_after is not None and summary_errors_after["count"]):
+ raise RepairRefusal(
+ "repair_formula_errors",
+ "The projected repair introduces a formula error or leaves one in its repair scope, "
+ "so no repair was applied.")
+ visual_before = visual_metadata(current)
+ if visual_metadata(projected) != visual_before:
+ raise RepairRefusal("repair_visual_metadata_changed",
+ "The projected repair changes Daily Sales visual metadata.")
+
+ exact_edits = [
+ {key: copy.deepcopy(row[key]) for key in ("sheet", "address", "before", "after")}
+ for row in edits]
+ diff_digest = sheets.workbook_digest({"edits": exact_edits})
+ backup_key = _write_backup(backup_store, current, digest, diff_digest, target_workbook_id)
+ write = sheets.apply_exact_cell_edits(
+ st, target_workbook_id, exact_edits,
+ revision, digest, actor=actor, actor_name=actor_name)
+ readback = sheets.workbook_snapshot(st, target_workbook_id)
+
+ if int(readback.get("rev") or 0) != revision + 1:
+ raise RepairRefusal("repair_readback_revision",
+ "The repaired workbook revision did not read back as expected.")
+ for row in edits:
+ actual = copy.deepcopy((_tab(readback, row["sheet"]).get("cells") or {}).get(row["address"]))
+ expected = None if row["after"] is None else sheets.clean_cell(row["after"], row["address"])
+ if actual != expected:
+ raise RepairRefusal("repair_readback_cell",
+ row["sheet"] + "!" + row["address"] + " did not read back exactly.")
+ checked_readback, readback_errors = _checked_projection(readback, copy.deepcopy(edits), formula_check)
+ readback_comparison = _formula_error_comparison(current, checked_readback, edits, readback_errors)
+ summary_errors_readback = (check_summary_formula_errors(checked_readback)
+ if summary_evidence is not None else None)
+ if readback_comparison["introduced"] or readback_comparison["remaining_in_repair"] \
+ or (summary_errors_readback is not None and summary_errors_readback["count"]) \
+ or _cell_map(checked_readback) != _cell_map(readback):
+ raise RepairRefusal("repair_readback_formula_state",
+ "The repaired workbook did not read back calculation-current.")
+ if visual_metadata(readback) != visual_before:
+ raise RepairRefusal("repair_readback_visual_metadata",
+ "Daily Sales visual metadata changed during repair.")
+ if summary_evidence is not None:
+ tab = _tab(readback, SUMMARY_TAB)
+ if any((tab.get("cells", {}).get(col + str(row)) or {}).get("kind")
+ != ("bound" if row in (31, 32) else "formula")
+ for col in SUMMARY_COLUMNS for row in range(31, 36)):
+ raise RepairRefusal("repair_summary_readback_kind",
+ "Every summary figure must read back as its planned binding or formula.")
+ summary_evidence["readback"] = summary_block_snapshot(readback)
+ summary_evidence["formula_error_check"] = {
+ "before": summary_errors_before, "projected": summary_errors_after,
+ "readback": summary_errors_readback,
+ }
+
+ source_readback = {row["name"]: copy.deepcopy(
+ (_tab(readback, row["sheet"]).get("cells") or {}).get(row["address"]))
+ for row in edits if row["kind"] == "source"}
+ bindings = {row["name"]: copy.deepcopy(
+ ((_tab(readback, row["sheet"]).get("cells") or {}).get(row["address"]) or {}).get("binding"))
+ for row in edits if row["kind"] == "binding"}
+ result = {
+ "workbook_id": target_workbook_id,
+ "revision_before": revision,
+ "revision_after": readback["rev"],
+ "backup_key": backup_key,
+ "backup_digest": digest,
+ "diff_digest": diff_digest,
+ "applied": write["edits"],
+ "paths": write["paths"],
+ "source_cells": source_readback,
+ "bindings": bindings,
+ "formula_errors": formula_error_census(readback),
+ "formula_error_comparison": readback_comparison,
+ "visual_metadata_preserved": True,
+ }
+ if summary_evidence is not None:
+ result["summary_block"] = summary_evidence
+ return result
diff --git a/platform/core/external_access.py b/platform/core/external_access.py
index 2312af7687425a10c923e79c08c44f4b4ff69e87..7d2dcc594d48a56a831c8ed76b7b9d06a4c8f598 100644
--- a/platform/core/external_access.py
+++ b/platform/core/external_access.py
@@ -8,6 +8,7 @@ credential, user, bucket, or tenant runtime.
"""
import calendar
from dataclasses import dataclass
+from datetime import datetime, timezone
import hashlib
import hmac
import secrets
@@ -25,6 +26,7 @@ ALL_DATABASES = "all"
MAX_SCOPE_DATABASES = 200
ALGORITHM = "sha256-salt-v1"
AUTH_ERROR = "invalid_external_credential"
+_UNSET = object()
_DUMMY_SALT = bytes(32)
_DUMMY_HASH = hashlib.sha256(_DUMMY_SALT + b"invalid").hexdigest()
#: Seconds between two durable `last_used` stamps for ONE credential.
@@ -114,6 +116,28 @@ def _now() -> str:
return time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
+def _epoch(value: str | None) -> float | None:
+ if value is None:
+ return None
+ try:
+ return datetime.strptime(value, "%Y-%m-%dT%H:%M:%SZ").replace(
+ tzinfo=timezone.utc).timestamp()
+ except (TypeError, ValueError, OverflowError):
+ return None
+
+
+def normalise_expiry(value, *, now: float | None = None) -> str | None:
+ """Return one canonical future UTC expiry, or ``None`` for no expiry."""
+ if value is None or value == "":
+ return None
+ if not isinstance(value, str):
+ raise ExternalAccessError("external_expiry_invalid")
+ parsed = _epoch(value.strip())
+ if parsed is None or parsed <= (time.time() if now is None else float(now)):
+ raise ExternalAccessError("external_expiry_invalid")
+ return time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime(parsed))
+
+
def _stamp_is_due(last_used, now: float) -> bool:
"""Is this credential's `last_used` stale enough to be worth a durable write?
@@ -164,7 +188,7 @@ def _assert_bound_runtime(st, tenant: str) -> None:
raise ExternalAccessError("external_identity_invalid")
-def _active_public_user(username: str, tenant: str) -> dict:
+def resolve_active_user(username: str, tenant: str) -> dict:
try:
registry = users.registry() or {}
except Exception as exc:
@@ -180,6 +204,20 @@ def _active_public_user(username: str, tenant: str) -> dict:
return public
+def _active_public_user(username: str, tenant: str) -> dict:
+ """Backward-compatible private spelling retained for the SDLC 62 seam."""
+ return resolve_active_user(username, tenant)
+
+
+def _normalise_label(label) -> str:
+ clean = str(label or "").strip()
+ if (not clean or len(clean) > 80 or any(ord(char) < 32 for char in clean)
+ or "lpb1." in clean or "lpo1." in clean or "lpr1." in clean
+ or "lpc1." in clean):
+ raise ExternalAccessError("external_label_invalid")
+ return clean
+
+
def _metadata(record: dict) -> dict:
"""Return list-safe metadata. Salt, digest, algorithm and owner never cross this seam."""
access, databases = scope_of(record)
@@ -189,6 +227,8 @@ def _metadata(record: dict) -> dict:
"created": record.get("created"),
"last_used": record.get("last_used"),
"revoked_at": record.get("revoked_at"),
+ "rotated_at": record.get("rotated_at"),
+ "expires_at": record.get("expires_at"),
"access": access,
"databases": ALL_DATABASES if databases == ALL_DATABASES else list(databases),
}
@@ -205,16 +245,16 @@ def label_for(st, key_id: str) -> str:
return label
-def mint(st, username: str, tenant: str, label: str, access=None, databases=None) -> tuple[dict, str]:
+def mint(st, username: str, tenant: str, label: str, access=None, databases=None,
+ expires_at=None) -> tuple[dict, str]:
"""Create one credential and return ``(metadata, token)``; the token is never persisted."""
access = normalise_access(access)
databases = normalise_databases(databases)
owner, tenant_key = _normalise_owner(username, tenant)
_assert_bound_runtime(st, tenant_key)
_active_public_user(owner, tenant_key)
- clean_label = str(label or "").strip()
- if not clean_label or len(clean_label) > 80:
- raise ExternalAccessError("external_label_invalid")
+ clean_label = _normalise_label(label)
+ expiry = normalise_expiry(expires_at)
secret = secrets.token_urlsafe(32)
salt = secrets.token_bytes(32)
@@ -244,6 +284,8 @@ def mint(st, username: str, tenant: str, label: str, access=None, databases=None
"created": created,
"last_used": None,
"revoked_at": None,
+ "rotated_at": None,
+ "expires_at": expiry,
}
book[key_id] = record
minted["record"] = record
@@ -262,6 +304,94 @@ def mint(st, username: str, tenant: str, label: str, access=None, databases=None
return _metadata(record), token
+def edit(st, username: str, tenant: str, key_id: str, *, label=_UNSET, access=_UNSET,
+ databases=_UNSET, expires_at=_UNSET) -> dict:
+ """Edit metadata/scope in one store update; credential material is left untouched."""
+ owner, tenant_key = _normalise_owner(username, tenant)
+ _assert_bound_runtime(st, tenant_key)
+ resolve_active_user(owner, tenant_key)
+ target = str(key_id or "").strip()
+ if label is access is databases is expires_at is _UNSET:
+ raise ExternalAccessError("external_edit_empty")
+ clean_label = _normalise_label(label) if label is not _UNSET else _UNSET
+ clean_access = normalise_access(access) if access is not _UNSET else _UNSET
+ clean_databases = normalise_databases(databases) if databases is not _UNSET else _UNSET
+ clean_expiry = normalise_expiry(expires_at) if expires_at is not _UNSET else _UNSET
+ edited: dict = {}
+
+ def _set(value):
+ if not isinstance(value, dict):
+ raise ExternalAccessError("external_key_not_found")
+ book = dict(value)
+ current = book.get(target)
+ if (not isinstance(current, dict) or current.get("username") != owner
+ or current.get("tenant") != tenant_key or current.get("revoked_at")):
+ raise ExternalAccessError("external_key_not_found")
+ record = dict(current)
+ if clean_label is not _UNSET:
+ record["label"] = clean_label
+ if clean_access is not _UNSET:
+ record["access"] = clean_access
+ if clean_databases is not _UNSET:
+ record["databases"] = (ALL_DATABASES if clean_databases == ALL_DATABASES
+ else list(clean_databases))
+ if clean_expiry is not _UNSET:
+ record["expires_at"] = clean_expiry
+ book[target] = record
+ edited["record"] = record
+ return book
+
+ try:
+ st.update(BUCKET, _set, flush="sync")
+ except ExternalAccessError:
+ raise
+ except Exception as exc:
+ raise ExternalAccessError("external_access_unavailable") from exc
+ record = edited.get("record")
+ if not isinstance(record, dict):
+ raise ExternalAccessError("external_access_unavailable")
+ return _metadata(record)
+
+
+def rotate(st, username: str, tenant: str, key_id: str) -> tuple[dict, str]:
+ """Replace one owned key's secret atomically and reveal the replacement exactly once."""
+ owner, tenant_key = _normalise_owner(username, tenant)
+ _assert_bound_runtime(st, tenant_key)
+ resolve_active_user(owner, tenant_key)
+ target = str(key_id or "").strip()
+ secret = secrets.token_urlsafe(32)
+ salt = secrets.token_bytes(32)
+ rotated: dict = {}
+
+ def _set(value):
+ if not isinstance(value, dict):
+ raise ExternalAccessError("external_key_not_found")
+ book = dict(value)
+ current = book.get(target)
+ if (not isinstance(current, dict) or current.get("username") != owner
+ or current.get("tenant") != tenant_key or current.get("revoked_at")):
+ raise ExternalAccessError("external_key_not_found")
+ record = dict(current)
+ record.update({
+ "salt": salt.hex(), "hash": _digest(salt, secret), "algo": ALGORITHM,
+ "rotated_at": _now(), "last_used": None,
+ })
+ book[target] = record
+ rotated["record"] = record
+ return book
+
+ try:
+ st.update(BUCKET, _set, flush="sync")
+ except ExternalAccessError:
+ raise
+ except Exception as exc:
+ raise ExternalAccessError("external_access_unavailable") from exc
+ record = rotated.get("record")
+ if not isinstance(record, dict):
+ raise ExternalAccessError("external_access_unavailable")
+ return _metadata(record), f"lpb1.{tenant_key}.{target}.{secret}"
+
+
def list_for_user(st, username: str, tenant: str) -> list[dict]:
"""List only this user's active credential metadata, newest first."""
owner, tenant_key = _normalise_owner(username, tenant)
@@ -355,6 +485,8 @@ def verify(token: str) -> VerifiedCredential:
and record.get("tenant") == tenant_key
and bool(record.get("username"))
and not record.get("revoked_at")
+ and (_epoch(record.get("expires_at")) is None
+ if not record.get("expires_at") else _epoch(record.get("expires_at")) > time.time())
)
if not record_ok or not matched:
raise ExternalAuthError(AUTH_ERROR)
diff --git a/platform/core/external_oauth.py b/platform/core/external_oauth.py
new file mode 100644
index 0000000000000000000000000000000000000000..6997d59fa7b4e804150f8d6687c8a118f77ed3cf
--- /dev/null
+++ b/platform/core/external_oauth.py
@@ -0,0 +1,723 @@
+"""Hash-only OAuth 2.1-style authorization for the existing remote MCP resource.
+
+The authorization endpoint is the only browser-session step. Code exchange, refresh, bearer
+verification, and disconnect resolve from opaque tokens and fresh tenant state without a cookie.
+Public clients use PKCE S256 and have no shared client secret to leak or impersonate a user.
+"""
+from __future__ import annotations
+
+import base64
+from dataclasses import dataclass
+import hashlib
+import hmac
+import secrets
+import time
+from urllib.parse import urlsplit, urlunsplit
+
+from core import external_access
+
+
+BUCKET = "external_oauth"
+ALGORITHM = "sha256-salt-v1"
+CODE_TTL_SECONDS = 5 * 60
+ACCESS_TTL_SECONDS = 60 * 60
+REFRESH_TTL_SECONDS = 30 * 24 * 60 * 60
+MAX_REFRESH_ROTATIONS = 256
+MAX_REDIRECT_URIS = 10
+SCOPES = ("mcp:read", "mcp:write", "offline_access")
+AUTH_ERROR = external_access.AUTH_ERROR
+_STAMP_EVERY = 900.0
+_DUMMY_SALT = bytes(32)
+_DUMMY_HASH = hashlib.sha256(_DUMMY_SALT + b"invalid").hexdigest()
+_UNSET = object()
+
+
+class ExternalOAuthError(external_access.ExternalAccessError):
+ """Stable OAuth refusal; HTTP vocabulary belongs to the API route."""
+
+
+class ExternalOAuthAuthError(ExternalOAuthError):
+ """Uniform bearer refusal with no tenant, client, or grant oracle."""
+
+
+@dataclass(frozen=True)
+class VerifiedOAuth:
+ tenant: str
+ grant_id: str
+ client_id: str
+ user: dict
+ runtime: object
+ access: str
+ databases: object
+ scopes: tuple[str, ...]
+ resource: str
+
+
+@dataclass(frozen=True)
+class DisconnectedOAuth:
+ """Safe attribution plus public metadata for one token-authenticated disconnect."""
+
+ tenant: str
+ username: str
+ connection: dict
+
+
+def _now(now=None) -> float:
+ return time.time() if now is None else float(now)
+
+
+def _iso(epoch: float | None) -> str | None:
+ if epoch is None:
+ return None
+ return time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime(epoch))
+
+
+def _digest(salt: bytes, secret: str) -> str:
+ return hashlib.sha256(salt + secret.encode("utf-8")).hexdigest()
+
+
+def _state_digest(state: str) -> str:
+ return hashlib.sha256(state.encode("utf-8")).hexdigest()
+
+
+def _pkce_challenge(verifier: str) -> str:
+ return base64.urlsafe_b64encode(hashlib.sha256(verifier.encode("ascii")).digest()).rstrip(b"=").decode("ascii")
+
+
+def _book(st, *, fresh=True) -> dict:
+ try:
+ value = st.get(BUCKET, fresh=fresh)
+ except Exception as exc:
+ raise ExternalOAuthError("oauth_unavailable") from exc
+ if value is None:
+ return {"clients": {}, "codes": {}, "states": {}, "grants": {}}
+ if not isinstance(value, dict):
+ raise ExternalOAuthError("oauth_unavailable")
+ out = {}
+ for key in ("clients", "codes", "states", "grants"):
+ child = value.get(key, {})
+ if not isinstance(child, dict):
+ raise ExternalOAuthError("oauth_unavailable")
+ out[key] = child
+ return out
+
+
+def _normalise_redirect_uris(value) -> tuple[str, ...]:
+ if not isinstance(value, (list, tuple)) or not 1 <= len(value) <= MAX_REDIRECT_URIS:
+ raise ExternalOAuthError("oauth_redirect_uri_invalid")
+ clean = []
+ for raw in value:
+ if not isinstance(raw, str) or not raw.strip() or len(raw.strip()) > 1000:
+ raise ExternalOAuthError("oauth_redirect_uri_invalid")
+ uri = raw.strip()
+ parsed = urlsplit(uri)
+ loopback = parsed.scheme == "http" and (parsed.hostname or "").lower() in {
+ "localhost", "127.0.0.1", "::1",
+ }
+ if (parsed.fragment or parsed.username or parsed.password or not parsed.netloc
+ or (parsed.scheme != "https" and not loopback)):
+ raise ExternalOAuthError("oauth_redirect_uri_invalid")
+ clean.append(uri)
+ return tuple(dict.fromkeys(clean))
+
+
+def _normalise_state(value) -> str:
+ if not isinstance(value, str) or not 16 <= len(value) <= 512 or any(ord(c) < 32 for c in value):
+ raise ExternalOAuthError("oauth_state_invalid")
+ return value
+
+
+def _normalise_challenge(value, method) -> str:
+ if method != "S256" or not isinstance(value, str) or not 43 <= len(value) <= 128:
+ raise ExternalOAuthError("oauth_pkce_required")
+ allowed = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_"
+ if any(char not in allowed for char in value):
+ raise ExternalOAuthError("oauth_pkce_required")
+ return value
+
+
+def _normalise_verifier(value) -> str:
+ if not isinstance(value, str) or not 43 <= len(value) <= 128:
+ raise ExternalOAuthError("oauth_invalid_grant")
+ allowed = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-._~"
+ if any(char not in allowed for char in value):
+ raise ExternalOAuthError("oauth_invalid_grant")
+ return value
+
+
+def _normalise_scopes(value, access: str) -> tuple[str, ...]:
+ requested = str(value or "mcp:read offline_access").split()
+ requested = list(dict.fromkeys(requested))
+ if (not requested or any(scope not in SCOPES for scope in requested)
+ or "mcp:read" not in requested
+ or ("mcp:write" in requested and access != external_access.ACCESS_READWRITE)):
+ raise ExternalOAuthError("oauth_scope_invalid")
+ return tuple(requested)
+
+
+def normalise_resource(value) -> str:
+ """Canonicalise one RFC 8707 audience URI while rejecting ambiguous targets."""
+ if not isinstance(value, str) or not value.strip() or len(value.strip()) > 1000:
+ raise ExternalOAuthError("oauth_resource_invalid")
+ parsed = urlsplit(value.strip())
+ if (parsed.scheme.lower() not in {"http", "https"} or not parsed.netloc
+ or parsed.fragment or parsed.username or parsed.password):
+ raise ExternalOAuthError("oauth_resource_invalid")
+ try:
+ port = parsed.port
+ except ValueError as exc:
+ raise ExternalOAuthError("oauth_resource_invalid") from exc
+ hostname = (parsed.hostname or "").lower()
+ if not hostname:
+ raise ExternalOAuthError("oauth_resource_invalid")
+ host = f"[{hostname}]" if ":" in hostname else hostname
+ netloc = host + (f":{port}" if port is not None else "")
+ return urlunsplit((parsed.scheme.lower(), netloc, parsed.path, parsed.query, ""))
+
+
+def _comparison_material(record, prefix: str) -> tuple[bool, bytes, str]:
+ usable = isinstance(record, dict) and record.get("algo") == ALGORITHM
+ if usable:
+ try:
+ salt = bytes.fromhex(str(record.get(prefix + "_salt") or ""))
+ expected = str(record.get(prefix + "_hash") or "")
+ if len(salt) != 32 or len(expected) != 64:
+ raise ValueError("bad digest")
+ int(expected, 16)
+ return True, salt, expected
+ except (TypeError, ValueError):
+ pass
+ return False, _DUMMY_SALT, _DUMMY_HASH
+
+
+def _used_refresh_history(record, secret: str) -> tuple[bool, list[dict], bool]:
+ """Validate and compare every consumed token retained for this live family."""
+ rows = record.get("used_refresh", ()) if isinstance(record, dict) else ()
+ if not isinstance(rows, list) or len(rows) > MAX_REFRESH_ROTATIONS:
+ return False, [], False
+ valid = True
+ matched = False
+ for item in rows:
+ usable, salt, expected = _comparison_material({
+ "algo": item.get("algo") if isinstance(item, dict) else None,
+ "used_salt": item.get("salt") if isinstance(item, dict) else None,
+ "used_hash": item.get("hash") if isinstance(item, dict) else None,
+ }, "used")
+ valid = valid and usable
+ matched = hmac.compare_digest(_digest(salt, secret), expected) and usable or matched
+ return valid, list(rows), matched
+
+
+def _token(prefix: str, tenant: str, identifier: str, secret: str) -> str:
+ return f"{prefix}.{tenant}.{identifier}.{secret}"
+
+
+def _parse_token(token, prefixes) -> tuple[str, str, str, str]:
+ parts = str(token or "").strip().split(".")
+ if (len(parts) != 4 or parts[0] not in prefixes or not all(parts[1:])
+ or len(parts[1]) > 128 or len(parts[2]) > 128 or len(parts[3]) > 256):
+ raise ExternalOAuthAuthError(AUTH_ERROR)
+ return parts[0], parts[1].lower(), parts[2], parts[3]
+
+
+def _metadata(record: dict, client: dict | None = None) -> dict:
+ return {
+ "id": record.get("id"),
+ "client_id": record.get("client_id"),
+ "label": (client or {}).get("label"),
+ "created": _iso(record.get("created_at")),
+ "last_used": _iso(record.get("last_used_at")),
+ "expires_at": _iso(record.get("refresh_expires_at")),
+ "disconnected_at": _iso(record.get("disconnected_at")),
+ "access": record.get("access", external_access.ACCESS_READ),
+ "databases": (external_access.ALL_DATABASES if record.get("databases") == external_access.ALL_DATABASES
+ else list(record.get("databases") or [])),
+ "scopes": list(record.get("scopes") or ()),
+ }
+
+
+def _client_metadata(record: dict) -> dict:
+ access, databases = external_access.scope_of(record)
+ return {
+ "id": record.get("id"), "label": record.get("label"),
+ "redirect_uris": list(record.get("redirect_uris") or []),
+ "created": _iso(record.get("created_at")), "revoked_at": _iso(record.get("revoked_at")),
+ "access": access,
+ "databases": (external_access.ALL_DATABASES if databases == external_access.ALL_DATABASES
+ else list(databases)),
+ "token_endpoint_auth_method": "none", "pkce": "S256",
+ }
+
+
+def register_client(st, username: str, tenant: str, label, redirect_uris, *, access=None,
+ databases=None) -> dict:
+ owner, tenant_key = external_access._normalise_owner(username, tenant)
+ external_access._assert_bound_runtime(st, tenant_key)
+ external_access.resolve_active_user(owner, tenant_key)
+ clean_label = external_access._normalise_label(label)
+ redirects = _normalise_redirect_uris(redirect_uris)
+ clean_access = external_access.normalise_access(access)
+ clean_databases = external_access.normalise_databases(databases)
+ created = _now()
+ made = {}
+
+ def _set(value):
+ book = _book_value(value)
+ clients = dict(book["clients"])
+ client_id = "lpc_" + secrets.token_urlsafe(18)
+ while client_id in clients:
+ client_id = "lpc_" + secrets.token_urlsafe(18)
+ record = {
+ "id": client_id, "username": owner, "tenant": tenant_key, "label": clean_label,
+ "redirect_uris": list(redirects), "access": clean_access,
+ "databases": (external_access.ALL_DATABASES if clean_databases == external_access.ALL_DATABASES
+ else list(clean_databases)),
+ "created_at": created, "revoked_at": None,
+ }
+ clients[client_id] = record
+ book["clients"] = clients
+ made["record"] = record
+ return book
+
+ _update(st, _set)
+ return _client_metadata(made["record"])
+
+
+def list_clients(st, username: str, tenant: str) -> list[dict]:
+ owner, tenant_key = external_access._normalise_owner(username, tenant)
+ external_access._assert_bound_runtime(st, tenant_key)
+ return sorted([
+ _client_metadata(row) for row in _book(st)["clients"].values()
+ if isinstance(row, dict) and row.get("username") == owner
+ and row.get("tenant") == tenant_key and not row.get("revoked_at")
+ ], key=lambda row: str(row.get("created") or ""), reverse=True)
+
+
+def edit_client(st, username: str, tenant: str, client_id: str, *, label=_UNSET,
+ redirect_uris=_UNSET, access=_UNSET, databases=_UNSET) -> dict:
+ owner, tenant_key = external_access._normalise_owner(username, tenant)
+ external_access._assert_bound_runtime(st, tenant_key)
+ external_access.resolve_active_user(owner, tenant_key)
+ if label is redirect_uris is access is databases is _UNSET:
+ raise ExternalOAuthError("oauth_edit_empty")
+ clean_label = external_access._normalise_label(label) if label is not _UNSET else _UNSET
+ clean_redirects = _normalise_redirect_uris(redirect_uris) if redirect_uris is not _UNSET else _UNSET
+ clean_access = external_access.normalise_access(access) if access is not _UNSET else _UNSET
+ clean_databases = external_access.normalise_databases(databases) if databases is not _UNSET else _UNSET
+ edited = {}
+
+ def _set(value):
+ book = _book_value(value)
+ clients = dict(book["clients"])
+ current = clients.get(client_id)
+ if (not isinstance(current, dict) or current.get("username") != owner
+ or current.get("tenant") != tenant_key or current.get("revoked_at")):
+ raise ExternalOAuthError("oauth_client_not_found")
+ row = dict(current)
+ if clean_label is not _UNSET:
+ row["label"] = clean_label
+ if clean_redirects is not _UNSET:
+ row["redirect_uris"] = list(clean_redirects)
+ if clean_access is not _UNSET:
+ row["access"] = clean_access
+ if clean_databases is not _UNSET:
+ row["databases"] = (external_access.ALL_DATABASES
+ if clean_databases == external_access.ALL_DATABASES
+ else list(clean_databases))
+ clients[client_id] = row
+ book["clients"] = clients
+ edited["record"] = row
+ return book
+
+ _update(st, _set)
+ return _client_metadata(edited["record"])
+
+
+def authorize(st, username: str, tenant: str, *, client_id, redirect_uri, state,
+ code_challenge, code_challenge_method, resource, scope=None, response_type="code",
+ now=None) -> dict:
+ owner, tenant_key = external_access._normalise_owner(username, tenant)
+ external_access._assert_bound_runtime(st, tenant_key)
+ external_access.resolve_active_user(owner, tenant_key)
+ if response_type != "code":
+ raise ExternalOAuthError("oauth_response_type_invalid")
+ clean_state = _normalise_state(state)
+ challenge = _normalise_challenge(code_challenge, code_challenge_method)
+ audience = normalise_resource(resource)
+ issued_at = _now(now)
+ code_id, secret, salt = secrets.token_urlsafe(18), secrets.token_urlsafe(32), secrets.token_bytes(32)
+ issued = {}
+
+ def _set(value):
+ book = _book_value(value)
+ client = book["clients"].get(client_id)
+ if (not isinstance(client, dict) or client.get("username") != owner
+ or client.get("tenant") != tenant_key or client.get("revoked_at")):
+ raise ExternalOAuthError("oauth_client_invalid")
+ if redirect_uri not in (client.get("redirect_uris") or []):
+ raise ExternalOAuthError("oauth_redirect_uri_invalid")
+ access, databases = external_access.scope_of(client)
+ scopes = _normalise_scopes(scope, access)
+ states = {key: row for key, row in book["states"].items()
+ if isinstance(row, dict) and float(row.get("expires_at") or 0) >= issued_at}
+ state_id = hashlib.sha256(f"{tenant_key}\0{owner}\0{client_id}\0{clean_state}".encode()).hexdigest()
+ if state_id in states:
+ raise ExternalOAuthError("oauth_state_replayed")
+ states[state_id] = {"expires_at": issued_at + CODE_TTL_SECONDS}
+ codes = {key: row for key, row in book["codes"].items()
+ if isinstance(row, dict) and float(row.get("expires_at") or 0) >= issued_at - CODE_TTL_SECONDS}
+ row = {
+ "id": code_id, "tenant": tenant_key, "username": owner, "client_id": client_id,
+ "redirect_uri": redirect_uri, "state_hash": _state_digest(clean_state),
+ "code_challenge": challenge, "resource": audience,
+ "scopes": list(scopes), "access": access,
+ "databases": (external_access.ALL_DATABASES if databases == external_access.ALL_DATABASES
+ else list(databases)),
+ "code_salt": salt.hex(), "code_hash": _digest(salt, secret), "algo": ALGORITHM,
+ "created_at": issued_at, "expires_at": issued_at + CODE_TTL_SECONDS,
+ "consumed_at": None,
+ }
+ codes[code_id] = row
+ book.update({"states": states, "codes": codes})
+ issued["record"] = row
+ return book
+
+ _update(st, _set)
+ return {"redirect_uri": redirect_uri, "state": clean_state,
+ "code": _token("lpc1", tenant_key, code_id, secret),
+ "expires_in": CODE_TTL_SECONDS}
+
+
+def exchange_code(code, *, client_id, redirect_uri, code_verifier, resource,
+ runtime_resolver, now=None) -> dict:
+ _, tenant_key, code_id, secret = _parse_token(code, {"lpc1"})
+ verifier = _normalise_verifier(code_verifier)
+ audience = normalise_resource(resource)
+ try:
+ st = runtime_resolver(tenant_key)
+ external_access._assert_bound_runtime(st, tenant_key)
+ except (KeyError, external_access.ExternalAccessError) as exc:
+ raise ExternalOAuthError("oauth_invalid_grant") from exc
+ snapshot = _book(st)
+ candidate = snapshot["codes"].get(code_id)
+ if isinstance(candidate, dict):
+ try:
+ external_access.resolve_active_user(candidate.get("username"), tenant_key)
+ except external_access.ExternalAccessError as exc:
+ raise ExternalOAuthError("oauth_invalid_grant") from exc
+ issued_at = _now(now)
+ access_secret, refresh_secret = secrets.token_urlsafe(32), secrets.token_urlsafe(32)
+ access_salt, refresh_salt = secrets.token_bytes(32), secrets.token_bytes(32)
+ grant_id = secrets.token_urlsafe(18)
+ result = {}
+
+ def _set(value):
+ book = _book_value(value)
+ row = book["codes"].get(code_id)
+ usable, salt, expected = _comparison_material(row, "code")
+ matched = hmac.compare_digest(_digest(salt, secret), expected)
+ client = book["clients"].get(client_id)
+ if (not usable or not matched or not isinstance(row, dict) or row.get("consumed_at")
+ or float(row.get("expires_at") or 0) < issued_at
+ or row.get("tenant") != tenant_key or row.get("client_id") != client_id
+ or row.get("redirect_uri") != redirect_uri
+ or row.get("resource") != audience
+ or not isinstance(client, dict) or client.get("revoked_at")
+ or client.get("username") != row.get("username")
+ or not hmac.compare_digest(_pkce_challenge(verifier), str(row.get("code_challenge") or ""))):
+ raise ExternalOAuthError("oauth_invalid_grant")
+ grants = dict(book["grants"])
+ while grant_id in grants:
+ raise ExternalOAuthError("oauth_unavailable")
+ grant = {
+ "id": grant_id, "tenant": tenant_key, "username": row["username"],
+ "client_id": client_id, "resource": audience,
+ "access": row["access"], "databases": row["databases"],
+ "scopes": list(row["scopes"]), "algo": ALGORITHM,
+ "access_salt": access_salt.hex(), "access_hash": _digest(access_salt, access_secret),
+ "refresh_salt": refresh_salt.hex(), "refresh_hash": _digest(refresh_salt, refresh_secret),
+ "used_refresh": [], "refresh_generation": 0,
+ "created_at": issued_at, "access_expires_at": issued_at + ACCESS_TTL_SECONDS,
+ "refresh_expires_at": issued_at + REFRESH_TTL_SECONDS,
+ "last_used_at": None, "disconnected_at": None,
+ }
+ grants[grant_id] = grant
+ codes = dict(book["codes"])
+ consumed = dict(row)
+ consumed["consumed_at"] = issued_at
+ codes[code_id] = consumed
+ book.update({"codes": codes, "grants": grants})
+ result["grant"] = grant
+ return book
+
+ _update(st, _set)
+ grant = result["grant"]
+ return {
+ "token_type": "Bearer", "access_token": _token("lpo1", tenant_key, grant_id, access_secret),
+ "expires_in": ACCESS_TTL_SECONDS,
+ "refresh_token": _token("lpr1", tenant_key, grant_id, refresh_secret),
+ "scope": " ".join(grant["scopes"]),
+ }
+
+
+def refresh(refresh_token, *, client_id, resource, runtime_resolver, now=None) -> dict:
+ _, tenant_key, grant_id, secret = _parse_token(refresh_token, {"lpr1"})
+ try:
+ st = runtime_resolver(tenant_key)
+ external_access._assert_bound_runtime(st, tenant_key)
+ except (KeyError, external_access.ExternalAccessError) as exc:
+ raise ExternalOAuthError("oauth_invalid_grant") from exc
+ snapshot = _book(st)
+ candidate = snapshot["grants"].get(grant_id)
+ if isinstance(candidate, dict):
+ try:
+ external_access.resolve_active_user(candidate.get("username"), tenant_key)
+ except external_access.ExternalAccessError as exc:
+ raise ExternalOAuthError("oauth_invalid_grant") from exc
+ issued_at = _now(now)
+ audience = normalise_resource(resource)
+ next_access, next_refresh = secrets.token_urlsafe(32), secrets.token_urlsafe(32)
+ next_access_salt, next_refresh_salt = secrets.token_bytes(32), secrets.token_bytes(32)
+ result = {}
+
+ def _set(value):
+ book = _book_value(value)
+ grants = dict(book["grants"])
+ row = grants.get(grant_id)
+ current_ok, current_salt, current_hash = _comparison_material(row, "refresh")
+ current_match = hmac.compare_digest(_digest(current_salt, secret), current_hash)
+ history_ok, used, used_match = _used_refresh_history(row, secret)
+ if (isinstance(row, dict) and not history_ok and not row.get("disconnected_at")):
+ corrupted = dict(row)
+ corrupted["disconnected_at"] = issued_at
+ grants[grant_id] = corrupted
+ book["grants"] = grants
+ result["error"] = "oauth_invalid_grant"
+ return book
+ if (isinstance(row, dict) and used_match and not row.get("disconnected_at")):
+ replayed = dict(row)
+ replayed["disconnected_at"] = issued_at
+ grants[grant_id] = replayed
+ book["grants"] = grants
+ result["error"] = "oauth_refresh_replayed"
+ return book
+ client = book["clients"].get(client_id)
+ if (not current_ok or not current_match or not isinstance(row, dict)
+ or row.get("disconnected_at") or row.get("tenant") != tenant_key
+ or row.get("client_id") != client_id
+ or row.get("resource") != audience
+ or float(row.get("refresh_expires_at") or 0) < issued_at
+ or not isinstance(client, dict) or client.get("revoked_at")
+ or client.get("username") != row.get("username")):
+ raise ExternalOAuthError("oauth_invalid_grant")
+ generation = int(row.get("refresh_generation") or 0)
+ if generation >= MAX_REFRESH_ROTATIONS or len(used) >= MAX_REFRESH_ROTATIONS:
+ exhausted = dict(row)
+ exhausted["disconnected_at"] = issued_at
+ grants[grant_id] = exhausted
+ book["grants"] = grants
+ result["error"] = "oauth_refresh_exhausted"
+ return book
+ used.append({"salt": row.get("refresh_salt"), "hash": row.get("refresh_hash"),
+ "algo": ALGORITHM})
+ rotated = dict(row)
+ rotated.update({
+ "used_refresh": used,
+ "refresh_generation": generation + 1,
+ "access_salt": next_access_salt.hex(),
+ "access_hash": _digest(next_access_salt, next_access),
+ "refresh_salt": next_refresh_salt.hex(),
+ "refresh_hash": _digest(next_refresh_salt, next_refresh),
+ "access_expires_at": issued_at + ACCESS_TTL_SECONDS,
+ "last_used_at": None,
+ })
+ grants[grant_id] = rotated
+ book["grants"] = grants
+ result["grant"] = rotated
+ return book
+
+ _update(st, _set)
+ if result.get("error"):
+ raise ExternalOAuthError(result["error"])
+ grant = result["grant"]
+ return {
+ "token_type": "Bearer", "access_token": _token("lpo1", tenant_key, grant_id, next_access),
+ "expires_in": ACCESS_TTL_SECONDS,
+ "refresh_token": _token("lpr1", tenant_key, grant_id, next_refresh),
+ "scope": " ".join(grant["scopes"]),
+ }
+
+
+def _intersect_databases(left, right):
+ if left == external_access.ALL_DATABASES:
+ return right
+ if right == external_access.ALL_DATABASES:
+ return left
+ allowed = set(right or ())
+ return tuple(item for item in (left or ()) if item in allowed)
+
+
+def verify_access(token: str, *, resource, runtime_resolver) -> VerifiedOAuth:
+ try:
+ _, tenant_key, grant_id, secret = _parse_token(token, {"lpo1"})
+ try:
+ st = runtime_resolver(tenant_key)
+ external_access._assert_bound_runtime(st, tenant_key)
+ except KeyError as exc:
+ raise ExternalOAuthAuthError(AUTH_ERROR) from exc
+ book = _book(st)
+ audience = normalise_resource(resource)
+ row = book["grants"].get(grant_id)
+ usable, salt, expected = _comparison_material(row, "access")
+ matched = hmac.compare_digest(_digest(salt, secret), expected)
+ client = book["clients"].get(row.get("client_id")) if isinstance(row, dict) else None
+ now = _now()
+ if (not usable or not matched or not isinstance(row, dict) or row.get("disconnected_at")
+ or row.get("tenant") != tenant_key or float(row.get("access_expires_at") or 0) < now
+ or row.get("resource") != audience
+ or not isinstance(client, dict) or client.get("revoked_at")
+ or client.get("username") != row.get("username")
+ or client.get("tenant") != tenant_key):
+ raise ExternalOAuthAuthError(AUTH_ERROR)
+ try:
+ public = external_access.resolve_active_user(row["username"], tenant_key)
+ except external_access.ExternalAccessError as exc:
+ raise ExternalOAuthAuthError(AUTH_ERROR) from exc
+ grant_access, grant_databases = external_access.scope_of(row)
+ client_access, client_databases = external_access.scope_of(client)
+ access = (external_access.ACCESS_READWRITE
+ if grant_access == client_access == external_access.ACCESS_READWRITE
+ and "mcp:write" in (row.get("scopes") or ())
+ else external_access.ACCESS_READ)
+ databases = _intersect_databases(grant_databases, client_databases)
+ if now - float(row.get("last_used_at") or 0) >= _STAMP_EVERY:
+ def _stamp(value):
+ latest = _book_value(value)
+ grants = dict(latest["grants"])
+ current = grants.get(grant_id)
+ if (not isinstance(current, dict) or current.get("disconnected_at")
+ or current.get("access_hash") != row.get("access_hash")):
+ raise ExternalOAuthAuthError(AUTH_ERROR)
+ current = dict(current)
+ current["last_used_at"] = now
+ grants[grant_id] = current
+ latest["grants"] = grants
+ return latest
+ _update(st, _stamp, auth=True)
+ return VerifiedOAuth(tenant_key, grant_id, row["client_id"], public, st, access,
+ databases, tuple(row.get("scopes") or ()), audience)
+ except ExternalOAuthAuthError:
+ raise
+ except Exception as exc:
+ raise ExternalOAuthAuthError(AUTH_ERROR) from exc
+
+
+def disconnect(token: str, *, runtime_resolver, now=None) -> DisconnectedOAuth:
+ prefix, tenant_key, grant_id, secret = _parse_token(token, {"lpo1", "lpr1"})
+ try:
+ st = runtime_resolver(tenant_key)
+ external_access._assert_bound_runtime(st, tenant_key)
+ except (KeyError, external_access.ExternalAccessError) as exc:
+ raise ExternalOAuthAuthError(AUTH_ERROR) from exc
+ stopped = {}
+ at = _now(now)
+ material = "access" if prefix == "lpo1" else "refresh"
+
+ def _set(value):
+ book = _book_value(value)
+ grants = dict(book["grants"])
+ row = grants.get(grant_id)
+ usable, salt, expected = _comparison_material(row, material)
+ if (not usable or not hmac.compare_digest(_digest(salt, secret), expected)
+ or not isinstance(row, dict) or row.get("tenant") != tenant_key
+ or not isinstance(row.get("username"), str) or not row.get("username")):
+ raise ExternalOAuthAuthError(AUTH_ERROR)
+ disconnected = dict(row)
+ if not disconnected.get("disconnected_at"):
+ disconnected["disconnected_at"] = at
+ grants[grant_id] = disconnected
+ book["grants"] = grants
+ stopped["record"] = disconnected
+ stopped["client"] = book["clients"].get(disconnected.get("client_id"))
+ return book
+
+ _update(st, _set, auth=True)
+ record = stopped["record"]
+ return DisconnectedOAuth(tenant_key, record["username"],
+ _metadata(record, stopped.get("client")))
+
+
+def disconnect_for_user(st, username: str, tenant: str, grant_id: str) -> dict:
+ owner, tenant_key = external_access._normalise_owner(username, tenant)
+ external_access._assert_bound_runtime(st, tenant_key)
+ stopped = {}
+ at = _now()
+
+ def _set(value):
+ book = _book_value(value)
+ grants = dict(book["grants"])
+ row = grants.get(grant_id)
+ if (not isinstance(row, dict) or row.get("username") != owner
+ or row.get("tenant") != tenant_key):
+ raise ExternalOAuthError("oauth_connection_not_found")
+ disconnected = dict(row)
+ if not disconnected.get("disconnected_at"):
+ disconnected["disconnected_at"] = at
+ grants[grant_id] = disconnected
+ book["grants"] = grants
+ stopped["record"] = disconnected
+ stopped["client"] = book["clients"].get(disconnected.get("client_id"))
+ return book
+
+ _update(st, _set)
+ return _metadata(stopped["record"], stopped.get("client"))
+
+
+def list_connections(st, username: str, tenant: str) -> list[dict]:
+ owner, tenant_key = external_access._normalise_owner(username, tenant)
+ external_access._assert_bound_runtime(st, tenant_key)
+ book = _book(st)
+ return sorted([
+ _metadata(row, book["clients"].get(row.get("client_id")))
+ for row in book["grants"].values()
+ if isinstance(row, dict) and row.get("username") == owner
+ and row.get("tenant") == tenant_key and not row.get("disconnected_at")
+ ], key=lambda row: str(row.get("created") or ""), reverse=True)
+
+
+def label_for(st, grant_id: str) -> str:
+ book = _book(st)
+ grant = book["grants"].get(grant_id)
+ client = book["clients"].get(grant.get("client_id")) if isinstance(grant, dict) else None
+ if (not isinstance(grant, dict) or grant.get("disconnected_at")
+ or not isinstance(client, dict) or client.get("revoked_at")):
+ raise ExternalOAuthAuthError(AUTH_ERROR)
+ label = client.get("label")
+ if not isinstance(label, str) or not label:
+ raise ExternalOAuthError("oauth_unavailable")
+ return label
+
+
+def _book_value(value) -> dict:
+ if value is None:
+ return {"clients": {}, "codes": {}, "states": {}, "grants": {}}
+ if not isinstance(value, dict):
+ raise ExternalOAuthError("oauth_unavailable")
+ book = dict(value)
+ for key in ("clients", "codes", "states", "grants"):
+ child = book.get(key, {})
+ if not isinstance(child, dict):
+ raise ExternalOAuthError("oauth_unavailable")
+ book[key] = dict(child)
+ return book
+
+
+def _update(st, callback, *, auth=False) -> None:
+ try:
+ st.update(BUCKET, callback, flush="sync")
+ except (ExternalOAuthError, ExternalOAuthAuthError):
+ raise
+ except Exception as exc:
+ if auth:
+ raise ExternalOAuthAuthError(AUTH_ERROR) from exc
+ raise ExternalOAuthError("oauth_unavailable") from exc
diff --git a/platform/core/external_tools.py b/platform/core/external_tools.py
index 57cfe7c52be5565876268cc3c0f09106753b813b..5553b62f08033aec5e2da696532af4cc0c521449 100644
--- a/platform/core/external_tools.py
+++ b/platform/core/external_tools.py
@@ -6,11 +6,10 @@ Adding a tool means adding a registry entry; writes also require the router's li
"""
import copy
import json
-import uuid
from dataclasses import dataclass
-from typing import Callable
+from typing import Callable, Optional
-from core import sheets, sheet_structure, sheet_templates
+from core import sheets, sheet_structure, sheet_templates, write_execution
@dataclass(frozen=True)
@@ -24,9 +23,15 @@ class ToolContext:
cache_binding: Callable
list_databases: Callable
describe_database: Callable
+ query_database: Callable
view_event: Callable
+ view_share_preview: Callable
app_upsert: Callable
app_delete: Callable
+ database_object: Optional[Callable] = None
+ sheet_object: Optional[Callable] = None
+ validated_batch: Optional[Callable] = None
+ confirm_change: Optional[Callable] = None
def _object(properties=None, required=(), **extra):
@@ -156,7 +161,7 @@ def _add_tab(ctx, args):
raise sheets.SheetRefusal(
"tab_name_taken",
"Tab " + wanted + " is already in this workbook. Choose another name.")
- tab = {"id": uuid.uuid4().hex[:12], "name": args["name"], "cells": {}}
+ tab = {"id": write_execution.new_id()[:12], "name": args["name"], "cells": {}}
book["sheets"].append(tab)
return _put(ctx, args, book, tab["id"])
@@ -252,7 +257,7 @@ def _chart(ctx, args):
book = _book(ctx, args, write=True)
tab = sheets.find_tab(book, args["tab"])
chart = copy.deepcopy(args["chart"])
- chart.setdefault("id", uuid.uuid4().hex[:12])
+ chart.setdefault("id", write_execution.new_id()[:12])
tab["charts"] = [row for row in tab.get("charts", []) if row["id"] != chart["id"]] + [chart]
return {**_put(ctx, args, book, tab["id"]), "chart_id": chart["id"]}
@@ -269,6 +274,55 @@ def _tool(description, schema, handler, *, write=False, grants=("analyst", "shee
"grants": grants, "handler": handler}
+def _callback(ctx, name, args):
+ callback = getattr(ctx, name, None)
+ if not callable(callback):
+ raise sheets.SheetRefusal(
+ "tool_capability_unsupported",
+ "This deployment does not publish that object operation in its capability manifest.")
+ return callback(args)
+
+
+_DATABASE_ACTIONS = (
+ "list", "describe", "query", "record_create", "record_update", "record_delete",
+ "field_create", "field_update", "field_delete",
+)
+_VIEW_ACTIONS = ("upsert", "delete", "share_preview")
+_SHEET_OBJECT_ACTIONS = {
+ "workbook_object": ("list", "get", "create", "named_ranges", "export"),
+ "worksheet_object": (
+ "list", "get", "add", "update", "delete", "geometry", "hidden_bands",
+ "freeze", "gridlines", "named_ranges",
+ ),
+ "range_object": (
+ "read", "write_values", "write_formulas", "write_styles", "number_format",
+ "fill_down", "clear",
+ ),
+ "calculation_object": ("calculate", "refresh"),
+}
+
+
+def _object_tool(name, actions):
+ target = {
+ "workbook_id": _STRING,
+ "worksheet": _STRING,
+ "range": _RANGE,
+ "revision": {"type": "integer"},
+ "arguments": {"type": "object"},
+ }
+ return _tool(
+ "Use the published Excel-shaped capability manifest for one "
+ + name.removesuffix("_object") + " operation.",
+ _object({"action": {"enum": list(actions)}, **target}, ("action",),
+ additionalProperties=False),
+ lambda ctx, args, object_name=name: _callback(
+ ctx, "sheet_object", {"object": object_name.removesuffix("_object"), **args}),
+ # The action classifier below distinguishes reads. Declaring the surface write-capable
+ # keeps the API's literal allow-list fail closed when a new action is added.
+ write=True,
+ )
+
+
TOOLS = {
"sheet_create_workbook": _tool("Create a private workbook with one initial tab.",
_object({"name": _STRING}, ("name",), additionalProperties=False), _create, write=True),
@@ -317,14 +371,44 @@ TOOLS = {
"describe_database": _tool("Read a database's field schema through the strict Assistant scope wall.",
_object({"database": _STRING}, ("database",), additionalProperties=False),
lambda ctx, args: ctx.describe_database(args["database"]), grants=("analyst",)),
+ "query_database": _tool(
+ "Query currently permitted records with bounded filters, stable sorts, cursor pagination and aggregates.",
+ _object({
+ "database": _STRING,
+ "fields": {"type": "array", "items": _STRING, "maxItems": 200},
+ "filters": {"oneOf": [_FILTER, {"type": "array", "items": {"type": "object"}}]},
+ "sorts": {"type": "array", "maxItems": 8, "items": _object({
+ "field": _STRING, "direction": {"enum": ["asc", "desc"]},
+ }, ("field", "direction"), additionalProperties=False)},
+ "limit": {"type": "integer", "minimum": 1, "maximum": 500},
+ "cursor": _STRING,
+ "aggregates": {"type": "array", "maxItems": 16, "items": _object({
+ "fn": {"enum": ["count", "count_distinct", "sum", "avg", "min", "max"]},
+ "field": _STRING,
+ }, ("fn",), additionalProperties=False)},
+ }, ("database",), additionalProperties=False),
+ lambda ctx, args: ctx.query_database(args["database"], {
+ key: value for key, value in args.items() if key != "database"
+ }), grants=("analyst",)),
"view_upsert": _tool(
"Create or update a saved View using the grid's filters, sorts, groupBy, visible fields and display config.",
_object({"database": _STRING, "view": _object({
"id": {"type": "string", "minLength": 1, "maxLength": 120},
"name": {"type": "string", "minLength": 1, "maxLength": 120},
- "config": _VIEW_CONFIG, "note": _STRING}, ("id", "name", "config"))},
+ "config": _VIEW_CONFIG, "note": _STRING}, ("id", "name", "config"),
+ additionalProperties=False)},
("database", "view"), additionalProperties=False),
lambda ctx, args: ctx.view_event("view_upsert", args), write=True, grants=("analyst",)),
+ "view_share_preview": _tool(
+ "Preview a personal draft View's C8 audience and field inheritance without sharing or mutating it.",
+ _object({
+ "database": _STRING,
+ "view_id": _STRING,
+ "entries": {"type": "array", "maxItems": 200, "items": _object({
+ "user": _STRING, "role": {"enum": ["view", "edit"]},
+ }, ("user", "role"), additionalProperties=False)},
+ }, ("database", "view_id", "entries"), additionalProperties=False),
+ lambda ctx, args: ctx.view_share_preview(args), grants=("analyst",)),
"view_delete": _tool("Delete a saved View through its ordinary owner and database permission walls.",
_object({"database": _STRING, "view_id": _STRING}, ("database", "view_id"),
additionalProperties=False),
@@ -339,9 +423,83 @@ TOOLS = {
_object({"app_id": _STRING, "confirmName": _STRING}, ("app_id", "confirmName"),
additionalProperties=False),
lambda ctx, args: ctx.app_delete(args), write=True, grants=("analyst",)),
+ "database_object": _tool(
+ "Discover/query a database or preview-confirm record and field CRUD through its live UI walls.",
+ _object({
+ "action": {"enum": list(_DATABASE_ACTIONS)},
+ "database": _STRING,
+ "record_id": {"oneOf": [_STRING, {"type": "integer"}]},
+ "values": {"type": "object"},
+ "field": {"type": "object"},
+ "field_key": _STRING,
+ "query": {"type": "object"},
+ }, ("action",), additionalProperties=False),
+ lambda ctx, args: _callback(ctx, "database_object", args), write=True,
+ grants=("analyst",)),
+ "view_object": _tool(
+ "Read or preview-confirm a personal View operation through the ordinary grid writer.",
+ _object({
+ "action": {"enum": list(_VIEW_ACTIONS)}, "database": _STRING,
+ "view": {"type": "object"}, "view_id": _STRING,
+ "entries": {"type": "array", "items": {"type": "object"}, "maxItems": 200},
+ }, ("action", "database"), additionalProperties=False),
+ lambda ctx, args: (
+ ctx.view_event("view_upsert", {"database": args["database"], "view": args["view"]})
+ if args["action"] == "upsert" else
+ ctx.view_event("view_delete", {"database": args["database"],
+ "view_id": args["view_id"]})
+ if args["action"] == "delete" else
+ ctx.view_share_preview({"database": args["database"],
+ "view_id": args["view_id"],
+ "entries": args.get("entries") or []})
+ ), write=True, grants=("analyst",)),
+ **{name: _object_tool(name, actions)
+ for name, actions in _SHEET_OBJECT_ACTIONS.items()},
+ "validated_batch": _tool(
+ "Validate a bounded object-operation batch; writes preview and confirm as one unit.",
+ _object({"operations": {"type": "array", "minItems": 1, "maxItems": 50,
+ "items": _object({"name": _STRING,
+ "arguments": {"type": "object"}},
+ ("name", "arguments"),
+ additionalProperties=False)}},
+ ("operations",), additionalProperties=False),
+ lambda ctx, args: _callback(ctx, "validated_batch", args), write=True,
+ grants=("analyst",)),
+ "confirm_change": _tool(
+ "Confirm one unexpired preview. Destructive/schema changes require its exact typed phrase.",
+ _object({"confirmation_token": _STRING, "confirmation_phrase": _STRING},
+ ("confirmation_token",), additionalProperties=False),
+ lambda ctx, args: _callback(ctx, "confirm_change", args), grants=("analyst",)),
}
+_OBJECT_READ_ACTIONS = {
+ "database_object": frozenset({"list", "describe", "query"}),
+ "view_object": frozenset({"share_preview"}),
+ "workbook_object": frozenset({"list", "get", "export"}),
+ "worksheet_object": frozenset({"list", "get"}),
+ "range_object": frozenset({"read"}),
+ "calculation_object": frozenset(),
+}
+
+
+def is_write(name, arguments) -> bool:
+ """Classify dynamic object tools without trusting an unvalidated action as a read."""
+ tool = TOOLS.get(name) if isinstance(name, str) else None
+ if tool is None or name == "confirm_change":
+ return False
+ if name == "validated_batch":
+ operations = arguments.get("operations") if isinstance(arguments, dict) else None
+ if not isinstance(operations, list):
+ return True
+ return any(is_write(item.get("name"), item.get("arguments"))
+ for item in operations if isinstance(item, dict))
+ if name in _OBJECT_READ_ACTIONS:
+ action = arguments.get("action") if isinstance(arguments, dict) else None
+ return action not in _OBJECT_READ_ACTIONS[name]
+ return bool(tool.get("write"))
+
+
def dispatch(ctx, name, arguments):
"""Dispatch one declared tool. The API translates core and grant refusals to MCP content."""
tool = TOOLS.get(name) if isinstance(name, str) else None
@@ -358,15 +516,20 @@ def dispatch(ctx, name, arguments):
raise sheets.SheetRefusal("tool_arguments_invalid", "Tool arguments must match the published input schema.")
# Validate only the transport shape here. Document semantics belong to the app writers.
for key, value in arguments.items():
- kind = schema["properties"].get(key, {}).get("type")
+ property_schema = schema["properties"].get(key, {})
+ kind = property_schema.get("type")
types = {"object": dict, "array": list, "string": str, "integer": int}
if kind in types and (not isinstance(value, types[kind]) or (kind == "integer" and isinstance(value, bool))):
raise sheets.SheetRefusal("tool_arguments_invalid", f"Tool argument {key} must be an {kind}.")
+ if "enum" in property_schema and value not in property_schema["enum"]:
+ raise sheets.SheetRefusal(
+ "tool_arguments_invalid", f"Tool argument {key} is not a published operation.")
# A dimension patch must never become an arbitrary tab-document replacement.
if name == "sheet_set_dimensions":
allowed = schema["properties"]["dimensions"]["properties"]
if arguments["dimensions"].keys() - allowed.keys():
raise sheets.SheetRefusal("tool_arguments_invalid", "Dimensions may contain only the published geometry fields.")
- if "lpb1." in json.dumps(arguments, ensure_ascii=False, allow_nan=False):
+ encoded_arguments = json.dumps(arguments, ensure_ascii=False, allow_nan=False)
+ if any(prefix in encoded_arguments for prefix in ("lpb1.", "lpo1.", "lpr1.", "lpc1.")):
raise sheets.SheetRefusal("credential_in_arguments", "Tool arguments must not contain credential values.")
return tool["handler"](ctx, arguments)
diff --git a/platform/core/field_permissions.py b/platform/core/field_permissions.py
index 27747b077a553e18eb4231a3d2f7f63138ea6397..7545358dea7a40a1bb5a184370fc483f48487d74 100644
--- a/platform/core/field_permissions.py
+++ b/platform/core/field_permissions.py
@@ -6,6 +6,7 @@ that could never be true. This module is the single normalization point for the
personal, collaborative, or specific users.
"""
+import copy
import time
import core.shared_overlay as shared_overlay
@@ -26,6 +27,18 @@ FIELD_ORIGINS = ("preset", "user")
FIELD_AUDIENCES = ("private", "users", "everyone")
FIELD_VALUE_STRATA = ("shared", "personal")
+#: SDLC63 C8 — the View-to-field edges live beside the View grants, in the SAME tenant-scoped
+#: document, but outside ``shares.KINDS``. A boundary names fields once; it never copies the
+#: View's current people into N field grant records. Consequently a later audience edit takes
+#: effect on the next permission read and removing one View cannot erase a direct or second-View
+#: path. ``core.shares`` deliberately remains the direct-grant registry; callers that need to
+#: explain provenance can therefore distinguish its record from this derived edge.
+VIEW_FIELD_BOUNDARIES_KEY = "view_field_boundaries"
+VIEW_FIELD_AUDIT_CAP = 200
+VIEW_SHARE_DIAGNOSTICS_KEY = "view_share_diagnostics"
+VIEW_SHARE_DIAGNOSTIC_CAP = 30
+VIEW_SHARE_DIAGNOSTIC_FIELDS_CAP = 500
+
#: ⭐⭐ W42-T47 / RULING R15 / CONTRACT C10 — WHICH STRATUM A DEFINITION WAS READ FROM.
#:
#: C10: *"`field_classes` reports, beside `{origin, audience, sharedBy, values, owner}`, WHICH
@@ -102,6 +115,377 @@ def grant_entries(permissions):
return []
+# ---------------------------------------------------------- SDLC63 / C8: View audience inheritance
+
+def view_field_keys(view):
+ """Fields deliberately used by one View, in stable encounter order.
+
+ ``config.order`` is intentionally absent. Both the browser reconciler and the server save
+ door append every database field to that list, including columns the author never selected;
+ treating it as an inclusion list is the old over-share. Visible columns and fields used by a
+ View rule/display are deliberate. Hiding a plain column therefore removes that inherited
+ path, while a hidden filter/group/sort input stays readable because the View still depends on
+ it.
+ """
+ if not isinstance(view, dict):
+ return []
+ cfg = view.get("config") if isinstance(view.get("config"), dict) else view
+ keys, seen = [], set()
+
+ def add(raw):
+ key = str(raw or "").strip()
+ if key and key not in seen:
+ seen.add(key)
+ keys.append(key)
+
+ def walk_filter(node):
+ if isinstance(node, list):
+ for child in node:
+ walk_filter(child)
+ return
+ if not isinstance(node, dict):
+ return
+ walk_filter(node.get("children"))
+ add(node.get("colId"))
+ rhs = node.get("rhs")
+ if isinstance(rhs, dict) and rhs.get("kind") != "stat":
+ add(rhs.get("colId"))
+
+ for key in cfg.get("visible") or ():
+ add(key)
+ walk_filter(cfg.get("filters"))
+ add(cfg.get("groupBy"))
+ colour = cfg.get("colorBy")
+ add(colour.get("colId") if isinstance(colour, dict) else colour)
+ sorts = cfg.get("sorts")
+ if isinstance(sorts, dict):
+ sorts = [sorts]
+ for spec in sorts or ():
+ if isinstance(spec, dict):
+ add(spec.get("colId"))
+ for key in (cfg.get("summary") or {}) if isinstance(cfg.get("summary"), dict) else ():
+ add(key)
+ display = cfg.get("display")
+ if isinstance(display, dict):
+ for name, value in display.items():
+ if str(name).endswith("Field"):
+ add(value)
+ for chart in display.get("charts") or ():
+ if isinstance(chart, dict):
+ add(chart.get("x"))
+ add(chart.get("y"))
+ for metric in display.get("calendarMetrics") or ():
+ if isinstance(metric, dict):
+ add(metric.get("field"))
+ for key in display.get("fields") or ():
+ add(key)
+ return keys
+
+
+def _st_for_boundaries(st=None):
+ """Use the same tenant handle ``core.shares`` uses without reaching into a second store."""
+ return st if st is not None else shares._st(None)
+
+
+def sync_view_field_boundary(view, grant_topic, actor, st=None, *, audit_additions=True,
+ force=False):
+ """Persist one View's current field edge and return its delta.
+
+ The audience snapshot used by an addition audit is read inside the same ``object_shares``
+ update as the boundary change. It is evidence only; permission resolution always reads the
+ live View grant, so the snapshot can never become an access list.
+ """
+ view_id = str((view or {}).get("id") or "").strip()
+ topic = str(grant_topic or "").strip()
+ who = _uname(actor)
+ if not view_id or not topic:
+ return {"added": [], "removed": [], "audience": [], "owner": None}
+ wanted = view_field_keys(view)
+ result = {"added": [], "removed": [], "audience": [], "owner": None}
+ handle = _st_for_boundaries(st)
+
+ # A private View needs no second write on every filter/resize autosave. Its first share calls
+ # this helper with ``force=True`` after the View grant lands; from then on the existing edge
+ # makes field additions/removals write-bearing. This keeps C8 off the ordinary private-view
+ # hot path without opening a share-before-boundary race.
+ try:
+ current = handle.get(shares.SHARES_KEY) or {}
+ except Exception: # noqa: BLE001
+ current = {}
+ current_boundaries = current.get(VIEW_FIELD_BOUNDARIES_KEY) or {}
+ current_grant = (current.get("view") or {}).get(view_id)
+ current_grant = current_grant if isinstance(current_grant, dict) else {}
+ current_audience = shares._clean_entries(current_grant.get("entries"))
+ current_owner = _uname(current_grant.get("owner")) or None
+ prior = (current_boundaries.get(view_id)
+ if isinstance(current_boundaries, dict) else None)
+ if not force and not isinstance(prior, dict) and not current_audience:
+ return {"added": [], "removed": [], "audience": [], "owner": current_owner}
+ if isinstance(prior, dict):
+ before = [str(key) for key in (prior.get("fields") or ()) if str(key)]
+ if prior.get("topic") == topic and before == wanted:
+ return {"added": [], "removed": [],
+ "audience": current_audience, "owner": current_owner}
+
+ def _apply(data):
+ data = data if isinstance(data, dict) else {}
+ boundaries = dict(data.get(VIEW_FIELD_BOUNDARIES_KEY) or {})
+ prior = boundaries.get(view_id) if isinstance(boundaries.get(view_id), dict) else {}
+ before = [str(key) for key in (prior.get("fields") or ()) if str(key)]
+ before_set, wanted_set = set(before), set(wanted)
+ added = [key for key in wanted if key not in before_set]
+ removed = [key for key in before if key not in wanted_set]
+ grant = (data.get("view") or {}).get(view_id)
+ grant = grant if isinstance(grant, dict) else {}
+ audience = shares._clean_entries(grant.get("entries"))
+ owner = _uname(grant.get("owner")) or None
+ audit = [row for row in (prior.get("audit") or ()) if isinstance(row, dict)]
+ if audit_additions and audience:
+ stamp = time.time()
+ snapshot = [{"user": row["user"], "role": row["role"]} for row in audience]
+ for key in added:
+ audit.append({"ts": stamp, "actor": who or None, "field": key,
+ "view": view_id, "audience": snapshot})
+ boundaries[view_id] = {
+ "topic": topic,
+ "fields": wanted,
+ "updatedBy": who or None,
+ "audit": audit[-VIEW_FIELD_AUDIT_CAP:],
+ }
+ data[VIEW_FIELD_BOUNDARIES_KEY] = boundaries
+ result.update({"added": added, "removed": removed,
+ "audience": audience, "owner": owner})
+ return data
+
+ handle.update(shares.SHARES_KEY, _apply, flush="async")
+ return result
+
+
+def set_view_grants_with_boundary(view, grant_topic, entries, *, owner, granter, expect,
+ st=None, _fail_after_prepare=False):
+ """Replace one View audience and its field edge in one store transaction.
+
+ The mutation is built on a detached copy. If comparison, normalization, a test-injected
+ mid-operation fault, or persistence fails, neither the audience nor its derived boundary is
+ visible. The callback writes one bounded View record plus one bounded list of field keys;
+ its store-call count is therefore constant for a View with one field or five hundred.
+
+ ``_fail_after_prepare`` is an internal verifier seam. It raises after both changes have
+ been prepared but before the detached document is returned, proving rollback at the only
+ point a half-applied mutation could otherwise escape.
+ """
+ view_id = str((view or {}).get("id") or "").strip()
+ topic = str(grant_topic or "").strip()
+ if not view_id or not topic:
+ raise ValueError("a View share must name its View and database")
+ clean = shares._clean_entries(entries)
+ wanted = view_field_keys(view)
+ owner_l = _uname(owner) or None
+ granter_l = _uname(granter) or None
+ expected = ([(row["user"], row["role"])
+ for row in shares._clean_entries(expect)]
+ if expect is not None else None)
+ answer = {}
+
+ def _apply(raw):
+ data = copy.deepcopy(raw) if isinstance(raw, dict) else {}
+ views = dict(data.get("view") or {})
+ prior = views.get(view_id) if isinstance(views.get(view_id), dict) else {}
+ prior_entries = shares._clean_entries(prior.get("entries"))
+ if expected is not None:
+ current = [(row["user"], row["role"]) for row in prior_entries]
+ if current != expected:
+ raise shares.GrantsChanged(
+ "this item was shared with somebody else while you were editing, so nothing "
+ "was saved. Reopen the sharing panel to see who has access now, then make "
+ "your change again.")
+
+ prior_by = {row["user"]: row.get("by") for row in prior_entries}
+ stamped = []
+ for row in clean:
+ saved = {"user": row["user"], "role": row["role"]}
+ by = prior_by[row["user"]] if row["user"] in prior_by else granter_l
+ if by:
+ saved["by"] = by
+ stamped.append(saved)
+ keep_owner = owner_l or _uname(prior.get("owner")) or None
+ if not stamped and not keep_owner:
+ views.pop(view_id, None)
+ else:
+ views[view_id] = {"owner": keep_owner, "entries": stamped}
+
+ boundaries = dict(data.get(VIEW_FIELD_BOUNDARIES_KEY) or {})
+ old_boundary = (boundaries.get(view_id)
+ if isinstance(boundaries.get(view_id), dict) else {})
+ boundaries[view_id] = {
+ "topic": topic,
+ "fields": list(wanted),
+ "updatedBy": granter_l,
+ "audit": [dict(row) for row in (old_boundary.get("audit") or ())
+ if isinstance(row, dict)][-VIEW_FIELD_AUDIT_CAP:],
+ }
+ data["view"] = views
+ data[VIEW_FIELD_BOUNDARIES_KEY] = boundaries
+ if _fail_after_prepare:
+ raise RuntimeError("forced View-share transaction failure")
+ answer.update({"owner": keep_owner, "entries": stamped,
+ "fieldCount": len(wanted)})
+ return data
+
+ _st_for_boundaries(st).update(shares.SHARES_KEY, _apply, flush="sync")
+ return dict(answer)
+
+
+def record_view_share_diagnostic(reference, view_id, actor, stage, fields, cause, st=None):
+ """Retain bounded server-only failure detail under a short support reference."""
+ ref = str(reference or "").strip().upper()[:40]
+ if not ref:
+ return False
+ clean_fields = []
+ seen = set()
+ for raw in fields or ():
+ key = str(raw or "").strip()[:160]
+ if key and key not in seen:
+ seen.add(key)
+ clean_fields.append(key)
+ if len(clean_fields) >= VIEW_SHARE_DIAGNOSTIC_FIELDS_CAP:
+ break
+
+ def _apply(raw):
+ data = copy.deepcopy(raw) if isinstance(raw, dict) else {}
+ rows = [dict(row) for row in (data.get(VIEW_SHARE_DIAGNOSTICS_KEY) or ())
+ if isinstance(row, dict)]
+ rows.append({
+ "reference": ref,
+ "ts": time.time(),
+ "view": str(view_id or "").strip()[:160],
+ "actor": _uname(actor) or None,
+ "stage": str(stage or "transaction").strip()[:80],
+ "fields": clean_fields,
+ "cause": (str(cause or "view share failed").strip()[:500]
+ if isinstance(cause, str)
+ else cause.__class__.__name__),
+ })
+ data[VIEW_SHARE_DIAGNOSTICS_KEY] = rows[-VIEW_SHARE_DIAGNOSTIC_CAP:]
+ return data
+
+ _st_for_boundaries(st).update(shares.SHARES_KEY, _apply, flush="sync")
+ return True
+
+
+def drop_view_field_boundary(view_id, st=None):
+ """Remove exactly one inherited source; direct and other-View grants remain untouched."""
+ view_id = str(view_id or "").strip()
+ if not view_id:
+ return False
+ removed = {"value": False}
+
+ def _apply(data):
+ boundaries = data.get(VIEW_FIELD_BOUNDARIES_KEY)
+ if isinstance(boundaries, dict) and view_id in boundaries:
+ boundaries.pop(view_id, None)
+ removed["value"] = True
+ return data
+
+ _st_for_boundaries(st).update(shares.SHARES_KEY, _apply, flush="async")
+ return removed["value"]
+
+
+def _field_role_sources_from_data(topic, key, me, is_admin, data):
+ """Resolve one field from one already-read share document."""
+ oid = shares.field_oid(topic, key)
+ raw_direct = (data.get("field") or {}).get(oid)
+ raw_direct = raw_direct if isinstance(raw_direct, dict) else {}
+ direct_rec = {"owner": _uname(raw_direct.get("owner")) or None,
+ "entries": shares._clean_entries(raw_direct.get("entries"))}
+ direct = shares.role_from(direct_rec, me, is_admin=is_admin)
+ if direct == "owner" or is_admin:
+ return {"role": "owner", "direct": direct, "views": []}
+ view_ids = []
+ boundaries = data.get(VIEW_FIELD_BOUNDARIES_KEY) or {}
+ view_grants = data.get("view") or {}
+ if isinstance(boundaries, dict) and isinstance(view_grants, dict):
+ for view_id, boundary in boundaries.items():
+ if not isinstance(boundary, dict) or boundary.get("topic") != topic:
+ continue
+ if key not in {str(value) for value in (boundary.get("fields") or ())}:
+ continue
+ raw = view_grants.get(str(view_id))
+ if not isinstance(raw, dict):
+ continue
+ rec = {"owner": _uname(raw.get("owner")) or None,
+ "entries": shares._clean_entries(raw.get("entries"))}
+ if shares.role_from(rec, me, is_admin=False) is not None:
+ view_ids.append(str(view_id))
+ role = direct if direct in ("edit", "view") else ("view" if view_ids else None)
+ return {"role": role, "direct": direct, "views": sorted(view_ids)}
+
+
+def field_roles_for_keys(grant_topic, field_keys, viewer, *, is_admin=False, st=None):
+ """Resolve many field roles from one share-document read."""
+ topic = str(grant_topic or "").strip()
+ me = _uname(viewer)
+ keys = []
+ seen = set()
+ for raw in field_keys or ():
+ key = str(raw or "").strip()
+ if key and key not in seen:
+ seen.add(key)
+ keys.append(key)
+ if not topic or not me:
+ return {key: {"role": None, "direct": None, "views": []} for key in keys}
+ try:
+ data = _st_for_boundaries(st).get(shares.SHARES_KEY) or {}
+ except Exception: # noqa: BLE001
+ data = {}
+ return {key: _field_role_sources_from_data(topic, key, me, is_admin, data)
+ for key in keys}
+
+
+def field_role_sources(grant_topic, field_key, viewer, *, is_admin=False, st=None):
+ """Resolve one field role and its direct/View provenance.
+
+ A View role inherits READ access only. ``Can edit`` on a View means editing the View's
+ configuration, not taking ownership of or editing a colleague's field definition/value.
+ Direct field roles remain stronger and are returned unchanged.
+ """
+ key = str(field_key or "").strip()
+ if not key:
+ return {"role": None, "direct": None, "views": []}
+ return field_roles_for_keys(
+ grant_topic, [key], viewer, is_admin=is_admin, st=st).get(
+ key, {"role": None, "direct": None, "views": []})
+
+
+# Save the unwrapped direct resolver once, including across module reloads in verifier processes.
+_DIRECT_FIELD_ROLE_FOR = getattr(shares.role_for, "_direct_field_role_for", shares.role_for)
+
+
+def install_view_field_resolver():
+ """Install C8 at the existing ``shares.role_for`` wall, once per process.
+
+ Lane V does not own ``core.shares`` or ``core.perm_scope``. The application imports its
+ share router at startup, and that router explicitly installs this owned resolver. Every
+ existing caller therefore keeps asking the same wall; only ``kind='field'`` gains the
+ View-boundary leg. The original callable remains attached for direct/provenance reads.
+ """
+ if getattr(shares.role_for, "_view_field_resolver", False):
+ return shares.role_for
+
+ def _role_for(kind, oid, user, is_admin=False, st=None):
+ if kind != "field":
+ return _DIRECT_FIELD_ROLE_FOR(kind, oid, user, is_admin=is_admin, st=st)
+ topic, key = shares.split_field_oid(oid)
+ if not topic or not key:
+ return None
+ return field_role_sources(topic, key, user, is_admin=is_admin, st=st)["role"]
+
+ _role_for._view_field_resolver = True
+ _role_for._direct_field_role_for = _DIRECT_FIELD_ROLE_FOR
+ shares.role_for = _role_for
+ return _role_for
+
+
def permissions_from_grants(entries):
"""⭐⭐ W40-T03 / CONTRACT C4 — THE CLASSIFICATION IS DERIVED FROM THE REGISTRY, NEVER STAMPED.
diff --git a/platform/core/grid_events.py b/platform/core/grid_events.py
index ed7311b29a7b9025555213d08fd29642957741df..f2c308477c631434959a219e36f24acfdf963bda 100644
--- a/platform/core/grid_events.py
+++ b/platform/core/grid_events.py
@@ -343,6 +343,93 @@ def _field_share_keys(ctx):
return raw, raw
+def _sync_view_field_inheritance(ctx, view):
+ """Apply C8 after a validated View save and notify the field owner/tenant admins.
+
+ ``ctx.fields`` is already the caller's permission-projected schema. A key the editor cannot
+ read therefore never reaches the saved config or this boundary. New personal overlay fields
+ are promoted without changing ``createdBy`` so the inherited READ role points at a definition
+ the audience can actually receive; database-owned ``ut_*`` fields already live tenant-wide.
+ """
+ from core import field_permissions as _fp
+
+ tops = _tops(ctx)
+ st = getattr(tops, 'st', None) or ctx.st
+ shared_key, grant_topic = _field_share_keys(ctx)
+ delta = _fp.sync_view_field_boundary(
+ view, grant_topic, ctx.uname, st=st, audit_additions=True)
+ added = list(delta.get('added') or ())
+ audience = list(delta.get('audience') or ())
+ if not audience:
+ return delta
+
+ by_key = {str(field.get('key')): field for field in (ctx.fields or ())
+ if isinstance(field, dict) and field.get('key')}
+ table_key = str(getattr(tops, 'table_key', '') or '')
+ # Check every included key, not only this save's delta. If a transient overlay failure left
+ # an edge pointing at a still-personal definition, the audience remains fail-closed and the
+ # next View save retries residency instead of treating the boundary as finished forever.
+ # Successful promotions are idempotent because the projected schema then reports shared=True.
+ included = _fp.view_field_keys(view)
+ if not table_key.startswith('ut_'):
+ for key in included:
+ field = by_key.get(key)
+ if not isinstance(field, dict) or field.get('shared') is True:
+ continue
+ owner = str(field.get('createdBy') or '').strip().lower()
+ if not owner or owner != str(ctx.uname or '').strip().lower():
+ continue
+ try:
+ promoted = _fp.promote_field(
+ table_key, shared_key, grant_topic, owner, field, st=st)
+ by_key[key] = promoted
+ except Exception as exc: # noqa: BLE001
+ # Fail closed: the boundary alone cannot surface a definition that stayed
+ # personal. Report the infrastructure failure without putting field values in
+ # logs; every later live-audience save retries the idempotent promotion.
+ _tel.error('grid_events:view-field-promotion', exc)
+
+ if not added:
+ return delta
+
+ # Addition notifications are working-inbox facts, not permission authority. Send one small
+ # row per field/recipient and never let an inbox outage roll back an already accepted View.
+ try:
+ import core.alerts as _alerts
+ import core.perms as _perms
+ import core.shares as _shares
+ import core.users as _users
+ registry = _users.registry() or {}
+ tenant = str(getattr(st, 'key', '') or '').strip().lower()
+ admins = {
+ str(username).strip().lower()
+ for username, record in registry.items()
+ if isinstance(record, dict) and _perms.is_admin(record)
+ and (not tenant or str(record.get('tenant') or 'royal-imports').strip().lower() == tenant)
+ }
+ view_id = str((view or {}).get('id') or '')
+ view_name = str((view or {}).get('name') or 'this View')[:160]
+ actor = str(ctx.uname or '').strip().lower()
+ for key in added:
+ field = by_key.get(key) or {}
+ direct = _shares.grants(
+ 'field', _shares.field_oid(grant_topic, key), st=st)
+ owner = str(direct.get('owner') or field.get('createdBy') or '').strip().lower()
+ recipients = set(admins)
+ if owner:
+ recipients.add(owner)
+ label = str(field.get('label') or key)[:160]
+ for recipient in sorted(recipients - {actor, ''}):
+ _alerts.notify(
+ recipient, 'Field added to a shared View', topic='share',
+ key=f'view-field:{view_id}:{key}', row_id=view_id,
+ detail=f'{label} was added to {view_name}. Its audience can now read it.',
+ actor=actor, st=st)
+ except Exception as exc: # noqa: BLE001
+ _tel.error('grid_events:view-field-notification', exc)
+ return delta
+
+
def _field_share_role(ctx, key, definition):
"""Return the specific Share field role, if this definition is governed by it.
@@ -1706,6 +1793,7 @@ def handle_one(event, ctx):
)
view['name'] = _unique_name(view.get('name'), _view_names)
ws['views'][view_id] = view
+ _sync_view_field_inheritance(ctx, view)
name_changed = view.get('name') != requested_name
# PERSISTED, but deliberately NO rerun — see the return-value contract in
# this function's docstring. This is the hot path: every filter, sort,
@@ -1770,6 +1858,8 @@ def handle_one(event, ctx):
try:
import core.shares as _shares
_shares.drop_objects([('view', view_id)], st=getattr(_tops(ctx), 'st', None))
+ from core import field_permissions as _fp
+ _fp.drop_view_field_boundary(view_id, st=getattr(_tops(ctx), 'st', None))
except Exception:
pass
return True
@@ -1779,11 +1869,18 @@ def handle_one(event, ctx):
try:
import core.shares as _shares
_shares.drop_objects([('view', view_id)], st=getattr(_tops(ctx), 'st', None))
+ from core import field_permissions as _fp
+ _fp.drop_view_field_boundary(view_id, st=getattr(_tops(ctx), 'st', None))
except Exception:
pass
else:
_session_ready()
ws['views'].pop(view_id, None)
+ try:
+ from core import field_permissions as _fp
+ _fp.drop_view_field_boundary(view_id, st=ctx.st)
+ except Exception:
+ pass
# Wave-6 item 3: the client removed the view optimistically; a deleted view simply
# stops being served, so the next render differs in nothing the client still shows.
return False
diff --git a/platform/core/registry.py b/platform/core/registry.py
index 585ff4873ac621d15dc4cae139298fbf49856bb7..ae3c90ed139055dca5823c8d1b6c4b4b95af3310 100644
--- a/platform/core/registry.py
+++ b/platform/core/registry.py
@@ -229,6 +229,53 @@ REGISTRY = [
BY_KEY = {m['key']: m for m in REGISTRY}
+# Source policies - the single source of truth for connector database set-up.
+#
+# The API resolves eligibility from these rows and each source's registered analysis. Adding a
+# source = add one row here + its provider and analysis in the connector's app layer. This is
+# DATA ONLY: core knows neither the mirror nor the API, and imports neither to answer a policy.
+#
+# Fields:
+# key stable source id (matches its registered analysis)
+# label human name
+# prefixes tuple of owned table-key prefixes; the longest matching prefix wins
+# max_rows materialised-home ceiling, or None to use the store default; never an ingest cap
+# window_max whole-table window ceiling, or None to use the datastore default
+# rank_ops "client" or "server": where set-level rank operators are evaluated
+# candidates "declared": only provider-returned grains; "prefix": every owned database
+#
+# None preserves the existing ceilings. Declared candidates preserve the existing grain set;
+# a prefix identifies the policy owner without making an undeclared database read-through.
+SOURCE_POLICIES = [
+ {'key': 'odoo', 'label': 'Odoo', 'prefixes': ('ut_odoo_',),
+ 'max_rows': None, 'window_max': None, 'rank_ops': 'client', 'candidates': 'declared'},
+ {'key': 'amazon', 'label': 'Amazon', 'prefixes': ('ut_amazon_',),
+ 'max_rows': None, 'window_max': None, 'rank_ops': 'client', 'candidates': 'declared'},
+ {'key': 'meta_ads', 'label': 'Meta Ads', 'prefixes': ('ut_meta_',),
+ 'max_rows': None, 'window_max': None, 'rank_ops': 'client', 'candidates': 'declared'},
+]
+DEFAULT_SOURCE_POLICY = {
+ 'key': 'default', 'label': 'Default', 'prefixes': (),
+ 'max_rows': None, 'window_max': None, 'rank_ops': 'client', 'candidates': 'declared',
+}
+
+
+def source_policy(table_key):
+ """The source row owning this key, or DEFAULT_SOURCE_POLICY when no prefix matches."""
+ key = str(table_key or '')
+ match, longest = DEFAULT_SOURCE_POLICY, -1
+ for row in SOURCE_POLICIES:
+ for prefix in row['prefixes']:
+ if key.startswith(prefix) and len(prefix) > longest:
+ match, longest = row, len(prefix)
+ return match
+
+
+def source_policies():
+ """The declared source rows, in declaration order; the fallback row is separate."""
+ return tuple(SOURCE_POLICIES)
+
+
def brand_filterable(key) -> bool:
return bool(BY_KEY.get(key, {}).get('brand'))
diff --git a/platform/core/sheet_binding.py b/platform/core/sheet_binding.py
index e1842a0fb86d34e4788f0a7b0926ce073b11208c..9dbbe8dafe122af327512c95a3d5bc2cacab5008 100644
--- a/platform/core/sheet_binding.py
+++ b/platform/core/sheet_binding.py
@@ -1,18 +1,134 @@
-"""Pure document cleaners shared by Sheet storage and Sheet query resolution.
+"""Binding metadata and server-owned Refresh snapshots shared by Sheet storage and routes.
-These functions decide only what binding metadata may be stored. They do not resolve data,
-permissions, or SQL. Keeping them in core lets the workbook document validate view bindings
-without importing the higher ``modules.sheet`` layer.
+Cleaners remain independent of query resolution. Change-token readers fingerprint source
+state without assembling a database or evaluating an individual cell binding.
"""
from __future__ import annotations
import datetime
+import hashlib
+import json
import re
+from copy import deepcopy
from core import windows as W
+def fingerprint(value):
+ return hashlib.sha256(json.dumps(value, sort_keys=True, default=str,
+ separators=(",", ":")).encode()).hexdigest()
+
+
+def database_change_token(st, database):
+ """Read a database fingerprint, without assembling rows or resolving a binding.
+
+ Authored databases share one JSON document. Its bucket revision cannot distinguish
+ siblings, so Postgres hashes the selected member IN the database and returns only a
+ token and row-free definition. This also sees connector writers which bypass the
+ ordinary row/field API. The HF adapter hashes the same documents locally.
+ """
+ from core import connector_sync, schema_overlay, store_pg, user_tables
+ workspace = {"customer_data": "customer_table_workspace",
+ "product_data": "product_table_workspace"}.get(
+ database, database + "_table_workspace")
+ shared = database if database.startswith("ut_") else workspace
+ buckets = [workspace, shared + "__shared"]
+ handle = st._store() if callable(getattr(st, "_store", None)) else st
+ if isinstance(handle, store_pg.PgStore):
+ key = getattr(st, "store_key", str)
+ # Values stay on the server. No row JSON crosses the connection on this path.
+ with store_pg._pool().connection() as con:
+ row = con.execute(store_pg._q(
+ "WITH source AS (SELECT value -> %s AS d FROM {tbl} WHERE key = %s) "
+ "SELECT md5(COALESCE((SELECT d::text FROM source), 'null') || "
+ "COALESCE((SELECT jsonb_object_agg(key, rev ORDER BY key)::text "
+ "FROM {tbl} WHERE key = ANY(%s)), 'null') || "
+ "COALESCE((SELECT (value -> %s)::text FROM {tbl} WHERE key = %s), 'null') || "
+ "COALESCE((SELECT (value -> %s)::text FROM {tbl} WHERE key = %s), 'null') || "
+ "COALESCE((SELECT jsonb_object_agg(e.key, e.value ORDER BY e.key)::text "
+ "FROM {tbl} kv, jsonb_each(kv.value) e WHERE kv.key = %s "
+ "AND e.value ->> 'databaseKey' = %s), 'null') || "
+ "COALESCE((SELECT jsonb_path_query_array(value, "
+ "'$.*.* ? (@.databases[*] == $db)', jsonb_build_object('db', %s::text))::text "
+ "FROM {tbl} WHERE key = %s), 'null')), "
+ "(SELECT d - 'rows' FROM source)", "store_kv", handle.tenant_slug),
+ (database, key(user_tables.STORE_KEY), [key(b) for b in buckets],
+ database, key(user_tables.ZONE_STORE_KEY), database, key(schema_overlay.STORE_KEY),
+ key(connector_sync.JOBS_KEY), database,
+ database, key(connector_sync.STATE_KEY))).fetchone()
+ token, definition = row[0], row[1] or {}
+ else:
+ definition = (st.get(user_tables.STORE_KEY, fresh=True) or {}).get(database) or {}
+ jobs = st.get(connector_sync.JOBS_KEY, fresh=True) or {}
+ states = st.get(connector_sync.STATE_KEY, fresh=True) or {}
+ token = fingerprint([definition, *(st.get(b, fresh=True) for b in buckets),
+ (st.get(user_tables.ZONE_STORE_KEY, fresh=True) or {}).get(database),
+ (st.get(schema_overlay.STORE_KEY, fresh=True) or {}).get(database),
+ {k: v for k, v in jobs.items() if v.get("databaseKey") == database},
+ [v for provider in sorted(states) for _, v in sorted(states[provider].items())
+ if database in (v.get("databases") or [])]])
+ # Registry topics and read-through connectors live in the observation plane. Sync
+ # receipts have second precision, so also fingerprint source content. A file-wide
+ # mtime would invalidate every database when just one source changed.
+ mirror = not definition or not user_tables.materialises(database, defn=definition)
+ if mirror:
+ token = fingerprint([token, mirror_change_token(st, database)])
+ return token
+
+
+#: ⛔⛔ THE MIRROR TOKEN IS PROVIDED FROM ABOVE, NOT IMPORTED FROM BELOW (SDLC 65 combine).
+#: S65-T03 wrote `mirror_change_token` here and it reached for `harness.datastore` and
+#: `harness.runtime` -- a NEW `core -> harness` edge, which `verify_layering.py` (D-460) caught as
+#: the wave's own violation of ARCHITECTURE.md section 1. ⛔ It is FIXED rather than grandfathered:
+#: `GRANDFATHERED_CORE` is the list of edges that PRE-DATE that gate, and adding this wave's own
+#: edge to it would be granting an exemption to the wave that created it.
+#: The body moved to `harness/mirror_tokens.py`, which registers itself here on import, so the
+#: dependency points DOWN the way every other layer does.
+_MIRROR_TOKEN_PROVIDER = None
+
+
+def set_mirror_token_provider(fn):
+ """Register the mirror reader. Called by `harness.mirror_tokens` at import."""
+ global _MIRROR_TOKEN_PROVIDER
+ _MIRROR_TOKEN_PROVIDER = fn
+
+
+def mirror_change_token(st, database):
+ """One SQL token read over source tables, delegated to the layer that owns the mirror.
+
+ ⚠ NO PROVIDER IS A RAISE, DELIBERATELY, and it is the safe direction: `database_change_token`'s
+ contract is that a token which cannot be READ resolves rather than reusing a value, so an
+ unregistered provider costs a recompute and can never serve a stale number.
+ """
+ if _MIRROR_TOKEN_PROVIDER is None:
+ raise RuntimeError("No mirror change-token provider is registered; import "
+ "harness.mirror_tokens before asking for one.")
+ return _MIRROR_TOKEN_PROVIDER(st, database)
+
+
+def read_refresh_cache(st, workbook_id):
+ from core import sheets
+ return st.get(sheets._scope_cache_key(workbook_id), fresh=True) or {}
+
+
+def save_refresh_cache(st, workbook_id, entries, databases):
+ """Server-only snapshots; ordinary workbook PUT/PATCH cannot manufacture these slots."""
+ from core import sheets
+
+ def apply(doc):
+ cells = doc.setdefault("refreshCells", {})
+ for cell_key, scope_key, entry in entries:
+ slots = cells.setdefault(cell_key, {})
+ if scope_key not in slots and len(slots) >= sheets._MAX_SCOPE_ENTRIES_PER_CELL:
+ slots.pop(next(iter(slots)))
+ slots[scope_key] = deepcopy(entry)
+ sheets._merge_data_origins(doc, databases=databases)
+ return doc, None
+
+ sheets._persist(st, sheets._scope_cache_key(workbook_id), apply)
+
+
PERIOD_UNITS = ("day", "week", "month", "quarter", "year")
PERIOD_START = re.compile(r"^\d{4}-(0[1-9]|1[0-2])$")
PERIOD_DAY = re.compile(r"^\d{4}-(0[1-9]|1[0-2])-(0[1-9]|[12]\d|3[01])$")
@@ -21,7 +137,9 @@ DRAG_ACROSS = ("period", "none")
DRAG_DOWN_NONE = "none"
_MESSAGES = {
+ "related_invalid": "Choose one declared relationship and a valid filter tree for this cell.",
"period_invalid": "{subject} is not a period this cell can use. Choose a valid start for the selected period unit.",
+ "period_window_invalid": "This cell's lookback window is invalid. Choose a valid window in the period control.",
"ending_invalid": "Ending {subject} cannot be used. Choose a valid ending date.",
"ending_not_supported": "{subject} does not support an Ending. Choose year to date, last year to date, this month or this quarter.",
"ending_cell_unreadable": "Ending from cell {subject} cannot be read. Use a cell containing a date in YYYY-MM-DD format.",
@@ -44,6 +162,58 @@ class BindingMetadataRefusal(Exception):
f"cannot be {verb}.", f"cannot be {verb} because {self.detail}")
+def clean_related(raw):
+ """One declared edge key and a structurally valid, losslessly copied filter tree.
+
+ Absence is handled by the caller. An explicitly supplied null is malformed, as are
+ extra relationship bags and nested relationships. Operator support belongs to the
+ query compiler, which reports unpushable conditions rather than dropping metadata.
+ """
+ def refuse():
+ raise BindingMetadataRefusal("related_invalid")
+
+ if not isinstance(raw, dict) or set(raw) != {"via", "filter"}:
+ refuse()
+ via = raw["via"]
+ if not isinstance(via, str) or not via.strip():
+ refuse()
+
+ def reject_nested(value):
+ if isinstance(value, dict):
+ if "related" in value:
+ refuse()
+ for child in value.values():
+ reject_nested(child)
+ elif isinstance(value, list):
+ for child in value:
+ reject_nested(child)
+
+ reject_nested(raw["filter"])
+
+ def nodes_ok(nodes):
+ if not isinstance(nodes, list):
+ refuse()
+ for node in nodes:
+ if not isinstance(node, dict) or "related" in node:
+ refuse()
+ if "children" in node:
+ if ("colId" in node or node.get("conj", "and") not in ("and", "or")):
+ refuse()
+ nodes_ok(node["children"])
+ elif (not isinstance(node.get("colId"), str) or not node["colId"].strip()
+ or not isinstance(node.get("op"), str) or not node["op"].strip()):
+ refuse()
+
+ tree = raw["filter"]
+ if isinstance(tree, dict):
+ if set(tree) - {"nodes", "conj"} or tree.get("conj", "and") not in ("and", "or"):
+ refuse()
+ nodes_ok(tree.get("nodes"))
+ else:
+ nodes_ok(tree)
+ return {"via": via.strip(), "filter": deepcopy(tree)}
+
+
def clean_period(raw):
"""Return one canonical Sheet period, or ``None`` when no window was named."""
if not isinstance(raw, dict):
@@ -135,7 +305,10 @@ def clean_period(raw):
out["ref"] = ref
if (raw.get("window") is not None and "unit" not in out and "start" not in out
and not out.get("frozen")):
- out["window"] = raw["window"]
+ window = W.normalize(raw["window"])
+ if window is None:
+ raise BindingMetadataRefusal("period_window_invalid")
+ out["window"] = window
if not ({"start", "window", "ref", "frozen"} & set(out)):
return None
return out
diff --git a/platform/core/sheet_compat.py b/platform/core/sheet_compat.py
new file mode 100644
index 0000000000000000000000000000000000000000..ecfe83911b0dc3bce0a42022205fafb6655bbdfc
--- /dev/null
+++ b/platform/core/sheet_compat.py
@@ -0,0 +1,875 @@
+"""Excel-shaped automation compatibility for Loopable Sheet.
+
+This module is the one capability declaration and the one operation planner used by both the
+HTTP door and the Python SDK. It deliberately models the bounded workbook operations exercised
+by the refresh-sales script and the refresh-fpa operating procedure. Excel process, window, VBA
+and modal-UI controls are inventoried too, but always refuse before the tenant store callback.
+
+Writes are operation-level compare-and-swap patches: every write needs the revision the caller
+read, and only the addressed cells or worksheet/workbook property is changed. This is not a
+second workbook persistence implementation; the final callback uses ``core.sheets``' store seam.
+"""
+
+from __future__ import annotations
+
+import base64
+import copy
+import json
+import math
+import os
+import re
+import tempfile
+from core import write_execution
+
+from core import sheets
+
+
+MANIFEST_VERSION = "2026-09-19"
+
+
+def _cap(name, supported, signature, semantics, evidence, *, access="read", reason=""):
+ row = {
+ "name": name,
+ "supported": bool(supported),
+ "access": access if supported else "unsupported",
+ "signature": signature,
+ "semantics": semantics if supported else "Refuses before any workbook store update.",
+ "evidence": list(evidence),
+ }
+ if not supported:
+ row["unsupportedReason"] = reason
+ return row
+
+
+_SALES = ".agents/skills/refresh-sales/scripts/refresh_daily_sales_performance.py"
+_FPA = ".agents/skills/refresh-fpa/SKILL.md"
+
+
+# The evidence references are intentionally literal and machine-readable. Section 26 of the
+# Sheet gate re-censuses the source operations and requires every token to land on one row here.
+_OPERATIONS = (
+ _cap("workbooks.open", True, "Open(id_or_name, UpdateLinks=0, ReadOnly=False)",
+ "Looks up one existing workbook by stable id, then exact case-insensitive name.",
+ [f"{_SALES}:421", f"{_FPA}:153"]),
+ _cap("worksheets.item", True, "Worksheets(name_or_id)",
+ "Looks up one worksheet by id, exact name, then case-insensitive name.",
+ [f"{_SALES}:425-426"]),
+ _cap("worksheet.range", True, "Range(A1 | A1:B9 | Cell, Cell)",
+ "Builds one inclusive rectangular A1 range inside Excel worksheet limits.",
+ [f"{_SALES}:392", f"{_SALES}:621-624"]),
+ _cap("worksheet.cells", True, "Cells(row, column)",
+ "Addresses one one-based cell and composes with Range.",
+ [f"{_SALES}:438", f"{_SALES}:581"]),
+ _cap("range.values.read", True, "Range.Value | Range.Value2",
+ "Returns a scalar for one cell or a row-major matrix for a rectangle; formulas return cached values.",
+ [f"{_SALES}:466", f"{_SALES}:584"]),
+ _cap("range.values.write", True, "Range.Value = scalar | matrix",
+ "Writes only addressed cell contents, retaining each cell's style and unrelated metadata.",
+ [f"{_SALES}:438", f"{_SALES}:624"], access="write"),
+ _cap("range.formulas.read", True, "Range.Formula | Range.FormulaR1C1",
+ "Returns authored formulas; FormulaR1C1 returns the preserved R1C1 source when present.",
+ [f"{_SALES}:393"]),
+ _cap("range.formulas.write", True, "Range.Formula[ R1C1] = scalar | matrix",
+ "Writes formulas only in the rectangle; R1C1 references are translated at each destination.",
+ [f"{_SALES}:403", f"{_FPA}:177-179"], access="write"),
+ _cap("range.styles.read", True, "Range.Style",
+ "Returns each cell's canonical style bag without interpreting unrelated cell content.",
+ [f"{_SALES}:433-436", f"{_FPA}:112" ]),
+ _cap("range.styles.write", True, "Range.Style = style | matrix",
+ "Replaces only the addressed cells' canonical style bags.",
+ [f"{_SALES}:433-436", f"{_FPA}:112"], access="write"),
+ _cap("range.number_formats.read", True, "Range.NumberFormat",
+ "Returns Excel format-code strings from canonical cell styles.",
+ [f"{_SALES}:10", f"{_FPA}:177" ]),
+ _cap("range.number_formats.write", True, "Range.NumberFormat = code | matrix",
+ "Changes only style.format on addressed cells; General removes the explicit format.",
+ [f"{_SALES}:10", f"{_FPA}:177"], access="write"),
+ _cap("range.fill_down", True, "Range.FillDown()",
+ "Copies the top row down inside the rectangle and translates A1/R1C1 formulas per row.",
+ [f"{_SALES}:404", f"{_FPA}:177-179"], access="write"),
+ _cap("range.clear_contents", True, "Range.ClearContents()",
+ "Clears values and formulas in the rectangle while retaining styles and worksheet properties.",
+ [f"{_SALES}:433-436", f"{_FPA}:139-143"], access="write"),
+ _cap("range.replace", True, "Range.Replace(What, Replacement, LookAt=xlWhole, MatchCase=False)",
+ "Replaces exact stored literal values only; formulas and partial strings are never rewritten.",
+ [f"{_SALES}:672-680"], access="write"),
+ _cap("range.copy", True, "target.CopyFrom(source, mode='all'|'values'|'formulas')",
+ "Copies equal-sized rectangles and translates formulas; values mode hardcodes cached values.",
+ [f"{_FPA}:121-130", f"{_FPA}:177-179"], access="write"),
+ _cap("range.paste_values", True, "target.PasteSpecial(xlPasteValues, source=source)",
+ "Hardcodes cached source values into an equal-sized target without an ambient process clipboard.",
+ [f"{_FPA}:177-179", f"{_FPA}:194-199"], access="write"),
+ _cap("worksheet.used_range", True, "Worksheet.UsedRange",
+ "Returns the inclusive true used rectangle, or A1 for an empty sheet.",
+ [f"{_SALES}:466", f"{_SALES}:656"]),
+ _cap("range.end_up", True, "Cells(Rows.Count, column).End(xlUp)",
+ "Returns the last nonblank row in the selected column, or row 1 when empty.",
+ [f"{_SALES}:581", f"{_SALES}:617"]),
+ _cap("worksheet.count_a", True, "WorksheetFunction.CountA(range)",
+ "Counts nonblank stored values or formulas inside the rectangle.",
+ [f"{_SALES}:457-463", f"{_SALES}:630"]),
+ _cap("worksheet.count_if", True, "WorksheetFunction.CountIf(range, criterion)",
+ "Supports the workflow's exact-cell, case-insensitive criterion.",
+ [f"{_SALES}:656"]),
+ _cap("worksheet.geometry.read", True, "Range.ColumnWidth | Range.RowHeight",
+ "Reads canonical per-column widths and per-row heights with worksheet defaults.",
+ [f"{_SALES}:10", f"{_FPA}:121-130"]),
+ _cap("worksheet.geometry.write", True, "Range.ColumnWidth|RowHeight = number",
+ "Changes only columns or rows intersecting the addressed range.",
+ [f"{_SALES}:10", f"{_FPA}:121-130"], access="write"),
+ _cap("worksheet.hidden.read", True, "Range.EntireRow.Hidden | EntireColumn.Hidden",
+ "Reads hidden state for rows or columns intersecting the addressed range.",
+ [f"{_SALES}:10"]),
+ _cap("worksheet.hidden.write", True, "Range.EntireRow|EntireColumn.Hidden = bool",
+ "Changes hidden state only for rows or columns intersecting the addressed range.",
+ [f"{_SALES}:10"], access="write"),
+ _cap("worksheet.freeze.read", True, "Worksheet.FreezePanes",
+ "Returns canonical frozen row and column counts.",
+ [f"{_SALES}:932-935"]),
+ _cap("worksheet.freeze.write", True, "Worksheet.FreezePanes = {rows, cols}",
+ "Changes only the worksheet freeze counts.",
+ [f"{_SALES}:932-935"], access="write"),
+ _cap("worksheet.gridlines.read", True, "Worksheet.DisplayGridlines",
+ "Reads the worksheet view's gridline flag, defaulting to shown.",
+ [f"{_SALES}:932-935"]),
+ _cap("worksheet.gridlines.write", True, "Worksheet.DisplayGridlines = bool",
+ "Changes only the worksheet view's gridline flag.",
+ [f"{_SALES}:932-935"], access="write"),
+ _cap("application.calculation.read", True, "Application.Calculation",
+ "Reads manual or automatic mode from workbook calculation metadata.",
+ [f"{_SALES}:423", f"{_FPA}:146-153"]),
+ _cap("application.calculation.write", True, "Application.Calculation = manual|automatic",
+ "Changes only workbook calculation mode and requires expected revision.",
+ [f"{_SALES}:423", f"{_SALES}:444", f"{_FPA}:151-157"], access="write"),
+ _cap("workbook.calculate_full", True, "Application.CalculateFull()",
+ "Runs the host formula evaluator over the workbook and persists refreshed formula caches.",
+ [f"{_SALES}:447", f"{_FPA}:187", f"{_FPA}:216"], access="write"),
+ _cap("workbook.refresh_all", True, "Workbook.RefreshAll()",
+ "Runs the same bounded calculation callback and stamps refreshed_at.",
+ [f"{_SALES}:447", f"{_FPA}:233"], access="write"),
+ _cap("workbook.names.item", True, "Workbook.Names(name)",
+ "Reads one workbook named range without resolving external names.",
+ [f"{_SALES}:63-66", f"{_FPA}:177"]),
+ _cap("workbook.names.write", True, "Workbook.Names.Add(name, refers_to)",
+ "Upserts one workbook-scoped A1 named range and preserves every other name.",
+ [f"{_SALES}:63-66", f"{_FPA}:177"], access="write"),
+ _cap("workbook.export_xlsx", True, "Workbook.Export(path|bytes)",
+ "Exports a data-free xlsx using the existing lossless Sheet export seam; it never runs VBA.",
+ [f"{_SALES}:506-538", f"{_FPA}:233"]),
+ _cap("workbook.save", True, "Workbook.Save()",
+ "Returns the current revision because Loopable operation writes are already atomic and durable.",
+ [f"{_SALES}:469", f"{_FPA}:157", f"{_FPA}:233"]),
+ _cap("workbook.read_only", True, "Workbook.ReadOnly",
+ "Returns false after the route's editor wall; the facade never bypasses authorization.",
+ [f"{_FPA}:160"]),
+
+ _cap("application.dispatch_ex", False, "DispatchEx('Excel.Application')", "", [f"{_SALES}:412"],
+ reason="Loopable has no local Excel process; choose the backend at the client boundary."),
+ _cap("workbooks.add", False, "Workbooks.Add()", "", [f"{_FPA}:147-153"],
+ reason="An Excel scratch workbook exists only to bind process-global calculation mode."),
+ _cap("application.visible", False, "Application.Visible = bool", "", [f"{_SALES}:413", f"{_SALES}:935"],
+ reason="Browser visibility is not a workbook data operation."),
+ _cap("application.window_state", False, "Application.WindowState = value", "", [f"{_SALES}:936"],
+ reason="Window maximization is local UI orchestration."),
+ _cap("application.display_alerts", False, "Application.DisplayAlerts = bool", "", [f"{_SALES}:414", f"{_SALES}:937"],
+ reason="Loopable has no Excel modal-alert process state."),
+ _cap("application.screen_updating", False, "Application.ScreenUpdating = bool", "", [f"{_SALES}:415"],
+ reason="Loopable writes documents without an Excel repaint loop."),
+ _cap("application.enable_events", False, "Application.EnableEvents = bool", "", [f"{_SALES}:416", f"{_SALES}:938"],
+ reason="Excel process events are not Sheet document data."),
+ _cap("application.automation_security", False, "Application.AutomationSecurity = value", "", [f"{_SALES}:417", f"{_SALES}:940"],
+ reason="Loopable never executes workbook macros."),
+ _cap("workbook.autosave_on", False, "Workbook.AutoSaveOn = bool", "", [f"{_SALES}:341-351"],
+ reason="Loopable writes are atomic store operations, not OneDrive AutoSave sessions."),
+ _cap("workbook.vbproject", False, "Workbook.VBProject", "", [f"{_SALES}:898"],
+ reason="VBA projects are preserved by import metadata but never exposed for execution or editing."),
+ _cap("vba.code_module", False, "CodeModule.Lines|DeleteLines|AddFromString", "", [f"{_SALES}:904-918"],
+ reason="Editing executable VBA is outside the Sheet document capability boundary."),
+ _cap("application.run", False, "Application.Run(macro, *args)", "", [f"{_SALES}:963-965"],
+ reason="Loopable never runs VBA; use workbook.export_xlsx for the supported export."),
+ _cap("application.get_save_as_filename", False, "Application.GetSaveAsFilename(...) ", "", [f"{_SALES}:806-863"],
+ reason="A modal file picker is browser/UI orchestration."),
+ _cap("application.msg_box", False, "MsgBox(message)", "", [f"{_SALES}:865-885"],
+ reason="A modal VBA message box is not a workbook data operation."),
+ _cap("workbook.close", False, "Workbook.Close(SaveChanges=False)", "", [f"{_SALES}:477", f"{_FPA}:160"],
+ reason="Loopable workbooks are request-scoped resources, not local process handles."),
+ _cap("application.quit", False, "Application.Quit()", "", [f"{_SALES}:478"],
+ reason="Loopable has no Excel process to terminate."),
+)
+
+
+CAPABILITY_MANIFEST = {
+ "schema": 1,
+ "version": MANIFEST_VERSION,
+ "hierarchy": ["Application", "Workbooks", "Workbook", "Worksheets", "Worksheet", "Range", "Cells"],
+ "inventory": {
+ "method": "mechanical Python attribute-call census plus the refresh-fpa operating workflow evidence",
+ "sources": [_SALES, _FPA],
+ "supported": sum(1 for row in _OPERATIONS if row["supported"]),
+ "unsupported": sum(1 for row in _OPERATIONS if not row["supported"]),
+ },
+ "operations": list(_OPERATIONS),
+}
+
+CAPABILITY_MANIFEST_BYTES = json.dumps(
+ CAPABILITY_MANIFEST, sort_keys=True, separators=(",", ":"), ensure_ascii=False
+).encode("utf-8")
+
+_BY_NAME = {row["name"]: row for row in _OPERATIONS}
+
+
+class CompatibilityRefusal(sheets.SheetRefusal):
+ """A named compatibility refusal suitable for the existing Sheet HTTP envelope."""
+
+
+def capability_manifest():
+ return copy.deepcopy(CAPABILITY_MANIFEST)
+
+
+def capability_manifest_bytes():
+ return CAPABILITY_MANIFEST_BYTES
+
+
+def operation_spec(name):
+ return copy.deepcopy(_BY_NAME.get(str(name or "")))
+
+
+def require_supported(name):
+ key = str(name or "").strip()
+ spec = _BY_NAME.get(key)
+ if spec is None:
+ raise CompatibilityRefusal(
+ "sheet_compat_operation_unknown", "That Sheet automation operation is not in the capability manifest.")
+ if not spec["supported"]:
+ raise CompatibilityRefusal(
+ "sheet_compat_operation_unsupported",
+ key + " is unsupported: " + spec["unsupportedReason"])
+ return spec
+
+
+_CELL = re.compile(r"^\$?([A-Z]{1,3})\$?([1-9][0-9]{0,6})$")
+_RECT = re.compile(r"^\$?([A-Z]{1,3})\$?([1-9][0-9]{0,6})(?::\$?([A-Z]{1,3})\$?([1-9][0-9]{0,6}))?$")
+
+
+def _col_number(letters):
+ value = 0
+ for letter in str(letters).upper():
+ value = value * 26 + ord(letter) - 64
+ return value
+
+
+def _letters(number):
+ out = ""
+ value = int(number)
+ while value:
+ value, rem = divmod(value - 1, 26)
+ out = chr(65 + rem) + out
+ return out
+
+
+def _addr(row, col):
+ return _letters(col) + str(row)
+
+
+def _range(raw):
+ text = str(raw or "").strip().upper().replace(" ", "")
+ found = _RECT.fullmatch(text)
+ if not found:
+ raise CompatibilityRefusal("sheet_compat_range_invalid", "A Sheet automation range needs one rectangular A1 address.")
+ c0, r0 = _col_number(found.group(1)), int(found.group(2))
+ c1, r1 = _col_number(found.group(3) or found.group(1)), int(found.group(4) or found.group(2))
+ if c0 > sheets.DOCUMENT_MAX_COLS or c1 > sheets.DOCUMENT_MAX_COLS \
+ or r0 > sheets.DOCUMENT_MAX_ROWS or r1 > sheets.DOCUMENT_MAX_ROWS:
+ raise CompatibilityRefusal("sheet_compat_range_invalid", "That Sheet automation range is outside Excel's dimensions.")
+ c0, c1 = sorted((c0, c1))
+ r0, r1 = sorted((r0, r1))
+ if (c1 - c0 + 1) * (r1 - r0 + 1) > sheets._MAX_CELLS_PER_SHEET:
+ raise CompatibilityRefusal("sheet_too_many_cells", "That Sheet automation range is larger than one operation accepts.")
+ return r0, c0, r1, c1
+
+
+def _range_text(rect):
+ r0, c0, r1, c1 = rect
+ first, last = _addr(r0, c0), _addr(r1, c1)
+ return first if first == last else first + ":" + last
+
+
+def _addresses(rect):
+ r0, c0, r1, c1 = rect
+ return [[_addr(row, col) for col in range(c0, c1 + 1)] for row in range(r0, r1 + 1)]
+
+
+def _matrix(value, height, width, what, *, scalar=True):
+ if scalar and not isinstance(value, (list, tuple)):
+ return [[copy.deepcopy(value) for _ in range(width)] for _ in range(height)]
+ if not isinstance(value, (list, tuple)):
+ raise CompatibilityRefusal("sheet_compat_matrix_invalid", what + " needs a row-major matrix.")
+ rows = list(value)
+ if height == 1 and len(rows) == width and (not rows or not isinstance(rows[0], (list, tuple))):
+ rows = [rows]
+ if len(rows) != height or any(not isinstance(row, (list, tuple)) or len(row) != width for row in rows):
+ raise CompatibilityRefusal(
+ "sheet_compat_matrix_shape", what + " must have exactly " + str(height) + " rows and " + str(width) + " columns.")
+ return [[copy.deepcopy(item) for item in row] for row in rows]
+
+
+def _tab(book, ref):
+ return sheets.find_tab(book, ref)
+
+
+def _cell_value(cell):
+ if not isinstance(cell, dict) or cell.get("kind") == "empty":
+ return None
+ return cell.get("value", cell.get("text"))
+
+
+def _display_matrix(tab, rect, getter):
+ grid = []
+ cells = tab.get("cells") or {}
+ for row in _addresses(rect):
+ grid.append([copy.deepcopy(getter(cells.get(addr))) for addr in row])
+ return grid[0][0] if len(grid) == len(grid[0]) == 1 else grid
+
+
+def _keep(cell, keys=("style", "validation", "note", "excel", "link", "sparkline")):
+ return {key: copy.deepcopy(cell[key]) for key in keys if isinstance(cell, dict) and key in cell}
+
+
+def _literal(value, prior):
+ if value is None:
+ held = _keep(prior)
+ return {"kind": "empty", "state": "typed", **held} if held else None
+ cell = {"kind": "literal", "state": "typed", "value": copy.deepcopy(value), **_keep(prior)}
+ if isinstance(value, str):
+ cell["text"] = value
+ return sheets.clean_cell(cell)
+
+
+_R1C1 = re.compile(r"(? sheets.DOCUMENT_MAX_ROWS or cc > sheets.DOCUMENT_MAX_COLS:
+ raise CompatibilityRefusal("sheet_compat_formula_reference", "That R1C1 formula points outside the worksheet.")
+ return _addr(rr, cc)
+ return _R1C1.sub(replace, str(formula or ""))
+
+
+def _formula(value, prior, addr, *, r1c1=False):
+ if value in (None, ""):
+ return _literal(None, prior)
+ text = str(value)
+ if not text.startswith("="):
+ text = "=" + text
+ found = _CELL.fullmatch(addr)
+ authored = _r1c1_to_a1(text, int(found.group(2)), _col_number(found.group(1))) if r1c1 else text
+ cell = {"kind": "formula", "state": "stale", "formula": authored, "value": None, **_keep(prior)}
+ if r1c1:
+ excel = dict(cell.get("excel") or {})
+ excel["formulaR1C1"] = text
+ cell["excel"] = excel
+ return sheets.clean_cell(cell, addr=addr)
+
+
+def _translate_formula(cell, origin, destination):
+ if not isinstance(cell, dict) or cell.get("kind") != "formula":
+ return copy.deepcopy(cell)
+ r1c1 = (cell.get("excel") or {}).get("formulaR1C1")
+ if isinstance(r1c1, str):
+ return _formula(r1c1, cell, destination, r1c1=True)
+ try:
+ from openpyxl.formula.translate import Translator
+ translated = Translator(str(cell.get("formula") or ""), origin=origin).translate_formula(destination)
+ except Exception:
+ translated = str(cell.get("formula") or "")
+ return _formula(translated, cell, destination)
+
+
+def _used(tab):
+ rect = tab.get("usedExtent") or sheets._used_extent(
+ tab.get("cells") or {}, (tab.get("excel") or {}).get("blankStyles"))
+ if not rect:
+ return (1, 1, 1, 1)
+ return (int(rect["row0"]) + 1, int(rect["col0"]) + 1,
+ int(rect["row1"]) + 1, int(rect["col1"]) + 1)
+
+
+def _write_required(args):
+ revision = args.get("expected_revision")
+ if isinstance(revision, bool) or not isinstance(revision, int) or revision < 0:
+ raise CompatibilityRefusal(
+ "sheet_compat_revision_required", "A Sheet automation write requires the exact workbook revision it read.")
+ return revision
+
+
+def _preflight(name, args):
+ spec = require_supported(name)
+ if not isinstance(args, dict):
+ raise CompatibilityRefusal("sheet_compat_arguments_invalid", "Sheet automation arguments must be an object.")
+ if spec["access"] == "write":
+ _write_required(args)
+ range_ops = {key for key in _BY_NAME if key.startswith("range.")} | {
+ "worksheet.count_a", "worksheet.count_if", "worksheet.geometry.read",
+ "worksheet.geometry.write", "worksheet.hidden.read", "worksheet.hidden.write",
+ }
+ if name.startswith("worksheet.") or name.startswith("range."):
+ if not str(args.get("sheet") or "").strip():
+ raise CompatibilityRefusal("sheet_compat_sheet_required", "That Sheet automation operation needs one worksheet.")
+ if name in range_ops:
+ _range(args.get("range"))
+ return spec
+
+
+def _read(book, name, args):
+ if name == "workbooks.open":
+ return {"id": book.get("id"), "name": book.get("name"), "rev": book.get("rev")}
+ if name == "worksheets.item":
+ tab = _tab(book, args.get("sheet"))
+ return {key: copy.deepcopy(tab.get(key)) for key in ("id", "name", "cols", "rows")}
+ if name in ("worksheet.range", "worksheet.cells"):
+ return {"sheet": args.get("sheet"), "range": _range_text(_range(args.get("range")))}
+ if name.startswith("range.") or name.startswith("worksheet."):
+ tab = _tab(book, args.get("sheet"))
+ rect = _range(args.get("range")) if args.get("range") else None
+ if name == "range.values.read":
+ return _display_matrix(tab, rect, _cell_value)
+ if name == "range.formulas.read":
+ mode = str(args.get("mode") or "a1").lower()
+ return _display_matrix(tab, rect, lambda cell: (
+ ((cell or {}).get("excel") or {}).get("formulaR1C1") if mode == "r1c1"
+ else (cell or {}).get("formula")))
+ if name == "range.styles.read":
+ return _display_matrix(tab, rect, lambda cell: (cell or {}).get("style"))
+ if name == "range.number_formats.read":
+ def number_format(cell):
+ fmt = ((cell or {}).get("style") or {}).get("format")
+ return _xlsx_number_format(fmt)
+ return _display_matrix(tab, rect, number_format)
+ if name == "worksheet.used_range":
+ return {"address": _range_text(_used(tab))}
+ if name == "range.end_up":
+ _r0, c0, _r1, _c1 = rect
+ occupied = [int(re.search(r"\d+$", addr).group()) for addr, cell in (tab.get("cells") or {}).items()
+ if _col_number(re.match(r"[A-Z]+", addr).group()) == c0 and _cell_value(cell) not in (None, "")]
+ row = max(occupied, default=1)
+ return {"address": _addr(row, c0), "row": row, "column": c0}
+ if name == "worksheet.count_a":
+ return sum(1 for row in _addresses(rect) for addr in row
+ if ((tab.get("cells") or {}).get(addr) or {}).get("kind") == "formula"
+ or _cell_value((tab.get("cells") or {}).get(addr)) not in (None, ""))
+ if name == "worksheet.count_if":
+ wanted = str(args.get("criterion") if args.get("criterion") is not None else "").casefold()
+ return sum(1 for row in _addresses(rect) for addr in row
+ if str(_cell_value((tab.get("cells") or {}).get(addr)) or "").casefold() == wanted)
+ if name == "worksheet.geometry.read":
+ r0, c0, r1, c1 = rect
+ return {"columns": {_letters(col): (tab.get("colWidths") or {}).get(_letters(col), sheets.DEFAULT_COL_WIDTH)
+ for col in range(c0, c1 + 1)},
+ "rows": {str(row): (tab.get("rowHeights") or {}).get(str(row), tab.get("rowHeight", 24))
+ for row in range(r0, r1 + 1)}}
+ if name == "worksheet.hidden.read":
+ r0, c0, r1, c1 = rect
+ hidden_rows, hidden_cols = set(tab.get("hiddenRows") or ()), set(tab.get("hiddenCols") or ())
+ return {"rows": {str(row): row in hidden_rows for row in range(r0, r1 + 1)},
+ "columns": {_letters(col): col in hidden_cols for col in range(c0, c1 + 1)}}
+ if name == "worksheet.freeze.read":
+ return copy.deepcopy(tab.get("freeze") or {"rows": 0, "cols": 0})
+ if name == "worksheet.gridlines.read":
+ return (tab.get("view") or {}).get("gridlines", True)
+ if name == "application.calculation.read":
+ return (book.get("calculation") or {}).get("mode", "automatic")
+ if name == "workbook.names.item":
+ wanted = str(args.get("name") or "").casefold()
+ rows = book.get("namedRanges") or []
+ if isinstance(rows, dict):
+ rows = [{"name": key, "ref": value} for key, value in rows.items()]
+ for row in rows:
+ if isinstance(row, dict) and str(row.get("name") or "").casefold() == wanted:
+ return copy.deepcopy(row)
+ raise CompatibilityRefusal("sheet_compat_name_missing", "That workbook named range does not exist.")
+ if name == "workbook.save":
+ return {"rev": int(book.get("rev") or 0), "saved": True}
+ if name == "workbook.read_only":
+ return False
+ if name == "workbook.export_xlsx":
+ from core import sheet_import
+ handle, path = tempfile.mkstemp(prefix="loopable-sheet-export-", suffix=".xlsx")
+ os.close(handle)
+ try:
+ sheet_import.export_imported_workbook(book, path)
+ _apply_canonical_export_styles(book, path, sheet_import)
+ with open(path, "rb") as source:
+ payload = base64.b64encode(source.read()).decode("ascii")
+ return {"contentType": "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
+ "filename": str(args.get("filename") or (book.get("name") or "workbook") + ".xlsx"),
+ "base64": payload}
+ finally:
+ try:
+ os.remove(path)
+ except OSError:
+ pass
+ raise CompatibilityRefusal("sheet_compat_operation_unimplemented", name + " is declared but has no planner.")
+
+
+def _xlsx_colour(value, palette):
+ """One canonical colour token as openpyxl's opaque ARGB value."""
+ raw = str(palette.get(str(value or ""), value or "") or "").strip().lstrip("#")
+ if re.fullmatch(r"[0-9a-fA-F]{6}", raw):
+ return "FF" + raw.upper()
+ if re.fullmatch(r"[0-9a-fA-F]{8}", raw):
+ return raw.upper()
+ return None
+
+
+def _xlsx_number_format(spec):
+ """Render the closed canonical number-format vocabulary without guessing custom codes."""
+ spec = spec if isinstance(spec, dict) else {}
+ kind = str(spec.get("kind") or "general")
+ if kind == "code":
+ return str(spec.get("code") or "General")
+ if kind == "date":
+ return "yyyy-mm-dd"
+ decimals = max(0, int(spec.get("decimals") or 0))
+ places = ("." + ("0" * decimals)) if decimals else ""
+ grouping = "#,##0" if spec.get("thousands") else "0"
+ if kind == "number":
+ return grouping + places
+ if kind == "percent":
+ return "0" + places + "%"
+ if kind == "currency":
+ symbols = {"USD": "$", "EUR": "€", "GBP": "£", "JPY": "¥"}
+ symbol = symbols.get(str(spec.get("currency") or "USD").upper(),
+ str(spec.get("currency") or "USD").upper() + " ")
+ return symbol + grouping + places
+ return "General"
+
+
+def _apply_canonical_export_styles(book, path, sheet_import):
+ """Add canonical Sheet styles where the import exporter has no source style table.
+
+ ``sheet_import.export_imported_workbook`` owns exact imported OOXML and must stay untouched.
+ A workbook authored through the compatibility facade has no ``stylesXml``, however, so that
+ seam intentionally emits defaults. In that one bounded case this overlay projects the
+ canonical style bags into the generated xlsx. Imported workbooks keep the lossless source
+ style path byte-for-byte and never pass through openpyxl a second time.
+ """
+ document = sheets.migrate_workbook(book, lossless=True)
+ if ((document.get("excel") or {}).get("stylesXml") or ""):
+ return
+
+ import openpyxl
+ from openpyxl.styles import Side
+
+ output = openpyxl.load_workbook(path)
+ size_steps = {"s": 9, "m": sheets.DEFAULT_FONT_PT, "l": 12, "xl": 14}
+ for tab in document.get("sheets") or []:
+ ws = output[str(tab.get("name") or "Sheet")[:31]]
+ for address, source in (tab.get("cells") or {}).items():
+ style = (source or {}).get("style") if isinstance(source, dict) else None
+ if not isinstance(style, dict) or not style:
+ continue
+ target = ws[address]
+
+ font = copy.copy(target.font)
+ font.bold = bool(style.get("bold"))
+ font.italic = bool(style.get("italic"))
+ font.underline = "single" if style.get("underline") else None
+ if style.get("font"):
+ font.name = str(style["font"])
+ size = style.get("size")
+ if isinstance(size, str):
+ font.sz = size_steps.get(size, sheets.DEFAULT_FONT_PT)
+ elif not isinstance(size, bool) and isinstance(size, (int, float)):
+ font.sz = float(size)
+ colour = _xlsx_colour(style.get("color"), sheet_import.PALETTE)
+ if colour:
+ font.color = colour
+ target.font = font
+
+ fill = _xlsx_colour(style.get("fill"), sheet_import.PALETTE)
+ if fill:
+ painted = copy.copy(target.fill)
+ painted.fill_type = "solid"
+ painted.fgColor = fill
+ painted.patternType = "solid"
+ target.fill = painted
+
+ alignment = copy.copy(target.alignment)
+ if style.get("align"):
+ alignment.horizontal = style["align"]
+ if style.get("valign"):
+ alignment.vertical = {"middle": "center"}.get(style["valign"], style["valign"])
+ alignment.wrap_text = bool(style.get("wrap"))
+ alignment.indent = int(style.get("indent") or 0)
+ alignment.text_rotation = int(style.get("textRotation") or 0)
+ target.alignment = alignment
+
+ borders = style.get("border") or {}
+ if borders:
+ border = copy.copy(target.border)
+ line = Side(style="thin", color="FFCBD1DC")
+ for side in ("top", "right", "bottom", "left"):
+ if borders.get(side):
+ setattr(border, side, line)
+ target.border = border
+ if style.get("format"):
+ target.number_format = _xlsx_number_format(style["format"])
+ output.save(path)
+ output.close()
+
+
+def _tab_property(tab, key, value):
+ candidate = copy.deepcopy(tab)
+ if value in (None, {}, []):
+ candidate.pop(key, None)
+ else:
+ candidate[key] = copy.deepcopy(value)
+ cleaned = sheets._clean_sheet(candidate, lossless=True)
+ if key in cleaned:
+ tab[key] = copy.deepcopy(cleaned[key])
+ else:
+ tab.pop(key, None)
+
+
+def _mutate(book, name, args, calculator=None):
+ touched = {}
+ result = None
+ tab = _tab(book, args.get("sheet")) if args.get("sheet") else None
+ rect = _range(args.get("range")) if args.get("range") else None
+ cells = tab.setdefault("cells", {}) if tab is not None else None
+
+ def mark(address):
+ touched.setdefault(str(tab.get("id") or tab.get("name")), []).append(address)
+
+ if name in ("range.values.write", "range.formulas.write", "range.styles.write", "range.number_formats.write"):
+ addresses = _addresses(rect)
+ height, width = len(addresses), len(addresses[0])
+ values = _matrix(args.get("values"), height, width, name)
+ for row_at, row in enumerate(addresses):
+ for col_at, address in enumerate(row):
+ prior, value = cells.get(address), values[row_at][col_at]
+ if name == "range.values.write":
+ after = _literal(value, prior)
+ elif name == "range.formulas.write":
+ after = _formula(value, prior, address, r1c1=str(args.get("mode") or "").lower() == "r1c1")
+ elif name == "range.styles.write":
+ style = sheets._clean_style(value)
+ after = copy.deepcopy(prior) if isinstance(prior, dict) else {"kind": "empty", "state": "typed"}
+ if style:
+ after["style"] = style
+ else:
+ after.pop("style", None)
+ after = sheets.clean_cell(after, addr=address)
+ else:
+ fmt = None if value in (None, "", "General", "general") else {"kind": "code", "code": str(value)}
+ after = copy.deepcopy(prior) if isinstance(prior, dict) else {"kind": "empty", "state": "typed"}
+ style = dict(after.get("style") or {})
+ if fmt:
+ style["format"] = fmt
+ else:
+ style.pop("format", None)
+ if style:
+ after["style"] = style
+ else:
+ after.pop("style", None)
+ after = sheets.clean_cell(after, addr=address)
+ if after is None:
+ cells.pop(address, None)
+ else:
+ cells[address] = after
+ mark(address)
+ result = {"cells": height * width}
+ elif name == "range.clear_contents":
+ for row in _addresses(rect):
+ for address in row:
+ after = _literal(None, cells.get(address))
+ if after is None:
+ cells.pop(address, None)
+ else:
+ cells[address] = after
+ mark(address)
+ result = {"cleared": sum(len(row) for row in _addresses(rect))}
+ elif name == "range.fill_down":
+ addresses = _addresses(rect)
+ sources = {address: copy.deepcopy(cells.get(address)) for address in addresses[0]}
+ for row in addresses[1:]:
+ for column_at, address in enumerate(row):
+ origin = addresses[0][column_at]
+ after = _translate_formula(sources[origin], origin, address)
+ if after is None:
+ cells.pop(address, None)
+ else:
+ cells[address] = after
+ mark(address)
+ result = {"filled": max(0, len(addresses) - 1) * len(addresses[0])}
+ elif name == "range.replace":
+ if str(args.get("look_at") or "whole").lower() != "whole":
+ raise CompatibilityRefusal("sheet_compat_replace_mode", "Only exact whole-cell replacement is supported.")
+ wanted, replacement = args.get("what"), args.get("replacement")
+ match_case = bool(args.get("match_case"))
+ matched = 0
+ for row in _addresses(rect):
+ for address in row:
+ prior = cells.get(address)
+ if not isinstance(prior, dict) or prior.get("kind") == "formula":
+ continue
+ value = _cell_value(prior)
+ equal = value == wanted if match_case else str(value or "").casefold() == str(wanted or "").casefold()
+ if equal:
+ cells[address] = _literal(replacement, prior)
+ mark(address)
+ matched += 1
+ result = {"replaced": matched}
+ elif name in ("range.copy", "range.paste_values"):
+ source = _range(args.get("source_range"))
+ source_tab = _tab(book, args.get("source_sheet") or args.get("sheet"))
+ source_addresses, target_addresses = _addresses(source), _addresses(rect)
+ if (len(source_addresses), len(source_addresses[0])) != (len(target_addresses), len(target_addresses[0])):
+ raise CompatibilityRefusal("sheet_compat_copy_shape", "Source and target copy ranges must have the same shape.")
+ mode = "values" if name == "range.paste_values" else str(args.get("mode") or "all").lower()
+ if mode not in ("all", "values", "formulas"):
+ raise CompatibilityRefusal("sheet_compat_copy_mode", "Copy mode must be all, values or formulas.")
+ source_cells = copy.deepcopy(source_tab.get("cells") or {})
+ for rr, row in enumerate(target_addresses):
+ for cc, address in enumerate(row):
+ origin = source_addresses[rr][cc]
+ source_cell, prior = source_cells.get(origin), cells.get(address)
+ if mode == "values":
+ after = _literal(_cell_value(source_cell), prior)
+ elif mode == "formulas":
+ after = _translate_formula(source_cell, origin, address) if (source_cell or {}).get("kind") == "formula" else prior
+ else:
+ after = _translate_formula(source_cell, origin, address)
+ if after is None:
+ cells.pop(address, None)
+ else:
+ cells[address] = after
+ mark(address)
+ result = {"copied": sum(len(row) for row in target_addresses), "mode": mode}
+ elif name == "worksheet.geometry.write":
+ r0, c0, r1, c1 = rect
+ if "column_width" in args:
+ widths = dict(tab.get("colWidths") or {})
+ for col in range(c0, c1 + 1):
+ widths[_letters(col)] = args["column_width"]
+ _tab_property(tab, "colWidths", widths)
+ if "row_height" in args:
+ heights = dict(tab.get("rowHeights") or {})
+ for row in range(r0, r1 + 1):
+ heights[str(row)] = args["row_height"]
+ _tab_property(tab, "rowHeights", heights)
+ result = {"range": _range_text(rect)}
+ elif name == "worksheet.hidden.write":
+ r0, c0, r1, c1 = rect
+ axis = str(args.get("axis") or "rows").lower()
+ hidden = bool(args.get("hidden"))
+ if axis not in ("rows", "columns"):
+ raise CompatibilityRefusal("sheet_compat_hidden_axis", "Hidden state needs rows or columns.")
+ key, members = ("hiddenRows", range(r0, r1 + 1)) if axis == "rows" else ("hiddenCols", range(c0, c1 + 1))
+ values = set(tab.get(key) or ())
+ values = values.union(members) if hidden else values.difference(members)
+ _tab_property(tab, key, sorted(values))
+ result = {"axis": axis, "hidden": hidden}
+ elif name == "worksheet.freeze.write":
+ _tab_property(tab, "freeze", {"rows": args.get("rows", 0), "cols": args.get("cols", 0)})
+ result = copy.deepcopy(tab.get("freeze") or {"rows": 0, "cols": 0})
+ elif name == "worksheet.gridlines.write":
+ view = dict(tab.get("view") or {})
+ view["gridlines"] = bool(args.get("shown"))
+ _tab_property(tab, "view", view)
+ result = view["gridlines"]
+ elif name == "application.calculation.write":
+ mode = str(args.get("mode") or "").lower()
+ if mode not in ("manual", "automatic"):
+ raise CompatibilityRefusal("sheet_compat_calculation_mode", "Calculation mode must be manual or automatic.")
+ calculation = dict(book.get("calculation") or {})
+ calculation["mode"] = mode
+ book["calculation"] = calculation
+ result = mode
+ elif name in ("workbook.calculate_full", "workbook.refresh_all"):
+ if not callable(calculator):
+ raise CompatibilityRefusal("sheet_compat_calculator_required", "This Sheet host has no formula evaluator configured.")
+ calculator(book)
+ stamp = write_execution.utcnow().replace(microsecond=0).isoformat()
+ calculation = dict(book.get("calculation") or {})
+ calculation["lastCalculatedAt"] = stamp
+ book["calculation"] = calculation
+ if name == "workbook.refresh_all":
+ book["refreshed_at"] = stamp
+ result = {"calculatedAt": stamp}
+ elif name == "workbook.names.write":
+ named, ref = str(args.get("name") or "").strip(), str(args.get("refers_to") or "").strip()
+ if not named or not ref:
+ raise CompatibilityRefusal("sheet_compat_name_invalid", "A named range needs a name and refers_to address.")
+ rows = book.get("namedRanges") or []
+ if isinstance(rows, dict):
+ rows = [{"name": key, "ref": value} for key, value in rows.items()]
+ kept = [copy.deepcopy(row) for row in rows if isinstance(row, dict)
+ and str(row.get("name") or "").casefold() != named.casefold()]
+ kept.append({"name": named, "ref": ref})
+ book["namedRanges"] = kept
+ result = copy.deepcopy(kept[-1])
+ else:
+ raise CompatibilityRefusal("sheet_compat_operation_unimplemented", name + " is declared but has no patcher.")
+
+ if tab is not None:
+ tab["usedExtent"] = sheets._used_extent(cells or {}, (tab.get("excel") or {}).get("blankStyles"))
+ if tab["usedExtent"]:
+ tab["cols"] = max(int(tab.get("cols") or 1), int(tab["usedExtent"]["col1"]) + 1)
+ tab["rows"] = max(int(tab.get("rows") or 1), int(tab["usedExtent"]["row1"]) + 1)
+ return result, touched
+
+
+def execute(st, workbook_id, operation, arguments=None, *, actor="", actor_name="", calculator=None):
+ """Execute one declared operation through the shared planner/patcher.
+
+ Unsupported and malformed operation-level requests are checked before ``st.update`` is ever
+ reachable. A supported write validates its expected revision again under the store lock.
+ """
+ if arguments is None:
+ args = {}
+ elif not isinstance(arguments, dict):
+ raise CompatibilityRefusal(
+ "sheet_compat_arguments_invalid", "Sheet automation arguments must be an object.")
+ else:
+ args = dict(arguments)
+ spec = _preflight(operation, args)
+ if spec["access"] != "write":
+ book = sheets.get_workbook(st, workbook_id)
+ return {"operation": operation, "rev": int(book.get("rev") or 0),
+ "result": _read(book, operation, args)}
+
+ expected = _write_required(args)
+ wid = sheets._slug_id(workbook_id)
+ saved, envelope = {}, {}
+
+ def apply(state):
+ if not isinstance(state, dict) or not state.get("id"):
+ raise sheets.SheetRefusal("workbook_missing", "That workbook no longer exists.")
+ current = int(state.get("rev") or 0)
+ if current != expected:
+ raise sheets.SheetStaleRefusal(
+ "workbook_revision_stale",
+ "That workbook changed somewhere else, so this automation operation was not applied.",
+ rev_current=current)
+ book = copy.deepcopy(state)
+ result, touched = _mutate(book, operation, args, calculator=calculator)
+ new_rev = current + 1
+ by_id = {str(tab.get("id")): tab for tab in book.get("sheets") or []}
+ for tab_id, addresses in touched.items():
+ if tab_id in by_id:
+ sheets._append_write_log(by_id[tab_id], new_rev, sorted(set(addresses)),
+ actor=actor, actor_name=actor_name)
+ book["rev"] = new_rev
+ book["updated_at"] = write_execution.utcnow().replace(microsecond=0).isoformat()
+ sheets._json_safe(book, "workbook")
+ saved.update(copy.deepcopy(book))
+ envelope.update({"operation": operation, "rev": new_rev, "result": copy.deepcopy(result)})
+ return book, envelope
+
+ out = sheets._persist(st, sheets.workbook_key(wid), apply)
+ sheets._index_upsert(st, saved)
+ return out
diff --git a/platform/core/sheet_events.py b/platform/core/sheet_events.py
new file mode 100644
index 0000000000000000000000000000000000000000..5b90b44a0f8e56ef658b4268104260755757156e
--- /dev/null
+++ b/platform/core/sheet_events.py
@@ -0,0 +1,101 @@
+"""Value-only workbook commit notifications; the API supplies the Automation consumer.
+
+No row or cell values cross this boundary. Publication happens after the synchronous store
+commit, outside its lock. A new workbook/tab is initial backfill, not a watched change.
+"""
+from __future__ import annotations
+
+import logging
+import re
+
+COMMIT_OBSERVER = None
+_ADDRESS = re.compile(r"^([A-Z]+)([1-9][0-9]*)$")
+
+
+def _value(cell):
+ cell = cell if isinstance(cell, dict) else {}
+ # Paint, formula text, refresh stamps and provenance do not change the computed insight.
+ return cell.get("value"), cell.get("error"), cell.get("refusal")
+
+
+def _binding_keys(cell, address):
+ binding = (cell or {}).get("binding") or {}
+ if not binding:
+ return set()
+ keys = {address}
+ for key in ("id", "key", "name"):
+ if binding.get(key):
+ keys.add(str(binding[key]))
+ field = binding.get("field") or binding.get("aggregate")
+ if field:
+ keys.add(str(field))
+ if binding.get("database"):
+ keys.add(f'{binding["database"]}.{field}')
+ return keys
+
+
+def _ranges(addresses):
+ """Compress adjacent rows only; sparse gaps must never trigger an unrelated watch."""
+ columns = {}
+ for address in addresses:
+ match = _ADDRESS.fullmatch(address)
+ if match:
+ columns.setdefault(match[1], []).append(int(match[2]))
+ ranges = []
+ for column, rows in sorted(columns.items()):
+ start = end = None
+ for row in sorted(rows):
+ if start is not None and row == end + 1:
+ end = row
+ continue
+ if start is not None:
+ ranges.append(f"{column}{start}" if start == end else f"{column}{start}:{column}{end}")
+ start = end = row
+ if start is not None:
+ ranges.append(f"{column}{start}" if start == end else f"{column}{start}:{column}{end}")
+ return ranges
+
+
+def committed_changes(before, after):
+ """Derive at most one event per existing tab from the transaction's actual before/after."""
+ if not isinstance(before, dict) or not isinstance(after, dict):
+ return []
+ workbook = str(after.get("id") or "")
+ if not workbook or before.get("id") != workbook:
+ return []
+ prior = {str(tab.get("id")): tab for tab in before.get("sheets") or []}
+ revision = int(after.get("rev") or 0)
+ events = []
+ for tab in after.get("sheets") or []:
+ sheet = str(tab.get("id") or "")
+ if sheet not in prior:
+ continue
+ old, new = prior[sheet].get("cells") or {}, tab.get("cells") or {}
+ changed, bindings = set(), set()
+ for address in old.keys() | new.keys():
+ left, right = old.get(address) or {}, new.get(address) or {}
+ value_changed = _value(left) != _value(right)
+ binding_changed = (left.get("binding") or {}) != (right.get("binding") or {})
+ if value_changed:
+ changed.add(address)
+ if value_changed or binding_changed:
+ bindings.update(_binding_keys(left, address) | _binding_keys(right, address))
+ if not changed and not bindings:
+ continue
+ events.append({
+ "kind": "sheet_insight_changed", "workbookId": workbook, "sheetId": sheet,
+ "batchId": f"{workbook}:{revision}:{sheet}", "revision": revision,
+ "committed": True, "changeKinds": (["value"] if changed else []) + (["binding"] if bindings else []),
+ "ranges": _ranges(changed), "bindings": sorted(bindings), "recordCount": len(changed),
+ })
+ return events
+
+
+def publish(st, events):
+ if events and callable(COMMIT_OBSERVER):
+ try:
+ COMMIT_OBSERVER(st, events)
+ except Exception:
+ # A consumer failure cannot turn a committed edit into a purported failed save.
+ # Never log cell contents, credentials, or the consumer's raw exception message.
+ logging.getLogger(__name__).error("Sheet commit notification failed after persistence")
diff --git a/platform/core/sheet_excel.py b/platform/core/sheet_excel.py
new file mode 100644
index 0000000000000000000000000000000000000000..35084e5b53c63ff5732a3323690b0c652b2f3ab6
--- /dev/null
+++ b/platform/core/sheet_excel.py
@@ -0,0 +1,65 @@
+"""Windows Excel adapter and shared spreadsheet-backend protocols.
+
+Importing this module is safe on Linux. The optional ``win32com`` dependency is loaded only when
+``excel_application`` is selected at the client boundary. Excel itself already implements the
+canonical object hierarchy, so the adapter returns the genuine COM Application rather than
+maintaining a second imitation of it.
+"""
+
+from __future__ import annotations
+
+from typing import Any, Protocol, runtime_checkable
+
+
+@runtime_checkable
+class OperationBackend(Protocol):
+ """Transport used by the Loopable facade."""
+
+ def manifest_bytes(self) -> bytes: ...
+
+ def open_workbook(self, identifier: str) -> dict[str, Any]: ...
+
+ def execute(self, workbook_id: str, operation: str, arguments: dict[str, Any]) -> dict[str, Any]: ...
+
+
+@runtime_checkable
+class SpreadsheetApplication(Protocol):
+ """The common boundary accepted by data-only workbook workflow functions."""
+
+ Workbooks: Any
+ WorksheetFunction: Any
+
+ def CalculateFull(self) -> Any: ...
+
+
+def excel_application(*, visible: bool | None = None, dispatch_ex: bool = True):
+ """Return a genuine Excel COM Application, loading pywin32 only on Windows.
+
+ Process/UI settings remain the Excel caller's responsibility. They are not projected onto
+ Loopable, where the public capability manifest names them as refusal-only operations.
+ """
+ try:
+ import win32com.client as win32
+ except (ImportError, ModuleNotFoundError) as exc: # pragma: no cover - Windows-only path
+ raise RuntimeError(
+ "The Excel backend needs Windows, desktop Excel and pywin32. Choose backend='loopable' on this host."
+ ) from exc
+ application = (win32.DispatchEx if dispatch_ex else win32.Dispatch)("Excel.Application")
+ if visible is not None:
+ application.Visible = bool(visible)
+ return application
+
+
+def select_application(backend: str, **options):
+ """Select Excel or Loopable once, outside business workflow logic."""
+ kind = str(backend or "").strip().lower()
+ if kind == "excel":
+ return excel_application(
+ visible=options.get("visible"), dispatch_ex=options.get("dispatch_ex", True))
+ if kind == "loopable":
+ from sdk.loopable_sheet import Application
+ transport = options.get("transport")
+ if transport is None:
+ raise ValueError("The Loopable backend needs an operation transport.")
+ return Application(transport)
+ raise ValueError("Spreadsheet backend must be 'excel' or 'loopable'.")
diff --git a/platform/core/sheet_import.py b/platform/core/sheet_import.py
index c1c433f81775ceec046d623622b5e4e39ed5e3d3..1bc4d6ce5726f9cf67e7db286b78602b3944eeb1 100644
--- a/platform/core/sheet_import.py
+++ b/platform/core/sheet_import.py
@@ -44,6 +44,9 @@ import re
import zipfile
import hashlib
import json
+import math
+import os
+import tempfile
import xml.etree.ElementTree as ET
from datetime import date, datetime, time, timezone
@@ -55,6 +58,15 @@ from core import sheets
#: parse in reasonable time.
MAX_IMPORT_BYTES = 25 * 1024 * 1024
+#: An xlsx is a ZIP and the compressed upload limit is therefore not a parse budget. These
+#: ceilings are checked from the central directory before openpyxl is imported or called. They
+#: deliberately leave more room than the 64 MiB stored document: worksheet XML repeats tags and
+#: shared strings before the importer turns them into the compact JSON document.
+MAX_ARCHIVE_UNCOMPRESSED_BYTES = 256 * 1024 * 1024
+MAX_ARCHIVE_MEMBER_BYTES = 128 * 1024 * 1024
+MAX_ARCHIVE_MEMBERS = 10_000
+MAX_ARCHIVE_COMPRESSION_RATIO = 250
+
#: C4's job states, in the order a job walks them. `refused` is the exit that replaces `done`.
IMPORT_STATES = ("reading", "cells", "formats", "recalc", "done", "refused")
@@ -126,10 +138,6 @@ PALETTE = {
DEFAULT_INK = "--lp-ink"
DEFAULT_FILL = "--lp-surface"
-#: The four font steps A1 froze, in points. A size is a STEP and never a pixel count, for
-#: `_clean_style`'s reason: a stored pixel size is a layout decision that outlives the layout.
-_SIZE_STEPS = ((9.5, "s"), (12.5, "m"), (16.5, "l"))
-
#: Excel's horizontal alignments that have a home in `CELL_ALIGNS`. Everything else is named in
#: `unsupported` rather than guessed at: `justify` and `distributed` are not `left`.
_ALIGNS = {"left": "left", "center": "center", "centre": "center", "right": "right"}
@@ -314,22 +322,13 @@ def classify_format(code):
# ── one cell's style ─────────────────────────────────────────────────────────────────────────
-def _size_step(points):
- if not isinstance(points, (int, float)):
- return ""
- for limit, step in _SIZE_STEPS:
- if points < limit:
- return step
- return "xl"
-
-
def style_of(cell, notes):
"""The C2 style for one openpyxl cell, or `{}` when nothing about it is worth storing.
- ⚠ FONT FAMILY IS DROPPED IN SILENCE AND THAT IS THE DESIGN, not an omission: the app is Inter
- everywhere (DESIGN.md), so carrying Arial would either be ignored or would put a second
- typeface on a screen the design forbids. It is still NAMED in the report, once, because a
- person who imported an Arial workbook should be told why it does not look like Arial.
+ Imported workbook styling is cell data, not product chrome. A supported font family and the
+ authored point size therefore enter the normal display model. Anything that model cannot
+ express still survives exactly through the cell's style id plus the workbook's original
+ styles XML; the import report names the display fallback instead of pretending it was exact.
"""
out = {}
font = getattr(cell, "font", None)
@@ -340,9 +339,17 @@ def style_of(cell, notes):
out["italic"] = True
if font.u:
out["underline"] = True
- step = _size_step(font.sz)
- if step and step != "m":
- out["size"] = step
+ size = getattr(font, "sz", None)
+ if not isinstance(size, bool) and isinstance(size, (int, float)) and math.isfinite(size):
+ points = max(sheets.MIN_FONT_PT, min(sheets.MAX_FONT_PT, float(size)))
+ if points != sheets.DEFAULT_FONT_PT:
+ out["size"] = points
+ name = str(getattr(font, "name", None) or "").strip()
+ if name in sheets.CELL_FONTS:
+ if name != sheets.DEFAULT_CELL_FONT:
+ out["font"] = name
+ elif name:
+ notes.add("the " + name[:60] + " font is kept for export but uses the default face on screen")
argb = argb_of(font.color)
if argb:
token = quantize(argb)
@@ -902,6 +909,134 @@ def _geometry_of(ws, notes):
return cols, rows, widths, freeze, sorted(set(hidden_rows)), sorted(set(hidden_cols)), heights
+def _view_of(ws):
+ """Explicit worksheet display state in the existing tab ``view`` vocabulary.
+
+ ``None`` means the source omitted the property and is intentionally different from an
+ explicit ``True`` or 100 percent. That distinction is what lets compact product defaults
+ apply only when the file said nothing while an authored value always wins.
+ """
+ source = getattr(ws, "sheet_view", None)
+ if source is None:
+ return {}
+ out = {}
+ if isinstance(getattr(source, "showGridLines", None), bool):
+ out["gridlines"] = source.showGridLines
+ if isinstance(getattr(source, "showRowColHeaders", None), bool):
+ out["headings"] = source.showRowColHeaders
+ zoom = getattr(source, "zoomScale", None)
+ if not isinstance(zoom, bool) and isinstance(zoom, (int, float)) and math.isfinite(zoom):
+ out["zoom"] = zoom
+ page_view = {"normal": "normal", "pageBreakPreview": "pageBreak",
+ "pageLayout": "pageLayout"}.get(getattr(source, "view", None))
+ if page_view:
+ out["pageView"] = page_view
+ return out
+
+
+def _archive_refusal(code, message):
+ raise ImportRefusal(code, message)
+
+
+def _address_in_document(ref):
+ point = _corner(ref)
+ return (point is not None and 0 <= point[0] < sheets.DOCUMENT_MAX_COLS
+ and 0 <= point[1] < sheets.DOCUMENT_MAX_ROWS)
+
+
+def _guard_worksheet_xml(stream):
+ """Stream one worksheet and reject impossible dimensions/cell counts before openpyxl.
+
+ ElementTree sees only bounded, central-directory-admitted bytes. It is used here as an
+ admission parser, not as the workbook parser: no cell value, formula or style is interpreted.
+ """
+ dimension_seen = False
+ cells = 0
+ try:
+ for _event, element in ET.iterparse(stream, events=("end",)):
+ tag = str(element.tag).rsplit("}", 1)[-1]
+ if tag == "dimension":
+ ref = str(element.get("ref") or "").replace("$", "").upper()
+ parts = ref.split(":")
+ if (len(parts) not in (1, 2) or not all(_address_in_document(part) for part in parts)
+ or (len(parts) == 2 and rect_of(ref) is None)):
+ _archive_refusal(
+ "import_declared_dimension_invalid",
+ "A worksheet declares cells outside Excel's supported dimensions, so it cannot be imported.")
+ dimension_seen = True
+ elif tag == "c":
+ cells += 1
+ ref = str(element.get("r") or "").replace("$", "").upper()
+ if not _address_in_document(ref):
+ _archive_refusal(
+ "import_declared_dimension_invalid",
+ "A worksheet contains a cell outside Excel's supported dimensions, so it cannot be imported.")
+ if cells > sheets._MAX_CELLS_PER_SHEET:
+ _archive_refusal(
+ "sheet_too_many_cells",
+ "A worksheet declares more than " + str(sheets._MAX_CELLS_PER_SHEET)
+ + " populated cells, so it cannot be imported.")
+ element.clear()
+ except ImportRefusal:
+ raise
+ except ET.ParseError as exc:
+ raise ImportRefusal(
+ "import_unreadable",
+ "That file could not be read as a workbook. Save it again as .xlsx and try once more.") from exc
+ # A dimension element is optional in OOXML. When it is absent, the independently bounded
+ # cell-address walk above is the authority; absence must not invent a default declaration.
+ return dimension_seen
+
+
+def guard_import_archive(path):
+ """Admit a bounded OOXML archive before any openpyxl workbook parse occurs."""
+ try:
+ byte_count = os.path.getsize(path)
+ except OSError as exc:
+ raise ImportRefusal("import_unreadable", "That workbook file could not be read.") from exc
+ if byte_count > MAX_IMPORT_BYTES:
+ raise ImportRefusal(
+ "import_too_large",
+ "That file is larger than the " + str(MAX_IMPORT_BYTES // (1024 * 1024))
+ + " MB import limit, so it cannot be read.")
+ try:
+ with zipfile.ZipFile(path) as archive:
+ members = archive.infolist()
+ if len(members) > MAX_ARCHIVE_MEMBERS:
+ _archive_refusal("import_zip_bloat",
+ "That workbook archive holds too many parts to import safely.")
+ names = [entry.filename for entry in members]
+ if len(set(names)) != len(names):
+ _archive_refusal("import_zip_bloat",
+ "That workbook archive repeats a part name, so it cannot be imported safely.")
+ total = sum(max(0, int(entry.file_size)) for entry in members)
+ if total > MAX_ARCHIVE_UNCOMPRESSED_BYTES:
+ _archive_refusal("import_zip_bloat",
+ "That workbook expands beyond the safe import limit.")
+ for entry in members:
+ if entry.flag_bits & 0x1:
+ _archive_refusal("import_encrypted",
+ "Password protected workbooks cannot be imported.")
+ if entry.file_size > MAX_ARCHIVE_MEMBER_BYTES:
+ _archive_refusal("import_zip_bloat",
+ "One workbook part expands beyond the safe import limit.")
+ ratio = entry.file_size / max(1, entry.compress_size)
+ if entry.file_size and ratio > MAX_ARCHIVE_COMPRESSION_RATIO:
+ _archive_refusal("import_zip_bloat",
+ "That workbook archive is compressed too densely to import safely.")
+ worksheets = [entry for entry in members
+ if re.fullmatch(r"xl/worksheets/sheet[^/]*\.xml", entry.filename)]
+ for entry in worksheets:
+ with archive.open(entry) as stream:
+ _guard_worksheet_xml(stream)
+ except ImportRefusal:
+ raise
+ except (OSError, zipfile.BadZipFile, zipfile.LargeZipFile) as exc:
+ raise ImportRefusal(
+ "import_unreadable",
+ "That file could not be read as a workbook. Save it again as .xlsx and try once more.") from exc
+
+
def _print_of(ws):
"""Store print setup and inclusive print areas; export executes these settings later."""
setup = {key: getattr(ws.page_setup, key) for key in ws.page_setup.__attrs__
@@ -977,8 +1112,6 @@ def import_workbook(path, *, source="", progress=None):
⚠ `progress(state, fraction)` IS CALLED FROM WHATEVER THREAD RUNS THIS. It must not raise and
must not block; the job's registry entry is the only thing it touches.
"""
- import openpyxl
-
def _say(state, fraction):
if progress:
try:
@@ -987,6 +1120,11 @@ def import_workbook(path, *, source="", progress=None):
pass
_say("reading", 0.02)
+ # The guard is deliberately before both the import and the first load_workbook call. A
+ # hostile declared dimension or ZIP bomb must not reach openpyxl's object allocator.
+ guard_import_archive(path)
+ import openpyxl
+
try:
formulas = openpyxl.load_workbook(path, data_only=False, read_only=False,
keep_links=False)
@@ -1088,6 +1226,9 @@ def import_workbook(path, *, source="", progress=None):
tab["hiddenRows"] = hidden_rows
if hidden_cols:
tab["hiddenCols"] = hidden_cols
+ view = _view_of(ws)
+ if view:
+ tab["view"] = view
charts = _charts_of(ws, name, notes)
if name in chart_tabs:
tab["id"] = chart_tabs[name]["id"]
@@ -1103,11 +1244,6 @@ def import_workbook(path, *, source="", progress=None):
values.close()
except Exception: # noqa: BLE001
pass
- #: ⭐ THE FONT NOTE IS WORKBOOK-WIDE AND UNCONDITIONAL, because the answer is the same on
- #: every file: the app is one typeface, so no imported font survives and the reader is told
- #: once rather than per tab.
- book_notes.add("every font was replaced for display; original fonts and formats are kept for export")
-
# Original styles are shared once, with integer references on cells, so exact formats,
# theme colours, border weights and blank-cell styles survive without repeating XML per cell.
original = {}
@@ -1129,6 +1265,132 @@ def import_workbook(path, *, source="", progress=None):
}
+def export_imported_workbook(book, path):
+ """Export current cell values with an imported workbook's exact source styles.
+
+ This is the lossless import round-trip seam, not the general Sheet exporter: cells without an
+ imported ``excel.styleId`` use the generated default style. The original style table is
+ restored only after openpyxl has written a structurally valid workbook, so current values,
+ formulas, dimensions, visibility, freeze and view state come from the cleaned document while
+ source fills, fonts, alignments, borders and number formats keep their exact OOXML records.
+ """
+ import openpyxl
+
+ document = sheets.migrate_workbook(book, lossless=True)
+ target = str(path)
+ output = openpyxl.Workbook()
+ output.remove(output.active)
+ source_styles = ((document.get("excel") or {}).get("stylesXml") or "")
+ source_theme = ((document.get("excel") or {}).get("themeXml") or "")
+ style_ids = []
+ for tab in document.get("sheets") or []:
+ ws = output.create_sheet(str(tab.get("name") or "Sheet")[:31])
+ ws.sheet_state = tab.get("visibility", "visible")
+ by_addr = {}
+ for addr, cell in (tab.get("cells") or {}).items():
+ if not isinstance(cell, dict):
+ continue
+ target_cell = ws[addr]
+ if cell.get("kind") == "formula" and isinstance(cell.get("formula"), str):
+ target_cell.value = cell["formula"]
+ elif cell.get("kind") != "empty":
+ target_cell.value = cell.get("value", cell.get("text"))
+ style_id = (cell.get("excel") or {}).get("styleId")
+ if isinstance(style_id, int) and not isinstance(style_id, bool) and style_id >= 0:
+ by_addr[addr] = style_id
+ for addr, style_id in ((tab.get("excel") or {}).get("blankStyles") or {}).items():
+ if isinstance(style_id, int) and not isinstance(style_id, bool) and style_id >= 0:
+ ws[addr] # Ensure the style-only cell exists in sheetData before the OOXML patch.
+ by_addr.setdefault(addr, style_id)
+
+ # Exact Excel dimension records win. The canonical pixel maps remain the fallback for a
+ # document whose source metadata predates this import seam.
+ column_records = (tab.get("excel") or {}).get("columnDimensions") or {}
+ for key, record in column_records.items():
+ if not isinstance(record, dict):
+ continue
+ dim = ws.column_dimensions[str(key)]
+ if record.get("width") is not None:
+ dim.width = float(record["width"])
+ if record.get("hidden") is not None:
+ dim.hidden = str(record["hidden"]).lower() in ("1", "true")
+ for key, width in (tab.get("colWidths") or {}).items():
+ if key not in column_records:
+ ws.column_dimensions[key].width = 0 if width == 0 else max(0, (float(width) - 5) / 7)
+ row_records = (tab.get("excel") or {}).get("rowDimensions") or {}
+ for key, record in row_records.items():
+ if not isinstance(record, dict) or not str(key).isdigit():
+ continue
+ dim = ws.row_dimensions[int(key)]
+ if record.get("ht") is not None:
+ dim.height = float(record["ht"])
+ if record.get("hidden") is not None:
+ dim.hidden = str(record["hidden"]).lower() in ("1", "true")
+ for key, height in (tab.get("rowHeights") or {}).items():
+ if key not in row_records:
+ ws.row_dimensions[int(key)].height = float(height) * 3 / 4
+ for number in tab.get("hiddenRows") or []:
+ ws.row_dimensions[int(number)].hidden = True
+ for number in tab.get("hiddenCols") or []:
+ ws.column_dimensions[_addr(int(number) - 1, 0).rstrip("0123456789")].hidden = True
+
+ freeze = tab.get("freeze") or {}
+ if freeze.get("rows") or freeze.get("cols"):
+ ws.freeze_panes = _addr(int(freeze.get("cols") or 0), int(freeze.get("rows") or 0))
+ view = tab.get("view") or {}
+ if "gridlines" in view:
+ ws.sheet_view.showGridLines = bool(view["gridlines"])
+ if "headings" in view:
+ ws.sheet_view.showRowColHeaders = bool(view["headings"])
+ if "zoom" in view:
+ ws.sheet_view.zoomScale = int(view["zoom"])
+ if view.get("pageView") in ("normal", "pageBreak", "pageLayout"):
+ ws.sheet_view.view = {"normal": "normal", "pageBreak": "pageBreakPreview",
+ "pageLayout": "pageLayout"}[view["pageView"]]
+ style_ids.append(by_addr)
+
+ output.save(target)
+ output.close()
+ if not source_styles:
+ return target
+
+ namespace = "http://schemas.openxmlformats.org/spreadsheetml/2006/main"
+ ET.register_namespace("", namespace)
+ with zipfile.ZipFile(target) as source:
+ parts = [(entry, source.read(entry.filename)) for entry in source.infolist()]
+ patched = []
+ for entry, blob in parts:
+ match = re.fullmatch(r"xl/worksheets/sheet([0-9]+)\.xml", entry.filename)
+ if match and int(match.group(1)) <= len(style_ids):
+ wanted = style_ids[int(match.group(1)) - 1]
+ if wanted:
+ root = ET.fromstring(blob)
+ for cell in root.iter("{" + namespace + "}c"):
+ style_id = wanted.get(cell.get("r"))
+ if style_id is not None:
+ cell.set("s", str(style_id))
+ blob = ET.tostring(root, encoding="utf-8", xml_declaration=True)
+ elif entry.filename == "xl/styles.xml":
+ blob = source_styles.encode("utf-8")
+ elif entry.filename == "xl/theme/theme1.xml" and source_theme:
+ blob = source_theme.encode("utf-8")
+ patched.append((entry, blob))
+ handle, replacement = tempfile.mkstemp(prefix="sheet-export-", suffix=".xlsx",
+ dir=os.path.dirname(os.path.abspath(target)))
+ os.close(handle)
+ try:
+ with zipfile.ZipFile(replacement, "w") as archive:
+ for entry, blob in patched:
+ archive.writestr(entry, blob)
+ os.replace(replacement, target)
+ finally:
+ try:
+ os.remove(replacement)
+ except OSError:
+ pass
+ return target
+
+
def take_imported_values(book):
"""Pull every `imported_value` off the document and return `{(tab, addr): cached}`.
diff --git a/platform/core/sheet_templates.py b/platform/core/sheet_templates.py
index 9e375e0d35b55b2888cda0e52635a2726bd8e324..5c90a44816e4e764e9c720fd2fb6b752c83d1738 100644
--- a/platform/core/sheet_templates.py
+++ b/platform/core/sheet_templates.py
@@ -9,7 +9,7 @@ from copy import deepcopy
from datetime import date, datetime, timedelta, timezone
import re
-from core import sheets, user_tables
+from core import sheets, user_tables, write_execution
# Do not use connector_spec.connector_of: it only covers SPEC'd tables and calls
@@ -205,7 +205,7 @@ def save_sheet_template(st, workbook, key, *, name="", description="", created_b
databases.add(database)
record = {"key": key, "name": name.strip() or book.get("name") or key,
"description": description, "databases": sorted(databases), "workbook": book,
- "created_by": created_by, "created_at": datetime.now(timezone.utc).isoformat(),
+ "created_by": created_by, "created_at": write_execution.utcnow().isoformat(),
"byName": actor_name or created_by}
def save(state):
diff --git a/platform/core/sheets.py b/platform/core/sheets.py
index 1740e147df692032d8aa59f92c0a53d4b577d059..097b64011319254cdc743bc8596042c64f7288b7 100644
--- a/platform/core/sheets.py
+++ b/platform/core/sheets.py
@@ -60,8 +60,7 @@ import hashlib
import json
import math
import re
-import uuid
-from datetime import datetime, timezone
+from core import write_execution
from core.sheet_binding import (
BindingMetadataRefusal,
@@ -247,7 +246,7 @@ SHEET_CHART_STACKING = ("clustered", "stacked", "percent")
#:
#: The order is the order a cleaned tab reads in: identity, then A2's geometry, then C2's strata.
SHEET_TAB_KEYS = ("id", "name", "cols", "rows", "cells", "usedExtent", "colWidths", "freeze",
- "rules", "hiddenRows", "hiddenCols", "charts", "imported", "rowHeight", "rowHeights",
+ "rules", "rowGroups", "hiddenRows", "hiddenCols", "charts", "imported", "rowHeight", "rowHeights",
"visibility", "view", "print", "excel")
#: The keys above that `_clean_sheet` writes UNCONDITIONALLY. Every other member of
@@ -261,8 +260,17 @@ _MAX_HIDDEN = 5000
_MAX_CHARTS = 50
_MAX_UNSUPPORTED = 200
-#: A colour is a `--lp-*` TOKEN NAME, not a hue. Hex is allowed so a future palette door is not
-#: locked out, but nothing free form reaches a canvas `fillStyle` from a stored document.
+#: ⭐⭐ SDLC 64 R2 / CONTRACT C1 — THE ACCEPTED FORM OF A CELL COLOUR, AS ONE NAMED PATTERN.
+#: Mirrored BYTE FOR BYTE in `aios-web/web/src/sheet/types.ts::CELL_COLOUR_PATTERN`, and
+#: `verify_sheet.py`'s C1 leg compares the two source strings, so the client cannot start
+#: writing a form this cleaner would drop.
+CELL_COLOUR_PATTERN = r"^(--lp-[a-z0-9-]{1,40}|#[0-9a-f]{6})$"
+_CELL_COLOUR = re.compile(CELL_COLOUR_PATTERN)
+
+#: R2 / C1: CELL `color` / `fill` use CELL_COLOUR_PATTERN above. This older guard stays on
+#: `_clean_rule_style` and `_clean_scale`: their RULE_FILLS / RULE_INKS galleries remain token
+#: only. Its existing storage allowance is unchanged; a cell fill's custom door does not widen
+#: the conditional-format palettes or make either guard dead.
_COLOUR = re.compile(r"^[A-Za-z0-9#_-]{1,40}$")
_MAX_DECIMALS = 10
@@ -280,7 +288,7 @@ _MAX_DECIMALS = 10
#: sheet at the ceiling is still far above `_MAX_CELLS_PER_SHEET`, so content runs out first.
DEFAULT_SHEET_COLS = 200
DEFAULT_SHEET_ROWS = 10000
-DEFAULT_COL_WIDTH = 120
+DEFAULT_COL_WIDTH = 100
MIN_SHEET_COLS = 1
MIN_SHEET_ROWS = 1
MAX_SHEET_COLS = 702
@@ -367,7 +375,7 @@ class SheetStaleRefusal(SheetRefusal):
def _now():
- return datetime.now(timezone.utc).replace(microsecond=0).isoformat()
+ return write_execution.utcnow().replace(microsecond=0).isoformat()
def workbook_key(workbook_id):
@@ -399,6 +407,7 @@ def _read(st, key):
def _persist(st, key, mutate, seed=None):
"""Read-modify-write under the store's own lock, mirroring `chart_system._persist`."""
+ from core import sheet_events
result = {}
def apply(current):
@@ -408,9 +417,12 @@ def _persist(st, key, mutate, seed=None):
state.setdefault(k, copy.deepcopy(v))
next_state, value = mutate(state)
result["value"] = copy.deepcopy(value)
+ # A store retry replaces this delta; no observer runs for a discarded transaction.
+ result["events"] = sheet_events.committed_changes(current, next_state)
return next_state
st.update(key, apply, flush="sync")
+ sheet_events.publish(st, result.get("events", []))
return result.get("value")
@@ -530,6 +542,29 @@ def _clean_format(raw):
return out
+def _normalise_colour(token):
+ """One cell colour in C1's accepted form, or "" when nothing survives.
+
+ ⭐⭐ C1's FORM IS LOWERCASE `#rrggbb`, AND THAT IS WHAT IS STORED, SO A RELOAD COMPARES
+ BYTE FOR BYTE EQUAL TO THE SAVE. A colour already in that form is returned unchanged:
+ normalisation is the identity on everything the client writes, which is the whole point.
+
+ ⚠ THE TWO COERCIONS ARE A RESCUE, NOT A CONVENIENCE. `#AABBCC` and `#abc` are forms an
+ IMPORTED workbook can carry, and this cleaner's house rule is to DROP what it does not
+ recognise, in silence. Dropping a colour somebody can see is the exact failure R2 named,
+ so a case difference and the three digit shorthand are widened onto the stored form
+ instead of being thrown away. Anything else is still dropped.
+ """
+ if not token:
+ return ""
+ if token.startswith("#"):
+ body = token[1:].lower()
+ if len(body) == 3 and all(c in "0123456789abcdef" for c in body):
+ body = body[0] * 2 + body[1] * 2 + body[2] * 2
+ token = "#" + body
+ return token if _CELL_COLOUR.match(token) else ""
+
+
def _clean_style(raw):
"""Normalize a cell's A1 style, or None when nothing survives.
@@ -552,8 +587,8 @@ def _clean_style(raw):
out[mark] = True
for key in ("color", "fill"):
- token = _text(raw.get(key), 40)
- if token and _COLOUR.match(token):
+ token = _normalise_colour(_text(raw.get(key), 40))
+ if token:
out[key] = token
align = _text(raw.get("align"), 12)
@@ -583,13 +618,16 @@ def _clean_style(raw):
#: picked, gone on the next read with nothing to say so. The type is decided FIRST, and a step
#: name still round trips exactly as it always did. A point size is CLAMPED, not refused, for
#: the rotation's reason below, and the default is stored as an absence just as `m` is.
+ #: Imported fractional points remain fractional: 11.5pt is authored workbook data, while
+ #: ribbon steps happen to be integers. Rounding here made an import unequal to its source on
+ #: the first clean/save cycle.
size = raw.get("size")
if isinstance(size, str):
step = _text(size, 4)
if step in CELL_SIZES and step != "m":
out["size"] = step
- elif not isinstance(size, bool) and isinstance(size, (int, float)):
- points = max(MIN_FONT_PT, min(MAX_FONT_PT, int(round(size))))
+ elif not isinstance(size, bool) and isinstance(size, (int, float)) and math.isfinite(size):
+ points = max(MIN_FONT_PT, min(MAX_FONT_PT, size))
if points != DEFAULT_FONT_PT:
out["size"] = points
@@ -1365,6 +1403,32 @@ def _clean_imported(raw):
return out if (out["at"] or out["source"] or notes) else None
+def _clean_row_groups(raw, rows):
+ """Inclusive sheet ranges, with independent nested collapse state and no empty stratum."""
+ if raw is None:
+ return None
+ if not isinstance(raw, list):
+ raise SheetRefusal("sheet_row_groups_invalid", "Row groups must be a list.")
+ out, ids = [], set()
+ for item in raw:
+ if not isinstance(item, dict):
+ raise SheetRefusal("sheet_row_groups_invalid", "A row group must be an object.")
+ gid, start, end, collapsed = (item.get(key) for key in ("id", "start", "end", "collapsed"))
+ if not isinstance(gid, str) or not gid.strip() or len(gid) > _MAX_ID or gid in ids \
+ or any(isinstance(n, bool) or not isinstance(n, int) for n in (start, end)) \
+ or not 1 <= start <= end <= rows or not isinstance(collapsed, bool):
+ raise SheetRefusal("sheet_row_groups_invalid", "A row group needs an id, a valid row range and a collapse state.")
+ for prior in out:
+ disjoint = end < prior["start"] or start > prior["end"]
+ nested = (start >= prior["start"] and end <= prior["end"]) \
+ or (start <= prior["start"] and end >= prior["end"])
+ if (start, end) == (prior["start"], prior["end"]) or not (disjoint or nested):
+ raise SheetRefusal("sheet_row_groups_overlap", "Groups can be nested, but cannot duplicate or partially overlap.")
+ ids.add(gid)
+ out.append({"id": gid, "start": start, "end": end, "collapsed": collapsed})
+ return out or None
+
+
def _clean_sheet(raw, index=0, *, lossless=False):
"""Normalize a tab's known strata and retain additive JSON metadata without interpretation.
@@ -1404,6 +1468,7 @@ def _clean_sheet(raw, index=0, *, lossless=False):
"colWidths": _clean_col_widths(raw.get("colWidths"), cols, lossless=lossless),
"freeze": _clean_freeze(raw.get("freeze"), cols, rows, lossless=lossless),
"rules": _clean_rules(raw.get("rules"), lossless=lossless),
+ "rowGroups": _clean_row_groups(raw.get("rowGroups"), rows),
"hiddenRows": _clean_hidden(raw.get("hiddenRows"), "rows"),
"hiddenCols": _clean_hidden(raw.get("hiddenCols"), "columns"),
"charts": _clean_charts(raw.get("charts")),
@@ -1481,13 +1546,13 @@ def migrate_workbook(raw, *, lossless=True):
def new_workbook(name="", created_by="", workbook_id=""):
"""The in-memory shape of a fresh workbook with ONE tab (R7: a workbook has tabs)."""
- wid = _slug_id(workbook_id) if workbook_id else uuid.uuid4().hex[:12]
+ wid = _slug_id(workbook_id) if workbook_id else write_execution.new_id()[:12]
stamp = _now()
return {
"id": wid,
"schemaVersion": SHEET_SCHEMA_VERSION,
"name": _text(name, _MAX_NAME) or "Untitled workbook",
- "sheets": [{"id": uuid.uuid4().hex[:12], "name": "Sheet1",
+ "sheets": [{"id": write_execution.new_id()[:12], "name": "Sheet1",
"cols": DEFAULT_SHEET_COLS, "rows": DEFAULT_SHEET_ROWS, "cells": {},
"usedExtent": None, "writesFrom": 0}],
"rev": 1,
@@ -1764,6 +1829,132 @@ def find_tab(book, tab_ref):
return tabs[at]
+def workbook_snapshot(st, workbook_id):
+ """Return the stored workbook bytes as JSON data, without migration or cleaning.
+
+ The ordinary read door intentionally migrates old workbooks. A narrowly scoped repair needs
+ a different primitive: its backup and compare-and-swap digest must describe the document that
+ is actually stored, or a migration performed only in memory could make the precondition look
+ stronger than it is. The returned copy is detached from the store.
+ """
+ wid = _slug_id(workbook_id)
+ doc = _read(st, workbook_key(wid))
+ if not doc or not doc.get("id"):
+ raise SheetRefusal("workbook_missing", "That workbook no longer exists.")
+ return copy.deepcopy(doc)
+
+
+def workbook_digest(book):
+ """A stable digest for an exact workbook snapshot."""
+ if not isinstance(book, dict):
+ raise SheetRefusal("workbook_not_an_object", "A workbook must be an object.")
+ encoded = json.dumps(book, sort_keys=True, separators=(",", ":"), ensure_ascii=False,
+ default=str).encode("utf-8")
+ return hashlib.sha256(encoded).hexdigest()
+
+
+def apply_exact_cell_edits(st, workbook_id, edits, expect_rev, expect_digest, *,
+ actor="", actor_name=""):
+ """Apply a preconditioned list of cell edits under the store lock.
+
+ This is deliberately narrower than PUT and stricter than the collaborative range PATCH. Each
+ edit names one tab/address and carries the exact value observed while the repair was planned.
+ The workbook revision, whole-document digest and every cell precondition must still match.
+ Nothing outside those cells, normal revision/write-audit fields and derived used extents is
+ rewritten. It is the storage primitive for exceptional, owner-authorized workbook repair;
+ normal Sheet editing continues to use ``put_workbook`` and ``patch_cells``.
+ """
+ wid = _slug_id(workbook_id)
+ if isinstance(expect_rev, bool) or not isinstance(expect_rev, int) or expect_rev < 0:
+ raise SheetRefusal("repair_revision_required",
+ "An exact workbook revision is required for this repair.")
+ if not isinstance(expect_digest, str) or not re.fullmatch(r"[0-9a-f]{64}", expect_digest):
+ raise SheetRefusal("repair_digest_required",
+ "An exact workbook snapshot digest is required for this repair.")
+ if not isinstance(edits, list) or not edits:
+ raise SheetRefusal("repair_edits_required", "This repair does not name any cell edits.")
+
+ prepared, seen = [], set()
+ for raw in edits:
+ if not isinstance(raw, dict):
+ raise SheetRefusal("repair_edit_invalid", "Every repair edit must be an object.")
+ tab_ref = str(raw.get("sheet") or "").strip()
+ addr = str(raw.get("address") or "").strip().upper().replace("$", "")
+ matched = _ADDR.fullmatch(addr)
+ if (not tab_ref or not matched
+ or _col_index(re.match(r"[A-Z]+", addr).group(0)) >= DOCUMENT_MAX_COLS
+ or int(re.search(r"[0-9]+$", addr).group(0)) > DOCUMENT_MAX_ROWS):
+ raise SheetRefusal("repair_edit_invalid",
+ "Every repair edit must name one sheet and one A1 address.")
+ slot = (tab_ref.casefold(), addr)
+ if slot in seen:
+ raise SheetRefusal("repair_edit_duplicate",
+ "A workbook repair may name each cell only once.")
+ seen.add(slot)
+ after = raw.get("after")
+ prepared.append({"sheet": tab_ref, "address": addr,
+ "before": copy.deepcopy(raw.get("before")),
+ "after": None if after is None else clean_cell(after, addr=addr)})
+ prepared.sort(key=lambda row: (row["sheet"].casefold(), row["address"]))
+
+ saved = {}
+
+ def _apply(state):
+ prior = state if state.get("id") else None
+ if prior is None:
+ raise SheetRefusal("workbook_missing", "That workbook no longer exists.")
+ if _slug_id(prior.get("id")) != wid:
+ raise SheetRefusal("repair_workbook_mismatch",
+ "The stored workbook identity does not match this repair target.")
+ current = int(prior.get("rev") or 0)
+ if current != expect_rev:
+ raise SheetStaleRefusal(
+ "workbook_revision_stale",
+ "That workbook changed after the repair was planned, so no repair was applied.",
+ rev_current=current)
+ if workbook_digest(prior) != expect_digest:
+ raise SheetRefusal(
+ "repair_snapshot_changed",
+ "That workbook changed without a revision change, so no repair was applied.")
+
+ book = copy.deepcopy(prior)
+ touched = {}
+ for edit in prepared:
+ at = _tab_index(book.get("sheets") or [], edit["sheet"])
+ if at is None:
+ raise SheetRefusal("tab_missing", "Tab " + edit["sheet"]
+ + " is not in this workbook.")
+ tab = book["sheets"][at]
+ cells = tab.setdefault("cells", {})
+ actual = copy.deepcopy(cells.get(edit["address"]))
+ if actual != edit["before"]:
+ raise SheetRefusal(
+ "repair_cell_changed",
+ "Cell " + edit["sheet"] + "!" + edit["address"]
+ + " changed after the repair was planned, so no repair was applied.")
+ if edit["after"] is None:
+ cells.pop(edit["address"], None)
+ else:
+ cells[edit["address"]] = copy.deepcopy(edit["after"])
+ touched.setdefault(at, []).append(edit["address"])
+
+ new_rev = current + 1
+ for at, addrs in touched.items():
+ tab = book["sheets"][at]
+ tab["usedExtent"] = _used_extent(
+ tab.get("cells") or {}, (tab.get("excel") or {}).get("blankStyles"))
+ _append_write_log(tab, new_rev, addrs, actor=actor, actor_name=actor_name)
+ book["rev"] = new_rev
+ book["updated_at"] = _now()
+ _json_safe(book, "workbook")
+ saved.update(book)
+ return book, {"rev": new_rev, "edits": len(prepared),
+ "paths": [row["sheet"] + "!" + row["address"] for row in prepared]}
+
+ out = _persist(st, workbook_key(wid), _apply)
+ return {**out, "digest": workbook_digest(saved)}
+
+
def workbook_meta(book):
"""The workbook WITHOUT any cells, for C3's `?tabs=meta`.
diff --git a/platform/core/table_store.py b/platform/core/table_store.py
index 0d1f47d6b62494a6e04642286bf4d0ede7a7f7bf..0ad87a7797f7aedc91e7c9aa6dc49f04d7eaea84 100644
--- a/platform/core/table_store.py
+++ b/platform/core/table_store.py
@@ -19,6 +19,7 @@ import time
import core.store as store
import core.view_audit as view_audit
+from core import write_execution
#: ``__shared__`` remains the tenant-wide FIELD-annotation stratum. Before Wave 48 it also
#: carried a second, legacy home for shared Views. That view map is migrated away on the first
@@ -1727,7 +1728,7 @@ class TableStore:
data.pop(ASSIGNED_KEY, None)
if isinstance(before, dict):
bucket = data.setdefault(owner, {}).setdefault(TRASH_KEY, {})
- stamp = time.time()
+ stamp = write_execution.timestamp()
bucket[vid] = {'view': before, 'deleted_at': float(stamp),
'deleted_by': who, 'assignments': carried}
purge_trash(bucket, now=stamp)
@@ -1818,7 +1819,7 @@ class TableStore:
current.get(owner), dict) else None
if isinstance(live, dict):
live.pop(vid, None)
- purge_trash(live, now=time.time())
+ purge_trash(live, now=write_execution.timestamp())
if not live:
current[owner].pop(TRASH_KEY, None)
audit(action=None, owner=owner, view_id=vid)
@@ -1854,7 +1855,7 @@ class TableStore:
bucket = ((data.get(owner) or {}).get(TRASH_KEY) or {})
if not isinstance(bucket, dict) or not bucket:
return []
- stamp = float(now if now is not None else time.time())
+ stamp = float(now if now is not None else write_execution.timestamp())
expired = [vid for vid, entry in bucket.items() if trash_expired(entry, now=stamp)]
# ⛔⛔ R11's ADOPTION RUNS BEFORE THE UPDATE AND OUTSIDE ITS CALLBACK (W51-T26).
# `sheets.py::save_folders` states the rule for this codebase in as many words: validation
@@ -1985,7 +1986,7 @@ class TableStore:
if isinstance(assignment, dict):
target[str(aid)] = dict(assignment)
bucket.pop(vid, None)
- purge_trash(bucket, now=time.time())
+ purge_trash(bucket, now=write_execution.timestamp())
if not bucket:
data[owner].pop(TRASH_KEY, None)
restored.clear()
diff --git a/platform/core/user_tables.py b/platform/core/user_tables.py
index a94ea82dfc3be527d10db159ac9b0a7332d9d708..8bfb00005799ed9a5c6a741a95124b3c6ee1f084 100644
--- a/platform/core/user_tables.py
+++ b/platform/core/user_tables.py
@@ -69,6 +69,22 @@ SHARED_NOTE_READS_KEY = 'shared_note_reads'
ODOO_SCHEMA_SNAPSHOT_KEY = 'odoo_schema_snapshot'
ODOO_SCHEMA_ENABLEMENTS_KEY = 'odoo_schema_enablements'
ODOO_SYNC_STATUS_KEY = 'odoo_sync_status'
+CONNECTOR_HEALTH_FIELD = 'connectorHealth'
+CONNECTOR_HEALTH_STATES = (
+ 'never_run', 'queued', 'running', 'succeeded', 'partial', 'failed', 'disabled',
+)
+_CONNECTOR_SAFE_ERRORS = {
+ 'maps_credential_missing': 'Add the scraper credential in Keychain before choosing Run now.',
+ 'scraper_credential_missing': 'Add the scraper credential in Keychain before choosing Run now.',
+ 'maps_unbounded_request': 'Google Maps discovery requires an explicit bounded result cap.',
+ 'maps_invalid_search': 'Google Maps discovery needs a valid category or search.',
+ 'maps_vendor_error': 'The Maps provider could not complete this run.',
+ 'maps_schema_drift': 'The Maps provider returned an unreadable result shape.',
+ 'scraper_run_failed': 'The provider could not complete this run. No result was written.',
+ 'provider_result_failed': 'The provider returned no usable result for this database.',
+ 'scraper_result_not_saved': 'The provider answered, but the result could not be saved.',
+ 'sync_status_not_saved': 'The run did not start because its durable status could not be saved.',
+}
# A reserved, derived multi-select field. It is table data backed by durable Zone membership,
# not a user-editable column and therefore cannot collide with a field a user creates.
ZONE_FIELD_KEY = '__zone__'
@@ -467,6 +483,74 @@ def connector_database_identity(table_key, rt=None, st=None, definition=None):
declared.get('model') or (getattr(spec, 'native', '') if spec else ''))
+def connector_health_receipt(state, *, last_success=None, source_timestamp=None,
+ materialized_timestamp=None, row_count=None, error=None):
+ """Validate the durable, data-free connector database health document."""
+ def stamp(value):
+ text = str(value or '').strip()
+ if not text:
+ return None
+ try:
+ parsed = _dt.datetime.fromisoformat(text.replace('Z', '+00:00'))
+ except (TypeError, ValueError, OverflowError):
+ return None
+ return parsed.isoformat()
+
+ clean_state = str(state or '').strip()
+ if clean_state not in CONNECTOR_HEALTH_STATES:
+ raise ValueError('unknown connector database health state')
+ count = None
+ if row_count is not None:
+ if type(row_count) is not int or row_count < 0:
+ raise ValueError('row_count must be a non-negative integer or None')
+ count = row_count
+ clean_error = None
+ if error is not None:
+ raw = error if isinstance(error, dict) else {}
+ requested_code = str(raw.get('code') or '').strip()
+ code = requested_code if requested_code in _CONNECTOR_SAFE_ERRORS else 'connector_sync_failed'
+ # Never persist provider/exception prose. Only named classifications get specific copy.
+ message = _CONNECTOR_SAFE_ERRORS.get(code, 'This database could not be synced.')
+ clean_error = {'code': code, 'message': message}
+ return {
+ 'state': clean_state,
+ 'lastSuccess': stamp(last_success),
+ 'sourceTimestamp': stamp(source_timestamp),
+ 'materializedTimestamp': stamp(materialized_timestamp),
+ 'rowCount': count,
+ 'error': clean_error,
+ }
+
+
+def record_connector_health(table_key, state, *, st=None, last_success=None,
+ source_timestamp=None, materialized_timestamp=None,
+ row_count=None, error=None):
+ """Persist one data-free database sync receipt beside its governed definition.
+
+ Provider runners supply already-sanitized error ``code``/``message`` values. This seam
+ deliberately accepts neither exceptions nor arbitrary provider payloads, so a failed run can
+ survive a process restart without turning the tenant store into a credential or row log.
+ """
+ key = str(table_key or '').strip()
+ if not key.startswith(KEY_PREFIX):
+ raise ValueError('a governed database key is required')
+ receipt = connector_health_receipt(
+ state, last_success=last_success, source_timestamp=source_timestamp,
+ materialized_timestamp=materialized_timestamp, row_count=row_count, error=error)
+
+ def _set(current):
+ document = dict(current or {})
+ definition = dict(document.get(key) or {})
+ if not definition:
+ raise KeyError(key)
+ definition[CONNECTOR_HEALTH_FIELD] = receipt
+ document[key] = definition
+ return document
+
+ landed = _st(st).update(STORE_KEY, _set, flush='sync')
+ return dict((landed.get(key) or {}).get(CONNECTOR_HEALTH_FIELD) or receipt)
+
+
def credential_footprint(entry_type, connection_id, rt=None, st=None):
"""Disclose what revoking this ONE credential purges and what its policy keeps.
diff --git a/platform/core/write_execution.py b/platform/core/write_execution.py
new file mode 100644
index 0000000000000000000000000000000000000000..64ad11cfc71512ec6ceb3a502eb3ccc11d64769b
--- /dev/null
+++ b/platform/core/write_execution.py
@@ -0,0 +1,45 @@
+"""Request-local server metadata for replaying an explicitly confirmed write.
+
+The gateway signs the seed and timestamp with the preview. Revalidation and the
+ordinary writer reuse them so generated ids and timestamps cannot change the diff.
+This does not replace document revision or permission checks. Unscoped UI writes
+continue to use fresh ids and the wall clock.
+"""
+from contextlib import contextmanager
+from contextvars import ContextVar
+from datetime import datetime, timezone
+import time
+import uuid
+
+
+_execution = ContextVar('confirmed_write_execution', default=None)
+
+
+@contextmanager
+def replay(metadata):
+ state = {'seed': str(metadata['seed']), 'timestamp': float(metadata['timestamp']), 'index': 0}
+ token = _execution.set(state)
+ try:
+ yield
+ finally:
+ _execution.reset(token)
+
+
+def new_id():
+ state = _execution.get()
+ if state is None:
+ return uuid.uuid4().hex
+ # Replace, never mutate, the context value: independently copied async contexts
+ # must not share a mutable sequence counter.
+ index = state['index']
+ _execution.set({**state, 'index': index + 1})
+ return uuid.uuid5(uuid.NAMESPACE_URL, f"loopable-write:{state['seed']}:{index}").hex
+
+
+def timestamp():
+ state = _execution.get()
+ return time.time() if state is None else state['timestamp']
+
+
+def utcnow():
+ return datetime.fromtimestamp(timestamp(), timezone.utc)
diff --git a/platform/harness/filter_sql.py b/platform/harness/filter_sql.py
index 0eb7b72310953dd8693499788991ea0fd57bd4d4..30e554655042da23a5105708efd122a15b1b6905 100644
--- a/platform/harness/filter_sql.py
+++ b/platform/harness/filter_sql.py
@@ -157,6 +157,9 @@ _DATE_CMP = dict(_NUM_CMP)
#: rather than WHERE; columns_used = colIds actually referenced in the emitted SQL (an
#: always-true `1=1` branch references nothing, and must not force HAVING routing).
Compiled = namedtuple('Compiled', 'sql params uses_aggregate columns_used')
+#: HOTFIX 2026-09-23: a cohort set given as a SELECT of ids rather than a list. A server-side
+#: rank answer (Top N, above average ...) over 256k rows is a subquery, never a 128k-id IN list.
+SqlMembership = namedtuple('SqlMembership', 'sql params')
# --------------------------------------------------------------------------- value coercion
@@ -612,6 +615,10 @@ def _compile_cohort(rule, state):
parts, params = [], []
for ids in members:
+ if isinstance(ids, SqlMembership):
+ parts.append(f"COALESCE({id_sql} IN ({ids.sql}), FALSE)")
+ params.extend(ids.params)
+ continue
ids = [int(x) for x in ids]
if not ids:
parts.append('1=0')
diff --git a/platform/harness/mirror_tokens.py b/platform/harness/mirror_tokens.py
new file mode 100644
index 0000000000000000000000000000000000000000..a444072e9ea28efc5981c902b73bb71c5c6c6f0d
--- /dev/null
+++ b/platform/harness/mirror_tokens.py
@@ -0,0 +1,64 @@
+"""The mirror's own change token, in the layer that owns the mirror (SDLC 65 combine).
+
+⛔⛔ WHY IT LIVES HERE AND NOT IN `core/sheet_binding.py`, WHERE S65-T03 WROTE IT.
+It reads the analytical mirror: `harness.datastore.ENTITIES` and `harness.runtime.mirror_cursor`.
+`core` never imports up (ARCHITECTURE.md section 1), and `verify_layering.py`'s D-460 leg caught
+the new `core -> harness` edge as this wave's own violation. Moving the body up and registering it
+back down is the fix the gate's first remedy names -- "move it down" read from the other end. The
+alternative, a `GRANDFATHERED_CORE` entry, was REFUSED: that list is edges which PRE-DATE the gate,
+and a wave adding its own edge to it exempts itself from the rule it just broke.
+
+⭐ Importing this module is what arms `core.sheet_binding.mirror_change_token`; `routes_sheet`
+does it. Until then that function RAISES, which `database_change_token` already treats as "cannot
+prove unchanged, therefore resolve" -- the safe direction.
+"""
+from __future__ import annotations
+import re
+
+from core import sheet_binding
+from core.sheet_binding import fingerprint
+
+
+def mirror_change_token(st, database):
+ """One SQL token read over source tables, not N binding queries or Python row copies.
+
+ The current mirror has no per-table monotonic revision. Content fingerprints close
+ that gap without changing its writer or adding a row cap. Count, xor and sum include
+ entire records; schema and sync receipts cover field changes and no-op syncs too.
+ Registry metrics depend on the Odoo observation set, so their token includes it.
+ """
+ from core import connector_spec
+ from harness import datastore
+ from harness.runtime import mirror_cursor
+ spec = connector_spec.spec_for(database)
+ if database in ("customer_data", "product_data"):
+ sources = set(datastore.ENTITIES)
+ elif spec:
+ sources = {spec.source}
+ # A connector grid also reads joined label/identity columns declared by its spec.
+ sources.update(target.source for _, key, _, _ in spec.links
+ if (target := connector_spec.spec_for(key)) is not None)
+ else:
+ raise RuntimeError("The connector has no declared source for its change token.")
+ if not sources or any(not re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]*", s) for s in sources):
+ raise RuntimeError("The connector source identity is invalid.")
+ sources = sorted(sources)
+ receipt_sources = sorted(set(sources) | ({"meta"} if any(s.startswith("meta_") for s in sources) else set()))
+ clauses = [f"SELECT '{source}' AS source, CAST(count(*) AS VARCHAR) AS n, "
+ f"CAST(bit_xor(hash(t)) AS VARCHAR) AS x, "
+ f"CAST(sum(CAST(hash(t) AS HUGEINT)) AS VARCHAR) AS s FROM \"{source}\" t"
+ for source in sources]
+ placeholders = ",".join("?" for _ in sources)
+ clauses.append("SELECT 'schema', table_name, column_name, data_type FROM duckdb_columns() "
+ f"WHERE table_name IN ({placeholders})")
+ # _sync_state is the resident mirror writer's receipt, not the app's status cache.
+ clauses.append("SELECT 'sync', entity, phase, concat_ws('|', last_id, cursor_wd, rows, updated_at) "
+ "FROM _sync_state WHERE split_part(entity, '.', 1) IN ("
+ + ",".join("?" for _ in receipt_sources) + ")")
+ rows = mirror_cursor(st).execute(" UNION ALL ".join(clauses) + " ORDER BY 1,2,3,4",
+ [*sources, *receipt_sources]).fetchall()
+ if not any(row[0] == "sync" for row in rows):
+ raise RuntimeError("The connector has no sync receipt.")
+ return fingerprint(rows)
+
+sheet_binding.set_mirror_token_provider(mirror_change_token)
diff --git a/platform/harness/semantic.py b/platform/harness/semantic.py
index f67713fc8e40aa5c35b6cf3288ac057cbd908ef8..edce984a37afdfd0a4ecba6134e2768de13ab840 100644
--- a/platform/harness/semantic.py
+++ b/platform/harness/semantic.py
@@ -561,6 +561,8 @@ _VALIDATORS = {
"gl_opex": _v_gl_opex,
"ar_outstanding": _v_ar_outstanding,
}
+from modules.finance_spine import FINANCE_VALIDATORS
+_VALIDATORS.update(FINANCE_VALIDATORS)
def validate_metric(key, date_from=None, date_to=None, team_id=None, tol=0.01):
diff --git a/platform/model/metrics/payables.yml b/platform/model/metrics/payables.yml
new file mode 100644
index 0000000000000000000000000000000000000000..b2964f111b41f9f05c72ab770195649457e8ddad
--- /dev/null
+++ b/platform/model/metrics/payables.yml
@@ -0,0 +1,15 @@
+# Metrics: payables — posted vendor documents at account.move grain.
+topic: odoo_bills
+metrics:
+ - key: ap_outstanding
+ label: AP outstanding
+ agg: sum
+ field: amount_residual_signed
+ negate: true
+ format: usd
+ empty: zero
+ description: "Net unpaid supplier balance over posted vendor bills and vendor credits."
+ ai_context: "Odoo signs vendor liabilities negative and vendor credits positive; negating the signed sum produces the payable balance without losing credit-note netting."
+ validate:
+ method: ap_outstanding
+ note: "The mirror's negated signed residual must equal the independently read live vendor-document aggregate to the cent."
diff --git a/platform/model/metrics/sales.yml b/platform/model/metrics/sales.yml
index ad5df59da35a431a6417ddec08a988df101e7a55..e3506c0c1d4d1b91c0b3769cd9c93d865fd68dc8 100644
--- a/platform/model/metrics/sales.yml
+++ b/platform/model/metrics/sales.yml
@@ -147,3 +147,29 @@ metrics:
format: int
description: "Confirmed wholesale orders Odoo marks fully invoiced (invoice_status = invoiced)."
ai_context: "Order-header count under the orders scope plus the fully-invoiced flag; partially invoiced and not-yet-invoiced orders are excluded. Reconciled store-vs-live per BU by store_parity."
+
+ - key: commission_accrued
+ label: Commission accrued
+ topic: commission_lines
+ agg: sum
+ field: amount
+ format: usd
+ empty: zero
+ description: "Commission amount accrued at invoice-line-agent grain."
+ ai_context: "One invoice line can have multiple commission rows. This is commission money, never a sales-line count."
+ validate:
+ method: commission_accrued
+ note: "The mirror aggregate must equal the live account.invoice.line.agent amount under the same invoice-date and BU scope."
+
+ - key: invoiced_sales
+ label: Invoiced sales
+ topic: customer_invoices
+ agg: sum
+ field: amount_untaxed_signed
+ format: usd
+ empty: zero
+ description: "Untaxed value of posted customer invoices."
+ ai_context: "Company-level posted out_invoice documents by invoice date; this is billing, not wholesale order-line revenue."
+ validate:
+ method: invoiced_sales
+ note: "The mirror document aggregate must equal live posted customer invoices to the cent."
diff --git a/platform/model/metrics/stock.yml b/platform/model/metrics/stock.yml
index f0410fc3ccb22a5aa91f62dc04d9160e5d233610..f9ce30bebaeb54e8312facd86fd2ec159b80f203 100644
--- a/platform/model/metrics/stock.yml
+++ b/platform/model/metrics/stock.yml
@@ -33,6 +33,9 @@ metrics:
empty: zero
description: "Units that ARRIVED at an internal location from outside it, over the window."
ai_context: "Receipts, returns inward and inventory adjustments upward. Two-sided by construction: an internal-to-internal transfer is NOT an arrival. Quantities are in each product's own unit, so never sum this across SKUs."
+ validate:
+ method: stock_movement_in
+ note: "Mirror arrivals must equal live done moves crossing from non-internal to internal locations."
- key: stock_out
label: Stock moved out
@@ -43,6 +46,9 @@ metrics:
empty: zero
description: "Units that LEFT an internal location for outside it, over the window."
ai_context: "Deliveries, scrap and adjustments downward. Two-sided by construction: an internal-to-internal transfer is NOT a departure. Quantities are in each product's own unit, so never sum this across SKUs."
+ validate:
+ method: stock_movement_out
+ note: "Mirror departures must equal live done moves crossing from internal to non-internal locations."
# ⚠ NET IS DERIVED, never a third sum. Deriving it guarantees `net = in - out` exactly, where a
# separately-summed net could disagree with its own two components under a filter and nothing
@@ -55,3 +61,6 @@ metrics:
empty: zero
description: "Arrivals minus departures over the window: the period change in units held."
ai_context: "Derived from stock_in and stock_out, so it always reconciles to them. It is a MOVEMENT figure, not the on-hand balance: the balance is `product_data.on_hand` from stock.quant."
+ validate:
+ method: stock_movement_net
+ note: "Derived mirror net must equal independently read live arrivals minus departures."
diff --git a/platform/model/topics/commission_lines.yml b/platform/model/topics/commission_lines.yml
index 6cb60740a9ce006d29649e8bd7a93627fd1bc8fb..acf9fa7317263e9fa709d5bcc474f14c75d87be3 100644
--- a/platform/model/topics/commission_lines.yml
+++ b/platform/model/topics/commission_lines.yml
@@ -12,10 +12,10 @@ semantic:
date_field: ""
date_col_note: "a.invoice_date"
scope: "Every commission row, including rows attached to non-product lines; sum commission amount in company currency by invoice date, and do not treat commission row counts as sales line counts."
- measure_not_yet:
- - key: commission_amount
- cause: "commission_amount resolves on commission_lines but declares no validate contract against an independent Odoo aggregate."
- fix: "Declare a validate method for commission_amount in model/metrics/invoice_lines.yml and add a _v_commission_amount reconciliation to harness/semantic.py::_VALIDATORS."
+ measure:
+ key: commission_accrued
+ source: commission_lines
+ validate: commission_accrued
label: Commission lines (agent × invoice line)
entity: account.invoice.line.agent
diff --git a/platform/model/topics/customer_invoices.yml b/platform/model/topics/customer_invoices.yml
index 0d5c647101cf797bebdbbbed27a009824fe9119c..82a7e62e8ab88527fd962cdbe49ae1cc29dd2665 100644
--- a/platform/model/topics/customer_invoices.yml
+++ b/platform/model/topics/customer_invoices.yml
@@ -14,10 +14,10 @@ semantic:
date_field: ""
date_col_note: "m.invoice_date"
scope: "Posted customer invoices only (out_invoice); company consolidated with no partner exclusion or business-unit filter; sum positive amount_untaxed_signed by invoice date."
- measure_not_yet:
- - key: invoiced
- cause: "invoiced resolves on customer_invoices but declares no validate contract against an independent Odoo aggregate."
- fix: "Declare a validate method for invoiced in model/metrics/returns.yml and add a _v_invoiced reconciliation to harness/semantic.py::_VALIDATORS."
+ measure:
+ key: invoiced_sales
+ source: customer_invoices
+ validate: invoiced_sales
label: Customer invoices
entity: account.move
domain_builder: customer_invoice_domain
diff --git a/platform/model/topics/odoo_bills.yml b/platform/model/topics/odoo_bills.yml
index 2de49e072c0c03492cd4d41d1f534fc3e6920494..75feb66d017831b34d218b01f13e0de4f6e2a05a 100644
--- a/platform/model/topics/odoo_bills.yml
+++ b/platform/model/topics/odoo_bills.yml
@@ -9,14 +9,14 @@ key: odoo_bills
semantic:
approved: [bill_no, odoo_id, vendor, vendor_id, invoice_date, due_date, amount_untaxed, residual, payment_state, move_type, vendor_link]
date_field: invoice_date
- scope: "Posted vendor invoices and vendor credit notes (in_invoice, in_refund); payable residuals are negative in Odoo, so a payable balance uses their absolute value."
- measure_not_yet:
- - key: ap_outstanding
- cause: "No payables fact topic or ap_outstanding metric exists. ar_outstanding resolves on receivables but covers customer documents, so it cannot reconcile a vendor bill balance."
- fix: "Add a payables topic for posted in_invoice and in_refund documents, declare ap_outstanding with a validate method in its metric yml, and add a _v_ap_outstanding reconciliation to harness/semantic.py::_VALIDATORS."
+ scope: "Posted vendor invoices and vendor credit notes (in_invoice, in_refund); negate the signed residual sum so vendor credits net against bills."
+ measure:
+ key: ap_outstanding
+ source: odoo_bills
+ validate: ap_outstanding
purposes:
invoice_date: "The vendor bill date used for dated bill questions."
- residual: "The outstanding amount on this posted vendor document; no validated payable aggregate is available yet."
+ residual: "The outstanding amount on this posted vendor document; ap_outstanding is the validated aggregate."
synonyms:
bill_no: [vendor bill, supplier invoice]
residual: [unpaid amount, payable balance]
@@ -31,42 +31,27 @@ subject: "odoo:account.move.vendor"
grain: "one row per posted VENDOR bill or vendor credit note (document grain)"
scope:
population: "move_type in (in_invoice, in_refund) AND state = posted"
- signs: "for payables the residual is negative on the Odoo side; take the absolute value for an AP figure"
+ signs: "vendor bills are negative and vendor credits positive; negate the signed sum so credits reduce AP"
store:
table: account_move
alias: m
date_col: "m.invoice_date"
+ scope_sql: "m.state = 'posted' AND m.move_type IN ('in_invoice','in_refund')"
dims:
- vendor: {label: "Vendor"}
- payment_state: {label: "Payment state"}
- move_type: {label: "Document type"}
+ move: {col: "m.id", name_col: "m.name", label: "Bill"}
+ vendor: {col: "m.partner_id", name_col: "m.partner_name", label: "Vendor"}
+ due_date: {col: "m.invoice_date_due", label: "Due date"}
+ payment_state: {col: "m.payment_state", label: "Payment state"}
+ move_type: {col: "m.move_type", label: "Document type"}
-# ⛔⛔ W49-T22 (R28) — NO `measures:` BLOCK, AND THE ABSENCE IS THE FINDING, NOT AN OVERSIGHT.
-# Its four siblings (orders, invoices, accounts, vendors) each gained one in this ticket; this
-# one could not, and offering a wrongly-keyed column would have been worse than offering none
-# (`semantic.entity_measures`: an offer is a promise).
-#
-# THE MODEL CARRIES NO AP FACT TOPIC AT ALL. The topic list is commission_lines, credit_notes,
-# customer_invoices, gl_lines, invoice_lines, odoo_*, receivables, sales_lines, sales_orders,
-# stock_moves. `receivables` looks like the answer and is not: its `scope_sql` is
-# `move_type IN ('out_invoice','out_refund')`, i.e. CUSTOMER documents only.
-#
-# AND THE ONE EXISTING BINDING THAT COULD KEY HERE IS ARITHMETICALLY OUT OF REACH. A `move` dim
-# on `gl_lines` (`l.move_id`) WOULD key to `account.move.id`, which is this grid's pid — but
-# `store_query` groups over the topic's whole scope, and tenant #0 has **234,364 distinct posted
-# `move_id`** against `semantic.MAX_GROUPS = 200_000`. `entity_measure_values` refuses a
-# truncated group set outright, so that binding raises rather than answering.
-# (`gl_lines.partner` groups by PARTNER, so it keys a vendor, never a bill.)
-#
-# THE FIX, and it is its own ticket: a `payables` topic — table `account_move`, alias `m`,
-# `scope_sql: m.state = 'posted' AND m.move_type IN ('in_invoice','in_refund')`,
-# `date_col: m.invoice_date`, dims `vendor`/`payment_state`/`move`, an `ap_outstanding` metric
-# (`agg: sum`, `field: amount_residual_signed`, `empty: zero`) and a `_v_ap_outstanding`
-# reconciliation in `harness/semantic.py::_VALIDATORS`. MEASURED as viable: 6,538 posted vendor
-# documents, so a `move` dim there returns 6,538 groups, far inside MAX_GROUPS.
-# EXIT CONDITION: `entity_measures("odoo_bills")` is non-empty under `strict=True` and
-# `entity_measure_values("odoo_bills", ["ap_outstanding"])` returns integer keys that are
-# `account.move.id` values in the posted vendor-document population.
+measures:
+ source: odoo_bills
+ dim: move
+ keys: [ap_outstanding]
+
+# W49-T22 measured 6,538 posted vendor documents, safely below MAX_GROUPS. Wave 63 closes that
+# debt with the validated, document-grain binding above. Receivables remains the separate
+# customer-document fact and is never reused for AP.
# key / label / type / kind, derived from the grid contract. `kind` says where the
# value COMES FROM: a `data` column is stored on the row; a `rollup` is computed
diff --git a/platform/model/topics/stock_moves.yml b/platform/model/topics/stock_moves.yml
index 519d47fa29c37ea25813ab7a9b2150c0193af5ef..da05ed89479e3c53b928d8a7f9d02754d28a8715 100644
--- a/platform/model/topics/stock_moves.yml
+++ b/platform/model/topics/stock_moves.yml
@@ -14,10 +14,10 @@ semantic:
date_field: ""
date_col_note: "sm.date"
scope: "Done stock moves only, by movement date and company consolidated; inbound crosses from non-internal to internal locations, outbound crosses the reverse way, and internal transfers count as neither; picking code does not define direction."
- measure_not_yet:
- - key: stock_in
- cause: "stock_in resolves on stock_moves but declares no validate contract against an independent Odoo aggregate; stock_out and stock_net also lack validate contracts."
- fix: "Declare validate methods in model/metrics/stock.yml and add _v_stock_in, _v_stock_out and _v_stock_net reconciliations to harness/semantic.py::_VALIDATORS using both location usages."
+ measure:
+ key: stock_net
+ source: stock_moves
+ validate: stock_movement_net
label: Stock movement
entity: stock.move
domain_builder: ~
diff --git a/platform/modules/ar.py b/platform/modules/ar.py
index 5494f3da49a591ff96fd3bfd8f30d2cb58462560..3fd931778ff4f0738d3c48bac0de7e9750b8840f 100644
--- a/platform/modules/ar.py
+++ b/platform/modules/ar.py
@@ -479,6 +479,69 @@ def credit_limit_proposals(exposure, behavior):
'with_limit': [e for e in exposure.get('_all_rows', []) if e.get('credit_limit')]}
+def prioritize_mirror_worklist(rows, *, today=None):
+ """Rank mirror-backed receivable documents without issuing an Odoo read.
+
+ The Finance application calls the canonical source-record endpoint and applies the same
+ ordering in the browser. This pure counterpart keeps the ranking and missing-data contract
+ independently testable: absent owners, promises, dates, or amounts remain ``None`` rather
+ than turning into a reassuring but false zero.
+ """
+ today = today or P.today()
+
+ def _number(value):
+ if value in (None, '') or isinstance(value, bool):
+ return None
+ try:
+ return float(value)
+ except (TypeError, ValueError):
+ return None
+
+ def _date(value):
+ try:
+ return dt.date.fromisoformat(str(value)[:10]) if value else None
+ except (TypeError, ValueError):
+ return None
+
+ ranked = []
+ for source in rows or ():
+ if not isinstance(source, dict):
+ continue
+ amount = _number(source.get('ar_outstanding'))
+ overdue_amount = _number(source.get('ar_overdue'))
+ due = _date(source.get('due_date'))
+ days_overdue = max((today - due).days, 0) if due else None
+ is_overdue = ((overdue_amount is not None and overdue_amount > 0)
+ or (days_overdue is not None and days_overdue > 0))
+ score = abs(amount) if amount is not None else None
+ ranked.append({
+ 'record_id': source.get('_rid'),
+ 'customer': source.get('partner') or None,
+ 'document': source.get('move') or None,
+ 'amount': amount,
+ 'overdue_amount': overdue_amount,
+ 'due_date': due.isoformat() if due else None,
+ 'days_overdue': days_overdue,
+ 'aging': ('Overdue' if is_overdue else 'Current')
+ if overdue_amount is not None or due else None,
+ 'owner': source.get('collection_owner') or None,
+ 'status': source.get('payment_state') or None,
+ 'promise_date': source.get('promise_date') or None,
+ 'promise_amount': _number(source.get('promise_amount')),
+ 'priority_score': score,
+ 'priority_reason': ('Overdue balance' if is_overdue else 'Open balance')
+ if amount is not None else None,
+ 'next_action': 'Preview outreach',
+ })
+ ranked.sort(key=lambda row: (
+ row['aging'] != 'Overdue',
+ row['priority_score'] is None,
+ -(row['priority_score'] or 0.0),
+ str(row['document'] or ''),
+ ))
+ return ranked
+
+
def validate(t=None, exposure=None, behavior=None):
"""exposure/behavior: pass precomputed credit_exposure()/days_to_pay() results to avoid
recomputing the heavy pulls when the caller (the page bundle) already has them; validate.py
diff --git a/platform/modules/collections_exec.py b/platform/modules/collections_exec.py
index 6f962e487e51947e23e0901ccfbc162984629d8a..64c799cb5e3176cb47e5bae16ff4f62c7ff8de18 100644
--- a/platform/modules/collections_exec.py
+++ b/platform/modules/collections_exec.py
@@ -356,6 +356,29 @@ def goal_status(key, value):
return target, ('Achieved' if ok else 'Unmet')
+def outreach_preview(item):
+ """Build a no-send collections preview from a mirror worklist item.
+
+ This seam deliberately has no mail client, recipient lookup, queue call, or mutation. The
+ ordinary Finance workflow can therefore help an owner prepare follow-up language while the
+ separately safeguarded statements sender remains the only sanctioned outward path.
+ """
+ item = item if isinstance(item, dict) else {}
+ customer = str(item.get('customer') or 'Unavailable customer')
+ document = str(item.get('document') or 'Unavailable document')
+ amount = item.get('amount')
+ amount_text = (f"${amount:,.2f}" if isinstance(amount, (int, float)) and not isinstance(amount, bool)
+ else 'an unavailable balance')
+ return {
+ 'mode': 'preview_only',
+ 'send_allowed': False,
+ 'recipient': None,
+ 'subject': f'Account follow-up for {document}',
+ 'body': (f'{customer}: please review {document}, which currently shows '
+ f'{amount_text} outstanding. Confirm the source record before any follow-up.'),
+ }
+
+
def validate(data, t=None):
"""Aging buckets (conserved) tie to BU receivable; the Overdue list total ties to the summary's
current overdue (the two independent code paths agree)."""
diff --git a/platform/modules/collections_send.py b/platform/modules/collections_send.py
index 6f1bc73bdb0fc316c2d438925a7d10f36e09ffd8..58854d9537c8cdbf1a9b362ba3c3a70ecf0a51e5 100644
--- a/platform/modules/collections_send.py
+++ b/platform/modules/collections_send.py
@@ -14,7 +14,7 @@ hard-blocking on all writes. The UI gates the section to admin users; SAFE_MODE
in the data layer so the UI cannot bypass it.
Env (Space secrets / .env): ODOO_URL, ODOO_DB, ODOO_USER, ODOO_API_KEY
-Optional: SAFE_MODE (default ON), SAFE_RECIPIENTS, SENDER_NAME, SENDER_EMAIL, REPLY_TO,
+Optional: SAFE_MODE (default ON; only 0/false/no turn it off), SAFE_RECIPIENTS, SENDER_NAME, SENDER_EMAIL, REPLY_TO,
COMPANY_NAME, ROYAL_MAIL_SERVER_ID, ROYAL_AUTHOR_ID
"""
import os
@@ -59,9 +59,24 @@ class SafeModeBlocked(RuntimeError):
# --- Testing guardrail -------------------------------------------------------
# While SAFE_MODE is on, NO email can be queued to any address outside the allow-list — enforced
-# here in the data layer so the UI cannot bypass it. Default: ON. To go live for real customers,
-# set the Space secret SAFE_MODE=0.
-SAFE_MODE = os.environ.get('SAFE_MODE', '1').strip().lower() not in ('0', 'false', 'no', '')
+# here in the data layer so the UI cannot bypass it. Default: ON; only an explicit recognised
+# opt-out turns it off. An empty or blank value is NOT an opt-out.
+# To go live for real customers, set the Space secret SAFE_MODE=0.
+#: The ONLY values that turn the guardrail off. Anything else -- unset, empty, whitespace,
+#: or a token nobody recognises -- leaves SAFE_MODE ON. [D-349]
+SAFE_MODE_OPT_OUT = frozenset({'0', 'false', 'no'})
+
+
+def safe_mode_from(value):
+ """True (guardrail ON) unless `value` is one of the recognised opt-out tokens.
+
+ Fails CLOSED on purpose: an empty or blank SAFE_MODE used to read as an opt-out, which
+ silently disarmed the only guard on the one path in the product allowed to write to Odoo.
+ """
+ return str(value if value is not None else '').strip().lower() not in SAFE_MODE_OPT_OUT
+
+
+SAFE_MODE = safe_mode_from(os.environ.get('SAFE_MODE', '1'))
SAFE_RECIPIENTS = {a.strip().lower() for a in
os.environ.get('SAFE_RECIPIENTS', 'farhan@teamroyalimports.com').split(',')
if a.strip()}
@@ -265,3 +280,25 @@ def queue_statement(odoo, row, subject_tpl=DEFAULT_SUBJECT,
'res_id': row['partner_id'],
}
return odoo.queue_mail(payload)
+
+
+def queue_statement_batch(odoo, rows, subject_tpl=DEFAULT_SUBJECT,
+ intro_tpl=DEFAULT_INTRO, footer_tpl=DEFAULT_FOOTER):
+ """Queue one statement per row through the existing guarded statement sender.
+
+ A missing address or a safe-mode refusal does not stop the remaining rows.
+ """
+ created, refused = [], []
+ for row in rows:
+ try:
+ mail_id = queue_statement(odoo, row, subject_tpl, intro_tpl, footer_tpl)
+ except SafeModeBlocked:
+ refused.append({"customer": row["Customer"],
+ "reason": "Safe mode does not allow this recipient address."})
+ except ValueError:
+ reason = ("No email address is saved for this customer." if not row.get("Email")
+ else "The statement could not be prepared from the current customer details.")
+ refused.append({"customer": row["Customer"], "reason": reason})
+ else:
+ created.append({"customer": row["Customer"], "to": row["Email"], "mailId": mail_id})
+ return {"created": created, "refused": refused, "safe_mode": bool(SAFE_MODE)}
diff --git a/platform/modules/customer_data.py b/platform/modules/customer_data.py
index 9a08488ed2587c2c10032094300c10e90b59736c..eb768ddfce7ccb86d1322630aa8d618e8d346437 100644
--- a/platform/modules/customer_data.py
+++ b/platform/modules/customer_data.py
@@ -360,20 +360,21 @@ def validate(team_id=None):
checks = []
# ── the population, and the three numbers that keep the address legs honest ───────────────
- # An Odoo `search` excludes archived records by default, so `n_visible` counts the partners
- # that are BOTH live and ours. Three populations therefore ride the pool and only one of them
- # can carry an address: rows Odoo can see, rows `_reconcile_ledger` REVIVED (I13 — Odoo
- # dropped them; every attribute is written '(none)'), and rows that are neither, i.e. archived
- # in Odoo but never retained. The third is the one that grows silently, so it is REPORTED as
- # a number rather than absorbed into a tolerance.
- revived = [r for r in rows if r.get('odoo_status') != 'Active']
+ # Every source query spells both active states explicitly. Otherwise Odoo's `active_test`
+ # context would exclude archived rows from the independent side while the pool now correctly
+ # includes their addresses, making the widened population look like an address defect.
+ # An archived Odoo record still exists at source and can carry current attributes. Only a
+ # ledger-only row was genuinely revived after disappearing from Odoo.
+ revived = [r for r in rows if r.get('_ledger_revived') is True]
rpids = [int(r['pid']) for r in revived if r.get('pid')]
- n_visible = odoo.search_count('res.partner', [('id', 'in', pids)])
+ archive_domain = [('active', 'in', [True, False])]
+ n_visible = odoo.search_count('res.partner', [('id', 'in', pids)] + archive_domain)
# ⛔ ASSERTED, NOT ASSUMED. "The retained rows are the ones Odoo lost" is trivially true when
# checked against the ledger that did the retaining — that is the self-sealing shape. So it is
# asked of ODOO: a retained pid Odoo can still see means `_reconcile_ledger` buried a LIVE
# customer's address behind six '(none)' cells.
- still_live = odoo.search_count('res.partner', [('id', 'in', rpids)]) if rpids else 0
+ still_live = (odoo.search_count('res.partner', [('id', 'in', rpids)] + archive_domain)
+ if rpids else 0)
checks.append({
'check': "every RETAINED customer (I13) is one Odoo genuinely cannot see; asked of "
"Odoo, never of the ledger that did the retaining",
@@ -387,7 +388,8 @@ def validate(team_id=None):
# so the oracle cannot inherit a bug from `_partner_attrs`, which is the code under test.
for key, field in ADDRESS_FIELDS.items():
ours = sum(1 for r in rows if not address_blank(r.get(key)))
- theirs = odoo.search_count('res.partner', [('id', 'in', pids), (field, '!=', False)])
+ theirs = odoo.search_count(
+ 'res.partner', [('id', 'in', pids), (field, '!=', False)] + archive_domain)
checks.append({
'check': f"the grid's `{key}` column is non-blank on exactly the pool customers Odoo "
f"says carry `{field}`; an INDEPENDENT search_count over the pool's own "
@@ -409,7 +411,7 @@ def validate(team_id=None):
if pids:
# `O.search_read`, the same door `_partner_attrs` reads through — the client's own method
# takes `domain=`/`fields=` as keywords and this wrapper is the one with a fixed shape.
- for got in O.search_read('res.partner', [('id', 'in', pids)], text_keys):
+ for got in O.search_read('res.partner', [('id', 'in', pids)] + archive_domain, text_keys):
for k in text_keys:
v = got.get(k)
if isinstance(v, str) and v and not v.strip():
@@ -465,7 +467,7 @@ def _book_pids(team_id=None, agent_pids=None):
customer assigned to an agent who has never ordered — precisely a retargeting target — did not
exist in the app at all.
- R3's definition, MEASURED against live Odoo 2026-08-05:
+ R3's original definition was measured against live Odoo 2026-08-05:
customer_rank>0 active .............. 3,614
ever ordered (confirmed, team 5/6) ... 1,748
UNION ............................... 3,723 (vs 1,555 in the old 24-month pool)
@@ -484,11 +486,16 @@ def _book_pids(team_id=None, agent_pids=None):
it — so the leg is subsumed rather than dropped. Item 27's actual ask (the 126 assigned
partners with no `sale.order` at all) is fully served: they are rank>0 and they are in.
- ⚠ ACTIVE ONLY. Archived partners stay out (R3): they are ex-customers, and putting them in
- every count would make "how many customers do we have" unanswerable. The agent-LOGIN scope
- (`customers.agent_partner_ids`) deliberately still includes archived — an agent's own book is
- their whole history — and the two remain compatible because that set is INTERSECTED with this
- pool, so archived rows drop out of the table without narrowing the agent's own permissions.
+ S63-T14 widens the rank leg to both Odoo active states. Archived accounts are source rows, not
+ deleted rows: the shipped default View owns the ordinary removable `Archived is not true`
+ condition. Keeping that distinction at the View layer is what lets removing the condition
+ reveal the same row under ordinary permissions instead of requiring a second data path.
+
+ S63-T15 makes the unscoped registry use the complete identity union already owned by the Odoo
+ customer source/mirror contract: rank-positive partners, every confirmed-order partner, and
+ every posted customer-document partner. The document leg is load-bearing for direct invoices
+ with no sale order and for historical records whose rank was reset. It is identity only;
+ revenue and permissions remain scoped by their own evaluators.
⚠ BU SCOPE IS APPLIED THROUGH ORDERS, NOT THROUGH THE PARTNER. `res.partner` carries no team,
so a BU-scoped caller gets the partners who have ORDERED in that BU (plus their own agent
@@ -507,17 +514,25 @@ def _book_pids(team_id=None, agent_pids=None):
book.discard(None)
else:
rank = O.search_read('res.partner',
- [('customer_rank', '>', 0), ('active', '=', True)],
+ [('customer_rank', '>', 0),
+ ('active', 'in', [True, False])],
['id'], limit=200000)
# `ever` is NOT redundant with rank>0: a partner can be archived-then-reactivated, or
# have had its rank reset, and an account that demonstrably bought from us belongs in
# the book whatever its flags say now. An ORDER is a fact; a rank is a setting.
rows = O.read_group('sale.order',
- [('state', 'in', ('sale', 'done')), ('team_id', 'in', (5, 6))],
+ [('state', 'in', ('sale', 'done'))],
['partner_id'], ['partner_id'], lazy=False)
ever = {O.m2o_id(r.get('partner_id')) for r in rows}
ever.discard(None)
- book = {r['id'] for r in rank} | ever
+ docs = O.read_group(
+ 'account.move',
+ [('state', '=', 'posted'),
+ ('move_type', 'in', ('out_invoice', 'out_refund'))],
+ ['partner_id'], ['partner_id'], lazy=False)
+ documented = {O.m2o_id(r.get('partner_id')) for r in docs}
+ documented.discard(None)
+ book = {r['id'] for r in rank} | ever | documented
# An agent filter NARROWS the book to that agent's partners — never widens it. The
# intersection is what keeps an agent-login user inside their own book while still
# gaining every no-order account assigned to them, which is the whole point of item 27.
@@ -543,8 +558,15 @@ def _book_pids(team_id=None, agent_pids=None):
return set()
-def _pool_build(agent_name, team_id, limit, fast=False):
- t = P.today()
+def _pool_identity(agent_name=None, team_id=None, t=None):
+ """The canonical customer row universe and the source maps that establish it.
+
+ This is the identity half of ``_pool_build``, extracted so sync reconciliation can compare
+ the mirror with the actual compiled database without paying for address, AR, product-mix,
+ mapping, or ledger cell families. There is still one evaluator: the pool itself consumes
+ this result.
+ """
+ t = t or P.today()
yf, yt = P.ytd(t)
lf, lt = P.ytd_last_year(t)
mf, mt = P.ltm(t)
@@ -554,6 +576,21 @@ def _pool_build(agent_name, team_id, limit, fast=False):
ltm = cust._cust_rev(mf, mt, team_id, pids)
cad = cust._cadence_bulk(t, team_id, agent_pids=pids)
all_pids = set(this) | set(last) | set(cad) | _book_pids(team_id, pids)
+ return {"today": t, "agentPids": pids, "this": this, "last": last,
+ "ltm": ltm, "cadence": cad, "pids": all_pids}
+
+
+def identity_pids(agent_name=None, team_id=None, t=None):
+ """The exact Odoo partner ids the compiled customer database will build before retention."""
+ return set(_pool_identity(agent_name, team_id, t)["pids"])
+
+
+def _pool_build(agent_name, team_id, limit, fast=False):
+ identity = _pool_identity(agent_name, team_id)
+ t = identity["today"]
+ pids = identity["agentPids"]
+ this, last, ltm = identity["this"], identity["last"], identity["ltm"]
+ cad, all_pids = identity["cadence"], identity["pids"]
total = len(all_pids)
if limit is not None and total > limit:
# the SAME ordering the assembled pool ships (see the sort below) — the slice must be
@@ -587,7 +624,8 @@ def _pool_build(agent_name, team_id, limit, fast=False):
if not ((this.get(pid) or {}).get('name') or (last.get(pid) or {}).get('name')
or (ltm.get(pid) or {}).get('name'))]
extra_names = ({r['id']: r['name'] for r in
- O.search_read('res.partner', [('id', 'in', unnamed)], ['name'])}
+ O.search_read('res.partner', [('id', 'in', unnamed),
+ ('active', 'in', [True, False])], ['name'])}
if unnamed else {})
rows = []
for pid in all_pids:
@@ -608,6 +646,9 @@ def _pool_build(agent_name, team_id, limit, fast=False):
'customer': tv.get('name') or lv.get('name')
or (ltm.get(pid) or {}).get('name') or extra_names.get(pid) or '?',
'status': status,
+ # A normal source boolean. Views decide whether to hide it; the population never
+ # disappears an archived source record below the permission/filter layers.
+ 'archived': bool(a.get('archived', False)),
'agent': a.get('agent', '(none)'),
# ⭐ W30-T33 (carried W29-T51) — the street lines beside the city they belong to.
'street': a.get('street', '(none)'), 'street2': a.get('street2', '(none)'),
@@ -667,7 +708,10 @@ def _pool_build(agent_name, team_id, limit, fast=False):
'typical_gap_days': gap, 'overdue_days': overdue,
})
for r in rows:
- r['odoo_status'] = 'Active'
+ # Kept for compatibility with existing saved layouts while `archived` is the canonical
+ # boolean new filters use.
+ r['odoo_status'] = 'Archived' if r['archived'] else 'Active'
+ r['_ledger_revived'] = False
# I13 (owner item 13, contract C6): RETAIN customers that disappear from Odoo.
# Only the FULL build reconciles the ledger — the fast slice is 100 rows by construction,
# so letting it write would mark ~1,450 living customers archived on every cold load.
@@ -728,6 +772,7 @@ def _reconcile_ledger(rows, total, agent_name, team_id):
revived.append({
'pid': pid, 'customer': e.get('name') or '?', 'status': 'Dormant',
'odoo_status': 'Archived',
+ 'archived': True, '_ledger_revived': True,
# Everything else is genuinely UNKNOWN now — the record is gone. Blank is the
# honest rendering; carrying the last-known numbers forward would state figures
# as current that nothing can reconcile ([[no-unverifiable-aggregates]]).
diff --git a/platform/modules/customers.py b/platform/modules/customers.py
index 0872f7bff3739c02b79e8ff0decbfb2197ae33d6..95369bcd366b36555795ad6cb47b0a7f427902b2 100644
--- a/platform/modules/customers.py
+++ b/platform/modules/customers.py
@@ -101,11 +101,10 @@ def agent_partner_ids(agent_name, customers_only=False):
this set, so it must be GENEROUS: archived accounts and Odoo address records included.
Narrowing it would hide an agent's own data from them, and the fails-closed trap in the
note below is what that costs.
- * **Display/reporting (`True`)** — "how many accounts does Martin have" must answer what
- Odoo answers. MEASURED 2026-08-05: the generous set is 503 for Martin and the honest one
- is **494**, the owner's number; the 9-row gap is entirely `type in (delivery, other)`
- ADDRESS records that ride the pool because they appear on orders, two of them with no
- name at all.
+ * **Display/reporting (`True`)** — a customer is a `customer_rank > 0` account, whether
+ active or archived. Address records stay excluded by rank while archived accounts remain
+ available to the ordinary `Archived is not true` View filter instead of disappearing
+ below the View layer.
Both are the SAME m2m-contains resolver — which is D-30's actual requirement. The bug was
never the generosity; it was that the Agent COLUMN used a THIRD rule (`agent_ids[0]`, first
@@ -122,9 +121,9 @@ def agent_partner_ids(agent_name, customers_only=False):
if not agent_name or agent_name in ('All agents', 'All'):
return None
_all = [('active', 'in', [True, False])]
- # The display policy: a real customer record, still active. See the docstring for the
- # measured 503-vs-494 this closes.
- _qual = [('customer_rank', '>', 0), ('active', '=', True)] if customers_only else _all
+ # A View owns archive visibility. The source population must include both boolean states or
+ # removing that View filter cannot reveal an archived account.
+ _qual = ([('customer_rank', '>', 0)] + _all) if customers_only else _all
if agent_name == '(none)':
rows = O.search_read('res.partner', [('customer_rank', '>', 0),
('agent_ids', '=', False)] + _all, ['id'], limit=100000)
@@ -585,10 +584,14 @@ def _partner_attrs(pids):
pids = list(pids)
if not pids:
return {}
- rows = O.search_read('res.partner', [('id', 'in', pids)],
+ # Odoo's active_test context otherwise hides archived partners even when their ids are named.
+ # The explicit two-state domain makes `archived` an ordinary source attribute rather than a
+ # conclusion inferred from a missing row.
+ rows = O.search_read('res.partner', [('id', 'in', pids),
+ ('active', 'in', [True, False])],
['street', 'street2', 'city', 'state_id', 'country_id', 'agent_ids',
'zip', 'property_payment_term_id', 'create_date', 'category_id',
- 'property_product_pricelist'])
+ 'property_product_pricelist', 'active'])
aids = {a for r in rows for a in (r.get('agent_ids') or [])}
anames = ({p['id']: p['name'] for p in O.search_read('res.partner', [('id', 'in', list(aids))], ['name'])}
if aids else {})
@@ -607,6 +610,7 @@ def _partner_attrs(pids):
# compares dates ISO-LEXICALLY, so a trailing time would sort and filter as text noise.
since = (r.get('create_date') or '')
out[r['id']] = {
+ 'archived': not bool(r.get('active', True)),
# ⭐ W30-T33 (carried W29-T51) — the STREET half of the address. `city`/`state`/`zip`/
# `country` were already here; the two street lines were read nowhere on the customer
# path, only by `modules/map.py` for geocoding, and a geocoder's input never reached
diff --git a/platform/modules/finance_apps.py b/platform/modules/finance_apps.py
new file mode 100644
index 0000000000000000000000000000000000000000..8d74a73311f5403474ebd822c32fb60c0cd10307
--- /dev/null
+++ b/platform/modules/finance_apps.py
@@ -0,0 +1,219 @@
+"""Static Finance application catalogue.
+
+The six entries below are navigation/application contracts, not data stores. Every fact key
+resolves through :mod:`modules.finance_spine`, which keeps one governed definition of scope,
+freshness, lineage, drill-through, and reconciliation for every consumer.
+"""
+from copy import deepcopy
+
+from modules import financial, health, inventory, management, o2c, procurement
+
+
+FINANCE_GROUP = {
+ "key": "finance",
+ "label": "Finance",
+ "description": "Receivables, spend, cash, margin, inventory, and planning.",
+ "routePrefix": "/finance",
+}
+
+
+# Workflow ownership lives here, at the application layer. Source modules describe only the
+# governed-data requirement and its read-only validation seam; they do not create queue state.
+_OWNED_WORK = {
+ "open_purchase_orders": ("Buyer", "Chase the late purchase order"),
+ "lead_time_coverage": ("Buyer", "Confirm the replenishment lead time"),
+ "replenishment": ("Buyer", "Create the replenishment plan"),
+ "inventory_value": ("Inventory manager", "Reconcile the inventory valuation"),
+ "excess_stock": ("Inventory manager", "Disposition the excess stock"),
+ "dead_stock": ("Inventory manager", "Choose the dead-stock disposition"),
+}
+
+
+def _owned_work_signals():
+ """Compose non-executable owned-work requirements without manufacturing queue records."""
+ signals = procurement.finance_workflow_contract() + inventory.finance_workflow_contract()
+ for signal in signals:
+ signal["state"] = signal.pop("availability")
+ signal["ownerRole"], signal["nextAction"] = _OWNED_WORK[signal["key"]]
+ signal["status"] = "unavailable"
+ return signals
+
+
+FINANCE_APPLICATIONS = (
+ {
+ "key": "ar-collections",
+ "label": "Collection",
+ "navLabel": "Collection",
+ "description": "Prioritize customer balances that need collection action.",
+ "mode": "workflow",
+ "permissionKey": "ar",
+ "composes": ["ar", "collections"],
+ "factKeys": ["ar_outstanding"],
+ "kpiKeys": ["ar_outstanding"],
+ "worklist": {
+ "title": "Priority queue",
+ "cadence": "Daily, 15 min",
+ "sourceFact": "ar_outstanding",
+ "pageSize": 500,
+ "maxRows": 10000,
+ "displayLimit": 50,
+ "entityField": "partner",
+ "documentField": "move",
+ "amountField": "ar_outstanding",
+ "overdueAmountField": "ar_overdue",
+ "dueDateField": "due_date",
+ "ownerField": "collection_owner",
+ "statusField": "payment_state",
+ "promiseDateField": "promise_date",
+ "promiseAmountField": "promise_amount",
+ "nextAction": "Preview outreach",
+ "previewOnly": True,
+ },
+ "question": "Which customer balances need an owner and a next action?",
+ "deepLink": "/finance/ar-collections",
+ },
+ {
+ "key": "ap-spend",
+ "label": "AP & Spend",
+ "navLabel": "AP & Spend",
+ "description": "Review supplier obligations before payment.",
+ "mode": "workflow",
+ "permissionKey": "spend",
+ "composes": ["spend", "vendor_bills"],
+ "factKeys": ["ap_outstanding"],
+ "kpiKeys": ["ap_outstanding"],
+ "worklist": {
+ "title": "Payment queue",
+ "cadence": "Daily, 15 min",
+ "sourceFact": "ap_outstanding",
+ "pageSize": 500,
+ "maxRows": 10000,
+ "displayLimit": 50,
+ "entityField": "vendor",
+ "documentField": "move",
+ "amountField": "ap_outstanding",
+ "dueDateField": "due_date",
+ "ownerField": "ap_owner",
+ "statusField": "payment_state",
+ "nextAction": "Review payable",
+ "previewOnly": False,
+ "concentration": True,
+ },
+ "question": "Which supplier obligations need review or payment action?",
+ "deepLink": "/finance/ap-spend",
+ },
+ {
+ "key": "cash-o2c",
+ "label": "Cash & O2C",
+ "navLabel": "Cash & O2C",
+ "description": "Track how invoiced demand converts into cash.",
+ "mode": "dashboard",
+ "permissionKey": "o2c",
+ "composes": ["o2c", "ar"],
+ "factKeys": ["invoiced_sales", "ar_outstanding"],
+ "kpiKeys": ["invoiced_sales", "ar_outstanding"],
+ "dashboard": {
+ "basisNotice": "Mirror facts use the selected window. Cycle measures name their own LTM or point-in-time basis.",
+ "sections": [
+ o2c.dashboard_contract(),
+ financial.cash_cycle_dashboard_contract(),
+ ],
+ },
+ "question": "How is invoiced demand converting into collectible cash?",
+ "deepLink": "/finance/cash-o2c",
+ },
+ {
+ "key": "pnl-gross-margin",
+ "label": "Management P&L & Gross Margin",
+ "navLabel": "P&L & Margin",
+ "description": "Understand what drives operating performance.",
+ "mode": "dashboard",
+ "permissionKey": "financial",
+ "composes": ["financial", "sales"],
+ "factKeys": ["revenue", "opex", "commission_accrued"],
+ "kpiKeys": ["revenue", "opex", "commission_accrued"],
+ "dashboard": {
+ "basisNotice": "Order, posted-GL, and commission actuals retain separate source bases and are never silently summed.",
+ "sections": [
+ management.comparison_dashboard_contract(),
+ financial.margin_dashboard_contract(),
+ ],
+ },
+ "question": "Where are revenue, operating expense, and selling cost moving?",
+ "deepLink": "/finance/pnl-gross-margin",
+ },
+ {
+ "key": "procurement-inventory",
+ "label": "Procurement & Inventory",
+ "navLabel": "Procurement & Inventory",
+ "description": "Prioritize buying and warehouse work from stock movements.",
+ "mode": "workflow",
+ "permissionKey": "procurement",
+ "composes": ["procurement", "inventory"],
+ "factKeys": ["stock_in", "stock_out", "stock_net"],
+ "kpiKeys": [],
+ "worklist": {
+ "kind": "unavailable_requirements",
+ "title": "Owned work requirements",
+ "cadence": "Daily, 15 min",
+ "basisNotice": ("Inventory and demand use one physical all-channel warehouse basis. "
+ "Open purchase orders stay company-wide."),
+ "signals": _owned_work_signals(),
+ },
+ "question": "Which physical stock movements require a buying or warehouse action?",
+ "deepLink": "/finance/procurement-inventory",
+ },
+ {
+ "key": "planning-data-health",
+ "label": "Planning & Data Health",
+ "navLabel": "Planning & Data Health",
+ "description": "Assess whether planning inputs are ready to use.",
+ "mode": "dashboard",
+ "permissionKey": "health",
+ "composes": ["planning", "health"],
+ "factKeys": ["revenue", "invoiced_sales", "commission_accrued"],
+ "kpiKeys": ["revenue", "invoiced_sales", "commission_accrued"],
+ "dashboard": {
+ "basisNotice": ("Actuals come only from governed Finance facts. Planning and close "
+ "exceptions remain named gaps until their canonical mirror facts exist."),
+ "sections": [
+ health.planning_dashboard_contract(),
+ *health.data_health_dashboard_contracts(),
+ ],
+ },
+ "question": "Are planning inputs fresh, traceable, and reconciled?",
+ "deepLink": "/finance/planning-data-health",
+ },
+)
+
+
+def finance_group():
+ """Return a caller-owned copy of the one Finance navigation group."""
+ return deepcopy(FINANCE_GROUP)
+
+
+# SDLC 64 / R6: also feeds GET /nav/applications through
+# application_registry.py::finance_application_records. platform/core/registry.py has no Finance
+# row, so may_open here is the ONLY grant between a session and a Finance application.
+def finance_applications(may_open=None):
+ """Return immutable descriptors, optionally filtered through the request permission wall."""
+ rows = FINANCE_APPLICATIONS
+ if may_open is not None:
+ rows = tuple(app for app in rows if may_open(app["permissionKey"]))
+ return deepcopy(list(rows))
+
+
+def finance_application(key, may_open=None):
+ """Resolve a direct-link descriptor without introducing a Finance-home aggregate."""
+ for app in FINANCE_APPLICATIONS:
+ if app["key"] == key:
+ if may_open is not None and not may_open(app["permissionKey"]):
+ raise PermissionError(f"access to {key} is not granted")
+ return deepcopy(app)
+ raise KeyError(key)
+
+
+def finance_templates(may_open=None):
+ """Offer one concise template per Finance application the caller may open."""
+ return [{"key": app["key"], "label": app["label"], "description": app["description"]}
+ for app in finance_applications(may_open=may_open)]
diff --git a/platform/modules/finance_spine.py b/platform/modules/finance_spine.py
new file mode 100644
index 0000000000000000000000000000000000000000..66a327b7039586046b3eef9b1c4ae43eaffcba4b
--- /dev/null
+++ b/platform/modules/finance_spine.py
@@ -0,0 +1,276 @@
+"""Canonical Finance fact, proof, freshness, lineage, and drill plane.
+
+Applications name facts from this module; they never copy numbers into an application store.
+Interactive values come only from the tenant mirror through ``harness.semantic``. Independent
+reconciliation reads live Odoo only when the caller explicitly requests it.
+"""
+from copy import deepcopy
+
+
+_COMMON = {
+ "source": {"observation": "tenant_mirror", "proof": "odoo_read_only"},
+ "drillThrough": {"kind": "source_records"},
+}
+
+
+FINANCE_FACTS = {
+ "ar_outstanding": {
+ "label": "AR outstanding", "topic": "receivables", "metric": "ar_outstanding",
+ "format": "usd", "scope": "hq_consolidated", "sourceEntities": ["account_move"],
+ "reconciliationMethod": "ar_outstanding",
+ },
+ "ap_outstanding": {
+ "label": "AP outstanding", "topic": "odoo_bills", "metric": "ap_outstanding",
+ "format": "usd", "scope": "hq_consolidated", "sourceEntities": ["account_move"],
+ "reconciliationMethod": "ap_outstanding",
+ },
+ "invoiced_sales": {
+ "label": "Invoiced sales", "topic": "customer_invoices", "metric": "invoiced_sales",
+ "format": "usd", "scope": "hq_consolidated", "sourceEntities": ["account_move"],
+ "reconciliationMethod": "invoiced_sales",
+ },
+ "revenue": {
+ "label": "Wholesale revenue", "topic": "sales_lines", "metric": "revenue",
+ "format": "usd", "scope": "business_unit", "sourceEntities": ["sale_order_line", "sale_order"],
+ "reconciliationMethod": "order_level_revenue",
+ },
+ "opex": {
+ "label": "Operating expense", "topic": "gl_lines", "metric": "opex",
+ "format": "usd", "scope": "hq_consolidated", "sourceEntities": ["account_move_line"],
+ "reconciliationMethod": "gl_opex",
+ },
+ "commission_accrued": {
+ "label": "Commission accrued", "topic": "commission_lines", "metric": "commission_accrued",
+ "format": "usd", "scope": "business_unit", "sourceEntities": ["account_invoice_line_agent", "account_move_line", "sale_order"],
+ "reconciliationMethod": "commission_accrued",
+ },
+ "stock_in": {
+ "label": "Stock moved in", "topic": "stock_moves", "metric": "stock_in",
+ "format": "int", "scope": "all_channel_inventory", "sourceEntities": ["stock_move", "stock_location"],
+ "reconciliationMethod": "stock_movement_in", "scalarAvailable": False, "drillMeasures": [],
+ "requiredGrain": "product_code",
+ },
+ "stock_out": {
+ "label": "Stock moved out", "topic": "stock_moves", "metric": "stock_out",
+ "format": "int", "scope": "all_channel_inventory", "sourceEntities": ["stock_move", "stock_location"],
+ "reconciliationMethod": "stock_movement_out", "scalarAvailable": False, "drillMeasures": [],
+ "requiredGrain": "product_code",
+ },
+ "stock_net": {
+ "label": "Stock moved net", "topic": "stock_moves", "metric": "stock_net",
+ "format": "int", "scope": "all_channel_inventory", "sourceEntities": ["stock_move", "stock_location"],
+ "reconciliationMethod": "stock_movement_net", "drillMeasures": [],
+ "scalarAvailable": False, "requiredGrain": "product_code",
+ },
+}
+
+
+def fact_spec(key):
+ """Return the governed descriptor for a Finance fact."""
+ if key not in FINANCE_FACTS:
+ raise KeyError(key)
+ item = {"key": key, **_COMMON, **FINANCE_FACTS[key]}
+ item["lineage"] = {
+ "topic": item["topic"], "metric": item["metric"],
+ "sourceEntities": list(item["sourceEntities"]),
+ }
+ item["drillThrough"] = {
+ **item["drillThrough"],
+ "href": f"/api/v1/finance/facts/{key}/records",
+ }
+ item["freshness"] = {"state": "not_observed", "sources": list(item["sourceEntities"])}
+ item["reconciliation"] = {
+ "state": "not_checked", "method": item["reconciliationMethod"],
+ "href": f"/api/v1/finance/facts/{key}/reconciliation",
+ }
+ return deepcopy(item)
+
+
+def fact_specs(keys=None):
+ return [fact_spec(key) for key in (FINANCE_FACTS if keys is None else keys)]
+
+
+def effective_team_id(spec, requested_team_id):
+ """Apply the declared scope. Company and stock facts cannot wear a BU label."""
+ if requested_team_id not in (None, 5, 6):
+ raise ValueError("Finance business-unit scope must be Fisch (5) or Royal (6)")
+ if spec["scope"] == "business_unit":
+ return requested_team_id
+ if spec["scope"] == "hq_consolidated" and requested_team_id is not None:
+ raise PermissionError("A business-unit-scoped account cannot read consolidated Finance facts")
+ # Inventory is one physical, all-channel warehouse basis. It is intentionally not allocated
+ # to Fisch or Royal; a pinned user receives the same company stock rows with no BU label.
+ return None
+
+
+def freshness(cursor, entities):
+ """Describe mirror readiness from its own sync ledger; never synthesize an as-of time."""
+ try:
+ rows = cursor.execute(
+ "SELECT entity, phase, cursor_wd, updated_at FROM _sync_state WHERE entity IN ("
+ + ",".join("?" for _ in entities) + ")", list(entities)).fetchall()
+ except Exception:
+ return {"state": "missing", "asOf": None, "sources": []}
+ found = {str(row[0]): row for row in rows}
+ sources = []
+ for entity in entities:
+ row = found.get(entity)
+ sources.append({"entity": entity, "state": str(row[1]) if row else "missing",
+ "asOf": (row[2] or row[3]) if row else None})
+ phases = {src["state"] for src in sources}
+ if sources and phases == {"live"}:
+ state = "ready"
+ elif "running" in phases or "pending" in phases:
+ state = "warming"
+ else:
+ state = "missing"
+ stamps = [src["asOf"] for src in sources if src["asOf"]]
+ return {"state": state, "asOf": min(stamps) if stamps else None, "sources": sources}
+
+
+def observe(key, cursor, *, date_from=None, date_to=None, team_id=None):
+ """Read one governed fact from the canonical mirror and attach its proof metadata."""
+ from harness import semantic
+ spec = fact_spec(key)
+ sync = freshness(cursor, spec["sourceEntities"])
+ spec["freshness"] = sync
+ if sync["state"] != "ready":
+ return {**spec, "state": sync["state"], "value": None}
+ if spec.get("scalarAvailable") is False:
+ return {**spec, "state": "requires_sku", "value": None,
+ "message": "Movement quantities use mixed units and are available at SKU grain."}
+ try:
+ scoped_team_id = effective_team_id(spec, team_id)
+ except PermissionError:
+ return {**spec, "state": "restricted", "value": None,
+ "message": "This fact is available only to consolidated Finance viewers."}
+ except ValueError as exc:
+ return {**spec, "state": "unavailable", "value": None, "message": str(exc)}
+ try:
+ result = semantic.store_query(
+ spec["topic"], [spec["metric"]], date_from=date_from, date_to=date_to,
+ team_id=scoped_team_id, limit=1)
+ rows = result.get("rows") or []
+ if not rows or rows[0].get(spec["metric"]) is None:
+ return {**spec, "state": "missing", "value": None}
+ return {**spec, "state": "ready", "value": rows[0][spec["metric"]]}
+ except Exception as exc:
+ return {**spec, "state": "unavailable", "value": None,
+ "message": _safe_message(exc)}
+
+
+def source_records(key, *, date_from=None, date_to=None, team_id=None, limit=100, offset=0):
+ """Return source-grain records plus whole-scope counts/totals from the same governed topic."""
+ from harness import semantic
+ spec = fact_spec(key)
+ measures = (spec.get("drillMeasures") if "drillMeasures" in spec
+ else [spec["metric"]])
+ payload = semantic.store_rows(
+ spec["topic"], date_from=date_from, date_to=date_to,
+ team_id=effective_team_id(spec, team_id), limit=limit, offset=offset,
+ aggregates=measures)
+ return {"fact": spec, **payload}
+
+
+def reconcile(key, *, date_from=None, date_to=None, team_id=None):
+ """Run the declared independent, read-only Odoo proof on explicit request."""
+ from harness import semantic
+ spec = fact_spec(key)
+ try:
+ result = semantic.validate_metric(
+ spec["metric"], date_from=date_from, date_to=date_to,
+ team_id=effective_team_id(spec, team_id))
+ return {"state": "passed" if result.get("ok") else "failed", **result}
+ except Exception as exc:
+ return {"state": "unavailable", "message": _safe_message(exc)}
+
+
+def _safe_message(exc):
+ text = str(exc).lower()
+ if "sync" in text or "store" in text or "table" in text:
+ return "The tenant mirror is still warming."
+ return "The fact is temporarily unavailable."
+
+
+def _store_value(topic, metric, date_from, date_to, team_id=None):
+ from harness import semantic
+ rows = semantic.store_query(topic, [metric], date_from=date_from, date_to=date_to,
+ team_id=team_id, limit=1).get("rows") or []
+ return (rows[0].get(metric) if rows else None) or 0
+
+
+def _date_domain(field, date_from, date_to):
+ dom = []
+ if date_from:
+ dom.append((field, ">=", date_from))
+ if date_to:
+ dom.append((field, "<=", date_to))
+ return dom
+
+
+def _v_commission_accrued(date_from, date_to, team_id):
+ import core.odoo as O
+ ours = _store_value("commission_lines", "commission_accrued", date_from, date_to, team_id)
+ dom = _date_domain("invoice_date", date_from, date_to)
+ if team_id:
+ dom.append(("object_id.sale_line_ids.order_id.team_id", "=", team_id))
+ return ours, O.sum_field("account.invoice.line.agent", dom, "amount")
+
+
+def _v_invoiced_sales(date_from, date_to, team_id):
+ import core.odoo as O
+ ours = _store_value("customer_invoices", "invoiced_sales", date_from, date_to)
+ dom = [("state", "=", "posted"), ("move_type", "=", "out_invoice")]
+ dom += _date_domain("invoice_date", date_from, date_to)
+ return ours, O.sum_field("account.move", dom, "amount_untaxed_signed")
+
+
+def _stock_live(date_from, date_to, inbound):
+ import core.odoo as O
+ dom = [("state", "=", "done")]
+ dom += _date_domain("date", date_from, date_to)
+ if inbound:
+ dom += [("location_dest_id.usage", "=", "internal"),
+ ("location_id.usage", "!=", "internal")]
+ else:
+ dom += [("location_id.usage", "=", "internal"),
+ ("location_dest_id.usage", "!=", "internal")]
+ return O.sum_field("stock.move", dom, "quantity_done")
+
+
+def _v_stock_in(date_from, date_to, team_id):
+ return _store_value("stock_moves", "stock_in", date_from, date_to), _stock_live(date_from, date_to, True)
+
+
+def _v_stock_out(date_from, date_to, team_id):
+ return _store_value("stock_moves", "stock_out", date_from, date_to), _stock_live(date_from, date_to, False)
+
+
+def _v_stock_net(date_from, date_to, team_id):
+ return (_store_value("stock_moves", "stock_net", date_from, date_to),
+ _stock_live(date_from, date_to, True) - _stock_live(date_from, date_to, False))
+
+
+def _v_ap_outstanding(date_from, date_to, team_id):
+ import core.odoo as O
+ ours = _store_value("odoo_bills", "ap_outstanding", date_from, date_to)
+ dom = [("state", "=", "posted"), ("move_type", "in", ["in_invoice", "in_refund"])]
+ dom += _date_domain("invoice_date", date_from, date_to)
+ return ours, -O.sum_field("account.move", dom, "amount_residual_signed")
+
+
+FINANCE_VALIDATORS = {
+ "commission_accrued": _v_commission_accrued,
+ "invoiced_sales": _v_invoiced_sales,
+ "stock_movement_in": _v_stock_in,
+ "stock_movement_out": _v_stock_out,
+ "stock_movement_net": _v_stock_net,
+ "ap_outstanding": _v_ap_outstanding,
+}
+
+
+def register_semantic_validators():
+ """Compatibility adapter until the out-of-fence semantic registry imports this module."""
+ from harness import semantic
+ semantic._VALIDATORS.update(FINANCE_VALIDATORS)
+ return tuple(FINANCE_VALIDATORS)
diff --git a/platform/modules/financial.py b/platform/modules/financial.py
index 016f1401c833d9a05c938ead52582bf12396ff7a..c2b535991e4f0dba71af111e5861d147bd3ed127 100644
--- a/platform/modules/financial.py
+++ b/platform/modules/financial.py
@@ -16,6 +16,7 @@ Data-quality caveats surfaced and reported, not hidden:
"""
import sys
import datetime as dt
+from copy import deepcopy
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
import core.odoo as O
@@ -24,6 +25,92 @@ import modules.inventory as inv_mod
import modules.ar as ar_mod
+# These descriptors are intentionally data-free. The Finance application registry may import
+# them on every ordinary page load without opening Odoo or inventing an application-specific
+# store. A cell becomes available only after its named input is promoted into finance_spine's
+# governed tenant-mirror fact plane.
+_CASH_CYCLE_DASHBOARD = {
+ 'key': 'cash-cycle',
+ 'title': 'Cash cycle',
+ 'kind': 'metric_strip',
+ 'basisLabel': 'LTM turnover with point-in-time working-capital balances',
+ 'metrics': [
+ {
+ 'key': 'dso_days', 'label': 'DSO', 'format': 'days',
+ 'state': 'unavailable', 'scope': 'hq_consolidated',
+ 'basisLabel': 'LTM posted customer-invoice sales and point-in-time AR',
+ 'missingSource': 'A dated AR balance snapshot and its LTM posted-invoice denominator are not yet joined in the canonical mirror.',
+ 'validationMethod': 'financial.validate',
+ },
+ {
+ 'key': 'dio_days', 'label': 'DIO', 'format': 'days',
+ 'state': 'unavailable', 'scope': 'hq_consolidated',
+ 'basisLabel': 'LTM posted inventory cost flow and point-in-time inventory value',
+ 'missingSource': 'Canonical inventory value and posted cost-flow facts do not yet share a validated turnover basis.',
+ 'validationMethod': 'financial.validate',
+ },
+ {
+ 'key': 'dpo_days', 'label': 'DPO', 'format': 'days',
+ 'state': 'unavailable', 'scope': 'hq_consolidated',
+ 'basisLabel': 'LTM posted vendor-bill throughput and point-in-time AP',
+ 'missingSource': 'A dated AP balance snapshot and validated vendor-bill throughput are not yet joined in the canonical mirror.',
+ 'validationMethod': 'financial.validate',
+ },
+ {
+ 'key': 'ccc_days', 'label': 'Cash conversion', 'format': 'days',
+ 'state': 'unavailable', 'scope': 'hq_consolidated',
+ 'basisLabel': 'DIO + DSO - DPO on one reconciled LTM basis',
+ 'missingSource': 'Cash conversion stays unavailable until DSO, DIO, and DPO are all available on the same governed basis.',
+ 'validationMethod': 'financial.validate',
+ },
+ ],
+}
+
+
+_MARGIN_DASHBOARD = {
+ 'key': 'margin-drill',
+ 'title': 'Margin drill',
+ 'kind': 'drill_matrix',
+ 'basisLabel': 'Odoo sale-line margin on confirmed wholesale orders',
+ 'metrics': [
+ {
+ 'key': 'category_margin', 'label': 'Category margin', 'format': 'usd',
+ 'state': 'unavailable', 'scope': 'business_unit', 'drillDimension': 'category',
+ 'sourceFact': 'revenue',
+ 'basisLabel': 'Confirmed sale/done wholesale lines; GIFTWARE DEALS excluded; sale-line margin',
+ 'missingSource': 'The canonical Finance spine does not yet expose a validated gross-margin fact grouped by category.',
+ 'validationMethod': 'financial.validate',
+ },
+ {
+ 'key': 'customer_margin', 'label': 'Customer margin', 'format': 'usd',
+ 'state': 'unavailable', 'scope': 'business_unit', 'drillDimension': 'order_partner',
+ 'sourceFact': 'revenue',
+ 'basisLabel': 'Confirmed sale/done wholesale lines; GIFTWARE DEALS excluded; sale-line margin',
+ 'missingSource': 'The canonical Finance spine does not yet expose a validated gross-margin fact grouped by customer.',
+ 'validationMethod': 'financial.validate',
+ },
+ {
+ 'key': 'sku_margin', 'label': 'SKU margin', 'format': 'usd',
+ 'state': 'unavailable', 'scope': 'business_unit', 'drillDimension': 'product_code',
+ 'sourceFact': 'revenue',
+ 'basisLabel': 'Confirmed sale/done wholesale lines; GIFTWARE DEALS excluded; sale-line margin',
+ 'missingSource': 'The canonical Finance spine does not yet expose a validated gross-margin fact grouped by SKU.',
+ 'validationMethod': 'financial.validate',
+ },
+ ],
+}
+
+
+def cash_cycle_dashboard_contract():
+ """Return the mirror-safe cash-cycle projection without performing a live read."""
+ return deepcopy(_CASH_CYCLE_DASHBOARD)
+
+
+def margin_dashboard_contract():
+ """Return governed margin drill requirements without performing a live read."""
+ return deepcopy(_MARGIN_DASHBOARD)
+
+
# ---------- core margin window ----------
def _window(date_from, date_to, team_id=None):
"""Revenue, margin, derived COGS and GM% for a sale-line window (RI+FFS scope)."""
diff --git a/platform/modules/health.py b/platform/modules/health.py
index 7c55f570e6f9791620d6fcf932630251cc442924..7c3a2ddd7cbe641628c40f72fa65b96f3c924d8f 100644
--- a/platform/modules/health.py
+++ b/platform/modules/health.py
@@ -525,6 +525,84 @@ def summary(date_from=None, date_to=None, t=None):
return summarize(run_all(date_from, date_to, t))
+def planning_dashboard_contract():
+ """Plan-versus-actual contract composed without reading live Odoo.
+
+ Actual cells name canonical Finance-spine facts. Planning, variance, and driver cells remain
+ explicit gaps until a governed planning source exists in the tenant mirror.
+ """
+ no_plan = 'No governed planning fact is available in the tenant mirror.'
+ no_variance = 'Variance is unavailable until a governed plan exists.'
+ no_driver = 'Driver decomposition is unavailable until governed plan dimensions exist.'
+ rows = []
+ for key, label, scope, basis, method in (
+ ('revenue', 'Wholesale revenue', 'business_unit',
+ 'Confirmed sale/done order lines with GIFTWARE DEALS excluded.', 'order_level_revenue'),
+ ('invoiced_sales', 'Invoiced sales', 'hq_consolidated',
+ 'Posted customer invoices on the governed invoice basis.', 'invoiced_sales'),
+ ('commission_accrued', 'Commission accrued', 'business_unit',
+ 'Commission lines joined to posted invoice and sale-order context.', 'commission_accrued'),
+ ):
+ rows.append({
+ 'key': key, 'label': label, 'scope': scope, 'basisLabel': basis,
+ 'actual': {'state': 'fact', 'factKey': key},
+ 'plan': {'state': 'unavailable', 'missingSource': no_plan},
+ 'variance': {'state': 'unavailable', 'missingSource': no_variance},
+ 'driver': {'state': 'unavailable', 'missingSource': no_driver},
+ 'validationMethod': method,
+ })
+ return {
+ 'key': 'plan-drivers', 'title': 'Plan drivers', 'kind': 'planning',
+ 'basisLabel': ('Actuals come only from governed Finance facts. Plan, variance, and drivers '
+ 'remain unavailable until the mirror carries a planning source.'),
+ 'planningRows': rows,
+ }
+
+
+def data_health_dashboard_contracts():
+ """Close and data-quality requirements; all stay unavailable on ordinary mirror loads."""
+ missing = ('This exception is not available as a governed fact in the tenant mirror. '
+ 'Run its explicit read-only validation seam to reconcile the source.')
+ def requirement(key, label, scope, basis, source):
+ return {
+ 'key': key, 'label': label, 'format': 'int', 'state': 'unavailable',
+ 'scope': scope, 'basisLabel': basis, 'missingSource': missing,
+ 'sourceModule': source, 'validationMethod': 'health.validate',
+ }
+ return [
+ {
+ 'key': 'close-exceptions', 'title': 'Close exceptions', 'kind': 'metric_strip',
+ 'basisLabel': 'Selected close window and current HQ balance exceptions.',
+ 'metrics': [
+ requirement('draft_postings', 'Draft postings', 'hq_consolidated',
+ 'Draft invoices, bills, credits, and journals inside the close window.',
+ 'health._draft_moves'),
+ requirement('grni', 'GRNI', 'hq_consolidated',
+ 'Received purchase-order quantity minus invoiced quantity at current close.',
+ 'health._grni'),
+ requirement('ar_reconciliation', 'AR reconciliation', 'hq_consolidated',
+ 'Open AR documents compared with the partner receivable balance.',
+ 'health._ar_recon'),
+ requirement('duplicate_bills', 'Duplicate bills', 'hq_consolidated',
+ 'Posted vendor bills matched by vendor, reference, and amount in the window.',
+ 'health._duplicate_bills'),
+ ],
+ },
+ {
+ 'key': 'data-quality', 'title': 'Data quality', 'kind': 'metric_strip',
+ 'basisLabel': 'Current physical inventory and product-master exceptions.',
+ 'metrics': [
+ requirement('negative_stock', 'Negative stock', 'all_channel_inventory',
+ 'Negative quantities in internal physical inventory locations.',
+ 'health._negative_stock'),
+ requirement('costing_gaps', 'Costing gaps', 'all_channel_inventory',
+ 'In-stock storable products with no standard cost.',
+ 'health._product_master'),
+ ],
+ },
+ ]
+
+
def validate(date_from=None, date_to=None, t=None):
"""Light, independent reconciliations (no full run_all): aggregate vs row-by-row."""
df, dt_ = _window(date_from, date_to, t)
@@ -556,4 +634,16 @@ def validate(date_from=None, date_to=None, t=None):
ids = o.search_read('stock.quant', sdom, ['id'])
out.append({'check': 'Negative-stock: search_count == len(search_read)',
'a': n_neg, 'b': len(ids), 'gap': n_neg - len(ids), 'ok': n_neg == len(ids)})
+
+ # Posted GL: group-by-account and ungrouped balance are independent live aggregations over the
+ # exact close window. This keeps Planning & Data Health tied to the GL without making ordinary
+ # dashboard loads cross the mirror/source boundary.
+ gl_dom = [('move_id.state', '=', 'posted'), ('date', '>=', df), ('date', '<=', dt_)]
+ by_account = o.read_group('account.move.line', gl_dom, ['balance:sum'], ['account_id'], lazy=False)
+ grouped_balance = sum(row.get('balance') or 0.0 for row in by_account)
+ ledger_balance = _sum('account.move.line', gl_dom, 'balance')
+ out.append({'check': 'Posted GL balance: account groups == ungrouped close-window balance',
+ 'a': round(grouped_balance, 2), 'b': round(ledger_balance, 2),
+ 'gap': round(grouped_balance - ledger_balance, 2),
+ 'ok': abs(grouped_balance - ledger_balance) <= 0.01})
return out
diff --git a/platform/modules/inventory.py b/platform/modules/inventory.py
index 85c43c19dc70feaa355cec9d0de550c681b532f0..c40089da0eb316fb76e61516c28cb6adf15112f1 100644
--- a/platform/modules/inventory.py
+++ b/platform/modules/inventory.py
@@ -304,8 +304,44 @@ def sku_inventory(target_days=180, carry_pct=_CARRY_PCT, t=None):
return out
+def finance_workflow_contract():
+ """Describe inventory work requirements without creating a second fact plane."""
+ return [
+ {
+ 'key': 'inventory_value', 'label': 'Inventory value', 'availability': 'unavailable',
+ 'scope': 'all_channel_inventory',
+ 'basisLabel': ('Internal-location on-hand quantity multiplied by standard cost at '
+ 'product grain.'),
+ 'missingSource': ('On-hand inventory valuation is not available as a governed Finance '
+ 'fact in the tenant mirror.'),
+ 'sourceModule': 'inventory.summary',
+ 'validationMethod': 'inventory.validate',
+ },
+ {
+ 'key': 'excess_stock', 'label': 'Excess stock', 'availability': 'unavailable',
+ 'scope': 'all_channel_inventory',
+ 'basisLabel': ('Physical on-hand compared with a 180-day target using all-channel LTM '
+ 'sales velocity.'),
+ 'missingSource': ('Product-grain excess quantity and value are not available as '
+ 'governed Finance facts in the tenant mirror.'),
+ 'sourceModule': 'inventory.analytics',
+ 'validationMethod': 'inventory.validate',
+ },
+ {
+ 'key': 'dead_stock', 'label': 'Dead stock', 'availability': 'unavailable',
+ 'scope': 'all_channel_inventory',
+ 'basisLabel': ('Inventory value of at least $1,000 with less than $100 of all-channel '
+ 'LTM revenue at product grain.'),
+ 'missingSource': ('Product-grain dead-stock value is not available as a governed '
+ 'Finance fact in the tenant mirror.'),
+ 'sourceModule': 'inventory.dead_stock',
+ 'validationMethod': 'inventory.validate',
+ },
+ ]
+
+
def validate(t=None):
- df, _ = _build(t)
+ df, (lf, lt) = _build(t)
o = O.get_odoo()
checks = []
@@ -331,7 +367,23 @@ def validate(t=None):
'gap': round(cat_sum - tot, 2),
'ok': abs(cat_sum - tot) <= 1.0})
- # 3. inv_value definition holds for a sampled SKU (qty×cost)
+ # 3. The velocity feeding coverage/excess/dead-stock is explicitly ALL CHANNELS. Reconcile
+ # the per-product frame against an independent ungrouped live aggregate over the same coded
+ # product population so an accidental return to the wholesale-only domain turns this red.
+ coded_ids = [int(pid) for pid in df['product_id'].tolist()]
+ sales_dom = O.sale_line_domain(lf, lt, all_channels=True)
+ if coded_ids:
+ sales_dom.append(('product_id', 'in', coded_ids))
+ live_units = (O.sum_field('sale.order.line', sales_dom, 'product_uom_qty')
+ if coded_ids and o.search_count('sale.order.line', sales_dom) else 0.0)
+ frame_units = float(df['qty_ltm'].sum())
+ checks.append({
+ 'check': 'Inventory LTM units: coded SKU Σ == all-channel line units',
+ 'a': round(frame_units, 2), 'b': round(live_units, 2),
+ 'gap': round(frame_units - live_units, 2),
+ 'ok': abs(frame_units - live_units) <= max(1.0, abs(live_units) * 0.001)})
+
+ # 4. inv_value definition holds for a sampled SKU (qty×cost)
smp = df[df['on_hand'] > 0].head(1)
if len(smp):
r = smp.iloc[0]
diff --git a/platform/modules/management.py b/platform/modules/management.py
index c6e4ef57268d200b20b6e77793d2312380984df0..00dae072c7098345cd49c3a6c83ff29b74fbcb41 100644
--- a/platform/modules/management.py
+++ b/platform/modules/management.py
@@ -11,6 +11,7 @@ module that ignores the global brand selector.
"""
import json
import os
+from copy import deepcopy
from core import management_pl as MP
from core import odoo as O
@@ -23,6 +24,81 @@ _SECTION_LABEL = {'income': 'Income', 'other_income': 'Other income', 'cogs': 'C
'expense': 'Operating expense', 'deprec': 'Depreciation', '': ''}
+_MANAGEMENT_COMPARISON_DASHBOARD = {
+ 'key': 'management-comparison',
+ 'title': 'Actual vs plan',
+ 'kind': 'comparison',
+ 'columns': ['Actual', 'Budget', 'Prior'],
+ 'rows': [
+ {
+ 'key': 'revenue', 'label': 'Revenue',
+ 'basisLabel': 'Order basis: confirmed sale/done wholesale lines, untaxed, GIFTWARE DEALS excluded',
+ 'actual': {'state': 'fact', 'factKey': 'revenue'},
+ 'budget': {
+ 'state': 'unavailable',
+ 'missingSource': 'No approved budget series is bound to the canonical Finance mirror.',
+ },
+ 'prior': {
+ 'state': 'unavailable',
+ 'missingSource': 'The application request has no governed prior-period window or comparison fact.',
+ },
+ 'validationMethod': 'financial.validate',
+ },
+ {
+ 'key': 'gross_margin', 'label': 'Gross margin',
+ 'basisLabel': 'Sale-line margin basis: confirmed sale/done wholesale lines with GIFTWARE DEALS excluded',
+ 'actual': {
+ 'state': 'unavailable',
+ 'missingSource': 'No validated gross-margin fact is exposed by the canonical Finance spine.',
+ },
+ 'budget': {
+ 'state': 'unavailable',
+ 'missingSource': 'No approved gross-margin budget series is bound to the canonical Finance mirror.',
+ },
+ 'prior': {
+ 'state': 'unavailable',
+ 'missingSource': 'The application request has no governed prior-period gross-margin window.',
+ },
+ 'validationMethod': 'financial.validate',
+ },
+ {
+ 'key': 'opex', 'label': 'Operating expense',
+ 'basisLabel': 'Posted GL basis: HQ consolidated operating-expense accounts',
+ 'actual': {'state': 'fact', 'factKey': 'opex'},
+ 'budget': {
+ 'state': 'unavailable',
+ 'missingSource': 'No approved GL budget series is bound to the canonical Finance mirror.',
+ },
+ 'prior': {
+ 'state': 'unavailable',
+ 'missingSource': 'The application request has no governed prior-period GL window.',
+ },
+ 'validationMethod': 'management.validate',
+ },
+ {
+ 'key': 'commission_accrued', 'label': 'Commission accrued',
+ 'basisLabel': 'Posted customer-invoice commission-line basis, scoped to the selected Business Unit',
+ 'actual': {'state': 'fact', 'factKey': 'commission_accrued'},
+ 'budget': {
+ 'state': 'unavailable',
+ 'missingSource': 'No approved commission budget series is bound to the canonical Finance mirror.',
+ },
+ 'prior': {
+ 'state': 'unavailable',
+ 'missingSource': 'The application request has no governed prior-period commission window.',
+ },
+ 'validationMethod': 'commission_accrued',
+ },
+ ],
+ 'notice': 'Actual rows retain their source basis and are not summed into a synthetic P&L.',
+}
+
+
+def comparison_dashboard_contract():
+ """Return the mirror-safe actual/budget/prior projection without loading a statement."""
+ return deepcopy(_MANAGEMENT_COMPARISON_DASHBOARD)
+
+
def months_available():
"""Months we can reproduce -M for = the months present in the extracted overlay
(Sep-2024 .. last reconciled month in the model). Ascending list of (year, month)."""
diff --git a/platform/modules/o2c.py b/platform/modules/o2c.py
index e97510bad2a969eb268501b111058b36df2bf167..4ceb28d396f6a839b12fae50e046619c88b8fe10 100644
--- a/platform/modules/o2c.py
+++ b/platform/modules/o2c.py
@@ -11,12 +11,49 @@ per-customer machinery in modules/ar.days_to_pay: dominant term, excess days, fr
Medians (not means) per stage — one whale order must not move the story.
"""
import datetime as dt
+from copy import deepcopy
import core.odoo as O
import core.periods as P
import modules.ar as ar
+_O2C_DASHBOARD = {
+ 'key': 'o2c-cycle',
+ 'title': 'Order to cash',
+ 'kind': 'metric_strip',
+ 'basisLabel': 'LTM confirmed wholesale sale/done orders; Fisch/Royal scoped; GIFTWARE DEALS excluded',
+ 'metrics': [
+ {
+ 'key': 'order_to_ship_days', 'label': 'Order to ship', 'format': 'days',
+ 'state': 'unavailable', 'scope': 'business_unit',
+ 'basisLabel': 'Median days from confirmed order date to completed delivery date',
+ 'missingSource': 'The canonical mirror does not yet join confirmed sale orders to completed delivery dates.',
+ 'validationMethod': 'o2c.validate',
+ },
+ {
+ 'key': 'ship_to_invoice_days', 'label': 'Ship to invoice', 'format': 'days',
+ 'state': 'unavailable', 'scope': 'business_unit',
+ 'basisLabel': 'Median days from completed delivery to first posted customer invoice',
+ 'missingSource': 'The canonical mirror does not yet join completed deliveries to posted customer invoices.',
+ 'validationMethod': 'o2c.validate',
+ },
+ {
+ 'key': 'invoice_to_paid_days', 'label': 'Invoice to paid', 'format': 'days',
+ 'state': 'unavailable', 'scope': 'business_unit',
+ 'basisLabel': 'Median days for posted invoices aged at least 120 days to full reconciliation',
+ 'missingSource': 'The canonical mirror does not yet expose scoped full-reconcile settlement dates.',
+ 'validationMethod': 'o2c.validate',
+ },
+ ],
+}
+
+
+def dashboard_contract():
+ """Return the mirror-safe O2C projection without performing a live read."""
+ return deepcopy(_O2C_DASHBOARD)
+
+
def _median(v):
v = sorted(v)
n = len(v)
@@ -52,22 +89,30 @@ def stages(team_id=None, t=None):
# invoices LTM joined by exact invoice_origin (multi-origin merges skipped — counted)
inv_dom = [('move_type', '=', 'out_invoice'), ('state', '=', 'posted'),
- ('invoice_date', '>=', lf)]
+ ('invoice_date', '>=', lf), ('invoice_date', '<=', lt)]
if ex:
inv_dom.append(('partner_id', 'not in', list(ex)))
invs = O.search_read('account.move', inv_dom,
- ['invoice_origin', 'invoice_date', 'name'])
+ ['id', 'invoice_origin', 'invoice_date', 'name'])
first_inv = {}
+ eligible_invoice_ids = []
multi_origin = 0
for iv in invs:
og = (iv.get('invoice_origin') or '').strip()
if not og:
continue
if ',' in og:
- multi_origin += 1
+ # Count only a merge that intersects this already-scoped order population. A
+ # company-wide count here would leak the other BU through a scoped response.
+ if any(part.strip() in by_name for part in og.split(',')):
+ multi_origin += 1
+ continue
+ if og not in by_name:
continue
+ if iv.get('id'):
+ eligible_invoice_ids.append(iv['id'])
d = str(iv.get('invoice_date') or '')[:10]
- if og in by_name and d and (og not in first_inv or d < first_inv[og]):
+ if d and (og not in first_inv or d < first_inv[og]):
first_inv[og] = d
# invoice → settled (receivable full-reconcile max counterpart date), LTM invoices
@@ -76,8 +121,11 @@ def stages(team_id=None, t=None):
('full_reconcile_id', '!=', False), ('date', '>=', lf)]
if ex:
pay_dom.append(('partner_id', 'not in', list(ex)))
- rec_lines = O.search_read('account.move.line', pay_dom,
- ['date', 'full_reconcile_id'])
+ # The invoice-to-paid cohort must stay inside the same sale-order scope. Pulling every
+ # receivable reconcile line made the consolidated pay median appear inside a BU response.
+ pay_dom.append(('move_id', 'in', eligible_invoice_ids))
+ rec_lines = (O.search_read('account.move.line', pay_dom, ['date', 'full_reconcile_id'])
+ if eligible_invoice_ids else [])
fr_ids = list({O.m2o_id(l['full_reconcile_id']) for l in rec_lines})
settle = {}
# raw lines + local max — a grouped read here makes the server echo the full id-list
@@ -144,7 +192,8 @@ def stages(team_id=None, t=None):
'by_bu': by_bu, 'pay_median': m_pay, 'n_pay': len(pay_days), 'trend': trend,
'ship_median': ship_all, 'n_ship': n_ship_all,
'inv_median': inv_all, 'n_inv': n_inv_all,
- 'n_orders': len(orders), 'n_invoices': len(invs), 'multi_origin': multi_origin,
+ 'n_orders': len(orders), 'n_invoices': len(eligible_invoice_ids),
+ 'multi_origin': multi_origin,
'window': (lf, lt),
}
@@ -197,11 +246,14 @@ def validate(t=None, team_id=None):
dom.append(('partner_id', 'not in', list(ex)))
b = stages(team_id, t)
n_orders = O.get_odoo().search_count('sale.order', dom)
+ scoped_orders = O.search_read('sale.order', dom, ['name'])
+ scoped_names = [row.get('name') for row in scoped_orders if row.get('name')]
inv_dom = [('move_type', '=', 'out_invoice'), ('state', '=', 'posted'),
- ('invoice_date', '>=', lf)]
+ ('invoice_date', '>=', lf), ('invoice_date', '<=', lt),
+ ('invoice_origin', 'in', scoped_names)]
if ex:
inv_dom.append(('partner_id', 'not in', list(ex)))
- n_inv = O.get_odoo().search_count('account.move', inv_dom)
+ n_inv = O.get_odoo().search_count('account.move', inv_dom) if scoped_names else 0
return [
{'check': 'LTM orders: pull complete', 'a': b['n_orders'], 'b': n_orders,
'gap': b['n_orders'] - n_orders, 'ok': b['n_orders'] == n_orders},
diff --git a/platform/modules/procurement.py b/platform/modules/procurement.py
index 7286d3d82033d535fc9ef0f93ca5d7987ccf3af4..d46fbd8fca1b6894e2f6068a15d8ecb822db8454 100644
--- a/platform/modules/procurement.py
+++ b/platform/modules/procurement.py
@@ -494,6 +494,65 @@ def vendor_book(t=None):
return {'recs': recs, 'multi_vendor': multi, 'n_multi': len(multi)}
+def finance_workflow_contract():
+ """Describe Procurement requirements without reading Odoo or inventing mirror facts.
+
+ These are requirements for an owned-work queue, not queued records. The canonical Finance
+ mirror does not yet carry the open-PO, lead-time, or coverage facts needed to derive real work
+ items, so ordinary application loads must render the gaps explicitly instead of calling the
+ live module functions above.
+ """
+ return [
+ {
+ 'key': 'open_purchase_orders', 'label': 'Open POs', 'availability': 'unavailable',
+ 'scope': 'hq_consolidated',
+ 'basisLabel': ('Confirmed purchase-order lines where ordered quantity minus received '
+ 'quantity is greater than zero.'),
+ 'missingSource': ('Open purchase-order lines are not available as a governed fact in '
+ 'the tenant mirror.'),
+ 'sourceModule': 'procurement.open_pos',
+ 'validationMethod': 'procurement.validate',
+ },
+ {
+ 'key': 'lead_time_coverage', 'label': 'Lead-time coverage', 'availability': 'unavailable',
+ 'scope': 'all_channel_inventory',
+ 'basisLabel': ('All-channel LTM demand, physical on-hand, incoming PO quantity, and '
+ 'supplier lead time at product grain.'),
+ 'missingSource': ('Product-grain coverage and supplier lead time are not available as '
+ 'governed Finance facts in the tenant mirror.'),
+ 'sourceModule': 'procurement.recommendations',
+ 'validationMethod': 'procurement.validate',
+ },
+ {
+ 'key': 'replenishment', 'label': 'Suggested replenishment', 'availability': 'unavailable',
+ 'scope': 'all_channel_inventory',
+ 'basisLabel': ('All-channel demand and incoming stock compared with the supplier '
+ 'lead-time reorder point.'),
+ 'missingSource': ('A governed product-grain replenishment fact is not available in '
+ 'the tenant mirror.'),
+ 'sourceModule': 'procurement.recommendations',
+ 'validationMethod': 'procurement.validate',
+ },
+ ]
+
+
+def _row_open_po_net(lines):
+ """Rebuild ``_incoming_po`` quantity from rows without sharing its aggregate path.
+
+ The production aggregate nets ordered and received quantity at product grain, then omits
+ products whose net is not positive. Summing raw lines first is not equivalent when one
+ product has an over-received line, so the independent proof must preserve that ordering.
+ """
+ by_product = {}
+ for line in lines:
+ product_id = O.m2o_id(line.get('product_id'))
+ if not product_id:
+ continue
+ by_product[product_id] = by_product.get(product_id, 0.0) + (
+ (line.get('product_qty') or 0.0) - (line.get('qty_received') or 0.0))
+ return sum(quantity for quantity in by_product.values() if quantity > 0)
+
+
def validate(t=None, team_id=None):
t = t or P.today()
_, pe = _proj_window(t)
@@ -506,12 +565,12 @@ def validate(t=None, team_id=None):
'a': round(per, 2), 'b': round(tot, 2), 'gap': round(per - tot, 2),
'ok': abs(per - tot) <= 1.0}]
- # Open POs: row-by-row NET (product_qty − qty_received over ALL confirmed lines — the same
- # semantics as _incoming_po, over-receipts included) vs the _incoming_po read_group aggregate.
- # Two aggregation paths, small live-drift tolerance (two RPC snapshots on a live ledger).
+ # Open POs: row-by-row net at PRODUCT grain vs the _incoming_po read_group aggregate. Product
+ # grain matters: _incoming_po omits only a non-positive product total, not each non-positive
+ # line. Two aggregation paths, small live-drift tolerance (two RPC snapshots on a live ledger).
ib = open_pos(t)
all_lines, _ = _open_po_lines()
- row_net = sum((l.get('product_qty') or 0) - (l.get('qty_received') or 0) for l in all_lines)
+ row_net = _row_open_po_net(all_lines)
agg_net = sum(_incoming_po().values())
checks.append({'check': 'Open POs: row-level Σ net qty == read_group Σ net qty (live-drift tol)',
'a': round(row_net, 2), 'b': round(agg_net, 2),
diff --git a/platform/modules/product_data.py b/platform/modules/product_data.py
index 673446dc32cf17f825723941a5d171b27f396074..e746c77fe0330391a648370b5c2f9a52c79311e0 100644
--- a/platform/modules/product_data.py
+++ b/platform/modules/product_data.py
@@ -590,6 +590,20 @@ def _catalogue_by_code():
return products.catalogue()
+def identity_rows():
+ """The exact identity rows the compiled product database builds from its catalogue.
+
+ This intentionally stops before inventory, pricing, demand, FBA, and revenue cells. It reads
+ the same catalogue seam as ``pool`` and runs the same pid-collision refusal, so reconciliation
+ observes the actual canonical row universe without turning an hourly sync into a full product
+ page build.
+ """
+ rows = [{"pid": sku_pid(code), "code": code, "product_id": meta.get("id")}
+ for code, meta in _catalogue_by_code().items()]
+ _assert_no_pid_collision(rows)
+ return rows
+
+
def _fba_by_code(cat):
"""`({code: units} | None, report)` — the FBA column's source, keyed like the pool's rows.
diff --git a/platform/modules/sheet.py b/platform/modules/sheet.py
index 2588c3d579c16a4f82499e0db4e76dc4c4a4d85a..62e3b176db4359eae374e146c53b24952b46ebd4 100644
--- a/platform/modules/sheet.py
+++ b/platform/modules/sheet.py
@@ -54,6 +54,9 @@ AGGREGATE_FNS = ("sum", "count", "avg", "min", "max")
#: C2's refusal codes, spelled once. ⚠ Each message reaches a screen, so none carries an em dash or
#: an en dash (standing rule 2) and each names both the cause and the subject.
REFUSALS = {
+ "related_invalid": "Choose one declared relationship and a valid filter tree for this cell.",
+ "related_unresolved": "Relationship {subject} has no resolved SQL join. Choose a readable "
+ "declared relationship and try again.",
"database_unreadable": "You cannot read {subject}.",
"view_missing": "View {subject} no longer exists.",
"result_too_large": "This result does not fit in one cell.",
@@ -66,6 +69,7 @@ REFUSALS = {
"field_unknown": "{subject} is not a column on this database.",
"period_invalid": "{subject} is not a period this cell can use. "
"Choose a valid start for the selected period unit.",
+ "period_window_invalid": "This cell's lookback window is invalid. Choose a valid window in the period control.",
"ending_invalid": "Ending {subject} cannot be used. Choose a valid ending date.",
"ending_not_supported": "{subject} does not support an Ending. Choose year to date, last year "
"to date, this month or this quarter.",
@@ -149,6 +153,15 @@ class SheetRefusal(Exception):
super().__init__(self.message)
+def clean_related(raw):
+ """Normalize the optional single-relationship contract in the pure core layer."""
+ from core.sheet_binding import BindingMetadataRefusal, clean_related as _clean
+ try:
+ return _clean(raw)
+ except BindingMetadataRefusal as exc:
+ raise SheetRefusal(exc.code, exc.subject, exc.detail) from exc
+
+
def clean_period(raw):
"""Normalize period metadata through the core document contract."""
from core.sheet_binding import BindingMetadataRefusal, clean_period as _clean
@@ -464,14 +477,34 @@ def clean_adopted_from(raw):
return out or None
-def validate(date_from, date_to, observed, *, team_id, partner_ids=None):
- """Reconcile T08's four sections to live Odoo, at the exact displayed scope.
+def validate(date_from, date_to, observed, *, team_id, partner_ids=None,
+ basis="order", include_refunds=None):
+ """Reconcile a displayed value to live Odoo at the exact displayed scope.
+ The default order basis retains T08's four sections and six check rows.
sales.validate owns the broader sales-module reconciliation. Its public API
takes a YTD as-of, not arbitrary weeks or customer sets. Reuse its order and
line domains here for the template's narrower independent checks; never use
customer_data.validate (which validates addresses).
+
+ For ut_odoo_invoices.amount_untaxed, pass basis="invoice" and observed as
+ {"invoiced_sales": cell_value}. Explicitly choose include_refunds=True for
+ the mirror's posted invoices plus refunds, or False for strict R9 invoices.
+ Both dates are inclusive invoice_date bounds: use the SAME range on the cell,
+ with date_to at or before the mirror's own max(invoice_date), never today's
+ date merely because live Odoo is newer. Date bounds do not reconstruct a sync
+ snapshot; invoice rows report that limitation even when the amounts tie.
+ Pass team_id=None: this mirror has no order-team scope. If needed, resolve
+ the cell's customer/BU filter into partner_ids (None=all, []=none).
"""
+ if basis == "invoice":
+ return _validate_invoices(date_from, date_to, observed, team_id=team_id,
+ partner_ids=partner_ids, include_refunds=include_refunds)
+ if basis != "order":
+ raise ValueError("Unknown validation basis. Choose order or invoice.")
+ if include_refunds is not None:
+ raise ValueError("Refund selection applies only to invoice validation. Choose basis invoice.")
+
import math
from core import odoo as O
from modules import sales
@@ -506,6 +539,51 @@ def validate(date_from, date_to, observed, *, team_id, partner_ids=None):
return checks
+def _validate_invoices(date_from, date_to, observed, *, team_id, partner_ids, include_refunds):
+ """Independent account.move sum; never read the mirror or inherit order scope."""
+ import math
+ from core import odoo as O
+
+ try:
+ first = datetime.date.fromisoformat(date_from)
+ last = datetime.date.fromisoformat(date_to)
+ except (TypeError, ValueError):
+ raise ValueError("Invoice reconciliation needs both invoice_date bounds in YYYY-MM-DD format. "
+ "Use the cell's start and cap its end at the mirror's as-of date.") from None
+ if first > last:
+ raise ValueError("Invoice reconciliation has a reversed date range. "
+ "Use the cell's start and cap its end at the mirror's as-of date.")
+ if not isinstance(include_refunds, bool):
+ raise ValueError("Invoice reconciliation needs an explicit refund choice. "
+ "Set include_refunds=False for R9 invoices or True for invoices plus refunds.")
+ if team_id is not None:
+ raise ValueError("The invoice mirror has no order-team scope. Set team_id=None and "
+ "pass the cell's resolved customer or business-unit filter in partner_ids.")
+
+ domain = [("state", "=", "posted"),
+ ("move_type", "in", ["out_invoice", "out_refund"] if include_refunds else ["out_invoice"]),
+ ("partner_id", "!=", False),
+ ("invoice_date", ">=", first.isoformat()), ("invoice_date", "<=", last.isoformat())]
+ if partner_ids is not None:
+ domain.append(("partner_id", "in", list(partner_ids)))
+ expected = O.sum_field("account.move", domain, "amount_untaxed_signed")
+ actual = observed.get("invoiced_sales")
+ valid = isinstance(actual, (float, int)) and not isinstance(actual, bool) and math.isfinite(actual)
+ # One cent is the monetary precision used by the existing order reconciliation;
+ # it absorbs float noise without a relative tolerance that grows with revenue.
+ return [{"check": "invoiced_sales", "a": actual, "b": expected,
+ "gap": actual - expected if valid else None,
+ "ok": valid and abs(actual - expected) <= 0.01,
+ "basis": "invoice", "include_refunds": include_refunds,
+ "date_from": first.isoformat(), "date_to": last.isoformat(),
+ "limits": [{"code": "invoice_date_not_snapshot",
+ "cause": "Live Odoo is bounded by invoice_date, not by the mirror's sync snapshot. "
+ "Older invoices may have changed since that sync.",
+ "recommendation": "Use the same date range on the cell and this check, ending "
+ "at or before the mirror's max(invoice_date). If a gap remains, "
+ "compare invoice IDs and amounts or refresh the mirror and reconcile again."}]}]
+
+
def sales_summary_partition(all_values, agent_values, unassigned):
"""Partition additive measures; AOV must be recomputed from the summed count."""
import math
@@ -530,7 +608,8 @@ def clean_binding(raw):
`Binding = {database, aggregate:{fn, field?} | {metric_key},
scope:{view_id} | {filter} (+ adoptedFrom?),
period?: {unit, start, locked?, ref?} | {unit, start, locked?, frozen: true, from, to}
- | {window}, onDrag?: {across, down}}`
+ | {window}, onDrag?: {across, down},
+ related?: {via: , filter: }}`
"""
if not isinstance(raw, dict):
raise SheetRefusal("binding_invalid")
@@ -589,6 +668,9 @@ def clean_binding(raw):
if on_drag:
out["onDrag"] = on_drag
+ if "related" in raw:
+ out["related"] = clean_related(raw["related"])
+
return out
@@ -838,8 +920,58 @@ def aggregate_sql(spec, aggregate, columns):
return f"{fn.upper()}({col['sql']})", key
+def _related_predicate(related, columns, *, today=None, cohort_sets=None):
+ """Compile a resolved declaration, including the far-side wall, into one SQL subquery.
+
+ This bag is supplied only by the API resolver. Join keys are looked up in the two
+ column maps here; neither the binding's via nor any record IDs enter the SQL.
+ """
+ rcols, rspec = related["columns"], related["spec"]
+ local = (columns.get(related["local_key"]) or {}).get("sql")
+ remote = (rcols.get(related["remote_key"]) or {}).get("sql")
+ source = rspec.get("from_sql") or rspec.get("table")
+ if not local or not remote or not source:
+ raise SheetRefusal("related_unresolved", related.get("via", "This relationship"))
+ where = [f"({part})" for part in (rspec.get("where"), related.get("where")) if part]
+ params, limits = [], []
+
+ def leaf_limits(nodes):
+ # Checking every leaf also catches a dropped condition on a column which a
+ # different, valid condition used. The legacy column-set check cannot see it.
+ out = []
+ for node in nodes or ():
+ if not isinstance(node, dict):
+ raise ValueError("malformed related filter condition")
+ if isinstance(node.get("children"), list):
+ out.extend(leaf_limits(node["children"]))
+ else:
+ if "children" in node or not node.get("colId"):
+ raise ValueError("malformed related filter condition")
+ part = compile_scope([node], "and", rcols, today=today, cohort_sets=cohort_sets)
+ out.extend(widen_limits([node], rcols, part))
+ return out
+
+ for nodes, conj, is_wall in (
+ (related.get("wall_nodes"), related.get("wall_conj", "and"), True),
+ (related["scope_nodes"], related["scope_conj"], False)):
+ compiled = compile_scope(nodes, conj, rcols, today=today, cohort_sets=cohort_sets)
+ dropped = leaf_limits(nodes)
+ if is_wall and dropped:
+ raise ValueError("unapplied related row wall")
+ limits.extend({**item, "subject": f"{related['database']}: {item['subject']}"}
+ for item in dropped)
+ if compiled is not None:
+ where.append(f"({compiled.sql})")
+ params.extend(compiled.params)
+ sql = f"{local} IN (SELECT {remote} FROM {source}"
+ if where:
+ sql += " WHERE " + " AND ".join(where)
+ return sql + ")", params, limits + list(related.get("limits") or ())
+
+
def build_aggregate_query(spec, binding, columns, *, scope_nodes, scope_conj,
- wall_nodes=None, wall_conj="and", today=None, cohort_sets=None):
+ wall_nodes=None, wall_conj="and", today=None, cohort_sets=None,
+ related=None):
"""`(sql, params, limits)` for one bound cell.
⛔⛔ **THE SPEC'S OWN `where` IS NOT OPTIONAL AND IT IS NOT AN OPTIMISATION.** It carries the
@@ -881,6 +1013,15 @@ def build_aggregate_query(spec, binding, columns, *, scope_nodes, scope_conj,
where.append(f"({compiled.sql})")
params.extend(list(compiled.params))
+ related_limits = []
+ if binding.get("related") is not None and related is None:
+ raise SheetRefusal("related_unresolved", binding["related"].get("via"))
+ if related is not None:
+ predicate, rparams, related_limits = _related_predicate(
+ related, columns, today=today, cohort_sets=cohort_sets)
+ where.append(f"({predicate})")
+ params.extend(rparams)
+
source = str(spec.get("from_sql") or "").strip() or str(spec.get("table") or "").strip()
if not source:
raise SheetRefusal("binding_invalid")
@@ -889,7 +1030,7 @@ def build_aggregate_query(spec, binding, columns, *, scope_nodes, scope_conj,
sql += " WHERE " + " AND ".join(where)
# ⭐ The limits are measured against what the compiler REALLY used, so a leaf that fell out for
# any reason is reported. See `widen_limits` for the measurement that made this necessary.
- return sql, params, widen_limits(scope_nodes, columns, scope_compiled)
+ return sql, params, widen_limits(scope_nodes, columns, scope_compiled) + related_limits
#: A drill hands `LinkGridModal` a list of record ids. It is capped because the popup renders them,
@@ -902,7 +1043,7 @@ MAX_DRILL_IDS = 500
def build_ids_query(spec, columns, *, scope_nodes, scope_conj, wall_nodes=None, wall_conj="and",
- today=None, cohort_sets=None, limit=MAX_DRILL_IDS):
+ today=None, cohort_sets=None, limit=MAX_DRILL_IDS, related=None):
"""`(sql, params, limits)` for the ROW IDS behind a bound cell (R6, R13).
Same WHERE as the aggregate, by construction: it is built by the same function so the popup
@@ -914,7 +1055,8 @@ def build_ids_query(spec, columns, *, scope_nodes, scope_conj, wall_nodes=None,
probe = {"aggregate": {"fn": "count"}}
sql, params, limits = build_aggregate_query(
spec, probe, columns, scope_nodes=scope_nodes, scope_conj=scope_conj,
- wall_nodes=wall_nodes, wall_conj=wall_conj, today=today, cohort_sets=cohort_sets)
+ wall_nodes=wall_nodes, wall_conj=wall_conj, today=today, cohort_sets=cohort_sets,
+ related=related)
sql = sql.replace("SELECT COUNT(*) FROM", f"SELECT {id_expr} FROM", 1)
return f"{sql} LIMIT {int(limit)}", params, limits
diff --git a/web/public/AIOS_DEPLOY_RECEIPT b/web/public/AIOS_DEPLOY_RECEIPT
index cb45e778be1e10f5d6fc5799d317d53799914c03..0b83153e284f93c56934e3402ef4ba98bf27d987 100644
--- a/web/public/AIOS_DEPLOY_RECEIPT
+++ b/web/public/AIOS_DEPLOY_RECEIPT
@@ -1 +1 @@
-9c7bd0c7d54ff05e2c3907769d9a34c9ffb0d78eb7672498b1e6ca8b806cf1cd
+4217633c110111c0798f4b260f5812ee5cb90bc279d7d7918f104e94dcb3ed88
diff --git a/web/public/sample_customers.json b/web/public/sample_customers.json
index 37d5b9ee33a9685fd715ba724ad40f2fdfd8a798..cbe9355f646ad6316c521e1a356960fed2ac5a5b 100644
--- a/web/public/sample_customers.json
+++ b/web/public/sample_customers.json
@@ -30,6 +30,14 @@
],
"description": "Whether this customer still exists in Odoo. Archived means deleted there."
},
+ {
+ "key": "archived",
+ "label": "Archived",
+ "type": "checkbox",
+ "source": "odoo",
+ "default": false,
+ "description": "Whether this customer is archived in Odoo."
+ },
{
"key": "agent",
"label": "Agent",
diff --git a/web/src/app/FinanceWorkspace.tsx b/web/src/app/FinanceWorkspace.tsx
new file mode 100644
index 0000000000000000000000000000000000000000..ebaec95b3628957ce30868af44679a2e52c19428
--- /dev/null
+++ b/web/src/app/FinanceWorkspace.tsx
@@ -0,0 +1,937 @@
+import { useEffect, useMemo, useState } from "react";
+import { useEscapeClose } from "../ui/useEscapeClose";
+import CollectionPage from "../apps/CollectionPage";
+import "./finance.css";
+
+type FactState = "ready" | "warming" | "missing" | "unavailable" | "restricted" | "requires_sku";
+
+type FinanceFact = {
+ key: string;
+ label: string;
+ format: "usd" | "int" | "pct";
+ scope: "business_unit" | "hq_consolidated" | "all_channel_inventory";
+ state: FactState;
+ value: number | null;
+ message?: string;
+ requiredGrain?: string;
+ freshness: { state: string; asOf?: string | null };
+ lineage: { topic: string; metric: string; sourceEntities: string[] };
+ drillThrough: { kind: "source_records"; href: string };
+ reconciliation: { state: string; method: string; href: string };
+};
+
+type MirrorWorklistContract = {
+ title: string;
+ cadence: string;
+ kind?: undefined;
+ sourceFact: string;
+ pageSize: number;
+ maxRows: number;
+ displayLimit: number;
+ entityField: string;
+ documentField: string;
+ amountField: string;
+ overdueAmountField?: string;
+ dueDateField: string;
+ ownerField: string;
+ statusField: string;
+ promiseDateField?: string;
+ promiseAmountField?: string;
+ nextAction: string;
+ previewOnly: boolean;
+ concentration?: boolean;
+};
+
+type UnavailableWorkSignal = {
+ key: string;
+ label: string;
+ state: "unavailable";
+ scope: FinanceFact["scope"];
+ basisLabel: string;
+ missingSource: string;
+ sourceModule: string;
+ validationMethod: string;
+ ownerRole: string;
+ status: "unavailable";
+ nextAction: string;
+};
+
+type RequirementsWorklistContract = {
+ kind: "unavailable_requirements";
+ title: string;
+ cadence: string;
+ basisNotice: string;
+ signals: UnavailableWorkSignal[];
+ sourceFact?: undefined;
+};
+
+type WorklistContract = MirrorWorklistContract | RequirementsWorklistContract;
+
+type DashboardUnavailable = {
+ state: "unavailable";
+ missingSource: string;
+};
+
+type DashboardFactCell = {
+ state: "fact";
+ factKey: string;
+};
+
+type DashboardMetric = {
+ key: string;
+ label: string;
+ format: "usd" | "int" | "pct" | "days";
+ state: "unavailable";
+ scope: FinanceFact["scope"];
+ basisLabel: string;
+ missingSource: string;
+ validationMethod: string;
+ drillDimension?: string;
+ sourceFact?: string;
+};
+
+type DashboardComparisonRow = {
+ key: string;
+ label: string;
+ basisLabel: string;
+ actual: DashboardFactCell | DashboardUnavailable;
+ budget: DashboardFactCell | DashboardUnavailable;
+ prior: DashboardFactCell | DashboardUnavailable;
+ validationMethod: string;
+};
+
+type DashboardPlanningRow = {
+ key: string;
+ label: string;
+ scope: FinanceFact["scope"];
+ basisLabel: string;
+ actual: DashboardFactCell;
+ plan: DashboardUnavailable;
+ variance: DashboardUnavailable;
+ driver: DashboardUnavailable;
+ validationMethod: string;
+};
+
+type DashboardSection = {
+ key: string;
+ title: string;
+ kind: "metric_strip" | "drill_matrix" | "comparison" | "planning";
+ basisLabel?: string;
+ metrics?: DashboardMetric[];
+ columns?: string[];
+ rows?: DashboardComparisonRow[];
+ planningRows?: DashboardPlanningRow[];
+ notice?: string;
+};
+
+type DashboardContract = {
+ basisNotice: string;
+ sections: DashboardSection[];
+};
+
+type FinanceApplication = {
+ key: string;
+ label: string;
+ navLabel: string;
+ mode: "workflow" | "dashboard";
+ question: string;
+ deepLink: string;
+ factKeys: string[];
+ kpiKeys: string[];
+ worklist?: WorklistContract;
+ dashboard?: DashboardContract;
+};
+
+type ApplicationPayload = {
+ state: string;
+ application: FinanceApplication;
+ businessUnit: number | null;
+ facts: FinanceFact[];
+ kpis: FinanceFact[];
+};
+
+type DrillPayload = {
+ fact?: FinanceFact;
+ state?: string;
+ rows?: Record[];
+ row_count?: number | null;
+ total_count?: number | null;
+ aggregates?: Record;
+};
+
+type WorklistPayload = DrillPayload & {
+ total_count?: number | null;
+};
+
+type WorklistState =
+ | { kind: "idle" }
+ | { kind: "loading" }
+ | { kind: "ready"; payload: WorklistPayload; complete: boolean }
+ | { kind: "error"; message: string };
+
+type WorkItem = {
+ key: string;
+ entity: string;
+ document: string;
+ amount: number | null;
+ overdueAmount: number | null;
+ dueDate: string;
+ aging: string;
+ owner: string;
+ status: string;
+ promise: string;
+ concentration: number | null;
+ concentrationAvailable: boolean;
+ exception: string;
+ priority: string;
+ nextAction: string;
+ source: Record;
+};
+
+type PageState =
+ | { kind: "loading" }
+ | { kind: "ready"; data: ApplicationPayload }
+ | { kind: "restricted" }
+ | { kind: "missing" }
+ | { kind: "error"; message: string };
+
+const CREDENTIALS: RequestCredentials = "include";
+
+function pathApplicationKey() {
+ if (typeof window === "undefined") return "";
+ const match = `${window.location.pathname}${window.location.hash}`.match(/\/finance\/([^/?#]+)/);
+ return match ? decodeURIComponent(match[1]) : "";
+}
+
+function displayValue(fact: FinanceFact) {
+ if (fact.state !== "ready" || fact.value == null) return "Not available";
+ if (fact.format === "usd") {
+ return new Intl.NumberFormat(undefined, { style: "currency", currency: "USD", maximumFractionDigits: 0 }).format(fact.value);
+ }
+ if (fact.format === "pct") return `${(fact.value * 100).toFixed(1)}%`;
+ return new Intl.NumberFormat().format(fact.value);
+}
+
+function stateLabel(state: string) {
+ return state.replaceAll("_", " ");
+}
+
+function isMirrorWorklist(contract: WorklistContract): contract is MirrorWorklistContract {
+ return contract.kind !== "unavailable_requirements";
+}
+
+function numberValue(value: unknown) {
+ if (value == null || value === "" || typeof value === "boolean") return null;
+ const parsed = typeof value === "number" ? value : Number(value);
+ return Number.isFinite(parsed) ? parsed : null;
+}
+
+function textValue(value: unknown) {
+ const text = typeof value === "string" || typeof value === "number" ? String(value).trim() : "";
+ return text || "Unavailable";
+}
+
+function money(value: number | null) {
+ return value == null
+ ? "Unavailable"
+ : new Intl.NumberFormat(undefined, { style: "currency", currency: "USD", maximumFractionDigits: 0 }).format(value);
+}
+
+function dueAge(value: unknown) {
+ const text = typeof value === "string" ? value.slice(0, 10) : "";
+ if (!/^\d{4}-\d{2}-\d{2}$/.test(text)) return { date: "Unavailable", days: null };
+ const due = new Date(`${text}T00:00:00Z`).getTime();
+ const now = new Date();
+ const today = Date.UTC(now.getUTCFullYear(), now.getUTCMonth(), now.getUTCDate());
+ return { date: text, days: Math.max(0, Math.floor((today - due) / 86_400_000)) };
+}
+
+function agingLabel(days: number | null, overdueAmount: number | null) {
+ if (days != null && days > 90) return "90+ days";
+ if (days != null && days > 60) return "61-90 days";
+ if (days != null && days > 30) return "31-60 days";
+ if (days != null && days > 0) return "1-30 days";
+ if (days === 0) return "Current";
+ if (overdueAmount != null) return overdueAmount > 0 ? "Overdue" : "Current";
+ return "Unavailable";
+}
+
+function workItems(contract: MirrorWorklistContract, payload: WorklistPayload, complete: boolean) {
+ const rows = payload.rows || [];
+ const entityTotals = new Map();
+ for (const row of rows) {
+ const entity = textValue(row[contract.entityField]);
+ const amount = numberValue(row[contract.amountField]);
+ if (amount != null) entityTotals.set(entity, (entityTotals.get(entity) || 0) + amount);
+ }
+ const payloadTotal = numberValue(payload.aggregates?.[contract.amountField]);
+ const computedTotal = rows.reduce((sum, row) => sum + (numberValue(row[contract.amountField]) || 0), 0);
+ const total = payloadTotal ?? (complete ? computedTotal : null);
+ const concentrationAvailable = Boolean(contract.concentration && complete && total != null && total > 0);
+
+ const items = rows.map((row, index): WorkItem => {
+ const entity = textValue(row[contract.entityField]);
+ const document = textValue(row[contract.documentField]);
+ const amount = numberValue(row[contract.amountField]);
+ const overdueAmount = contract.overdueAmountField
+ ? numberValue(row[contract.overdueAmountField])
+ : null;
+ const due = dueAge(row[contract.dueDateField]);
+ const aging = agingLabel(due.days, overdueAmount);
+ const concentration = concentrationAvailable
+ ? Math.max(0, entityTotals.get(entity) || 0) / (total as number)
+ : null;
+ const promiseDate = contract.promiseDateField ? textValue(row[contract.promiseDateField]) : "Unavailable";
+ const promiseAmount = contract.promiseAmountField ? numberValue(row[contract.promiseAmountField]) : null;
+ const promise = promiseDate !== "Unavailable" || promiseAmount != null
+ ? [promiseDate !== "Unavailable" ? promiseDate : null, promiseAmount != null ? money(promiseAmount) : null].filter(Boolean).join(" · ")
+ : "Unavailable";
+ const overdue = (due.days != null && due.days > 0) || (overdueAmount != null && overdueAmount > 0);
+ const credit = amount != null && amount < 0;
+ const concentrated = concentration != null && concentration >= 0.25;
+ const exception = credit ? "Vendor credit" : overdue ? "Past due" : concentrated ? "High concentration" : "No exception";
+ const nextAction = contract.previewOnly
+ ? contract.nextAction
+ : credit ? "Apply vendor credit" : overdue ? "Resolve overdue bill" : contract.nextAction;
+ return {
+ key: String(row._rid ?? `${entity}-${document}-${index}`),
+ entity,
+ document,
+ amount,
+ overdueAmount,
+ dueDate: due.date,
+ aging,
+ owner: textValue(row[contract.ownerField]),
+ status: textValue(row[contract.statusField]),
+ promise,
+ concentration,
+ concentrationAvailable,
+ exception,
+ priority: amount == null ? "Amount unavailable" : `${overdue ? "Overdue" : "Open"} · ${money(Math.abs(amount))}`,
+ nextAction,
+ source: row,
+ };
+ });
+ const actionable = items.filter(item => item.amount == null || Math.abs(item.amount) >= 0.005);
+ actionable.sort((left, right) => {
+ const leftOverdue = left.aging !== "Current" && left.aging !== "Unavailable";
+ const rightOverdue = right.aging !== "Current" && right.aging !== "Unavailable";
+ if (leftOverdue !== rightOverdue) return leftOverdue ? -1 : 1;
+ if (left.amount == null) return right.amount == null ? left.document.localeCompare(right.document) : 1;
+ if (right.amount == null) return -1;
+ return Math.abs(right.amount) - Math.abs(left.amount) || left.document.localeCompare(right.document);
+ });
+ return actionable.slice(0, contract.displayLimit);
+}
+
+function appendScope(url: string, businessUnit?: string, dateFrom?: string, dateTo?: string) {
+ const target = new URL(url, window.location.origin);
+ if (businessUnit) target.searchParams.set("bu", businessUnit);
+ if (dateFrom) target.searchParams.set("date_from", dateFrom);
+ if (dateTo) target.searchParams.set("date_to", dateTo);
+ return `${target.pathname}${target.search}`;
+}
+
+async function responseMessage(response: Response) {
+ try {
+ const body = await response.json() as { detail?: { error?: { message?: string } } };
+ return body.detail?.error?.message || "Finance is temporarily unavailable.";
+ } catch {
+ return "Finance is temporarily unavailable.";
+ }
+}
+
+function FactCard({ fact, onDrill, onReconcile }: {
+ fact: FinanceFact;
+ onDrill: (fact: FinanceFact) => void;
+ onReconcile: (fact: FinanceFact) => void;
+}) {
+ return (
+
+
{plural(data.requested, "customer", "customers")} selected. {plural(expected, "statement", "statements")} can be queued; {count.format(data.refused.length)} will be refused.
+ {confirming ? <>
+
Queue one statement per eligible customer in Odoo's outbox?
+
Balances and recipients are read again when you confirm.
+
+ > : <>
+
From {data.sender.name} ({data.sender.email}). Replies go to {data.sender.replyTo}.
+ {statement && <>
+
+
+
To
{statement.to || "No email address"}
+
Subject
{statement.subject}
+
+ {statement.refusal &&
{statement.refusal}
}
+
+ {/* Scripts and remote resources are blocked. The frame is read-only and sized to
+ its content, so scrolling and Escape stay in the parent overlay. */}
+
+ Creating a database saves its recipe for $0. Credentials stay in Keychain and are
+ never stored with the recipe. A provider is contacted only after you choose Run now.
+