fsanyoto commited on
Commit
2e7e617
·
verified ·
1 Parent(s): c1fc133

Deploy AIOS web (React glide grid + FastAPI slice)

Browse files
This view is limited to 50 files because it contains too many changes.   See raw diff
Files changed (50) hide show
  1. RELEASES.json +1 -1
  2. VERSION +1 -1
  3. api/application_registry.py +193 -9
  4. api/automation_engine.py +96 -13
  5. api/connectors_ig.py +30 -0
  6. api/connectors_tt.py +43 -3
  7. api/deps.py +29 -0
  8. api/formula_runtime.py +51 -9
  9. api/main.py +329 -83
  10. api/odoo_relational.py +444 -20
  11. api/providers.py +344 -0
  12. api/routes_activity.py +392 -0
  13. api/routes_admin.py +8 -1
  14. api/routes_applications.py +258 -0
  15. api/routes_automation.py +16 -4
  16. api/routes_connected_tables.py +4 -1
  17. api/routes_connectors.py +1180 -38
  18. api/routes_customers.py +73 -0
  19. api/routes_external.py +0 -0
  20. api/routes_geo.py +52 -0
  21. api/routes_module_index.py +25 -2
  22. api/routes_odoo_tables.py +351 -35
  23. api/routes_shares.py +173 -308
  24. api/routes_sheet.py +600 -216
  25. api/routes_slack.py +55 -9
  26. api/routes_templates.py +36 -1
  27. api/test_s63_integration.py +240 -0
  28. api/test_sheet_commit_events.py +162 -0
  29. platform/aios_grid.py +5 -2
  30. platform/aios_grid_fields.json +8 -0
  31. platform/core/audit.py +29 -0
  32. platform/core/daily_sales_repair.py +977 -0
  33. platform/core/external_access.py +137 -5
  34. platform/core/external_oauth.py +723 -0
  35. platform/core/external_tools.py +171 -8
  36. platform/core/field_permissions.py +384 -0
  37. platform/core/grid_events.py +97 -0
  38. platform/core/registry.py +47 -0
  39. platform/core/sheet_binding.py +178 -5
  40. platform/core/sheet_compat.py +875 -0
  41. platform/core/sheet_events.py +101 -0
  42. platform/core/sheet_excel.py +65 -0
  43. platform/core/sheet_import.py +289 -27
  44. platform/core/sheet_templates.py +2 -2
  45. platform/core/sheets.py +204 -13
  46. platform/core/table_store.py +5 -4
  47. platform/core/user_tables.py +84 -0
  48. platform/core/write_execution.py +45 -0
  49. platform/harness/filter_sql.py +7 -0
  50. platform/harness/mirror_tokens.py +64 -0
RELEASES.json CHANGED
@@ -1,5 +1,5 @@
1
  {
2
- "current": "v75 (2c51469)",
3
  "releases": [
4
  {
5
  "version": "v75",
 
1
  {
2
+ "current": "df68553 (df68553)",
3
  "releases": [
4
  {
5
  "version": "v75",
VERSION CHANGED
@@ -1 +1 @@
1
- v75 (2c51469)
 
1
+ df68553 (df68553)
api/application_registry.py CHANGED
@@ -2,12 +2,20 @@
2
  from __future__ import annotations
3
 
4
  import copy
5
- import datetime as dt
6
- import uuid
 
 
 
 
7
 
8
 
9
  APPLICATION_TYPES = ("chart", "route", "map", "timeseries")
10
- APPLICATION_TEMPLATE_KINDS = ("route", "catalog")
 
 
 
 
11
  _KEY = "applications_v1"
12
  _MAX = 500
13
 
@@ -48,6 +56,15 @@ _TEMPLATE_CONTRACTS = {
48
  ],
49
  "defaultPlacement": {"section": "app", "order": 0},
50
  },
 
 
 
 
 
 
 
 
 
51
  }
52
 
53
 
@@ -57,7 +74,130 @@ class ApplicationError(ValueError):
57
 
58
  def template_contracts():
59
  """The trusted renderer contracts offered by the Application creation flow."""
60
- return [copy.deepcopy(_TEMPLATE_CONTRACTS[kind]) for kind in APPLICATION_TEMPLATE_KINDS]
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
61
 
62
 
63
  def blank_doc():
@@ -205,6 +345,24 @@ def _template_definition(kind, payload, *, semantic_records=None, field_contract
205
  template_kind = str(payload.get("templateKind") or "").strip().lower()
206
  if template_kind != kind:
207
  raise ApplicationError(f"{kind} applications require templateKind {kind!r}")
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
208
  source = str(payload.get("sourceDatabase") or "").strip()[:120]
209
  if not source:
210
  raise ApplicationError("sourceDatabase is required")
@@ -305,6 +463,13 @@ def _public(app, user, is_admin=False, *, allowed_databases=None, target_databas
305
  "default":bool(is_admin)}
306
  row["starred"] = user in (app.get("starredBy") or ())
307
  row["isDefault"] = bool(is_default)
 
 
 
 
 
 
 
308
  target_database = target_database if target_database in compatible else (compatible[0] if compatible else None)
309
  row["openTarget"]={"kind":"application","id":row["id"],"type":row["type"],
310
  "database":target_database}
@@ -318,7 +483,8 @@ def list_apps(runtime, user, is_admin=False, *, database=None, kind=None, starre
318
  for app in doc["applications"].values():
319
  if not may_read(app,user,is_admin): continue
320
  if database and database not in (app.get("compatibleDatabases") or ()): continue
321
- if not _allowed(app.get("compatibleDatabases"), allowed_databases): continue
 
322
  if kind and app.get("type") != kind: continue
323
  if starred is not None and bool(user in (app.get("starredBy") or ())) != bool(starred): continue
324
  default_key = f"{database}::{app.get('type')}" if database else ""
@@ -328,6 +494,22 @@ def list_apps(runtime, user, is_admin=False, *, database=None, kind=None, starre
328
  return sorted(rows,key=lambda x:(str(x.get("updatedAt") or ""),str(x.get("id"))),reverse=True)
329
 
330
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
331
  def create(runtime, tenant, user, payload, *, semantic_records=None, field_contracts=None,
332
  allowed_databases=None):
333
  payload=payload if isinstance(payload,dict) else {}
@@ -339,8 +521,9 @@ def create(runtime, tenant, user, payload, *, semantic_records=None, field_contr
339
  allowed_databases=allowed_databases)
340
  name=" ".join(str(payload.get("name") or "").split())[:120]
341
  if not name: raise ApplicationError("application name is required")
342
- app_id=f"app_{uuid.uuid4().hex[:16]}"; now=dt.datetime.now(dt.timezone.utc).isoformat()
343
- compatible = ([definition["sourceDatabase"]] if definition
 
344
  else _compat(payload.get("compatibleDatabases")))
345
  if _allowed(compatible, allowed_databases) != set(compatible):
346
  raise PermissionError("an Application cannot bind a database you cannot open")
@@ -368,7 +551,8 @@ def mutate(runtime, app_id, user, is_admin, patch, *, semantic_records=None,
368
  allowed_databases=None, target_database=None):
369
  patch=patch if isinstance(patch,dict) else {}; landed={}
370
  immutable = sorted(set(patch) & {
371
- "id", "tenant", "type", "templateKind", "sourceDatabase", "fieldMap", "navPlacement"
 
372
  })
373
  if immutable:
374
  raise ApplicationError("Application identity and template definition are immutable: "
@@ -409,7 +593,7 @@ def mutate(runtime, app_id, user, is_admin, patch, *, semantic_records=None,
409
  stars=set(app.get("starredBy") or ())
410
  (stars.add if patch["starred"] else stars.discard)(user); app["starredBy"]=sorted(stars)
411
  app["revision"]=int(app.get("revision") or 0)+1
412
- app["updatedAt"]=dt.datetime.now(dt.timezone.utc).isoformat(); landed.update(app)
413
  return doc
414
  runtime.update(_KEY,apply)
415
  return _public(landed,user,is_admin,allowed_databases=allowed_databases,
 
2
  from __future__ import annotations
3
 
4
  import copy
5
+ from core import write_execution
6
+
7
+ from modules.finance_apps import (finance_application as _finance_application,
8
+ finance_applications as _finance_applications,
9
+ finance_group as _finance_group)
10
+ from modules.finance_spine import fact_specs as _finance_fact_specs
11
 
12
 
13
  APPLICATION_TYPES = ("chart", "route", "map", "timeseries")
14
+ # The wizard binds a database and fields. Finance templates use the catalogue POST instead;
15
+ # validation accepts both creation doors without offering Finance to the five-step wizard.
16
+ APPLICATION_WIZARD_TEMPLATE_KINDS = ("route", "catalog")
17
+ FINANCE_TEMPLATE_KIND = "finance"
18
+ APPLICATION_TEMPLATE_KINDS = APPLICATION_WIZARD_TEMPLATE_KINDS + (FINANCE_TEMPLATE_KIND,)
19
  _KEY = "applications_v1"
20
  _MAX = 500
21
 
 
56
  ],
57
  "defaultPlacement": {"section": "app", "order": 0},
58
  },
59
+ "finance": {
60
+ "kind": "finance",
61
+ "label": "Finance",
62
+ # Like Catalog, templateKind selects a trusted surface without growing APPLICATION_TYPES.
63
+ "applicationType": "chart",
64
+ "displayMode": "chart",
65
+ "fieldRoles": [],
66
+ "defaultPlacement": {"section": "app", "order": 0},
67
+ },
68
  }
69
 
70
 
 
74
 
75
  def template_contracts():
76
  """The trusted renderer contracts offered by the Application creation flow."""
77
+ return [copy.deepcopy(_TEMPLATE_CONTRACTS[kind]) for kind in APPLICATION_WIZARD_TEMPLATE_KINDS]
78
+
79
+
80
+ def finance_group():
81
+ """The one immutable Finance group consumed by navigation and direct-link routes."""
82
+ return _finance_group()
83
+
84
+
85
+ def finance_applications(may_open=None):
86
+ """Six separately addressable Finance application descriptors."""
87
+ return _finance_applications(may_open=may_open)
88
+
89
+
90
+ # R6: Finance records are read-only projections of finance_apps.py, never validated, created,
91
+ # mutated, or persisted. Adding "finance" to APPLICATION_TYPES buys nothing and weakens
92
+ # validate_application's governed type vocabulary.
93
+ # This remains true for the SEEDED six. Users may now persist their own Finance applications
94
+ # through templateKind, pointing to the same trusted surface without modifying those projections.
95
+ def finance_application_records(may_open=None):
96
+ """Project the grant-filtered Finance catalogue as seeded Application records."""
97
+ return [
98
+ {"id": f"finance_{app['key']}",
99
+ "key": app["key"],
100
+ "name": app["navLabel"],
101
+ "type": "finance",
102
+ "templateKind": None,
103
+ "origin": "finance",
104
+ "permissionKey": app["permissionKey"],
105
+ "mode": app["mode"],
106
+ "question": app["question"],
107
+ "deepLink": app["deepLink"],
108
+ "compatibleDatabases": [],
109
+ "navPlacement": {"section": "finance", "order": index},
110
+ "openTarget": {"kind": "finance", "id": f"finance_{app['key']}", "type": "finance",
111
+ "database": None},
112
+ "capabilities": {"read": True, "edit": False, "share": False, "delete": False,
113
+ "duplicate": False, "star": False, "default": False},
114
+ "starred": False,
115
+ "isDefault": False}
116
+ for index, app in enumerate(finance_applications(may_open=may_open))
117
+ ]
118
+
119
+
120
+ _FINANCE_SCOPES = {"business_unit", "hq_consolidated", "all_channel_inventory"}
121
+
122
+
123
+ def _require_unavailable(value, context):
124
+ if not isinstance(value, dict) or value.get("state") != "unavailable":
125
+ raise RuntimeError(f"{context} must be explicitly unavailable")
126
+ if "value" in value or not value.get("missingSource"):
127
+ raise RuntimeError(f"{context} cannot fabricate a missing value")
128
+
129
+
130
+ def validate_finance_application_contract(app):
131
+ """Fail closed when an app bypasses its declared workflow/dashboard or fact boundary."""
132
+ key = str(app.get("key") or "unknown")
133
+ mode = app.get("mode")
134
+ worklist = app.get("worklist")
135
+ dashboard = app.get("dashboard")
136
+ fact_keys = set(app.get("factKeys") or ())
137
+ if mode not in {"workflow", "dashboard"}:
138
+ raise RuntimeError(f"invalid Finance application mode for {key}")
139
+ if mode == "workflow":
140
+ if not isinstance(worklist, dict) or dashboard:
141
+ raise RuntimeError(f"invalid Finance worklist contract for {key}")
142
+ source_fact = worklist.get("sourceFact")
143
+ if source_fact:
144
+ if source_fact not in fact_keys:
145
+ raise RuntimeError(f"invalid Finance worklist source for {key}")
146
+ required = ("ownerField", "statusField", "dueDateField", "nextAction")
147
+ if not all(worklist.get(name) for name in required):
148
+ raise RuntimeError(f"incomplete Finance worklist fields for {key}")
149
+ elif worklist.get("kind") == "unavailable_requirements":
150
+ signals = worklist.get("signals")
151
+ if not isinstance(signals, list) or not signals:
152
+ raise RuntimeError(f"Finance requirements worklist {key} has no signals")
153
+ for signal in signals:
154
+ _require_unavailable(signal, f"Finance work signal in {key}")
155
+ if signal.get("scope") not in _FINANCE_SCOPES:
156
+ raise RuntimeError(f"Finance work signal in {key} has no governed scope")
157
+ if not all(signal.get(name) for name in (
158
+ "key", "label", "basisLabel", "sourceModule", "validationMethod",
159
+ "ownerRole", "status", "nextAction")):
160
+ raise RuntimeError(f"Finance work signal in {key} is ungrounded")
161
+ if signal.get("status") != "unavailable":
162
+ raise RuntimeError(f"Finance work signal in {key} claims executable state")
163
+ else:
164
+ raise RuntimeError(f"Finance worklist {key} has no governed source")
165
+ elif worklist or not isinstance(dashboard, dict):
166
+ raise RuntimeError(f"invalid Finance dashboard contract for {key}")
167
+
168
+ for section in (dashboard or {}).get("sections", []):
169
+ for metric in section.get("metrics", []):
170
+ if metric.get("sourceFact") and metric["sourceFact"] not in fact_keys:
171
+ raise RuntimeError(f"dashboard {key} references an ungoverned source fact")
172
+ if metric.get("state") == "unavailable":
173
+ _require_unavailable(metric, f"dashboard metric in {key}")
174
+ if metric.get("scope") not in _FINANCE_SCOPES:
175
+ raise RuntimeError(f"dashboard metric in {key} has no governed scope")
176
+ if not metric.get("basisLabel") or not metric.get("validationMethod"):
177
+ raise RuntimeError(f"dashboard metric in {key} is ungrounded")
178
+ for row in section.get("rows", []):
179
+ for column in ("actual", "budget", "prior"):
180
+ cell = row.get(column) or {}
181
+ if cell.get("state") == "fact" and cell.get("factKey") not in fact_keys:
182
+ raise RuntimeError(f"dashboard {key} references an ungoverned fact")
183
+ for row in section.get("planningRows", []):
184
+ if row.get("scope") not in _FINANCE_SCOPES or not row.get("basisLabel"):
185
+ raise RuntimeError(f"planning row in {key} is ungrounded")
186
+ actual = row.get("actual") or {}
187
+ if actual.get("state") != "fact" or actual.get("factKey") not in fact_keys:
188
+ raise RuntimeError(f"planning row in {key} references an ungoverned actual")
189
+ for column in ("plan", "variance", "driver"):
190
+ _require_unavailable(row.get(column), f"planning {column} in {key}")
191
+ return True
192
+
193
+
194
+ def finance_application(key, may_open=None):
195
+ """Resolve one Finance application without constructing a Finance-home payload."""
196
+ app = _finance_application(key, may_open=may_open)
197
+ validate_finance_application_contract(app)
198
+ app["facts"] = _finance_fact_specs(app["factKeys"])
199
+ app["kpis"] = _finance_fact_specs(app["kpiKeys"])
200
+ return app
201
 
202
 
203
  def blank_doc():
 
345
  template_kind = str(payload.get("templateKind") or "").strip().lower()
346
  if template_kind != kind:
347
  raise ApplicationError(f"{kind} applications require templateKind {kind!r}")
348
+ if kind == FINANCE_TEMPLATE_KIND:
349
+ finance_key = str(payload.get("financeKey") or "").strip()
350
+ try:
351
+ app = _finance_application(finance_key)
352
+ except KeyError:
353
+ raise ApplicationError(f"Finance application {finance_key!r} does not exist.")
354
+ if payload.get("sourceDatabase") or payload.get("fieldMap") or payload.get("compatibleDatabases"):
355
+ raise ApplicationError("Finance applications do not bind a database or fields.")
356
+ raw_placement = payload.get("navPlacement")
357
+ if not isinstance(raw_placement, dict) or set(raw_placement) != {"section", "order"}:
358
+ raise ApplicationError("navPlacement must contain section and order")
359
+ if raw_placement.get("section") != "app":
360
+ raise ApplicationError("Finance applications must be placed in App.")
361
+ order = raw_placement.get("order")
362
+ if type(order) is not int or not 0 <= order <= 999:
363
+ raise ApplicationError("navPlacement order must be an integer from 0 to 999")
364
+ return {"templateKind": kind, "financeKey": finance_key, "deepLink": app["deepLink"],
365
+ "navPlacement": {"section": "app", "order": order}}
366
  source = str(payload.get("sourceDatabase") or "").strip()[:120]
367
  if not source:
368
  raise ApplicationError("sourceDatabase is required")
 
463
  "default":bool(is_admin)}
464
  row["starred"] = user in (app.get("starredBy") or ())
465
  row["isDefault"] = bool(is_default)
466
+ if row.get("templateKind") == FINANCE_TEMPLATE_KIND:
467
+ # The database-bound mutation doors cannot edit Finance definitions. Honour that in
468
+ # every card rather than displaying controls that can only refuse.
469
+ row["capabilities"].update({key: False for key in (
470
+ "edit", "share", "delete", "duplicate", "star", "default")})
471
+ row["starred"] = False
472
+ row["isDefault"] = False
473
  target_database = target_database if target_database in compatible else (compatible[0] if compatible else None)
474
  row["openTarget"]={"kind":"application","id":row["id"],"type":row["type"],
475
  "database":target_database}
 
483
  for app in doc["applications"].values():
484
  if not may_read(app,user,is_admin): continue
485
  if database and database not in (app.get("compatibleDatabases") or ()): continue
486
+ if (app.get("templateKind") != FINANCE_TEMPLATE_KIND
487
+ and not _allowed(app.get("compatibleDatabases"), allowed_databases)): continue
488
  if kind and app.get("type") != kind: continue
489
  if starred is not None and bool(user in (app.get("starredBy") or ())) != bool(starred): continue
490
  default_key = f"{database}::{app.get('type')}" if database else ""
 
494
  return sorted(rows,key=lambda x:(str(x.get("updatedAt") or ""),str(x.get("id"))),reverse=True)
495
 
496
 
497
+ def nav_applications(runtime, user, is_admin=False, *, may_open=None, allowed_databases=None):
498
+ """The ONE generic navigation read for stored and seeded Application records (R6)."""
499
+ rows = [{"id": app["id"], "name": app.get("name"),
500
+ "type": app.get("type"), "templateKind": app.get("templateKind"),
501
+ "deepLink": app.get("deepLink"),
502
+ "origin": "tenant",
503
+ "compatibleDatabases": app.get("compatibleDatabases") or [],
504
+ "openTarget": app.get("openTarget"),
505
+ "navPlacement": app.get("navPlacement") or {"section": "app", "order": index},
506
+ "capabilities": app.get("capabilities"), "starred": app.get("starred"),
507
+ "isDefault": app.get("isDefault")}
508
+ for index, app in enumerate(list_apps(runtime, user, is_admin,
509
+ allowed_databases=allowed_databases))]
510
+ return rows + finance_application_records(may_open=may_open)
511
+
512
+
513
  def create(runtime, tenant, user, payload, *, semantic_records=None, field_contracts=None,
514
  allowed_databases=None):
515
  payload=payload if isinstance(payload,dict) else {}
 
521
  allowed_databases=allowed_databases)
522
  name=" ".join(str(payload.get("name") or "").split())[:120]
523
  if not name: raise ApplicationError("application name is required")
524
+ app_id=f"app_{write_execution.new_id()[:16]}"; now=write_execution.utcnow().isoformat()
525
+ compatible = ([] if template_kind == FINANCE_TEMPLATE_KIND
526
+ else [definition["sourceDatabase"]] if definition
527
  else _compat(payload.get("compatibleDatabases")))
528
  if _allowed(compatible, allowed_databases) != set(compatible):
529
  raise PermissionError("an Application cannot bind a database you cannot open")
 
551
  allowed_databases=None, target_database=None):
552
  patch=patch if isinstance(patch,dict) else {}; landed={}
553
  immutable = sorted(set(patch) & {
554
+ "id", "tenant", "type", "templateKind", "sourceDatabase", "fieldMap", "navPlacement",
555
+ "financeKey", "deepLink"
556
  })
557
  if immutable:
558
  raise ApplicationError("Application identity and template definition are immutable: "
 
593
  stars=set(app.get("starredBy") or ())
594
  (stars.add if patch["starred"] else stars.discard)(user); app["starredBy"]=sorted(stars)
595
  app["revision"]=int(app.get("revision") or 0)+1
596
+ app["updatedAt"]=write_execution.utcnow().isoformat(); landed.update(app)
597
  return doc
598
  runtime.update(_KEY,apply)
599
  return _public(landed,user,is_admin,allowed_databases=allowed_databases,
api/automation_engine.py CHANGED
@@ -34,7 +34,7 @@ import re
34
  import socket
35
  import threading
36
  import time
37
- from urllib.parse import urljoin, urlparse
38
  from zoneinfo import ZoneInfo, ZoneInfoNotFoundError
39
 
40
  import requests
@@ -61,6 +61,25 @@ STORE_KEY = "automations"
61
  AGENT_CONTRACT_KEY = "agent_contracts"
62
  AGENT_EVENT_TYPES = ("transcript", "message", "gathered_data", "run", "consent")
63
  AGENT_ACTIONS = ("read", "create", "update", "overwrite", "delete")
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
64
  #: ⭐⭐ W47-T21 / R3 — THE THREE NAMED REASONS AN AGENT'S ROW FILTER REFUSES WITH.
65
  #:
66
  #: R3, verbatim: an agent's row filter is *"its whole world: it may neither read nor write a
@@ -2218,13 +2237,18 @@ def clean_definition(raw, previous=None, username="", notes=None, rt=None):
2218
  """Whole-definition validation. Returns `(defn, error)`."""
2219
  raw = raw if isinstance(raw, dict) else {}
2220
  prev = previous or {}
 
 
 
 
 
2221
  # ⭐ WAVE 24 · C-TRIG — THE TRIGGER IS RESOLVED FIRST NOW, because law 1 makes it the thing
2222
  # that CHOOSES the kind. Reading `raw["trigger"]["key"]` here instead would be a second
2223
  # reader of the trigger vocabulary, free to disagree with `clean_trigger` about which keys
2224
  # exist and which are refused. Gate-visible consequence, recorded in AMENDMENT A1: a payload
2225
  # invalid in BOTH its trigger and its config now answers with the trigger's sentence.
2226
- trigger, terr = clean_trigger(raw.get("trigger") if "trigger" in raw
2227
- else prev.get("trigger"), prev.get("trigger"))
2228
  if terr:
2229
  return None, terr
2230
  # C-TRIG law 2 (wave 24): a definition that names no kind is a `plain` one. Before R6 this
@@ -2619,6 +2643,56 @@ def retire_automation_board_state(rt, tables=None):
2619
  return {"definitions": len(plan), **fields}
2620
 
2621
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2622
  #: ⭐⭐ WAVE 31 · T35 (D-134) — the definitions memo, and the two things that make it safe.
2623
  #: `tenant -> (monotonic_at, defs)`. Read ONLY through `all_definitions(..., cached=True)`, which
2624
  #: exactly one caller uses (`grid_hook`).
@@ -2696,7 +2770,8 @@ def all_definitions(rt, cached=False):
2696
  raw = dict(rt.get(STORE_KEY) or {})
2697
  except Exception:
2698
  return {}
2699
- out = {str(aid): _without_retired_board(defn)[0] for aid, defn in raw.items()}
 
2700
  if cached:
2701
  _DEFS_MEMO[str(getattr(rt, "key", "") or "")] = (time.monotonic(), out)
2702
  return out
@@ -2813,7 +2888,11 @@ def agent_contract_dto(rt, agent_id):
2813
 
2814
 
2815
  def authorize_agent_principal(rt, agent_id, actor, is_admin=False):
2816
- rec = _agent_contracts(rt).get(str(agent_id or "")) or {}
 
 
 
 
2817
  sharing = rec.get("sharing") or {}
2818
  if is_admin or rec.get("createdBy") == str(actor or "") or str(actor or "") in set(sharing.get("sharedWith") or []):
2819
  return True
@@ -3353,7 +3432,7 @@ def patch(rt, auto_id, raw, username="", notes=None):
3353
  # DISCARDED — the client shows a saved flow that the store never received, and nothing goes
3354
  # red. `flow` joined it in the same edit that created `flow` (wave 23), which is the only
3355
  # ordering that never has a window where the bug exists.
3356
- for k in ("name", "config", "schedule", "kind", "trigger", "flow"):
3357
  if k in (raw or {}):
3358
  merged[k] = raw[k]
3359
  # 2026-08-10 — the PATCH door needs it too, and this is the one that actually bit: picking the
@@ -9958,8 +10037,6 @@ ACTION_CATALOG = [
9958
  "ready": True, "menu": False,
9959
  "detail": "Fill an AI column for the records this flow walks. Configured on the column, not "
9960
  "here"},
9961
- {"kind": "slack", "label": "Send Slack message", "group": "Connected", "ready": False,
9962
- "detail": "Needs the Slack connector"},
9963
  # ── 4. ADVANCED LOGIC ────────────────────────────────────────────────────────────────────
9964
  # `group` is relabelled "If / then" (C-ACT): "Conditional logic" described the mechanism,
9965
  # and R8 made it a FORK with lettered branches, which is a thing people already have a name
@@ -15492,7 +15569,10 @@ def _row_gate(rt, defn, trg):
15492
  """
15493
  key = trg.get("key")
15494
  if key == "enters_view":
15495
- tree, fields, problem = view_filter(rt, trg.get("table"), trg.get("viewId"))
 
 
 
15496
  if problem:
15497
  return None, problem
15498
  import harness.filter_eval as filter_eval
@@ -15527,7 +15607,7 @@ def _settle_eval(rt, tenant, auto_id, log=print):
15527
  return
15528
  fired = []
15529
  if gate is not None:
15530
- rows = (ut_get(rt, trg.get("table") or "") or {}).get("rows") or {}
15531
  disarmed = set((d.get("state") or {}).get("eventDisarmed") or [])
15532
  nxt = set(disarmed)
15533
  for rid in touched:
@@ -15572,7 +15652,7 @@ def _seed_event_state(rt, defn):
15572
  gate, problem = _row_gate(rt, defn, trg)
15573
  if gate is None or problem:
15574
  return # no match gate ⇒ nothing to arm; a broken gate seeds nothing
15575
- rows = (ut_get(rt, trg.get("table") or "") or {}).get("rows") or {}
15576
  matching = sorted(str(rid) for rid, row in rows.items() if gate(row or {}))
15577
  set_state(rt, defn.get("id"), {"eventDisarmed": matching[:5000]})
15578
 
@@ -15600,10 +15680,11 @@ def grid_hook(evt):
15600
  for aid, d in defs.items():
15601
  trg = (d or {}).get("trigger") or {}
15602
  key = str(trg.get("key") or "")
 
15603
  if trg.get("paused") or not trg.get("enabled", True) \
15604
  or not trg.get("configured", True) \
15605
  or key not in TRIGGER_ROW_KEYS \
15606
- or str(trg.get("table") or "") != table:
15607
  continue
15608
  if kind == "record_created" and key == "record_created":
15609
  hw = _ig_int((d.get("state") or {}).get("rcHighwater")) or 0
@@ -16060,7 +16141,9 @@ def run_now(rt, tenant, auto_id, username="automation", log=print, rows=None):
16060
  # inside `dispatch_authorized_slack_automation` before its reply sentence is composed;
16061
  # notification trouble must never rewrite the truthful outcome of work that already ran.
16062
  try:
16063
- dispatch_authorized_slack_automation(rt, defn, {"state": state})
 
 
16064
  except Exception as e: # noqa: BLE001
16065
  log(f"[aios-auto] Slack notification for {auto_id} failed: {type(e).__name__}: {e}")
16066
  return _commit_run(rt, auto_id, state, summary, counts, state != "error", affected,
 
34
  import socket
35
  import threading
36
  import time
37
+ from urllib.parse import quote, urljoin, urlparse
38
  from zoneinfo import ZoneInfo, ZoneInfoNotFoundError
39
 
40
  import requests
 
61
  AGENT_CONTRACT_KEY = "agent_contracts"
62
  AGENT_EVENT_TYPES = ("transcript", "message", "gathered_data", "run", "consent")
63
  AGENT_ACTIONS = ("read", "create", "update", "overwrite", "delete")
64
+
65
+ #: MIGRATION NOTE (SDLC 64 / R3, 2026-09-21) — the only surviving occurrence of the
66
+ #: removed preset name. D-295 records why this must remain: a person's stored automation
67
+ #: must be dropped with a named reason, not refused on every save forever. Every read
68
+ #: drops these definitions; `retire_preset_automations` also sweeps them from the store.
69
+ RETIRED_PRESETS = {
70
+ "customer_intelligence": (
71
+ "Customer intelligence has been removed from Loopable. This automation's trigger and "
72
+ "its insight action no longer exist, so it can no longer run and has been removed."),
73
+ }
74
+
75
+
76
+ def retired_preset_reason(definition):
77
+ """The sentence to show for a stored definition whose preset is gone, else ``""``."""
78
+ if not isinstance(definition, dict):
79
+ return ""
80
+ return RETIRED_PRESETS.get(str(definition.get("preset") or "").strip(), "")
81
+
82
+
83
  #: ⭐⭐ W47-T21 / R3 — THE THREE NAMED REASONS AN AGENT'S ROW FILTER REFUSES WITH.
84
  #:
85
  #: R3, verbatim: an agent's row filter is *"its whole world: it may neither read nor write a
 
2237
  """Whole-definition validation. Returns `(defn, error)`."""
2238
  raw = raw if isinstance(raw, dict) else {}
2239
  prev = previous or {}
2240
+ preset = _s(raw.get("preset") if "preset" in raw else prev.get("preset"), 60).strip()
2241
+ if preset in RETIRED_PRESETS:
2242
+ return None, RETIRED_PRESETS[preset]
2243
+ if preset:
2244
+ return None, f"unknown automation preset {preset!r}"
2245
  # ⭐ WAVE 24 · C-TRIG — THE TRIGGER IS RESOLVED FIRST NOW, because law 1 makes it the thing
2246
  # that CHOOSES the kind. Reading `raw["trigger"]["key"]` here instead would be a second
2247
  # reader of the trigger vocabulary, free to disagree with `clean_trigger` about which keys
2248
  # exist and which are refused. Gate-visible consequence, recorded in AMENDMENT A1: a payload
2249
  # invalid in BOTH its trigger and its config now answers with the trigger's sentence.
2250
+ trigger_raw = raw.get("trigger") if "trigger" in raw else prev.get("trigger")
2251
+ trigger, terr = clean_trigger(trigger_raw, prev.get("trigger"))
2252
  if terr:
2253
  return None, terr
2254
  # C-TRIG law 2 (wave 24): a definition that names no kind is a `plain` one. Before R6 this
 
2643
  return {"definitions": len(plan), **fields}
2644
 
2645
 
2646
+ def _no_preset_retirement():
2647
+ """The receipt when nothing was retired: the SAME keys the success path answers with.
2648
+
2649
+ A FUNCTION rather than a module constant, because the receipt is handed to a caller and a
2650
+ shared literal would let one caller's edit reach the next one.
2651
+ """
2652
+ return {"definitions": 0, "ids": [], "reasons": {}, "removed": []}
2653
+
2654
+
2655
+ def retire_preset_automations(rt):
2656
+ """Persist retired-definition removal and name each removed automation in the log.
2657
+
2658
+ ⭐⭐ THE RECEIPT CARRIES `removed` SO THE CALLER NEEDS NO SECOND READ. `list_automations` has
2659
+ to tell the reader WHICH automation went and why, and the obvious way to get the names — read
2660
+ the bucket again beside the sweep — is the defect the WAVE 30 · T12 comment block above that
2661
+ call site is the tombstone of: a whole-tenant document re-serialised under the store lock on
2662
+ every single request, to report something that is empty on all but one of them. This function
2663
+ has already read every row it removed, so it hands them back.
2664
+ """
2665
+ try:
2666
+ stored = dict(rt.get(STORE_KEY) or {})
2667
+ except Exception:
2668
+ return _no_preset_retirement()
2669
+ plan = {aid for aid, definition in stored.items() if retired_preset_reason(definition)}
2670
+ if not plan:
2671
+ return _no_preset_retirement()
2672
+ removed = {}
2673
+
2674
+ def _up(cur):
2675
+ cur = cur if isinstance(cur, dict) else {}
2676
+ # A store retry replays this callback. Report only the successful attempt, and
2677
+ # recheck each row so an intervening edit to a live definition is preserved.
2678
+ removed.clear()
2679
+ for aid in sorted(plan):
2680
+ definition = cur.get(aid)
2681
+ reason = retired_preset_reason(definition)
2682
+ if reason:
2683
+ removed[str(aid)] = (str(definition.get("name") or "Unnamed automation"), reason)
2684
+ cur.pop(aid)
2685
+ return cur
2686
+
2687
+ _store_update(rt, _up, flush="sync")
2688
+ for aid, (name, reason) in removed.items():
2689
+ print(f"[aios-auto] removed automation {aid!r} ({name!r}): {reason}")
2690
+ return {"definitions": len(removed), "ids": list(removed),
2691
+ "reasons": {aid: reason for aid, (_name, reason) in removed.items()},
2692
+ "removed": [{"id": aid, "name": name, "reason": reason}
2693
+ for aid, (name, reason) in removed.items()]}
2694
+
2695
+
2696
  #: ⭐⭐ WAVE 31 · T35 (D-134) — the definitions memo, and the two things that make it safe.
2697
  #: `tenant -> (monotonic_at, defs)`. Read ONLY through `all_definitions(..., cached=True)`, which
2698
  #: exactly one caller uses (`grid_hook`).
 
2770
  raw = dict(rt.get(STORE_KEY) or {})
2771
  except Exception:
2772
  return {}
2773
+ out = {str(aid): _without_retired_board(defn)[0] for aid, defn in raw.items()
2774
+ if not retired_preset_reason(defn)}
2775
  if cached:
2776
  _DEFS_MEMO[str(getattr(rt, "key", "") or "")] = (time.monotonic(), out)
2777
  return out
 
2888
 
2889
 
2890
  def authorize_agent_principal(rt, agent_id, actor, is_admin=False):
2891
+ rec = _agent_contracts(rt).get(str(agent_id or ""))
2892
+ # An admin may read a live private Agent; that role cannot resurrect a deleted Agent
2893
+ # or replace the user identity required for authorization.
2894
+ if not str(actor or "").strip() or not isinstance(rec, dict) or not rec:
2895
+ raise PermissionError("this Agent or its principal is no longer available")
2896
  sharing = rec.get("sharing") or {}
2897
  if is_admin or rec.get("createdBy") == str(actor or "") or str(actor or "") in set(sharing.get("sharedWith") or []):
2898
  return True
 
3432
  # DISCARDED — the client shows a saved flow that the store never received, and nothing goes
3433
  # red. `flow` joined it in the same edit that created `flow` (wave 23), which is the only
3434
  # ordering that never has a window where the bug exists.
3435
+ for k in ("name", "config", "schedule", "kind", "trigger", "flow", "preset"):
3436
  if k in (raw or {}):
3437
  merged[k] = raw[k]
3438
  # 2026-08-10 — the PATCH door needs it too, and this is the one that actually bit: picking the
 
10037
  "ready": True, "menu": False,
10038
  "detail": "Fill an AI column for the records this flow walks. Configured on the column, not "
10039
  "here"},
 
 
10040
  # ── 4. ADVANCED LOGIC ────────────────────────────────────────────────────────────────────
10041
  # `group` is relabelled "If / then" (C-ACT): "Conditional logic" described the mechanism,
10042
  # and R8 made it a FORK with lettered branches, which is a thing people already have a name
 
15569
  """
15570
  key = trg.get("key")
15571
  if key == "enters_view":
15572
+ # Native View intelligence keeps the C2 trigger exact (`kind` + `viewId`); its table is
15573
+ # the definition's source binding rather than a second, client-editable trigger field.
15574
+ tree, fields, problem = view_filter(
15575
+ rt, trg.get("table") or _flow_table(defn), trg.get("viewId"))
15576
  if problem:
15577
  return None, problem
15578
  import harness.filter_eval as filter_eval
 
15607
  return
15608
  fired = []
15609
  if gate is not None:
15610
+ rows = (ut_get(rt, trg.get("table") or _flow_table(d)) or {}).get("rows") or {}
15611
  disarmed = set((d.get("state") or {}).get("eventDisarmed") or [])
15612
  nxt = set(disarmed)
15613
  for rid in touched:
 
15652
  gate, problem = _row_gate(rt, defn, trg)
15653
  if gate is None or problem:
15654
  return # no match gate ⇒ nothing to arm; a broken gate seeds nothing
15655
+ rows = (ut_get(rt, trg.get("table") or _flow_table(defn)) or {}).get("rows") or {}
15656
  matching = sorted(str(rid) for rid, row in rows.items() if gate(row or {}))
15657
  set_state(rt, defn.get("id"), {"eventDisarmed": matching[:5000]})
15658
 
 
15680
  for aid, d in defs.items():
15681
  trg = (d or {}).get("trigger") or {}
15682
  key = str(trg.get("key") or "")
15683
+ source_table = str(trg.get("table") or _flow_table(d) or "")
15684
  if trg.get("paused") or not trg.get("enabled", True) \
15685
  or not trg.get("configured", True) \
15686
  or key not in TRIGGER_ROW_KEYS \
15687
+ or source_table != table:
15688
  continue
15689
  if kind == "record_created" and key == "record_created":
15690
  hw = _ig_int((d.get("state") or {}).get("rcHighwater")) or 0
 
16141
  # inside `dispatch_authorized_slack_automation` before its reply sentence is composed;
16142
  # notification trouble must never rewrite the truthful outcome of work that already ran.
16143
  try:
16144
+ dispatch_authorized_slack_automation(
16145
+ rt, defn, {"state": state, "summary": summary,
16146
+ "rowIds": affected or []})
16147
  except Exception as e: # noqa: BLE001
16148
  log(f"[aios-auto] Slack notification for {auto_id} failed: {type(e).__name__}: {e}")
16149
  return _commit_run(rt, auto_id, state, summary, counts, state != "error", affected,
api/connectors_ig.py CHANGED
@@ -1652,3 +1652,33 @@ def pull_profile(url, max_posts=None, log=print,
1652
  "no provider could read this profile "
1653
  f"({'; '.join(attempts) or 'blocked'})")}
1654
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1652
  "no provider could read this profile "
1653
  f"({'; '.join(attempts) or 'blocked'})")}
1654
 
1655
+
1656
+ def pull_ig_recipe(seed, limit, post_type="video", log=print):
1657
+ """Run one saved Instagram database recipe through the established paid runner.
1658
+
1659
+ This deliberately stays a very small adapter over :func:`pull_profile`. The database route
1660
+ owns persistence and the explicit ``Run now`` boundary; this module owns the vendor's native
1661
+ last-N/type vocabulary. Passing ``post_groups`` is load-bearing: it reaches the existing
1662
+ reels-only specialization (Apify when configured, otherwise Bright Data's native
1663
+ ``post_type: reel`` request) instead of buying a mixed window and filtering it afterwards.
1664
+ """
1665
+ try:
1666
+ n = int(limit)
1667
+ except (TypeError, ValueError):
1668
+ return {"state": "error", "profile": {}, "posts": [], "comments": [],
1669
+ "via": "", "note": "the post limit must be a whole number"}
1670
+ kind = str(post_type or "").strip().lower()
1671
+ if kind not in ("video", "image", "carousel"):
1672
+ return {"state": "error", "profile": {}, "posts": [], "comments": [],
1673
+ "via": "", "note": "choose video, image, or carousel posts"}
1674
+ if n < 1 or n > MAX_POST_WINDOW:
1675
+ return {"state": "error", "profile": {}, "posts": [], "comments": [],
1676
+ "via": "", "note": f"the post limit must be between 1 and {MAX_POST_WINDOW}"}
1677
+ return pull_profile(
1678
+ seed,
1679
+ max_posts=n,
1680
+ post_metrics=True,
1681
+ comment_metrics=False,
1682
+ post_groups=[{"type": kind, "limit": n}],
1683
+ log=log,
1684
+ )
api/connectors_tt.py CHANGED
@@ -337,7 +337,7 @@ def tt_handle(url):
337
  return s if s and all(c.isalnum() or c in "._" for c in s) else ""
338
 
339
 
340
- def tt_post_urls(node, limit=0):
341
  """⭐ WAVE 30 · T10 — the profile row's own post permalinks, newest-first as the vendor sends.
342
 
343
  ⛔ THIS IS WHY TIKTOK POST CAPTURE COSTS NO EXTRA DISCOVERY. `top_videos` rides the PROFILE row
@@ -363,6 +363,7 @@ def tt_post_urls(node, limit=0):
363
  not a number this function may invent.
364
  """
365
  raw = (node or {}).get("top_videos")
 
366
  out = []
367
  for item in raw if isinstance(raw, list) else []:
368
  # ⛔⛔ MEASURED ON A REAL ROW 2026-08-12, AND IT IS NOT WHAT THE SCHEMA SAID.
@@ -389,8 +390,17 @@ def tt_post_urls(node, limit=0):
389
  # whichever array happened to be longer is how one creator's window silently differs from
390
  # another's, and that reasoning is unchanged by this correction.
391
  if isinstance(item, dict):
 
 
 
 
 
 
 
392
  u = str(item.get("video_url") or item.get("url") or "").strip()
393
  else:
 
 
394
  u = str(item or "").strip()
395
  if u.lower().startswith("http") and u not in out:
396
  out.append(u)
@@ -458,7 +468,8 @@ def _media_deferrals(deferred):
458
 
459
 
460
  def pull_profile_tt(url, log=print, pending_profile=None, prefetch=None,
461
- max_posts=0, post_metrics=False, comment_metrics=False):
 
462
  """ONE TikTok profile from the vendor. Same return contract as `pull_profile`.
463
 
464
  `{state, profile, posts, comments, via, note}` with `state ∈ ok | partial | blocked | error`,
@@ -529,7 +540,7 @@ def pull_profile_tt(url, log=print, pending_profile=None, prefetch=None,
529
  return {"state": "partial", "profile": profile, "posts": [], "comments": [],
530
  "via": "brightdata",
531
  "note": note or "profile read; post capture is off for this step"}
532
- urls = tt_post_urls(node, limit=max_posts)
533
  if not urls:
534
  # ⚠ NOT AN ERROR AND NOT A RETRY. A creator with no `top_videos` has nothing to buy, and
535
  # saying so is what stops the next run paying to be told the same thing.
@@ -566,3 +577,32 @@ def pull_profile_tt(url, log=print, pending_profile=None, prefetch=None,
566
  return {"state": "ok", "profile": profile, "posts": posts, "comments": comments,
567
  "via": "brightdata", "deferredMedia": deferred_media,
568
  "note": c_note or note or ""}
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
337
  return s if s and all(c.isalnum() or c in "._" for c in s) else ""
338
 
339
 
340
+ def tt_post_urls(node, limit=0, post_types=None):
341
  """⭐ WAVE 30 · T10 — the profile row's own post permalinks, newest-first as the vendor sends.
342
 
343
  ⛔ THIS IS WHY TIKTOK POST CAPTURE COSTS NO EXTRA DISCOVERY. `top_videos` rides the PROFILE row
 
363
  not a number this function may invent.
364
  """
365
  raw = (node or {}).get("top_videos")
366
+ wanted = {str(value or "").strip().lower() for value in (post_types or ()) if str(value or "").strip()}
367
  out = []
368
  for item in raw if isinstance(raw, list) else []:
369
  # ⛔⛔ MEASURED ON A REAL ROW 2026-08-12, AND IT IS NOT WHAT THE SCHEMA SAID.
 
390
  # whichever array happened to be longer is how one creator's window silently differs from
391
  # another's, and that reasoning is unchanged by this correction.
392
  if isinstance(item, dict):
393
+ # ``top_videos`` is the vendor's native video window. Keep the type selection at
394
+ # this boundary so an excluded type never reaches the paid Posts dataset. A future
395
+ # vendor row that starts declaring photo-mode entries can use its own ``post_type``;
396
+ # today's measured rows have no such key and are videos by construction.
397
+ delivered_type = str(item.get("post_type") or "video").strip().lower()
398
+ if wanted and delivered_type not in wanted:
399
+ continue
400
  u = str(item.get("video_url") or item.get("url") or "").strip()
401
  else:
402
+ if wanted and "video" not in wanted:
403
+ continue
404
  u = str(item or "").strip()
405
  if u.lower().startswith("http") and u not in out:
406
  out.append(u)
 
468
 
469
 
470
  def pull_profile_tt(url, log=print, pending_profile=None, prefetch=None,
471
+ max_posts=0, post_metrics=False, comment_metrics=False,
472
+ post_types=None):
473
  """ONE TikTok profile from the vendor. Same return contract as `pull_profile`.
474
 
475
  `{state, profile, posts, comments, via, note}` with `state ∈ ok | partial | blocked | error`,
 
540
  return {"state": "partial", "profile": profile, "posts": [], "comments": [],
541
  "via": "brightdata",
542
  "note": note or "profile read; post capture is off for this step"}
543
+ urls = tt_post_urls(node, limit=max_posts, post_types=post_types)
544
  if not urls:
545
  # ⚠ NOT AN ERROR AND NOT A RETRY. A creator with no `top_videos` has nothing to buy, and
546
  # saying so is what stops the next run paying to be told the same thing.
 
577
  return {"state": "ok", "profile": profile, "posts": posts, "comments": comments,
578
  "via": "brightdata", "deferredMedia": deferred_media,
579
  "note": c_note or note or ""}
580
+
581
+
582
+ def pull_tt_recipe(seed, limit, post_type="video", log=print):
583
+ """Run one saved TikTok database recipe through the established paid runner.
584
+
585
+ TikTok's measured profile payload exposes a native ``top_videos`` window, so ``limit`` and
586
+ ``post_types`` are applied before the paid Posts scrape. Keeping this adapter beside the
587
+ vendor runner makes the route unable to silently regress to an after-the-fact UI filter.
588
+ """
589
+ try:
590
+ n = int(limit)
591
+ except (TypeError, ValueError):
592
+ return {"state": "error", "profile": {}, "posts": [], "comments": [],
593
+ "via": "", "note": "the post limit must be a whole number"}
594
+ kind = str(post_type or "").strip().lower()
595
+ if kind != "video":
596
+ return {"state": "error", "profile": {}, "posts": [], "comments": [],
597
+ "via": "", "note": "TikTok's native profile window supports video posts"}
598
+ if n < 1 or n > 30:
599
+ return {"state": "error", "profile": {}, "posts": [], "comments": [],
600
+ "via": "", "note": "the post limit must be between 1 and 30"}
601
+ return pull_profile_tt(
602
+ seed,
603
+ max_posts=n,
604
+ post_metrics=True,
605
+ comment_metrics=False,
606
+ post_types=(kind,),
607
+ log=log,
608
+ )
api/deps.py CHANGED
@@ -37,6 +37,35 @@ def err(status, code, message):
37
  return HTTPException(status_code=status, detail={"error": {"code": code, "message": message}})
38
 
39
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
40
  @dataclass
41
  class Session:
42
  """A verified request identity. `user` is the PUBLIC record (never a hash or a salt)."""
 
37
  return HTTPException(status_code=status, detail={"error": {"code": code, "message": message}})
38
 
39
 
40
+
41
+ def running_main():
42
+ """The `main` module THIS PROCESS IS ACTUALLY RUNNING, or None. Never an `import`.
43
+
44
+ ⛔⛔ D-781 — `import main` IN A ROUTE MODULE STARTED A SECOND COPY OF EVERY RESIDENT DAEMON.
45
+ The Space runs `uvicorn api.main:app`, so the module executing is `api.main`. Python keys
46
+ `sys.modules` by the imported NAME, so a plain `import main` from a route file is a DIFFERENT
47
+ module object and **re-executes main.py's whole body** — including its module-level
48
+ `threading.Thread(target=_prewarm, …).start()` and `threading.Thread(target=_store_resync_loop,
49
+ …).start()`. Two resync loops, two prewarms, in one single-worker container.
50
+
51
+ ⛔ AND IT COST REAL MONEY. Staging's own log showed the tell twice over: two `prewarm done`
52
+ lines from ONE boot (46.3s and 110.3s, 2026-09-21T11:35:24) and two
53
+ `odoo relational rebuild required`/`done` pairs per resync pass, every pass — double the
54
+ whole-document Neon reads, on a bill whose entire overage was read egress.
55
+
56
+ ⛔⛔ AND THE STATUS ROUTE WAS READING THE SHADOW. `_resync_state` pulled `RESYNC_STATE` from the
57
+ duplicate module — a second, separately-populated dict belonging to the second loop — so the
58
+ operator surface answering *"is the mirror advancing"* was describing the copy nobody meant to
59
+ start. That is the worse half: the cost was invisible and the disclosure was wrong.
60
+
61
+ ⚠ IT RESOLVES, IT NEVER IMPORTS, and that restraint is the whole fix: falling back to
62
+ `import main` when neither name is loaded would reintroduce exactly the defect. `api.main`
63
+ first (uvicorn), then `main` (the gates, which import it under that name) — and None when this
64
+ process is running neither, which each caller already handles.
65
+ """
66
+ return sys.modules.get("api.main") or sys.modules.get("main")
67
+
68
+
69
  @dataclass
70
  class Session:
71
  """A verified request identity. `user` is the PUBLIC record (never a hash or a salt)."""
api/formula_runtime.py CHANGED
@@ -84,7 +84,10 @@ FN_ARITY = {
84
  "OFFSET": (3, 5), "INDIRECT": (1, 2), "RAND": (0, 0),
85
  "FILTER": (2, 3), "SORT": (1, 4),
86
  }
87
- ALIASES = {"CONCAT": "CONCATENATE"}
 
 
 
88
 
89
  #: ⭐ THE TRANSPORT SPELLING OF A DOTTED EXCEL NAME, AND IT IS NOT AN ALIAS.
90
  #:
@@ -1755,6 +1758,26 @@ def _eval_value(node, row, today, diagnostics=None):
1755
  if isinstance(row, _SheetInputs):
1756
  left, right = 0 if left is None else left, 0 if right is None else right
1757
  left, right = int(left) if isinstance(left, bool) else left, int(right) if isinstance(right, bool) else right
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1758
  a, b = _num(left), _num(right)
1759
  if a is None or b is None: return None
1760
  if node["op"] == "/" and b == 0:
@@ -3257,9 +3280,17 @@ class SheetCalculation:
3257
  token = id(node)
3258
  if token not in self._volatile_values:
3259
  now = dt.datetime.fromisoformat(str(self.today)) if self.today else self._clock.replace(tzinfo=None)
3260
- epoch = dt.datetime(1904, 1, 1) if self.date_system == "1904" else dt.datetime(1899, 12, 30)
3261
- self._volatile_values[token] = (random.random() if fn == "RAND" else
3262
- (now - epoch).total_seconds() / 86400 if fn == "NOW" else (now - epoch).days)
 
 
 
 
 
 
 
 
3263
  return self._volatile_values[token]
3264
 
3265
  def _connect(self, key):
@@ -3612,7 +3643,7 @@ def _sheet_error_message(code):
3612
  }.get(code, "This formula could not be calculated.")
3613
 
3614
 
3615
- def recompute_document(book, *, cache_key=None, changes=None):
3616
  """Recalculate a C7 document in place; return C8 values/errors/provenance/work counters.
3617
 
3618
  Only a trusted (tenant, user, workbook) tuple can opt into this process-local bounded LRU.
@@ -3622,12 +3653,18 @@ def recompute_document(book, *, cache_key=None, changes=None):
3622
  formula values; it never scans a sheet's cell map on a valid delta.
3623
  """
3624
  with _document_calculations_lock:
 
 
 
 
 
 
3625
  tabs = [tab for tab in book.get("sheets", ()) if isinstance(tab, dict)]
3626
  names = _document_scopes(tabs)
3627
  tab_by_scope = {name.upper(): tab for tab, name in zip(tabs, names)}
3628
  signature = json.dumps([[(tab.get("id"), name) for tab, name in zip(tabs, names)],
3629
  book.get("namedRanges"), book.get("dateSystem"),
3630
- dt.date.today().isoformat()], sort_keys=True)
3631
  revision = book.get("rev")
3632
  valid_revision = type(revision) is int and revision >= 0
3633
  trusted = (isinstance(cache_key, tuple) and len(cache_key) == 3
@@ -3650,7 +3687,8 @@ def recompute_document(book, *, cache_key=None, changes=None):
3650
  else:
3651
  formula_keys.discard(key)
3652
  if incremental:
3653
- engine.update(delta, revision=revision)
 
3654
  if not incremental:
3655
  specs, formula_keys = {}, set()
3656
  for tab, name in zip(tabs, names):
@@ -3661,8 +3699,12 @@ def recompute_document(book, *, cache_key=None, changes=None):
3661
  specs[key] = _document_spec(cell, name)
3662
  if cell.get("kind") == "formula":
3663
  formula_keys.add(key)
3664
- engine = SheetCalculation(specs, revision=revision if valid_revision else 0, sheets=names,
3665
- named_ranges=book.get("namedRanges"), date_system=book.get("dateSystem", "1900"))
 
 
 
 
3666
  projected = 0
3667
  for key in formula_keys:
3668
  scope, _, addr = key.rpartition("!")
 
84
  "OFFSET": (3, 5), "INDIRECT": (1, 2), "RAND": (0, 0),
85
  "FILTER": (2, 3), "SORT": (1, 4),
86
  }
87
+ # Excel workbooks in the Daily Sales lineage use both spellings. Keep the authored formula text
88
+ # untouched in the workbook; normalization maps AVG to the same AST/function the catalogue's
89
+ # Average action emits, so quoted-sheet references and cached formula text still round-trip.
90
+ ALIASES = {"CONCAT": "CONCATENATE", "AVG": "AVERAGE"}
91
 
92
  #: ⭐ THE TRANSPORT SPELLING OF A DOTTED EXCEL NAME, AND IT IS NOT AN ALIAS.
93
  #:
 
1758
  if isinstance(row, _SheetInputs):
1759
  left, right = 0 if left is None else left, 0 if right is None else right
1760
  left, right = int(left) if isinstance(left, bool) else left, int(right) if isinstance(right, bool) else right
1761
+ # ISO is the one date value shape shared by import, rendering and database filters. Give
1762
+ # that same value Excel's day arithmetic here instead of forcing a caller to replace it
1763
+ # with a serial number that the date filter cannot read. Date-date subtraction answers a
1764
+ # day count; date +/- number answers another ISO date. Other operators remain numeric.
1765
+ if node["op"] in ("+", "-"):
1766
+ left_day, right_day = _date(left), _date(right)
1767
+ left_num, right_num = _num(left), _num(right)
1768
+ try:
1769
+ if left_day is not None and right_day is not None and node["op"] == "-":
1770
+ return (left_day - right_day).days
1771
+ if left_day is not None and right_num is not None:
1772
+ shift = right_num if node["op"] == "+" else -right_num
1773
+ if int(shift) == shift:
1774
+ return (left_day + dt.timedelta(days=int(shift))).isoformat()
1775
+ if right_day is not None and left_num is not None and node["op"] == "+":
1776
+ if int(left_num) == left_num:
1777
+ return (right_day + dt.timedelta(days=int(left_num))).isoformat()
1778
+ except (OverflowError, ValueError):
1779
+ _diagnose(diagnostics, "bad_value")
1780
+ return None
1781
  a, b = _num(left), _num(right)
1782
  if a is None or b is None: return None
1783
  if node["op"] == "/" and b == 0:
 
3280
  token = id(node)
3281
  if token not in self._volatile_values:
3282
  now = dt.datetime.fromisoformat(str(self.today)) if self.today else self._clock.replace(tzinfo=None)
3283
+ # Formula dates have one representation in this runtime: ISO, the same representation
3284
+ # imported date literals, DATE/DATEVALUE, filters and the cell display path consume.
3285
+ # The old document graph special-cased volatile dates as Excel serials while the
3286
+ # compatibility evaluator returned ISO, so TODAY() could render as 46284 while
3287
+ # DATE(...) beside it rendered 2026-09-19. dateSystem still governs imported Excel
3288
+ # serial conversion at the input boundary; it must not create a second stored value
3289
+ # type after that conversion.
3290
+ self._volatile_values[token] = (
3291
+ random.random() if fn == "RAND" else
3292
+ now.strftime("%Y-%m-%d %H:%M:%S") if fn == "NOW" else
3293
+ now.date().isoformat())
3294
  return self._volatile_values[token]
3295
 
3296
  def _connect(self, key):
 
3643
  }.get(code, "This formula could not be calculated.")
3644
 
3645
 
3646
+ def recompute_document(book, *, cache_key=None, changes=None, today=None):
3647
  """Recalculate a C7 document in place; return C8 values/errors/provenance/work counters.
3648
 
3649
  Only a trusted (tenant, user, workbook) tuple can opt into this process-local bounded LRU.
 
3653
  formula values; it never scans a sheet's cell map on a valid delta.
3654
  """
3655
  with _document_calculations_lock:
3656
+ explicit_today = today is not None
3657
+ calculation_day = str(today or dt.date.today().isoformat())[:10]
3658
+ try:
3659
+ calculation_day = dt.date.fromisoformat(calculation_day).isoformat()
3660
+ except ValueError as exc:
3661
+ raise ValueError("document calculation today must be an ISO date") from exc
3662
  tabs = [tab for tab in book.get("sheets", ()) if isinstance(tab, dict)]
3663
  names = _document_scopes(tabs)
3664
  tab_by_scope = {name.upper(): tab for tab, name in zip(tabs, names)}
3665
  signature = json.dumps([[(tab.get("id"), name) for tab, name in zip(tabs, names)],
3666
  book.get("namedRanges"), book.get("dateSystem"),
3667
+ calculation_day], sort_keys=True)
3668
  revision = book.get("rev")
3669
  valid_revision = type(revision) is int and revision >= 0
3670
  trusted = (isinstance(cache_key, tuple) and len(cache_key) == 3
 
3687
  else:
3688
  formula_keys.discard(key)
3689
  if incremental:
3690
+ engine.update(delta, revision=revision,
3691
+ today=calculation_day if explicit_today else None)
3692
  if not incremental:
3693
  specs, formula_keys = {}, set()
3694
  for tab, name in zip(tabs, names):
 
3699
  specs[key] = _document_spec(cell, name)
3700
  if cell.get("kind") == "formula":
3701
  formula_keys.add(key)
3702
+ engine = SheetCalculation(
3703
+ specs, revision=revision if valid_revision else 0, sheets=names,
3704
+ named_ranges=book.get("namedRanges"), date_system=book.get("dateSystem", "1900"),
3705
+ # A supplied day makes an exceptional repair deterministic. Ordinary workbook
3706
+ # recalculation keeps the live clock, so NOW() does not silently become midnight.
3707
+ today=calculation_day if explicit_today else None)
3708
  projected = 0
3709
  for key in formula_keys:
3710
  scope, _, addr = key.rpartition("!")
api/main.py CHANGED
@@ -13,7 +13,8 @@ WHAT CHANGED IN EXIT WAVE 1 (2026-07-30) — three things, all of them load-bear
13
  user RECORD. `/api/health` stays unauthenticated (liveness only).
14
  * **The overlay fork is deleted.** `aios-web/api/data/overlay.json` was a second writable home
15
  for user-owned fields the Streamlit app keeps in the tenant store. ONE store now (C1c).
16
- * **The SSL shim is env-gated.** It used to run at import, unconditionally.
 
17
 
18
  ⚠ TENANT RESOLUTION IS PER REQUEST (X7). Nothing tenant-shaped is held at module level; the
19
  session's tenant claim resolves through `harness.runtime.get_runtime`, which is LRU-bounded. This
@@ -25,18 +26,6 @@ import os
25
  import sys
26
  from pathlib import Path
27
 
28
- # --- the local Odoo SSL quirk, now BEHIND A GATE -------------------------------------------------
29
- # The Windows trust store reports the (valid) Odoo cert as expired, so local runs need an
30
- # unverified default context; the HF Space and the container are unaffected. This used to run
31
- # unconditionally at import — i.e. the shipped container disabled TLS verification for every
32
- # outbound HTTPS call it ever made, to Odoo and to everything else, forever. That is a
33
- # man-in-the-middle away from being someone else's data. It is now opt-in, must be set
34
- # deliberately, and is never on by default.
35
- # ⛔ NEVER set AIOS_INSECURE_SSL in a deployed environment. It exists for one developer laptop.
36
- if os.environ.get("AIOS_INSECURE_SSL") == "1":
37
- import ssl
38
- ssl._create_default_https_context = ssl._create_unverified_context # noqa: S323
39
-
40
  # --- reuse the tenant #0 data layer verbatim. RI_DIR overrides the location in the container
41
  # (where the layout differs from the local sibling-dir default). ---
42
  _HERE = Path(__file__).resolve()
@@ -93,6 +82,7 @@ import routes_feedback # noqa: E402 (wave 35 R8/C6 — feedback to the operato
93
  import routes_agent_harness # noqa: E402 (wave 36 R8/C4 — the agent harness store; E's router)
94
  import routes_geo # noqa: E402 (wave 37 R14/A11 — the keyless map provider seam; D's router)
95
  from core import grid_events # noqa: E402
 
96
  # D-315 / D-305 (2026-08-18) — the two store REFUSALS get their own app-level handlers below.
97
  # ⚠ Neither is a `StoreUnavailable` subclass, on purpose: routes that degrade a store outage into a
98
  # session fallback must NOT degrade a refusal, or the user is told a change was saved that was not.
@@ -324,6 +314,8 @@ app.include_router(routes_nav.router)
324
  app.include_router(routes_customers.router)
325
  app.include_router(routes_products.router)
326
  app.include_router(routes_assets.router)
 
 
327
  app.include_router(routes_tables.router)
328
  app.include_router(routes_grid.router)
329
  app.include_router(routes_records.router)
@@ -541,9 +533,6 @@ def _startup_notes():
541
  print("[aios-api] AIOS_SESSION_SECRET is not set. Signing with a random per-process "
542
  "key. Sessions will not survive a restart and will not work across workers. "
543
  "Set it in production.")
544
- if os.environ.get("AIOS_INSECURE_SSL") == "1":
545
- print("[aios-api] AIOS_INSECURE_SSL=1. TLS verification is DISABLED for outbound "
546
- "requests. Local development only; never in a deployed environment.")
547
 
548
 
549
  _assert_contract_floor()
@@ -782,49 +771,93 @@ class _SyncNowRefused(Exception):
782
  """An application-authored, credential-free sentence for a Sync now receipt."""
783
 
784
 
785
- def _sync_now_failure(cause):
786
  """Classify actual provider failures without returning vendor text or exception reprs."""
787
  import re
788
- text = str(cause).lower()
 
789
  causes = (
790
- (("reduce the amount of data", "response too large"),
791
  "The provider refused the pull because the response contains too much data."),
792
- (("no ad accounts",), "The Meta token cannot access any ad accounts."),
793
- (("max_pages", "page cap"), "The pull stopped at the provider page limit."),
794
- (("sandbox",), "Amazon sync requires a production credential; this credential is for the sandbox."),
795
- (("deletion", "retention"), "Amazon sync is blocked by an outstanding data deletion requirement."),
796
- (("no stored", "no odoo credentials", "access_token is not set"),
 
 
797
  "This tenant has no usable credential for the connector."),
798
- (("enablement document", "capability unavailable"),
799
  "The connector database settings could not be read."),
800
- (("throttl", "rate limit", "too many requests"),
801
  "The provider rate limit stopped the sync. Retry after the provider delay."),
802
- (("timed out", "timeout"), "The sync failed because the provider request timed out."),
803
- (("permission", "access denied", "unauthorized", "invalid oauth", "invalid token",
804
  "expired", "authentication"), "The provider refused the connector credential or its permissions."),
805
- (("duplicate key", "primary key", "constraint"),
806
  "The mirror rejected rows because they violate a database constraint."),
807
- (("connection", "network", "name resolution"),
808
  "The sync could not connect to the provider or tenant database."),
809
  # A-META / D-201: a Meta pull on a container whose mirror never held `meta_*` raises
810
  # DuckDB's CatalogException. Standing rule 1 wants the CAUSE, not "unexpected error":
811
  # "the loader is broken" and "this server has never held these tables" are different
812
  # operator actions, and only the second one is true here.
813
- (("does not exist", "no such table", "catalog error"),
814
  "This server's mirror does not hold this connector's tables yet, so there was nothing "
815
  "to update. Run the connector's first load on the server that owns this tenant."),
816
  )
817
- for needles, sentence in causes:
818
  if any(needle in text for needle in needles):
819
- return sentence
820
  status = re.search(r"\bhttp\s+(\d{3})\b", text)
821
  if status:
822
- return f"The provider refused the sync request with HTTP status {status.group(1)}."
823
  if isinstance(cause, TypeError):
824
- return "The connector failed because its provider call used incompatible arguments."
825
  if isinstance(cause, (KeyError, ValueError)):
826
- return "The connector could not read an expected value in the sync response."
827
- return "The connector encountered an unexpected error while processing the sync response."
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
828
 
829
 
830
  def _run_sync_requests():
@@ -854,28 +887,47 @@ def _run_sync_requests():
854
  if provider == "odoo":
855
  import core.odoo as odoo
856
  import odoo_relational as rel
 
857
  if rt.odoo_paused() or rel.frozen(rt):
858
  raise _SyncNowRefused("Odoo is paused or disconnected. Resume the connection before syncing.")
859
- source = rt.odoo_source()
860
- if source is None:
861
- raise _SyncNowRefused("This tenant has no Odoo credential. Connect Odoo before syncing.")
862
- odoo_requested = True
863
- previous_client = getattr(odoo._tlocal, "client", None)
864
- previous_context = getattr(odoo._tlocal, "tenant_ctx", None)
865
- try:
866
- # Raw mirror pulls use the resolved tenant credential and a dedicated
867
- # transport. Do not borrow a request's cached XML-RPC client.
868
- odoo._tlocal.client = odoo.OdooClient(creds=source._creds)
869
- odoo._tlocal.tenant_ctx = None
870
- report = datastore.sync_all(log=lambda *a, **k: None)
871
- finally:
872
- odoo._tlocal.client = previous_client
873
- odoo._tlocal.tenant_ctx = previous_context
874
- if not report:
875
- raise _SyncNowRefused("Odoo did not run a sync. Check whether the connection is paused.")
876
- if any(row.get("phase") != "live" for row in report.values()):
877
- raise _SyncNowRefused("Odoo backfill is incomplete. Run Sync now again to continue it.")
878
- problems = []
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
879
  elif provider == "amazon_sp":
880
  from harness import amazon_store as amazon
881
  if not amazon.credential(tenant, rt=rt):
@@ -911,13 +963,13 @@ def _run_sync_requests():
911
  if problems:
912
  detail += " " + _sync_now_failure(problems[0])
913
  elif problems:
914
- detail = _sync_now_failure(problems[0])
915
  else:
916
  ok, detail = True, "Sync finished."
917
  except _SyncNowRefused as exc:
918
  detail = str(exc)
919
  except Exception as exc: # noqa: BLE001
920
- detail = _sync_now_failure(exc)
921
  if first_load_progress is not None:
922
  connector_sync.finish_sync_request(provider, tenant, ok=ok, detail=detail,
923
  first_load=first_load_progress)
@@ -1117,7 +1169,8 @@ def _sweep_automation_schemas(why):
1117
  print(f"[aios-api] schema sweep ({why}) done over {len(tenants)} tenant(s)")
1118
 
1119
 
1120
- def _rebuild_odoo_relational(why, previous_meta_fingerprint=None, meta_state_out=None):
 
1121
  """Spawn/refresh the four locked Odoo databases, then compute their cells. `why` is 'boot' or
1122
  'resync' and rides every log line, because "it failed" and "it failed at boot, before anyone
1123
  could have asked" are different diagnoses.
@@ -1127,21 +1180,38 @@ def _rebuild_odoo_relational(why, previous_meta_fingerprint=None, meta_state_out
1127
  and was fine" from "the rebuild was never reached". Three failure markers and no success
1128
  marker means silence is ambiguous; now it is not.
1129
 
1130
- ⚠ SCOPED TO ROYAL-IMPORTS, deliberately, and it is NOT the D-29 shortcut it resembles. The
1131
- caller has just advanced whichever DuckDB file this process holds open — tenant #0's — and
1132
- royal is the only tenant with an Odoo mirror to derive from (R1). Iterating tenants here walks
1133
- straight into D-29's documented hazard: `harness/datastore` is a ONE-FILE-AT-A-TIME
1134
- process-global, and rebinding it in a daemon thread under live readers can serve one tenant's
1135
- rows to another with nothing raised. `is_royal` stays the authority on which slugs qualify.
1136
  """
1137
  try:
 
1138
  import odoo_relational as _rel
1139
  from harness import runtime as _runtime
1140
- if not _rel.is_royal("royal-imports"):
1141
- return False
1142
- _rt = _runtime.get_runtime("royal-imports")
1143
  from core import user_tables as _ut
1144
- _ut.record_odoo_sync_status(phase="syncing", st=_rt)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1145
  # ⭐⭐ W54-T46 / R4 / R7 — EVERY Odoo CONNECTION, not just the first. `all_connections`
1146
  # delegates to `odoo_relational.refresh_all`, which walks this tenant's Odoo
1147
  # connections in slot order and records a failure against the connection it belongs
@@ -1156,7 +1226,7 @@ def _rebuild_odoo_relational(why, previous_meta_fingerprint=None, meta_state_out
1156
  # because a gate that counts RAW occurrences of a call would count the one in this
1157
  # comment too. That gate happens to strip comments first; the next one may not.
1158
  # See `odoo_relational.refresh` and its `all_connections` arm.
1159
- counts = _rel.refresh(_rt, "royal-imports", all_connections=True)
1160
  raw_models, schema_errors = _rel.authoritative_schema_snapshot()
1161
  _ut.record_odoo_schema_snapshot(raw_models, complete=not schema_errors,
1162
  errors=schema_errors, st=_rt)
@@ -1183,19 +1253,134 @@ def _rebuild_odoo_relational(why, previous_meta_fingerprint=None, meta_state_out
1183
  except Exception as e: # noqa: BLE001
1184
  rebuild_errors.append(f"relations: {type(e).__name__}: {e}")
1185
  print(f"[aios-api] odoo relation cells failed ({why}): {type(e).__name__}: {e}")
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1186
  try:
1187
  import rollup_sql as _rollup
 
1188
  for _b, _key, _l, _f in _rel.TABLES:
1189
- _rollup.compute(_rt, _key)
1190
  except Exception as e: # noqa: BLE001
1191
  rebuild_errors.append(f"rollups: {type(e).__name__}: {e}")
1192
  print(f"[aios-api] odoo source rollups failed ({why}): {type(e).__name__}: {e}")
1193
  print(f"[aios-api] odoo relational rebuild done ({why}): "
1194
  + ", ".join(f"{k}={v}" for k, v in sorted((counts or {}).items())))
1195
- if rebuild_errors:
1196
- _ut.record_odoo_sync_status(phase="error", errors=rebuild_errors, st=_rt)
1197
- else:
1198
- _ut.record_odoo_sync_status(phase="ok", st=_rt)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1199
  meta_state = _rebuild_meta_relational(
1200
  why, _rt, previous_fingerprint=previous_meta_fingerprint, force_relations=True)
1201
  if isinstance(meta_state_out, dict):
@@ -1207,7 +1392,7 @@ def _rebuild_odoo_relational(why, previous_meta_fingerprint=None, meta_state_out
1207
  # The isolated cells/rollups/meta handlers above are deliberately allowed to report and
1208
  # continue. This return value answers the narrower resync-policy question: did the Odoo
1209
  # table refresh itself complete, so a confirmed zero-change pass may be skipped?
1210
- return True
1211
  except Exception as e: # noqa: BLE001
1212
  # ⚠ The TYPE is named here as it is in the two inner handlers. The original printed only
1213
  # `{e}`, and a bare message is exactly what made D-107 unreadable from the outside: a
@@ -1217,7 +1402,14 @@ def _rebuild_odoo_relational(why, previous_meta_fingerprint=None, meta_state_out
1217
  try:
1218
  if '_rt' in locals():
1219
  from core import user_tables as _ut
1220
- _ut.record_odoo_sync_status(phase="error", errors=[f"{type(e).__name__}: {e}"], st=_rt)
 
 
 
 
 
 
 
1221
  except Exception:
1222
  pass
1223
  return False
@@ -1297,8 +1489,10 @@ RESYNC_STATE = {
1297
  "disagreement": None,
1298
  "passes": 0,
1299
  "lastPassAt": None,
 
1300
  "lastPassOk": None,
1301
  "lastError": None,
 
1302
  "consecutiveFailures": 0,
1303
  "bootSyncError": None,
1304
  }
@@ -1467,15 +1661,29 @@ def _classify_resync_error(exc):
1467
  ⚠ NOT the whole of D-314: naming the offending TABLE and ROW ID belongs to the DuckDB writer
1468
  that raised, which is not this loop.
1469
  """
 
1470
  text = f"{type(exc).__name__}: {exc}"
1471
  low = text.lower()
 
 
 
 
 
1472
  persistent = any(t in low for t in
1473
  ("duplicate key", "primary key", "unique constraint", "constraint error",
1474
  "binder", "conversion error"))
1475
- return {"type": type(exc).__name__, "message": str(exc)[:400],
1476
- "kind": "persistent" if persistent else "transient",
 
 
 
 
 
 
 
 
1477
  "meaning": ("this failure repeats until someone repairs the mirror; retrying the same "
1478
- "pass will not clear it" if persistent else
1479
  "this failure usually clears on the next pass")}
1480
 
1481
 
@@ -1543,6 +1751,14 @@ def _store_resync_loop():
1543
  "tenantSeconds": _cad["tenantSeconds"], "envSeconds": _cad["envSeconds"],
1544
  "resolvedSeconds": _every, "manual": _manual,
1545
  "disagreement": _cad["disagreement"]})
 
 
 
 
 
 
 
 
1546
  # ⚠ ON CHANGE, NOT EVERY CYCLE. An hourly warning is a skimmed warning.
1547
  if _cad["disagreement"] != _RESYNC_NOTICE[0]:
1548
  _RESYNC_NOTICE[0] = _cad["disagreement"]
@@ -1566,6 +1782,8 @@ def _store_resync_loop():
1566
  RESYNC_STATE["lastError"] = _err
1567
  if _err is None:
1568
  RESYNC_STATE["consecutiveFailures"] = 0
 
 
1569
  else:
1570
  RESYNC_STATE["consecutiveFailures"] = int(RESYNC_STATE["consecutiveFailures"]) + 1
1571
  # D-314: the TYPE and the persistent/transient reading, both in the line and in the
@@ -1619,7 +1837,17 @@ def _store_resync_loop():
1619
  _meta_state = {}
1620
  if _reason is not None:
1621
  print(f"[aios-api] odoo relational rebuild required ({_reason})")
1622
- if _rebuild_odoo_relational("resync", _meta_previous, _meta_state):
 
 
 
 
 
 
 
 
 
 
1623
  last_relational_success_at = _t.time()
1624
  else:
1625
  # A prior successful projection no longer proves the changed mirror was applied.
@@ -1797,7 +2025,7 @@ if os.environ.get("AIOS_PREWARM") == "1":
1797
  # ⭐⭐ W50-T29 / R8 — AND THE DEPLOYMENT NOW OPTS IN, ON STAGING AND ON LIVE. The owner's ruling is
1798
  # that the Space resyncs its own mirror hourly rather than waiting for someone to visit; without
1799
  # it a Space serves whatever the seed was cut with, forever, and August reads zero while Odoo has
1800
- # 723 orders. `deploy_web.py` pushes `AIOS_STORE_RESYNC=1` plus `AIOS_STORE_RESYNC_SECONDS=3600`.
1801
  #
1802
  # ⛔ THE OLD REASON FOR KEEPING IT OFF WAS "so a Pg-backed Live Space can scale its Neon compute to
1803
  # zero between real requests", AND THAT REASON IS ONLY HALF SATISFIED. `_resync_mirror_pass` is
@@ -1806,6 +2034,24 @@ if os.environ.get("AIOS_PREWARM") == "1":
1806
  # `_rebuild_meta_relational` write through `core.user_tables`, which is the tenant product store,
1807
  # and under `STORE_BACKEND=pg` that is Neon. Named here rather than left for the next reader to
1808
  # discover from a bill, and carried as a register row by this ticket's handoff.
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1809
  if os.environ.get("AIOS_STORE_RESYNC") == "1":
1810
  import threading as _threading_resync
1811
  _threading_resync.Thread(target=_store_resync_loop, daemon=True, name="store-resync").start()
 
13
  user RECORD. `/api/health` stays unauthenticated (liveness only).
14
  * **The overlay fork is deleted.** `aios-web/api/data/overlay.json` was a second writable home
15
  for user-owned fields the Streamlit app keeps in the tenant store. ONE store now (C1c).
16
+ * **Outbound TLS always uses the verified default context.** A certificate failure is a source
17
+ failure, never permission to accept an untrusted connection.
18
 
19
  ⚠ TENANT RESOLUTION IS PER REQUEST (X7). Nothing tenant-shaped is held at module level; the
20
  session's tenant claim resolves through `harness.runtime.get_runtime`, which is LRU-bounded. This
 
26
  import sys
27
  from pathlib import Path
28
 
 
 
 
 
 
 
 
 
 
 
 
 
29
  # --- reuse the tenant #0 data layer verbatim. RI_DIR overrides the location in the container
30
  # (where the layout differs from the local sibling-dir default). ---
31
  _HERE = Path(__file__).resolve()
 
82
  import routes_agent_harness # noqa: E402 (wave 36 R8/C4 — the agent harness store; E's router)
83
  import routes_geo # noqa: E402 (wave 37 R14/A11 — the keyless map provider seam; D's router)
84
  from core import grid_events # noqa: E402
85
+
86
  # D-315 / D-305 (2026-08-18) — the two store REFUSALS get their own app-level handlers below.
87
  # ⚠ Neither is a `StoreUnavailable` subclass, on purpose: routes that degrade a store outage into a
88
  # session fallback must NOT degrade a refusal, or the user is told a change was saved that was not.
 
314
  app.include_router(routes_customers.router)
315
  app.include_router(routes_products.router)
316
  app.include_router(routes_assets.router)
317
+ import routes_applications
318
+ app.include_router(routes_applications.router)
319
  app.include_router(routes_tables.router)
320
  app.include_router(routes_grid.router)
321
  app.include_router(routes_records.router)
 
533
  print("[aios-api] AIOS_SESSION_SECRET is not set. Signing with a random per-process "
534
  "key. Sessions will not survive a restart and will not work across workers. "
535
  "Set it in production.")
 
 
 
536
 
537
 
538
  _assert_contract_floor()
 
771
  """An application-authored, credential-free sentence for a Sync now receipt."""
772
 
773
 
774
+ def _sync_now_cause(cause):
775
  """Classify actual provider failures without returning vendor text or exception reprs."""
776
  import re
777
+ raw = str(cause)
778
+ text = raw.lower()
779
  causes = (
780
+ ("response_too_large", ("reduce the amount of data", "response too large"),
781
  "The provider refused the pull because the response contains too much data."),
782
+ ("meta_no_ad_accounts", ("no ad accounts",), "The Meta token cannot access any ad accounts."),
783
+ ("page_cap", ("max_pages", "page cap"), "The pull stopped at the provider page limit."),
784
+ ("sandbox_credential", ("sandbox",),
785
+ "Amazon sync requires a production credential; this credential is for the sandbox."),
786
+ ("deletion_outstanding", ("deletion", "retention"),
787
+ "Amazon sync is blocked by an outstanding data deletion requirement."),
788
+ ("no_credential", ("no stored", "no odoo credentials", "access_token is not set"),
789
  "This tenant has no usable credential for the connector."),
790
+ ("enablement_unreadable", ("enablement document", "capability unavailable"),
791
  "The connector database settings could not be read."),
792
+ ("rate_limited", ("throttl", "rate limit", "too many requests"),
793
  "The provider rate limit stopped the sync. Retry after the provider delay."),
794
+ ("timeout", ("timed out", "timeout"), "The sync failed because the provider request timed out."),
795
+ ("credential_refused", ("permission", "access denied", "unauthorized", "invalid oauth", "invalid token",
796
  "expired", "authentication"), "The provider refused the connector credential or its permissions."),
797
+ ("mirror_constraint", ("duplicate key", "primary key", "constraint"),
798
  "The mirror rejected rows because they violate a database constraint."),
799
+ ("connect_failed", ("connection", "network", "name resolution"),
800
  "The sync could not connect to the provider or tenant database."),
801
  # A-META / D-201: a Meta pull on a container whose mirror never held `meta_*` raises
802
  # DuckDB's CatalogException. Standing rule 1 wants the CAUSE, not "unexpected error":
803
  # "the loader is broken" and "this server has never held these tables" are different
804
  # operator actions, and only the second one is true here.
805
+ ("mirror_missing_tables", ("does not exist", "no such table", "catalog error"),
806
  "This server's mirror does not hold this connector's tables yet, so there was nothing "
807
  "to update. Run the connector's first load on the server that owns this tenant."),
808
  )
809
+ for classifier, needles, sentence in causes:
810
  if any(needle in text for needle in needles):
811
+ return classifier, sentence
812
  status = re.search(r"\bhttp\s+(\d{3})\b", text)
813
  if status:
814
+ return "http_status", f"The provider refused the sync request with HTTP status {status.group(1)}."
815
  if isinstance(cause, TypeError):
816
+ return "bad_arguments", "The connector failed because its provider call used incompatible arguments."
817
  if isinstance(cause, (KeyError, ValueError)):
818
+ return "unreadable_value", "The connector could not read an expected value in the sync response."
819
+ if isinstance(cause, str):
820
+ # amazon_store._sync_problem appends the application-authored fix LAST. The
821
+ # exception message may itself contain delimiters; none of it is display text.
822
+ problem, separator, fix = raw.rpartition("; fix: ")
823
+ header = re.match(
824
+ r"(?P<table>[A-Za-z_][A-Za-z0-9_.-]*): "
825
+ r"(?P<operation>[A-Za-z_][A-Za-z0-9_.-]*) (?P<state>[A-Z]+): "
826
+ r"(?P<exception>[A-Za-z_][A-Za-z0-9_]*): ", problem)
827
+ if separator and header and fix.strip():
828
+ # Keep the authored action, with sentence punctuation suitable for the UI.
829
+ action = fix.strip().replace("\u2014", ". ").replace("\u2013", " to ")
830
+ sentence = (
831
+ f"The Amazon table {header['table']} could not be read. "
832
+ f"The {header['operation']} call was {header['state'].lower()} "
833
+ f"with {header['exception']}. What to do: {action}"
834
+ )
835
+ return "connector_reported", sentence if sentence.endswith(".") else sentence + "."
836
+ return "unclassified", (
837
+ "The connector reported an unclassified plain-text failure. "
838
+ "Ask an administrator to inspect the connector's sync report and retry.")
839
+ return "unclassified", (
840
+ f"The connector failed with an unclassified {type(cause).__name__}. "
841
+ "Ask an administrator to inspect the connector's sync report and retry.")
842
+
843
+
844
+ def _sync_now_failure(cause):
845
+ """Classify actual provider failures without returning vendor text or exception reprs."""
846
+ return _sync_now_cause(cause)[1]
847
+
848
+
849
+ def _sync_now_detail(provider, tenant, *, ok, detail="", cause=None):
850
+ """Record one classified failure; leave successful receipt text untouched."""
851
+ if ok:
852
+ return detail
853
+ classifier, sentence = _sync_now_cause(cause)
854
+ print(f"[aios-api] sync FAILED provider={provider} tenant={tenant} classifier={classifier}")
855
+ suffix = f" (cause code: {classifier})"
856
+ # finish_sync_request stores only 300 characters. Reserve the suffix before it clips.
857
+ budget = 300 - len(suffix)
858
+ if len(sentence) > budget:
859
+ sentence = sentence[:budget - 3].rstrip() + "..."
860
+ return sentence + suffix
861
 
862
 
863
  def _run_sync_requests():
 
887
  if provider == "odoo":
888
  import core.odoo as odoo
889
  import odoo_relational as rel
890
+ import routes_connectors as _sync_routes
891
  if rt.odoo_paused() or rel.frozen(rt):
892
  raise _SyncNowRefused("Odoo is paused or disconnected. Resume the connection before syncing.")
893
+ requested_cid = (_sync_routes.sync_request_connection(provider, tenant)
894
+ or _sync_routes._odoo_connection_slot(rt, None))
895
+ default_cid = _sync_routes._odoo_connection_slot(rt, None)
896
+ if requested_cid != default_cid:
897
+ # Named connections have their own provider reader and must never be pulled
898
+ # through the process-global default mirror under another connection's id.
899
+ from types import SimpleNamespace
900
+ report = _sync_routes._odoo_sync_runner(
901
+ SimpleNamespace(runtime=rt, tenant=tenant,
902
+ uname=receipt.get("requestedBy") or "automation"),
903
+ requested_cid)
904
+ problems = list(report.get("errors") or ())
905
+ if report.get("state") in ("error", "partial") and not problems:
906
+ problems.append("The named Odoo connection did not finish cleanly.")
907
+ else:
908
+ source = rt.odoo_source()
909
+ if source is None:
910
+ raise _SyncNowRefused(
911
+ "This tenant has no Odoo credential. Connect Odoo before syncing.")
912
+ odoo_requested = True
913
+ previous_client = getattr(odoo._tlocal, "client", None)
914
+ previous_context = getattr(odoo._tlocal, "tenant_ctx", None)
915
+ try:
916
+ # Raw mirror pulls use the resolved tenant credential and a dedicated
917
+ # transport. Do not borrow a request's cached XML-RPC client.
918
+ odoo._tlocal.client = odoo.OdooClient(creds=source._creds)
919
+ odoo._tlocal.tenant_ctx = None
920
+ report = datastore.sync_all(log=lambda *a, **k: None)
921
+ finally:
922
+ odoo._tlocal.client = previous_client
923
+ odoo._tlocal.tenant_ctx = previous_context
924
+ if not report:
925
+ raise _SyncNowRefused(
926
+ "Odoo did not run a sync. Check whether the connection is paused.")
927
+ if any(row.get("phase") != "live" for row in report.values()):
928
+ raise _SyncNowRefused(
929
+ "Odoo backfill is incomplete. Run Sync now again to continue it.")
930
+ problems = []
931
  elif provider == "amazon_sp":
932
  from harness import amazon_store as amazon
933
  if not amazon.credential(tenant, rt=rt):
 
963
  if problems:
964
  detail += " " + _sync_now_failure(problems[0])
965
  elif problems:
966
+ detail = _sync_now_detail(provider, tenant, ok=False, cause=problems[0])
967
  else:
968
  ok, detail = True, "Sync finished."
969
  except _SyncNowRefused as exc:
970
  detail = str(exc)
971
  except Exception as exc: # noqa: BLE001
972
+ detail = _sync_now_detail(provider, tenant, ok=False, cause=exc)
973
  if first_load_progress is not None:
974
  connector_sync.finish_sync_request(provider, tenant, ok=ok, detail=detail,
975
  first_load=first_load_progress)
 
1169
  print(f"[aios-api] schema sweep ({why}) done over {len(tenants)} tenant(s)")
1170
 
1171
 
1172
+ def _rebuild_odoo_relational(why, previous_meta_fingerprint=None, meta_state_out=None, *,
1173
+ tenant=None, cadence_seconds=None):
1174
  """Spawn/refresh the four locked Odoo databases, then compute their cells. `why` is 'boot' or
1175
  'resync' and rides every log line, because "it failed" and "it failed at boot, before anyone
1176
  could have asked" are different diagnoses.
 
1180
  and was fine" from "the rebuild was never reached". Three failure markers and no success
1181
  marker means silence is ambiguous; now it is not.
1182
 
1183
+ The tenant is the owner of the process-global mirror resolved by `_resync_tenant`; this
1184
+ function never loops or rebinds datastore paths. That preserves D-29's isolation rule while
1185
+ removing the old hard-coded tenant/status path.
 
 
 
1186
  """
1187
  try:
1188
+ import time as _t
1189
  import odoo_relational as _rel
1190
  from harness import runtime as _runtime
1191
+ tenant_key = str(tenant or _resync_tenant()[0])
1192
+ _rt = _runtime.get_runtime(tenant_key)
 
1193
  from core import user_tables as _ut
1194
+ default_cid = routes_connectors._odoo_connection_slot(_rt, None)
1195
+ connection_ids = [default_cid]
1196
+ try:
1197
+ import core.keychain as _keychain
1198
+ connection_ids.extend(str(row.get("connection_id") or row.get("id"))
1199
+ for row in _keychain.list_entries(_rt)
1200
+ if row.get("type") == "odoo")
1201
+ except Exception: # noqa: BLE001
1202
+ pass
1203
+ connection_ids = list(dict.fromkeys(cid for cid in connection_ids if cid))
1204
+
1205
+ def _health(cid, stage, state, **kwargs):
1206
+ try:
1207
+ routes_connectors.record_odoo_stage(
1208
+ _rt, cid, stage, state, cadence_seconds=cadence_seconds, **kwargs)
1209
+ except Exception: # noqa: BLE001
1210
+ pass # The established lifecycle receipt below remains independently visible.
1211
+
1212
+ for cid in connection_ids:
1213
+ _ut.record_odoo_sync_status(phase="syncing", st=_rt, connection_id=cid)
1214
+ _health(cid, "materialization", "running")
1215
  # ⭐⭐ W54-T46 / R4 / R7 — EVERY Odoo CONNECTION, not just the first. `all_connections`
1216
  # delegates to `odoo_relational.refresh_all`, which walks this tenant's Odoo
1217
  # connections in slot order and records a failure against the connection it belongs
 
1226
  # because a gate that counts RAW occurrences of a call would count the one in this
1227
  # comment too. That gate happens to strip comments first; the next one may not.
1228
  # See `odoo_relational.refresh` and its `all_connections` arm.
1229
+ counts = _rel.refresh(_rt, tenant_key, all_connections=True)
1230
  raw_models, schema_errors = _rel.authoritative_schema_snapshot()
1231
  _ut.record_odoo_schema_snapshot(raw_models, complete=not schema_errors,
1232
  errors=schema_errors, st=_rt)
 
1253
  except Exception as e: # noqa: BLE001
1254
  rebuild_errors.append(f"relations: {type(e).__name__}: {e}")
1255
  print(f"[aios-api] odoo relation cells failed ({why}): {type(e).__name__}: {e}")
1256
+ # ⭐⭐ 2026-09-21 — ONE DOCUMENT READ FOR ALL EIGHT TABLES, NOT EIGHT. This loop used to
1257
+ # call `compute(_rt, _key)` with no `tables=`, and that argument is the whole cost of the
1258
+ # pass. Without it `rollup_sql.compute` does `rt.get(ut.STORE_KEY)` ITSELF, once per table
1259
+ # — and under `STORE_BACKEND=pg` the store is key -> whole-JSONB-document, so each of
1260
+ # those is the ENTIRE tenant workspace (~28 MB for tenant #0) coming back through Neon's
1261
+ # public proxy. Eight tables meant EIGHT full reads per rebuild before a single cell was
1262
+ # computed, and up to sixteen in the worst case, because each table that HAS source
1263
+ # rollups and finds a changed cell re-reads the document in its `persist` write.
1264
+ # ⚠ EIGHT IS THE MEASURED FLOOR AND SIXTEEN IS A CEILING, NOT AN OBSERVATION: `compute()`
1265
+ # takes its read BEFORE it checks `source_fields(tdef)`, so every table pays the read even
1266
+ # when it returns immediately — but only a table WITH source rollups can reach the write.
1267
+ # Two of the eight (`order_lines`, `gl_lines`) are read-through grains with no stored rows.
1268
+ #
1269
+ # ⛔ AND EGRESS IS THE HALF THAT IS BILLED. Neon's public network transfer is defined as
1270
+ # *"data sent from your databases through the Neon proxy to clients"* — outbound only.
1271
+ # Writes are free; these READS are the 823.47 GB that took September past a 500 GB
1272
+ # allowance. Skipping redundant WRITES would have saved nothing, which is why this is the
1273
+ # change that was made and that one was not.
1274
+ #
1275
+ # ⭐ `tables=`/`persist=` ARE NOT NEW AND THIS IS NOT A NEW PATTERN — it is the calling
1276
+ # convention `automation_engine._refresh_source_rollups` has always used
1277
+ # (`rollup_sql.compute(rt, table_key, tables=snapshot, persist=True)`). The `owned` branch
1278
+ # applies every table's plan against the snapshot in MEMORY and only opens a write when a
1279
+ # cell actually changed, so a quiet pass now costs ONE read instead of eight.
1280
+ # ⚠ THE SHARED SNAPSHOT IS DELIBERATE, not an accident of reuse: table N sees the cells
1281
+ # table N-1 just computed, which is one consistent view of the workspace rather than eight
1282
+ # reads of a document that other writers can move underneath the loop.
1283
+ # ⚠ `persist=True` IS LOAD-BEARING — the `owned` branch returns WITHOUT writing when it is
1284
+ # false, so dropping it would turn this into an expensive no-op that still reports counts.
1285
+ # ⛔ THE SCOPE STAYS `_rel.TABLES`, not `snapshot.items()` like the automation helper: this
1286
+ # is the Odoo rebuild and it must not start computing rollups for unrelated tables.
1287
  try:
1288
  import rollup_sql as _rollup
1289
+ _rollup_blob = _rt.get(_ut.STORE_KEY) or {}
1290
  for _b, _key, _l, _f in _rel.TABLES:
1291
+ _rollup.compute(_rt, _key, tables=_rollup_blob, persist=True)
1292
  except Exception as e: # noqa: BLE001
1293
  rebuild_errors.append(f"rollups: {type(e).__name__}: {e}")
1294
  print(f"[aios-api] odoo source rollups failed ({why}): {type(e).__name__}: {e}")
1295
  print(f"[aios-api] odoo relational rebuild done ({why}): "
1296
  + ", ".join(f"{k}={v}" for k, v in sorted((counts or {}).items())))
1297
+ reports = ((counts or {}).get("connections")
1298
+ if isinstance((counts or {}).get("connections"), dict) else {})
1299
+ if not reports:
1300
+ reports = {default_cid: counts or {}}
1301
+ any_connection_failed = False
1302
+ from datetime import datetime as _datetime, timezone as _timezone
1303
+ refreshed_at = _datetime.now(_timezone.utc).isoformat()
1304
+ for cid, report in reports.items():
1305
+ cid = str(cid or default_cid)
1306
+ report = report if isinstance(report, dict) else {}
1307
+ failed = report.get("failed")
1308
+ database_failures, unmatched_failures = (
1309
+ routes_connectors._odoo_result_database_failures(_rt, cid, report))
1310
+ failure = (routes_connectors._odoo_failure(failed, "materialization")
1311
+ if failed else None)
1312
+ if not failure and (database_failures or unmatched_failures):
1313
+ failure = routes_connectors._odoo_failure(
1314
+ "one or more databases failed", "materialization")
1315
+ tables = report.get("tables") if isinstance(report.get("tables"), dict) else {}
1316
+ database_receipts = report.get("databases")
1317
+ database_receipts = database_receipts if isinstance(database_receipts, dict) else {}
1318
+ canonical_receipts = report.get("canonicalDatabases")
1319
+ canonical_receipts = canonical_receipts if isinstance(canonical_receipts, dict) else {}
1320
+ all_receipts = {**database_receipts, **canonical_receipts}
1321
+ if failure:
1322
+ any_connection_failed = True
1323
+ _health(cid, "materialization", "failed", failure=failure)
1324
+ _health(cid, "freshness", "failed",
1325
+ failure={**failure, "stage": "freshness"})
1326
+ else:
1327
+ _health(cid, "source", "ready")
1328
+ _health(cid, "materialization", "ready")
1329
+ # A receipt is the authoritative per-database result. ``tables`` alone omits
1330
+ # read-through grains and the two compiled canonical databases, which previously
1331
+ # made their rich reconciliation evidence disappear at this scheduler boundary.
1332
+ database_keys = list(dict.fromkeys([*all_receipts, *tables]))
1333
+ for key in database_keys:
1334
+ table_counts = tables.get(key)
1335
+ if key in database_failures:
1336
+ continue
1337
+ receipt = all_receipts.get(key)
1338
+ receipt = receipt if isinstance(receipt, dict) else None
1339
+ nrows = (receipt.get("servingCount") if receipt is not None
1340
+ else table_counts.get("rows") if isinstance(table_counts, dict)
1341
+ else None)
1342
+ _health(cid, "materialization", "ready", database_key=key,
1343
+ row_count=nrows, receipt=receipt)
1344
+ if rebuild_errors:
1345
+ recon_failure = routes_connectors._odoo_failure(
1346
+ "relation or rollup pass failed", "reconciliation")
1347
+ _health(cid, "reconciliation", "failed", database_key=key,
1348
+ row_count=nrows, failure=recon_failure, receipt=receipt)
1349
+ _health(cid, "freshness", "failed", database_key=key,
1350
+ row_count=nrows, failure={**recon_failure, "stage": "freshness"},
1351
+ receipt=receipt)
1352
+ else:
1353
+ _health(cid, "reconciliation", "ready", database_key=key,
1354
+ row_count=nrows, receipt=receipt)
1355
+ _health(cid, "freshness", "ready", database_key=key,
1356
+ row_count=nrows, refreshed_at=refreshed_at, receipt=receipt)
1357
+ for key, db_failure in database_failures.items():
1358
+ receipt = all_receipts.get(key)
1359
+ receipt = receipt if isinstance(receipt, dict) else None
1360
+ failed_stage = str(db_failure.get("stage") or "materialization")
1361
+ failed_stage = (failed_stage if failed_stage in
1362
+ routes_connectors._ODOO_HEALTH_STAGES else "materialization")
1363
+ _health(cid, failed_stage, "failed", database_key=key,
1364
+ failure=db_failure, receipt=receipt)
1365
+ _health(cid, "freshness", "failed", database_key=key,
1366
+ failure={**db_failure, "stage": "freshness"}, receipt=receipt)
1367
+ if rebuild_errors:
1368
+ recon_failure = routes_connectors._odoo_failure(
1369
+ "relation or rollup pass failed", "reconciliation")
1370
+ _health(cid, "reconciliation", "failed", failure=recon_failure)
1371
+ _health(cid, "freshness", "failed",
1372
+ failure={**recon_failure, "stage": "freshness"})
1373
+ elif not failure:
1374
+ _health(cid, "reconciliation", "ready")
1375
+ _health(cid, "freshness", "ready")
1376
+ safe_errors = []
1377
+ if failure:
1378
+ safe_errors.append(f"{failure['code']}: {failure['message']}")
1379
+ if rebuild_errors:
1380
+ safe_errors.append("reconciliation_failed: Odoo reconciliation did not complete.")
1381
+ _ut.record_odoo_sync_status(
1382
+ phase="error" if safe_errors else "ok", errors=safe_errors,
1383
+ st=_rt, connection_id=cid)
1384
  meta_state = _rebuild_meta_relational(
1385
  why, _rt, previous_fingerprint=previous_meta_fingerprint, force_relations=True)
1386
  if isinstance(meta_state_out, dict):
 
1392
  # The isolated cells/rollups/meta handlers above are deliberately allowed to report and
1393
  # continue. This return value answers the narrower resync-policy question: did the Odoo
1394
  # table refresh itself complete, so a confirmed zero-change pass may be skipped?
1395
+ return not any_connection_failed and not rebuild_errors
1396
  except Exception as e: # noqa: BLE001
1397
  # ⚠ The TYPE is named here as it is in the two inner handlers. The original printed only
1398
  # `{e}`, and a bare message is exactly what made D-107 unreadable from the outside: a
 
1402
  try:
1403
  if '_rt' in locals():
1404
  from core import user_tables as _ut
1405
+ failure = routes_connectors._odoo_failure(e, "materialization")
1406
+ cid = locals().get("default_cid") or "default"
1407
+ routes_connectors.record_odoo_stage(
1408
+ _rt, cid, failure["stage"], "failed", failure=failure,
1409
+ cadence_seconds=cadence_seconds)
1410
+ _ut.record_odoo_sync_status(
1411
+ phase="error", errors=[f"{failure['code']}: {failure['message']}"],
1412
+ st=_rt, connection_id=cid)
1413
  except Exception:
1414
  pass
1415
  return False
 
1489
  "disagreement": None,
1490
  "passes": 0,
1491
  "lastPassAt": None,
1492
+ "lastSuccessAt": None,
1493
  "lastPassOk": None,
1494
  "lastError": None,
1495
+ "connectionId": "",
1496
  "consecutiveFailures": 0,
1497
  "bootSyncError": None,
1498
  }
 
1661
  ⚠ NOT the whole of D-314: naming the offending TABLE and ROW ID belongs to the DuckDB writer
1662
  that raised, which is not this loop.
1663
  """
1664
+ import ssl as _ssl
1665
  text = f"{type(exc).__name__}: {exc}"
1666
  low = text.lower()
1667
+ tls = (isinstance(exc, _ssl.SSLCertVerificationError)
1668
+ or "certificate_verify_failed" in low
1669
+ or "certificate verify failed" in low
1670
+ or "certificate has expired" in low
1671
+ or "self signed certificate" in low)
1672
  persistent = any(t in low for t in
1673
  ("duplicate key", "primary key", "unique constraint", "constraint error",
1674
  "binder", "conversion error"))
1675
+ code = ("tls_certificate_invalid" if tls else
1676
+ "mirror_integrity_failure" if persistent else "mirror_pull_failed")
1677
+ message = ("Odoo TLS certificate verification failed because the certificate is expired or "
1678
+ "untrusted. No source data was accepted." if tls else
1679
+ "The analytical mirror failed an integrity check and was not marked fresh."
1680
+ if persistent else
1681
+ "The Odoo mirror pull failed and freshness was not advanced.")
1682
+ return {"stage": "source" if tls else "mirrorPull", "code": code,
1683
+ "type": type(exc).__name__, "message": message,
1684
+ "kind": "persistent" if persistent or tls else "transient",
1685
  "meaning": ("this failure repeats until someone repairs the mirror; retrying the same "
1686
+ "pass will not clear it" if persistent or tls else
1687
  "this failure usually clears on the next pass")}
1688
 
1689
 
 
1751
  "tenantSeconds": _cad["tenantSeconds"], "envSeconds": _cad["envSeconds"],
1752
  "resolvedSeconds": _every, "manual": _manual,
1753
  "disagreement": _cad["disagreement"]})
1754
+ try:
1755
+ from harness import runtime as _health_runtime
1756
+ import odoo_relational as _health_rel
1757
+ RESYNC_STATE["connectionId"] = str(
1758
+ _health_rel.default_connection_id(
1759
+ _health_runtime.get_runtime(_cad["tenant"])) or "default")
1760
+ except Exception: # noqa: BLE001
1761
+ RESYNC_STATE["connectionId"] = "default"
1762
  # ⚠ ON CHANGE, NOT EVERY CYCLE. An hourly warning is a skimmed warning.
1763
  if _cad["disagreement"] != _RESYNC_NOTICE[0]:
1764
  _RESYNC_NOTICE[0] = _cad["disagreement"]
 
1782
  RESYNC_STATE["lastError"] = _err
1783
  if _err is None:
1784
  RESYNC_STATE["consecutiveFailures"] = 0
1785
+ if not _manual:
1786
+ RESYNC_STATE["lastSuccessAt"] = RESYNC_STATE["lastPassAt"]
1787
  else:
1788
  RESYNC_STATE["consecutiveFailures"] = int(RESYNC_STATE["consecutiveFailures"]) + 1
1789
  # D-314: the TYPE and the persistent/transient reading, both in the line and in the
 
1837
  _meta_state = {}
1838
  if _reason is not None:
1839
  print(f"[aios-api] odoo relational rebuild required ({_reason})")
1840
+ # Test/recovery harnesses replace the rebuild seam with its historical three-argument
1841
+ # callable. Bind tenant/cadence whenever the active implementation accepts them,
1842
+ # without turning an internal TypeError into a second materialization attempt.
1843
+ import inspect as _inspect
1844
+ _rebuild_params = _inspect.signature(_rebuild_odoo_relational).parameters
1845
+ _rebuild_variadic = any(
1846
+ p.kind == _inspect.Parameter.VAR_KEYWORD for p in _rebuild_params.values())
1847
+ _rebuild_scope = ({"tenant": _cad["tenant"], "cadence_seconds": _every}
1848
+ if _rebuild_variadic or "tenant" in _rebuild_params else {})
1849
+ if _rebuild_odoo_relational(
1850
+ "resync", _meta_previous, _meta_state, **_rebuild_scope):
1851
  last_relational_success_at = _t.time()
1852
  else:
1853
  # A prior successful projection no longer proves the changed mirror was applied.
 
2025
  # ⭐⭐ W50-T29 / R8 — AND THE DEPLOYMENT NOW OPTS IN, ON STAGING AND ON LIVE. The owner's ruling is
2026
  # that the Space resyncs its own mirror hourly rather than waiting for someone to visit; without
2027
  # it a Space serves whatever the seed was cut with, forever, and August reads zero while Odoo has
2028
+ # 723 orders. `deploy_web.py` pushes `AIOS_STORE_RESYNC=1` plus `AIOS_STORE_RESYNC_SECONDS=14400`.
2029
  #
2030
  # ⛔ THE OLD REASON FOR KEEPING IT OFF WAS "so a Pg-backed Live Space can scale its Neon compute to
2031
  # zero between real requests", AND THAT REASON IS ONLY HALF SATISFIED. `_resync_mirror_pass` is
 
2034
  # `_rebuild_meta_relational` write through `core.user_tables`, which is the tenant product store,
2035
  # and under `STORE_BACKEND=pg` that is Neon. Named here rather than left for the next reader to
2036
  # discover from a bill, and carried as a register row by this ticket's handoff.
2037
+ #
2038
+ # ⛔⛔ AND IT WAS LEFT FOR THE NEXT READER TO DISCOVER FROM A BILL — 2026-09-21, THIRTY-ONE DAYS
2039
+ # LATER. The register row never became work, and the paragraph above was the only thing in the
2040
+ # codebase that knew. September's Neon transfer reached 823.47 GB against a 500 GB allowance by
2041
+ # day 21 (compute 165 of 300 CU-hours, storage 96 MB of 10 GiB — both fine), which is an
2042
+ # over-$50 bill on a $19 plan, pacing to ~$87. The two Spaces' own run logs showed 61 of 61
2043
+ # hourly passes on live and two loops an hour on staging: 72 full rebuilds a day at ~545 MB each,
2044
+ # because `core.store_pg.update()` is a whole-JSONB-document read-modify-write and one pass makes
2045
+ # about ten of them against tenant #0's ~28.6 MB document.
2046
+ #
2047
+ # ⭐ THE HOTFIX MOVED THE CADENCE ONLY, 3600 -> 14400 (R11's existing four-hourly preset), which
2048
+ # is safe at any value because the wait below is `sync_wake_event().wait(_every)` and a Sync now
2049
+ # SETS it. TWO STRUCTURAL CAUSES REMAIN OPEN AS REGISTER ROWS, and neither is a hotfix:
2050
+ # * `_resync_rebuild_reason` treats `pulled > 0` as a change, but `pulled` counts rows FETCHED,
2051
+ # not rows CHANGED — live rebuilt on passes its own report scored `added/updated/removed: 0`.
2052
+ # Narrowing it is what would make an hourly cadence nearly free.
2053
+ # * the whole-document store write itself. A partial-JSONB write is a wave.
2054
+ # ⚠ SO DO NOT PUT THIS BACK TO 3600 to "fix staleness" without closing one of those two first.
2055
  if os.environ.get("AIOS_STORE_RESYNC") == "1":
2056
  import threading as _threading_resync
2057
  _threading_resync.Thread(target=_store_resync_loop, daemon=True, name="store-resync").start()
api/odoo_relational.py CHANGED
@@ -1546,13 +1546,15 @@ def read_customers(cur, excluded=None):
1546
  The two DOCUMENT legs are the original pair: Amazon books as direct invoices with no sale
1547
  order (the `odoo-api` gotcha), so a sale-order leg alone would silently drop a real customer.
1548
 
1549
- ⭐⭐ THE THIRD IS `customer_rank > 0 AND active` (wave 29, item 22 / R12 via finding F2 —
 
1550
  the owner's *"never an arbitrary limit… applies to ALL connected database"*). The old
1551
  docstring said partners with no document are *"left out on purpose — a row that can never
1552
  appear in any topic has nothing to roll up"*; that reasoning is RETIRED. It is the same
1553
  join-drop class as the Product grid's 2,717, and it dropped **~1,149 real customer records**
1554
  (MEASURED 2026-08-11: `rank>0 active` = 3,617 against a document union of ~2,000). A customer
1555
- a salesperson has not sold to yet is exactly the row a prospecting view needs.
 
1556
 
1557
  ⛔ IT IS A UNION AND NOT A REPLACEMENT, AND THAT IS MEASURED, NOT TIDINESS. Swapping the
1558
  document legs for the rank leg would drop **16 partners that hold posted documents** (7
@@ -1585,18 +1587,18 @@ def read_customers(cur, excluded=None):
1585
  agent = ("ag.name" if "agent_id" in have else "NULL")
1586
  agent_id_col = ("p.agent_id" if "agent_id" in have else "NULL")
1587
  join = ("LEFT JOIN res_partner ag ON ag.id = p.agent_id " if "agent_id" in have else "")
1588
- # ⚠ BOTH columns must be present, not just `customer_rank`: `active` is what keeps an
1589
- # archived prospect out, and a rank test without it would re-admit the 47 archived partners
1590
- # the mirror carries. Absent ⇒ the leg is dropped WHOLE, exactly like the agent join above.
1591
- rank_leg = (" OR (p.customer_rank > 0 AND p.active) "
1592
- if {"customer_rank", "active"} <= have else "")
1593
  # ⭐⭐ W37-T11 / D-165 — the POSTAL columns join city/state/country here. They ride `_col` like
1594
  # every other optional column, so a mirror predating the 2026-08-15 backfill degrades the CELL
1595
  # to blank rather than failing the spawn with a Binder error (see `columns()`'s header for the
1596
  # live 500 that rule was written after).
1597
  sql = (f"SELECT p.id, p.name, {_col(have, 'p.city')}, {_col(have, 'p.state_name')}, "
1598
  f" {_col(have, 'p.country_name')}, {agent}, {agent_id_col}, "
1599
- f" {_col(have, 'p.street')}, {_col(have, 'p.street2')}, {_col(have, 'p.zip')} "
 
1600
  "FROM res_partner p "
1601
  f"{join}"
1602
  "WHERE p.id IN ("
@@ -1607,7 +1609,7 @@ def read_customers(cur, excluded=None):
1607
  f"{rank_leg}")
1608
  out = []
1609
  for r in cur.execute(sql).fetchall():
1610
- (pid, name, city, state, country, agent, agent_id, street, street2, zipc) = r
1611
  out.append({
1612
  "_id": str(pid),
1613
  "customer": str(name or ""),
@@ -1620,6 +1622,7 @@ def read_customers(cur, excluded=None):
1620
  "country": str(country or ""),
1621
  "agent": str(agent or ""),
1622
  AGENT_JOIN_KEY: int(agent_id) if agent_id else "",
 
1623
  "wholesale_scope": _in_scope(pid, excluded),
1624
  })
1625
  return out
@@ -2771,6 +2774,292 @@ TABLE_SUBJECTS = {
2771
  GL_LINES_KEY: "odoo:account.move.line",
2772
  }
2773
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2774
  #: ⛔⛔ THE TWO RETIRED KEYS — W33-T44 / AMENDMENT A2. The rows a tenant ALREADY HAS.
2775
  #:
2776
  #: Dropping the `TABLES` rows above stops these being planned or re-created; it does NOT remove the
@@ -3999,6 +4288,119 @@ def available(rt=None, tenant_key=None, connection_id=None):
3999
  return False
4000
 
4001
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
4002
  def refresh(rt, tenant, username="automation", cur=None, today=None, connection_id=None,
4003
  all_connections=False, table_keys=None, on_apply=None):
4004
  """THE entry point — the store-resync path and the route both call this.
@@ -4105,18 +4507,36 @@ def refresh(rt, tenant, username="automation", cur=None, today=None, connection_
4105
  if table_key_for(key, ns) in requested}
4106
  skipped = sorted(table_key_for(key, ns)
4107
  for bucket, key, _l, _f in TABLES if bucket not in keep)
4108
- built = plan(cur, rt=rt, connection_id=ns, buckets=keep)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
4109
  # ⭐ W33-T44: a table this pass REMOVED is reported, not inferred from a grid going missing.
4110
  # Same rule as `skipped` below — R6's second sentence: what was deliberately not built (or no
4111
  # longer built) SAYS SO, with the keys.
4112
- plan_report = {}
4113
- if on_apply is not None:
4114
- try:
4115
- on_apply()
4116
- except Exception: # noqa: BLE001
4117
- pass # Optional status reporting cannot fail a valid plan's publication.
4118
- written = apply_plan(rt, built, username=username, today=today, report=plan_report,
4119
- connection_id=ns, retire=requested is None)
4120
  # The fixed mirror roster above and the discovered long tail deliberately use separate
4121
  # materialisers. Only an explicit long-tail enablement reaches this read-only path; disabled
4122
  # databases are not passed an empty set and therefore cannot be erased by reconciliation.
@@ -4131,7 +4551,9 @@ def refresh(rt, tenant, username="automation", cur=None, today=None, connection_
4131
  # FROZEN workspace returned before reaching here, so a disconnected connector is not
4132
  # re-recorded. See `_refresh_declared_schema` for why it writes the declared half only.
4133
  declared_report = _refresh_declared_schema(rt) if requested is None else None
4134
- out = {"tables": written,
 
 
4135
  **({"schemaDeclared": declared_report} if declared_report else {}),
4136
  **({"discovered": discovered_report}
4137
  if discovered_report and (discovered_report.get("tables")
@@ -4143,6 +4565,8 @@ def refresh(rt, tenant, username="automation", cur=None, today=None, connection_
4143
  # ⚠ THE DEFAULT CONNECTION'S RETURN SHAPE DOES NOT MOVE BY ONE KEY. The two additions below
4144
  # are reachable only on the namespaced path, because `verify_odoo_relational` asserts on this
4145
  # dict as it stands for the default.
 
 
4146
  if ns is not None:
4147
  out["connection"] = cid
4148
  if built.get("limits"):
@@ -4191,7 +4615,7 @@ def refresh_all(rt, tenant, username="automation", cur=None, today=None):
4191
  cur=(cur if named is None else None),
4192
  today=today, connection_id=named)
4193
  except Exception as exc: # noqa: BLE001
4194
- one = {"tables": {}, "failed": f"{type(exc).__name__}: {exc}"}
4195
  reports[str(entry_cid or default or DEFAULT_CONNECTION)] = one
4196
  if named is None and primary is None:
4197
  primary = one
 
1546
  The two DOCUMENT legs are the original pair: Amazon books as direct invoices with no sale
1547
  order (the `odoo-api` gotcha), so a sale-order leg alone would silently drop a real customer.
1548
 
1549
+ ⭐⭐ THE THIRD IS `customer_rank > 0` across both active states (wave 29, item 22 / R12
1550
+ via finding F2, widened by S63-T14 —
1551
  the owner's *"never an arbitrary limit… applies to ALL connected database"*). The old
1552
  docstring said partners with no document are *"left out on purpose — a row that can never
1553
  appear in any topic has nothing to roll up"*; that reasoning is RETIRED. It is the same
1554
  join-drop class as the Product grid's 2,717, and it dropped **~1,149 real customer records**
1555
  (MEASURED 2026-08-11: `rank>0 active` = 3,617 against a document union of ~2,000). A customer
1556
+ a salesperson has not sold to yet is exactly the row a prospecting view needs. Archived is a
1557
+ projected boolean; the default View hides it with an ordinary removable condition.
1558
 
1559
  ⛔ IT IS A UNION AND NOT A REPLACEMENT, AND THAT IS MEASURED, NOT TIDINESS. Swapping the
1560
  document legs for the rank leg would drop **16 partners that hold posted documents** (7
 
1587
  agent = ("ag.name" if "agent_id" in have else "NULL")
1588
  agent_id_col = ("p.agent_id" if "agent_id" in have else "NULL")
1589
  join = ("LEFT JOIN res_partner ag ON ag.id = p.agent_id " if "agent_id" in have else "")
1590
+ # The rank leg intentionally includes active and archived accounts. An absent rank column
1591
+ # still drops the leg whole; an absent active column keeps row arity with a conservative live
1592
+ # default while the health gate reports the schema gap separately.
1593
+ rank_leg = (" OR (p.customer_rank > 0) " if "customer_rank" in have else "")
 
1594
  # ⭐⭐ W37-T11 / D-165 — the POSTAL columns join city/state/country here. They ride `_col` like
1595
  # every other optional column, so a mirror predating the 2026-08-15 backfill degrades the CELL
1596
  # to blank rather than failing the spawn with a Binder error (see `columns()`'s header for the
1597
  # live 500 that rule was written after).
1598
  sql = (f"SELECT p.id, p.name, {_col(have, 'p.city')}, {_col(have, 'p.state_name')}, "
1599
  f" {_col(have, 'p.country_name')}, {agent}, {agent_id_col}, "
1600
+ f" {_col(have, 'p.street')}, {_col(have, 'p.street2')}, {_col(have, 'p.zip')}, "
1601
+ f" {_col(have, 'p.active', 'TRUE')} "
1602
  "FROM res_partner p "
1603
  f"{join}"
1604
  "WHERE p.id IN ("
 
1609
  f"{rank_leg}")
1610
  out = []
1611
  for r in cur.execute(sql).fetchall():
1612
+ (pid, name, city, state, country, agent, agent_id, street, street2, zipc, active) = r
1613
  out.append({
1614
  "_id": str(pid),
1615
  "customer": str(name or ""),
 
1622
  "country": str(country or ""),
1623
  "agent": str(agent or ""),
1624
  AGENT_JOIN_KEY: int(agent_id) if agent_id else "",
1625
+ "archived": not bool(active),
1626
  "wholesale_scope": _in_scope(pid, excluded),
1627
  })
1628
  return out
 
2774
  GL_LINES_KEY: "odoo:account.move.line",
2775
  }
2776
 
2777
+ # One receipt identity per curated database. The model is not enough: invoices/bills share
2778
+ # ``account.move`` and agents/vendors/customers share ``res.partner`` while presenting different
2779
+ # governed populations. These keys are public metadata only; no source row enters a receipt.
2780
+ TABLE_TOPICS = {
2781
+ INVOICES_KEY: "odoo_invoices",
2782
+ ORDERS_KEY: "odoo_orders",
2783
+ AGENTS_KEY: "odoo_agents",
2784
+ ACCOUNTS_KEY: "odoo_accounts",
2785
+ VENDORS_KEY: "odoo_vendors",
2786
+ BILLS_KEY: "odoo_bills",
2787
+ ORDER_LINES_KEY: "sales_lines",
2788
+ GL_LINES_KEY: "gl_lines",
2789
+ }
2790
+
2791
+ _SOURCE_COUNT_SPECS = {
2792
+ "invoices": {"model": "account.move", "mode": "count", "domain": [
2793
+ ("state", "=", "posted"), ("move_type", "in", ["out_invoice", "out_refund"]),
2794
+ ("partner_id", "!=", False)]},
2795
+ "orders": {"model": "sale.order", "mode": "count", "domain": [
2796
+ ("state", "in", ["sale", "done"]), ("partner_id", "!=", False)]},
2797
+ "agents": {"model": "res.partner", "mode": "agent_union", "domain": [
2798
+ ("agent", "=", True)]},
2799
+ "accounts": {"model": "account.account", "mode": "count", "domain": []},
2800
+ "vendors": {"model": "account.move", "mode": "distinct_partner", "domain": [
2801
+ ("state", "=", "posted"), ("move_type", "in", ["in_invoice", "in_refund"]),
2802
+ ("partner_id", "!=", False)]},
2803
+ "bills": {"model": "account.move", "mode": "count", "domain": [
2804
+ ("state", "=", "posted"), ("move_type", "in", ["in_invoice", "in_refund"]),
2805
+ ("partner_id", "!=", False)]},
2806
+ "order_lines": {"model": "sale.order.line", "mode": "count", "domain": [
2807
+ ("order_id.state", "in", ["sale", "done"])]},
2808
+ "gl_lines": {"model": "account.move.line", "mode": "count", "domain": []},
2809
+ }
2810
+
2811
+ # The two canonical registry databases replaced the retired ``ut_odoo_customers`` and
2812
+ # ``ut_odoo_products`` twins. They still need an identity receipt after every full default-
2813
+ # connection sync; otherwise the eight curated-table receipts can all be green while either of
2814
+ # the two databases people actually use is short. These are public schema/domain facts only.
2815
+ CANONICAL_RECEIPT_SPECS = {
2816
+ "customer_data": {
2817
+ "topicKey": "odoo_customers", "domainKey": "odoo:res.partner",
2818
+ "model": "res.partner", "selection": "customer eligibility union",
2819
+ },
2820
+ "product_data": {
2821
+ "topicKey": "odoo_products", "domainKey": "odoo:product.product",
2822
+ "model": "product.product", "selection": "active catalogue by business key",
2823
+ },
2824
+ }
2825
+
2826
+
2827
+ def _relation_id(value):
2828
+ """Return the id from an Odoo many2one/group value without retaining its label."""
2829
+ if isinstance(value, (list, tuple)) and value:
2830
+ value = value[0]
2831
+ try:
2832
+ return int(value) if value not in (None, False, "") else None
2833
+ except (TypeError, ValueError):
2834
+ return None
2835
+
2836
+
2837
+ def _source_domain_count(client, bucket):
2838
+ """Exact source population for one curated database domain, plus its counting method."""
2839
+ spec = _SOURCE_COUNT_SPECS[bucket]
2840
+ if spec["mode"] == "count":
2841
+ return int(client.search_count(spec["model"], spec["domain"])), "search_count"
2842
+ if spec["mode"] == "distinct_partner":
2843
+ groups = client.read_group(spec["model"], spec["domain"], ["partner_id"],
2844
+ ["partner_id"], lazy=False)
2845
+ ids = {_relation_id(row.get("partner_id")) for row in groups}
2846
+ ids.discard(None)
2847
+ return len(ids), "distinct_partner"
2848
+ if spec["mode"] == "agent_union":
2849
+ flagged = client.search_read("res.partner", spec["domain"], ["id"], limit=100000)
2850
+ commissioned = client.read_group(
2851
+ "account.invoice.line.agent", [("agent_id", "!=", False)], ["agent_id"],
2852
+ ["agent_id"], lazy=False)
2853
+ ids = {_relation_id(row.get("id")) for row in flagged}
2854
+ ids.update(_relation_id(row.get("agent_id")) for row in commissioned)
2855
+ ids.discard(None)
2856
+ return len(ids), "union_distinct_agent"
2857
+ raise Refused("the database source-count mode is not supported")
2858
+
2859
+
2860
+ def _group_identity_ids(client, model, domain, field):
2861
+ """Return a read-only grouped identity set without retaining relation labels."""
2862
+ groups = client.read_group(model, domain, [field], [field], lazy=False)
2863
+ ids = {_relation_id(row.get(field)) for row in groups}
2864
+ ids.discard(None)
2865
+ return ids
2866
+
2867
+
2868
+ def _canonical_source_population(client, database_key):
2869
+ """Return the exact source identities used by a canonical registry database.
2870
+
2871
+ Customer eligibility is deliberately a union: rank-positive partners in either active state,
2872
+ confirmed-order partners, and posted customer-document partners. Products are the active
2873
+ catalogue. Every operation is a verified read method; a capped read is refused rather than
2874
+ reported as a complete population.
2875
+ """
2876
+ if database_key == "customer_data":
2877
+ rank_domain = [("customer_rank", ">", 0), ("active", "in", [True, False])]
2878
+ ranked = client.search_read("res.partner", rank_domain, ["id"], limit=200000)
2879
+ ranked_count = int(client.search_count("res.partner", rank_domain))
2880
+ if len(ranked) != ranked_count:
2881
+ raise Refused("the customer identity read was incomplete")
2882
+ ids = {_relation_id(row.get("id")) for row in ranked}
2883
+ ids.update(_group_identity_ids(
2884
+ client, "sale.order",
2885
+ [("state", "in", ["sale", "done"]), ("partner_id", "!=", False)],
2886
+ "partner_id"))
2887
+ ids.update(_group_identity_ids(
2888
+ client, "account.move",
2889
+ [("state", "=", "posted"),
2890
+ ("move_type", "in", ["out_invoice", "out_refund"]),
2891
+ ("partner_id", "!=", False)], "partner_id"))
2892
+ ids.discard(None)
2893
+ return [{"id": value} for value in sorted(ids)]
2894
+ if database_key == "product_data":
2895
+ domain = [("active", "=", True)]
2896
+ rows = client.search_read(
2897
+ "product.product", domain, ["id", "default_code"], limit=100000)
2898
+ count = int(client.search_count("product.product", domain))
2899
+ if len(rows) != count:
2900
+ raise Refused("the product identity read was incomplete")
2901
+ return [{"id": _relation_id(row.get("id")),
2902
+ "code": str(row.get("default_code") or "")}
2903
+ for row in rows]
2904
+ raise Refused("the canonical database identity is not supported")
2905
+
2906
+
2907
+ def _identity_counts(values):
2908
+ """Return a normalized identity list, set, and duplicate count."""
2909
+ normalized = [value for value in values if value not in (None, False, "")]
2910
+ return normalized, set(normalized), len(normalized) - len(set(normalized))
2911
+
2912
+
2913
+ def canonical_reconciliation_receipts(cur, source_client=None, captured_at=None):
2914
+ """Data-free source -> mirror -> canonical receipts for Customers and Products.
2915
+
2916
+ The canonical grids are compiled registry databases, not ``ut_*`` tables. Customer rows keep
2917
+ the Odoo partner id. Product rows are SKU-grained, so multiple active product records sharing
2918
+ one business key are an explained merge rather than an unexplained missing row. Only counts
2919
+ and public schema/domain metadata leave this function; source ids and SKU values never do.
2920
+ """
2921
+ captured = str(captured_at or _dt.datetime.now(_dt.timezone.utc).isoformat())
2922
+ receipts = {}
2923
+ for database_key, spec in CANONICAL_RECEIPT_SPECS.items():
2924
+ identity = {
2925
+ "databaseKey": database_key, "schemaKey": database_key,
2926
+ "topicKey": spec["topicKey"], "domainKey": spec["domainKey"],
2927
+ "model": spec["model"],
2928
+ "selection": {"rule": spec["selection"], "enabled": True},
2929
+ }
2930
+ source_rows, source_failure = None, None
2931
+ if source_client is not None:
2932
+ try:
2933
+ source_rows = _canonical_source_population(source_client, database_key)
2934
+ except Exception as exc: # noqa: BLE001
2935
+ source_failure = {
2936
+ "code": "source_reconciliation_failed", "type": type(exc).__name__,
2937
+ "message": "This canonical Odoo population could not be reconciled.",
2938
+ }
2939
+ try:
2940
+ if database_key == "customer_data":
2941
+ mirror_rows = read_customers(cur)
2942
+ mirror_values = [row.get(JOIN_KEY) for row in mirror_rows]
2943
+ mirror_canonical_values = list(mirror_values)
2944
+ if source_client is not None:
2945
+ import modules.customer_data as customer_data
2946
+ # This is the same identity evaluator consumed by ``_pool_build``. Reading
2947
+ # the mirror again here would let a dropped canonical row certify itself.
2948
+ canonical_values = list(customer_data.identity_pids())
2949
+ else:
2950
+ canonical_values = None
2951
+ source_values = ([row.get("id") for row in source_rows]
2952
+ if source_rows is not None else None)
2953
+ source_canonical_values = source_values
2954
+ explained_merges = 0
2955
+ else:
2956
+ mirror_rows = [row for row in read_products(cur) if row.get("active") == "1"]
2957
+ mirror_values = [row.get(PRODUCT_JOIN_KEY) for row in mirror_rows]
2958
+ # product_data is one row per business key. An uncoded product has a stable,
2959
+ # per-record fallback, matching modules.products.catalogue().
2960
+ mirror_canonical_values = [
2961
+ (str(row.get("code") or "").strip()
2962
+ or f"pid:{row.get(PRODUCT_JOIN_KEY)}") for row in mirror_rows]
2963
+ if source_client is not None:
2964
+ import modules.product_data as product_data
2965
+ # ``identity_rows`` uses the catalogue seam and collision guard that the
2966
+ # shipped product pool consumes. It intentionally omits expensive cells.
2967
+ canonical_values = [row.get("code") for row in product_data.identity_rows()]
2968
+ else:
2969
+ canonical_values = None
2970
+ source_values = ([row.get("id") for row in source_rows]
2971
+ if source_rows is not None else None)
2972
+ source_canonical_values = ([
2973
+ (str(row.get("code") or "").strip() or f"pid:{row.get('id')}")
2974
+ for row in source_rows] if source_rows is not None else None)
2975
+ explained_merges = (len(mirror_canonical_values)
2976
+ - len(set(mirror_canonical_values)))
2977
+
2978
+ mirror_list, mirror_set, mirror_duplicates = _identity_counts(mirror_values)
2979
+ _mirror_canonical_list, mirror_canonical_set, mirror_key_duplicates = (
2980
+ _identity_counts(mirror_canonical_values))
2981
+ if canonical_values is None:
2982
+ canonical_list, canonical_set, canonical_duplicates = None, None, None
2983
+ else:
2984
+ canonical_list, canonical_set, canonical_duplicates = _identity_counts(
2985
+ canonical_values)
2986
+ if source_values is None:
2987
+ source_list, source_set, source_duplicates = None, None, None
2988
+ source_to_mirror = "unconfirmed"
2989
+ missing, extra = None, None
2990
+ canonical_missing, canonical_extra = None, None
2991
+ else:
2992
+ source_list, source_set, source_duplicates = _identity_counts(source_values)
2993
+ missing = len(source_set - mirror_set)
2994
+ extra = len(mirror_set - source_set)
2995
+ _source_canonical_list, source_canonical_set, _source_canonical_duplicates = (
2996
+ _identity_counts(source_canonical_values))
2997
+ canonical_missing = (len(source_canonical_set - canonical_set)
2998
+ if canonical_set is not None else None)
2999
+ canonical_extra = (len(canonical_set - source_canonical_set)
3000
+ if canonical_set is not None else None)
3001
+ source_to_mirror = (
3002
+ "matched" if not missing and not extra and not source_duplicates
3003
+ and not mirror_duplicates else "mismatch")
3004
+ if canonical_set is None:
3005
+ mirror_to_canonical = "unconfirmed"
3006
+ elif (mirror_canonical_set == canonical_set and not mirror_duplicates
3007
+ and not canonical_duplicates):
3008
+ mirror_to_canonical = (
3009
+ "matched_with_explained_merges"
3010
+ if database_key == "product_data" and explained_merges else "matched")
3011
+ else:
3012
+ mirror_to_canonical = "mismatch"
3013
+ # ID and canonical-key gaps are two views of the same records, so use the larger
3014
+ # layer gap rather than double-counting one absent row twice.
3015
+ unexplained = (max((missing or 0) + (extra or 0),
3016
+ (canonical_missing or 0) + (canonical_extra or 0))
3017
+ + (source_duplicates or 0)
3018
+ + mirror_duplicates
3019
+ + (canonical_duplicates or 0)
3020
+ + (0 if database_key == "product_data"
3021
+ else mirror_key_duplicates))
3022
+ partial = (source_to_mirror != "matched" or mirror_to_canonical not in {
3023
+ "matched", "matched_with_explained_merges"}
3024
+ or unexplained > 0 or source_failure is not None)
3025
+ receipts[database_key] = {
3026
+ **identity, "state": "partial" if partial else "succeeded",
3027
+ "sourceCount": len(source_list) if source_list is not None else None,
3028
+ "sourceCountMode": "identity_set" if source_list is not None else "unavailable",
3029
+ "sourceReconciliation": source_to_mirror,
3030
+ "mirrorCount": len(mirror_list), "materializedCount": None,
3031
+ "canonicalCount": (len(canonical_set) if canonical_set is not None else None),
3032
+ "servingCount": (len(canonical_set) if canonical_set is not None else None),
3033
+ "readThrough": True,
3034
+ "duplicateCount": canonical_duplicates,
3035
+ "explainedMergeCount": explained_merges,
3036
+ "unexplainedMissingCount": missing,
3037
+ "unexplainedExtraCount": extra,
3038
+ "unexplainedCanonicalMissingCount": canonical_missing,
3039
+ "unexplainedCanonicalExtraCount": canonical_extra,
3040
+ "unexplainedCount": unexplained,
3041
+ "reconciliation": {"sourceToMirror": source_to_mirror,
3042
+ "mirrorToCanonical": mirror_to_canonical},
3043
+ "freshness": {"state": "partial" if partial else "current",
3044
+ "capturedAt": captured},
3045
+ **({"sourceError": source_failure} if source_failure else {}),
3046
+ }
3047
+ except Exception as exc: # noqa: BLE001
3048
+ failure = {"code": "canonical_reconciliation_failed", "type": type(exc).__name__,
3049
+ "message": "This canonical Odoo database could not be reconciled."}
3050
+ receipts[database_key] = {
3051
+ **identity, "state": "failed", "sourceCount": None,
3052
+ "sourceCountMode": "unavailable", "sourceReconciliation": "unconfirmed",
3053
+ "mirrorCount": None, "materializedCount": None, "canonicalCount": None,
3054
+ "servingCount": None, "readThrough": True, "duplicateCount": None,
3055
+ "explainedMergeCount": None, "unexplainedMissingCount": None,
3056
+ "unexplainedExtraCount": None, "unexplainedCanonicalMissingCount": None,
3057
+ "unexplainedCanonicalExtraCount": None, "unexplainedCount": None,
3058
+ "freshness": {"state": "failed", "capturedAt": captured}, "error": failure,
3059
+ **({"sourceError": source_failure} if source_failure else {}),
3060
+ }
3061
+ return receipts
3062
+
3063
  #: ⛔⛔ THE TWO RETIRED KEYS — W33-T44 / AMENDMENT A2. The rows a tenant ALREADY HAS.
3064
  #:
3065
  #: Dropping the `TABLES` rows above stops these being planned or re-created; it does NOT remove the
 
4288
  return False
4289
 
4290
 
4291
+ def refresh_curated(rt, cur, buckets, *, connection_id=None, username="automation",
4292
+ today=None, retire=True, on_apply=None, source_client=None):
4293
+ """Refresh each enabled database independently; retain failed databases unchanged.
4294
+
4295
+ Counts describe the database's own domain, never its shared Odoo model. When a read-only
4296
+ source client is available, each receipt asks Odoo independently for that exact population;
4297
+ a failed source/count or mirror reader marks only that database and cannot stale siblings.
4298
+ """
4299
+ from copy import deepcopy
4300
+ built, written, errors, receipts, report = {}, {}, {}, {}, {}
4301
+ captured_at = _dt.datetime.now(_dt.timezone.utc).isoformat()
4302
+ for bucket, base_key, _label, _fields in TABLES:
4303
+ if bucket not in buckets:
4304
+ continue
4305
+ key = table_key_for(base_key, connection_id)
4306
+ source_count = None
4307
+ source_mode = "unavailable"
4308
+ source_failure = None
4309
+ if source_client is not None:
4310
+ try:
4311
+ source_count, source_mode = _source_domain_count(source_client, bucket)
4312
+ except Exception as exc: # noqa: BLE001
4313
+ source_failure = {
4314
+ "code": "source_reconciliation_failed", "type": type(exc).__name__,
4315
+ "message": "This Odoo database source population could not be counted.",
4316
+ }
4317
+ identity = {
4318
+ "databaseKey": key,
4319
+ "schemaKey": key,
4320
+ "topicKey": TABLE_TOPICS[base_key],
4321
+ "domainKey": TABLE_SUBJECTS[base_key],
4322
+ "model": ODOO_TABLE_MODELS.get(base_key),
4323
+ "countModel": _SOURCE_COUNT_SPECS[bucket]["model"],
4324
+ "domain": deepcopy(_SOURCE_COUNT_SPECS[bucket]["domain"]),
4325
+ "selection": {"bucket": bucket, "enabled": True},
4326
+ }
4327
+ try:
4328
+ one = plan(cur, rt=rt, connection_id=connection_id, buckets={bucket})
4329
+ rows = one[bucket]
4330
+ ids = [row.get("_id") for row in rows]
4331
+ if len(ids) != len(set(ids)) or any(value is None for value in ids):
4332
+ raise Refused("the database reader returned duplicate or missing record identities")
4333
+ if one.get("problems"):
4334
+ raise Refused("; ".join(one["problems"]))
4335
+ read_through = is_read_through_key(key) or _ut().row_limit(key, st=rt) == 0
4336
+ count = len(rows)
4337
+ if read_through and cur is not None:
4338
+ from routes_odoo_tables import population
4339
+ count = population(key, cur=cur, rt=rt)
4340
+ if count is None:
4341
+ raise Refused("the database has no readable mirror binding")
4342
+ if on_apply is not None:
4343
+ on_apply()
4344
+ # Each callback receives a detached document so even a store double or an
4345
+ # in-memory backend cannot leak a partially failed database update.
4346
+ class DetachedRuntime:
4347
+ def __getattr__(self, name):
4348
+ return getattr(rt, name)
4349
+
4350
+ def update(self, name, fn, **kwargs):
4351
+ return rt.update(name, lambda value: fn(deepcopy(value)), **kwargs)
4352
+
4353
+ result = apply_plan(DetachedRuntime(), one, username=username, today=today,
4354
+ connection_id=connection_id, retire=False)
4355
+ written.update(result)
4356
+ built[bucket] = rows
4357
+ if one.get("limits"):
4358
+ built.setdefault("limits", {}).update(one["limits"])
4359
+ materialized_count = 0 if read_through else result[key]["rows"]
4360
+ source_reconciliation = (
4361
+ "unconfirmed" if source_count is None else
4362
+ "matched" if source_count == count else "mismatch")
4363
+ mirror_reconciliation = (
4364
+ "served_read_through" if read_through else
4365
+ "matched" if count == materialized_count else "mismatch")
4366
+ partial = bool(one.get("limits") or source_failure
4367
+ or source_reconciliation == "mismatch"
4368
+ or mirror_reconciliation == "mismatch")
4369
+ receipts[key] = {
4370
+ **identity, "state": "partial" if partial else "succeeded",
4371
+ "sourceCount": source_count, "sourceCountMode": source_mode,
4372
+ "sourceReconciliation": source_reconciliation,
4373
+ "mirrorCount": count, "materializedCount": materialized_count,
4374
+ "servingCount": count, "readThrough": read_through, "duplicateCount": 0,
4375
+ "reconciliation": {"sourceToMirror": source_reconciliation,
4376
+ "mirrorToMaterialized": mirror_reconciliation},
4377
+ "freshness": {"state": "partial" if partial else "current",
4378
+ "capturedAt": captured_at},
4379
+ **({"sourceError": source_failure} if source_failure else {}),
4380
+ }
4381
+ if source_failure:
4382
+ errors[key] = source_failure
4383
+ except Exception as exc:
4384
+ # Exception prose may contain connection or record values. Keep only the
4385
+ # structural class and a fixed product message in returned/persisted evidence.
4386
+ failure = {"code": "materialization_failed", "type": type(exc).__name__,
4387
+ "message": "This Odoo database could not be materialized."}
4388
+ errors[key] = failure
4389
+ receipts[key] = {
4390
+ **identity, "state": "failed", "sourceCount": source_count,
4391
+ "sourceCountMode": source_mode, "sourceReconciliation": "unconfirmed",
4392
+ "mirrorCount": None, "materializedCount": None, "servingCount": None,
4393
+ "readThrough": is_read_through_key(key), "duplicateCount": None,
4394
+ "freshness": {"state": "failed", "capturedAt": captured_at},
4395
+ "error": failure,
4396
+ **({"sourceError": source_failure} if source_failure else {}),
4397
+ }
4398
+ if retire:
4399
+ apply_plan(rt, {}, username=username, today=today, report=report,
4400
+ connection_id=connection_id, retire=True)
4401
+ return built, written, errors, receipts, report
4402
+
4403
+
4404
  def refresh(rt, tenant, username="automation", cur=None, today=None, connection_id=None,
4405
  all_connections=False, table_keys=None, on_apply=None):
4406
  """THE entry point — the store-resync path and the route both call this.
 
4507
  if table_key_for(key, ns) in requested}
4508
  skipped = sorted(table_key_for(key, ns)
4509
  for bucket, key, _l, _f in TABLES if bucket not in keep)
4510
+ source = None
4511
+ try:
4512
+ source = (rt.odoo_source() if connection_id is None else rt.odoo_source(connection_id))
4513
+ except Exception: # noqa: BLE001
4514
+ source = None
4515
+ canonical_receipts = {}
4516
+ if source is None:
4517
+ built, written, database_errors, database_receipts, plan_report = refresh_curated(
4518
+ rt, cur, keep, connection_id=ns, username=username, today=today,
4519
+ retire=requested is None, on_apply=on_apply)
4520
+ if ns is None and requested is None and cur is not None:
4521
+ canonical_receipts = canonical_reconciliation_receipts(cur)
4522
+ else:
4523
+ # The binding selects this connection's verified read-only client. Authentication failure
4524
+ # leaves source counts unconfirmed but does not discard a usable mirror materialization.
4525
+ with source._bound(): # tenant credential + scope
4526
+ try:
4527
+ from core.odoo import get_odoo
4528
+ source_client = get_odoo()
4529
+ except Exception: # noqa: BLE001
4530
+ source_client = None
4531
+ built, written, database_errors, database_receipts, plan_report = refresh_curated(
4532
+ rt, cur, keep, connection_id=ns, username=username, today=today,
4533
+ retire=requested is None, on_apply=on_apply, source_client=source_client)
4534
+ if ns is None and requested is None and cur is not None:
4535
+ canonical_receipts = canonical_reconciliation_receipts(
4536
+ cur, source_client=source_client)
4537
  # ⭐ W33-T44: a table this pass REMOVED is reported, not inferred from a grid going missing.
4538
  # Same rule as `skipped` below — R6's second sentence: what was deliberately not built (or no
4539
  # longer built) SAYS SO, with the keys.
 
 
 
 
 
 
 
 
4540
  # The fixed mirror roster above and the discovered long tail deliberately use separate
4541
  # materialisers. Only an explicit long-tail enablement reaches this read-only path; disabled
4542
  # databases are not passed an empty set and therefore cannot be erased by reconciliation.
 
4551
  # FROZEN workspace returned before reaching here, so a disconnected connector is not
4552
  # re-recorded. See `_refresh_declared_schema` for why it writes the declared half only.
4553
  declared_report = _refresh_declared_schema(rt) if requested is None else None
4554
+ out = {"tables": written, "databases": database_receipts,
4555
+ **({"canonicalDatabases": canonical_receipts} if canonical_receipts else {}),
4556
+ **({"databaseErrors": database_errors} if database_errors else {}),
4557
  **({"schemaDeclared": declared_report} if declared_report else {}),
4558
  **({"discovered": discovered_report}
4559
  if discovered_report and (discovered_report.get("tables")
 
4565
  # ⚠ THE DEFAULT CONNECTION'S RETURN SHAPE DOES NOT MOVE BY ONE KEY. The two additions below
4566
  # are reachable only on the namespaced path, because `verify_odoo_relational` asserts on this
4567
  # dict as it stands for the default.
4568
+ if built.get("limits"):
4569
+ out["limits"] = built["limits"]
4570
  if ns is not None:
4571
  out["connection"] = cid
4572
  if built.get("limits"):
 
4615
  cur=(cur if named is None else None),
4616
  today=today, connection_id=named)
4617
  except Exception as exc: # noqa: BLE001
4618
+ one = {"tables": {}, "failed": f"{type(exc).__name__}: Odoo connection refresh failed"}
4619
  reports[str(entry_cid or default or DEFAULT_CONNECTION)] = one
4620
  if named is None and primary is None:
4621
  primary = one
api/providers.py CHANGED
@@ -37,6 +37,7 @@ import json
37
  import os
38
  import time
39
  from dataclasses import dataclass, field
 
40
  from typing import Callable
41
 
42
  # ⚠ TOP-LEVEL AND SAFE, unlike the `anthropic` import in `anthropic_sdk()`: `requests` is pinned in
@@ -58,6 +59,32 @@ import requests
58
  #: reads a live price — a modelled figure that says so is honest; one that pretends is not.
59
  _COST_BRIGHTDATA = float(os.environ.get("AIOS_COST_BRIGHTDATA") or 0.0015)
60
  _COST_APIFY = float(os.environ.get("AIOS_COST_APIFY") or 0.0027)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
61
 
62
 
63
  @dataclass(frozen=True)
@@ -135,6 +162,9 @@ PROVIDERS: dict[str, Provider] = {
135
  "apify": Provider(
136
  key="apify", label="Apify", key_env="AIOS_APIFY_KEY",
137
  caps={
 
 
 
138
  # ⭐ MEASURED 2026-08-08 against the public Reels grid: `videoPlayCount` 137,684 and
139
  # 299,493 vs a browser-read ground truth of 134K-137K and 299K. Exact.
140
  "ig_post_views": Capability(True, _COST_APIFY,
@@ -146,6 +176,7 @@ PROVIDERS: dict[str, Provider] = {
146
 
147
  #: The DEFAULT chain per capability, cheapest-capable-first. Overridable — see `chain()`.
148
  DEFAULT_CHAINS: dict[str, tuple] = {
 
149
  "ig_profile": ("brightdata", "apify"),
150
  "ig_post_metrics": ("brightdata", "apify"),
151
  # Only one entry, and that is the point: Bright Data is declared incapable, so listing it here
@@ -239,6 +270,11 @@ def run(capability, work, satisfied=None, log=None):
239
  started = time.time()
240
  try:
241
  result, note = work(provider)
 
 
 
 
 
242
  except Exception as e: # noqa: BLE001
243
  # ⚠ TYPE NAME ONLY. A vendor client's str() can carry the URL, and the credential is
244
  # one refactor away from being a query param; this line must not be what leaks it.
@@ -264,6 +300,8 @@ def _count(result):
264
  if isinstance(result, (list, tuple, set)):
265
  return len(result)
266
  if isinstance(result, dict):
 
 
267
  return len(result)
268
  return 1 if result else 0
269
 
@@ -888,6 +926,312 @@ def normalize_profile_apify(row):
888
  return {k: v for k, v in out.items() if v is not None and v != ""}
889
 
890
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
891
  def canonical_gaps(row, want=("views",)):
892
  """Which requested canonical fields this row does NOT carry — the `satisfied` input.
893
 
 
37
  import os
38
  import time
39
  from dataclasses import dataclass, field
40
+ from datetime import datetime, timezone
41
  from typing import Callable
42
 
43
  # ⚠ TOP-LEVEL AND SAFE, unlike the `anthropic` import in `anthropic_sdk()`: `requests` is pinned in
 
59
  #: reads a live price — a modelled figure that says so is honest; one that pretends is not.
60
  _COST_BRIGHTDATA = float(os.environ.get("AIOS_COST_BRIGHTDATA") or 0.0015)
61
  _COST_APIFY = float(os.environ.get("AIOS_COST_APIFY") or 0.0027)
62
+ #: Google Maps store discovery is a different Apify actor and therefore has its own modelled
63
+ #: list rate. This is a planning/receipt rate, not an invoice; deployments may replace it without
64
+ #: a release when the actor's published price changes. Keeping it separate from Instagram avoids
65
+ #: making a Maps estimate silently inherit another actor's commercial terms.
66
+ _COST_APIFY_MAPS = float(os.environ.get("AIOS_COST_APIFY_MAPS") or 0.004)
67
+
68
+ #: One actor run produces one dataset. Pagination reads that dataset; it never launches a second
69
+ #: paid run. The result cap remains the hard provider-record and modelled-spend ceiling.
70
+ APIFY_BASE = "https://api.apify.com/v2"
71
+ APIFY_MAPS_ACTOR = os.environ.get("AIOS_APIFY_MAPS_ACTOR") or "compass~crawler-google-places"
72
+ APIFY_MAPS_WAIT = float(os.environ.get("AIOS_APIFY_MAPS_WAIT") or 240)
73
+ APIFY_MAPS_PAGE_SIZE = 50
74
+ APIFY_MAPS_MAX_RECORDS = 500
75
+
76
+
77
+ class ProviderRefusal(RuntimeError):
78
+ """A safe, named provider refusal whose message may cross the API boundary."""
79
+
80
+ def __init__(self, code: str, public_message: str):
81
+ super().__init__(public_message)
82
+ self.code = str(code)
83
+ self.public_message = str(public_message)
84
+
85
+
86
+ class MapsProviderRefusal(ProviderRefusal):
87
+ """Named Google Maps refusal; never contains a credential, URL, or vendor response body."""
88
 
89
 
90
  @dataclass(frozen=True)
 
162
  "apify": Provider(
163
  key="apify", label="Apify", key_env="AIOS_APIFY_KEY",
164
  caps={
165
+ "google_maps_stores": Capability(
166
+ True, _COST_APIFY_MAPS,
167
+ "Google Maps places by category/search and geography; modelled per place"),
168
  # ⭐ MEASURED 2026-08-08 against the public Reels grid: `videoPlayCount` 137,684 and
169
  # 299,493 vs a browser-read ground truth of 134K-137K and 299K. Exact.
170
  "ig_post_views": Capability(True, _COST_APIFY,
 
176
 
177
  #: The DEFAULT chain per capability, cheapest-capable-first. Overridable — see `chain()`.
178
  DEFAULT_CHAINS: dict[str, tuple] = {
179
+ "google_maps_stores": ("apify",),
180
  "ig_profile": ("brightdata", "apify"),
181
  "ig_post_metrics": ("brightdata", "apify"),
182
  # Only one entry, and that is the point: Bright Data is declared incapable, so listing it here
 
270
  started = time.time()
271
  try:
272
  result, note = work(provider)
273
+ except ProviderRefusal:
274
+ # A named refusal is part of the product contract, not a fallback signal. In
275
+ # particular, retrying after Maps schema drift can buy the same unusable dataset
276
+ # twice and replace the actionable error code with a generic provider failure.
277
+ raise
278
  except Exception as e: # noqa: BLE001
279
  # ⚠ TYPE NAME ONLY. A vendor client's str() can carry the URL, and the credential is
280
  # one refactor away from being a query param; this line must not be what leaks it.
 
300
  if isinstance(result, (list, tuple, set)):
301
  return len(result)
302
  if isinstance(result, dict):
303
+ if isinstance(result.get("rows"), list):
304
+ return len(result["rows"])
305
  return len(result)
306
  return 1 if result else 0
307
 
 
926
  return {k: v for k, v in out.items() if v is not None and v != ""}
927
 
928
 
929
+ def _maps_json(raw, *, subject: str):
930
+ """Decode one Apify Maps envelope without echoing its body into a refusal."""
931
+ try:
932
+ return json.loads(bytes(raw or b"").decode("utf-8", "replace"))
933
+ except Exception as exc:
934
+ raise MapsProviderRefusal(
935
+ "maps_schema_drift",
936
+ f"Apify's Google Maps {subject} was not readable JSON. No result was written.") from exc
937
+
938
+
939
+ def _apify_maps_page_source(provider, search, geography, cap):
940
+ """Start one bounded actor run and return a reader for its one resulting dataset.
941
+
942
+ The actor is launched once. The returned closure pages the resulting dataset by offset, so
943
+ walking a 500-place result never re-buys the search. Credentials remain in request URLs and
944
+ are consequently never copied into an exception or receipt.
945
+ """
946
+ from automation_engine import Refused, fetch, fetch_json # lazy: only explicit Run now pays
947
+
948
+ key = (os.environ.get(provider.key_env) or "").strip()
949
+ if not key:
950
+ raise MapsProviderRefusal(
951
+ "maps_credential_missing",
952
+ "Add the Apify credential in Keychain before choosing Run now.")
953
+ token = requests.utils.quote(key, safe="")
954
+ actor = requests.utils.quote(APIFY_MAPS_ACTOR, safe="~")
955
+ start_url = (f"{APIFY_BASE}/acts/{actor}/runs?token={token}"
956
+ f"&waitForFinish={max(1, min(60, int(APIFY_MAPS_WAIT)))}")
957
+ body = {
958
+ "searchStringsArray": [str(search)],
959
+ "locationQuery": str(geography),
960
+ "maxCrawledPlacesPerSearch": int(cap),
961
+ "maxCrawledPlaces": int(cap),
962
+ # Details required by the stable schema, without unrelated review/image payloads.
963
+ "maxReviews": 0,
964
+ "maxImages": 0,
965
+ "scrapePlaceDetailPage": True,
966
+ }
967
+ try:
968
+ status, raw = fetch_json(start_url, body, timeout=min(APIFY_MAPS_WAIT, 60) + 30,
969
+ max_kb=1024)
970
+ except Refused as exc:
971
+ raise MapsProviderRefusal(
972
+ "maps_vendor_error",
973
+ "The Apify Google Maps address was refused by the provider safety rail. "
974
+ "No result was written.") from exc
975
+ except Exception as exc: # noqa: BLE001
976
+ raise MapsProviderRefusal(
977
+ "maps_vendor_error",
978
+ f"Apify Google Maps did not answer ({type(exc).__name__}). No result was written.") from exc
979
+ if not 200 <= int(status or 0) < 300:
980
+ raise MapsProviderRefusal(
981
+ "maps_vendor_error",
982
+ "Apify Google Maps refused the bounded run. No result was written.")
983
+ envelope = _maps_json(raw, subject="run envelope")
984
+ data = envelope.get("data") if isinstance(envelope, dict) else None
985
+ if not isinstance(data, dict):
986
+ raise MapsProviderRefusal(
987
+ "maps_schema_drift",
988
+ "Apify's Google Maps run envelope changed. No result was written.")
989
+
990
+ deadline = time.monotonic() + max(1.0, float(APIFY_MAPS_WAIT))
991
+ terminal = {"SUCCEEDED", "FAILED", "ABORTED", "TIMED-OUT"}
992
+ run_id = str(data.get("id") or "").strip()
993
+ run_status = str(data.get("status") or "").strip().upper()
994
+ while run_status not in terminal:
995
+ remaining = deadline - time.monotonic()
996
+ if remaining <= 0 or not run_id:
997
+ raise MapsProviderRefusal(
998
+ "maps_vendor_error",
999
+ "Apify Google Maps did not finish within the bounded wait. No result was written.")
1000
+ wait_seconds = max(1, min(60, int(remaining)))
1001
+ run_url = (f"{APIFY_BASE}/actor-runs/{requests.utils.quote(run_id, safe='')}"
1002
+ f"?token={token}&waitForFinish={wait_seconds}")
1003
+ try:
1004
+ poll_status, _final, poll_raw = fetch(
1005
+ run_url, timeout=wait_seconds + 30, max_kb=1024)
1006
+ except Exception as exc: # noqa: BLE001
1007
+ raise MapsProviderRefusal(
1008
+ "maps_vendor_error",
1009
+ f"Apify Google Maps status did not answer ({type(exc).__name__}). "
1010
+ "No result was written.") from exc
1011
+ if not 200 <= int(poll_status or 0) < 300:
1012
+ raise MapsProviderRefusal(
1013
+ "maps_vendor_error",
1014
+ "Apify Google Maps status could not be read. No result was written.")
1015
+ polled = _maps_json(poll_raw, subject="status envelope")
1016
+ data = polled.get("data") if isinstance(polled, dict) else None
1017
+ if not isinstance(data, dict):
1018
+ raise MapsProviderRefusal(
1019
+ "maps_schema_drift",
1020
+ "Apify's Google Maps status envelope changed. No result was written.")
1021
+ run_status = str(data.get("status") or "").strip().upper()
1022
+
1023
+ if run_status != "SUCCEEDED":
1024
+ raise MapsProviderRefusal(
1025
+ "maps_vendor_error",
1026
+ f"Apify Google Maps ended in {run_status.lower() or 'an error'}. No result was written.")
1027
+ dataset_id = str(data.get("defaultDatasetId") or "").strip()
1028
+ if not dataset_id:
1029
+ raise MapsProviderRefusal(
1030
+ "maps_schema_drift",
1031
+ "Apify's Google Maps result did not name its dataset. No result was written.")
1032
+
1033
+ def _page(offset, limit):
1034
+ dataset = requests.utils.quote(dataset_id, safe="")
1035
+ page_url = (f"{APIFY_BASE}/datasets/{dataset}/items?token={token}&clean=true&format=json"
1036
+ f"&offset={int(offset)}&limit={int(limit)}")
1037
+ try:
1038
+ page_status, _final, page_raw = fetch(page_url, timeout=60, max_kb=16384)
1039
+ except Exception as exc: # noqa: BLE001
1040
+ raise MapsProviderRefusal(
1041
+ "maps_vendor_error",
1042
+ f"Apify Google Maps results did not answer ({type(exc).__name__}). "
1043
+ "No result was written.") from exc
1044
+ if not 200 <= int(page_status or 0) < 300:
1045
+ raise MapsProviderRefusal(
1046
+ "maps_vendor_error",
1047
+ "Apify Google Maps results could not be read. No result was written.")
1048
+ page = _maps_json(page_raw, subject="dataset page")
1049
+ if not isinstance(page, list):
1050
+ raise MapsProviderRefusal(
1051
+ "maps_schema_drift",
1052
+ "Apify's Google Maps dataset page changed shape. No result was written.")
1053
+ return page
1054
+
1055
+ return _page
1056
+
1057
+
1058
+ def _maps_coordinate(value, name):
1059
+ try:
1060
+ number = float(value)
1061
+ except (TypeError, ValueError) as exc:
1062
+ raise MapsProviderRefusal(
1063
+ "maps_schema_drift",
1064
+ f"Apify's Google Maps result has no usable {name}. No result was written.") from exc
1065
+ if name == "latitude" and not -90 <= number <= 90:
1066
+ raise MapsProviderRefusal(
1067
+ "maps_schema_drift",
1068
+ "Apify's Google Maps result has an invalid latitude. No result was written.")
1069
+ if name == "longitude" and not -180 <= number <= 180:
1070
+ raise MapsProviderRefusal(
1071
+ "maps_schema_drift",
1072
+ "Apify's Google Maps result has an invalid longitude. No result was written.")
1073
+ return number
1074
+
1075
+
1076
+ def normalize_google_maps_store(row, *, captured_at=""):
1077
+ """One Apify Google Maps place -> the stable store-discovery database schema.
1078
+
1079
+ Identity, name, physical address, category and coordinates are the structural minimum. Phone
1080
+ and website are optional facts about a real place, not schema failures. The capture timestamp
1081
+ falls back to the run timestamp so every observation says when its source was read.
1082
+ """
1083
+ if not isinstance(row, dict):
1084
+ raise MapsProviderRefusal(
1085
+ "maps_schema_drift",
1086
+ "Apify's Google Maps result is not a place object. No result was written.")
1087
+ location = row.get("location") if isinstance(row.get("location"), dict) else {}
1088
+ place_id = str(row.get("placeId") or row.get("place_id") or "").strip()
1089
+ name = str(row.get("title") or row.get("name") or "").strip()
1090
+ address = str(row.get("address") or row.get("completeAddress") or "").strip()
1091
+ category = str(row.get("categoryName") or row.get("category") or "").strip()
1092
+ if not category and isinstance(row.get("categories"), list):
1093
+ category = str(next((item for item in row["categories"] if item), "")).strip()
1094
+ missing = [label for label, value in (
1095
+ ("placeId", place_id), ("title", name), ("address", address),
1096
+ ("categoryName", category)) if not value]
1097
+ if missing:
1098
+ raise MapsProviderRefusal(
1099
+ "maps_schema_drift",
1100
+ "Apify's Google Maps result schema changed; required fields are missing: "
1101
+ f"{', '.join(missing)}. No result was written.")
1102
+ latitude = _maps_coordinate(location.get("lat", row.get("latitude")), "latitude")
1103
+ longitude = _maps_coordinate(location.get("lng", row.get("longitude")), "longitude")
1104
+ source_timestamp = str(
1105
+ row.get("scrapedAt") or row.get("scraped_at") or row.get("sourceTimestamp")
1106
+ or captured_at or datetime.now(timezone.utc).isoformat()).strip()
1107
+ out = {
1108
+ "place_id": place_id,
1109
+ "name": name,
1110
+ "category": category,
1111
+ "address": address,
1112
+ "city": str(row.get("city") or "").strip(),
1113
+ "region": str(row.get("state") or row.get("region") or "").strip(),
1114
+ "postal_code": str(row.get("postalCode") or row.get("postal_code") or "").strip(),
1115
+ "country": str(row.get("countryCode") or row.get("country") or "").strip(),
1116
+ "phone": str(row.get("phone") or row.get("phoneUnformatted") or "").strip(),
1117
+ "website": str(row.get("website") or row.get("webSite") or "").strip(),
1118
+ "latitude": latitude,
1119
+ "longitude": longitude,
1120
+ "rating": row.get("totalScore", row.get("rating")),
1121
+ "reviews": _int_or_none(row.get("reviewsCount", row.get("reviews"))),
1122
+ "source_timestamp": source_timestamp,
1123
+ "source_payload": json.dumps(row, default=str, separators=(",", ":"))[:32_000],
1124
+ }
1125
+ return {key: value for key, value in out.items() if value is not None and value != ""}
1126
+
1127
+
1128
+ def discover_google_maps_stores(search, geography, cap, *, page_source=None,
1129
+ page_size=APIFY_MAPS_PAGE_SIZE, captured_at=""):
1130
+ """Run one bounded Maps discovery through the provider chain and return rows + receipts.
1131
+
1132
+ `page_source(provider, search, geography, cap) -> page(offset, limit)` is the network seam used
1133
+ by the recorded gate fixture. Production uses one Apify actor run and pages its dataset.
1134
+ Provider rows, including duplicates, consume the explicit cap; the runner never buys extra
1135
+ rows merely to replace duplicate place IDs.
1136
+ """
1137
+ query = str(search or "").strip()
1138
+ area = str(geography or "").strip()
1139
+ if not query or not area:
1140
+ raise MapsProviderRefusal(
1141
+ "maps_invalid_search",
1142
+ "Google Maps discovery needs a category or search term and a geography.")
1143
+ try:
1144
+ limit = int(cap)
1145
+ except (TypeError, ValueError) as exc:
1146
+ raise MapsProviderRefusal(
1147
+ "maps_unbounded_request",
1148
+ "Google Maps discovery requires an explicit result cap between 1 and 500.") from exc
1149
+ if limit < 1 or limit > APIFY_MAPS_MAX_RECORDS:
1150
+ raise MapsProviderRefusal(
1151
+ "maps_unbounded_request",
1152
+ "Google Maps discovery requires an explicit result cap between 1 and 500.")
1153
+ try:
1154
+ chunk = max(1, min(int(page_size), APIFY_MAPS_PAGE_SIZE, limit))
1155
+ except (TypeError, ValueError):
1156
+ chunk = min(APIFY_MAPS_PAGE_SIZE, limit)
1157
+ if not chain("google_maps_stores"):
1158
+ raise MapsProviderRefusal(
1159
+ "maps_credential_missing",
1160
+ "Add the Apify credential in Keychain before choosing Run now.")
1161
+ captured = str(captured_at or datetime.now(timezone.utc).isoformat())
1162
+ factory = page_source or _apify_maps_page_source
1163
+
1164
+ def _work(provider):
1165
+ page = factory(provider, query, area, limit)
1166
+ if not callable(page):
1167
+ raise MapsProviderRefusal(
1168
+ "maps_schema_drift",
1169
+ "The Google Maps provider did not supply a page reader. No result was written.")
1170
+ offset = 0
1171
+ pages = 0
1172
+ raw_count = 0
1173
+ duplicates = 0
1174
+ seen = set()
1175
+ rows = []
1176
+ while raw_count < limit:
1177
+ ask = min(chunk, limit - raw_count)
1178
+ batch = page(offset, ask)
1179
+ pages += 1
1180
+ if not isinstance(batch, list):
1181
+ raise MapsProviderRefusal(
1182
+ "maps_schema_drift",
1183
+ "The Google Maps provider returned a non-list dataset page. "
1184
+ "No result was written.")
1185
+ if len(batch) > ask:
1186
+ raise MapsProviderRefusal(
1187
+ "maps_vendor_error",
1188
+ "The Google Maps provider exceeded the requested page bound. "
1189
+ "No result was written.")
1190
+ if not batch:
1191
+ break
1192
+ for vendor_row in batch:
1193
+ normal = normalize_google_maps_store(vendor_row, captured_at=captured)
1194
+ raw_count += 1
1195
+ place_id = normal["place_id"]
1196
+ if place_id in seen:
1197
+ duplicates += 1
1198
+ continue
1199
+ seen.add(place_id)
1200
+ rows.append(normal)
1201
+ offset += len(batch)
1202
+ if len(batch) < ask:
1203
+ break
1204
+ rate = float((provider.cap("google_maps_stores") or Capability()).cost_per_record or 0.0)
1205
+ cost_cap = round(rate * limit, 4)
1206
+ payload = {
1207
+ "state": "ok",
1208
+ "via": provider.key,
1209
+ "rows": rows,
1210
+ "counts": {"cap": limit, "pages": pages, "raw": raw_count,
1211
+ "normalized": raw_count, "duplicates": duplicates,
1212
+ "returned": len(rows)},
1213
+ "cost": {"currency": "USD", "basis": "modelled per provider record",
1214
+ "ratePerRecordUsd": rate, "estimatedUsd": cost_cap,
1215
+ "costCapUsd": cost_cap,
1216
+ "incurredUsd": round(rate * raw_count, 4)},
1217
+ }
1218
+ return payload, ""
1219
+
1220
+ result, attempts = run(
1221
+ "google_maps_stores", _work, satisfied=lambda answer: isinstance(answer, dict))
1222
+ if not isinstance(result, dict):
1223
+ raise MapsProviderRefusal(
1224
+ "maps_vendor_error",
1225
+ "Apify Google Maps could not complete the bounded run. No result was written.")
1226
+ result["attempts"] = [
1227
+ {"provider": attempt.provider, "ok": attempt.ok,
1228
+ "records": (len(result.get("rows") or []) if attempt.ok else 0),
1229
+ "seconds": attempt.seconds}
1230
+ for attempt in attempts
1231
+ ]
1232
+ return result
1233
+
1234
+
1235
  def canonical_gaps(row, want=("views",)):
1236
  """Which requested canonical fields this row does NOT carry — the `satisfied` input.
1237
 
api/routes_activity.py ADDED
@@ -0,0 +1,392 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Value-free activity history, persisted only through the tenant store (S65-T16)."""
2
+ import json
3
+ import math
4
+ import re
5
+ import time
6
+ import uuid
7
+ from urllib.parse import urlsplit
8
+
9
+ from fastapi import APIRouter, Depends, Query, Request
10
+ from starlette.concurrency import run_in_threadpool
11
+
12
+ from deps import Session, err, require_session
13
+
14
+
15
+ RETENTION_DAYS = 30
16
+ MAX_SESSIONS = 200
17
+ MAX_EVENTS_PER_SESSION = 2000
18
+ MAX_MOVEMENT_SAMPLES = 500
19
+ MOVEMENT_WINDOW_MS = 30_000
20
+ MAX_BATCH_EVENTS = 100
21
+ MAX_PAYLOAD_BYTES = 512 * 1024
22
+ MAX_STRING_LENGTH = 256
23
+ MAX_CLASS_LENGTH = 128
24
+ MAX_DATA_ATTRIBUTES = 16
25
+ MAX_PAGE_EVENTS = 100
26
+ BUCKET = "activity_log"
27
+
28
+ # The entire client field vocabulary, including nested objects, is here. Each
29
+ # shape below selects only its own subset; no arbitrary recursive object survives.
30
+ ACTIVITY_ALLOWED_KEYS = frozenset({
31
+ "sessionId", "events", "dropped", "kind", "ts", "route", "from", "to", "ms",
32
+ "target", "method", "url", "status", "message", "source", "line", "name",
33
+ "movement", "movementDropped", "x", "y", "key", "tagName", "role", "type",
34
+ "id", "data", "className", "aria-label", "path",
35
+ })
36
+ SECRET_KEYS = frozenset({
37
+ "value", "defaultvalue", "textcontent", "innertext", "innerhtml", "placeholder",
38
+ "password", "token", "apikey", "authorization", "cookie", "secret", "credential",
39
+ "accesstoken", "refreshtoken", "clientsecret", "clipboard", "body", "query",
40
+ "cellvalue", "celldata", "text", "content", "payload",
41
+ })
42
+ _BEARER = re.compile(r"\bbearer\s+\S+", re.I)
43
+ _JWT = re.compile(r"\beyJ[A-Za-z0-9_-]*\.")
44
+ _CREDENTIAL = re.compile(r"\blp[bcor]1\.[A-Za-z0-9._~-]+")
45
+ _TOKEN = re.compile(r"^[a-zA-Z][a-zA-Z0-9_-]{0,79}$")
46
+ _CLASS = re.compile(r"^(?:set|shell|sheet|cg|pg|activity|lp|is)-[a-z][a-z0-9-]*$")
47
+ _SID = re.compile(r"^[a-f0-9-]{32,36}$")
48
+ _TAGS = set("a button input textarea select option div span svg path rect circle canvas td th tr table body html main section nav header footer label form ul ol li h1 h2 h3 h4 p img dialog details summary video audio".split())
49
+ _ROLES = set("button link textbox checkbox radio switch combobox listbox option grid gridcell cell row columnheader rowheader dialog tab tabpanel menu menuitem navigation region slider spinbutton searchbox".split())
50
+ _TYPES = set("button submit reset text password email number search tel url date time checkbox radio range file hidden color".split())
51
+ _ERROR_NAMES = {"Error", "TypeError", "ReferenceError", "SyntaxError", "RangeError", "URIError", "EvalError", "AggregateError", "NetworkError", "HttpError", "ResourceError"}
52
+ _MESSAGES = {"Application error.", "Unhandled rejection.", "Script error.", "A request failed.", "A network request failed.", "A resource failed to load."}
53
+ _LABELS = {"Open", "Close", "Save", "Cancel", "Delete", "Edit", "Search", "Settings", "Close settings", "Activity log", "Refresh", "Previous", "Next", "Add", "Remove", "Menu", "More", "Retry"}
54
+ # Attribute values may hold tenant data even when their names look harmless.
55
+ # Retain only structural enums; all other safe data-* names record presence.
56
+ _DATA_ENUMS = {
57
+ "data-section": {"account", "scope", "security", "shortcuts", "activity", "users", "database", "keychains", "connectors", "padmin"},
58
+ "data-state": {"open", "closed", "active", "inactive", "loading", "ready"},
59
+ "data-view-kind": {"sheet", "grid", "table", "chart", "timeline"},
60
+ }
61
+ # Routes describe the surface, not record ids, searches, or cell text embedded in
62
+ # a URL. Unknown segments become :item on BOTH sides of the wire.
63
+ _ROUTE_SEGMENTS = set("api v1 auth me login logout refresh nav workspace settings account scope security shortcuts activity sessions users database databases grid sheet sheets view views table tables records rows columns fields cells query sort filter data applications finance facts orders inventory customers products sales collections procurement agents agent automation automations connectors keychains external oauth connections admin operator subscriptions billing health status run runs events logs actions export import search schema preferences folders pages charts catalog reports reconciliation permissions detail assets".split())
64
+
65
+ # ⛔ NO `prefix=` HERE, AND THAT IS DELIBERATE. This router is included INSIDE
66
+ # `routes_applications.router` (main.py belongs to another session this wave), and a child
67
+ # router that carries its own `prefix=` has that prefix baked into its own `.routes` at
68
+ # DECLARATION time. `verify_api.py::_mounted_paths` then adds `nested.prefix` a SECOND time
69
+ # while walking the nested include, so `/api/v1/activity` is derived as
70
+ # `/api/v1/api/v1/activity` and D-48 reports a client call with no route -- a false RED over a
71
+ # route the app genuinely serves (`app.openapi()` proves it). Spelling the full path on each
72
+ # decorator makes the declared path the served path, which is the honest reading either way.
73
+ router = APIRouter(tags=["activity"])
74
+
75
+
76
+ class ActivityRefused(ValueError):
77
+ """Never put a refused value into an exception, traceback, or response."""
78
+
79
+ def __init__(self, code, message):
80
+ self.code = code
81
+ self.message = message
82
+ super().__init__(message)
83
+
84
+
85
+ def _text(value, maximum=MAX_STRING_LENGTH):
86
+ if not isinstance(value, str):
87
+ return ""
88
+ if (_BEARER.search(value) or _JWT.search(value) or _CREDENTIAL.search(value)
89
+ or value.startswith("Atzr|")):
90
+ return ""
91
+ return re.sub(r"[\x00-\x1f\x7f\u2013\u2014]", " ", value)[:maximum]
92
+
93
+
94
+ def _shape(value):
95
+ if not isinstance(value, dict):
96
+ return {}
97
+ return {key: val for key, val in value.items() if key in ACTIVITY_ALLOWED_KEYS}
98
+
99
+
100
+ def _number(value, default=0, maximum=10**15):
101
+ if type(value) not in (int, float) or (isinstance(value, float) and not math.isfinite(value)):
102
+ return default
103
+ return min(maximum, max(0, round(value)))
104
+
105
+
106
+ def _path(value):
107
+ value = _text(value)
108
+ if not value:
109
+ return ""
110
+ try:
111
+ # Hash routes are paths too, but hash query parameters never survive.
112
+ raw = value[1:] if value.startswith("#") else urlsplit(value).path
113
+ raw = raw.split("?", 1)[0].split("#", 1)[0]
114
+ segments = raw.strip("/").split("/")
115
+ clean = [part if part in _ROUTE_SEGMENTS or re.fullmatch(r"[a-zA-Z0-9_-]+\.(?:js|css|tsx|ts)", part) else ":item" for part in segments if part]
116
+ return ("/" + "/".join(clean))[:MAX_STRING_LENGTH]
117
+ except ValueError:
118
+ return ""
119
+
120
+
121
+ def _classes(value):
122
+ return " ".join(part for part in _text(value).split() if _CLASS.fullmatch(part))[:MAX_CLASS_LENGTH]
123
+
124
+
125
+ def _element(raw):
126
+ raw = _shape(raw)
127
+ tag = _text(raw.get("tagName")).lower()
128
+ role = _text(raw.get("role"))
129
+ result = {"tagName": tag if tag in _TAGS else "element"}
130
+ if role in _ROLES:
131
+ result["role"] = role
132
+ element_type = _text(raw.get("type"), 32)
133
+ if element_type in _TYPES:
134
+ result["type"] = element_type
135
+ # The collector marks descendants of a grid/editor with role=gridcell/textbox.
136
+ # Defend direct forged input too; never accept rich metadata on these shapes.
137
+ protected = (tag in {"input", "textarea", "option", "select", "canvas", "td", "th"}
138
+ or role in {"grid", "gridcell", "cell", "textbox", "searchbox", "option"}
139
+ or "sheet-cell" in str(raw.get("className", ""))
140
+ or "cg-cell" in str(raw.get("className", "")))
141
+ hops = raw.get("path", [])
142
+ if isinstance(hops, list):
143
+ protected = protected or any(
144
+ _text(hop).split(".")[0] in {"td", "th", "input", "textarea", "option", "canvas"}
145
+ or re.search(r"\.(?:sheet|cg)-cell(?:\.|$)", _text(hop))
146
+ for hop in hops[:5])
147
+ path = []
148
+ for hop in hops[:5]:
149
+ parts = _text(hop).split(".")
150
+ if parts[0] in _TAGS:
151
+ classes = _classes(" ".join(parts[1:])) if not protected else ""
152
+ path.append(parts[0] + ("." + classes.replace(" ", ".") if classes else ""))
153
+ result["path"] = path
154
+ if protected:
155
+ return result
156
+ classes = _classes(raw.get("className"))
157
+ if classes:
158
+ result["className"] = classes
159
+ for key in ("id", "name"):
160
+ val = _text(raw.get(key), 80)
161
+ if val == "root" or (_TOKEN.fullmatch(val) and _CLASS.fullmatch(val)):
162
+ result[key] = val
163
+ label = _text(raw.get("aria-label"))
164
+ if label in _LABELS:
165
+ result["aria-label"] = label
166
+ data = raw.get("data")
167
+ if isinstance(data, dict):
168
+ safe = {}
169
+ for key, val in list(data.items())[:MAX_DATA_ATTRIBUTES]:
170
+ if not isinstance(key, str) or not re.fullmatch(r"data-[a-z][a-z-]{0,47}", key):
171
+ continue
172
+ normalized = re.sub(r"[^a-z]", "", key[5:])
173
+ if any(word in normalized for word in SECRET_KEYS):
174
+ continue
175
+ val = _text(val)
176
+ if not val:
177
+ continue
178
+ safe[key] = val if val in _DATA_ENUMS.get(key, set()) else "present"
179
+ if safe:
180
+ result["data"] = safe
181
+ return result
182
+
183
+
184
+ def _key(value, target):
185
+ value = _text(value, 64)
186
+ parts = value.split("+")
187
+ key = parts[-1]
188
+ if any(part not in {"Control", "Alt", "Shift", "Meta"} for part in parts[:-1]):
189
+ return "Key"
190
+ named = {"Enter", "Escape", "Tab", "Backspace", "Delete", "ArrowUp", "ArrowDown", "ArrowLeft", "ArrowRight", "Home", "End", "PageUp", "PageDown", "Control", "Alt", "Shift", "Meta", "CapsLock", "Space", "Character"}
191
+ if key not in named and not re.fullmatch(r"(?:[a-zA-Z0-9]|F(?:[1-9]|1[0-2]))", key):
192
+ return "Key"
193
+ editable = target.get("tagName") in {"input", "textarea", "option", "select"} or target.get("role") in {"textbox", "searchbox", "gridcell", "cell", "grid"}
194
+ shortcut = any(part in {"Control", "Meta"} for part in parts[:-1]) and "Alt" not in parts[:-1]
195
+ if editable and (len(key) == 1 or key == "Space") and not shortcut:
196
+ return "Character"
197
+ return value
198
+
199
+
200
+ def project_event(raw, now_ms, *, movement=False):
201
+ raw = _shape(raw)
202
+ kind = raw.get("kind")
203
+ allowed = {"hover", "scroll", "focus", "keypress", "pointer"} if movement else {"route", "click", "api", "error"}
204
+ if not isinstance(kind, str) or kind not in allowed:
205
+ return None
206
+ ts = _number(raw.get("ts"), now_ms, now_ms)
207
+ if ts < now_ms - RETENTION_DAYS * 86400_000:
208
+ return None
209
+ event = {"kind": kind, "ts": ts}
210
+ if movement or kind == "click":
211
+ event["target"] = _element(raw.get("target"))
212
+ if movement:
213
+ if kind in {"pointer", "scroll"}:
214
+ event.update({key: _number(raw.get(key), maximum=10**7) for key in ("x", "y")})
215
+ if kind == "keypress":
216
+ event["key"] = _key(raw.get("key"), event["target"])
217
+ return event
218
+ event["route"] = _path(raw.get("route"))
219
+ if kind == "route":
220
+ event.update({"from": _path(raw.get("from")), "to": _path(raw.get("to")), "ms": _number(raw.get("ms"))})
221
+ elif kind == "api":
222
+ method = _text(raw.get("method"), 16).upper()
223
+ event.update({"method": method if method in {"GET", "POST", "PUT", "PATCH", "DELETE", "HEAD", "OPTIONS"} else "OTHER",
224
+ "url": _path(raw.get("url")), "status": _number(raw.get("status"), maximum=599), "ms": _number(raw.get("ms"))})
225
+ elif kind == "error":
226
+ name = _text(raw.get("name"), 64)
227
+ message = _text(raw.get("message"))
228
+ event.update({"name": name if name in _ERROR_NAMES else "Error",
229
+ # Raw exception text often interpolates the offending cell.
230
+ "message": message if message in _MESSAGES else "Application error.",
231
+ "source": _path(raw.get("source")), "line": _number(raw.get("line"), maximum=10**7)})
232
+ samples = raw.get("movement")
233
+ samples = samples if isinstance(samples, list) else []
234
+ trail = [project_event(sample, now_ms, movement=True) for sample in samples[-MAX_MOVEMENT_SAMPLES:]]
235
+ event["movement"] = sorted([sample for sample in trail if sample and ts - MOVEMENT_WINDOW_MS <= sample["ts"] <= ts], key=lambda sample: sample["ts"])
236
+ event["movementDropped"] = _number(raw.get("movementDropped")) + len(samples) - len(event["movement"])
237
+ return event
238
+
239
+
240
+ def _limits():
241
+ return {"retentionDays": RETENTION_DAYS, "maxSessions": MAX_SESSIONS,
242
+ "maxEventsPerSession": MAX_EVENTS_PER_SESSION, "maxMovementSamples": MAX_MOVEMENT_SAMPLES,
243
+ "movementWindowSeconds": MOVEMENT_WINDOW_MS // 1000, "pageSize": MAX_PAGE_EVENTS}
244
+
245
+
246
+ def _retained(doc, tenant, now_ms):
247
+ if not isinstance(doc, dict) or not isinstance(doc.get("sessions"), list):
248
+ return []
249
+ cutoff = now_ms - RETENTION_DAYS * 86400_000
250
+ rows = [row for row in doc["sessions"] if isinstance(row, dict) and row.get("tenant") == tenant
251
+ and isinstance(row.get("events"), list) and _number(row.get("endedAt")) >= cutoff]
252
+ retained = []
253
+ for row in rows:
254
+ events = [event for event in row["events"] if isinstance(event, dict) and _number(event.get("ts")) >= cutoff]
255
+ if events:
256
+ retained.append({**row, "events": events[-MAX_EVENTS_PER_SESSION:]})
257
+ return sorted(retained, key=lambda row: row["endedAt"])[-MAX_SESSIONS:]
258
+
259
+
260
+ def _record(session, session_id, events, dropped, now_ms):
261
+ from harness import runtime
262
+
263
+ def _append(doc):
264
+ rows = _retained(doc, session.tenant, now_ms)
265
+ row = next((item for item in rows if item["sessionId"] == session_id and item["user"] == session.uname), None)
266
+ if row is None:
267
+ row = {"sessionId": session_id, "user": session.uname, "tenant": session.tenant,
268
+ "startedAt": min(event["ts"] for event in events), "endedAt": now_ms,
269
+ "route": next((event.get("to") or event.get("route") for event in events if event.get("to") or event.get("route")), "/"),
270
+ "events": [], "dropped": 0, "prunedEvents": 0}
271
+ rows.append(row)
272
+ merged = sorted(row["events"] + events, key=lambda event: event["ts"])
273
+ if merged[0]["ts"] < row["startedAt"]:
274
+ row["route"] = merged[0].get("to") or merged[0].get("route") or "/"
275
+ row["startedAt"] = min(row["startedAt"], merged[0]["ts"])
276
+ row["endedAt"] = merged[-1]["ts"]
277
+ row["prunedEvents"] += max(0, len(merged) - MAX_EVENTS_PER_SESSION)
278
+ row["dropped"] += dropped
279
+ row["events"] = merged[-MAX_EVENTS_PER_SESSION:]
280
+ return {"sessions": sorted(rows, key=lambda item: item["endedAt"])[-MAX_SESSIONS:]}
281
+
282
+ # Async coalescing avoids blocking uploads and one HF commit per batch.
283
+ # A container loss before the worker flushes may lose cached telemetry.
284
+ runtime.get_runtime(str(session.tenant)).update(BUCKET, _append, flush="async")
285
+ # platform/ is available after session resolution, not necessarily at import.
286
+ try:
287
+ from harness.telemetry import error as telemetry_error
288
+ except ImportError:
289
+ return
290
+ for event in events:
291
+ if event["kind"] == "error":
292
+ try:
293
+ # This is the existing diagnostic sink, not a second activity store.
294
+ # Never forward exc, the incoming payload, or the movement snapshot.
295
+ telemetry_error("activity", name=event["name"], message=event["message"],
296
+ source=event["source"], line=event["line"])
297
+ except Exception:
298
+ pass
299
+
300
+
301
+ def _batch(body):
302
+ events = body.get("events")
303
+ session_id = _text(body.get("sessionId"), 80)
304
+ if not _SID.fullmatch(session_id) or not isinstance(events, list):
305
+ raise ActivityRefused("activity_invalid", "Activity requires a tab session and an event list.")
306
+ if len(events) > MAX_BATCH_EVENTS:
307
+ raise ActivityRefused("activity_batch_too_large", f"Activity batches must contain at most {MAX_BATCH_EVENTS} events.")
308
+ return session_id, events
309
+
310
+
311
+ @router.post("/api/v1/activity")
312
+ async def ingest_activity(request: Request, session: Session = Depends(require_session)):
313
+ data = bytearray()
314
+ async for chunk in request.stream():
315
+ if len(data) + len(chunk) > MAX_PAYLOAD_BYTES:
316
+ raise err(400, "activity_payload_too_large", f"Activity batches must be at most {MAX_PAYLOAD_BYTES} bytes.")
317
+ data.extend(chunk)
318
+ try:
319
+ body = _shape(json.loads(data))
320
+ except (ValueError, UnicodeError, RecursionError):
321
+ raise err(400, "activity_invalid", "Activity requires a valid JSON object.") from None
322
+ try:
323
+ session_id, events = _batch(body)
324
+ except ActivityRefused as exc:
325
+ raise err(400, exc.code, exc.message) from None
326
+ now_ms = round(time.time() * 1000)
327
+ clean = []
328
+ for raw in events:
329
+ event = project_event(raw, now_ms)
330
+ if event is not None:
331
+ clean.append({**event, "eventId": uuid.uuid4().hex})
332
+ dropped = len(events) - len(clean)
333
+ clean.sort(key=lambda event: event["ts"])
334
+ if clean:
335
+ try:
336
+ await run_in_threadpool(_record, session, session_id, clean,
337
+ _number(body.get("dropped")) + dropped, now_ms)
338
+ except Exception:
339
+ raise err(503, "activity_unavailable", "Activity could not be saved.") from None
340
+ return {"accepted": len(clean), "dropped": dropped}
341
+
342
+
343
+ def _user(session, requested):
344
+ if requested is not None and requested != session.uname and not session.admin:
345
+ raise err(403, "activity_forbidden", "You can only view your own activity.")
346
+ return session.uname if requested is None else requested
347
+
348
+
349
+ def _read(session):
350
+ from harness import runtime
351
+ try:
352
+ return _retained(runtime.get_runtime(str(session.tenant)).get(BUCKET), session.tenant, round(time.time() * 1000))
353
+ except Exception:
354
+ raise err(503, "activity_unavailable", "Activity could not be loaded.") from None
355
+
356
+
357
+ @router.get("/api/v1/activity/sessions")
358
+ def activity_sessions(user: str | None = None, session: Session = Depends(require_session)):
359
+ owner = _user(session, user)
360
+ rows = []
361
+ for row in reversed(_read(session)):
362
+ if row["user"] == owner:
363
+ rows.append({key: row[key] for key in ("sessionId", "startedAt", "endedAt", "route", "dropped", "prunedEvents")})
364
+ rows[-1].update(events=len(row["events"]), errors=sum(event["kind"] == "error" for event in row["events"]))
365
+ return {"sessions": rows, "limits": _limits()}
366
+
367
+
368
+ @router.get("/api/v1/activity/sessions/{session_id}")
369
+ def activity_session(session_id: str, user: str | None = None,
370
+ limit: int = Query(MAX_PAGE_EVENTS, ge=1, le=MAX_PAGE_EVENTS),
371
+ offset: int = Query(0, ge=0), session: Session = Depends(require_session)):
372
+ owner = _user(session, user)
373
+ row = next((item for item in _read(session) if item["user"] == owner and item["sessionId"] == session_id), None)
374
+ if row is None:
375
+ raise err(404, "activity_session_missing", "This activity session is no longer available.")
376
+ # Re-project the displayed slice as defense against a legacy or corrupt document.
377
+ now_ms = round(time.time() * 1000)
378
+ events = []
379
+ for raw in row["events"][offset:offset + limit]:
380
+ clean = project_event(raw, now_ms)
381
+ if clean:
382
+ events.append({**clean, "eventId": _text(raw.get("eventId"), 36)})
383
+ return {"sessionId": session_id, "events": events, "total": len(row["events"]),
384
+ "offset": offset, "limit": limit, "dropped": row["dropped"],
385
+ "prunedEvents": row["prunedEvents"], "limits": _limits()}
386
+
387
+
388
+ @router.get("/api/v1/activity/users")
389
+ def activity_users(session: Session = Depends(require_session)):
390
+ if not session.admin:
391
+ raise err(403, "activity_forbidden", "Only administrators can view other users' activity.")
392
+ return {"users": sorted({row["user"] for row in _read(session)})}
api/routes_admin.py CHANGED
@@ -2091,8 +2091,13 @@ def _confines_to_one_bu(tree):
2091
  return any(kids) if node.get("conj") == "or" else all(kids)
2092
  if node.get("colId") != "dba":
2093
  return True # every other condition: assume satisfied
2094
- want = str(node.get("value") or "").strip().lower()
2095
  op = node.get("op")
 
 
 
 
 
 
2096
  if op == "eq":
2097
  return brand.lower() == want
2098
  if op == "neq":
@@ -2107,6 +2112,8 @@ def _confines_to_one_bu(tree):
2107
  return True
2108
  return True # an op that cannot judge a brand does not confine
2109
 
 
 
2110
  nodes, conj = (tree.get("nodes") or []), tree.get("conj", "and")
2111
  root = {"conj": conj, "children": nodes}
2112
  admitted = {b for b in ("Fisch", "Royal", "Both") if admits(root, b)}
 
2091
  return any(kids) if node.get("conj") == "or" else all(kids)
2092
  if node.get("colId") != "dba":
2093
  return True # every other condition: assume satisfied
 
2094
  op = node.get("op")
2095
+ if op in fe.MULTI_VALUE_OPS:
2096
+ # HOTFIX 2026-09-23: `dba is any of [Royal]` is the shape the condition builder writes
2097
+ # now, and `perm_scope` already reads it. Falling to "cannot judge" here made a save
2098
+ # that KEPT the BU condition read as one that removed it, and refused it as bu_widening.
2099
+ return fe._match_multi_value(op, brand, fe.parse_filter_values(node.get("values")))
2100
+ want = str(node.get("value") or "").strip().lower()
2101
  if op == "eq":
2102
  return brand.lower() == want
2103
  if op == "neq":
 
2112
  return True
2113
  return True # an op that cannot judge a brand does not confine
2114
 
2115
+ from harness import filter_eval as fe
2116
+
2117
  nodes, conj = (tree.get("nodes") or []), tree.get("conj", "and")
2118
  root = {"conj": conj, "children": nodes}
2119
  admitted = {b for b in ("Fisch", "Royal", "Both") if admits(root, b)}
api/routes_applications.py ADDED
@@ -0,0 +1,258 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Permissioned, directly addressable Finance application contracts (S63-T16)."""
2
+ from fastapi import APIRouter, Body, Depends, Query
3
+
4
+ import application_registry
5
+ from deps import Session, err, require_session
6
+ from harness import runtime as mirror_runtime
7
+ from modules import finance_spine
8
+ from pages import resolve_bu
9
+ from routes_activity import router as activity_router
10
+ from routes_statements import _gate as statement_gate
11
+
12
+
13
+ finance_router = APIRouter(prefix="/api/v1/finance", tags=["finance"])
14
+ nav_router = APIRouter(prefix="/api/v1", tags=["applications"])
15
+ # ⛔ NO `prefix=` HERE, for the same reason S65-T16 spelled full paths on `routes_activity`:
16
+ # this router is included INSIDE this module's own `router`, and a child router carrying
17
+ # `prefix=` has that prefix baked into its `.routes` at DECLARATION time, so
18
+ # `verify_api.py::_mounted_paths` adds it a SECOND time and D-48 reads a route the app really
19
+ # serves as one it does not. The full path on each decorator is also the honest reading.
20
+ collection_router = APIRouter(tags=["collection"], dependencies=[Depends(require_session)])
21
+
22
+
23
+ def _may_open(session, permission_key):
24
+ try:
25
+ session.require(permission_key)
26
+ return True
27
+ except Exception:
28
+ return False
29
+
30
+
31
+ def _require_app(session, key):
32
+ try:
33
+ return application_registry.finance_application(
34
+ key, may_open=lambda permission: _may_open(session, permission))
35
+ except PermissionError:
36
+ raise err(403, "finance_forbidden", "your account does not have access to this Finance application")
37
+ except KeyError:
38
+ raise err(404, "finance_app_not_found", "Finance application not found")
39
+
40
+
41
+ def _require_fact(session, key):
42
+ candidates = [app for app in application_registry.finance_applications()
43
+ if key in app["factKeys"]]
44
+ if not candidates:
45
+ raise err(404, "finance_fact_not_found", "Finance fact not found")
46
+ if not any(_may_open(session, app["permissionKey"]) for app in candidates):
47
+ raise err(403, "finance_forbidden", "your account does not have access to this Finance fact")
48
+ return finance_spine.fact_spec(key)
49
+
50
+
51
+ @finance_router.get("/applications")
52
+ def list_finance_applications(session: Session = Depends(require_session)):
53
+ """One Finance group containing only the applications this session may open."""
54
+ apps = application_registry.finance_applications(
55
+ may_open=lambda permission: _may_open(session, permission))
56
+ apps = [{**app, "id": app["key"], "name": app["navLabel"],
57
+ "navPlacement": {"section": "finance", "order": index}}
58
+ for index, app in enumerate(apps)]
59
+ return {"group": application_registry.finance_group(), "applications": apps}
60
+
61
+
62
+ @finance_router.get("/applications/{app_key}")
63
+ def get_finance_application(app_key: str, bu: str | None = None,
64
+ date_from: str | None = None, date_to: str | None = None,
65
+ session: Session = Depends(require_session)):
66
+ """Render one deep-linked app; this endpoint deliberately has no Finance-home aggregate."""
67
+ app = _require_app(session, app_key)
68
+ team_id = resolve_bu(session, bu)
69
+ try:
70
+ cursor = mirror_runtime.mirror_cursor(session.runtime)
71
+ except mirror_runtime.DatastoreMismatch:
72
+ raise err(503, "finance_store_mismatch", "the Finance mirror belongs to a different workspace")
73
+ except RuntimeError:
74
+ facts = [{**finance_spine.fact_spec(key), "state": "warming", "value": None}
75
+ for key in app["factKeys"]]
76
+ return {"group": application_registry.finance_group(), "application": app,
77
+ "businessUnit": team_id, "state": "warming",
78
+ "facts": facts,
79
+ "kpis": [fact for fact in facts if fact["key"] in app["kpiKeys"]]}
80
+ facts = [finance_spine.observe(key, cursor, date_from=date_from, date_to=date_to,
81
+ team_id=team_id) for key in app["factKeys"]]
82
+ kpis = [fact for fact in facts if fact["key"] in app["kpiKeys"]]
83
+ state = "ready" if all(fact["state"] in {"ready", "requires_sku"} for fact in facts) else "partial"
84
+ return {"group": application_registry.finance_group(), "application": app,
85
+ "businessUnit": team_id, "state": state, "facts": facts, "kpis": kpis}
86
+
87
+
88
+ @finance_router.get("/facts/{fact_key}/records")
89
+ def get_finance_fact_records(fact_key: str, bu: str | None = None,
90
+ date_from: str | None = None, date_to: str | None = None,
91
+ limit: int = Query(100, ge=1, le=500), offset: int = Query(0, ge=0),
92
+ session: Session = Depends(require_session)):
93
+ """Source-record drill-through under the same permission and tenant mirror walls."""
94
+ spec = _require_fact(session, fact_key)
95
+ team_id = resolve_bu(session, bu)
96
+ try:
97
+ finance_spine.effective_team_id(spec, team_id)
98
+ mirror_runtime.mirror_cursor(session.runtime)
99
+ return finance_spine.source_records(
100
+ fact_key, date_from=date_from, date_to=date_to, team_id=team_id,
101
+ limit=limit, offset=offset)
102
+ except PermissionError:
103
+ raise err(403, "finance_scope_forbidden", "this fact requires consolidated Finance access")
104
+ except ValueError as exc:
105
+ raise err(400, "finance_scope_invalid", str(exc))
106
+ except mirror_runtime.DatastoreMismatch:
107
+ raise err(503, "finance_store_mismatch", "the Finance mirror belongs to a different workspace")
108
+ except RuntimeError:
109
+ return {"fact": spec, "state": "warming", "rows": [], "row_count": None,
110
+ "total_count": None, "aggregates": {}}
111
+
112
+
113
+ @finance_router.get("/facts/{fact_key}/reconciliation")
114
+ def reconcile_finance_fact(fact_key: str, bu: str | None = None,
115
+ date_from: str | None = None, date_to: str | None = None,
116
+ session: Session = Depends(require_session)):
117
+ """Explicit independent Odoo read; ordinary page loads never issue live source reads."""
118
+ spec = _require_fact(session, fact_key)
119
+ team_id = resolve_bu(session, bu)
120
+ try:
121
+ finance_spine.effective_team_id(spec, team_id)
122
+ mirror_runtime.mirror_cursor(session.runtime)
123
+ except PermissionError:
124
+ raise err(403, "finance_scope_forbidden", "this fact requires consolidated Finance access")
125
+ except ValueError as exc:
126
+ raise err(400, "finance_scope_invalid", str(exc))
127
+ except mirror_runtime.DatastoreMismatch:
128
+ raise err(503, "finance_store_mismatch", "the Finance mirror belongs to a different workspace")
129
+ except RuntimeError:
130
+ return {"state": "warming", "message": "The tenant mirror is still warming."}
131
+ return finance_spine.reconcile(
132
+ fact_key, date_from=date_from, date_to=date_to, team_id=team_id)
133
+
134
+
135
+ @nav_router.get("/nav/applications")
136
+ def list_nav_applications(session: Session = Depends(require_session)):
137
+ """Every Application this session may open, in ONE generic navigation shape (R6).
138
+
139
+ Finance's six arrive here as seeded Application records rather than as new template kinds,
140
+ so the App module reaches them by the same path Routing and Catalog ride.
141
+ """
142
+ if not session.runtime.available():
143
+ raise err(503, "store_unavailable", "the tenant store is unavailable")
144
+ return {"applications": application_registry.nav_applications(
145
+ session.runtime, session.uname, session.admin,
146
+ may_open=lambda permission: _may_open(session, permission))}
147
+
148
+
149
+ def _collection_ids(body, *, confirm=False):
150
+ body = body or {}
151
+ if confirm and body.get("confirm") is not True:
152
+ raise err(400, "collection_confirmation_required", "Confirm the whole batch before sending.")
153
+ allowed = {"partnerIds", "confirm"} if confirm else {"partnerIds"}
154
+ if set(body) - allowed:
155
+ raise err(400, "collection_request_invalid", "Only customer identifiers are accepted.")
156
+ ids = body.get("partnerIds")
157
+ if not isinstance(ids, list) or not ids:
158
+ raise err(400, "collection_selection_required", "Select at least one customer.")
159
+ if any(type(pid) is not int or pid <= 0 for pid in ids):
160
+ raise err(400, "collection_selection_invalid", "Customer identifiers must be positive integers.")
161
+ return list(dict.fromkeys(ids))
162
+
163
+
164
+ def _collection_source(cs):
165
+ try:
166
+ odoo = cs.Odoo()
167
+ return odoo, cs.load_collection_list(odoo)
168
+ except Exception:
169
+ raise err(502, "collection_odoo_unavailable", "The Collection worklist could not be read from Odoo.")
170
+
171
+
172
+ def _collection_selection(rows, ids):
173
+ by_id = {row["partner_id"]: row for row in rows}
174
+ selected = [by_id[pid] for pid in ids if pid in by_id]
175
+ refused = [{"customer": f"Customer {pid}", "partnerId": pid,
176
+ "reason": "This customer is no longer on the Collection worklist."}
177
+ for pid in ids if pid not in by_id]
178
+ return selected, refused
179
+
180
+
181
+ def _collection_refusal(cs, row):
182
+ """Preview the existing data-layer guard; the sender still enforces it independently."""
183
+ if not row.get("Email"):
184
+ return "No email address is saved for this customer."
185
+ if not cs.safe_recipient_ok(row["Email"]):
186
+ return "Safe mode does not allow this recipient address."
187
+ return None
188
+
189
+
190
+ @collection_router.get("/api/v1/apps/collection/worklist")
191
+ def collection_worklist(session: Session = Depends(statement_gate)):
192
+ import modules.collections_send as cs
193
+ _, rows = _collection_source(cs)
194
+ public = []
195
+ for row in rows:
196
+ docs = row.get("_docs") or []
197
+ public.append({**{key: value for key, value in row.items() if key != "_docs"},
198
+ "_docs": len(docs),
199
+ "documents": [{"name": doc["name"],
200
+ "date": doc.get("invoice_date") or None,
201
+ "dueDate": doc.get("invoice_date_due") or None,
202
+ "daysOverdue": doc["days_overdue"],
203
+ "openBalance": doc["open_signed"]} for doc in docs],
204
+ "sendRefusal": _collection_refusal(cs, row)})
205
+ return {"rows": public, "safeMode": bool(cs.SAFE_MODE),
206
+ "safeRecipients": sorted(cs.SAFE_RECIPIENTS),
207
+ "sender": {"name": cs.SENDER_NAME, "email": cs.SENDER_EMAIL, "replyTo": cs.REPLY_TO}}
208
+
209
+
210
+ @collection_router.post("/api/v1/apps/collection/preview")
211
+ def collection_preview(body: dict = Body(default=None), session: Session = Depends(statement_gate)):
212
+ """Read-only rendered statements, using the same defaults the batch will queue."""
213
+ import datetime as dt
214
+ import modules.collections_send as cs
215
+ ids = _collection_ids(body)
216
+ _, rows = _collection_source(cs)
217
+ selected, refused = _collection_selection(rows, ids)
218
+ statements = []
219
+ for row in selected:
220
+ reason = _collection_refusal(cs, row)
221
+ if reason:
222
+ refused.append({"customer": row["Customer"], "reason": reason})
223
+ statements.append({"partnerId": row["partner_id"], "customer": row["Customer"],
224
+ "to": row["Email"], "refusal": reason,
225
+ "subject": cs.DEFAULT_SUBJECT.format(
226
+ customer=row["Customer"], company=cs.COMPANY,
227
+ month=dt.date.today().strftime("%B %Y")),
228
+ "html": cs.render_statement_html(row)})
229
+ return {"statements": statements, "refused": refused, "requested": len(ids),
230
+ "safeMode": bool(cs.SAFE_MODE), "safeRecipients": sorted(cs.SAFE_RECIPIENTS),
231
+ "sender": {"name": cs.SENDER_NAME, "email": cs.SENDER_EMAIL, "replyTo": cs.REPLY_TO}}
232
+
233
+
234
+ @collection_router.post("/api/v1/apps/collection/send")
235
+ def collection_send(body: dict = Body(default=None), session: Session = Depends(statement_gate)):
236
+ import modules.collections_send as cs
237
+ ids = _collection_ids(body, confirm=True)
238
+ odoo, rows = _collection_source(cs)
239
+ selected, refused = _collection_selection(rows, ids)
240
+ try:
241
+ result = cs.queue_statement_batch(odoo, selected)
242
+ except Exception:
243
+ # An interrupted response may follow a successful create. Never invite a blind retry.
244
+ raise err(502, "collection_send_unconfirmed",
245
+ "Odoo did not confirm the batch result. Some statements may already be in the "
246
+ "outbox. Check Odoo before trying again.")
247
+ return {"created": len(result["created"]), "refused": refused + result["refused"],
248
+ "safeMode": result["safe_mode"]}
249
+
250
+
251
+ # main.py includes exactly one object named router. These sub-routers keep
252
+ # /nav/applications outside the Finance prefix while every Finance URL stays unchanged.
253
+ router = APIRouter()
254
+ router.include_router(finance_router)
255
+ router.include_router(nav_router)
256
+ # main.py belongs to another session this wave; this router is already mounted there.
257
+ router.include_router(activity_router)
258
+ router.include_router(collection_router)
api/routes_automation.py CHANGED
@@ -433,6 +433,7 @@ def _wire(defn, tenant, rt=None):
433
  "config": defn.get("config") or {}, "schedule": sched, "status": status,
434
  "runs": list(defn.get("runs") or [])[:engine.MAX_RUNS],
435
  "created": defn.get("created"), "createdBy": defn.get("createdBy"),
 
436
  "nextRunAt": nxt.strftime("%Y-%m-%d %H:%M") if nxt else "",
437
  "running": bool(live),
438
  # ⭐ WAVE 27 (contract C6) — DEBT D-70: "a paid search is already outstanding at the
@@ -1210,8 +1211,9 @@ def list_automations(session: Session = Depends(_GATE)):
1210
  #
1211
  # ⛔ WHY SKIPPING THE SCAN ON CALLS 2..N IS SAFE, and it is not "because it is idempotent":
1212
  # `all_definitions` STRIPS retired board state on every read (`automation_engine.py`'s
1213
- # `_without_retired_board`), so the persist step is hygiene, not correctness. No client can
1214
- # ever be shown state this skip left behind. Nothing writes new `stage_auto_` cells either —
 
1215
  # wave 26's R6 deleted the board that made them.
1216
  # ⭐⭐ WAVE 30 · T12 (owner items 4/5, the complaint that has now survived TWO waves).
1217
  # ⛔ W29-T01 GUARDED THE SCAN AND NOT THE READ, and that is the whole of what was left. The
@@ -1221,6 +1223,13 @@ def list_automations(session: Session = Depends(_GATE)):
1221
  # behind each other) — while on calls 2..N the result was DISCARDED: `tables` had exactly two
1222
  # consumers, the `_BOARD_RETIRED` scan (skipped after call 1) and a picker DEFAULT STRING.
1223
  # A whole-tenant document, per request, to render a rail showing a name and a toggle.
 
 
 
 
 
 
 
1224
  tables = None
1225
  if session.runtime.available() and session.tenant not in _BOARD_RETIRED:
1226
  tables = engine.ut_all(session.runtime)
@@ -1228,6 +1237,7 @@ def list_automations(session: Session = Depends(_GATE)):
1228
  # Idempotent Board retirement removes only engine-marked stage fields and legacy Board
1229
  # state. User-created Status/Stage columns remain intact.
1230
  engine.retire_automation_board_state(session.runtime, tables=tables)
 
1231
  # ⭐⭐ WAVE 35 · T37 / OWNER RULING R10 — ROYAL IMPORTS' STATEMENTS AGENT EXISTS BEFORE ANYBODY
1232
  # ASKS FOR IT. It is minted ONCE, by the container, SWITCHED OFF; every subsequent call finds
1233
  # it present and returns immediately. See `engine.seed_statements_agent` for why this one is
@@ -1254,6 +1264,7 @@ def list_automations(session: Session = Depends(_GATE)):
1254
  items = sorted(items + _field_agent_rows(session) + _sys_rows,
1255
  key=lambda r: str(r.get("name") or "").lower())
1256
  return {"automations": items,
 
1257
  # ⭐ WAVE 24 — DERIVED from the engine's public `PICKER_KINDS`, not a hand-written trio. It was
1258
  # three literals that happened to match, which is a second copy of a server
1259
  # vocabulary; R6 has just made two of them uncreatable and `plain` has joined, so a
@@ -2114,7 +2125,7 @@ def create_automation(body: dict = Body(default=None), session: Session = Depend
2114
  # ⚠ It rides the SAVE's own response, beside `unconfigured`, because that is the moment the
2115
  # person is looking — a note in a log they never open is the same silence with extra steps.
2116
  _notes = []
2117
- defn, error = engine.create(session.runtime, body or {}, username=session.uname,
2118
  notes=_notes)
2119
  if error:
2120
  raise err(400, "invalid_automation", error)
@@ -2159,8 +2170,9 @@ def patch_automation(auto_id: str, body: dict = Body(default=None),
2159
  # attribute in the builder is a suggestion; this is the refusal. It permits a DELETE of the
2160
  # step and permits an ordinary save of a flow that merely contains one.
2161
  _agent_owned_guard(session, auto_id, body or {})
 
2162
  _notes = []
2163
- defn, error = engine.patch(session.runtime, auto_id, body or {}, username=session.uname,
2164
  notes=_notes)
2165
  if error:
2166
  raise err(400 if error != "no such automation" else 404,
 
433
  "config": defn.get("config") or {}, "schedule": sched, "status": status,
434
  "runs": list(defn.get("runs") or [])[:engine.MAX_RUNS],
435
  "created": defn.get("created"), "createdBy": defn.get("createdBy"),
436
+ **({"preset": str(defn.get("preset"))} if defn.get("preset") else {}),
437
  "nextRunAt": nxt.strftime("%Y-%m-%d %H:%M") if nxt else "",
438
  "running": bool(live),
439
  # ⭐ WAVE 27 (contract C6) — DEBT D-70: "a paid search is already outstanding at the
 
1211
  #
1212
  # ⛔ WHY SKIPPING THE SCAN ON CALLS 2..N IS SAFE, and it is not "because it is idempotent":
1213
  # `all_definitions` STRIPS retired board state on every read (`automation_engine.py`'s
1214
+ # `_without_retired_board`) and drops retired presets, so persistence is hygiene, not
1215
+ # correctness. No client can see state this skip left behind. Nothing writes new
1216
+ # `stage_auto_` cells either —
1217
  # wave 26's R6 deleted the board that made them.
1218
  # ⭐⭐ WAVE 30 · T12 (owner items 4/5, the complaint that has now survived TWO waves).
1219
  # ⛔ W29-T01 GUARDED THE SCAN AND NOT THE READ, and that is the whole of what was left. The
 
1223
  # behind each other) — while on calls 2..N the result was DISCARDED: `tables` had exactly two
1224
  # consumers, the `_BOARD_RETIRED` scan (skipped after call 1) and a picker DEFAULT STRING.
1225
  # A whole-tenant document, per request, to render a rail showing a name and a toggle.
1226
+ # ⛔⛔ AND THE REMOVAL NOTICE OBEYS THE SAME RULE, which is why it is built HERE and not above.
1227
+ # The first draft of this read the whole `automations` document on EVERY request to name the
1228
+ # retired rows, and then a SECOND time to fetch their names — the defect this very comment
1229
+ # block is the tombstone of, re-introduced two lines under it. The sweep has already read
1230
+ # every row it removed, so its receipt carries them: calls 2..N report nothing and read
1231
+ # nothing, which is correct as well as cheap, because after the sweep no retired row exists.
1232
+ retired = []
1233
  tables = None
1234
  if session.runtime.available() and session.tenant not in _BOARD_RETIRED:
1235
  tables = engine.ut_all(session.runtime)
 
1237
  # Idempotent Board retirement removes only engine-marked stage fields and legacy Board
1238
  # state. User-created Status/Stage columns remain intact.
1239
  engine.retire_automation_board_state(session.runtime, tables=tables)
1240
+ retired = engine.retire_preset_automations(session.runtime)["removed"]
1241
  # ⭐⭐ WAVE 35 · T37 / OWNER RULING R10 — ROYAL IMPORTS' STATEMENTS AGENT EXISTS BEFORE ANYBODY
1242
  # ASKS FOR IT. It is minted ONCE, by the container, SWITCHED OFF; every subsequent call finds
1243
  # it present and returns immediately. See `engine.seed_statements_agent` for why this one is
 
1264
  items = sorted(items + _field_agent_rows(session) + _sys_rows,
1265
  key=lambda r: str(r.get("name") or "").lower())
1266
  return {"automations": items,
1267
+ "retired": retired,
1268
  # ⭐ WAVE 24 — DERIVED from the engine's public `PICKER_KINDS`, not a hand-written trio. It was
1269
  # three literals that happened to match, which is a second copy of a server
1270
  # vocabulary; R6 has just made two of them uncreatable and `plain` has joined, so a
 
2125
  # ⚠ It rides the SAVE's own response, beside `unconfigured`, because that is the moment the
2126
  # person is looking — a note in a log they never open is the same silence with extra steps.
2127
  _notes = []
2128
+ defn, error = engine.create(session.runtime, body, username=session.uname,
2129
  notes=_notes)
2130
  if error:
2131
  raise err(400, "invalid_automation", error)
 
2170
  # attribute in the builder is a suggestion; this is the refusal. It permits a DELETE of the
2171
  # step and permits an ordinary save of a flow that merely contains one.
2172
  _agent_owned_guard(session, auto_id, body or {})
2173
+ previous = engine.all_definitions(session.runtime).get(str(auto_id))
2174
  _notes = []
2175
+ defn, error = engine.patch(session.runtime, auto_id, body, username=session.uname,
2176
  notes=_notes)
2177
  if error:
2178
  raise err(400 if error != "no such automation" else 404,
api/routes_connected_tables.py CHANGED
@@ -62,6 +62,8 @@ def connected_table_rows(table_key: str,
62
  sorts: str = Query(default=None),
63
  search: str = Query(default=None),
64
  related: str = Query(default=None),
 
 
65
  session: Session = Depends(require_session)):
66
  """ONE WINDOW over a connected grid — `{fields, rows, total, totalUnfiltered, offset, limit,
67
  limits, identity, recordsMutable}`, whichever connector the rows come from.
@@ -89,4 +91,5 @@ def connected_table_rows(table_key: str,
89
  import routes_odoo_tables
90
  return routes_odoo_tables.odoo_table_rows(
91
  table_key, offset=offset, limit=limit, filters=filters, filterConj=filterConj,
92
- sorts=sorts, search=search, related=related, session=session)
 
 
62
  sorts: str = Query(default=None),
63
  search: str = Query(default=None),
64
  related: str = Query(default=None),
65
+ groupBy: str = Query(default=None),
66
+ groupKey: str = Query(default=None),
67
  session: Session = Depends(require_session)):
68
  """ONE WINDOW over a connected grid — `{fields, rows, total, totalUnfiltered, offset, limit,
69
  limits, identity, recordsMutable}`, whichever connector the rows come from.
 
91
  import routes_odoo_tables
92
  return routes_odoo_tables.odoo_table_rows(
93
  table_key, offset=offset, limit=limit, filters=filters, filterConj=filterConj,
94
+ sorts=sorts, search=search, related=related, groupBy=groupBy, groupKey=groupKey,
95
+ session=session)
api/routes_connectors.py CHANGED
@@ -21,6 +21,7 @@ that reported another user's Gmail as "connected" would be both wrong and a smal
21
  from __future__ import annotations
22
 
23
  import os
 
24
  import time
25
  import threading
26
  import xmlrpc.client
@@ -30,7 +31,7 @@ from datetime import datetime, timezone
30
  from fastapi import APIRouter, Body, Depends
31
 
32
  import oauth_connect
33
- from deps import Session, err, require_session
34
  #: ⭐⭐ W49-T18 / R27 / C4 — THE TAXONOMY LIVES IN THE DATA LAYER, NOT HERE. `core` is already on
35
  #: the path by the time this module loads (`deps` imports three of its modules), so this is the
36
  #: same import shape the rest of the API uses. It is in `platform/core/` rather than beside this
@@ -82,6 +83,61 @@ from routes_monid import router as monid_router
82
  router = APIRouter(prefix="/api/v1")
83
 
84
  CONNECTOR_DETAIL_DEADLINE_SECONDS = 30
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
85
 
86
 
87
  # Measured reads take 1.34 s cold, 0.01 s warm, and above 8 s under boot contention.
@@ -150,13 +206,16 @@ PLANNED = [
150
  #: 'Scraper' … would I see the option to choose either Instagram OR TikTok."* So the directory
151
  #: mirrors the automation menus exactly: ONE card for the capability, and the platforms are behind
152
  #: it rather than beside it.
153
- #: ⚠ These are not connectors and must never become rows: they have no state, no credential and
154
- #: no manage door of their own — the Scraper's key is what is configured, and it configures both.
 
155
  SCRAPER_PLATFORMS = [
156
  {"key": "instagram", "label": "Instagram",
157
  "desc": "Profiles, posts and comments, by search or by handle."},
158
  {"key": "tiktok", "label": "TikTok",
159
  "desc": "The same three, on TikTok's own schema."},
 
 
160
  ]
161
 
162
  #: Token connectors an admin can wire up TODAY by pasting a key — no OAuth client, no app review.
@@ -1063,13 +1122,16 @@ def _scraper_row():
1063
  # ⭐ WAVE 30 · R4 — the sentence names the CAPABILITY; the platforms are the drill below.
1064
  # It used to say "Instagram profile and post capture", which stopped being true the day
1065
  # TikTok discovery shipped and would have to be re-edited for every platform after it.
1066
- "desc": "Profile, post and comment capture for the discovery automations.",
1067
  "state": "connected" if ready else "unconfigured",
1068
  "needs": "" if ready else "AIOS_BRIGHTDATA_KEY is not set on this deployment",
1069
  "manage": "",
1070
- # ⛔ ONE CREDENTIAL, TWO PLATFORMS — which is why this is a field on the row and not two
1071
- # rows. `bd_ready()` above answers for both; a per-platform state would be a second
1072
- # answer to a question that has one.
 
 
 
1073
  "platforms": [dict(p) for p in SCRAPER_PLATFORMS],
1074
  }
1075
 
@@ -1129,6 +1191,8 @@ def connectors_directory(session: Session = Depends(require_session)):
1129
  # trap; `chip_counts` carries the argument).
1130
  served = connector_directory.to_client_rows(rows)
1131
  for row in served:
 
 
1132
  _safe_connector_status(row.get("key"), row)
1133
  _safe_connector_status("amazon_sp", {"incomplete": incomplete})
1134
  return {"connectors": served,
@@ -1162,6 +1226,357 @@ def connectors_directory(session: Session = Depends(require_session)):
1162
 
1163
  _ODOO_SCHEMA_ENABLEMENTS = "odoo_schema_enablements"
1164
  _ODOO_SYNC_STATUS = "odoo_sync_status"
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1165
 
1166
 
1167
  def _odoo_admin(session):
@@ -1178,8 +1593,7 @@ def _connection_jobs(rt, provider, connection_id):
1178
  """A status endpoint selects one connection; the jobs inventory may enumerate them all."""
1179
  cid = str(connection_id or "").strip() or "default"
1180
  if provider == "odoo" and cid == "default":
1181
- import odoo_relational
1182
- cid = odoo_relational.default_connection_id(rt) or "default"
1183
  return connector_sync.jobs(rt, provider, cid)
1184
 
1185
 
@@ -1200,17 +1614,86 @@ def _sync_payload(connector_key, connection_id, row=None):
1200
  return _safe_connector_status(connector_key, answer)
1201
 
1202
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1203
  def _odoo_sync_runner(session, connection_id):
1204
  """Run Odoo through C6 without moving provider imports into the data layer."""
1205
- connector_sync.record(session.runtime, "odoo", connection_id, "syncing")
 
 
 
1206
  import odoo_relational
1207
- result = odoo_relational.refresh(session.runtime, session.tenant,
1208
- username=session.uname or "automation",
1209
- connection_id=connection_id,
1210
- on_apply=lambda: connector_sync.record(
1211
- session.runtime, "odoo", connection_id, "materializing"))
 
 
 
 
 
 
 
 
 
 
 
1212
  tables = result.get("tables") if isinstance(result, dict) else {}
1213
- databases = sorted(str(key) for key in (tables or {}).keys())
 
 
 
 
 
 
1214
  sync_errors = []
1215
  if isinstance(result, dict):
1216
  if result.get("failed"):
@@ -1220,12 +1703,57 @@ def _odoo_sync_runner(session, connection_id):
1220
  sync_errors.extend(str(value) for value in (discovered.get("errors") or ()))
1221
  discovered_tables = discovered.get("tables")
1222
  if isinstance(discovered_tables, dict):
 
1223
  databases.extend(str(key) for key in discovered_tables)
1224
  if result.get("frozen"):
1225
  sync_errors.append(str(result.get("note") or "the connector is frozen"))
1226
- state = ("partial" if databases else "error") if sync_errors else "ready"
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1227
  return connector_sync.record(
1228
- session.runtime, "odoo", connection_id, state,
1229
  detail=str(result.get("note") or "") if isinstance(result, dict) else "",
1230
  errors=sync_errors, databases=sorted(set(databases)),
1231
  )
@@ -1339,6 +1867,9 @@ def _run_database_job(rt, candidate):
1339
  return
1340
  fenced = _MaterializationRuntime(rt, job)
1341
  key = job["databaseKey"]
 
 
 
1342
  def finish(success=False, cause=""):
1343
  # The pre-claim check cannot cover a disconnect during source I/O or publication.
1344
  # Only the current lease may requeue; disable and a newer Save still win.
@@ -1398,22 +1929,64 @@ def _run_database_job(rt, candidate):
1398
  fenced, job, rows=rows, total=total))}
1399
  discovered = result.get("discovered") or {}
1400
  published = {**(result.get("tables") or {}), **(discovered.get("tables") or {})}
 
 
 
1401
  success = (key in published and not result.get("failed") and not result.get("frozen")
1402
- and not result.get("limits") and not discovered.get("errors"))
1403
- finish(success=success, cause=result.get("note") or discovered.get("note") or "")
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1404
  except _SupersededMaterialization:
1405
  # Disable/new Save already invalidated the table marker; never restore it here.
1406
  finish()
1407
  except rel.Refused as exc:
1408
- finish(cause=str(exc))
 
 
 
 
 
1409
  except Exception as exc:
1410
- # Raw provider exceptions may carry credentials or source values. Store only safe copy.
1411
- cause = ""
 
1412
  if _odoo_read_access_refused(exc):
1413
- cause = (f"Odoo refused to read {job['model']} because the connected Odoo user has no "
1414
- "read access, so retrying cannot load rows. "
1415
- "Give that user read access in Odoo, or switch this database off.")
1416
- finish(cause=cause)
 
 
 
 
 
1417
  finally:
1418
  stopped.set()
1419
  if heartbeat_thread:
@@ -1644,17 +2217,20 @@ def _odoo_field_key(rt, database_key, model, raw_name, connection_id=...):
1644
  def odoo_status(connection_id: str = None, session: Session = Depends(require_session)):
1645
  _odoo_admin(session)
1646
  import odoo_relational
 
1647
  cfg = odoo_relational.read_config(session.runtime)
1648
  stored = session.runtime.get(_ODOO_SYNC_STATUS) or {}
1649
  errors = stored.get("errors") if isinstance(stored.get("errors"), list) else []
1650
  phase = str(stored.get("phase") or "unknown")
1651
- lifecycle = connector_sync.read(session.runtime, "odoo", connection_id)
1652
  durable = connector_sync.job_summary(_connection_jobs(session.runtime, "odoo", connection_id))
 
1653
  if durable:
1654
  return {"schedule": cfg.get("syncEvery"),
1655
  "health": "paused" if cfg.get("frozen") else durable["state"],
1656
  "lastRun": durable["at"], "frozen": cfg.get("frozen") is True,
1657
- **durable, "stateAt": durable["at"], "connectionId": connection_id or "default"}
 
1658
  # The old keys stay the old default-connection answer. A caller that names a connection gets
1659
  # that slot's lifecycle instead of the tenant-wide legacy document that caused the C6 defect.
1660
  named = connection_id is not None
@@ -1678,7 +2254,8 @@ def odoo_status(connection_id: str = None, session: Session = Depends(require_se
1678
  "retryable": state in ("error", "partial"),
1679
  "errorsTotal": int(errors_total or 0),
1680
  "connectionId": (lifecycle or {}).get("connectionId")
1681
- or str(connection_id or "").strip() or "default"}
 
1682
 
1683
 
1684
  @router.get("/connectors/{connector_key}/sync")
@@ -1694,13 +2271,25 @@ def connector_sync_status(connector_key: str, connection_id: str = None,
1694
 
1695
 
1696
  @router.post("/connectors/{connector_key}/sync/now")
1697
- def connector_sync_now(connector_key: str, session: Session = Depends(require_session)):
 
1698
  _sync_admin(session)
1699
  if connector_key not in connector_sync.providers():
1700
  raise err(404, "sync_provider_unknown", "This connector is not registered for sync.")
1701
  try:
1702
- return connector_sync.request_sync(connector_key, session.runtime.key,
1703
- requested_by=session.uname)
 
 
 
 
 
 
 
 
 
 
 
1704
  except ValueError:
1705
  raise err(404, "sync_provider_unknown", "This connector is not registered for sync.") from None
1706
 
@@ -1710,7 +2299,11 @@ def connector_sync_now_status(connector_key: str, session: Session = Depends(req
1710
  _sync_admin(session)
1711
  if connector_key not in connector_sync.providers():
1712
  raise err(404, "sync_provider_unknown", "This connector is not registered for sync.")
1713
- return connector_sync.sync_request(connector_key, session.runtime.key) or {
 
 
 
 
1714
  "provider": connector_key, "tenant": session.runtime.key, "state": "idle",
1715
  "requestedAt": None, "startedAt": None, "finishedAt": None,
1716
  "requestedBy": "", "detail": "", "accepted": False,
@@ -1760,8 +2353,11 @@ def connector_database_jobs(connector_key: str, session: Session = Depends(requi
1760
  _sync_admin(session)
1761
  if connector_key not in connector_sync.providers():
1762
  raise err(404, "sync_provider_unknown", "This connector has no database sync runner.")
1763
- return _safe_connector_status(connector_key, {"provider": connector_key, "jobs": connector_sync.jobs(
1764
- session.runtime, connector_key, connection_id)})
 
 
 
1765
 
1766
 
1767
  @router.get("/connectors/odoo/schema")
@@ -2241,6 +2837,114 @@ def _safe_connector_status(connector_key, payload):
2241
  return payload
2242
 
2243
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2244
  def _detail_connect(entry_type, is_admin):
2245
  """C4's `keychain.connect` half: what the Connect form must ask for, and what it may store.
2246
 
@@ -2624,6 +3328,173 @@ def _named_databases(rt, is_admin, uname, rows):
2624
  return named
2625
 
2626
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2627
  def _detail_databases(rt, is_admin, connector_key, uname="", connection_id=None):
2628
  """C4's `databases` half: what this connector materialises, and the fields behind each one.
2629
 
@@ -2652,6 +3523,8 @@ def _detail_databases(rt, is_admin, connector_key, uname="", connection_id=None)
2652
  reason: what a database is CALLED is a fact about the reader's own workspace. Only the WRITE
2653
  is walled, per row, by `canRename`.
2654
  """
 
 
2655
  if not connector_directory.is_introspectable(connector_key):
2656
  return []
2657
  spec = _connector_spec_tables(connector_key)
@@ -2668,6 +3541,9 @@ def _detail_databases(rt, is_admin, connector_key, uname="", connection_id=None)
2668
  census = odoo_relational.population_census(rt, namespace)
2669
  except Exception: # noqa: BLE001
2670
  return []
 
 
 
2671
  census_at = str(census.get("capturedAt") or "")
2672
  counts = census.get("counts") if isinstance(census.get("counts"), dict) else {}
2673
  catalog = {}
@@ -2712,6 +3588,12 @@ def _detail_databases(rt, is_admin, connector_key, uname="", connection_id=None)
2712
  # Counts are model-level facts. Databases backed by the same model deliberately show the
2713
  # same sourceRows number; inventing a per-database domain here would make the number lie.
2714
  source_rows = counts.get(model) if census_at and model in counts else None
 
 
 
 
 
 
2715
  out.append({"key": key,
2716
  "identity": connector_sync.database_identity(
2717
  "odoo", resolved_connection_id, key, model),
@@ -2721,6 +3603,12 @@ def _detail_databases(rt, is_admin, connector_key, uname="", connection_id=None)
2721
  "enabled": bool(choice.get("enabled")),
2722
  "sourceRows": source_rows,
2723
  "populated": None if source_rows is None else source_rows > 0,
 
 
 
 
 
 
2724
  "fields": scoped_fields,
2725
  "excluded": [dict(item) for item in excluded],
2726
  "sharesFieldsWith": []})
@@ -2811,6 +3699,8 @@ def _amazon_databases(rt, is_admin, spec=None):
2811
  # what `native` carries: the vendor's own name for the thing, never one we invented. It is
2812
  # empty for a table whose report type is unknown, and the tab says so rather than guessing.
2813
  declared = (spec or {}).get(key) or {}
 
 
2814
  out.append({"key": key,
2815
  "label": str(declared.get("label") or LABELS_AMAZON.get(key) or key),
2816
  "defaultLabel": str(declared.get("label")
@@ -2819,8 +3709,9 @@ def _amazon_databases(rt, is_admin, spec=None):
2819
  # W55-T19 / R7 — AMAZON'S OWN tenant-local answer. This is deliberately not
2820
  # read from Odoo's grid config, and disabling it leaves every stored row and
2821
  # definition untouched: only this provider config document changes.
2822
- "enabled": enabled.get(key) is True,
2823
- **({"sync": (states or {}).get(name)} if is_admin else {}),
 
2824
  "fields": fields,
2825
  "excluded": excluded,
2826
  "sharesFieldsWith": [k for k in by_report.get(rep, ()) if k != key]})
@@ -2888,6 +3779,14 @@ def _detail_introspection(rt, is_admin, row):
2888
  stops "administrator only" being rendered as "this database has no fields".
2889
  """
2890
  key = str(row.get("key") or "")
 
 
 
 
 
 
 
 
2891
  if not connector_directory.is_introspectable(key):
2892
  reason = ("Nothing is connected behind this listing yet, so there is no schema to read."
2893
  if str(row.get("state") or "") == connector_directory.CATALOGUE_STATE
@@ -3008,6 +3907,229 @@ def _detail_enablement(rt, is_admin, row):
3008
  "updatePath": "", "preservesData": True, "available": False}
3009
 
3010
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
3011
  def connector_detail(rt, user, uname, is_admin, connector_key):
3012
  """The whole C4 detail DTO for one connector, or `None` when this reader has no such card.
3013
 
@@ -3106,6 +4228,26 @@ def amazon_database_enablement(database_key: str, body: dict = Body(default=None
3106
  return _set_amazon_database_enablement(database_key, body, session)
3107
 
3108
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
3109
  # ⚠ DECLARED AFTER THE LITERAL `/connectors/odoo/*` ROUTES. FastAPI matches in declaration order,
3110
  # and while none of those ends in `/detail` — so no shadowing is possible today — the ordering is
3111
  # the habit that keeps it impossible tomorrow. `/connectors/directory` is two segments and cannot
 
21
  from __future__ import annotations
22
 
23
  import os
24
+ import ssl
25
  import time
26
  import threading
27
  import xmlrpc.client
 
31
  from fastapi import APIRouter, Body, Depends
32
 
33
  import oauth_connect
34
+ from deps import Session, err, require_session, running_main
35
  #: ⭐⭐ W49-T18 / R27 / C4 — THE TAXONOMY LIVES IN THE DATA LAYER, NOT HERE. `core` is already on
36
  #: the path by the time this module loads (`deps` imports three of its modules), so this is the
37
  #: same import shape the rest of the API uses. It is in `platform/core/` rather than beside this
 
83
  router = APIRouter(prefix="/api/v1")
84
 
85
  CONNECTOR_DETAIL_DEADLINE_SECONDS = 30
86
+ SCRAPER_RECIPE_KINDS = frozenset(("instagram", "tiktok", "google_maps"))
87
+ SCRAPER_RECIPE_MAX_ROWS = 30
88
+ SCRAPER_RUN_HISTORY_LIMIT = 20
89
+ DATABASE_SYNC_STATES = frozenset(
90
+ ("never_run", "queued", "running", "succeeded", "partial", "failed", "disabled")
91
+ )
92
+ _DATABASE_SAFE_ERRORS = {
93
+ "tls_certificate_invalid": ("Odoo TLS certificate verification failed because the certificate "
94
+ "is expired or untrusted. No source data was accepted."),
95
+ "source_read_refused": "Odoo refused the configured account's read access.",
96
+ "source_auth_failed": "Odoo rejected the configured read-only credential.",
97
+ "source_unavailable": "Odoo could not be reached for this read-only sync.",
98
+ "source_read_failed": "Odoo source validation failed.",
99
+ "mirror_pull_failed": "The Odoo mirror pull failed.",
100
+ "materialization_failed": "The database could not be materialized.",
101
+ "reconciliation_failed": "The database reconciliation failed.",
102
+ "freshness_unconfirmed": "Freshness could not be confirmed for this database.",
103
+ "sync_status_unavailable": "Sync status is unavailable for this database.",
104
+ "sync_stalled": "This database sync is stalled.",
105
+ "sync_state_unreadable": "This database's saved sync state could not be read.",
106
+ "maps_credential_missing": "Add the scraper credential in Keychain before choosing Run now.",
107
+ "scraper_credential_missing": "Add the scraper credential in Keychain before choosing Run now.",
108
+ "maps_unbounded_request": "Google Maps discovery requires an explicit bounded result cap.",
109
+ "maps_invalid_search": "Google Maps discovery needs a valid category or search.",
110
+ "maps_vendor_error": "The Maps provider could not complete this run.",
111
+ "maps_schema_drift": "The Maps provider returned an unreadable result shape.",
112
+ "scraper_run_failed": "The provider could not complete this run. No result was written.",
113
+ "provider_result_failed": "The provider returned no usable result for this database.",
114
+ "scraper_result_not_saved": "The provider answered, but the result could not be saved.",
115
+ }
116
+
117
+ # T11 established the saved database contract; T12's provider normalises every Apify answer into
118
+ # exactly this schema. Vendor field names never reach a saved View or downstream automation.
119
+ _MAPS_RECIPE_FIELDS = [
120
+ {"key": "place_id", "label": "Place id", "type": "text", "source": "automation"},
121
+ {"key": "name", "label": "Name", "type": "text", "source": "automation"},
122
+ {"key": "category", "label": "Category", "type": "text", "source": "automation"},
123
+ {"key": "address", "label": "Address", "type": "text", "source": "automation"},
124
+ {"key": "city", "label": "City", "type": "text", "source": "automation"},
125
+ {"key": "region", "label": "Region", "type": "text", "source": "automation"},
126
+ {"key": "postal_code", "label": "Postal code", "type": "text", "source": "automation"},
127
+ {"key": "country", "label": "Country", "type": "text", "source": "automation"},
128
+ {"key": "phone", "label": "Phone", "type": "phone", "source": "automation"},
129
+ {"key": "website", "label": "Website", "type": "url", "source": "automation"},
130
+ # Coordinates remain text because `number` is not a user-table field kind; the Map binding
131
+ # parses the canonical scalar value. Declaring an unsupported kind would silently drop both
132
+ # columns at database creation.
133
+ {"key": "latitude", "label": "Latitude", "type": "text", "source": "automation"},
134
+ {"key": "longitude", "label": "Longitude", "type": "text", "source": "automation"},
135
+ {"key": "rating", "label": "Rating", "type": "rating", "source": "automation"},
136
+ {"key": "reviews", "label": "Reviews", "type": "int", "source": "automation"},
137
+ {"key": "source_timestamp", "label": "Source timestamp", "type": "date",
138
+ "source": "automation"},
139
+ {"key": "source_payload", "label": "Source data", "type": "json", "source": "automation"},
140
+ ]
141
 
142
 
143
  # Measured reads take 1.34 s cold, 0.01 s warm, and above 8 s under boot contention.
 
206
  #: 'Scraper' … would I see the option to choose either Instagram OR TikTok."* So the directory
207
  #: mirrors the automation menus exactly: ONE card for the capability, and the platforms are behind
208
  #: it rather than beside it.
209
+ #: ⚠ These are not connectors and must never become rows: they have no state or manage door of
210
+ #: their own. Instagram and TikTok share the Scraper credential; Maps uses the Apify credential
211
+ #: through the same governed provider registry.
212
  SCRAPER_PLATFORMS = [
213
  {"key": "instagram", "label": "Instagram",
214
  "desc": "Profiles, posts and comments, by search or by handle."},
215
  {"key": "tiktok", "label": "TikTok",
216
  "desc": "The same three, on TikTok's own schema."},
217
+ {"key": "google_maps", "label": "Google Maps",
218
+ "desc": "Places by category or search and geography, through a bounded Apify run."},
219
  ]
220
 
221
  #: Token connectors an admin can wire up TODAY by pasting a key — no OAuth client, no app review.
 
1122
  # ⭐ WAVE 30 · R4 — the sentence names the CAPABILITY; the platforms are the drill below.
1123
  # It used to say "Instagram profile and post capture", which stopped being true the day
1124
  # TikTok discovery shipped and would have to be re-edited for every platform after it.
1125
+ "desc": "Create saved Instagram, TikTok and Google Maps database recipes.",
1126
  "state": "connected" if ready else "unconfigured",
1127
  "needs": "" if ready else "AIOS_BRIGHTDATA_KEY is not set on this deployment",
1128
  "manage": "",
1129
+ # The directory taxonomy normally derives this flag from connected table prefixes.
1130
+ # Scraper recipes are user databases, so the route stamps it again after normalisation.
1131
+ "spawns_tables": True,
1132
+ # Platforms remain a drill behind one recipe/database capability, never directory rows.
1133
+ # `bd_ready()` describes today's social runners only; Maps' own sentence names that its
1134
+ # execution provider is intentionally absent until T12.
1135
  "platforms": [dict(p) for p in SCRAPER_PLATFORMS],
1136
  }
1137
 
 
1191
  # trap; `chip_counts` carries the argument).
1192
  served = connector_directory.to_client_rows(rows)
1193
  for row in served:
1194
+ if str(row.get("key") or "") == "scraper":
1195
+ row["spawns_tables"] = True
1196
  _safe_connector_status(row.get("key"), row)
1197
  _safe_connector_status("amazon_sp", {"incomplete": incomplete})
1198
  return {"connectors": served,
 
1226
 
1227
  _ODOO_SCHEMA_ENABLEMENTS = "odoo_schema_enablements"
1228
  _ODOO_SYNC_STATUS = "odoo_sync_status"
1229
+ _ODOO_HEALTH_STATUS = "odoo_health_status_v1"
1230
+ _ODOO_HEALTH_STAGES = ("source", "mirrorPull", "materialization", "reconciliation", "freshness")
1231
+ _SYNC_REQUEST_CONNECTIONS = {}
1232
+ _SYNC_REQUEST_CONNECTION_LOCK = threading.Lock()
1233
+
1234
+
1235
+ def _odoo_connection_slot(rt, connection_id=None):
1236
+ """Return the durable Odoo connection id used by jobs, status, and health receipts."""
1237
+ requested = str(connection_id or "").strip()
1238
+ if requested and requested != "default":
1239
+ return requested
1240
+ try:
1241
+ import odoo_relational
1242
+ return str(odoo_relational.default_connection_id(rt) or "default")
1243
+ except Exception: # noqa: BLE001
1244
+ return "default"
1245
+
1246
+
1247
+ def sync_request_connection(provider, tenant):
1248
+ """Return the connection bound to a process-local Sync now intent."""
1249
+ with _SYNC_REQUEST_CONNECTION_LOCK:
1250
+ return _SYNC_REQUEST_CONNECTIONS.get((str(provider), str(tenant)))
1251
+
1252
+
1253
+ def _odoo_failure(exc, stage="source"):
1254
+ """Classify a source/database exception without persisting its potentially sensitive text."""
1255
+ stage = stage if stage in _ODOO_HEALTH_STAGES else "source"
1256
+ text = str(exc or "").casefold()
1257
+ exc_type = type(exc).__name__ if isinstance(exc, BaseException) else "SourceError"
1258
+ if (isinstance(exc, ssl.SSLCertVerificationError)
1259
+ or "certificate_verify_failed" in text
1260
+ or "certificate verify failed" in text
1261
+ or "certificate has expired" in text
1262
+ or "self signed certificate" in text):
1263
+ return {"stage": "source", "code": "tls_certificate_invalid",
1264
+ "type": exc_type,
1265
+ "message": ("Odoo TLS certificate verification failed because the certificate "
1266
+ "is expired or untrusted. No source data was accepted.")}
1267
+ if isinstance(exc, xmlrpc.client.Fault) and any(
1268
+ token in text for token in ("accesserror", "access denied", "security restrictions")):
1269
+ return {"stage": "source", "code": "source_read_refused", "type": exc_type,
1270
+ "message": "Odoo refused the configured account's read access."}
1271
+ if any(token in text for token in ("authentication", "access denied", "invalid api key")):
1272
+ return {"stage": "source", "code": "source_auth_failed", "type": exc_type,
1273
+ "message": "Odoo rejected the configured read-only credential."}
1274
+ if isinstance(exc, (TimeoutError, ConnectionError)) or any(
1275
+ token in text for token in ("timed out", "timeout", "name resolution", "connection refused")):
1276
+ return {"stage": "source", "code": "source_unavailable", "type": exc_type,
1277
+ "message": "Odoo could not be reached for this read-only sync."}
1278
+ labels = {
1279
+ "source": ("source_read_failed", "Odoo source validation failed."),
1280
+ "mirrorPull": ("mirror_pull_failed", "The Odoo mirror pull failed."),
1281
+ "materialization": ("materialization_failed", "The Odoo database could not be materialized."),
1282
+ "reconciliation": ("reconciliation_failed", "The Odoo database reconciliation failed."),
1283
+ "freshness": ("freshness_unconfirmed", "Freshness could not be confirmed for this database."),
1284
+ }
1285
+ code, message = labels[stage]
1286
+ return {"stage": stage, "code": code, "type": exc_type, "message": message}
1287
+
1288
+
1289
+ def _clean_odoo_failure(failure, stage):
1290
+ if failure is None:
1291
+ return None
1292
+ if not isinstance(failure, dict):
1293
+ return _odoo_failure(failure, stage)
1294
+ return {"stage": str(failure.get("stage") or stage),
1295
+ "code": str(failure.get("code") or f"{stage}_failed"),
1296
+ "type": str(failure.get("type") or "SourceError")[:80],
1297
+ "message": str(failure.get("message") or "Odoo sync failed.")[:300]}
1298
+
1299
+
1300
+ def _odoo_stage_failure(stages):
1301
+ """Return the earliest unresolved lifecycle failure, preserving its named cause."""
1302
+ return next((stages.get(name, {}).get("failure") for name in _ODOO_HEALTH_STAGES
1303
+ if stages.get(name, {}).get("state") == "failed"
1304
+ and stages.get(name, {}).get("failure")), None)
1305
+
1306
+
1307
+ def _clean_odoo_database_receipt(receipt, database_key):
1308
+ """Allow-list one data-free reconciliation receipt for durable health storage."""
1309
+ if not isinstance(receipt, dict):
1310
+ return None
1311
+ key = str(database_key or "").strip()
1312
+ if str(receipt.get("databaseKey") or "").strip() != key:
1313
+ raise ValueError("the reconciliation receipt belongs to a different database")
1314
+ out = {"databaseKey": key}
1315
+ for name in ("schemaKey", "topicKey", "domainKey", "model", "countModel", "state",
1316
+ "sourceCountMode", "sourceReconciliation"):
1317
+ value = receipt.get(name)
1318
+ if value is not None:
1319
+ out[name] = str(value)[:160]
1320
+ for name in ("sourceCount", "mirrorCount", "materializedCount", "canonicalCount",
1321
+ "servingCount", "duplicateCount", "explainedMergeCount",
1322
+ "unexplainedMissingCount", "unexplainedExtraCount",
1323
+ "unexplainedCanonicalMissingCount", "unexplainedCanonicalExtraCount",
1324
+ "unexplainedCount"):
1325
+ value = receipt.get(name)
1326
+ out[name] = None if value is None else max(0, int(value))
1327
+ if "readThrough" in receipt:
1328
+ out["readThrough"] = bool(receipt.get("readThrough"))
1329
+ selection = receipt.get("selection")
1330
+ if isinstance(selection, dict):
1331
+ out["selection"] = {
1332
+ name: (bool(selection.get(name)) if name == "enabled"
1333
+ else str(selection.get(name) or "")[:160])
1334
+ for name in ("bucket", "rule", "enabled") if name in selection}
1335
+ domain = receipt.get("domain")
1336
+ if isinstance(domain, list):
1337
+ # Curated domains contain public field/operator/value predicates. Refuse nested objects,
1338
+ # which could otherwise turn this metadata seam into durable source-row storage.
1339
+ clean_domain = []
1340
+ for predicate in domain:
1341
+ if not isinstance(predicate, (list, tuple)) or len(predicate) != 3:
1342
+ continue
1343
+ field, operator, value = predicate
1344
+ if isinstance(value, (list, tuple)):
1345
+ clean_value = [item for item in value
1346
+ if isinstance(item, (str, int, float, bool))][:50]
1347
+ elif isinstance(value, (str, int, float, bool)) or value is None:
1348
+ clean_value = value
1349
+ else:
1350
+ continue
1351
+ clean_domain.append([str(field)[:100], str(operator)[:40], clean_value])
1352
+ out["domain"] = clean_domain
1353
+ reconciliation = receipt.get("reconciliation")
1354
+ if isinstance(reconciliation, dict):
1355
+ out["reconciliation"] = {
1356
+ name: str(reconciliation.get(name) or "")[:80]
1357
+ for name in ("sourceToMirror", "mirrorToMaterialized", "mirrorToCanonical")
1358
+ if name in reconciliation}
1359
+ freshness = receipt.get("freshness")
1360
+ if isinstance(freshness, dict):
1361
+ out["freshness"] = {
1362
+ name: str(freshness.get(name) or "")[:100]
1363
+ for name in ("state", "capturedAt") if name in freshness}
1364
+ for name in ("sourceError", "error"):
1365
+ if receipt.get(name) is not None:
1366
+ out[name] = _clean_odoo_failure(receipt.get(name), "reconciliation")
1367
+ return out
1368
+
1369
+
1370
+ def record_odoo_stage(rt, connection_id, stage, state, *, database_key=None, row_count=None,
1371
+ refreshed_at=None, cadence_seconds=None, failure=None, receipt=None,
1372
+ at=None):
1373
+ """Persist one data-free Odoo stage transition for one connection/database.
1374
+
1375
+ This is additive observability over the established ``connector_sync`` and materialization
1376
+ job documents. It stores identifiers, counts, timestamps, and classified failures only;
1377
+ credentials and source rows never cross this seam.
1378
+ """
1379
+ if stage not in _ODOO_HEALTH_STAGES:
1380
+ raise ValueError(f"unknown Odoo health stage: {stage}")
1381
+ if state not in ("unknown", "queued", "running", "ready", "failed", "paused"):
1382
+ raise ValueError(f"unknown Odoo health state: {state}")
1383
+ cid = _odoo_connection_slot(rt, connection_id)
1384
+ stamp = str(at or datetime.now(timezone.utc).isoformat())
1385
+ failed = _clean_odoo_failure(failure, stage) if state == "failed" else None
1386
+ db_key = str(database_key or "").strip()
1387
+ safe_receipt = _clean_odoo_database_receipt(receipt, db_key) if receipt is not None else None
1388
+
1389
+ def _update(current):
1390
+ document = dict(current or {})
1391
+ connections = dict(document.get("connections") or {})
1392
+ connection = dict(connections.get(cid) or {})
1393
+ stages = dict(connection.get("stages") or {})
1394
+ target_stages = stages
1395
+ database = None
1396
+ databases = None
1397
+ if db_key:
1398
+ databases = dict(connection.get("databases") or {})
1399
+ database = dict(databases.get(db_key) or {"databaseKey": db_key})
1400
+ target_stages = dict(database.get("stages") or {})
1401
+ previous = dict(target_stages.get(stage) or {})
1402
+ stage_row = {**previous, "state": state, "at": stamp, "failure": failed}
1403
+ if state == "running":
1404
+ stage_row["startedAt"] = stamp
1405
+ stage_row["finishedAt"] = None
1406
+ elif state in ("ready", "failed", "paused"):
1407
+ stage_row["finishedAt"] = stamp
1408
+ if state == "ready":
1409
+ stage_row["lastSuccess"] = stamp
1410
+ target_stages[stage] = stage_row
1411
+ connection.update({"connectionId": cid, "stages": stages, "lastAttempt": stamp})
1412
+ if cadence_seconds is not None:
1413
+ connection["cadenceSeconds"] = int(cadence_seconds)
1414
+
1415
+ if db_key:
1416
+ database.update({"databaseKey": db_key, "stages": target_stages,
1417
+ "lastAttempt": stamp})
1418
+ if row_count is not None:
1419
+ database["rowCount"] = max(0, int(row_count))
1420
+ if refreshed_at:
1421
+ database["recordRefreshed"] = str(refreshed_at)
1422
+ if safe_receipt is not None:
1423
+ database["reconciliationReceipt"] = safe_receipt
1424
+ db_failure = _odoo_stage_failure(target_stages)
1425
+ if db_failure:
1426
+ database["failure"] = db_failure
1427
+ else:
1428
+ database.pop("failure", None)
1429
+ if state == "ready" and stage == "freshness":
1430
+ database["lastSuccess"] = stamp
1431
+ databases[db_key] = database
1432
+ connection["databases"] = databases
1433
+ else:
1434
+ stages[stage] = stage_row
1435
+ connection["stages"] = stages
1436
+ connection_failure = _odoo_stage_failure(stages)
1437
+ if connection_failure:
1438
+ connection["failure"] = connection_failure
1439
+ else:
1440
+ connection.pop("failure", None)
1441
+ if state == "ready" and stage == "freshness":
1442
+ connection["lastSuccess"] = stamp
1443
+ connections[cid] = connection
1444
+ document["connections"] = connections
1445
+ return document
1446
+
1447
+ rt.update(_ODOO_HEALTH_STATUS, _update, flush="sync")
1448
+ return odoo_health_receipt(rt, cid)
1449
+
1450
+
1451
+ def _lag_seconds(stamp, now=None):
1452
+ if not stamp:
1453
+ return None
1454
+ try:
1455
+ parsed = datetime.fromisoformat(str(stamp).replace("Z", "+00:00"))
1456
+ if parsed.tzinfo is None:
1457
+ parsed = parsed.replace(tzinfo=timezone.utc)
1458
+ current = now or datetime.now(timezone.utc)
1459
+ return max(0, int((current - parsed.astimezone(timezone.utc)).total_seconds()))
1460
+ except (TypeError, ValueError):
1461
+ return None
1462
+
1463
+
1464
+ def _empty_odoo_stages():
1465
+ return {stage: {"state": "unknown", "at": None, "startedAt": None,
1466
+ "finishedAt": None, "lastSuccess": None, "failure": None}
1467
+ for stage in _ODOO_HEALTH_STAGES}
1468
+
1469
+
1470
+ def odoo_health_receipt(rt, connection_id=None, *, now=None):
1471
+ """Project the truthful Odoo receipt for one connection and each of its databases."""
1472
+ cid = _odoo_connection_slot(rt, connection_id)
1473
+ try:
1474
+ document = rt.get(_ODOO_HEALTH_STATUS, fresh=True) or {}
1475
+ except TypeError:
1476
+ document = rt.get(_ODOO_HEALTH_STATUS) or {}
1477
+ except Exception: # noqa: BLE001
1478
+ document = {}
1479
+ stored = dict(((document.get("connections") or {}).get(cid)) or {})
1480
+ stages = _empty_odoo_stages()
1481
+ stages.update({key: dict(value) for key, value in (stored.get("stages") or {}).items()
1482
+ if key in stages and isinstance(value, dict)})
1483
+ databases = {str(key): dict(value) for key, value in (stored.get("databases") or {}).items()
1484
+ if isinstance(value, dict)}
1485
+
1486
+ # Durable materialization receipts are the authority for database work and survive restarts.
1487
+ for job in connector_sync.jobs(rt, "odoo", cid):
1488
+ key = str(job.get("databaseKey") or "")
1489
+ if not key:
1490
+ continue
1491
+ database = dict(databases.get(key) or {"databaseKey": key})
1492
+ db_stages = _empty_odoo_stages()
1493
+ db_stages.update({name: dict(value) for name, value in (database.get("stages") or {}).items()
1494
+ if name in db_stages and isinstance(value, dict)})
1495
+ job_state = str(job.get("state") or "unknown")
1496
+ material_state = {"configured": "queued", "catalogue-ready": "queued",
1497
+ "queued": "queued", "syncing": "running", "ready": "ready",
1498
+ "failed": "failed"}.get(job_state, "unknown")
1499
+ job_at = None
1500
+ if job.get("at") is not None:
1501
+ try:
1502
+ job_at = datetime.fromtimestamp(float(job["at"]), timezone.utc).isoformat()
1503
+ except (TypeError, ValueError, OSError):
1504
+ job_at = None
1505
+ job_failure = (_odoo_failure(job.get("error"), "materialization")
1506
+ if material_state == "failed" else None)
1507
+ db_stages["materialization"] = {
1508
+ **db_stages["materialization"], "state": material_state,
1509
+ "at": job_at, "finishedAt": job_at if material_state in ("ready", "failed") else None,
1510
+ "lastSuccess": (job_at if material_state == "ready" else
1511
+ db_stages["materialization"].get("lastSuccess")),
1512
+ "failure": job_failure,
1513
+ }
1514
+ progress = job.get("progress") if isinstance(job.get("progress"), dict) else {}
1515
+ if progress.get("rows") is not None:
1516
+ database["rowCount"] = max(0, int(progress["rows"]))
1517
+ database.update({"databaseKey": key, "model": str(job.get("model") or ""),
1518
+ "state": material_state, "stages": db_stages})
1519
+ persisted_failure = database.get("failure") or _odoo_stage_failure(db_stages)
1520
+ if persisted_failure or job_failure:
1521
+ database["failure"] = persisted_failure or job_failure
1522
+ elif material_state == "ready" and not database.get("failure"):
1523
+ database["lastSuccess"] = database.get("lastSuccess") or job_at
1524
+ databases[key] = database
1525
+
1526
+ # The resident mirror state is process-local by design; overlay it only on the connection
1527
+ # that owns this tenant's mirror. A named sibling must never inherit default's fresh badge.
1528
+ cadence = stored.get("cadenceSeconds")
1529
+ try:
1530
+ _main = running_main() # D-781: resolve, never import
1531
+ resident = dict(_main.RESYNC_STATE)
1532
+ resident_cid = str(resident.get("connectionId") or "")
1533
+ if resident_cid == cid:
1534
+ cadence = resident.get("resolvedSeconds") or cadence
1535
+ mirror_failure = resident.get("lastError")
1536
+ mirror_state = ("paused" if resident.get("manual") else
1537
+ "failed" if resident.get("lastPassOk") is False else
1538
+ "ready" if resident.get("lastPassOk") is True else "unknown")
1539
+ stages["mirrorPull"] = {**stages["mirrorPull"], "state": mirror_state,
1540
+ "at": resident.get("lastPassAt"),
1541
+ "finishedAt": resident.get("lastPassAt"),
1542
+ "lastSuccess": resident.get("lastSuccessAt"),
1543
+ "failure": (_clean_odoo_failure(mirror_failure, "mirrorPull")
1544
+ if mirror_failure else None)}
1545
+ except Exception: # noqa: BLE001
1546
+ pass
1547
+
1548
+ for database in databases.values():
1549
+ db_stages = _empty_odoo_stages()
1550
+ db_stages.update({name: dict(value) for name, value in (database.get("stages") or {}).items()
1551
+ if name in db_stages and isinstance(value, dict)})
1552
+ # Connection source/mirror/reconciliation facts apply to each database, while its own
1553
+ # materialization/freshness facts remain independent.
1554
+ for stage in ("source", "mirrorPull", "reconciliation"):
1555
+ if db_stages[stage].get("state") == "unknown":
1556
+ db_stages[stage] = dict(stages[stage])
1557
+ database["stages"] = db_stages
1558
+ success = database.get("lastSuccess") or db_stages["freshness"].get("lastSuccess")
1559
+ database["lastSuccess"] = success
1560
+ database["currentLagSeconds"] = _lag_seconds(success, now)
1561
+ database.setdefault("rowCount", None)
1562
+ database.setdefault("recordRefreshed", None)
1563
+ stage_failure = _odoo_stage_failure(db_stages)
1564
+ if stage_failure:
1565
+ database["failure"] = stage_failure
1566
+ elif not database.get("failure"):
1567
+ database["failure"] = None
1568
+
1569
+ successes = [row.get("lastSuccess") for row in databases.values() if row.get("lastSuccess")]
1570
+ last_success = max(successes, default=stored.get("lastSuccess"))
1571
+ row_counts = [row.get("rowCount") for row in databases.values()
1572
+ if row.get("rowCount") is not None]
1573
+ failure = (_odoo_stage_failure(stages) or stored.get("failure") or next(
1574
+ (row.get("failure") for row in databases.values() if row.get("failure")), None))
1575
+ return {"connectionId": cid, "cadenceSeconds": cadence,
1576
+ "lastSuccess": last_success, "currentLagSeconds": _lag_seconds(last_success, now),
1577
+ "rowCount": sum(row_counts) if row_counts else None,
1578
+ "failure": failure, "stages": stages,
1579
+ "databases": [databases[key] for key in sorted(databases)]}
1580
 
1581
 
1582
  def _odoo_admin(session):
 
1593
  """A status endpoint selects one connection; the jobs inventory may enumerate them all."""
1594
  cid = str(connection_id or "").strip() or "default"
1595
  if provider == "odoo" and cid == "default":
1596
+ cid = _odoo_connection_slot(rt, connection_id)
 
1597
  return connector_sync.jobs(rt, provider, cid)
1598
 
1599
 
 
1614
  return _safe_connector_status(connector_key, answer)
1615
 
1616
 
1617
+ def _odoo_result_database_failures(rt, connection_id, result):
1618
+ """Map provider-reported model/table failures to their durable database keys."""
1619
+ import odoo_relational as rel
1620
+ try:
1621
+ choices = rel.grid_choices(rt, connection_id)
1622
+ except Exception: # noqa: BLE001
1623
+ choices = []
1624
+ by_model = {str(row.get("model") or ""): str(row.get("key") or "")
1625
+ for row in choices if row.get("model") and row.get("key")}
1626
+ failures = {str(key): _clean_odoo_failure(value, "materialization")
1627
+ for key, value in (result.get("databaseErrors") or {}).items()}
1628
+ receipt_groups = []
1629
+ for name in ("databases", "canonicalDatabases"):
1630
+ group = result.get(name) if isinstance(result, dict) else None
1631
+ if isinstance(group, dict):
1632
+ receipt_groups.append(group)
1633
+ for group in receipt_groups:
1634
+ for raw_key, receipt in group.items():
1635
+ if not isinstance(receipt, dict) or receipt.get("state") == "succeeded":
1636
+ continue
1637
+ key = str(raw_key or "")
1638
+ if not key or key in failures:
1639
+ continue
1640
+ raw_failure = receipt.get("error") or receipt.get("sourceError") or {
1641
+ "stage": "reconciliation", "code": "reconciliation_incomplete",
1642
+ "type": "ReconciliationError",
1643
+ "message": "The Odoo database reconciliation is incomplete.",
1644
+ }
1645
+ failures[key] = _clean_odoo_failure(raw_failure, "reconciliation")
1646
+ unmatched = []
1647
+ discovered = result.get("discovered") if isinstance(result, dict) else {}
1648
+ for raw in ((discovered or {}).get("errors") or ()):
1649
+ model = str(raw).split(":", 1)[0].strip()
1650
+ key = by_model.get(model)
1651
+ if key:
1652
+ failures[key] = _odoo_failure(raw, "materialization")
1653
+ else:
1654
+ unmatched.append(str(raw))
1655
+ limits = result.get("limits") if isinstance(result, dict) else None
1656
+ if isinstance(limits, dict):
1657
+ for raw_key, raw in limits.items():
1658
+ key = by_model.get(str(raw_key or ""), str(raw_key or ""))
1659
+ if key:
1660
+ failures[key] = _odoo_failure(raw, "materialization")
1661
+ elif limits:
1662
+ unmatched.append(str(limits))
1663
+ return failures, unmatched
1664
+
1665
+
1666
  def _odoo_sync_runner(session, connection_id):
1667
  """Run Odoo through C6 without moving provider imports into the data layer."""
1668
+ cid = _odoo_connection_slot(session.runtime, connection_id)
1669
+ connector_sync.record(session.runtime, "odoo", cid, "syncing")
1670
+ record_odoo_stage(session.runtime, cid, "source", "running")
1671
+ record_odoo_stage(session.runtime, cid, "materialization", "queued")
1672
  import odoo_relational
1673
+ def _materializing():
1674
+ connector_sync.record(session.runtime, "odoo", cid, "materializing")
1675
+ record_odoo_stage(session.runtime, cid, "materialization", "running")
1676
+ try:
1677
+ result = odoo_relational.refresh(session.runtime, session.tenant,
1678
+ username=session.uname or "automation",
1679
+ connection_id=connection_id,
1680
+ on_apply=_materializing)
1681
+ except Exception as exc:
1682
+ failure = _odoo_failure(exc, "source")
1683
+ record_odoo_stage(session.runtime, cid, failure["stage"], "failed", failure=failure)
1684
+ record_odoo_stage(session.runtime, cid, "freshness", "failed",
1685
+ failure={**failure, "stage": "freshness"})
1686
+ connector_sync.record(session.runtime, "odoo", cid, "error",
1687
+ errors=[f"{failure['code']}: {failure['message']}"])
1688
+ raise
1689
  tables = result.get("tables") if isinstance(result, dict) else {}
1690
+ published = dict(tables or {})
1691
+ database_receipts = result.get("databases") if isinstance(result, dict) else {}
1692
+ database_receipts = database_receipts if isinstance(database_receipts, dict) else {}
1693
+ canonical_receipts = result.get("canonicalDatabases") if isinstance(result, dict) else {}
1694
+ canonical_receipts = canonical_receipts if isinstance(canonical_receipts, dict) else {}
1695
+ reconciliation_receipts = {**database_receipts, **canonical_receipts}
1696
+ databases = sorted(str(key) for key in published)
1697
  sync_errors = []
1698
  if isinstance(result, dict):
1699
  if result.get("failed"):
 
1703
  sync_errors.extend(str(value) for value in (discovered.get("errors") or ()))
1704
  discovered_tables = discovered.get("tables")
1705
  if isinstance(discovered_tables, dict):
1706
+ published.update(discovered_tables)
1707
  databases.extend(str(key) for key in discovered_tables)
1708
  if result.get("frozen"):
1709
  sync_errors.append(str(result.get("note") or "the connector is frozen"))
1710
+ database_failures, unmatched_errors = _odoo_result_database_failures(
1711
+ session.runtime, connection_id, result if isinstance(result, dict) else {})
1712
+ databases.extend(reconciliation_receipts)
1713
+ databases.extend(database_failures)
1714
+ sync_errors.extend(f"{key}: {failure['code']}" for key, failure in database_failures.items())
1715
+ sync_errors.extend(raw for raw in unmatched_errors if raw not in sync_errors)
1716
+ state = ("partial" if published else "error") if sync_errors else "ready"
1717
+ record_odoo_stage(session.runtime, cid, "source", "ready")
1718
+ refreshed_at = datetime.now(timezone.utc).isoformat()
1719
+ database_keys = list(dict.fromkeys([*reconciliation_receipts, *published]))
1720
+ for key in database_keys:
1721
+ if key in database_failures:
1722
+ continue
1723
+ counts = published.get(key)
1724
+ receipt = reconciliation_receipts.get(key)
1725
+ receipt = receipt if isinstance(receipt, dict) else None
1726
+ nrows = (receipt.get("servingCount") if receipt is not None
1727
+ else counts.get("rows") if isinstance(counts, dict) else None)
1728
+ record_odoo_stage(session.runtime, cid, "source", "ready", database_key=key,
1729
+ receipt=receipt)
1730
+ record_odoo_stage(session.runtime, cid, "materialization", "ready",
1731
+ database_key=key, row_count=nrows, receipt=receipt)
1732
+ record_odoo_stage(session.runtime, cid, "reconciliation", "ready",
1733
+ database_key=key, row_count=nrows, receipt=receipt)
1734
+ record_odoo_stage(session.runtime, cid, "freshness", "ready",
1735
+ database_key=key, row_count=nrows, refreshed_at=refreshed_at,
1736
+ receipt=receipt)
1737
+ for key, failure in database_failures.items():
1738
+ receipt = reconciliation_receipts.get(key)
1739
+ receipt = receipt if isinstance(receipt, dict) else None
1740
+ failed_stage = str(failure.get("stage") or "materialization")
1741
+ failed_stage = failed_stage if failed_stage in _ODOO_HEALTH_STAGES else "materialization"
1742
+ record_odoo_stage(session.runtime, cid, failed_stage, "failed",
1743
+ database_key=key, failure=failure, receipt=receipt)
1744
+ record_odoo_stage(session.runtime, cid, "freshness", "failed", database_key=key,
1745
+ failure={**failure, "stage": "freshness"}, receipt=receipt)
1746
+ if sync_errors:
1747
+ failure = _odoo_failure("provider result contained errors", "materialization")
1748
+ record_odoo_stage(session.runtime, cid, "materialization", "failed", failure=failure)
1749
+ record_odoo_stage(session.runtime, cid, "freshness", "failed",
1750
+ failure={**failure, "stage": "freshness"})
1751
+ else:
1752
+ record_odoo_stage(session.runtime, cid, "materialization", "ready")
1753
+ record_odoo_stage(session.runtime, cid, "reconciliation", "ready")
1754
+ record_odoo_stage(session.runtime, cid, "freshness", "ready")
1755
  return connector_sync.record(
1756
+ session.runtime, "odoo", cid, state,
1757
  detail=str(result.get("note") or "") if isinstance(result, dict) else "",
1758
  errors=sync_errors, databases=sorted(set(databases)),
1759
  )
 
1867
  return
1868
  fenced = _MaterializationRuntime(rt, job)
1869
  key = job["databaseKey"]
1870
+ cid = job["connectionId"]
1871
+ record_odoo_stage(rt, cid, "source", "running", database_key=key)
1872
+ record_odoo_stage(rt, cid, "materialization", "running", database_key=key)
1873
  def finish(success=False, cause=""):
1874
  # The pre-claim check cannot cover a disconnect during source I/O or publication.
1875
  # Only the current lease may requeue; disable and a newer Save still win.
 
1929
  fenced, job, rows=rows, total=total))}
1930
  discovered = result.get("discovered") or {}
1931
  published = {**(result.get("tables") or {}), **(discovered.get("tables") or {})}
1932
+ receipt = (result.get("databases") or {}).get(key)
1933
+ receipt = receipt if isinstance(receipt, dict) else None
1934
+ database_errors = result.get("databaseErrors") or {}
1935
  success = (key in published and not result.get("failed") and not result.get("frozen")
1936
+ and not result.get("limits") and not discovered.get("errors")
1937
+ and key not in database_errors
1938
+ and (receipt is None or receipt.get("state") == "succeeded"))
1939
+ counts = published.get(key) if isinstance(published.get(key), dict) else {}
1940
+ nrows = (receipt.get("servingCount") if receipt is not None
1941
+ else counts.get("rows") if isinstance(counts, dict) else None)
1942
+ if success:
1943
+ refreshed_at = datetime.now(timezone.utc).isoformat()
1944
+ record_odoo_stage(rt, cid, "source", "ready", database_key=key,
1945
+ receipt=receipt)
1946
+ record_odoo_stage(rt, cid, "materialization", "ready",
1947
+ database_key=key, row_count=nrows, receipt=receipt)
1948
+ record_odoo_stage(rt, cid, "reconciliation", "ready",
1949
+ database_key=key, row_count=nrows, receipt=receipt)
1950
+ record_odoo_stage(rt, cid, "freshness", "ready", database_key=key,
1951
+ row_count=nrows, refreshed_at=refreshed_at, receipt=receipt)
1952
+ finish(success=True)
1953
+ else:
1954
+ raw_failure = ((database_errors.get(key) if isinstance(database_errors, dict) else None)
1955
+ or (receipt or {}).get("error") or (receipt or {}).get("sourceError"))
1956
+ failure = (_clean_odoo_failure(raw_failure, "reconciliation") if raw_failure
1957
+ else _odoo_failure("provider result was incomplete", "materialization"))
1958
+ failed_stage = str(failure.get("stage") or "materialization")
1959
+ failed_stage = (failed_stage if failed_stage in _ODOO_HEALTH_STAGES
1960
+ else "materialization")
1961
+ record_odoo_stage(rt, cid, failed_stage, "failed",
1962
+ database_key=key, failure=failure, receipt=receipt)
1963
+ record_odoo_stage(rt, cid, "freshness", "failed", database_key=key,
1964
+ failure={**failure, "stage": "freshness"}, receipt=receipt)
1965
+ finish(cause=f"{failure['code']}: {failure['message']}")
1966
  except _SupersededMaterialization:
1967
  # Disable/new Save already invalidated the table marker; never restore it here.
1968
  finish()
1969
  except rel.Refused as exc:
1970
+ failure = _odoo_failure(exc, "materialization")
1971
+ record_odoo_stage(rt, cid, failure["stage"], "failed",
1972
+ database_key=key, failure=failure)
1973
+ record_odoo_stage(rt, cid, "freshness", "failed", database_key=key,
1974
+ failure={**failure, "stage": "freshness"})
1975
+ finish(cause=f"{failure['code']}: {failure['message']}")
1976
  except Exception as exc:
1977
+ # Raw provider exceptions may carry credentials or source values. Store only a named,
1978
+ # data-free classification, including the fail-closed TLS case.
1979
+ failure = _odoo_failure(exc, "source")
1980
  if _odoo_read_access_refused(exc):
1981
+ failure = {"stage": "source", "code": "source_read_refused",
1982
+ "type": type(exc).__name__,
1983
+ "message": (f"Odoo refused read access to {job['model']}. Give the "
1984
+ "connected account read access, or switch this database off.")}
1985
+ record_odoo_stage(rt, cid, failure["stage"], "failed",
1986
+ database_key=key, failure=failure)
1987
+ record_odoo_stage(rt, cid, "freshness", "failed", database_key=key,
1988
+ failure={**failure, "stage": "freshness"})
1989
+ finish(cause=f"{failure['code']}: {failure['message']}")
1990
  finally:
1991
  stopped.set()
1992
  if heartbeat_thread:
 
2217
  def odoo_status(connection_id: str = None, session: Session = Depends(require_session)):
2218
  _odoo_admin(session)
2219
  import odoo_relational
2220
+ resolved_connection = _odoo_connection_slot(session.runtime, connection_id)
2221
  cfg = odoo_relational.read_config(session.runtime)
2222
  stored = session.runtime.get(_ODOO_SYNC_STATUS) or {}
2223
  errors = stored.get("errors") if isinstance(stored.get("errors"), list) else []
2224
  phase = str(stored.get("phase") or "unknown")
2225
+ lifecycle = connector_sync.read(session.runtime, "odoo", resolved_connection)
2226
  durable = connector_sync.job_summary(_connection_jobs(session.runtime, "odoo", connection_id))
2227
+ receipt = odoo_health_receipt(session.runtime, resolved_connection)
2228
  if durable:
2229
  return {"schedule": cfg.get("syncEvery"),
2230
  "health": "paused" if cfg.get("frozen") else durable["state"],
2231
  "lastRun": durable["at"], "frozen": cfg.get("frozen") is True,
2232
+ **durable, "stateAt": durable["at"], "connectionId": resolved_connection,
2233
+ "receipt": receipt}
2234
  # The old keys stay the old default-connection answer. A caller that names a connection gets
2235
  # that slot's lifecycle instead of the tenant-wide legacy document that caused the C6 defect.
2236
  named = connection_id is not None
 
2254
  "retryable": state in ("error", "partial"),
2255
  "errorsTotal": int(errors_total or 0),
2256
  "connectionId": (lifecycle or {}).get("connectionId")
2257
+ or resolved_connection,
2258
+ "receipt": receipt}
2259
 
2260
 
2261
  @router.get("/connectors/{connector_key}/sync")
 
2271
 
2272
 
2273
  @router.post("/connectors/{connector_key}/sync/now")
2274
+ def connector_sync_now(connector_key: str, connection_id: str = None,
2275
+ session: Session = Depends(require_session)):
2276
  _sync_admin(session)
2277
  if connector_key not in connector_sync.providers():
2278
  raise err(404, "sync_provider_unknown", "This connector is not registered for sync.")
2279
  try:
2280
+ requested_connection = (_odoo_connection_slot(session.runtime, connection_id)
2281
+ if connector_key == "odoo" else
2282
+ str(connection_id or "default").strip() or "default")
2283
+ key = (connector_key, session.runtime.key)
2284
+ with _SYNC_REQUEST_CONNECTION_LOCK:
2285
+ # Bind under one API-side lock so a duplicate request cannot race ahead of the
2286
+ # accepted request and attach its different connection id to the queued intent.
2287
+ receipt = connector_sync.request_sync(connector_key, session.runtime.key,
2288
+ requested_by=session.uname)
2289
+ if receipt.get("accepted") or key not in _SYNC_REQUEST_CONNECTIONS:
2290
+ _SYNC_REQUEST_CONNECTIONS[key] = requested_connection
2291
+ receipt["connectionId"] = _SYNC_REQUEST_CONNECTIONS[key]
2292
+ return receipt
2293
  except ValueError:
2294
  raise err(404, "sync_provider_unknown", "This connector is not registered for sync.") from None
2295
 
 
2299
  _sync_admin(session)
2300
  if connector_key not in connector_sync.providers():
2301
  raise err(404, "sync_provider_unknown", "This connector is not registered for sync.")
2302
+ receipt = connector_sync.sync_request(connector_key, session.runtime.key)
2303
+ if receipt is not None:
2304
+ receipt["connectionId"] = sync_request_connection(connector_key, session.runtime.key)
2305
+ return receipt
2306
+ return {
2307
  "provider": connector_key, "tenant": session.runtime.key, "state": "idle",
2308
  "requestedAt": None, "startedAt": None, "finishedAt": None,
2309
  "requestedBy": "", "detail": "", "accepted": False,
 
2353
  _sync_admin(session)
2354
  if connector_key not in connector_sync.providers():
2355
  raise err(404, "sync_provider_unknown", "This connector has no database sync runner.")
2356
+ payload = {"provider": connector_key, "jobs": connector_sync.jobs(
2357
+ session.runtime, connector_key, connection_id)}
2358
+ if connector_key == "odoo":
2359
+ payload["receipt"] = odoo_health_receipt(session.runtime, connection_id)
2360
+ return _safe_connector_status(connector_key, payload)
2361
 
2362
 
2363
  @router.get("/connectors/odoo/schema")
 
2837
  return payload
2838
 
2839
 
2840
+ def _sync_stamp(value):
2841
+ """Return a parseable public timestamp, never a creation-time freshness substitute."""
2842
+ text = str(value or "").strip()
2843
+ if not text:
2844
+ return None
2845
+ try:
2846
+ parsed = datetime.fromisoformat(text.replace("Z", "+00:00"))
2847
+ if parsed.tzinfo is None:
2848
+ parsed = parsed.replace(tzinfo=timezone.utc)
2849
+ return parsed.astimezone(timezone.utc).isoformat()
2850
+ except (TypeError, ValueError, OverflowError):
2851
+ return None
2852
+
2853
+
2854
+ def _database_sync_health(*, enabled=True, state="never_run", last_success=None,
2855
+ source_timestamp=None, materialized_timestamp=None, row_count=None,
2856
+ error=None, now=None):
2857
+ """Normalize one database's durable facts into S63-T34's shared public vocabulary."""
2858
+ clean_state = str(state or "never_run")
2859
+ if clean_state not in DATABASE_SYNC_STATES:
2860
+ clean_state = "failed"
2861
+ error = {"code": "sync_state_unreadable",
2862
+ "message": "This database's saved sync state could not be read."}
2863
+ if not enabled:
2864
+ clean_state = "disabled"
2865
+ count = row_count if type(row_count) is int and row_count >= 0 else None
2866
+ success = _sync_stamp(last_success)
2867
+ source = _sync_stamp(source_timestamp)
2868
+ materialized = _sync_stamp(materialized_timestamp)
2869
+ safe_error = None
2870
+ if error is not None:
2871
+ raw = error if isinstance(error, dict) else {}
2872
+ requested_code = str(raw.get("code") or "")
2873
+ code = requested_code if requested_code in _DATABASE_SAFE_ERRORS else "connector_sync_failed"
2874
+ safe_error = {
2875
+ "code": code,
2876
+ # Raw provider and exception text never crosses this boundary.
2877
+ "message": _DATABASE_SAFE_ERRORS.get(code, "This database could not be synced."),
2878
+ }
2879
+ # A current failure may retain the prior success timestamp, but it can never look current.
2880
+ lag = _lag_seconds(success, now) if success else None
2881
+ return {"state": clean_state, "lastSuccess": success,
2882
+ "sourceTimestamp": source, "materializedTimestamp": materialized,
2883
+ "rowCount": count, "lagSeconds": lag, "error": safe_error}
2884
+
2885
+
2886
+ def _odoo_database_sync_health(raw, enabled=True, *, now=None):
2887
+ row = raw if isinstance(raw, dict) else {}
2888
+ stages = row.get("stages") if isinstance(row.get("stages"), dict) else {}
2889
+ failure = row.get("failure") if isinstance(row.get("failure"), dict) else None
2890
+ raw_state = str(row.get("state") or "unknown")
2891
+ stage_states = {str((value or {}).get("state") or "unknown")
2892
+ for value in stages.values() if isinstance(value, dict)}
2893
+ last_success = row.get("lastSuccess")
2894
+ if failure or "failed" in stage_states or raw_state == "failed":
2895
+ state = "partial" if last_success else "failed"
2896
+ elif raw_state == "running" or "running" in stage_states:
2897
+ state = "running"
2898
+ elif raw_state == "queued" or "queued" in stage_states:
2899
+ state = "queued"
2900
+ elif last_success or raw_state == "ready":
2901
+ state = "succeeded"
2902
+ else:
2903
+ state = "never_run"
2904
+ source_stage = stages.get("source") if isinstance(stages.get("source"), dict) else {}
2905
+ material_stage = (stages.get("materialization")
2906
+ if isinstance(stages.get("materialization"), dict) else {})
2907
+ return _database_sync_health(
2908
+ enabled=enabled, state=state, last_success=last_success,
2909
+ source_timestamp=source_stage.get("lastSuccess"),
2910
+ materialized_timestamp=material_stage.get("lastSuccess"),
2911
+ row_count=row.get("rowCount"), error=failure, now=now)
2912
+
2913
+
2914
+ def _amazon_database_sync_health(raw, enabled=True, *, now=None):
2915
+ if not enabled:
2916
+ return _database_sync_health(enabled=False)
2917
+ if not isinstance(raw, dict):
2918
+ return _database_sync_health(
2919
+ state="failed", error={"code": "sync_status_unavailable",
2920
+ "message": "Sync status is unavailable for this database."})
2921
+ success = raw.get("lastSuccess")
2922
+ if raw.get("stuck") is True:
2923
+ state = "partial" if success else "failed"
2924
+ error = {"code": "sync_stalled",
2925
+ "message": "This database sync is stalled."}
2926
+ elif raw.get("inProgress") is True:
2927
+ state, error = "running", None
2928
+ elif success:
2929
+ state, error = "succeeded", None
2930
+ elif raw.get("deferred"):
2931
+ state, error = "queued", None
2932
+ else:
2933
+ state, error = "never_run", None
2934
+ return _database_sync_health(
2935
+ state=state, last_success=success, materialized_timestamp=success,
2936
+ row_count=raw.get("rowsLoaded"), error=error, now=now)
2937
+
2938
+
2939
+ def _stored_database_sync_health(raw, enabled=True, *, now=None):
2940
+ row = raw if isinstance(raw, dict) else {}
2941
+ return _database_sync_health(
2942
+ enabled=enabled, state=row.get("state") or "never_run",
2943
+ last_success=row.get("lastSuccess"), source_timestamp=row.get("sourceTimestamp"),
2944
+ materialized_timestamp=row.get("materializedTimestamp"), row_count=row.get("rowCount"),
2945
+ error=row.get("error"), now=now)
2946
+
2947
+
2948
  def _detail_connect(entry_type, is_admin):
2949
  """C4's `keychain.connect` half: what the Connect form must ask for, and what it may store.
2950
 
 
3328
  return named
3329
 
3330
 
3331
+ def _clean_scraper_recipe(raw):
3332
+ """Return T11's persisted recipe shape and explicit structural validation errors.
3333
+
3334
+ This function is intentionally pure. Preview and creation both call it, so opening a saved
3335
+ recipe cannot reinterpret it differently, and validation cannot accidentally become a paid
3336
+ provider probe.
3337
+ """
3338
+ value = raw if isinstance(raw, dict) else {}
3339
+ kind = str(value.get("kind") or "").strip().lower()
3340
+ errors = []
3341
+ if kind not in SCRAPER_RECIPE_KINDS:
3342
+ errors.append("Choose Instagram, TikTok, or Google Maps.")
3343
+ return {}, errors
3344
+
3345
+ limits = value.get("limits") if isinstance(value.get("limits"), dict) else {}
3346
+ schedule = value.get("schedule") if isinstance(value.get("schedule"), dict) else {}
3347
+ schedule_mode = str(schedule.get("mode") or "manual").strip().lower()
3348
+ if schedule_mode != "manual":
3349
+ errors.append("This recipe must use the manual Run now schedule.")
3350
+
3351
+ if kind == "google_maps":
3352
+ seed = value.get("seed") if isinstance(value.get("seed"), dict) else {}
3353
+ search = str(seed.get("search") or value.get("search") or "").strip()
3354
+ geography = str(seed.get("geography") or value.get("geography") or "").strip()
3355
+ raw_cap = limits.get("records", value.get("limit"))
3356
+ if not search:
3357
+ errors.append("Enter a category or search term.")
3358
+ if not geography:
3359
+ errors.append("Enter a geography.")
3360
+ if raw_cap in (None, ""):
3361
+ errors.append("Enter a result cap.")
3362
+ cap = 0
3363
+ else:
3364
+ try:
3365
+ cap = int(raw_cap)
3366
+ except (TypeError, ValueError):
3367
+ cap = 0
3368
+ errors.append("The result cap must be a whole number.")
3369
+ if raw_cap not in (None, "") and (cap < 1 or cap > 500):
3370
+ errors.append("The result cap must be between 1 and 500.")
3371
+ return ({"kind": kind, "source": "places",
3372
+ "seed": {"search": search, "geography": geography},
3373
+ "limits": {"records": cap}, "schedule": {"mode": "manual"}}, errors)
3374
+
3375
+ seed = str(value.get("seed") or "").strip()
3376
+ raw_limit = limits.get("posts", value.get("limit"))
3377
+ post_type = str(limits.get("postType") or value.get("postType") or "video").strip().lower()
3378
+ if not seed:
3379
+ errors.append(f"Enter an {kind.title()} profile seed.")
3380
+ if raw_limit in (None, ""):
3381
+ errors.append("Enter a last-N post limit.")
3382
+ limit = 0
3383
+ else:
3384
+ try:
3385
+ limit = int(raw_limit)
3386
+ except (TypeError, ValueError):
3387
+ limit = 0
3388
+ errors.append("The post limit must be a whole number.")
3389
+ if raw_limit not in (None, "") and (limit < 1 or limit > SCRAPER_RECIPE_MAX_ROWS):
3390
+ errors.append(f"The post limit must be between 1 and {SCRAPER_RECIPE_MAX_ROWS}.")
3391
+ allowed_types = ("video", "image", "carousel") if kind == "instagram" else ("video",)
3392
+ if post_type not in allowed_types:
3393
+ errors.append("Choose video, image, or carousel posts."
3394
+ if kind == "instagram" else
3395
+ "TikTok's native profile window supports video posts only.")
3396
+ return ({"kind": kind, "source": "profile", "seed": seed,
3397
+ "limits": {"posts": limit, "postType": post_type},
3398
+ "schedule": {"mode": "manual"}}, errors)
3399
+
3400
+
3401
+ def _scraper_recipe_fields(kind):
3402
+ """Stable user-table schema for one recipe, derived from the shipped normalisers."""
3403
+ if kind == "google_maps":
3404
+ return [dict(field) for field in _MAPS_RECIPE_FIELDS]
3405
+ import automation_engine as engine # noqa: PLC0415 - existing schema owner
3406
+ source = engine.POST_FIELDS if kind == "instagram" else engine.TT_POST_FIELDS
3407
+ # Links and rollups target the shared automation tables. A recipe database is independent,
3408
+ # so copying those references would create a plausible but broken relationship graph.
3409
+ return [dict(field) for field in source
3410
+ if isinstance(field, dict) and field.get("type") not in ("link", "rollup")]
3411
+
3412
+
3413
+ def _scraper_plan(recipe):
3414
+ """Network-free setup preview: declared provider, modelled spend, cap, and credential state."""
3415
+ kind = str((recipe or {}).get("kind") or "")
3416
+ limit_key = "records" if kind == "google_maps" else "posts"
3417
+ limit = int(((recipe or {}).get("limits") or {}).get(limit_key) or 0)
3418
+ if kind == "google_maps":
3419
+ import providers # noqa: PLC0415 - registry owns cost/capability
3420
+ apify = providers.PROVIDERS["apify"]
3421
+ capability = apify.cap("google_maps_stores")
3422
+ per_record = float(capability.cost_per_record or 0.0) if capability else 0.0
3423
+ amount = round(per_record * limit, 4)
3424
+ return {"provider": apify.label, "credentialReady": apify.configured(),
3425
+ "estimatedUsd": amount, "costCapUsd": amount, "recordsCap": limit,
3426
+ "canRun": True,
3427
+ "note": "Modelled from the declared per-place rate; the provider receipt "
3428
+ "reconciles the bounded result count."}
3429
+
3430
+ import providers # noqa: PLC0415 - registry is the authority
3431
+ bright = providers.PROVIDERS["brightdata"]
3432
+ apify = providers.PROVIDERS["apify"]
3433
+ if kind == "tiktok":
3434
+ per_record = float(bright.cap("tt_post_metrics").cost_per_record or 0.0)
3435
+ profile_cost = float(bright.cap("tt_profile").cost_per_record or 0.0)
3436
+ estimate_usd = profile_cost + (per_record * limit)
3437
+ provider_label = bright.label
3438
+ credential_ready = bright.configured()
3439
+ else:
3440
+ # The established Instagram runner can use Bright Data and its Apify fallback. The cap
3441
+ # includes both post metrics and view enrichment, so it never hides the worst declared
3442
+ # cost from the person authorising Run now.
3443
+ bd_cost = float(bright.cap("ig_post_metrics").cost_per_record or 0.0)
3444
+ profile_cost = float(bright.cap("ig_profile").cost_per_record or 0.0)
3445
+ view_cost = float(apify.cap("ig_post_views").cost_per_record or 0.0)
3446
+ estimate_usd = profile_cost + ((bd_cost + view_cost) * limit)
3447
+ provider_label = (bright.label if bright.configured() or not apify.configured()
3448
+ else apify.label)
3449
+ credential_ready = bright.configured() or apify.configured()
3450
+ amount = round(estimate_usd, 4)
3451
+ return {"provider": provider_label, "credentialReady": bool(credential_ready),
3452
+ "estimatedUsd": amount, "costCapUsd": amount, "recordsCap": limit,
3453
+ "canRun": True, "note": "Modelled from declared per-record list rates."}
3454
+
3455
+
3456
+ def _scraper_databases(rt, is_admin, uname):
3457
+ """Saved recipe databases this reader may open, with no row payloads or secrets."""
3458
+ try:
3459
+ import core.user_tables as user_tables
3460
+ definitions = user_tables.all_defs(rt)
3461
+ lent = user_tables.lend(rt, **{user_tables.STORE_KEY: definitions})
3462
+ except Exception: # noqa: BLE001
3463
+ return []
3464
+ rows = []
3465
+ for key, definition in (definitions or {}).items():
3466
+ recipe = definition.get("scraperRecipe") if isinstance(definition, dict) else None
3467
+ if not isinstance(recipe, dict):
3468
+ continue
3469
+ try:
3470
+ if not user_tables.may_open(key, uname, bool(is_admin), st=lent):
3471
+ continue
3472
+ except Exception: # noqa: BLE001
3473
+ continue
3474
+ cleaned, errors = _clean_scraper_recipe(recipe)
3475
+ if errors:
3476
+ continue
3477
+ plan = _scraper_plan(cleaned)
3478
+ # Recipe existence alone is not enablement. The creation door stamps both ownership
3479
+ # markers; a legacy/malformed row without them is disclosed as disabled and cannot run.
3480
+ enabled = (definition.get("connectorKey") == "scraper"
3481
+ and definition.get("recordMode") == "automation")
3482
+ rows.append({"key": str(key), "label": str(definition.get("label") or key),
3483
+ "defaultLabel": str(definition.get("label") or key),
3484
+ "native": cleaned["kind"], "enabled": enabled, "exists": True,
3485
+ "canRename": True, "sourceRows": None, "populated": None,
3486
+ "fields": [dict(field) for field in (definition.get("fields") or ())
3487
+ if isinstance(field, dict)],
3488
+ "excluded": [], "sharesFieldsWith": [], "scraperRecipe": cleaned,
3489
+ "syncHealth": _stored_database_sync_health(
3490
+ definition.get(user_tables.CONNECTOR_HEALTH_FIELD), enabled),
3491
+ "scraperRuns": [dict(item) for item in (definition.get("scraperRuns") or ())
3492
+ if isinstance(item, dict)],
3493
+ "runPath": f"/connectors/scraper/databases/{key}/run",
3494
+ "canRun": enabled and plan["canRun"] is True})
3495
+ return _named_databases(rt, is_admin, uname, rows)
3496
+
3497
+
3498
  def _detail_databases(rt, is_admin, connector_key, uname="", connection_id=None):
3499
  """C4's `databases` half: what this connector materialises, and the fields behind each one.
3500
 
 
3523
  reason: what a database is CALLED is a fact about the reader's own workspace. Only the WRITE
3524
  is walled, per row, by `canRename`.
3525
  """
3526
+ if str(connector_key) == "scraper":
3527
+ return _scraper_databases(rt, is_admin, uname)
3528
  if not connector_directory.is_introspectable(connector_key):
3529
  return []
3530
  spec = _connector_spec_tables(connector_key)
 
3541
  census = odoo_relational.population_census(rt, namespace)
3542
  except Exception: # noqa: BLE001
3543
  return []
3544
+ health = odoo_health_receipt(rt, connection_id)
3545
+ health_by_key = {str(item.get("databaseKey") or ""): item
3546
+ for item in (health.get("databases") or ())}
3547
  census_at = str(census.get("capturedAt") or "")
3548
  counts = census.get("counts") if isinstance(census.get("counts"), dict) else {}
3549
  catalog = {}
 
3588
  # Counts are model-level facts. Databases backed by the same model deliberately show the
3589
  # same sourceRows number; inventing a per-database domain here would make the number lie.
3590
  source_rows = counts.get(model) if census_at and model in counts else None
3591
+ database_health = health_by_key.get(key) or {
3592
+ # Source rows is a model census. It is not this governed database's materialized
3593
+ # row count, especially when two curated databases share one source model.
3594
+ "databaseKey": key, "state": "unknown", "rowCount": None,
3595
+ "lastSuccess": None, "currentLagSeconds": None, "recordRefreshed": None,
3596
+ "failure": None, "stages": _empty_odoo_stages()}
3597
  out.append({"key": key,
3598
  "identity": connector_sync.database_identity(
3599
  "odoo", resolved_connection_id, key, model),
 
3603
  "enabled": bool(choice.get("enabled")),
3604
  "sourceRows": source_rows,
3605
  "populated": None if source_rows is None else source_rows > 0,
3606
+ # Data-free client receipt: source/mirror/materialization/reconciliation/
3607
+ # freshness are distinct, and one database's failure cannot make a sibling
3608
+ # look newly successful.
3609
+ "sync": database_health,
3610
+ "syncHealth": _odoo_database_sync_health(
3611
+ database_health, bool(choice.get("enabled"))),
3612
  "fields": scoped_fields,
3613
  "excluded": [dict(item) for item in excluded],
3614
  "sharesFieldsWith": []})
 
3699
  # what `native` carries: the vendor's own name for the thing, never one we invented. It is
3700
  # empty for a table whose report type is unknown, and the tab says so rather than guessing.
3701
  declared = (spec or {}).get(key) or {}
3702
+ raw_sync = (states or {}).get(name) if is_admin else None
3703
+ is_enabled = enabled.get(key) is True
3704
  out.append({"key": key,
3705
  "label": str(declared.get("label") or LABELS_AMAZON.get(key) or key),
3706
  "defaultLabel": str(declared.get("label")
 
3709
  # W55-T19 / R7 — AMAZON'S OWN tenant-local answer. This is deliberately not
3710
  # read from Odoo's grid config, and disabling it leaves every stored row and
3711
  # definition untouched: only this provider config document changes.
3712
+ "enabled": is_enabled,
3713
+ **({"sync": raw_sync} if is_admin else {}),
3714
+ "syncHealth": _amazon_database_sync_health(raw_sync, is_enabled),
3715
  "fields": fields,
3716
  "excluded": excluded,
3717
  "sharesFieldsWith": [k for k in by_report.get(rep, ()) if k != key]})
 
3779
  stops "administrator only" being rendered as "this database has no fields".
3780
  """
3781
  key = str(row.get("key") or "")
3782
+ if key == "scraper":
3783
+ # Recipe databases publish their saved, stable schema to every reader who may open the
3784
+ # database. They have no generic schema-toggle route, so no write capability is implied.
3785
+ return {"introspectable": True, "reason": "", "fieldsVisible": True,
3786
+ "canToggleFields": False, "canToggleTables": False,
3787
+ "catalogueComplete": True, "capturedAt": "", "excludedBasis": "",
3788
+ "censusAt": "", "censusComplete": False, "censusDurationMs": 0,
3789
+ "censusModelsCounted": 0, "censusModelsWithRows": 0}
3790
  if not connector_directory.is_introspectable(key):
3791
  reason = ("Nothing is connected behind this listing yet, so there is no schema to read."
3792
  if str(row.get("state") or "") == connector_directory.CATALOGUE_STATE
 
3907
  "updatePath": "", "preservesData": True, "available": False}
3908
 
3909
 
3910
+ def _scraper_preview(body):
3911
+ request = body if isinstance(body, dict) else {}
3912
+ raw_recipe = request.get("recipe") if isinstance(request.get("recipe"), dict) else request
3913
+ recipe, errors = _clean_scraper_recipe(raw_recipe)
3914
+ if errors:
3915
+ raise err(400, "invalid_scraper_recipe", " ".join(errors))
3916
+ plan = _scraper_plan(recipe)
3917
+ missing = [] if plan["credentialReady"] else ["credential"]
3918
+ return {"recipe": recipe, "inputs": recipe["seed"], "provider": plan["provider"],
3919
+ "credentialReady": plan["credentialReady"], "estimatedUsd": plan["estimatedUsd"],
3920
+ "costCapUsd": plan["costCapUsd"], "recordsCap": plan["recordsCap"],
3921
+ "canRun": plan["canRun"], "missing": missing, "note": plan["note"],
3922
+ # This is a contract field, not a measurement. A preview and a create never call a
3923
+ # provider, so both truthfully report zero incurred spend.
3924
+ "incurredUsd": 0.0}
3925
+
3926
+
3927
+ def _create_scraper_database(rt, uname, body):
3928
+ """Persist one recipe database. Contains no provider import or runner call by construction."""
3929
+ if not rt.available():
3930
+ raise err(503, "store_unavailable", "the tenant store is unavailable")
3931
+ request = body if isinstance(body, dict) else {}
3932
+ label = str(request.get("label") or "").strip()
3933
+ if not label:
3934
+ raise err(400, "missing_database_name", "Enter a database name.")
3935
+ preview = _scraper_preview(request)
3936
+ recipe = preview["recipe"]
3937
+ import core.user_tables as user_tables # noqa: PLC0415 - existing create door
3938
+ key = user_tables.create(label, uname, fields=_scraper_recipe_fields(recipe["kind"]),
3939
+ source=user_tables.AUTOMATION_SOURCE, st=rt)
3940
+ if not key:
3941
+ raise err(409, "database_not_created",
3942
+ "The database name is unavailable or this workspace reached its database limit.")
3943
+
3944
+ def _attach(cur):
3945
+ current = dict(cur or {})
3946
+ definition = dict(current.get(key) or {})
3947
+ if not definition:
3948
+ raise RuntimeError("the new database disappeared before its recipe was attached")
3949
+ definition.update({"connectorKey": "scraper", "recordMode": "automation",
3950
+ "scraperRecipe": recipe, "scraperRuns": [],
3951
+ user_tables.CONNECTOR_HEALTH_FIELD:
3952
+ user_tables.connector_health_receipt("never_run")})
3953
+ current[key] = definition
3954
+ return current
3955
+
3956
+ try:
3957
+ rt.update(user_tables.STORE_KEY, _attach, flush="sync")
3958
+ except Exception: # noqa: BLE001
3959
+ user_tables.delete(key, st=rt)
3960
+ raise err(503, "recipe_not_saved",
3961
+ "The recipe could not be saved, so the new database was removed.")
3962
+ return {"key": key, "label": label, "recipe": recipe, "preview": preview,
3963
+ "createdWithoutProviderCall": True, "incurredUsd": 0.0}
3964
+
3965
+
3966
+ def _scraper_database_run(rt, uname, is_admin, database_key):
3967
+ """The one explicit paid-work boundary for any saved scraper recipe."""
3968
+ if not rt.available():
3969
+ raise err(503, "store_unavailable", "the tenant store is unavailable")
3970
+ import core.user_tables as user_tables
3971
+ key = str(database_key or "").strip()
3972
+ definitions = user_tables.lend_defs(rt)
3973
+ definition = user_tables.get(key, st=definitions) or {}
3974
+ if not definition:
3975
+ raise err(404, "unknown_scraper_database", "This saved scraper database was not found.")
3976
+ raw_recipe = definition.get("scraperRecipe")
3977
+ recipe, errors = _clean_scraper_recipe(raw_recipe)
3978
+ if errors:
3979
+ raw_kind = str((raw_recipe or {}).get("kind") or "") if isinstance(raw_recipe, dict) else ""
3980
+ raw_limits = ((raw_recipe or {}).get("limits") or {}) if isinstance(raw_recipe, dict) else {}
3981
+ raw_cap = raw_limits.get("records") if isinstance(raw_limits, dict) else None
3982
+ try:
3983
+ bounded = 1 <= int(raw_cap) <= 500
3984
+ except (TypeError, ValueError):
3985
+ bounded = False
3986
+ if raw_kind == "google_maps" and not bounded:
3987
+ raise err(400, "maps_unbounded_request",
3988
+ "Google Maps discovery requires an explicit result cap between 1 and 500.")
3989
+ raise err(409, "invalid_saved_scraper_recipe",
3990
+ "This saved scraper recipe is no longer valid. Review its settings before running it.")
3991
+ # Run now both spends against a provider account and mutates this database. A viewer grant
3992
+ # may expose the rows without granting either power, so the ordinary open wall is too broad.
3993
+ # Reuse the database editor authority that every other mutation surface already honours.
3994
+ if not user_tables.may_edit_database(key, uname, bool(is_admin), st=definitions):
3995
+ raise err(403, "database_forbidden", "You cannot run this database recipe.")
3996
+ plan = _scraper_plan(recipe)
3997
+ previous_health = definition.get(user_tables.CONNECTOR_HEALTH_FIELD)
3998
+ previous_health = previous_health if isinstance(previous_health, dict) else {}
3999
+
4000
+ def record_attempt(state, error=None):
4001
+ try:
4002
+ return user_tables.record_connector_health(
4003
+ key, state, st=rt,
4004
+ last_success=previous_health.get("lastSuccess"),
4005
+ source_timestamp=previous_health.get("sourceTimestamp"),
4006
+ materialized_timestamp=previous_health.get("materializedTimestamp"),
4007
+ row_count=previous_health.get("rowCount"), error=error)
4008
+ except Exception: # noqa: BLE001
4009
+ raise err(503, "sync_status_not_saved",
4010
+ "The run did not start because its durable status could not be saved.") from None
4011
+
4012
+ if not plan["credentialReady"]:
4013
+ code = "maps_credential_missing" if recipe["kind"] == "google_maps" else \
4014
+ "scraper_credential_missing"
4015
+ message = "Add the scraper credential in Keychain before choosing Run now."
4016
+ record_attempt("failed", {"code": code, "message": message})
4017
+ raise err(409, code, "Add the scraper credential in Keychain before choosing Run now.")
4018
+
4019
+ # Imports and provider calls occur only below the explicit Run-now endpoint. Preview, create,
4020
+ # detail, and page load never reach either runner.
4021
+ started_at = datetime.now(timezone.utc).isoformat()
4022
+ record_attempt("running")
4023
+ key_field = "shortcode"
4024
+ if recipe["kind"] == "google_maps":
4025
+ import providers
4026
+ try:
4027
+ result = providers.discover_google_maps_stores(
4028
+ recipe["seed"]["search"], recipe["seed"]["geography"],
4029
+ recipe["limits"]["records"], captured_at=started_at)
4030
+ except providers.MapsProviderRefusal as exc:
4031
+ record_attempt("failed", {"code": exc.code, "message": exc.public_message})
4032
+ status = 409 if exc.code == "maps_credential_missing" else \
4033
+ (400 if exc.code in ("maps_unbounded_request", "maps_invalid_search") else 502)
4034
+ raise err(status, exc.code, exc.public_message)
4035
+ incoming_key = "rows"
4036
+ key_field = "place_id"
4037
+ else:
4038
+ limit = recipe["limits"]["posts"]
4039
+ post_type = recipe["limits"]["postType"]
4040
+ try:
4041
+ if recipe["kind"] == "instagram":
4042
+ import connectors_ig as runner
4043
+ run_recipe = runner.pull_ig_recipe
4044
+ else:
4045
+ import connectors_tt as runner
4046
+ run_recipe = runner.pull_tt_recipe
4047
+ result = run_recipe(recipe["seed"], limit, post_type=post_type,
4048
+ log=lambda *_args: None)
4049
+ except Exception: # noqa: BLE001
4050
+ record_attempt("failed", {
4051
+ "code": "scraper_run_failed",
4052
+ "message": "The provider could not complete this run. No result was written.",
4053
+ })
4054
+ raise err(502, "scraper_run_failed",
4055
+ "The provider could not complete this run. No result was written.")
4056
+ incoming_key = "posts"
4057
+ result = result if isinstance(result, dict) else {}
4058
+ incoming = [dict(row) for row in (result.get(incoming_key) or ()) if isinstance(row, dict)]
4059
+ state = str(result.get("state") or "error")
4060
+ if state not in ("ok", "partial"):
4061
+ incoming = []
4062
+ finished_at = datetime.now(timezone.utc).isoformat()
4063
+ source_stamps = [_sync_stamp(row.get("source_timestamp")) for row in incoming]
4064
+ source_timestamp = max((stamp for stamp in source_stamps if stamp), default=None)
4065
+ provider_counts = dict(result.get("counts") or {})
4066
+ provider_counts.setdefault("returned", len(incoming))
4067
+ provider_cost = dict(result.get("cost") or {})
4068
+ provider_cost.setdefault("currency", "USD")
4069
+ provider_cost.setdefault("estimatedUsd", plan["estimatedUsd"])
4070
+ provider_cost.setdefault("costCapUsd", plan["costCapUsd"])
4071
+ receipt = {"id": finished_at, "state": state,
4072
+ "provider": str(result.get("via") or plan["provider"]),
4073
+ "rows": len(incoming), "startedAt": started_at, "finishedAt": finished_at,
4074
+ "estimatedUsd": plan["estimatedUsd"], "costCapUsd": plan["costCapUsd"],
4075
+ "cost": provider_cost, "providerCounts": provider_counts,
4076
+ "attempts": [dict(item) for item in (result.get("attempts") or ())
4077
+ if isinstance(item, dict)],
4078
+ # Provider notes can echo a submitted handle. Run history stores operational
4079
+ # metadata only; the saved recipe already owns the input and no customer value is
4080
+ # duplicated into an audit-like receipt.
4081
+ "note": f"Provider returned {state}."}
4082
+ counts = {"inserted": 0, "updated": 0, "unchanged": 0, "skipped": 0,
4083
+ "duplicates": 0, "orphans": 0, "capped": 0}
4084
+
4085
+ def _write(cur):
4086
+ nonlocal counts
4087
+ current = dict(cur or {})
4088
+ stored = dict(current.get(key) or {})
4089
+ if not stored:
4090
+ raise RuntimeError("the database disappeared before the run could be written")
4091
+ if incoming:
4092
+ import automation_engine as engine
4093
+ rows, counts = engine.upsert_rows(stored.get("rows") or {}, incoming, key_field)
4094
+ stored["rows"] = rows
4095
+ processed = sum(int(counts.get(name) or 0)
4096
+ for name in ("inserted", "updated", "unchanged", "skipped", "capped"))
4097
+ receipt["writeCounts"] = dict(counts)
4098
+ receipt["countReconciled"] = (
4099
+ int(provider_counts.get("returned") or 0) == len(incoming) == processed)
4100
+ history = [dict(item) for item in (stored.get("scraperRuns") or ())
4101
+ if isinstance(item, dict)]
4102
+ stored["scraperRuns"] = [receipt, *history][:SCRAPER_RUN_HISTORY_LIMIT]
4103
+ final_state = "succeeded" if state == "ok" else "partial" if state == "partial" else "failed"
4104
+ final_error = None if final_state != "failed" else {
4105
+ "code": "provider_result_failed",
4106
+ "message": "The provider returned no usable result for this database.",
4107
+ }
4108
+ stored[user_tables.CONNECTOR_HEALTH_FIELD] = user_tables.connector_health_receipt(
4109
+ final_state,
4110
+ last_success=finished_at if final_state in ("succeeded", "partial")
4111
+ else previous_health.get("lastSuccess"),
4112
+ source_timestamp=source_timestamp,
4113
+ materialized_timestamp=finished_at if final_state in ("succeeded", "partial") else None,
4114
+ row_count=len(stored.get("rows") or {}), error=final_error)
4115
+ current[key] = stored
4116
+ return current
4117
+
4118
+ try:
4119
+ rt.update(user_tables.STORE_KEY, _write, flush="sync")
4120
+ except Exception: # noqa: BLE001
4121
+ try:
4122
+ record_attempt("failed", {
4123
+ "code": "scraper_result_not_saved",
4124
+ "message": "The provider answered, but the result could not be saved.",
4125
+ })
4126
+ except Exception: # noqa: BLE001
4127
+ pass
4128
+ raise err(503, "scraper_result_not_saved",
4129
+ "The provider answered, but the result could not be saved. Reload before retrying.")
4130
+ return {"key": key, "state": state, "counts": counts, "receipt": receipt}
4131
+
4132
+
4133
  def connector_detail(rt, user, uname, is_admin, connector_key):
4134
  """The whole C4 detail DTO for one connector, or `None` when this reader has no such card.
4135
 
 
4228
  return _set_amazon_database_enablement(database_key, body, session)
4229
 
4230
 
4231
+ @router.post("/connectors/scraper/databases/preview")
4232
+ def scraper_database_preview(body: dict = Body(default=None),
4233
+ session: Session = Depends(require_session)):
4234
+ """Validate and price a recipe locally. This endpoint makes zero provider calls."""
4235
+ return _scraper_preview(body)
4236
+
4237
+
4238
+ @router.post("/connectors/scraper/databases")
4239
+ def scraper_database_create(body: dict = Body(default=None),
4240
+ session: Session = Depends(require_session)):
4241
+ """Create a saved recipe database for $0; paid work begins only at its Run-now door."""
4242
+ return _create_scraper_database(session.runtime, session.uname, body)
4243
+
4244
+
4245
+ @router.post("/connectors/scraper/databases/{database_key}/run")
4246
+ def scraper_database_run(database_key: str, session: Session = Depends(require_session)):
4247
+ """Explicitly execute one bounded saved scraper recipe through its provider seam."""
4248
+ return _scraper_database_run(session.runtime, session.uname, bool(session.admin), database_key)
4249
+
4250
+
4251
  # ⚠ DECLARED AFTER THE LITERAL `/connectors/odoo/*` ROUTES. FastAPI matches in declaration order,
4252
  # and while none of those ends in `/detail` — so no shadowing is possible today — the ordering is
4253
  # the habit that keeps it impossible tomorrow. `/connectors/directory` is two segments and cannot
api/routes_customers.py CHANGED
@@ -22,8 +22,10 @@ owner-blocked on B-3), and no test here may claim read-your-writes ACROSS runtim
22
  proof is single-process and would report green on exactly the thing that is still broken.
23
  """
24
  import datetime as _dt
 
25
  import hashlib
26
  import json
 
27
  import time
28
 
29
  from fastapi import APIRouter, Body, Depends
@@ -39,6 +41,14 @@ MODULE = "customer_data"
39
 
40
  _CACHE_TTL = 900 # the pool build is slow (Odoo + reconciliation); 15 min, as before
41
 
 
 
 
 
 
 
 
 
42
 
43
  def _pool_rows(session: Session):
44
  """The reconciled Odoo pool for this session's SCOPE — the slow part, and the only part that
@@ -304,6 +314,65 @@ def _mark_offerability(fields):
304
 
305
  def grid_assembly(session: Session, scope: str = "customer", storage_key: str = "",
306
  consume_corrections: bool = True):
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
307
  """ONE assembly of this session's grid state, shared by `/customers`, `/workspace` and the
308
  events route (2026-07-31 — the standalone measure gap, owner item 1).
309
 
@@ -330,6 +399,10 @@ def grid_assembly(session: Session, scope: str = "customer", storage_key: str =
330
 
331
  import core.perm_scope as perm_scope
332
 
 
 
 
 
333
  rt = session.runtime
334
  team_id, agent = _team_agent(session)
335
  rows_src = _pool_for(rt, team_id, agent)
 
22
  proof is single-process and would report green on exactly the thing that is still broken.
23
  """
24
  import datetime as _dt
25
+ from copy import deepcopy
26
  import hashlib
27
  import json
28
+ import threading
29
  import time
30
 
31
  from fastapi import APIRouter, Body, Depends
 
41
 
42
  _CACHE_TTL = 900 # the pool build is slow (Odoo + reconciliation); 15 min, as before
43
 
44
+ # A slow producer must not indefinitely hold another threadpool request; timeout recomputes.
45
+ _GRID_ASSEMBLY_WAIT_SECONDS = 10.0
46
+ _grid_assembly_lock = threading.Lock()
47
+ _grid_assembly_inflight = {}
48
+ _grid_assembly_enabled = True
49
+ grid_assembly_computed = 0
50
+ grid_assembly_shared = 0
51
+
52
 
53
  def _pool_rows(session: Session):
54
  """The reconciled Odoo pool for this session's SCOPE — the slow part, and the only part that
 
314
 
315
  def grid_assembly(session: Session, scope: str = "customer", storage_key: str = "",
316
  consume_corrections: bool = True):
317
+ """Share only overlapping assemblies of the same user's grid, never completed results."""
318
+ global grid_assembly_shared
319
+ rt = session.runtime
320
+ team_id, agent = _team_agent(session)
321
+ key = (session.tenant, session.uname, session.admin, team_id, agent, scope,
322
+ consume_corrections, _pool_stamp(rt, team_id, agent),
323
+ # A changed permission record must not borrow an assembly from the old record.
324
+ # ⛔ `default=str` IS NOT TIDINESS. This runs on every `/customers` and `/workspace`,
325
+ # the product's main grid; a `TypeError` from an unserialisable value in a stored user
326
+ # record would 500 BOTH doors, which is a far worse failure than the duplicated work
327
+ # this key exists to remove. The record round-trips JSON by construction today, so the
328
+ # fallback should never fire -- and a key that cannot raise is the only kind worth
329
+ # putting in front of a hot path.
330
+ json.dumps(session.user, sort_keys=True, separators=(",", ":"), default=str))
331
+
332
+ def own_copy(result):
333
+ # The pool is already shared by scope_cache and rows_from_pool never mutates it.
334
+ # Copy the rest together to preserve internal aliases without lending mutable state.
335
+ out = deepcopy({k: v for k, v in result.items() if k != "rows_src"})
336
+ out["rows_src"] = result["rows_src"]
337
+ out["workspace"]["storageKey"] = storage_key
338
+ return out
339
+
340
+ if not _grid_assembly_enabled:
341
+ return own_copy(_compute_grid_assembly(session, scope, storage_key, consume_corrections))
342
+
343
+ with _grid_assembly_lock:
344
+ entry = _grid_assembly_inflight.get(key)
345
+ producer = entry is None
346
+ if producer:
347
+ entry = {"event": threading.Event(), "result": None, "error": None}
348
+ _grid_assembly_inflight[key] = entry
349
+
350
+ if not producer:
351
+ if entry["event"].wait(_GRID_ASSEMBLY_WAIT_SECONDS):
352
+ if entry["error"] is not None:
353
+ raise entry["error"]
354
+ result = own_copy(entry["result"])
355
+ with _grid_assembly_lock:
356
+ grid_assembly_shared += 1
357
+ return result
358
+ # A timeout changes neither the caller's answer nor the producer's ownership.
359
+ return own_copy(_compute_grid_assembly(session, scope, storage_key, consume_corrections))
360
+
361
+ try:
362
+ entry["result"] = _compute_grid_assembly(session, scope, storage_key, consume_corrections)
363
+ except BaseException as exc:
364
+ entry["error"] = exc
365
+ raise
366
+ finally:
367
+ with _grid_assembly_lock:
368
+ if _grid_assembly_inflight.get(key) is entry:
369
+ del _grid_assembly_inflight[key]
370
+ entry["event"].set()
371
+ return own_copy(entry["result"])
372
+
373
+
374
+ def _compute_grid_assembly(session: Session, scope: str = "customer", storage_key: str = "",
375
+ consume_corrections: bool = True):
376
  """ONE assembly of this session's grid state, shared by `/customers`, `/workspace` and the
377
  events route (2026-07-31 — the standalone measure gap, owner item 1).
378
 
 
399
 
400
  import core.perm_scope as perm_scope
401
 
402
+ global grid_assembly_computed
403
+ with _grid_assembly_lock:
404
+ grid_assembly_computed += 1
405
+
406
  rt = session.runtime
407
  team_id, agent = _team_agent(session)
408
  rows_src = _pool_for(rt, team_id, agent)
api/routes_external.py CHANGED
The diff for this file is too large to render. See raw diff
 
api/routes_geo.py CHANGED
@@ -998,6 +998,53 @@ def _route_endpoint_index(pids, raw, label):
998
  return matches[0]
999
 
1000
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1001
  @router.post("/geo/route")
1002
  def geo_route(body: dict = Body(...), session: Session = Depends(require_session)):
1003
  """Road distance, duration and the drawn line, for the order given AND for a better one.
@@ -1015,6 +1062,8 @@ def geo_route(body: dict = Body(...), session: Session = Depends(require_session
1015
 
1016
  Request body:
1017
  `stops` [{lat, lon, pid?}] the order to measure, at least two.
 
 
1018
  `roundTrip` bool. The server closes the measured route at its selected start. A legacy
1019
  closing duplicate is accepted too and never becomes a visitable stop.
1020
  `startPid` optional selected start among the submitted `pid` values.
@@ -1067,6 +1116,7 @@ def geo_route(body: dict = Body(...), session: Session = Depends(require_session
1067
  round_trip = bool(body.get("roundTrip")) or has_closing_stop
1068
  core = points[:-1] if has_closing_stop else points
1069
  core_pids = pids[:-1] if has_closing_stop else pids
 
1070
  start_index = _route_endpoint_index(core_pids, body.get("startPid"), "start")
1071
  end_index = _route_endpoint_index(core_pids, body.get("endPid"), "end")
1072
  if round_trip and end_index is not None:
@@ -1139,6 +1189,8 @@ def geo_route(body: dict = Body(...), session: Session = Depends(require_session
1139
  "attribution": credit,
1140
  # ---- C2, for the order as submitted
1141
  "ordered": core_pids,
 
 
1142
  "meters": current["meters"],
1143
  "seconds": current["seconds"],
1144
  "source": current["source"],
 
998
  return matches[0]
999
 
1000
 
1001
+ def _route_location_partition(body, located_pids):
1002
+ """Validate C1's explicit client partition without trusting labels as coordinates.
1003
+
1004
+ Older map callers submit only ``stops`` and keep their established response shape. RouteNav
1005
+ submits both keys together; once either is present the pair is atomic and every pid must occur
1006
+ exactly once across the two halves.
1007
+ """
1008
+ has_located = "locatedPids" in body
1009
+ has_unlocated = "unlocated" in body
1010
+ if not has_located and not has_unlocated:
1011
+ return list(located_pids), []
1012
+ if not has_located or not has_unlocated:
1013
+ raise err(400, "invalid_route_partition",
1014
+ "Route locations must identify both located and unlocated stops.")
1015
+ raw_located, raw_unlocated = body.get("locatedPids"), body.get("unlocated")
1016
+ if not isinstance(raw_located, list) or not isinstance(raw_unlocated, list):
1017
+ raise err(400, "invalid_route_partition",
1018
+ "Route locations must be lists of located and unlocated stops.")
1019
+ try:
1020
+ claimed_located = [int(pid) for pid in raw_located]
1021
+ except (TypeError, ValueError):
1022
+ raise err(400, "invalid_route_partition", "Every located Route stop needs a record id.")
1023
+ if claimed_located != list(located_pids) or len(set(claimed_located)) != len(claimed_located):
1024
+ raise err(400, "invalid_route_partition",
1025
+ "The located Route stop ids must match the submitted coordinates in order.")
1026
+ cleaned, unlocated_ids = [], set()
1027
+ for item in raw_unlocated:
1028
+ if not isinstance(item, dict):
1029
+ raise err(400, "invalid_route_partition",
1030
+ "Every unlocated Route stop needs a record id, label, and reason.")
1031
+ try:
1032
+ pid = int(item.get("pid"))
1033
+ except (TypeError, ValueError):
1034
+ raise err(400, "invalid_route_partition", "Every unlocated Route stop needs a record id.")
1035
+ label = " ".join(str(item.get("label") or "").split())[:160]
1036
+ reason = " ".join(str(item.get("reason") or "").split())[:240]
1037
+ if pid in unlocated_ids or pid in claimed_located or not label or not reason:
1038
+ raise err(400, "invalid_route_partition",
1039
+ "Every Route stop must occur once with a non-empty label and location reason.")
1040
+ unlocated_ids.add(pid)
1041
+ cleaned.append({"pid": pid, "label": label, "reason": reason})
1042
+ if len(cleaned) > MAX_ROUTE_STOPS:
1043
+ raise err(400, "too_many_stops",
1044
+ f"Identify at most {MAX_ROUTE_STOPS} unlocated Route stops at once.")
1045
+ return claimed_located, cleaned
1046
+
1047
+
1048
  @router.post("/geo/route")
1049
  def geo_route(body: dict = Body(...), session: Session = Depends(require_session)):
1050
  """Road distance, duration and the drawn line, for the order given AND for a better one.
 
1062
 
1063
  Request body:
1064
  `stops` [{lat, lon, pid?}] the order to measure, at least two.
1065
+ `locatedPids` [pid] matching ``stops`` in order when RouteNav submits C1's partition.
1066
+ `unlocated` [{pid,label,reason}] named fixed anchors excluded from both measurements.
1067
  `roundTrip` bool. The server closes the measured route at its selected start. A legacy
1068
  closing duplicate is accepted too and never becomes a visitable stop.
1069
  `startPid` optional selected start among the submitted `pid` values.
 
1116
  round_trip = bool(body.get("roundTrip")) or has_closing_stop
1117
  core = points[:-1] if has_closing_stop else points
1118
  core_pids = pids[:-1] if has_closing_stop else pids
1119
+ _claimed_located, unlocated = _route_location_partition(body, core_pids)
1120
  start_index = _route_endpoint_index(core_pids, body.get("startPid"), "start")
1121
  end_index = _route_endpoint_index(core_pids, body.get("endPid"), "end")
1122
  if round_trip and end_index is not None:
 
1189
  "attribution": credit,
1190
  # ---- C2, for the order as submitted
1191
  "ordered": core_pids,
1192
+ "locatedPids": core_pids,
1193
+ "unlocated": unlocated,
1194
  "meters": current["meters"],
1195
  "seconds": current["seconds"],
1196
  "source": current["source"],
api/routes_module_index.py CHANGED
@@ -65,10 +65,28 @@ def _readable_ids(module, session):
65
  voice._store(session.runtime)["voice_agent"].values() if isinstance(row, dict)})
66
 
67
 
 
 
 
 
 
 
 
 
 
 
 
 
 
68
  def _project(bucket, ids):
69
  folders, placed, _ = validate_folder_tree(bucket.get("folders"), bucket.get("itemFolders"))
70
- return {"folders": folders, "itemFolders": {k: v for k, v in placed.items() if k in ids},
71
- "rev": bucket.get("rev", 0)}
 
 
 
 
 
72
 
73
 
74
  def _available(session):
@@ -111,6 +129,7 @@ def write_module_index(module: str, body: dict = Body(default=None), user: str =
111
  raise SheetFolderRefusal("module_index_item_forbidden",
112
  "A folder placement names an item you cannot read in this module.")
113
  folders, placed, _ = validate_folder_tree(raw["folders"], placements)
 
114
  saved = {}
115
 
116
  def apply(current):
@@ -122,6 +141,10 @@ def write_module_index(module: str, body: dict = Body(default=None), user: str =
122
  raise SheetFolderRefusal("index_revision_stale",
123
  "Folders changed somewhere else. Reload folders and try again.")
124
  saved.update(folders=folders, itemFolders=placed, rev=rev + 1)
 
 
 
 
125
  users[owner] = dict(saved)
126
  state["schema"] = 1
127
  return state
 
65
  voice._store(session.runtime)["voice_agent"].values() if isinstance(row, dict)})
66
 
67
 
68
+ def _collapsed_groups(raw):
69
+ """The module-index document owns per-user App chrome disclosure preferences.
70
+
71
+ Finance is the only registry group mounted in chrome today. Refuse unknown identifiers rather
72
+ than turning this preference bucket into unbounded client storage.
73
+ """
74
+ if not isinstance(raw, list) or any(not isinstance(value, str) for value in raw):
75
+ raise SheetFolderRefusal("module_index_invalid", "collapsedGroups must be a list of supported group names.")
76
+ if any(value != "finance" for value in raw):
77
+ raise SheetFolderRefusal("module_index_invalid", "collapsedGroups names an unsupported App group.")
78
+ return list(dict.fromkeys(raw))
79
+
80
+
81
  def _project(bucket, ids):
82
  folders, placed, _ = validate_folder_tree(bucket.get("folders"), bucket.get("itemFolders"))
83
+ projected = {"folders": folders, "itemFolders": {k: v for k, v in placed.items() if k in ids},
84
+ "rev": bucket.get("rev", 0)}
85
+ # Additive and presence-preserving: old buckets retain their byte-for-byte response shape,
86
+ # while a user who explicitly opens Finance can persist an empty list as a real preference.
87
+ if "collapsedGroups" in bucket:
88
+ projected["collapsedGroups"] = _collapsed_groups(bucket.get("collapsedGroups"))
89
+ return projected
90
 
91
 
92
  def _available(session):
 
129
  raise SheetFolderRefusal("module_index_item_forbidden",
130
  "A folder placement names an item you cannot read in this module.")
131
  folders, placed, _ = validate_folder_tree(raw["folders"], placements)
132
+ collapsed = _collapsed_groups(raw.get("collapsedGroups")) if "collapsedGroups" in raw else None
133
  saved = {}
134
 
135
  def apply(current):
 
141
  raise SheetFolderRefusal("index_revision_stale",
142
  "Folders changed somewhere else. Reload folders and try again.")
143
  saved.update(folders=folders, itemFolders=placed, rev=rev + 1)
144
+ if collapsed is not None:
145
+ saved["collapsedGroups"] = collapsed
146
+ elif "collapsedGroups" in (users.get(owner) or {}):
147
+ saved["collapsedGroups"] = _collapsed_groups(users[owner].get("collapsedGroups"))
148
  users[owner] = dict(saved)
149
  state["schema"] = 1
150
  return state
api/routes_odoo_tables.py CHANGED
@@ -22,13 +22,15 @@ import base64
22
  import hashlib
23
  import hmac
24
  import json
 
25
  import time
26
  from contextlib import contextmanager
27
 
28
  from fastapi import Body, Depends, HTTPException
29
  from fastapi import APIRouter, Query
30
 
31
- from deps import (Session, err, module_cache_rows, module_cache_source, require_session)
 
32
 
33
  router = APIRouter(prefix="/api/v1")
34
 
@@ -120,13 +122,14 @@ def refresh_odoo_tables(session: Session = Depends(require_session)):
120
  # section and its negative control.
121
  if not session.admin:
122
  raise err(403, "forbidden", "Refreshing the Odoo databases is an admin action.")
123
- if not rel.is_royal(session.tenant):
124
  # A plain 400 with the reason: this tenant has no Odoo mirror, and spawning two empty
125
  # locked databases it can never fill would be worse than refusing.
126
  raise err(400, "not_applicable",
127
- "These databases are built from the Royal Imports Odoo mirror (R1).")
128
  try:
129
- out = rel.refresh(session.runtime, session.tenant, username=session.user)
 
130
  # ⭐⭐ W30-T31 — AND THEN THE ROWS THAT SHOULD NOT BE HERE LEAVE AGAIN.
131
  #
132
  # The spawn writes every bucket's rows straight into the tenant document by mutating it
@@ -209,7 +212,19 @@ def odoo_tables_status(brief: bool = Query(False),
209
  # while those were the only two; the 2026-08-09 widening added products and orders, and a
210
  # hard-coded list here would have reported "everything is fine" over two databases it had
211
  # stopped knowing about ([[gate-answers-the-wrong-question]] — the hard-coded count).
212
- for _bucket, key, _label, _fields in rel.TABLES:
 
 
 
 
 
 
 
 
 
 
 
 
213
  table = tables.get(key) or {}
214
  rows = (table.get("rows") or {})
215
  stamps = [str((r or {}).get("refreshed") or "") for r in rows.values()]
@@ -220,6 +235,8 @@ def odoo_tables_status(brief: bool = Query(False),
220
  # says a worklist that quietly stopped updating must be legible without a refresh. So a
221
  # read-through table is counted from the MIRROR and says where its count came from.
222
  materialised = user_tables.materialises(key, defn=table)
 
 
223
  if not materialised:
224
  try:
225
  rows = {}
@@ -233,11 +250,36 @@ def odoo_tables_status(brief: bool = Query(False),
233
  order_by=spec["id"], limit=1, cur=cur, **frm)["total"]
234
  else:
235
  n = -1 # unknown, and never reported as zero
 
 
 
236
  except Exception: # noqa: BLE001
237
  n = -1
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
238
  out[key] = {
239
  "exists": bool(table),
240
  "rows": (len(rows) if materialised else n),
 
241
  # ⭐ W31-T45: a refused mirror says `refused`, never `mirror` — a size that came from
242
  # nowhere must not be labelled with the source it did not come from.
243
  "rowsFrom": ("document" if materialised
@@ -245,8 +287,14 @@ def odoo_tables_status(brief: bool = Query(False),
245
  "materialised": materialised,
246
  # why this grid still keeps its rows in the tenant document, when it does not have to
247
  "materialisedBecause": (why_not.get(key, "") if materialised else ""),
248
- "refreshed": (max([s for s in stamps if s], default="") if materialised
249
- else "live, read through the mirror"),
 
 
 
 
 
 
250
  "locked": table.get("recordMode") == user_tables.AUTOMATION_RECORD_MODE,
251
  # W30/R7: whether this database is served THROUGH the mirror rather than from the copy
252
  # above. Reported per table, because they convert one at a time and an operator
@@ -316,8 +364,8 @@ def _seed_state():
316
  from harness import datastore as _ds
317
  block["present"] = bool(_ds.DB_PATH.exists())
318
  if not block["present"]:
319
- import main as _main
320
- state = dict(_main.MIRROR_SEED)
321
  block["manifest"] = "absent"
322
  block["cause"] = state.get("cause") or "this container has no analytical mirror yet"
323
  block["recommendation"] = state.get("recommendation") or (
@@ -456,10 +504,11 @@ def _resync_state(cur):
456
  block = {"armed": False, "tenant": "", "tenantSource": "", "tenantSeconds": None,
457
  "envSeconds": None, "resolvedSeconds": None, "manual": False,
458
  "disagreement": None, "passes": 0, "lastPassAt": None, "lastPassOk": None,
459
- "lastError": None, "bootSyncError": None, "entities": None,
 
460
  "entitiesCause": "", "stuck": []}
461
  try:
462
- import main as _main
463
  block.update({k: v for k, v in dict(_main.RESYNC_STATE).items() if k in block})
464
  cadence = block.get("resolvedSeconds") or _main._RESYNC_DEFAULT_SECONDS
465
  except Exception: # noqa: BLE001
@@ -545,6 +594,7 @@ GRID_SOURCES = {}
545
  #: ⚠ A provider registers a CALLABLE, not a dict, so its module is not imported until the first
546
  #: read — the same lazy rule `_sources` already follows for `odoo_relational`.
547
  _SOURCE_PROVIDERS = []
 
548
 
549
 
550
  def register_source_provider(fn):
@@ -558,6 +608,25 @@ def register_source_provider(fn):
558
  return len(_SOURCE_PROVIDERS)
559
 
560
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
561
  def _sources():
562
  """Build `GRID_SOURCES` lazily by asking EVERY registered provider — Odoo is one of them.
563
 
@@ -929,10 +998,24 @@ def _odoo_sources():
929
  return GRID_SOURCES
930
 
931
 
 
 
 
 
 
 
 
 
 
 
 
 
 
932
  #: Odoo registers itself, at import, exactly as any other connector does. ⚠ The order providers
933
  #: are registered in is the order their specs land; keys are namespaced by connector (`ut_odoo_`,
934
  #: `ut_meta_`), so a later provider cannot shadow an earlier one's grid.
935
  register_source_provider(_odoo_sources)
 
936
 
937
 
938
  def sync_read_through(cur=None):
@@ -942,7 +1025,7 @@ def sync_read_through(cur=None):
942
  registration is additive and `strip_materialised` is a no-op once a table is empty.
943
 
944
  ⛔ IT IS ALSO THE ONLY PLACE THAT MAY CALL `register_read_through`, because the eligibility
945
- question needs BOTH halves — `odoo_relational`'s field declarations (for the fold matrix) and
946
  the mirror (for the size) — and `core` may import neither.
947
  """
948
  eligible, reasons = _unmaterialisable(cur)
@@ -1034,7 +1117,7 @@ def _unmaterialisable(cur=None):
1034
 
1035
  ⛔ SIZE FORCES READ-THROUGH; IT NEVER BLOCKS IT — and getting that backwards was a real bug in
1036
  the first cut of this function. A table too big for the tenant document has NO materialised
1037
- option at all, so making it ineligible would have handed `odoo_relational.plan` a `row_limit`
1038
  of None and invited it to build 963,783 python dicts. The window ceiling is a different, much
1039
  softer thing: it only limits what the whole-table CLIENT door can serve today.
1040
 
@@ -1046,19 +1129,42 @@ def _unmaterialisable(cur=None):
1046
  ⚠ Without a mirror cursor the size questions cannot be asked, so only a grid that was never
1047
  part of the materialised spawn is eligible — a table is never freed by a question we skipped.
1048
  """
1049
- rel = _rel()
1050
- folds, eligible, reasons = _fold_reasons(rel), {}, {}
1051
- # ⛔ "WAS THIS PART OF THE MATERIALISED SPAWN?", and being in `TABLES` STOPPED ANSWERING IT.
1052
- # W30-T35 declared the two line grains, which have no python row builder anywhere — their
1053
- # absence from `_READERS` IS that statement — so a table the spawn could never materialise
1054
- # started reading as spawned, and on a mirror-less deployment fell through to "its size could
1055
- # not be read" and came back INELIGIBLE. That inverts this function's own first law (size
1056
- # forces read-through; it never blocks it). Ask the question through the reader map, which is
1057
- # what actually decides whether a row could ever have been built.
1058
- spawned = {key for _b, key, _l, _f in rel.TABLES if _b in getattr(rel, "_READERS", {})}
1059
  from harness import datastore
1060
  ut = _ut()
1061
- for key, spec in _sources().items():
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1062
  total = None
1063
  if cur is not None:
1064
  try:
@@ -1068,7 +1174,7 @@ def _unmaterialisable(cur=None):
1068
  order_by=spec["id"], limit=1, cur=cur, **frm)["total"]
1069
  except Exception: # noqa: BLE001
1070
  total = None
1071
- if total is not None and total > ut.MAX_ROWS:
1072
  eligible[key] = True # no other home exists; this is not a choice
1073
  continue
1074
  if total is None and key not in spawned:
@@ -1079,8 +1185,8 @@ def _unmaterialisable(cur=None):
1079
  why.append("its size could not be read from the mirror on this deployment")
1080
  if key in folds:
1081
  why.append(folds[key])
1082
- if total is not None and total > datastore.WINDOW_MAX:
1083
- why.append(f"its {total:,} rows exceed the {datastore.WINDOW_MAX:,}-row window and "
1084
  f"the grid client still asks for whole tables, so un-materialising it "
1085
  f"today could only truncate")
1086
  eligible[key] = not why
@@ -2197,6 +2303,155 @@ def _views_as_cohorts(tree, table_key, session, spec, cols, base_where, wall_par
2197
  return tree, sets
2198
 
2199
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2200
  def _may_see_view(session, vid, store):
2201
  """May this caller open that saved view? The same predicate the view rail asks.
2202
 
@@ -2437,8 +2692,12 @@ def _compiled_read(table_key, session, *, filters=None, filterConj="and", sorts=
2437
  # the reader is held to, exactly as `viewSets` resolves inside the rows the client holds) and
2438
  # before `_leaf_cols` below, so the leaf never reaches the sentence that explains a column with
2439
  # no mirror expression. One place learns it; `/rows` and `/aggregates` share this compiler.
 
2440
  tree, cohort_sets = _views_as_cohorts(tree, table_key, session, spec, cols,
2441
  base_where, wall_params, cur, limits)
 
 
 
2442
 
2443
  # ── the predicate, pushed down ────────────────────────────────────────────────────────────
2444
  # ⛔ `_PSEUDO_COLS` is subtracted BEFORE the missing-column test. `__cohort__` and `__view__`
@@ -2563,8 +2822,12 @@ def _materialised_read(table_key, session, query, related):
2563
  "cols": bindings, "where": None}
2564
  cols = _filter_columns(spec, fields)
2565
  limits = list(payload.get("limits") or [])
 
2566
  tree, cohorts = _views_as_cohorts(query["filters"], table_key, session, spec, cols,
2567
  None, [], cur, limits)
 
 
 
2568
  missing = _leaf_cols(tree) - _PSEUDO_COLS - set(cols)
2569
  if missing:
2570
  raise err(400, "filter_unsupported", "A filter names a column unavailable to this account.")
@@ -2585,7 +2848,42 @@ def _materialised_read(table_key, session, query, related):
2585
  "frm": {"table": "pinned_rows"}, "limits": limits, "payload": payload}
2586
 
2587
 
2588
- def _materialised_window(table_key, session, offset, limit, filters, filterConj, sorts, search, related):
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2589
  from harness import datastore
2590
 
2591
  query = _selection_query({"filters": _json_arg(filters, "filters"), "filterConj": filterConj,
@@ -2593,8 +2891,13 @@ def _materialised_window(table_key, session, offset, limit, filters, filterConj,
2593
  with _materialised_read(table_key, session, query, related) as read:
2594
  spec = read["spec"]
2595
  order = _fs().compile_order_by(query["sorts"], read["cols"], tiebreak_sql=spec["id"])
2596
- win = datastore.window(select=f'{spec["id"]} AS pid', where=read["where"],
2597
- params=tuple(read["params"]), order_by=order or spec["id"],
 
 
 
 
 
2598
  offset=offset, limit=limit or None, cur=read["cur"], **read["frm"])
2599
  payload = read["payload"]
2600
  by_id = {int(row["pid"]): row for row in payload["rows"]}
@@ -2617,6 +2920,8 @@ def odoo_table_rows(table_key: str,
2617
  sorts: str = Query(default=None),
2618
  search: str = Query(default=None),
2619
  related: str = Query(default=None),
 
 
2620
  session: Session = Depends(require_session)):
2621
  """ONE WINDOW over a connected grid, read straight from the mirror (contract C2).
2622
 
@@ -2647,7 +2952,7 @@ def odoo_table_rows(table_key: str,
2647
 
2648
  if _materialised_table(table_key):
2649
  return _materialised_window(table_key, session, offset, limit, filters, filterConj,
2650
- sorts, search, related)
2651
 
2652
  # ⛔⛔ ONE COMPILER, AND THE AGGREGATES DOOR CALLS THE SAME ONE. The permission wall, the
2653
  # hidden-field closure, the user predicate and the search all live in `_compiled_read` above,
@@ -2661,6 +2966,10 @@ def odoo_table_rows(table_key: str,
2661
  cols, limits, frm = read["cols"], read["limits"], read["frm"]
2662
  base_where, wall_params = read["base_where"], read["wall_params"]
2663
  where, params, sort_spec = read["where"], read["params"], read["sort_spec"]
 
 
 
 
2664
 
2665
  # ── the order, made TOTAL ─────────────────────────────────────────────────────────────────
2666
  # ⛔ `tiebreak_sql` is not optional here: without a total order, LIMIT/OFFSET may return one
@@ -2699,7 +3008,7 @@ def odoo_table_rows(table_key: str,
2699
  # ⭐ D-155 — the freshness stamp rides the SAME builder, so both read-through doors answer the
2700
  # `refreshed` column identically instead of one of them leaving it blank.
2701
  rows_src = _pool_from_window(spec, win,
2702
- stamp=mirror_stamp(spec, cur, table_key,
2703
  session.runtime))
2704
  overlays = {str(r["pid"]): {k: v for k, v in r.items() if k != "pid"} for r in rows_src}
2705
 
@@ -3063,7 +3372,7 @@ def odoo_table_aggregates(table_key: str, body: dict = Body(default=None),
3063
  return body_out
3064
 
3065
 
3066
- def mirror_stamp(spec, cur=None, table_key="", rt=None):
3067
  """⭐ D-155 — WHEN THIS READ-THROUGH GRID'S DATA WAS LAST RECONCILED AGAINST ODOO, as a date.
3068
 
3069
  ⛔ THE COLUMN EXISTED AND THE BINDING DID NOT, which is the whole of D-155. Every Odoo table
@@ -3108,9 +3417,12 @@ def mirror_stamp(spec, cur=None, table_key="", rt=None):
3108
  # the materialised carve-out above, made structural. ⚠ Fail-QUIET to `""`: when we cannot
3109
  # tell whether this grid stores rows, the safe answer is the behaviour that shipped (blank),
3110
  # never a stamp that might overwrite a per-row one.
 
 
 
3111
  if table_key:
3112
  try:
3113
- if _ut().materialises(str(table_key), st=rt):
3114
  return ""
3115
  except Exception: # noqa: BLE001
3116
  return ""
@@ -3227,7 +3539,11 @@ def whole_pool(table_key, cur=None, rt=None):
3227
  f"{table_key}: {win['total']:,} rows is more than one {datastore.WINDOW_MAX:,}-row "
3228
  f"window, and this database no longer stores rows in the tenant document; it can "
3229
  f"only be read a page at a time (`/odoo-tables/{table_key}/rows`)")
3230
- return _pool_from_window(spec, win, stamp=mirror_stamp(spec, cur, table_key, rt))
 
 
 
 
3231
 
3232
 
3233
  def _fe():
 
22
  import hashlib
23
  import hmac
24
  import json
25
+ import math
26
  import time
27
  from contextlib import contextmanager
28
 
29
  from fastapi import Body, Depends, HTTPException
30
  from fastapi import APIRouter, Query
31
 
32
+ from deps import (Session, err, module_cache_rows, module_cache_source,
33
+ require_session, running_main)
34
 
35
  router = APIRouter(prefix="/api/v1")
36
 
 
122
  # section and its negative control.
123
  if not session.admin:
124
  raise err(403, "forbidden", "Refreshing the Odoo databases is an admin action.")
125
+ if not rel.available(session.runtime) and not rel.frozen(session.runtime):
126
  # A plain 400 with the reason: this tenant has no Odoo mirror, and spawning two empty
127
  # locked databases it can never fill would be worse than refusing.
128
  raise err(400, "not_applicable",
129
+ "Connect an Odoo source before refreshing its databases.")
130
  try:
131
+ out = rel.refresh(session.runtime, session.tenant, username=session.uname,
132
+ all_connections=True)
133
  # ⭐⭐ W30-T31 — AND THEN THE ROWS THAT SHOULD NOT BE HERE LEAVE AGAIN.
134
  #
135
  # The spawn writes every bucket's rows straight into the tenant document by mutating it
 
212
  # while those were the only two; the 2026-08-09 widening added products and orders, and a
213
  # hard-coded list here would have reported "everything is fine" over two databases it had
214
  # stopped knowing about ([[gate-answers-the-wrong-question]] — the hard-coded count).
215
+ declared = list(rel.TABLES)
216
+ declared_keys = {key for _bucket, key, _label, _fields in declared}
217
+ # Include every namespaced/discovered Odoo database already present in this tenant. The old
218
+ # compiled-only loop made connection two invisible and therefore falsely healthy by omission.
219
+ for key, table in tables.items():
220
+ connector = table.get("connector") if isinstance(table, dict) else None
221
+ if (key not in declared_keys and isinstance(connector, dict)
222
+ and connector.get("key") == "odoo"):
223
+ declared.append(("stored", key, str(table.get("label") or key), lambda: ()))
224
+ declared_keys.add(key)
225
+
226
+ health_cache = {}
227
+ for _bucket, key, _label, _fields in declared:
228
  table = tables.get(key) or {}
229
  rows = (table.get("rows") or {})
230
  stamps = [str((r or {}).get("refreshed") or "") for r in rows.values()]
 
235
  # says a worklist that quietly stopped updating must be legible without a refresh. So a
236
  # read-through table is counted from the MIRROR and says where its count came from.
237
  materialised = user_tables.materialises(key, defn=table)
238
+ spec = None
239
+ mirror_refreshed = ""
240
  if not materialised:
241
  try:
242
  rows = {}
 
250
  order_by=spec["id"], limit=1, cur=cur, **frm)["total"]
251
  else:
252
  n = -1 # unknown, and never reported as zero
253
+ mirror_refreshed = mirror_stamp(
254
+ spec or {}, cur=cur, table_key=key, rt=session.runtime,
255
+ defn=(table if isinstance(table, dict) and table else None))
256
  except Exception: # noqa: BLE001
257
  n = -1
258
+ connection_id = str(((table.get("connector") or {}).get("connectionId")
259
+ if isinstance(table, dict) else "") or "")
260
+ if not connection_id:
261
+ try:
262
+ connection_id = str(rel.connection_for_table_key(key, session.runtime) or "default")
263
+ except Exception: # noqa: BLE001
264
+ connection_id = "default"
265
+ if connection_id not in health_cache:
266
+ try:
267
+ import routes_connectors as _connectors
268
+ receipt = _connectors.odoo_health_receipt(session.runtime, connection_id)
269
+ health_cache[connection_id] = {
270
+ str(item.get("databaseKey") or ""): item
271
+ for item in (receipt.get("databases") or ())}
272
+ except Exception: # noqa: BLE001
273
+ health_cache[connection_id] = {}
274
+ db_health = health_cache[connection_id].get(key) or {
275
+ "databaseKey": key, "state": "unknown", "lastSuccess": None,
276
+ "currentLagSeconds": None, "failure": None, "stages": {}}
277
+ refreshed = (max([s for s in stamps if s], default="") if materialised
278
+ else mirror_refreshed)
279
  out[key] = {
280
  "exists": bool(table),
281
  "rows": (len(rows) if materialised else n),
282
+ "rowCount": (len(rows) if materialised else n),
283
  # ⭐ W31-T45: a refused mirror says `refused`, never `mirror` — a size that came from
284
  # nowhere must not be labelled with the source it did not come from.
285
  "rowsFrom": ("document" if materialised
 
287
  "materialised": materialised,
288
  # why this grid still keeps its rows in the tenant document, when it does not have to
289
  "materialisedBecause": (why_not.get(key, "") if materialised else ""),
290
+ "refreshed": refreshed,
291
+ # The row-level receipt carries an exact instant; mirror metadata may be date-only.
292
+ "recordRefreshed": db_health.get("recordRefreshed") or refreshed,
293
+ "lastSuccess": db_health.get("lastSuccess"),
294
+ "currentLagSeconds": db_health.get("currentLagSeconds"),
295
+ "failure": db_health.get("failure"),
296
+ "stages": db_health.get("stages") or {},
297
+ "connectionId": connection_id,
298
  "locked": table.get("recordMode") == user_tables.AUTOMATION_RECORD_MODE,
299
  # W30/R7: whether this database is served THROUGH the mirror rather than from the copy
300
  # above. Reported per table, because they convert one at a time and an operator
 
364
  from harness import datastore as _ds
365
  block["present"] = bool(_ds.DB_PATH.exists())
366
  if not block["present"]:
367
+ _main = running_main() # D-781: resolve, never import
368
+ state = dict(getattr(_main, "MIRROR_SEED", None) or {})
369
  block["manifest"] = "absent"
370
  block["cause"] = state.get("cause") or "this container has no analytical mirror yet"
371
  block["recommendation"] = state.get("recommendation") or (
 
504
  block = {"armed": False, "tenant": "", "tenantSource": "", "tenantSeconds": None,
505
  "envSeconds": None, "resolvedSeconds": None, "manual": False,
506
  "disagreement": None, "passes": 0, "lastPassAt": None, "lastPassOk": None,
507
+ "lastSuccessAt": None, "lastError": None, "bootSyncError": None,
508
+ "connectionId": "", "entities": None,
509
  "entitiesCause": "", "stuck": []}
510
  try:
511
+ _main = running_main() # D-781: resolve, never import
512
  block.update({k: v for k, v in dict(_main.RESYNC_STATE).items() if k in block})
513
  cadence = block.get("resolvedSeconds") or _main._RESYNC_DEFAULT_SECONDS
514
  except Exception: # noqa: BLE001
 
594
  #: ⚠ A provider registers a CALLABLE, not a dict, so its module is not imported until the first
595
  #: read — the same lazy rule `_sources` already follows for `odoo_relational`.
596
  _SOURCE_PROVIDERS = []
597
+ _SOURCE_ANALYSES = {}
598
 
599
 
600
  def register_source_provider(fn):
 
608
  return len(_SOURCE_PROVIDERS)
609
 
610
 
611
+ def register_source_analysis(source_key, *, folds, spawned):
612
+ """Register lazy `() -> {table_key: reason}` and `() -> set[table_key]` analyses.
613
+
614
+ Source ids come from core.registry's policies. Re-registering replaces that source's pair
615
+ without adding another entry. Returns the number of sources whose analysis is now known.
616
+ """
617
+ if callable(folds) and callable(spawned):
618
+ _SOURCE_ANALYSES[source_key] = {"folds": folds, "spawned": spawned}
619
+ return len(_SOURCE_ANALYSES)
620
+
621
+
622
+ def _source_analysis(source_key):
623
+ """Resolve one source's analysis lazily; an unregistered source has two empty results."""
624
+ analysis = _SOURCE_ANALYSES.get(source_key)
625
+ if analysis is None:
626
+ return {}, set()
627
+ return analysis["folds"](), analysis["spawned"]()
628
+
629
+
630
  def _sources():
631
  """Build `GRID_SOURCES` lazily by asking EVERY registered provider — Odoo is one of them.
632
 
 
998
  return GRID_SOURCES
999
 
1000
 
1001
+ def _odoo_spawned():
1002
+ """The Odoo grains that have a materialised row builder."""
1003
+ rel = _rel()
1004
+ # ⛔ "WAS THIS PART OF THE MATERIALISED SPAWN?", and being in `TABLES` STOPPED ANSWERING IT.
1005
+ # W30-T35 declared the two line grains, which have no python row builder anywhere — their
1006
+ # absence from `_READERS` IS that statement — so a table the spawn could never materialise
1007
+ # started reading as spawned, and on a mirror-less deployment fell through to "its size could
1008
+ # not be read" and came back INELIGIBLE. That inverts eligibility's first law (size
1009
+ # forces read-through; it never blocks it). Ask the question through the reader map, which is
1010
+ # what actually decides whether a row could ever have been built.
1011
+ return {key for _b, key, _l, _f in rel.TABLES if _b in getattr(rel, "_READERS", {})}
1012
+
1013
+
1014
  #: Odoo registers itself, at import, exactly as any other connector does. ⚠ The order providers
1015
  #: are registered in is the order their specs land; keys are namespaced by connector (`ut_odoo_`,
1016
  #: `ut_meta_`), so a later provider cannot shadow an earlier one's grid.
1017
  register_source_provider(_odoo_sources)
1018
+ register_source_analysis("odoo", folds=lambda: _fold_reasons(_rel()), spawned=_odoo_spawned)
1019
 
1020
 
1021
  def sync_read_through(cur=None):
 
1025
  registration is additive and `strip_materialised` is a no-op once a table is empty.
1026
 
1027
  ⛔ IT IS ALSO THE ONLY PLACE THAT MAY CALL `register_read_through`, because the eligibility
1028
+ question needs BOTH halves — the source's registered analysis (for the fold matrix) and
1029
  the mirror (for the size) — and `core` may import neither.
1030
  """
1031
  eligible, reasons = _unmaterialisable(cur)
 
1117
 
1118
  ⛔ SIZE FORCES READ-THROUGH; IT NEVER BLOCKS IT — and getting that backwards was a real bug in
1119
  the first cut of this function. A table too big for the tenant document has NO materialised
1120
+ option at all, so making it ineligible would have handed the materialisation plan a `row_limit`
1121
  of None and invited it to build 963,783 python dicts. The window ceiling is a different, much
1122
  softer thing: it only limits what the whole-table CLIENT door can serve today.
1123
 
 
1129
  ⚠ Without a mirror cursor the size questions cannot be asked, so only a grid that was never
1130
  part of the materialised spawn is eligible — a table is never freed by a question we skipped.
1131
  """
1132
+ from core import registry
 
 
 
 
 
 
 
 
 
1133
  from harness import datastore
1134
  ut = _ut()
1135
+ analyses, eligible, reasons = {}, {}, {}
1136
+ candidates = dict(_sources())
1137
+ if any(policy["candidates"] == "prefix" for policy in registry.source_policies()):
1138
+ # ⚠ Every declared policy is "declared" today, so this walk does not run. It is here so a
1139
+ # source CAN opt in without editing this branch, which is the ticket. The store read is
1140
+ # guarded because a policy is a declaration and a missing store is not its problem.
1141
+ try:
1142
+ owned = list(ut.all_defs() or {})
1143
+ except Exception: # noqa: BLE001
1144
+ owned = []
1145
+ for key in owned:
1146
+ if registry.source_policy(key)["candidates"] == "prefix":
1147
+ candidates.setdefault(key, {})
1148
+ for key, spec in candidates.items():
1149
+ policy = registry.source_policy(key)
1150
+ max_rows = policy["max_rows"] if policy["max_rows"] is not None else ut.MAX_ROWS
1151
+ window_max = (policy["window_max"] if policy["window_max"] is not None
1152
+ else datastore.WINDOW_MAX)
1153
+ source_key = policy["key"]
1154
+ if source_key not in analyses:
1155
+ analyses[source_key] = _source_analysis(source_key)
1156
+ folds, spawned = analyses[source_key]
1157
+ # ⛔ A CANDIDATE WITH NO SPEC CANNOT BE JUDGED, AND MUST THEREFORE BE REFUSED, NOT FREED.
1158
+ # A `candidates: "prefix"` source contributes keys its provider never described, so the
1159
+ # size question below has nothing to ask. Falling through would reach the `total is None
1160
+ # and key not in spawned` arm and hand back ELIGIBLE, un-materialising a database whose
1161
+ # size was never read - the exact inversion of this function's first law, arrived at by
1162
+ # an absence rather than a decision. [S65-T13]
1163
+ if not (spec.get("table") or spec.get("from_sql")):
1164
+ eligible[key] = False
1165
+ reasons[key] = ("its source declares no read-through binding, so its size cannot be "
1166
+ "read and un-materialising it would be a guess")
1167
+ continue
1168
  total = None
1169
  if cur is not None:
1170
  try:
 
1174
  order_by=spec["id"], limit=1, cur=cur, **frm)["total"]
1175
  except Exception: # noqa: BLE001
1176
  total = None
1177
+ if total is not None and total > max_rows:
1178
  eligible[key] = True # no other home exists; this is not a choice
1179
  continue
1180
  if total is None and key not in spawned:
 
1185
  why.append("its size could not be read from the mirror on this deployment")
1186
  if key in folds:
1187
  why.append(folds[key])
1188
+ if total is not None and total > window_max:
1189
+ why.append(f"its {total:,} rows exceed the {window_max:,}-row window and "
1190
  f"the grid client still asks for whole tables, so un-materialising it "
1191
  f"today could only truncate")
1192
  eligible[key] = not why
 
2303
  return tree, sets
2304
 
2305
 
2306
+ def _ranks_as_cohorts(tree, conj, spec, cols, base_where, wall_params, cur, cohort_sets,
2307
+ had_cohort, today, limits=None):
2308
+ """Resolve every ACTIVE rank leaf (Top N, quartile, above average ...) into a synthetic
2309
+ `__cohort__` leaf over the ids it selects. `(tree, cohort_sets)`.
2310
+
2311
+ A rank is a property of the SET, so `filter_sql` refuses it at row grain. The set is answered
2312
+ here the way the client answers it: the residue (every non-rank condition) under the wall is
2313
+ the domain, and `filter_eval.resolve_rank_sets` (the Python port of the client's set pass,
2314
+ ties by ascending pid) picks the members. Only the id and the ranked columns are read, uncapped.
2315
+ """
2316
+ fe = _fe()
2317
+ leaves = fe._rank_leaves(tree or [], cols)
2318
+ if not leaves:
2319
+ return tree, cohort_sets
2320
+ if had_cohort:
2321
+ raise err(400, "filter_unsupported",
2322
+ "a condition on cohort membership cannot be evaluated on this database's live "
2323
+ "connection, because a cohort is resolved from this tenant's own records rather "
2324
+ "than from the connection. Filter on a saved view instead, or open this "
2325
+ "database's stored copy")
2326
+ residue = fe._strip_rank_leaves(tree)
2327
+ try:
2328
+ pred = _fs().compile_filter_tree(residue, conj=conj, columns=cols, id_sql=spec["id"],
2329
+ today=today, cohort_sets=cohort_sets)
2330
+ except ValueError as exc:
2331
+ raise err(400, "filter_unsupported", str(exc))
2332
+ if pred is not None and pred.uses_aggregate:
2333
+ raise err(400, "filter_unsupported",
2334
+ "This condition cannot be evaluated for individual rows.")
2335
+ where = base_where
2336
+ params = list(wall_params)
2337
+ if pred is not None:
2338
+ where = f"({where}) AND {pred.sql}" if where else pred.sql
2339
+ params.extend(pred.params)
2340
+ frm_body = spec["from_sql"] if spec.get("from_sql") else spec["table"]
2341
+ answers = {}
2342
+ python_leaves = []
2343
+ for n in leaves:
2344
+ got = _rank_ids_sql(n, cols, spec["id"], frm_body, where, params, cur)
2345
+ if got is None:
2346
+ python_leaves.append(n)
2347
+ else:
2348
+ answers[id(n)] = got
2349
+ if python_leaves:
2350
+ # A GROUP-scoped rank (rare on this door) keeps the reference resolver: read the id and
2351
+ # the ranked columns once, uncapped, and let `resolve_rank_sets` answer it.
2352
+ ranked = sorted({str(n.get("colId")) for n in python_leaves if n.get("colId") in cols})
2353
+ select = ", ".join([spec["id"]] + [cols[c]["sql"] for c in ranked])
2354
+ sql = f"SELECT {select} FROM {frm_body}" + (f" WHERE {where}" if where else "")
2355
+ rows = []
2356
+ for r in cur.execute(sql, params).fetchall():
2357
+ if not r or r[0] is None:
2358
+ continue
2359
+ row = {"pid": int(r[0])}
2360
+ for i, c in enumerate(ranked, start=1):
2361
+ row[c] = r[i]
2362
+ rows.append(row)
2363
+ resolved = fe.resolve_rank_sets({"nodes": python_leaves, "conj": "and"}, rows, cols)
2364
+ for n in python_leaves:
2365
+ answers[id(n)] = sorted(resolved.sets.get(fe._rank_key(n)) or ())
2366
+ sets = dict(cohort_sets or {})
2367
+ counter = [0]
2368
+ wanted = {id(n) for n in leaves}
2369
+
2370
+ def rewrite(nodes):
2371
+ out = []
2372
+ for n in (nodes or []):
2373
+ if isinstance(n, dict) and isinstance(n.get("children"), list):
2374
+ out.append(dict(n, children=rewrite(n["children"])))
2375
+ elif isinstance(n, dict) and id(n) in wanted:
2376
+ counter[0] += 1
2377
+ key = "__rank_%d__" % counter[0]
2378
+ got = answers.get(id(n))
2379
+ sets[key] = got if got is not None else []
2380
+ if (limits is not None and isinstance(got, list)
2381
+ and len(got) >= _PIN_REPORT_AT):
2382
+ limits.append({
2383
+ "subject": str(n.get("colId") or ""), "effect": "slow",
2384
+ "cause": f"this ranking condition selects {len(got):,} rows, and "
2385
+ f"each page of this database names every one of them in the "
2386
+ f"query that narrows it",
2387
+ "recommendation": "rank a smaller slice (a lower Top N or percentage), "
2388
+ "or add a condition beside it"})
2389
+ out.append({"colId": "__cohort__", "op": "anyOf", "value": key})
2390
+ else:
2391
+ out.append(n)
2392
+ return out
2393
+
2394
+ return rewrite(tree), sets
2395
+
2396
+
2397
+ def _rank_ids_sql(leaf, cols, id_sql, frm_body, where, params, cur):
2398
+ """One GLOBAL rank leaf as SQL: a `SqlMembership` selecting its ids, or None for the resolver.
2399
+
2400
+ Measured 2026-09-23: `resolve_rank_sets` over 256,542 order lines took 3.6 s in Python, and a
2401
+ paged grid pays it on every page; as a SQL window the same answer is tens of milliseconds,
2402
+ and as a subquery it never becomes a six-figure IN list. Semantics are the resolver's, line
2403
+ for line (parity-tested over all eight ops with ties and blanks): blank or non-finite values
2404
+ are ABSENT, ties break by ascending id in both directions, a bound outside the builder's
2405
+ whole-number range is unanswerable (empty), and a quartile or decile of fewer rows than
2406
+ buckets is empty. The domain's count and mean are read once here and inlined as literals.
2407
+ """
2408
+ fe = _fe()
2409
+ Sql = _fs().SqlMembership
2410
+ empty = Sql("SELECT NULL WHERE FALSE", [])
2411
+ scope, _field = fe._rank_scope(leaf, None)
2412
+ if scope != "global":
2413
+ return None
2414
+ col = cols.get(leaf.get("colId")) or {}
2415
+ if col.get("type") not in fe.NUMERIC_TYPES:
2416
+ return empty
2417
+ v = f"TRY_CAST({col['sql']} AS DOUBLE)"
2418
+ dom = (f"SELECT {id_sql} AS pid, {v} AS v FROM {frm_body} WHERE "
2419
+ + (f"({where}) AND " if where else "") + f"{v} IS NOT NULL AND isfinite({v})")
2420
+ params = list(params)
2421
+ op = leaf.get("op")
2422
+ if op in ("topN", "bottomN"):
2423
+ bound = fe._rank_bound(leaf.get("value"), 1, 10000)
2424
+ if bound is None:
2425
+ return empty
2426
+ direction = "DESC" if op == "topN" else "ASC"
2427
+ return Sql(f"SELECT pid FROM ({dom}) d ORDER BY v {direction}, pid ASC LIMIT {bound}",
2428
+ params)
2429
+ count, total = cur.execute(f"SELECT count(*), sum(v) FROM ({dom}) d", params).fetchone()
2430
+ count = int(count or 0)
2431
+ if not count:
2432
+ return empty
2433
+ if op in ("inTopPct", "inBottomPct"):
2434
+ pct = fe._rank_bound(leaf.get("value"), 1, 100)
2435
+ if pct is None:
2436
+ return empty
2437
+ take = min(count, max(1, math.ceil(count * pct / 100)))
2438
+ direction = "DESC" if op == "inTopPct" else "ASC"
2439
+ return Sql(f"SELECT pid FROM ({dom}) d ORDER BY v {direction}, pid ASC LIMIT {take}",
2440
+ params)
2441
+ if op in ("aboveAvg", "belowAvg"):
2442
+ mean = float(total) / count
2443
+ cmp = ">" if op == "aboveAvg" else "<"
2444
+ return Sql(f"SELECT pid FROM ({dom}) d WHERE v {cmp} {mean!r}", params)
2445
+ if op in ("inQuartile", "inDecile"):
2446
+ buckets = 4 if op == "inQuartile" else 10
2447
+ wanted = fe._rank_bound(leaf.get("value"), 1, buckets)
2448
+ if wanted is None or count < buckets:
2449
+ return empty
2450
+ return Sql(f"SELECT pid FROM (SELECT pid, row_number() OVER (ORDER BY v ASC, pid ASC) - 1"
2451
+ f" AS idx FROM ({dom}) d) r WHERE least({buckets}, (idx * {buckets}) // {count}"
2452
+ f" + 1) = {int(wanted)}", params)
2453
+ return None
2454
+
2455
  def _may_see_view(session, vid, store):
2456
  """May this caller open that saved view? The same predicate the view rail asks.
2457
 
 
2692
  # the reader is held to, exactly as `viewSets` resolves inside the rows the client holds) and
2693
  # before `_leaf_cols` below, so the leaf never reaches the sentence that explains a column with
2694
  # no mirror expression. One place learns it; `/rows` and `/aggregates` share this compiler.
2695
+ _had_cohort = bool(_active_cohort_leaves(tree))
2696
  tree, cohort_sets = _views_as_cohorts(tree, table_key, session, spec, cols,
2697
  base_where, wall_params, cur, limits)
2698
+ tree, cohort_sets = _ranks_as_cohorts(
2699
+ tree, (filterConj if filterConj in ("and", "or") else "and"), spec, cols, base_where,
2700
+ wall_params, cur, cohort_sets, _had_cohort, _today(), limits)
2701
 
2702
  # ── the predicate, pushed down ────────────────────────────────────────────────────────────
2703
  # ⛔ `_PSEUDO_COLS` is subtracted BEFORE the missing-column test. `__cohort__` and `__view__`
 
2822
  "cols": bindings, "where": None}
2823
  cols = _filter_columns(spec, fields)
2824
  limits = list(payload.get("limits") or [])
2825
+ _had_cohort = bool(_active_cohort_leaves(query["filters"]))
2826
  tree, cohorts = _views_as_cohorts(query["filters"], table_key, session, spec, cols,
2827
  None, [], cur, limits)
2828
+ tree, cohorts = _ranks_as_cohorts(tree, query["filterConj"], spec, cols, None, [], cur,
2829
+ cohorts, _had_cohort,
2830
+ payload.get("today") or _today(), limits)
2831
  missing = _leaf_cols(tree) - _PSEUDO_COLS - set(cols)
2832
  if missing:
2833
  raise err(400, "filter_unsupported", "A filter names a column unavailable to this account.")
 
2848
  "frm": {"table": "pinned_rows"}, "limits": limits, "payload": payload}
2849
 
2850
 
2851
+ def _group_pin(cols, group_by, group_key):
2852
+ """`(sql, params)` narrowing a window to ONE group of `/aggregates`' grouping, or `(None, [])`.
2853
+
2854
+ HOTFIX 2026-09-23 (owner: "whatever is GROUPED we can add pagination per group"). A group's
2855
+ records page through this door, so the narrowing is the SAME expression `_aggregate_over`
2856
+ groups by (`filter_sql.group_sql`), never a client-built `eq` leaf: `_value_sql` lowercases
2857
+ text and rounds numbers, which would make a group's pages disagree with its count.
2858
+ `group_key` is the JSON of the key `/aggregates` put on the wire (`null` = the blank bucket).
2859
+ """
2860
+ # A plain-Python caller that omits the argument hands us FastAPI's `Query` sentinel, not
2861
+ # None (the `related` scar in `routes_connected_tables`), so anything but a string is "none".
2862
+ gcol = group_by.strip() if isinstance(group_by, str) else ""
2863
+ if not gcol:
2864
+ return None, []
2865
+ group_key = group_key if isinstance(group_key, str) else None
2866
+ gspec = cols.get(gcol)
2867
+ if gspec is None:
2868
+ raise err(400, "bad_group", "this column cannot be grouped on this database's live "
2869
+ "connection, or it is not visible to your account")
2870
+ try:
2871
+ key = json.loads(group_key) if group_key not in (None, "") else None
2872
+ except ValueError:
2873
+ raise err(400, "bad_group", "groupKey must be the JSON of a group key")
2874
+ gexpr = _fs().group_sql(gspec)
2875
+ if key is None:
2876
+ return f"({gexpr}) IS NULL", []
2877
+ if gspec["type"] in _fs().NUMERIC_TYPES:
2878
+ try:
2879
+ return f"({gexpr}) = ?", [float(key)]
2880
+ except (TypeError, ValueError):
2881
+ raise err(400, "bad_group", "groupKey must be a number for a number column")
2882
+ return f"({gexpr}) = ?", [str(key)]
2883
+
2884
+
2885
+ def _materialised_window(table_key, session, offset, limit, filters, filterConj, sorts, search, related,
2886
+ groupBy=None, groupKey=None):
2887
  from harness import datastore
2888
 
2889
  query = _selection_query({"filters": _json_arg(filters, "filters"), "filterConj": filterConj,
 
2891
  with _materialised_read(table_key, session, query, related) as read:
2892
  spec = read["spec"]
2893
  order = _fs().compile_order_by(query["sorts"], read["cols"], tiebreak_sql=spec["id"])
2894
+ where, params = read["where"], list(read["params"])
2895
+ gsql, gparams = _group_pin(read["cols"], groupBy, groupKey)
2896
+ if gsql:
2897
+ where = f"({where}) AND {gsql}" if where else gsql
2898
+ params.extend(gparams)
2899
+ win = datastore.window(select=f'{spec["id"]} AS pid', where=where,
2900
+ params=tuple(params), order_by=order or spec["id"],
2901
  offset=offset, limit=limit or None, cur=read["cur"], **read["frm"])
2902
  payload = read["payload"]
2903
  by_id = {int(row["pid"]): row for row in payload["rows"]}
 
2920
  sorts: str = Query(default=None),
2921
  search: str = Query(default=None),
2922
  related: str = Query(default=None),
2923
+ groupBy: str = Query(default=None),
2924
+ groupKey: str = Query(default=None),
2925
  session: Session = Depends(require_session)):
2926
  """ONE WINDOW over a connected grid, read straight from the mirror (contract C2).
2927
 
 
2952
 
2953
  if _materialised_table(table_key):
2954
  return _materialised_window(table_key, session, offset, limit, filters, filterConj,
2955
+ sorts, search, related, groupBy, groupKey)
2956
 
2957
  # ⛔⛔ ONE COMPILER, AND THE AGGREGATES DOOR CALLS THE SAME ONE. The permission wall, the
2958
  # hidden-field closure, the user predicate and the search all live in `_compiled_read` above,
 
2966
  cols, limits, frm = read["cols"], read["limits"], read["frm"]
2967
  base_where, wall_params = read["base_where"], read["wall_params"]
2968
  where, params, sort_spec = read["where"], read["params"], read["sort_spec"]
2969
+ _gsql, _gparams = _group_pin(cols, groupBy, groupKey)
2970
+ if _gsql:
2971
+ where = f"({where}) AND {_gsql}" if where else _gsql
2972
+ params = list(params) + list(_gparams)
2973
 
2974
  # ── the order, made TOTAL ─────────────────────────────────────────────────────────────────
2975
  # ⛔ `tiebreak_sql` is not optional here: without a total order, LIMIT/OFFSET may return one
 
3008
  # ⭐ D-155 — the freshness stamp rides the SAME builder, so both read-through doors answer the
3009
  # `refreshed` column identically instead of one of them leaving it blank.
3010
  rows_src = _pool_from_window(spec, win,
3011
+ stamp=mirror_stamp(spec, cur, table_key, defn=defn, rt=
3012
  session.runtime))
3013
  overlays = {str(r["pid"]): {k: v for k, v in r.items() if k != "pid"} for r in rows_src}
3014
 
 
3372
  return body_out
3373
 
3374
 
3375
+ def mirror_stamp(spec, cur=None, table_key="", rt=None, defn=None):
3376
  """⭐ D-155 — WHEN THIS READ-THROUGH GRID'S DATA WAS LAST RECONCILED AGAINST ODOO, as a date.
3377
 
3378
  ⛔ THE COLUMN EXISTED AND THE BINDING DID NOT, which is the whole of D-155. Every Odoo table
 
3417
  # the materialised carve-out above, made structural. ⚠ Fail-QUIET to `""`: when we cannot
3418
  # tell whether this grid stores rows, the safe answer is the behaviour that shipped (blank),
3419
  # never a stamp that might overwrite a per-row one.
3420
+ # HOTFIX 2026-09-23: `defn` is LENT by a caller already holding the definition. Without it
3421
+ # `materialises` falls through to a whole-document `user_tables` read, measured on staging
3422
+ # at ~3 s of every 50-row page of order lines (the `timing` phase `pool_and_stamp`).
3423
  if table_key:
3424
  try:
3425
+ if _ut().materialises(str(table_key), st=rt, defn=defn):
3426
  return ""
3427
  except Exception: # noqa: BLE001
3428
  return ""
 
3539
  f"{table_key}: {win['total']:,} rows is more than one {datastore.WINDOW_MAX:,}-row "
3540
  f"window, and this database no longer stores rows in the tenant document; it can "
3541
  f"only be read a page at a time (`/odoo-tables/{table_key}/rows`)")
3542
+ # HOTFIX 2026-09-23: this door serves only a table whose rows have LEFT the tenant document
3543
+ # (its docstring), so that fact is lent rather than re-derived: without `defn`,
3544
+ # `materialises` re-reads the whole `user_tables` document, ~3 s on the pg store.
3545
+ return _pool_from_window(spec, win, stamp=mirror_stamp(spec, cur, table_key, rt,
3546
+ defn={"readThrough": True}))
3547
 
3548
 
3549
  def _fe():
api/routes_shares.py CHANGED
@@ -90,10 +90,13 @@ wall that does not exist for this kind. Scoping user tables is booked, not done
90
  *"Shared with me"*, who may open a source View, and who may re-share. Saved Views carry no
91
  view-level permissions member or secondary access model.
92
  """
 
 
93
  from fastapi import APIRouter, Body, Depends
94
 
95
  import core.shares as shares
96
  import core.users as users
 
97
  from deps import Session, err, require_session
98
  # ⭐ W32-T28 (C3) — the SHARE notification's topic word, imported from the module that CLASSIFIES
99
  # it (`routes_alerts.notification_view`) rather than typed again here. The producer and the
@@ -103,6 +106,11 @@ from routes_alerts import SHARE_TOPIC as _SHARE_TOPIC
103
 
104
  router = APIRouter(prefix="/api/v1")
105
 
 
 
 
 
 
106
 
107
  def _kind_or_400(raw):
108
  try:
@@ -249,11 +257,10 @@ def _field_resolve(session, table_key, field_key, strata=_FIELD_STRATA):
249
  none, because residency IS authorship in a per-account document.
250
  4. **the connector catalogue** — the columns the database itself declares.
251
 
252
- ⚠ `strata` NARROWS THE WALK, AND IT IS A COST CONTROL AS WELL AS A CORRECTNESS ONE.
253
- `_share_view_fields` asks per VIEW KEY, and `_catalogue_fields` re-parses the canonical
254
- contract file on every built-in call — so an unfiltered walk would `json.loads`
255
- `aios_grid_fields.json` once per column per view share, where before this ticket it was zero.
256
- That arm passes `("shared", "mine")`, which is exactly the pair it could see before W50-T23.
257
 
258
  ⛔ EACH STRATUM RESOLVES IN ITS OWN `try`, AND THE BARE `except Exception: pass` THAT WRAPPED
259
  ALL OF THEM IS GONE. It conflated *"the store threw"* with *"no such field"*, so one
@@ -765,54 +772,12 @@ def _unremovable(held_rows, entries, uname):
765
  stuck.add(user)
766
  return stuck
767
 
768
- # ── ⭐⭐ WAVE 42 · T25 — SHARING A VIEW SHARES THE FIELDS IT NAMES ──────────────────────────────
769
- #
770
- # A saved view is a SELECTION expressed in columns: an order, a visible set, a group, a colour,
771
- # a sort and a filter tree, every one of which names a field key. Granting the view alone hands
772
- # the receiver a rail row whose columns resolve to nothing they may read, so the feature that
773
- # looked shipped puts an empty grid on their screen. This block closes that, and it refuses
774
- # LOUDLY in the three places where a grant would otherwise be a lie.
775
- #
776
- # ⛔⛔ TRAP 1 — A FIELD GRANT IS TOPIC-WIDE, NOT VIEW-SCOPED. It is keyed
777
- # `shares.field_oid(topic, key)`, so it hands the receiver that column on EVERY view of that
778
- # database, for as long as the grant stands. That is the existing grant primitive and this
779
- # ticket does not narrow it; what it does is refuse to let the response PRETEND otherwise, which
780
- # is why `sharedFieldScope` rides back beside the list.
781
- #
782
- # ⛔⛔ TRAP 2 — THE SHARER CANNOT GRANT A COLUMN THEY DO NOT OWN. `field` is absent from
783
- # `shares.RESHARE_KINDS` and `_owns_object`'s field branch demands `createdBy == session.uname`,
784
- # so a view built over a colleague's column simply cannot carry that column with it. Silently
785
- # skipping it ships an empty column; silently granting it is a permission hole. It is REFUSED
786
- # BY NAME, inside the 200 response, with the sentence that says who to ask.
787
- #
788
- # ⛔⛔ TRAP 3 — A GRANT CANNOT UN-HIDE AN ADMIN-WALLED COLUMN. `perm_scope.hidden_keys` UNIONS
789
- # the administrator's `hiddenFields` with the field-grant source, so granting a walled column
790
- # answers 200, writes the record, and the receiver still sees nothing: granted with no effect,
791
- # which is the failure this ticket exists to end. Detected BEFORE the grant, refused by name,
792
- # and the receiver it is walled from is named too.
793
- #
794
- # ⚠ THE WALL IS READ FROM THE ADMIN SOURCE DIRECTLY AND NEVER THROUGH `hidden_keys`, AND THAT IS
795
- # NOT AN OPTIMISATION. `hidden_keys` unions `field_grant_hidden` — "hidden because you were not
796
- # granted it" — which is true of every column this route is ABOUT to grant. Asking it here would
797
- # refuse the whole set on the grounds that the grant has not happened yet.
798
-
799
-
800
- def _view_configs(view):
801
- """The dicts a saved view keeps its column state in.
802
-
803
- `SavedView.config` is the contract (`types.ts::ViewConfig`) and is where every key below
804
- actually lives. The record ITSELF is read as well because a legacy row that stored the same
805
- members flat costs one dict lookup to cover and would otherwise hand back an empty key set,
806
- which is the shape that grants nothing and reports nothing wrong.
807
- """
808
- out = []
809
- if not isinstance(view, dict):
810
- return out
811
- cfg = view.get("config")
812
- if isinstance(cfg, dict):
813
- out.append(cfg)
814
- out.append(view)
815
- return out
816
 
817
 
818
  def _view_field_keys(view):
@@ -826,45 +791,57 @@ def _view_field_keys(view):
826
  ⚠ AND `filters` IS A TREE, NOT A LIST. Condition groups nest under `children`, so the walk
827
  recurses rather than iterating once: a column named only inside a group is still named.
828
  """
829
- keys, seen = [], set()
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
830
 
831
- def add(raw):
 
 
832
  key = str(raw or "").strip()
833
- if key and key not in seen:
834
- seen.add(key)
835
- keys.append(key)
836
-
837
- def walk(node):
838
- if isinstance(node, list):
839
- for child in node:
840
- walk(child)
841
- return
842
- if not isinstance(node, dict):
843
- return
844
- kids = node.get("children")
845
- if isinstance(kids, list):
846
- walk(kids)
847
- add(node.get("colId"))
848
- rhs = node.get("rhs")
849
- if isinstance(rhs, dict) and rhs.get("kind") != "stat":
850
- add(rhs.get("colId"))
851
-
852
- for cfg in _view_configs(view):
853
- for key in (cfg.get("order") or ()):
854
- add(key)
855
- for key in (cfg.get("visible") or ()):
856
- add(key)
857
- walk(cfg.get("filters"))
858
- add(cfg.get("groupBy"))
859
- colour = cfg.get("colorBy")
860
- add(colour.get("colId") if isinstance(colour, dict) else colour)
861
- sorts = cfg.get("sorts")
862
- if isinstance(sorts, dict):
863
- sorts = [sorts]
864
- for spec in (sorts or ()):
865
- if isinstance(spec, dict):
866
- add(spec.get("colId"))
867
- return keys
868
 
869
 
870
  def _find_view_topic(session, oid):
@@ -891,7 +868,7 @@ def _find_view_topic(session, oid):
891
  return None, None
892
 
893
 
894
- def _promote_field_for_grant(session, table_key, field_key):
895
  """Promote a private column exactly once, stamp it shared, and answer its GRANT OID.
896
 
897
  ⛔⛔ ONE SEQUENCE, TWO ARMS, AND THAT IS THE WHOLE REASON THIS IS A FUNCTION. `put_share`'s
@@ -921,8 +898,8 @@ def _promote_field_for_grant(session, table_key, field_key):
921
  person who did not make it. It was a 404 before this ticket and it stays one.
922
  """
923
  from core import field_permissions, shared_overlay
924
- defn, stratum, workspace_key, shared_key, grant_topic = _field_resolve(
925
- session, table_key, field_key)
926
  if not isinstance(defn, dict) or stratum == "colleague":
927
  return None
928
  if stratum == "mine":
@@ -936,38 +913,6 @@ def _promote_field_for_grant(session, table_key, field_key):
936
  return shares.field_oid(grant_topic, field_key)
937
 
938
 
939
- def _grant_receivers(session, entries):
940
- """`{username: display name}` for everybody a grant list actually reaches.
941
-
942
- `shares.EVERYONE` is a real audience, not a placeholder, so it expands to the tenant roster
943
- `_people` already computes for the picker. Reading it from a second place would be a second
944
- answer to "who is in this workspace".
945
- """
946
- people = None
947
- out = {}
948
- for entry in shares._clean_entries(entries):
949
- who = str(entry.get("user") or "").strip().lower()
950
- if not who:
951
- continue
952
- if who == shares.EVERYONE:
953
- if people is None:
954
- people = _people(session.tenant)
955
- for person in people:
956
- out.setdefault(str(person["username"]).strip().lower(), person["name"])
957
- continue
958
- out.setdefault(who, who)
959
- if out:
960
- try:
961
- reg = users.registry() or {}
962
- except Exception: # noqa: BLE001
963
- reg = {}
964
- for uname in list(out):
965
- rec = reg.get(uname)
966
- if isinstance(rec, dict) and rec.get("name"):
967
- out[uname] = str(rec["name"])
968
- return out
969
-
970
-
971
  def _admin_walled(session, grant_topic, defs, receivers):
972
  """`{field key: [display names]}` for receivers an ADMINISTRATOR has already walled off.
973
 
@@ -1020,187 +965,74 @@ def _admin_walled(session, grant_topic, defs, receivers):
1020
  return walled
1021
 
1022
 
1023
- #: ⭐ THE THREE REASONS A VIEW'S COLUMN IS NOT GRANTED, EACH ITS OWN CODE. One fused
1024
- #: "could not share" would make the three walls unprovable apart, which is the
1025
- #: [[a-declared-gate-is-an-unchecked-claim]] failure `verify_field_doors` section 8 exists to
1026
- #: prevent: every cause fails by its own name, or it is not gated at all.
1027
- #: ⛔ STANDING RULE 2 — no em dash and no en dash in any sentence below; these reach a screen.
1028
- FIELD_REFUSAL_CODES = ("not_a_shared_column", "not_owned", "admin_hidden")
1029
-
1030
- #: ⭐ TRAP 1, SAID OUT LOUD. A column grant is keyed by DATABASE, so it is not scoped to the view
1031
- #: that occasioned it. The response carries this sentence rather than letting a caller infer a
1032
- #: narrower promise than the store actually makes.
1033
- FIELD_GRANT_SCOPE = ("A column shared this way is shared on the whole database, so the people "
1034
- "you named can see it on every view of it, not only on this one.")
1035
-
1036
-
1037
- def _refusal(key, code, names=()):
1038
- """One refused column, with the sentence a person reads."""
1039
- if code == "not_a_shared_column":
1040
- why = ('"{k}" belongs to the database itself rather than to a person, so there was '
1041
- 'nothing to grant. Anybody who can open the database already sees it.')
1042
- elif code == "not_owned":
1043
- why = ('"{k}" is somebody else\'s column, and only its owner (or an administrator) can '
1044
- 'share it. Ask them to share it, then share this view again.')
1045
- else:
1046
- why = ('"{k}" is hidden from {who} by an administrator, so sharing it would change '
1047
- 'nothing. Ask an administrator to unhide it first.')
1048
- return {"field": key, "reason": code,
1049
- "message": why.format(k=key, who=_named(names) if names else "them")}
1050
-
1051
-
1052
- def _classify_view_fields(keys, owned, defined, walled):
1053
- """`(sharedFieldKeys, refusedFieldKeys)` for the columns a view names. PURE.
1054
-
1055
- ⛔ EVERY KEY IS DECIDED BEFORE ANY OF THEM IS WRITTEN, which is the same property the ceiling
1056
- block above claims for the three role walls: a refusal must not be able to leave part of the
1057
- set promoted and part of it untouched.
1058
-
1059
- ⚠ THE ORDER OF THE CAUSES IS FIXED AND IT DECIDES WHICH SENTENCE A PERSON READS. A built-in
1060
- column is owned by nobody, so "there was nothing to grant" has to be asked before "you do not
1061
- own it", or every database column in `visible` would be reported as somebody else's.
1062
- """
1063
- granted, refused = [], []
1064
- for key in keys:
1065
- if key not in defined:
1066
- refused.append(_refusal(key, "not_a_shared_column"))
1067
- elif key not in owned:
1068
- refused.append(_refusal(key, "not_owned"))
1069
- elif walled.get(key):
1070
- refused.append(_refusal(key, "admin_hidden", walled.get(key)))
1071
- else:
1072
- granted.append(key)
1073
- return granted, refused
1074
-
1075
-
1076
- def _merge_field_entries(existing, incoming):
1077
- """A column's grant list with `incoming` ADDED to it. Nobody is removed, nobody is lowered.
1078
-
1079
- ⛔⛔ THE SHARE PUT REPLACES, AND A COLUMN'S AUDIENCE IS NOT THE VIEW'S AUDIENCE. This route's
1080
- contract is a REPLACING set (`PUT ... entries` at the top of the file), which is right for the
1081
- object the caller is looking at and catastrophic for the columns underneath it: a column grant
1082
- is TOPIC-WIDE, so handing the view's list straight to `set_grants` would rewrite the column's
1083
- audience to whoever this view happens to be shared with. A colleague granted that column
1084
- through the `field` door would lose it, on every view of the database, silently, inside a 200.
1085
- That is exactly the harm D-474 closed at the view door, arriving one door over.
1086
-
1087
- ⚠ AND A LENGTH CHECK DOES NOT SEE IT. Replacing a one-person list with a different one-person
1088
- list keeps the count identical, so the gate asserts the HELD USER IS STILL THERE AT THEIR OWN
1089
- ROLE rather than counting rows.
1090
-
1091
- ⭐ RAISED, NEVER LOWERED. A receiver already held at `view` who arrives on an `edit` share is
1092
- raised, because only the column's owner (or an administrator) reaches this path at all and
1093
- that is the access they just asked to hand over. An `edit` holder meeting a `view` share keeps
1094
- `edit`: a share is an addition, and nothing here is a revocation.
1095
-
1096
- ⚠ `by` RIDES ON EXISTING ROWS AND IS NEVER TAKEN FROM AN INCOMING ONE. Provenance decides who
1097
- may revoke a grant (`_unremovable`), so an incoming payload must not be able to claim it;
1098
- `set_grants` stamps a genuinely new row from `granter`.
1099
- """
1100
- rank = {"view": 0, "edit": 1}
1101
- order, merged = [], {}
1102
- for row in (existing or ()):
1103
- if not isinstance(row, dict):
1104
- continue
1105
- who = str(row.get("user") or "").strip().lower()
1106
- role = str(row.get("role") or "").strip().lower()
1107
- if not who or role not in rank or who in merged:
1108
- continue
1109
- kept = {"user": who, "role": role}
1110
- stamp = str(row.get("by") or "").strip().lower()
1111
- if stamp:
1112
- kept["by"] = stamp
1113
- merged[who] = kept
1114
- order.append(who)
1115
- for row in (incoming or ()):
1116
- if not isinstance(row, dict):
1117
- continue
1118
- who = str(row.get("user") or "").strip().lower()
1119
- role = str(row.get("role") or "").strip().lower()
1120
- if not who or role not in rank:
1121
- continue
1122
- if who not in merged:
1123
- merged[who] = {"user": who, "role": role}
1124
- order.append(who)
1125
- elif rank[role] > rank[merged[who]["role"]]:
1126
- merged[who]["role"] = role
1127
- return [merged[who] for who in order]
1128
 
 
 
 
1129
 
1130
- def _share_view_fields(session, oid, entries):
1131
- """The `view` arm's field half: classify every column the view names, then grant the ones
1132
- that survive. `(sharedFieldKeys, refusedFieldKeys)`.
1133
 
1134
- ⛔ CLASSIFY EVERYTHING, THEN WRITE. `_classify_view_fields` sees the whole key set before
1135
- `_promote_field_for_grant` touches a single definition, so a refusal cannot leave half the
1136
- view's columns promoted.
1137
 
1138
- ⚠ THE VIEW'S OWN GRANT HAS ALREADY BEEN WRITTEN when this runs, and that order is the safe
1139
- one: a `409 grants_changed` on the view must not be preceded by a set of column promotions
1140
- for a share that then did not happen.
 
 
 
 
1141
  """
1142
  topic, view = _find_view_topic(session, oid)
1143
  if not topic or not isinstance(view, dict):
1144
- return [], []
1145
  keys = _view_field_keys(view)
1146
- if not keys:
1147
- return [], []
1148
  table_key = f"{topic}_table_workspace"
1149
  grant_topic = _field_storage_keys(table_key)[2]
1150
- defs, defined, owned, owners = {}, set(), set(), {}
 
 
 
 
 
 
 
1151
  for key in keys:
1152
- defn, stratum = _field_resolve(session, table_key, key, strata=("shared", "mine"))[:2]
1153
- # ⛔⛔ W50-T23 — THE VIEW ARM SEES EXACTLY THE TWO STRATA IT SAW BEFORE THIS TICKET.
1154
- # `_field_resolve` added `catalogue` and `colleague`, and BOTH are regressions here:
1155
- # * `catalogue` on a built-in topic is the WHOLE canonical contract, so an admin
1156
- # sharing one ordinary view would `defined` every contract column, promote each into
1157
- # `<topic>__shared` and stamp `granted: True` — the exact mark
1158
- # `perm_scope.field_grant_hidden` walls on. One view share would turn the product
1159
- # contract into grant-governed columns and hide them from everybody without a grant.
1160
- # W42-T25 states the intent in as many words: *"Throwing instead would make the
1161
- # ordinary case (a view over one built-in column) unshareable."*
1162
- # * `colleague` would put a refusal row per colleague-private column into the dialog's
1163
- # refused list, for columns the sharer never named.
1164
- # The `field` arm keeps both, because there the column IS the request.
1165
- if not isinstance(defn, dict) or stratum in ("catalogue", "colleague"):
1166
  continue
1167
- defs[key] = defn
1168
- defined.add(key)
1169
- # ⚠ THE COLUMN'S OWNER, NOT THIS SESSION. `set_grants`' owner is sticky, so stamping the
1170
- # SHARER here would hand an administrator sharing somebody else's column the ownership
1171
- # of it as a side effect of sharing a view. Same resolver the claim wall uses.
1172
- owners[key] = _field_owner(session, defn, stratum == "shared", stratum=stratum)
1173
- # ⛔ ONE OWNERSHIP PREDICATE. `_owns_object` is the wall the `field` arm claims through
1174
- # and the DELETE door reads; a second "does this person own this column" here is how the
1175
- # two answers come apart. It short circuits True for an administrator, which is right:
1176
- # an administrator may already share any column through the `field` door.
1177
- if _owns_object(session, "field", shares.field_oid(grant_topic, key)):
1178
- owned.add(key)
1179
- walled = _admin_walled(session, grant_topic, defs, _grant_receivers(session, entries))
1180
- granted, refused = _classify_view_fields(keys, owned, defined, walled)
1181
- written = []
1182
- for key in granted:
1183
- field_oid = _promote_field_for_grant(session, table_key, key)
1184
- if not field_oid:
1185
- refused.append(_refusal(key, "not_a_shared_column"))
1186
  continue
1187
- # ⛔⛔ ADDED TO, NEVER REPLACED. `set_grants` writes the list it is given, and this route
1188
- # PUTs a REPLACING set, so handing it the view's `entries` would make the view's audience
1189
- # the COLUMN's audience: a colleague granted this column through the `field` door would
1190
- # lose it, on every view of the database, inside a 200. `_merge_field_entries` says what
1191
- # that sentence means once, and it is the only writer of a column list on this path.
1192
- held = shares.grants("field", field_oid, st=session.runtime)
1193
- # ⚠ NO `expect` HERE, AND THAT IS NOT AN OVERSIGHT. The compare-and-set on the view's own
1194
- # grant guards a record TWO PEOPLE ARE EDITING through the same dialog. A column's record
1195
- # was just read (and possibly created) inside this call, nobody is looking at it in a
1196
- # dialog, and a `409` raised at this point would fail a PUT whose view grant has ALREADY
1197
- # been written, which is the half-applied state the refusals above exist to avoid.
1198
- shares.set_grants("field", field_oid,
1199
- _merge_field_entries(held.get("entries"), entries),
1200
- owner=held.get("owner") or owners.get(key) or session.uname,
1201
- granter=session.uname, st=session.runtime)
1202
- written.append(key)
1203
- return written, refused
 
 
1204
 
1205
 
1206
  @router.get("/share/mine")
@@ -1253,7 +1085,16 @@ def get_share(kind: str, oid: str, session: Session = Depends(require_session)):
1253
  # administrator-shaped Farhan session must not turn Shantal's absent grant into a record.
1254
  # Other share kinds retain their established administrator behaviour.
1255
  _admin_share = bool(session.admin) and kind != "cohort"
1256
- role = shares.role_for(kind, oid, session.uname, is_admin=_admin_share, st=session.runtime)
 
 
 
 
 
 
 
 
 
1257
  may_admin = shares.may_administer(kind, oid, session.uname, is_admin=_admin_share,
1258
  st=session.runtime)
1259
  # W39-T29 — the dialog opens with GET before its first PUT. Until that PUT exists the grant
@@ -1561,6 +1402,44 @@ def put_share(kind: str, oid: str, body: dict = Body(default=None),
1561
  # write rather than at the moment of the read: a concurrent save lands between the two, and
1562
  # a wave-40 adversarial probe drove a re-sharer's PUT being ACCEPTED while the grant the
1563
  # owner had just added disappeared. See `shares.set_grants`' own note.
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1564
  try:
1565
  out = shares.set_grants(kind, oid, entries,
1566
  owner=session.uname if claiming else rec["owner"],
@@ -1570,20 +1449,6 @@ def put_share(kind: str, oid: str, body: dict = Body(default=None),
1570
  except shares.GrantsChanged as exc:
1571
  raise err(409, "grants_changed", str(exc))
1572
  _notify_new_grantees(session, kind, oid, before=rec["entries"], after=out.get("entries") or [])
1573
- # ⭐⭐ W42-T25 — SHARING A VIEW SHARES THE FIELDS IT NAMES. A view is a selection expressed
1574
- # in columns, so a grant that carries the view and none of its columns puts a rail row on
1575
- # the receiver's screen over an empty grid. The three refusals are named rather than thrown:
1576
- # this PUT succeeded, the view IS shared, and what could not follow it is reported by name
1577
- # with the reason. Throwing instead would make the ordinary case (a view over one built-in
1578
- # column) unshareable.
1579
- # ⛔ THE EXISTING KEYS ARE UNTOUCHED. `{owner, entries}` is what every current client reads
1580
- # off this route; the field report is additive, and a caller that ignores it is unaffected.
1581
- if kind == "view":
1582
- shared_fields, refused_fields = _share_view_fields(session, oid, entries)
1583
- return {**out,
1584
- "sharedFieldKeys": shared_fields,
1585
- "refusedFieldKeys": refused_fields,
1586
- "sharedFieldScope": FIELD_GRANT_SCOPE}
1587
  return out
1588
 
1589
 
 
90
  *"Shared with me"*, who may open a source View, and who may re-share. Saved Views carry no
91
  view-level permissions member or secondary access model.
92
  """
93
+ import secrets
94
+
95
  from fastapi import APIRouter, Body, Depends
96
 
97
  import core.shares as shares
98
  import core.users as users
99
+ from core import field_permissions as _field_permissions
100
  from deps import Session, err, require_session
101
  # ⭐ W32-T28 (C3) — the SHARE notification's topic word, imported from the module that CLASSIFIES
102
  # it (`routes_alerts.notification_view`) rather than typed again here. The producer and the
 
106
 
107
  router = APIRouter(prefix="/api/v1")
108
 
109
+ # C8's derived field leg belongs to the field-permission seam, while every existing read/write
110
+ # wall already calls ``shares.role_for``. Install it explicitly at application-router startup;
111
+ # no caller gets a second permission predicate and non-field kinds are byte-for-byte delegated.
112
+ _field_permissions.install_view_field_resolver()
113
+
114
 
115
  def _kind_or_400(raw):
116
  try:
 
257
  none, because residency IS authorship in a per-account document.
258
  4. **the connector catalogue** — the columns the database itself declares.
259
 
260
+ ⚠ `strata` NARROWS THE WALK, AND IT IS A COST CONTROL AS WELL AS A CORRECTNESS ONE. A caller
261
+ that already knows which definition strata are relevant can avoid the remaining reads; the
262
+ View share preflight deliberately checks all four because unreadable references must refuse
263
+ the whole transaction rather than be misclassified as database-owned.
 
264
 
265
  ⛔ EACH STRATUM RESOLVES IN ITS OWN `try`, AND THE BARE `except Exception: pass` THAT WRAPPED
266
  ALL OF THEM IS GONE. It conflated *"the store threw"* with *"no such field"*, so one
 
772
  stuck.add(user)
773
  return stuck
774
 
775
+ # ── SDLC63 / C8 — ONE VIEW AUDIENCE, ONE DERIVED FIELD BOUNDARY ───────────────────────────────
776
+ # The pre-SDLC63 route copied a View audience into every named field grant and returned every
777
+ # field/refusal to the browser. The current path resolves the deliberate field set once,
778
+ # preflights the editor's own readability, and commits only the View audience plus its compact
779
+ # field boundary. Field detail leaves the server only as a count on success; failures retain it
780
+ # under a support reference rather than turning a large View into a large error message.
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
781
 
782
 
783
  def _view_field_keys(view):
 
791
  ⚠ AND `filters` IS A TREE, NOT A LIST. Condition groups nest under `children`, so the walk
792
  recurses rather than iterating once: a column named only inside a group is still named.
793
  """
794
+ return _field_permissions.view_field_keys(view)
795
+
796
+
797
+ def _view_field_definitions(session, table_key, keys):
798
+ """Resolve a View's field definitions with a constant number of source reads."""
799
+ workspace_key, shared_key, grant_topic = _field_storage_keys(table_key)
800
+ shared, mine, document, catalogue = {}, {}, {}, {}
801
+ try:
802
+ from core import shared_overlay
803
+ shared = shared_overlay.fields(shared_key, st=session.runtime) or {}
804
+ except Exception: # noqa: BLE001
805
+ pass
806
+ try:
807
+ import core.table_store as table_store
808
+ mine = (table_store.make(workspace_key, st=session.runtime)
809
+ .workspace(session.uname).get("fields") or {})
810
+ except Exception: # noqa: BLE001
811
+ pass
812
+ try:
813
+ document = session.runtime.get(workspace_key) or {}
814
+ except Exception: # noqa: BLE001
815
+ pass
816
+ try:
817
+ catalogue = _catalogue_fields(session, table_key, workspace_key, grant_topic)
818
+ except Exception: # noqa: BLE001
819
+ pass
820
 
821
+ me = str(session.uname or "").strip().lower()
822
+ out = {}
823
+ for raw in keys or ():
824
  key = str(raw or "").strip()
825
+ defn, stratum = None, None
826
+ if isinstance(shared, dict) and isinstance(shared.get(key), dict):
827
+ defn, stratum = shared[key], "shared"
828
+ elif isinstance(mine, dict) and isinstance(mine.get(key), dict):
829
+ defn, stratum = mine[key], "mine"
830
+ elif isinstance(document, dict):
831
+ for member, workspace in document.items():
832
+ if str(member).strip().lower() == me or not isinstance(workspace, dict):
833
+ continue
834
+ theirs = (workspace.get("fields") or {}).get(key)
835
+ if isinstance(theirs, dict):
836
+ defn = dict(theirs)
837
+ if not str(defn.get("createdBy") or "").strip():
838
+ defn["createdBy"] = str(member).strip().lower()
839
+ stratum = "colleague"
840
+ break
841
+ if defn is None and isinstance(catalogue, dict) and isinstance(catalogue.get(key), dict):
842
+ defn, stratum = catalogue[key], "catalogue"
843
+ out[key] = (defn, stratum, workspace_key, shared_key, grant_topic)
844
+ return out
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
845
 
846
 
847
  def _find_view_topic(session, oid):
 
868
  return None, None
869
 
870
 
871
+ def _promote_field_for_grant(session, table_key, field_key, resolved=None):
872
  """Promote a private column exactly once, stamp it shared, and answer its GRANT OID.
873
 
874
  ⛔⛔ ONE SEQUENCE, TWO ARMS, AND THAT IS THE WHOLE REASON THIS IS A FUNCTION. `put_share`'s
 
898
  person who did not make it. It was a 404 before this ticket and it stays one.
899
  """
900
  from core import field_permissions, shared_overlay
901
+ defn, stratum, workspace_key, shared_key, grant_topic = (
902
+ resolved if resolved is not None else _field_resolve(session, table_key, field_key))
903
  if not isinstance(defn, dict) or stratum == "colleague":
904
  return None
905
  if stratum == "mine":
 
913
  return shares.field_oid(grant_topic, field_key)
914
 
915
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
916
  def _admin_walled(session, grant_topic, defs, receivers):
917
  """`{field key: [display names]}` for receivers an ADMINISTRATOR has already walled off.
918
 
 
965
  return walled
966
 
967
 
968
+ class _ViewSharePreparationError(RuntimeError):
969
+ """Server-only field details for a refused all-or-none View share."""
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
970
 
971
+ def __init__(self, details):
972
+ self.details = [str(value) for value in (details or ()) if str(value)]
973
+ super().__init__("one or more View fields could not be prepared")
974
 
 
 
 
975
 
976
+ def _prepare_view_share(session, oid, *, preview=False):
977
+ """Resolve one View and make its personal definitions tenant-readable before the CAS.
 
978
 
979
+ Database-owned fields need no field write and some connector columns have no overlay
980
+ definition at all. A personal definition is promoted while it
981
+ still has no derived audience; if a later transaction fails, it remains an owner-controlled
982
+ shared definition rather than leaking a value. Resolved unreadable references refuse the
983
+ whole operation and ride only in server diagnostics; keys absent from definition strata are
984
+ database-owned fields already admitted by the validated View-save door.
985
+ Preview performs the same admission checks without promoting personal definitions.
986
  """
987
  topic, view = _find_view_topic(session, oid)
988
  if not topic or not isinstance(view, dict):
989
+ raise _ViewSharePreparationError(["view:not_found"])
990
  keys = _view_field_keys(view)
 
 
991
  table_key = f"{topic}_table_workspace"
992
  grant_topic = _field_storage_keys(table_key)[2]
993
+ details = []
994
+ definitions = {}
995
+ database_owned = 0
996
+ resolved = _view_field_definitions(session, table_key, keys)
997
+ shared_roles = _field_permissions.field_roles_for_keys(
998
+ grant_topic,
999
+ [key for key, row in resolved.items() if row[1] == "shared"],
1000
+ session.uname, is_admin=session.admin, st=session.runtime)
1001
  for key in keys:
1002
+ row = resolved.get(key) or (None, None, None, None, grant_topic)
1003
+ defn, stratum = row[:2]
1004
+ if not isinstance(defn, dict):
1005
+ # The validated View-save door already intersected references with the editor's
1006
+ # served schema. Absence from the overlay/catalogue resolver therefore means a
1007
+ # database-owned column, not a private definition to publish. It belongs in the
1008
+ # compact boundary and needs no per-field ownership or grant mutation.
1009
+ database_owned += 1
 
 
 
 
 
 
1010
  continue
1011
+ definitions[key] = defn
1012
+ if stratum == "catalogue":
1013
+ database_owned += 1
1014
+ if stratum == "colleague":
1015
+ details.append(f"{key}:not_readable")
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1016
  continue
1017
+ if stratum == "shared":
1018
+ if (shared_roles.get(key) or {}).get("role") is None:
1019
+ details.append(f"{key}:not_readable")
1020
+ continue
1021
+ if stratum == "mine" and not preview:
1022
+ field_oid = _promote_field_for_grant(session, table_key, key, resolved=row)
1023
+ if not field_oid:
1024
+ details.append(f"{key}:promotion_failed")
1025
+
1026
+ # The account's permanent hidden-field wall wins even for catalogue fields. The View save
1027
+ # path already removes these references; repeating that answer here keeps a stale or imported
1028
+ # View from becoming a permission bypass at its first share.
1029
+ walled = _admin_walled(
1030
+ session, grant_topic, definitions,
1031
+ {session.uname: _display_name(session.uname)})
1032
+ details.extend(f"{key}:not_readable" for key in sorted(walled))
1033
+ if details:
1034
+ raise _ViewSharePreparationError(details)
1035
+ return topic, view, grant_topic, keys, database_owned
1036
 
1037
 
1038
  @router.get("/share/mine")
 
1085
  # administrator-shaped Farhan session must not turn Shantal's absent grant into a record.
1086
  # Other share kinds retain their established administrator behaviour.
1087
  _admin_share = bool(session.admin) and kind != "cohort"
1088
+ if kind == "field":
1089
+ table_key, field_key = shares.split_field_oid(oid)
1090
+ # C8's inherited path grants the data, not the column's Manage Access roster. Use the
1091
+ # direct leg here so a View audience cannot enumerate a field owner's separate grants.
1092
+ role = _field_permissions.field_role_sources(
1093
+ table_key, field_key, session.uname,
1094
+ is_admin=_admin_share, st=session.runtime)["direct"]
1095
+ else:
1096
+ role = shares.role_for(kind, oid, session.uname,
1097
+ is_admin=_admin_share, st=session.runtime)
1098
  may_admin = shares.may_administer(kind, oid, session.uname, is_admin=_admin_share,
1099
  st=session.runtime)
1100
  # W39-T29 — the dialog opens with GET before its first PUT. Until that PUT exists the grant
 
1402
  # write rather than at the moment of the read: a concurrent save lands between the two, and
1403
  # a wave-40 adversarial probe drove a re-sharer's PUT being ACCEPTED while the grant the
1404
  # owner had just added disappeared. See `shares.set_grants`' own note.
1405
+ if kind == "view":
1406
+ fields = []
1407
+ stage = "prepare"
1408
+ try:
1409
+ _topic, view, grant_topic, fields, database_owned = _prepare_view_share(session, oid)
1410
+ stage = "transaction"
1411
+ out = _field_permissions.set_view_grants_with_boundary(
1412
+ view, grant_topic, entries,
1413
+ owner=session.uname if claiming else rec["owner"],
1414
+ granter=session.uname, expect=rec["entries"], st=session.runtime)
1415
+ except shares.GrantsChanged as exc:
1416
+ raise err(409, "grants_changed", str(exc))
1417
+ except Exception as exc: # noqa: BLE001
1418
+ # The browser gets one short, actionable sentence. Field names and the internal
1419
+ # cause remain in the tenant-scoped diagnostic ring under the same support reference.
1420
+ reference = f"VS-{secrets.token_hex(4).upper()}"
1421
+ details = getattr(exc, "details", None) or fields
1422
+ try:
1423
+ _field_permissions.record_view_share_diagnostic(
1424
+ reference, oid, session.uname, stage, details, exc,
1425
+ st=session.runtime)
1426
+ except Exception: # noqa: BLE001
1427
+ pass
1428
+ raise err(
1429
+ 500, "view_share_failed",
1430
+ "We could not update this View's access. Try again. If it keeps failing, "
1431
+ f"contact support with reference {reference}.")
1432
+ _notify_new_grantees(
1433
+ session, kind, oid, before=rec["entries"], after=out.get("entries") or [])
1434
+ # One field-free compatibility reason keeps older API callers from mistaking connector
1435
+ # columns for promoted custom definitions. It is never rendered and never scales with
1436
+ # the field count; all actual field detail stays server-side under a support reference.
1437
+ compatibility = ([{"reason": "not_a_shared_column"}] if database_owned else [])
1438
+ return {"owner": out.get("owner"), "entries": out.get("entries") or [],
1439
+ "fieldShare": {"includedCount": int(out.get("fieldCount") or 0),
1440
+ "databaseOwnedCount": database_owned},
1441
+ "refusedFieldKeys": compatibility}
1442
+
1443
  try:
1444
  out = shares.set_grants(kind, oid, entries,
1445
  owner=session.uname if claiming else rec["owner"],
 
1449
  except shares.GrantsChanged as exc:
1450
  raise err(409, "grants_changed", str(exc))
1451
  _notify_new_grantees(session, kind, oid, before=rec["entries"], after=out.get("entries") or [])
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1452
  return out
1453
 
1454
 
api/routes_sheet.py CHANGED
@@ -44,6 +44,7 @@ import threading
44
  import uuid
45
 
46
  from fastapi import APIRouter, Body, Depends, File, HTTPException, UploadFile
 
47
 
48
  from deps import Session, err, require_session
49
 
@@ -974,7 +975,36 @@ def _binding_scope_key(session: Session, database: str, binding: dict) -> str:
974
  import core.perm_scope as perm_scope
975
  wall_tree = ((perm_scope.entry(session.user, database) or {}).get("filter")
976
  if perm_scope.row_scope_applies(session.user, database) else None)
977
- return sheets.scope_hash(database, wall_tree, (binding or {}).get("scope") or {})
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
978
 
979
 
980
  def _has_ending(binding):
@@ -988,128 +1018,131 @@ def _has_cell_inputs(binding):
988
  return True
989
  scope = binding.get("scope") if isinstance(binding, dict) else None
990
  nodes, _ = _fe_parts(scope.get("filter") if isinstance(scope, dict) else None)
 
 
 
991
  # Linked values and related-table grants can change independently of the orders' cache key.
992
  if sheet_mod.linked_filter_ids(nodes):
993
  return True
994
  return any(sheet_mod.filter_cell_nodes(nodes))
995
 
996
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
997
  def _reresolved_for_reader(session: Session, book, *, endings_only=False):
998
- """A COPY of `book` whose every bound cell was recomputed under THIS reader's scope (C5).
999
-
1000
- ⛔⛔ IN MEMORY, AND IT MUST NEVER PERSIST. `refresh_sheet` runs the same loop and ends with
1001
- `put_workbook`, and a read path that copied that body would write the READER's numbers into
1002
- the OWNER's document: the same leak as serving stale owner values, in the opposite direction,
1003
- caused by a GET, and therefore reachable by anybody the workbook was shared with. So this
1004
- works on a `copy.deepcopy` and returns it without saving the reader's workbook values.
1005
- R10 may create a missing Metric field through the database's ordinary field writer.
1006
-
1007
- ⭐ THE WALL RUNS ONCE PER DISTINCT DATABASE, exactly as `refresh_sheet` does (R16): a reader
1008
- who has lost access to a bound database is refused on every cell of it rather than shown the
1009
- owner's cached number.
1010
-
1011
- ⚠ `refreshedFor` IS STAMPED ONLY WHEN A BOUND CELL WAS ACTUALLY RECOMPUTED. A workbook of
1012
- literals has nothing to re-resolve, and a stamp there would be a claim with nothing behind it.
1013
-
1014
- ⭐⭐ W52-T12 / R3: A CACHE HIT SKIPS `_resolve_binding` ENTIRELY, AND A MISS FILLS ONLY THE
1015
- CALLER'S OWN SLOT. The cache lives in a document `core.sheets` owns and keys by
1016
- `(database, wall, binding scope)` (`_binding_scope_key`), never by user id or role name, so
1017
- two readers who share a row restriction share an answer and a reader whose restriction differs
1018
- never reads the other's slot. Only `refresh_sheet` may ever clear it; a GET here only ever
1019
- fills the ONE slot the caller themselves is entitled to.
1020
- """
1021
- from core import sheets
1022
- import copy as _copy
1023
- fresh = _copy.deepcopy(book)
1024
- stamp = _now_iso()
1025
- seen_db = {}
1026
- # ⭐ The same per-request memo as `refresh_sheet`, for the same reason and with the same
1027
- # lifetime: this loop resolves cell by cell too, and a reader's GET must not rebuild one
1028
- # database's workspace assembly once per bound cell.
1029
- view_memo = {}
1030
- touched = 0
1031
- store = _store(session)
1032
- wid = book.get("id") or ""
1033
- for tab in fresh.get("sheets") or ():
1034
- tab_ref = str(tab.get("id") or tab.get("name") or "")
1035
  for addr, cell in (tab.get("cells") or {}).items():
1036
- if not isinstance(cell, dict) or cell.get("kind") != "bound":
1037
  continue
1038
- if endings_only and not _has_cell_inputs(cell.get("binding")):
1039
  continue
1040
- touched += 1
1041
  binding = cell.get("binding") or {}
1042
  db = str(binding.get("database") or "")
1043
- if db not in seen_db:
1044
  try:
1045
  _require_database(session, db)
1046
- seen_db[db] = None
1047
- except Exception as exc: # noqa: BLE001
1048
- seen_db[db] = exc
1049
- failure = seen_db.get(db)
1050
- if failure is not None:
1051
- cell.update(_binding_refusal("database_unreadable", f"You cannot read {db}."))
1052
- cell.pop("limits", None)
1053
- continue
1054
-
1055
- cell_key = tab_ref + "!" + addr
1056
- scope_key = _binding_scope_key(session, db, binding)
1057
- hit = None
1058
- if wid and scope_key and not _has_cell_inputs(binding):
 
 
 
 
 
 
 
 
1059
  try:
1060
- hit = sheets.cached_scope_value(store, wid, cell_key, scope_key)
1061
- except Exception: # noqa: BLE001
1062
- hit = None
1063
- if hit is not None:
1064
- cell["state"] = "fresh"
1065
- cell["value"] = hit.get("value")
1066
- cell["display"] = "" if hit.get("value") is None else str(hit.get("value"))
1067
- cell["refreshed_at"] = hit.get("refreshedAt")
1068
- cell.pop("refusal", None)
1069
- #: ⚠ NOT CLAIMED ON A CACHE HIT: `limits` describes THIS resolve's own truncation,
1070
- #: and a cached slot never recorded one, so a prior (possibly stale) value on the
1071
- #: cell is dropped rather than shown as if it described the answer just served.
1072
- cell.pop("limits", None)
1073
- continue
1074
-
1075
- try:
1076
- out = _resolve_cell_binding(session, binding, fresh, tab_ref, view_memo=view_memo)
1077
- except HTTPException as exc:
1078
- detail = getattr(exc, "detail", None) or {}
1079
- info = detail.get("error") if isinstance(detail, dict) else None
1080
- cell.update(_binding_refusal(
1081
- (info or {}).get("code", "sheet_unresolved"),
1082
- (info or {}).get("message", "This cell could not be refreshed."),
1083
- ))
1084
- cell.pop("limits", None)
1085
- continue
1086
- cell["state"] = "fresh"
1087
- cell["value"] = out["value"]
1088
- cell["display"] = "" if out["value"] is None else str(out["value"])
1089
- cell["refreshed_at"] = stamp
1090
- cell.pop("refusal", None)
1091
- if out["limits"]:
1092
- cell["limits"] = out["limits"]
1093
  else:
1094
- cell.pop("limits", None)
1095
- #: ⚠ FILLED AFTER THE CELL ABOVE IS DONE READING `out`, and best effort: a cache-write
1096
- #: failure only costs a future recompute, never a wrong number, so it never raises past
1097
- #: here.
1098
- if wid and scope_key:
1099
- try:
1100
- sheets.fill_scope_cache(store, wid, cell_key, scope_key, out["value"], stamp,
1101
- out["database"])
1102
- except Exception: # noqa: BLE001
1103
- pass
1104
- if not touched:
1105
- return fresh
1106
- # ⭐ EVERY FORMULA IS RE-EVALUATED TOO, over the numbers this reader is entitled to. Leaving
1107
- # them alone would serve the owner's totals sitting on top of the reader's inputs, which is
1108
- # the leak this whole function exists to close, one layer up.
1109
- from modules import sheet as sheet_mod
1110
- import formula_runtime as fr
1111
- _recompute_formulas(fresh, fr, sheet_mod)
1112
- fresh["refreshedFor"] = str(session.uname or "")
 
 
 
1113
  return fresh
1114
 
1115
 
@@ -1121,17 +1154,12 @@ def _response_workbook(book):
1121
 
1122
 
1123
  def _read_for(session: Session, book):
1124
- """The workbook this session is entitled to see: stored for the owner, re-resolved otherwise.
1125
 
1126
  W60-T09, 2026-09-14: R28 checks visibility before this projection. Admitted legacy readers
1127
- and grantees still receive their own scoped numbers, never the owner's cached numbers.
1128
  """
1129
  book = _response_workbook(book)
1130
- if _role_on_workbook(session, book) == "owner":
1131
- if any(_has_cell_inputs(cell.get("binding"))
1132
- for tab in book.get("sheets") or [] for cell in (tab.get("cells") or {}).values()):
1133
- return _reresolved_for_reader(session, book, endings_only=True)
1134
- return book
1135
  return _reresolved_for_reader(session, book)
1136
 
1137
 
@@ -1484,7 +1512,9 @@ async def start_sheet_import(file: UploadFile = File(...),
1484
  if not name.lower().endswith(".xlsx"):
1485
  raise err(400, "import_not_xlsx",
1486
  "Only an .xlsx workbook can be imported. Save your file as .xlsx and try again.")
1487
- data = await file.read()
 
 
1488
  if len(data) > sheet_import.MAX_IMPORT_BYTES:
1489
  raise HTTPException(status_code=413, detail={"error": {
1490
  "code": "import_too_large",
@@ -1530,6 +1560,51 @@ def _mb(byte_count):
1530
  return "%.1f MB" % (float(byte_count) / (1024.0 * 1024.0))
1531
 
1532
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1533
  @router.get("/sheets/{workbook_id}")
1534
  def read_sheet(workbook_id: str, tabs: str = "",
1535
  session: Session = Depends(require_session)):
@@ -1695,6 +1770,8 @@ def _recomputed_tab_cells(session: Session, workbook_id: str, tab_ref: str, *, c
1695
  cache_key=(session.tenant, session.uname, workbook_id), changes=edits)
1696
  # Date and filter-cell edits also change database totals. Recompute all readers, then formulas.
1697
  ending_cells = set()
 
 
1698
  for source in book.get("sheets") or []:
1699
  for addr, cell in (source.get("cells") or {}).items():
1700
  binding = cell.get("binding") or {}
@@ -1703,6 +1780,11 @@ def _recomputed_tab_cells(session: Session, workbook_id: str, tab_ref: str, *, c
1703
  try:
1704
  result = _resolve_cell_binding(session, binding, book, source.get("id"))
1705
  cell.update(value=result["value"], display="" if result["value"] is None else str(result["value"]), state="fresh")
 
 
 
 
 
1706
  cell.pop("refusal", None)
1707
  except HTTPException as exc:
1708
  info = ((exc.detail or {}).get("error") or {})
@@ -1711,7 +1793,16 @@ def _recomputed_tab_cells(session: Session, workbook_id: str, tab_ref: str, *, c
1711
  info.get("message", "The Ending date could not be read. Choose a valid date."),
1712
  ))
1713
  ending_cells.add((source.get("id"), addr))
 
 
 
 
 
1714
  if ending_cells:
 
 
 
 
1715
  _recompute_formulas(book, fr, sheet_mod)
1716
  found = sheets.find_tab(book, tab_ref)
1717
  out = {}
@@ -1887,117 +1978,221 @@ def unshare_sheet(workbook_id: str, who: str, session: Session = Depends(require
1887
 
1888
  @router.post("/sheets/{workbook_id}/refresh")
1889
  def refresh_sheet(workbook_id: str, session: Session = Depends(require_session)):
1890
- """R10: recompute every bound cell, then every formula, and restamp `refreshed_at`.
1891
-
1892
- ⭐ THE WALL RUNS PER DISTINCT DATABASE, ON OPEN, NOT ONLY PER CELL (R16). A workbook that binds
1893
- forty cells to one database asks once; a reader who has lost access to that database gets a
1894
- refusal on every one of its cells rather than a stale number quietly repainting.
1895
-
1896
- ⚠ Each distinct `view_id` is resolved ONCE, and as of the wave-53 follow-up to W53-T12 that
1897
- sentence is finally TRUE OF THE CODE. It was written of `seen_db`, which deduplicates the
1898
- DATABASE WALL and nothing else: the resolve underneath it ran per cell, so a forty-cell
1899
- workbook did forty `table_store.find_view` calls and forty legacy migrations with them.
1900
- `view_memo` below is the dedupe this sentence describes.
1901
-
1902
- ⛔⛔ C5 — THIS IS A WRITE DOOR, BECAUSE IT ENDS IN `put_workbook`, AND A VIEWER IS REFUSED
1903
- HERE FOR A REASON THAT IS NOT ABOUT PERMISSION TO SEE. A viewer whose refresh were allowed
1904
- would compute every bound cell under THEIR scope and then SAVE those numbers into the owner's
1905
- document, so a narrowly walled reader pressing Refresh would silently shrink the owner's
1906
- workbook. The reader's own re-resolve happens on the READ path, in memory, and never lands.
1907
- ⚠ An EDITOR still writes their own numbers here, which is what an edit grant means and is
1908
- booked as a residual rather than widened away.
1909
  """
1910
- from core import sheets
 
1911
  from modules import sheet as sheet_mod
1912
  import formula_runtime as fr
 
1913
 
 
 
1914
  try:
1915
- book = sheets.get_workbook(_store(session), workbook_id)
1916
- _require_editor(session, book)
1917
  except sheets.SheetRefusal as exc:
1918
  raise _refuse(exc)
1919
-
 
 
 
 
1920
  stamp = _now_iso()
1921
- seen_db = {}
1922
- # ⭐ ONE MEMO FOR THE WHOLE REFRESH, keyed `(database, view_id)`. `seen_db` deduplicates the
1923
- # WALL and never the scope, so forty cells on one saved view used to resolve that view forty
1924
- # times. See `_view_filter`: the second lookup behind it builds a workspace assembly, which is
1925
- # work no workbook should pay for per cell. Per REQUEST, never on the runtime: a view's filter
1926
- # may be edited between two refreshes and a longer lived cache would serve the old rows.
1927
- view_memo = {}
1928
- for tab in book.get("sheets") or ():
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1929
  for addr, cell in (tab.get("cells") or {}).items():
1930
- if not isinstance(cell, dict) or cell.get("kind") != "bound":
1931
  continue
1932
- if _is_view_binding(cell):
1933
- continue # C2: a placed view refreshes through its own door
1934
  binding = cell.get("binding") or {}
1935
  db = str(binding.get("database") or "")
1936
- if db not in seen_db:
1937
- try:
1938
- _require_database(session, db)
1939
- seen_db[db] = None
1940
- except Exception as exc: # noqa: BLE001
1941
- seen_db[db] = exc
1942
- failure = seen_db.get(db)
1943
- if failure is not None:
1944
- cell.update(_binding_refusal("database_unreadable", f"You cannot read {db}."))
1945
- cell.pop("limits", None)
1946
- continue
1947
  try:
1948
- out = _resolve_cell_binding(session, binding, book, tab.get("id"), view_memo=view_memo)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1949
  except HTTPException as exc:
1950
- detail = getattr(exc, "detail", None) or {}
1951
- info = detail.get("error") if isinstance(detail, dict) else None
1952
- cell.update(_binding_refusal(
1953
- (info or {}).get("code", "sheet_unresolved"),
1954
- (info or {}).get("message", "This cell could not be refreshed."),
1955
- ))
1956
- cell.pop("limits", None)
1957
- continue
1958
- cell["state"] = "fresh"
1959
- cell["value"] = out["value"]
1960
- cell["display"] = "" if out["value"] is None else str(out["value"])
1961
- cell["refreshed_at"] = stamp
1962
- cell.pop("refusal", None)
1963
- if out["limits"]:
1964
- cell["limits"] = out["limits"]
1965
- else:
1966
- cell.pop("limits", None)
 
 
 
 
 
 
 
 
 
 
 
 
 
1967
 
1968
- # ⭐⭐ R4 / A1: THE RECALC IS TIMED AND THE TIME IS SHOWN. The prototype measured 2.9 s as the
1969
- # FLOOR for the owner's 11,300 formulas (2.9 to 6.2 s observed), so a refresh is a wait a
1970
- # person notices, and a wait nobody explains reads as a hang. `recalc_ms` is what the client
1971
- # renders as "Recalculated in N.N s".
1972
- #
1973
- # ⚠ IT TIMES THE RECALC, NOT THE ROUTE. The bound-cell loop above is database work whose
1974
- # duration says nothing about the formula engine, and the save below is store work; folding
1975
- # either into this number would make the figure on the screen unable to answer the question it
1976
- # exists for, which is whether the ENGINE is inside A1's 5 s target.
1977
- from time import perf_counter
1978
  started = perf_counter()
1979
  _recompute_formulas(book, fr, sheet_mod)
1980
  recalc_ms = round((perf_counter() - started) * 1000.0, 1)
1981
- try:
1982
- # R19: book came from get_workbook; clamping on refresh would destroy imported widths.
1983
- saved = sheets.put_workbook(_store(session), workbook_id,
1984
- {"name": book.get("name"), "sheets": book.get("sheets"),
1985
- "refreshed_at": stamp, "rev": book.get("rev")},
1986
- actor=session.uname,
1987
- actor_name=session.user.get("name") or session.uname,
1988
- lossless=True)
1989
- except sheets.SheetRefusal as exc:
1990
- raise _refuse(exc)
1991
- # ⭐⭐ R3 / W52-T12: REFRESH IS THE ONE EVENT ALLOWED TO INVALIDATE A READER'S CACHED SCOPE
1992
- # VALUE, so it is the one door that clears the whole per-scope cache. Every reader's next GET
1993
- # recomputes once, under their own scope, and fills its own slot again from there.
1994
- sheets.clear_scope_cache(_store(session), workbook_id)
1995
- return {"workbook": _response_workbook(saved), "recalc_ms": recalc_ms}
 
1996
 
1997
 
1998
  def _now_iso():
1999
- from datetime import datetime, timezone
2000
- return datetime.now(timezone.utc).replace(microsecond=0).isoformat()
2001
 
2002
 
2003
  #: ⛔ THE NAME A TAB GETS WHEN IT CANNOT KEEP ITS OWN, and it is built to be UNTYPEABLE on two
@@ -2288,8 +2483,11 @@ def _read_binding_cell(session, ref, *, book=None, tab=None, today=None, seen=()
2288
  refuse("the cell or its sheet is missing or blank.")
2289
  if cell.get("kind") == "bound":
2290
  try:
2291
- value = _resolve_binding(session, cell.get("binding"), today=today,
2292
- book=book, tab=key.rpartition("!")[0], ending_seen=(*seen, key))["value"]
 
 
 
2293
  except HTTPException as exc:
2294
  reason = ((exc.detail or {}).get("error") or {}).get("message", "its source is refused.")
2295
  refuse(reason)
@@ -2317,8 +2515,11 @@ def _read_binding_cell(session, ref, *, book=None, tab=None, today=None, seen=()
2317
  for dependency in needed:
2318
  attempted.add(dependency)
2319
  try:
2320
- result = _resolve_binding(session, bound[dependency].get("binding"), today=today,
2321
- book=book, tab=dependency.rpartition("!")[0], ending_seen=(*seen, key, dependency))
 
 
 
2322
  changes[dependency] = {"value": result["value"]}
2323
  except HTTPException as exc:
2324
  info = ((exc.detail or {}).get("error") or {})
@@ -2389,6 +2590,116 @@ def _resolve_cell_binding(session, binding, book, tab, **kwargs):
2389
  return _resolve_binding(session, binding, **kwargs)
2390
 
2391
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2392
  def _resolve_binding(session: Session, raw_binding, *, today=None, view_memo=None,
2393
  book=None, tab=None, ending_seen=()):
2394
  """Evaluate ONE C2 binding and return `{value, limits, sql_columns}`.
@@ -2450,6 +2761,9 @@ def _resolve_binding(session: Session, raw_binding, *, today=None, view_memo=Non
2450
  if binding["aggregate"].get("field"):
2451
  needed.add(str(binding["aggregate"]["field"]))
2452
 
 
 
 
2453
  cur, spec, cols = _mirror(session)(database, metric_window=window, needed=needed)
2454
  cols = dict(cols)
2455
  scope_nodes = _linked_filter_scope(session, database, scope_nodes, cur, spec, cols, today=today)
@@ -2491,9 +2805,12 @@ def _resolve_binding(session: Session, raw_binding, *, today=None, view_memo=Non
2491
  # spec's column expressions; applying it a second time would double-count the exclusion and
2492
  # break every joined spec, whose partner column is `sol.order_partner_id` or `aml.partner_id`.
2493
  try:
 
 
 
2494
  sql, params, limits = sheet_mod.build_aggregate_query(
2495
  spec, binding, cols, scope_nodes=scope_nodes, scope_conj=scope_conj,
2496
- wall_nodes=wall_nodes, wall_conj=wall_conj, today=today)
2497
  except sheet_mod.SheetRefusal as exc:
2498
  raise err(400, exc.code, exc.message)
2499
  except ValueError:
@@ -2534,6 +2851,50 @@ def _resolve_binding(session: Session, raw_binding, *, today=None, view_memo=Non
2534
  return {"value": value, "limits": limits, "database": database, "binding": binding}
2535
 
2536
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2537
  def _fe_parts(tree):
2538
  """`(nodes, conj)` for a stored `{conj, nodes}` filter, through the ONE reader of that shape."""
2539
  from modules import sheet as sheet_mod
@@ -2716,6 +3077,11 @@ def compute_cell(workbook_id: str, addr: str, body: dict = Body(default=None),
2716
  saved["refreshed_at"] = stamp
2717
  saved.pop("refusal", None)
2718
 
 
 
 
 
 
2719
  try:
2720
  landed = sheets.patch_cells(_store(session), workbook_id, tab_ref, {key: saved},
2721
  actor=session.uname,
@@ -2735,6 +3101,13 @@ def compute_cell(workbook_id: str, addr: str, body: dict = Body(default=None),
2735
  except Exception: # noqa: BLE001
2736
  pass
2737
 
 
 
 
 
 
 
 
2738
  wire = dict(saved)
2739
  #: ⚠ `limits` RIDES THE ANSWER AND IS NOT STORED, because `core.sheets.clean_cell` carries no
2740
  #: such key: it describes THIS resolve's truncation, not the cell.
@@ -2916,24 +3289,35 @@ def drill_cell(workbook_id: str, addr: str, session: Session = Depends(require_s
2916
  window = _ending_window(session, binding, book=book, tab=tab.get("id"))
2917
  except sheet_mod.SheetRefusal as exc:
2918
  raise err(400, exc.code, exc.message)
 
 
 
 
 
2919
  cur, spec, cols = _mirror(session)(
2920
  database, metric_window=window,
2921
- needed=(sheet_mod.filter_columns_named(scope_nodes)
2922
- | sheet_mod.filter_columns_named(wall_nodes)))
2923
  cols = dict(cols)
2924
  scope_nodes = _linked_filter_scope(session, database, scope_nodes, cur, spec, cols)
2925
 
2926
  try:
 
 
 
2927
  count_sql, count_params, limits = sheet_mod.build_aggregate_query(
2928
  spec, {"aggregate": {"fn": "count"}}, cols, scope_nodes=scope_nodes,
2929
- scope_conj=scope_conj, wall_nodes=wall_nodes, wall_conj=wall_conj)
 
 
 
2930
  except sheet_mod.SheetRefusal as exc:
2931
  raise err(400, exc.code, exc.message)
 
 
 
 
2932
  total = (cur.execute(count_sql, count_params).fetchone() or [0])[0]
2933
 
2934
- ids_sql, ids_params, _ = sheet_mod.build_ids_query(
2935
- spec, cols, scope_nodes=scope_nodes, scope_conj=scope_conj,
2936
- wall_nodes=wall_nodes, wall_conj=wall_conj)
2937
  record_ids = [int(r[0]) for r in cur.execute(ids_sql, ids_params).fetchall() if r and r[0]]
2938
 
2939
  out = {"database": database, "record_ids": record_ids, "count": int(total or 0),
 
44
  import uuid
45
 
46
  from fastapi import APIRouter, Body, Depends, File, HTTPException, UploadFile
47
+ from fastapi.responses import Response
48
 
49
  from deps import Session, err, require_session
50
 
 
975
  import core.perm_scope as perm_scope
976
  wall_tree = ((perm_scope.entry(session.user, database) or {}).get("filter")
977
  if perm_scope.row_scope_applies(session.user, database) else None)
978
+ from core.sheet_binding import fingerprint
979
+ # Compare effective access, including field grants. Equivalent callers may share a
980
+ # snapshot unless a personal workspace makes their database values caller-specific.
981
+ memo = getattr(session, "_sheet_scope_context", None)
982
+ if memo is None:
983
+ memo = session._sheet_scope_context = {}
984
+ if database not in memo:
985
+ from core import shared_overlay, shares, user_tables, view_templates
986
+ st = _store(session)
987
+ try:
988
+ if "_definitions" not in memo:
989
+ memo["_definitions"] = st.get_projection(user_tables.STORE_KEY, drop=("rows",)) or {}
990
+ memo["_grants"] = fingerprint((st.get(shares.SHARES_KEY, fresh=True) or {}).get("field"))
991
+ definition = memo["_definitions"].get(database) or {}
992
+ workspace = view_templates.workspace_key(database)
993
+ personal = (st.get_projection(workspace, drop=("overlays",)) or {}).get(session.uname) or {}
994
+ shared_key = database if database.startswith("ut_") else workspace
995
+ fields = {f["key"]: f for f in definition.get("fields", []) if f.get("key")}
996
+ fields.update(shared_overlay.fields(shared_key, st=st))
997
+ fields.update(personal.get("fields") or {})
998
+ memo[database] = [sorted(perm_scope.hidden_keys(session.user, database,
999
+ list(fields.values()), st=st)), session.uname if personal else None]
1000
+ except Exception:
1001
+ # An unprovable wall cannot read a previous slot.
1002
+ memo[database] = uuid.uuid4().hex
1003
+ policy = {key: value for key, value in session.user.items()
1004
+ if key not in ("username", "name", "email", "password", "password_hash", "epoch",
1005
+ "created_at", "updated_at", "last_login")}
1006
+ return fingerprint([sheets.scope_hash(database, wall_tree, (binding or {}).get("scope") or {}),
1007
+ binding, policy, memo[database], memo.get("_grants")])
1008
 
1009
 
1010
  def _has_ending(binding):
 
1018
  return True
1019
  scope = binding.get("scope") if isinstance(binding, dict) else None
1020
  nodes, _ = _fe_parts(scope.get("filter") if isinstance(scope, dict) else None)
1021
+ related = binding.get("related") if isinstance(binding, dict) else None
1022
+ related_nodes, _ = _fe_parts(related.get("filter") if isinstance(related, dict) else None)
1023
+ nodes = [*nodes, *related_nodes]
1024
  # Linked values and related-table grants can change independently of the orders' cache key.
1025
  if sheet_mod.linked_filter_ids(nodes):
1026
  return True
1027
  return any(sheet_mod.filter_cell_nodes(nodes))
1028
 
1029
 
1030
+ def _refresh_context(book, binding, memo=None):
1031
+ """Authored inputs and relative dates invalidate refresh, but never an open's snapshot."""
1032
+ from core.sheet_binding import fingerprint
1033
+ from datetime import date
1034
+ payload = {}
1035
+ if _has_cell_inputs(binding):
1036
+ if memo is None:
1037
+ memo = {}
1038
+ if "inputs" not in memo:
1039
+ memo["inputs"] = fingerprint([[tab.get("id"), tab.get("name"), {
1040
+ addr: {key: cell.get(key) for key in ("kind", "text", "formula", "binding", "value")
1041
+ if key != "value" or cell.get("kind") in ("literal", "empty")}
1042
+ for addr, cell in (tab.get("cells") or {}).items()}]
1043
+ for tab in book.get("sheets") or []])
1044
+ payload["inputs"] = memo["inputs"]
1045
+ # Relative filters and measure windows can change their answer at midnight without a write.
1046
+ encoded = json.dumps(binding, sort_keys=True)
1047
+ if any(word in encoded for word in ('"window"', '"within"', '"ending"', '"view_id"')):
1048
+ payload["today"] = date.today().isoformat()
1049
+ return fingerprint(payload)
1050
+
1051
+
1052
+ def _snapshot_cell(cell, snapshot):
1053
+ from copy import deepcopy
1054
+ for key in ("value", "display", "state", "refreshed_at", "refusal", "limits"):
1055
+ cell.pop(key, None)
1056
+ cell.update(deepcopy(snapshot))
1057
+
1058
+
1059
  def _reresolved_for_reader(session: Session, book, *, endings_only=False):
1060
+ """C2/R4: project last resolved values only. This GET path never resolves a binding."""
1061
+ from copy import deepcopy
1062
+ from core import sheet_binding
1063
+ session._sheet_scope_context = {}
1064
+ fresh = deepcopy(book)
1065
+ owner = _role_on_workbook(session, book) == "owner"
1066
+ try:
1067
+ cache = sheet_binding.read_refresh_cache(_store(session), book["id"])
1068
+ except Exception:
1069
+ cache = {}
1070
+ seen = {}
1071
+ changed_values = False
1072
+ for tab in fresh.get("sheets") or []:
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1073
  for addr, cell in (tab.get("cells") or {}).items():
1074
+ if cell.get("kind") != "bound":
1075
  continue
1076
+ if _is_view_binding(cell):
1077
  continue
1078
+ before = (cell.get("value"), cell.get("state"))
1079
  binding = cell.get("binding") or {}
1080
  db = str(binding.get("database") or "")
1081
+ if db not in seen:
1082
  try:
1083
  _require_database(session, db)
1084
+ seen[db] = True
1085
+ except HTTPException:
1086
+ seen[db] = False
1087
+ slot = _binding_scope_key(session, db, binding)
1088
+ hit = (cache.get("refreshCells", {}).get(str(tab.get("id")) + "!" + addr, {})
1089
+ .get(slot)) if seen[db] else None
1090
+ if not hit and seen[db]:
1091
+ # ⛔⛔ THE BRIDGE S65-T03 LEFT OUT (SDLC 65 combine). It moved this reader onto
1092
+ # `sheet_binding.read_refresh_cache` while `core.sheets`' scope cache kept its two
1093
+ # writers: `compute_cell` fills BOTH, but the MCP gateway
1094
+ # (`routes_external.py::cache_binding`) fills only the legacy one. So a binding
1095
+ # written through MCP was invisible to its own next read and answered
1096
+ # `sheet_refresh_required` with NO VALUE. Measured by `verify_perm_scope.py::An
1097
+ # editor binding write replaces the old scope cache before reading its result`:
1098
+ # got None, want 75.
1099
+ # ⭐ A READ, never a second write: the legacy slot is keyed by the SAME
1100
+ # `_binding_scope_key`, so C5's per-scope isolation is the one this already
1101
+ # enforces, and nothing here resolves a binding (C2/R4 is untouched). Writing the
1102
+ # new cache from the gateway instead was tried and REJECTED: it makes
1103
+ # `refreshCells` non-empty, which retires the pre-C2 passthrough below for every
1104
+ # OTHER cell and blanked a bound cell that had a perfectly good stored value.
1105
  try:
1106
+ from core import sheets as _sheets
1107
+ legacy = _sheets.cached_scope_value(
1108
+ _store(session), book["id"], str(tab.get("id")) + "!" + addr, slot)
1109
+ except Exception:
1110
+ legacy = None
1111
+ if legacy is not None:
1112
+ value = legacy.get("value")
1113
+ hit = {"cell": {"value": value, "state": "fresh",
1114
+ "display": "" if value is None else str(value),
1115
+ "refreshed_at": legacy.get("refreshedAt")}}
1116
+ if hit:
1117
+ _snapshot_cell(cell, hit["cell"])
1118
+ elif not seen[db]:
1119
+ _snapshot_cell(cell, _binding_refusal("database_unreadable", f"You cannot read {db}."))
1120
+ elif owner and not cache.get("refreshCells") and session.user.get("role") == "admin":
1121
+ # Existing workbooks keep their pre-C2 last result until the first Refresh.
1122
+ pass
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1123
  else:
1124
+ _snapshot_cell(cell, _binding_refusal("sheet_refresh_required",
1125
+ "Refresh this workbook to resolve this cell under your current access."))
1126
+ changed_values |= before != (cell.get("value"), cell.get("state"))
1127
+ # ⛔⛔ A STALE FORMULA IS A REASON TO RECOMPUTE TOO, AND MISSING IT MADE THE TWO SERVER READS
1128
+ # DISAGREE (SDLC 65 combine). C2/R4 says this GET path never RESOLVES A BINDING, and it still
1129
+ # does not: `_recompute_formulas` evaluates the formula graph over cell values that are already
1130
+ # present and opens no database. Gating it on `changed_values` alone meant that after a
1131
+ # structure edit -- which stores every dependent formula `stale` with no value and moves no
1132
+ # bound cell -- the UI read returned `{'kind': 'formula', 'state': 'stale'}` with no figure
1133
+ # while `sheet_read_cells` (MCP) returned the computed one. `verify_perm_scope.py::The UI read
1134
+ # and MCP read agree on stored cells, figures and formats` measured exactly that pair: D1 and
1135
+ # F1 fresh 30.0 / 60.0 from MCP against stale-and-blank from the UI. A non-browser consumer has
1136
+ # no client engine to cover for it, so the blank is what it would have shown.
1137
+ stale_formula = any(
1138
+ cell.get("kind") == "formula" and (cell.get("state") != "fresh" or "value" not in cell)
1139
+ for tab in (fresh.get("sheets") or []) for cell in (tab.get("cells") or {}).values())
1140
+ if changed_values or stale_formula:
1141
+ from modules import sheet as sheet_mod
1142
+ import formula_runtime as fr
1143
+ _recompute_formulas(fresh, fr, sheet_mod)
1144
+ if not owner:
1145
+ fresh["refreshedFor"] = str(session.uname or "")
1146
  return fresh
1147
 
1148
 
 
1154
 
1155
 
1156
  def _read_for(session: Session, book):
1157
+ """Project saved results under the current access wall without resolving any binding.
1158
 
1159
  W60-T09, 2026-09-14: R28 checks visibility before this projection. Admitted legacy readers
1160
+ and grantees receive a proven scoped snapshot or a request to Refresh.
1161
  """
1162
  book = _response_workbook(book)
 
 
 
 
 
1163
  return _reresolved_for_reader(session, book)
1164
 
1165
 
 
1512
  if not name.lower().endswith(".xlsx"):
1513
  raise err(400, "import_not_xlsx",
1514
  "Only an .xlsx workbook can be imported. Save your file as .xlsx and try again.")
1515
+ # Read at most one byte past the cap. UploadFile may be backed by a large temporary file; an
1516
+ # unbounded read would allocate hostile input merely to discover that it is too large.
1517
+ data = await file.read(sheet_import.MAX_IMPORT_BYTES + 1)
1518
  if len(data) > sheet_import.MAX_IMPORT_BYTES:
1519
  raise HTTPException(status_code=413, detail={"error": {
1520
  "code": "import_too_large",
 
1560
  return "%.1f MB" % (float(byte_count) / (1024.0 * 1024.0))
1561
 
1562
 
1563
+ # ── C5: Excel-shaped bounded automation (S63-T22) ──────────────────────────────────────────
1564
+
1565
+ @router.get("/sheets/automation/manifest")
1566
+ def sheet_automation_manifest(session: Session = Depends(require_session)):
1567
+ """The canonical capability bytes shared with the Python SDK.
1568
+
1569
+ The session wall is deliberate: this is product capability metadata, but keeping it beside
1570
+ the authenticated workbook doors avoids creating a second unauthenticated API convention.
1571
+ """
1572
+ del session
1573
+ from core import sheet_compat
1574
+ return Response(content=sheet_compat.capability_manifest_bytes(), media_type="application/json")
1575
+
1576
+
1577
+ def _sheet_automation_calculator(book):
1578
+ """Inject the existing formula runtime without making ``platform/core`` import the API."""
1579
+ from modules import sheet as sheet_mod
1580
+ import formula_runtime as fr
1581
+ _recompute_formulas(book, fr, sheet_mod)
1582
+
1583
+
1584
+ @router.post("/sheets/{workbook_id}/automation")
1585
+ def execute_sheet_automation(workbook_id: str, body: dict = Body(default=None),
1586
+ session: Session = Depends(require_session)):
1587
+ """Execute one manifest operation through the shared core planner and targeted patcher."""
1588
+ from core import sheet_compat, sheets
1589
+ raw = dict(body or {})
1590
+ operation = str(raw.get("operation") or "")
1591
+ arguments = raw.get("arguments")
1592
+ try:
1593
+ # Unsupported and malformed operation-level calls stop here, before any store update
1594
+ # callback can be entered. Authorization then applies to the declared access class.
1595
+ spec = sheet_compat.require_supported(operation)
1596
+ book = sheets.get_workbook(_store(session), workbook_id)
1597
+ _require_reader(session, book)
1598
+ if spec["access"] == "write":
1599
+ _require_editor(session, book)
1600
+ return sheet_compat.execute(
1601
+ _store(session), workbook_id, operation, arguments,
1602
+ actor=session.uname, actor_name=session.user.get("name") or session.uname,
1603
+ calculator=_sheet_automation_calculator)
1604
+ except sheets.SheetRefusal as exc:
1605
+ raise _refuse(exc)
1606
+
1607
+
1608
  @router.get("/sheets/{workbook_id}")
1609
  def read_sheet(workbook_id: str, tabs: str = "",
1610
  session: Session = Depends(require_session)):
 
1770
  cache_key=(session.tenant, session.uname, workbook_id), changes=edits)
1771
  # Date and filter-cell edits also change database totals. Recompute all readers, then formulas.
1772
  ending_cells = set()
1773
+ refreshed_entries = []
1774
+ context_memo = {}
1775
  for source in book.get("sheets") or []:
1776
  for addr, cell in (source.get("cells") or {}).items():
1777
  binding = cell.get("binding") or {}
 
1780
  try:
1781
  result = _resolve_cell_binding(session, binding, book, source.get("id"))
1782
  cell.update(value=result["value"], display="" if result["value"] is None else str(result["value"]), state="fresh")
1783
+ cell["refreshed_at"] = _now_iso()
1784
+ if result.get("limits"):
1785
+ cell["limits"] = result["limits"]
1786
+ else:
1787
+ cell.pop("limits", None)
1788
  cell.pop("refusal", None)
1789
  except HTTPException as exc:
1790
  info = ((exc.detail or {}).get("error") or {})
 
1793
  info.get("message", "The Ending date could not be read. Choose a valid date."),
1794
  ))
1795
  ending_cells.add((source.get("id"), addr))
1796
+ refreshed_entries.append((str(source.get("id")) + "!" + addr,
1797
+ _binding_scope_key(session, str(binding.get("database") or ""), binding),
1798
+ {"tokens": {}, "context": _refresh_context(book, binding, context_memo),
1799
+ "cell": {k: cell[k] for k in ("value", "display", "state", "refreshed_at", "limits", "refusal")
1800
+ if k in cell}}))
1801
  if ending_cells:
1802
+ from core.sheet_binding import save_refresh_cache
1803
+ save_refresh_cache(_store(session), workbook_id, refreshed_entries,
1804
+ {c.get("binding", {}).get("database") for t in book.get("sheets", [])
1805
+ for c in t.get("cells", {}).values() if c.get("kind") == "bound"})
1806
  _recompute_formulas(book, fr, sheet_mod)
1807
  found = sheets.find_tab(book, tab_ref)
1808
  out = {}
 
1978
 
1979
  @router.post("/sheets/{workbook_id}/refresh")
1980
  def refresh_sheet(workbook_id: str, session: Session = Depends(require_session)):
1981
+ """C2/R4: Refresh resolves only databases whose token or binding inputs changed.
1982
+
1983
+ Viewers refresh their own server cache. Only editors save the workbook document.
1984
+ One token is read per distinct database on the unchanged path, regardless of cell count.
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1985
  """
1986
+ from copy import deepcopy
1987
+ from core import sheets, sheet_binding
1988
  from modules import sheet as sheet_mod
1989
  import formula_runtime as fr
1990
+ from time import perf_counter
1991
 
1992
+ store = _store(session)
1993
+ session._sheet_scope_context = {}
1994
  try:
1995
+ book = sheets.get_workbook(store, workbook_id)
1996
+ _require_reader(session, book)
1997
  except sheets.SheetRefusal as exc:
1998
  raise _refuse(exc)
1999
+ editable = _role_on_workbook(session, book) in ("owner", "edit")
2000
+ try:
2001
+ cache = sheet_binding.read_refresh_cache(store, workbook_id)
2002
+ except Exception:
2003
+ cache = {}
2004
  stamp = _now_iso()
2005
+ seen_db, tokens, entries, status = {}, {}, [], {}
2006
+ view_memo, related_memo = {}, {}
2007
+ context_memo = {}
2008
+ token_reads = 0
2009
+ input_graph = None
2010
+
2011
+ def input_databases(binding, tab_ref):
2012
+ nonlocal input_graph
2013
+ nodes, _ = _fe_parts((binding.get("scope") or {}).get("filter"))
2014
+ related_nodes, _ = _fe_parts((binding.get("related") or {}).get("filter"))
2015
+ refs = [str((node.get("valueRef") or {}).get("ref") or "")
2016
+ for node in sheet_mod.filter_cell_nodes([*nodes, *related_nodes])]
2017
+ period = binding.get("period") or {}
2018
+ if period.get("ref"):
2019
+ refs.append(period["ref"])
2020
+ if not refs:
2021
+ return set()
2022
+ tabs = book.get("sheets") or []
2023
+ names = _tab_scope_names(tabs)
2024
+ if input_graph is None:
2025
+ documents = {fr.a1_key(addr, name): cell for tab, name in zip(tabs, names)
2026
+ for addr, cell in (tab.get("cells") or {}).items()}
2027
+ engine = fr.SheetCalculation({key: fr._document_spec(cell, key.rpartition("!")[0])
2028
+ for key, cell in documents.items()}, sheets=names,
2029
+ named_ranges=book.get("namedRanges"), date_system=book.get("dateSystem", "1900"))
2030
+ input_graph = documents, engine
2031
+ documents, engine = input_graph
2032
+ home = _resolve_home_tab(book, tabs, names, tab_ref)
2033
+ pending, visited, deps = [_asked_key(ref, home, fr) for ref in refs], set(), set()
2034
+ while pending:
2035
+ key = pending.pop()
2036
+ if key in visited:
2037
+ continue
2038
+ visited.add(key)
2039
+ cell = documents.get(key) or {}
2040
+ if cell.get("kind") == "bound":
2041
+ deps.add(str((cell.get("binding") or {}).get("database") or ""))
2042
+ pending.extend(engine.consumed.get(key, ()))
2043
+ return deps
2044
+
2045
+ def token(db):
2046
+ nonlocal token_reads
2047
+ if db not in tokens:
2048
+ token_reads += 1
2049
+ try:
2050
+ # ⭐ Arms `core.sheet_binding.mirror_change_token` (SDLC 65 combine). The mirror
2051
+ # reader lives in `harness` because `core` never imports up, and the API is the
2052
+ # layer allowed to reach both ways, so the wiring belongs exactly here.
2053
+ import harness.mirror_tokens # noqa: F401 (registers the provider on import)
2054
+ tokens[db] = sheet_binding.database_change_token(store, db)
2055
+ except Exception:
2056
+ # Failure to prove unchanged means resolve, never silently reuse a value.
2057
+ tokens[db] = None
2058
+ return tokens[db]
2059
+
2060
+ def dependencies(db, binding, tab_ref):
2061
+ deps = {db}
2062
+ if binding.get("related"):
2063
+ key = (db, binding["related"].get("via"))
2064
+ if key not in related_memo:
2065
+ related_memo[key] = _binding_related_entry(session, db, binding)
2066
+ deps.add(related_memo[key]["target"])
2067
+ nodes, _ = _fe_parts((binding.get("scope") or {}).get("filter"))
2068
+ linked = sheet_mod.linked_filter_ids(nodes)
2069
+ if linked:
2070
+ names = {sheet_mod.linked_field_parts(key)[0] for key in linked}
2071
+ deps.update(b["source"] for b in _linked_customer_bridges(db) if b["key"] in names)
2072
+ deps.update(input_databases(binding, tab_ref))
2073
+ return deps
2074
+
2075
+ bound_inputs = {fr.a1_key(addr, name): (tab.get("id"), addr, cell)
2076
+ for tab, name in zip(book.get("sheets") or [], _tab_scope_names(book.get("sheets") or []))
2077
+ for addr, cell in (tab.get("cells") or {}).items() if cell.get("kind") == "bound"}
2078
+
2079
+ def cached_input(key):
2080
+ item = bound_inputs.get(key)
2081
+ if item is None:
2082
+ return None
2083
+ tab_ref, addr, cell = item
2084
+ binding = cell.get("binding") or {}
2085
+ db = str(binding.get("database") or "")
2086
+ _require_database(session, db)
2087
+ scope_key = _binding_scope_key(session, db, binding)
2088
+ hit = cache.get("refreshCells", {}).get(str(tab_ref) + "!" + addr, {}).get(scope_key)
2089
+ sources = {db: token(db) for db in dependencies(db, binding, tab_ref)}
2090
+ if (hit and all(sources.values()) and hit.get("tokens") == sources
2091
+ and hit.get("context") == _refresh_context(book, binding, context_memo)
2092
+ and hit["cell"].get("state") == "fresh"):
2093
+ return hit["cell"]
2094
+ return None
2095
+
2096
+ for tab in book.get("sheets") or []:
2097
  for addr, cell in (tab.get("cells") or {}).items():
2098
+ if cell.get("kind") != "bound" or _is_view_binding(cell):
2099
  continue
 
 
2100
  binding = cell.get("binding") or {}
2101
  db = str(binding.get("database") or "")
2102
+ cell_key = str(tab.get("id")) + "!" + addr
2103
+ scope_key = _binding_scope_key(session, db, binding)
2104
+ context = _refresh_context(book, binding, context_memo)
2105
+ hit = cache.get("refreshCells", {}).get(cell_key, {}).get(scope_key)
2106
+ source_tokens = {}
 
 
 
 
 
 
2107
  try:
2108
+ if db not in seen_db:
2109
+ try:
2110
+ _require_database(session, db)
2111
+ seen_db[db] = None
2112
+ except HTTPException as exc:
2113
+ seen_db[db] = exc
2114
+ if seen_db[db] is not None:
2115
+ raise seen_db[db]
2116
+ source_tokens = {key: token(key) for key in sorted(dependencies(db, binding, tab.get("id")))}
2117
+ if (hit and all(source_tokens.values()) and hit.get("tokens") == source_tokens
2118
+ and hit.get("context") == context
2119
+ and hit["cell"].get("state") == "fresh"):
2120
+ _snapshot_cell(cell, hit["cell"])
2121
+ status.setdefault(db, "skipped")
2122
+ continue
2123
+ status[db] = "resolved"
2124
+ previous = getattr(session, "_sheet_refresh_input", None)
2125
+ session._sheet_refresh_input = cached_input
2126
+ try:
2127
+ out = _resolve_cell_binding(session, binding, book, tab.get("id"), view_memo=view_memo)
2128
+ finally:
2129
+ session._sheet_refresh_input = previous
2130
+ limits = list(out.get("limits") or [])
2131
+ if not all(source_tokens.values()):
2132
+ limits.append({"code": "change_token_unavailable",
2133
+ "cause": "The database change token could not be read. This Refresh resolved the cell.",
2134
+ "recommendation": "Restore the database store and connector sync receipts to enable skipping."})
2135
+ snapshot = {"state": "fresh", "value": out["value"],
2136
+ "display": "" if out["value"] is None else str(out["value"]),
2137
+ "refreshed_at": stamp}
2138
+ if limits:
2139
+ snapshot["limits"] = limits
2140
  except HTTPException as exc:
2141
+ info = ((exc.detail or {}).get("error") or {})
2142
+ snapshot = _binding_refusal(info.get("code", "sheet_unresolved"),
2143
+ info.get("message", "This cell could not be refreshed."))
2144
+ status[db] = "refused"
2145
+ _snapshot_cell(cell, snapshot)
2146
+ entries.append((cell_key, scope_key, {"tokens": source_tokens,
2147
+ "context": context, "cell": deepcopy(snapshot)}))
2148
+
2149
+ # Read again only after resolving. A concurrent write must not certify a mixed result
2150
+ # as reusable. The unchanged path still costs exactly one token read per database.
2151
+ resolved_sources = {db for _, _, entry in entries for db in entry["tokens"]}
2152
+ changed = set()
2153
+ for db in resolved_sources:
2154
+ if tokens[db] is None:
2155
+ continue # Already disclosed as unverified and never eligible for a hit.
2156
+ token_reads += 1
2157
+ try:
2158
+ if sheet_binding.database_change_token(store, db) != tokens[db]:
2159
+ changed.add(db)
2160
+ except Exception:
2161
+ changed.add(db)
2162
+ for cell_key, _, entry in entries:
2163
+ if changed.intersection(entry["tokens"]):
2164
+ entry["tokens"] = {} # no next-refresh hit, even if the writer rolls back
2165
+ if entry["cell"].get("state") == "fresh":
2166
+ entry["cell"].setdefault("limits", []).append({"code": "change_during_refresh",
2167
+ "cause": "This database could not be confirmed unchanged while the cell was being resolved.",
2168
+ "recommendation": "Refresh again to resolve the cell from the updated database."})
2169
+ tab_ref, addr = cell_key.split("!", 1)
2170
+ _snapshot_cell(sheets.find_tab(book, tab_ref)["cells"][addr], entry["cell"])
2171
 
 
 
 
 
 
 
 
 
 
 
2172
  started = perf_counter()
2173
  _recompute_formulas(book, fr, sheet_mod)
2174
  recalc_ms = round((perf_counter() - started) * 1000.0, 1)
2175
+ if editable:
2176
+ try:
2177
+ saved = sheets.put_workbook(store, workbook_id,
2178
+ {"name": book.get("name"), "sheets": book.get("sheets"),
2179
+ "refreshed_at": stamp, "rev": book.get("rev")},
2180
+ actor=session.uname, actor_name=session.user.get("name") or session.uname,
2181
+ lossless=True)
2182
+ except sheets.SheetRefusal as exc:
2183
+ raise _refuse(exc)
2184
+ else:
2185
+ saved = book
2186
+ saved["refreshedFor"] = str(session.uname or "")
2187
+ if entries:
2188
+ sheet_binding.save_refresh_cache(store, workbook_id, entries, status)
2189
+ return {"workbook": _response_workbook(saved), "recalc_ms": recalc_ms,
2190
+ "binding_refresh": {"databases": status, "token_reads": token_reads}}
2191
 
2192
 
2193
  def _now_iso():
2194
+ from core import write_execution
2195
+ return write_execution.utcnow().replace(microsecond=0).isoformat()
2196
 
2197
 
2198
  #: ⛔ THE NAME A TAB GETS WHEN IT CANNOT KEEP ITS OWN, and it is built to be UNTYPEABLE on two
 
2483
  refuse("the cell or its sheet is missing or blank.")
2484
  if cell.get("kind") == "bound":
2485
  try:
2486
+ lookup = getattr(session, "_sheet_refresh_input", None)
2487
+ cached = lookup(key) if lookup is not None else None
2488
+ value = (cached["value"] if cached is not None else _resolve_binding(
2489
+ session, cell.get("binding"), today=today, book=book,
2490
+ tab=key.rpartition("!")[0], ending_seen=(*seen, key))["value"])
2491
  except HTTPException as exc:
2492
  reason = ((exc.detail or {}).get("error") or {}).get("message", "its source is refused.")
2493
  refuse(reason)
 
2515
  for dependency in needed:
2516
  attempted.add(dependency)
2517
  try:
2518
+ lookup = getattr(session, "_sheet_refresh_input", None)
2519
+ result = lookup(dependency) if lookup is not None else None
2520
+ if result is None:
2521
+ result = _resolve_binding(session, bound[dependency].get("binding"), today=today,
2522
+ book=book, tab=dependency.rpartition("!")[0], ending_seen=(*seen, key, dependency))
2523
  changes[dependency] = {"value": result["value"]}
2524
  except HTTPException as exc:
2525
  info = ((exc.detail or {}).get("error") or {})
 
2590
  return _resolve_binding(session, binding, **kwargs)
2591
 
2592
 
2593
+ def _binding_related_entry(session, database, binding):
2594
+ """Resolve one authored edge key against the permission-scoped declarations."""
2595
+ if "related" not in binding:
2596
+ return None
2597
+ from core import perm_scope, relations, user_tables
2598
+ from routes_grid import _source_fields
2599
+
2600
+ fields = _source_fields(session, database)
2601
+ defs = {}
2602
+ for key, definition in (user_tables.all_defs(st=session.runtime) or {}).items():
2603
+ if perm_scope.nav_may_open(session.user, key, st=session.runtime):
2604
+ declared = definition.get("fields") or []
2605
+ hidden = perm_scope.hidden_keys(session.user, key, declared, st=session.runtime)
2606
+ defs[key] = {**definition, "fields": [f for f in declared if f.get("key") not in hidden]}
2607
+ entries = relations.related(database, fields=fields, defs=defs)
2608
+ name = binding["related"]["via"]
2609
+ matches = [entry for entry in entries if entry["key"] == name]
2610
+ via = matches[0]["via"] if len(matches) == 1 else {"key": name}
2611
+ entry = relations.find(database, via, fields=fields, defs=defs)
2612
+ if entry is None:
2613
+ candidates = [item["key"] for item in relations.via_candidates(entries, via)
2614
+ if perm_scope.nav_may_open(session.user, item["target"], st=session.runtime)]
2615
+ offered = (" Available relationships: " + ", ".join(candidates) + ".") if candidates else ""
2616
+ # Write our own sentence: the shared pin refusal does not name via.key.
2617
+ raise err(409, "unknown_relation",
2618
+ f"Relationship '{name}' cannot be resolved on '{database}'."
2619
+ + offered + " Choose a declared relationship for this database.")
2620
+ target = _require_database(session, entry["target"])
2621
+ if not entry.get("served"):
2622
+ raise err(409, "relation_target_unavailable",
2623
+ f"Relationship '{name}' cannot read '{target}' because it has no read-through "
2624
+ "binding. Choose a relationship to a database with a live connection.")
2625
+ # A through/manual link cannot be represented by this one declared column pair.
2626
+ declaration = next((f for f in fields if f.get("key") == entry["via"].get("field")), {})
2627
+ if entry["via"]["direction"] == "reverse":
2628
+ declaration = next((f for f in defs.get(target, {}).get("fields", [])
2629
+ if f.get("key") == entry["via"].get("field")), {})
2630
+ link = declaration.get("link") or {}
2631
+ if link.get("through") or link.get("selectionMode") == "manual":
2632
+ raise err(409, "relation_source_unresolvable",
2633
+ f"Relationship '{name}' does not declare a direct column join. "
2634
+ "Choose a relationship with one readable column on each database.")
2635
+ return entry
2636
+
2637
+
2638
+ def _binding_related_query(session, binding, entry, cur, spec, columns, *, window=None,
2639
+ today=None, book=None, tab=None, ending_seen=()):
2640
+ """Materialise the far side on the near cursor, then hand metadata to the compiler."""
2641
+ if entry is None:
2642
+ return None
2643
+ from core import perm_scope, relations
2644
+ from modules import sheet as sheet_mod
2645
+
2646
+ database, target = binding["database"], entry["target"]
2647
+ name = binding["related"]["via"]
2648
+ local_key, remote_key = entry["via"].get("from"), entry["via"].get("on")
2649
+
2650
+ def map_join(key, table, source_spec, cols, side):
2651
+ hidden = perm_scope.hidden_keys(session.user, table, source_spec.get("fields") or [],
2652
+ st=session.runtime)
2653
+ if key and key not in hidden:
2654
+ expr = (cols.get(key) or {}).get("sql")
2655
+ if not expr:
2656
+ # Identity and join columns can be absent from the visible field list.
2657
+ # Only the trusted mirror map may supply their expressions.
2658
+ mapped = (source_spec.get("cols") or {}).get(key)
2659
+ expr = (source_spec.get("id") if key == relations.IDENTITY_COL else
2660
+ mapped[0] if mapped else None)
2661
+ if expr and str(expr).strip().upper() != "NULL":
2662
+ cols[key] = {**cols.get(key, {}), "sql": expr}
2663
+ return
2664
+ code = "relation_source_unresolvable" if side == "source" else "relation_target_unavailable"
2665
+ raise err(409, code,
2666
+ f"Relationship '{name}' cannot read its {side} column '{key}' on '{table}'. "
2667
+ "Choose a relationship whose join columns are available on the live connection.")
2668
+
2669
+ map_join(local_key, database, spec, columns, "source")
2670
+ wall = ((perm_scope.entry(session.user, target) or {}).get("filter")
2671
+ if perm_scope.row_scope_applies(session.user, target) else None)
2672
+ wall_nodes, wall_conj = _fe_parts(wall)
2673
+ scope_nodes, scope_conj = _fe_parts(binding["related"]["filter"])
2674
+ scope_nodes = _filter_cell_values(session, scope_nodes, book=book, tab=tab,
2675
+ today=today, seen=ending_seen)
2676
+ needed = (sheet_mod.filter_columns_named(scope_nodes)
2677
+ | sheet_mod.filter_columns_named(wall_nodes) | {remote_key})
2678
+ try:
2679
+ rcur, rspec, rcols = _mirror(session)(target, metric_window=window, needed=needed)
2680
+ except HTTPException as exc:
2681
+ detail = exc.detail if isinstance(exc.detail, dict) else {}
2682
+ if exc.status_code == 404 or (detail.get("error") or {}).get("code") == "not_read_through":
2683
+ raise err(409, "relation_target_unavailable",
2684
+ f"Relationship '{name}' cannot read '{target}' on this live connection. "
2685
+ "Choose a relationship to an available database.") from exc
2686
+ raise
2687
+ if rcur is not cur:
2688
+ raise err(409, "relation_cursor_mismatch",
2689
+ f"Relationship '{name}' cannot be queried on the same live connection as "
2690
+ f"'{database}'. Retry after both databases share a connection.")
2691
+ hidden = perm_scope.hidden_keys(session.user, target, rspec.get("fields") or [],
2692
+ st=session.runtime)
2693
+ rcols = {key: value for key, value in rcols.items() if key not in hidden}
2694
+ map_join(remote_key, target, rspec, rcols, "target")
2695
+ depends = sheet_mod.filter_columns_named(scope_nodes)
2696
+ return {"via": name, "database": target, "local_key": local_key, "remote_key": remote_key,
2697
+ "spec": rspec, "columns": rcols, "where": entry.get("where"),
2698
+ "scope_nodes": scope_nodes, "scope_conj": scope_conj,
2699
+ "wall_nodes": wall_nodes, "wall_conj": wall_conj,
2700
+ "limits": [row for key, row in (rspec.get("unbounded") or {}).items() if key in depends]}
2701
+
2702
+
2703
  def _resolve_binding(session: Session, raw_binding, *, today=None, view_memo=None,
2704
  book=None, tab=None, ending_seen=()):
2705
  """Evaluate ONE C2 binding and return `{value, limits, sql_columns}`.
 
2761
  if binding["aggregate"].get("field"):
2762
  needed.add(str(binding["aggregate"]["field"]))
2763
 
2764
+ related_entry = _binding_related_entry(session, database, binding)
2765
+ if related_entry:
2766
+ needed.add(related_entry["via"].get("from"))
2767
  cur, spec, cols = _mirror(session)(database, metric_window=window, needed=needed)
2768
  cols = dict(cols)
2769
  scope_nodes = _linked_filter_scope(session, database, scope_nodes, cur, spec, cols, today=today)
 
2805
  # spec's column expressions; applying it a second time would double-count the exclusion and
2806
  # break every joined spec, whose partner column is `sol.order_partner_id` or `aml.partner_id`.
2807
  try:
2808
+ related = _binding_related_query(
2809
+ session, binding, related_entry, cur, spec, cols, window=window, today=today,
2810
+ book=book, tab=tab, ending_seen=ending_seen)
2811
  sql, params, limits = sheet_mod.build_aggregate_query(
2812
  spec, binding, cols, scope_nodes=scope_nodes, scope_conj=scope_conj,
2813
+ wall_nodes=wall_nodes, wall_conj=wall_conj, today=today, related=related)
2814
  except sheet_mod.SheetRefusal as exc:
2815
  raise err(400, exc.code, exc.message)
2816
  except ValueError:
 
2851
  return {"value": value, "limits": limits, "database": database, "binding": binding}
2852
 
2853
 
2854
+ def execute_daily_sales_repair(session: Session, request, *, backup_store, workbook_id=None):
2855
+ """Run the explicit T21 repair through the normal scoped binding and formula engines.
2856
+
2857
+ This is intentionally not an HTTP route. A normal workbook request must never acquire an
2858
+ exceptional tenant-data mutation surface. The approved operator calls this helper with the
2859
+ exact marker/revision/source oracle and a backup store, naming the workbook with workbook_id.
2860
+ Omission retains the owner's workbook default. The helper explicitly points the core workflow
2861
+ at that target; the core refuses any workbook it was not pointed at and performs its own marker,
2862
+ snapshot, backup and readback checks.
2863
+ """
2864
+ from core import daily_sales_repair, sheets
2865
+ import formula_runtime
2866
+
2867
+ try:
2868
+ if str(session.tenant or "") != daily_sales_repair.TARGET_TENANT:
2869
+ raise daily_sales_repair.RepairRefusal(
2870
+ "repair_tenant_mismatch",
2871
+ "This operation may run only in the Royal Imports workspace.")
2872
+ target = str(workbook_id or "").strip() or daily_sales_repair.TARGET_WORKBOOK_ID
2873
+ book = sheets.get_workbook(_store(session), target)
2874
+ _require_editor(session, book)
2875
+
2876
+ view_memo = {}
2877
+
2878
+ def resolve(binding, projected, tab):
2879
+ tab_ref = str(tab.get("id") or tab.get("name") or "")
2880
+ return _resolve_cell_binding(
2881
+ session, binding, projected, tab_ref, view_memo=view_memo)
2882
+
2883
+ def formula_check(projected):
2884
+ formula_runtime.recompute_document(
2885
+ projected, today=(request or {}).get("calculation_date"))
2886
+ return {"book": projected,
2887
+ "errors": daily_sales_repair.formula_error_census(projected)}
2888
+
2889
+ return daily_sales_repair.execute(
2890
+ _store(session), backup_store, request,
2891
+ target_workbook_id=target,
2892
+ resolve_binding=resolve, formula_check=formula_check,
2893
+ actor=session.uname, actor_name=session.user.get("name") or session.uname)
2894
+ except sheets.SheetRefusal as exc:
2895
+ raise _refuse(exc)
2896
+
2897
+
2898
  def _fe_parts(tree):
2899
  """`(nodes, conj)` for a stored `{conj, nodes}` filter, through the ONE reader of that shape."""
2900
  from modules import sheet as sheet_mod
 
3077
  saved["refreshed_at"] = stamp
3078
  saved.pop("refusal", None)
3079
 
3080
+ if limits:
3081
+ saved["limits"] = limits
3082
+ else:
3083
+ saved.pop("limits", None)
3084
+
3085
  try:
3086
  landed = sheets.patch_cells(_store(session), workbook_id, tab_ref, {key: saved},
3087
  actor=session.uname,
 
3101
  except Exception: # noqa: BLE001
3102
  pass
3103
 
3104
+ from core.sheet_binding import save_refresh_cache
3105
+ save_refresh_cache(_store(session), workbook_id, [(tab_ref + "!" + key,
3106
+ _binding_scope_key(session, str(binding.get("database") or ""), binding),
3107
+ {"tokens": {}, "context": _refresh_context(book, binding),
3108
+ "cell": {k: saved[k] for k in ("value", "display", "state", "refreshed_at", "limits", "refusal")
3109
+ if k in saved}})], [binding.get("database")])
3110
+
3111
  wire = dict(saved)
3112
  #: ⚠ `limits` RIDES THE ANSWER AND IS NOT STORED, because `core.sheets.clean_cell` carries no
3113
  #: such key: it describes THIS resolve's truncation, not the cell.
 
3289
  window = _ending_window(session, binding, book=book, tab=tab.get("id"))
3290
  except sheet_mod.SheetRefusal as exc:
3291
  raise err(400, exc.code, exc.message)
3292
+ related_entry = _binding_related_entry(session, database, binding)
3293
+ needed = (sheet_mod.filter_columns_named(scope_nodes)
3294
+ | sheet_mod.filter_columns_named(wall_nodes))
3295
+ if related_entry:
3296
+ needed.add(related_entry["via"].get("from"))
3297
  cur, spec, cols = _mirror(session)(
3298
  database, metric_window=window,
3299
+ needed=needed)
 
3300
  cols = dict(cols)
3301
  scope_nodes = _linked_filter_scope(session, database, scope_nodes, cur, spec, cols)
3302
 
3303
  try:
3304
+ related = _binding_related_query(
3305
+ session, binding, related_entry, cur, spec, cols, window=window,
3306
+ book=book, tab=tab.get("id"))
3307
  count_sql, count_params, limits = sheet_mod.build_aggregate_query(
3308
  spec, {"aggregate": {"fn": "count"}}, cols, scope_nodes=scope_nodes,
3309
+ scope_conj=scope_conj, wall_nodes=wall_nodes, wall_conj=wall_conj, related=related)
3310
+ ids_sql, ids_params, _ = sheet_mod.build_ids_query(
3311
+ spec, cols, scope_nodes=scope_nodes, scope_conj=scope_conj,
3312
+ wall_nodes=wall_nodes, wall_conj=wall_conj, related=related)
3313
  except sheet_mod.SheetRefusal as exc:
3314
  raise err(400, exc.code, exc.message)
3315
+ except ValueError:
3316
+ raise err(409, "scope_not_applicable",
3317
+ "This cell's scope uses a restriction that cannot be applied on its live "
3318
+ "connection. Edit the scope or ask an administrator to update the restriction.")
3319
  total = (cur.execute(count_sql, count_params).fetchone() or [0])[0]
3320
 
 
 
 
3321
  record_ids = [int(r[0]) for r in cur.execute(ids_sql, ids_params).fetchall() if r and r[0]]
3322
 
3323
  out = {"database": database, "record_ids": record_ids, "count": int(total or 0),
api/routes_slack.py CHANGED
@@ -197,11 +197,11 @@ def agent_may_read(agent, module):
197
  return bool(perm_scope.may_access(agent_principal(agent), module))
198
 
199
 
200
- def agent_visible_fields(agent, fields, module):
201
  """`fields` minus this channel's hidden closure — the SAME transitive closure a person gets,
202
  so a formula over a hidden column cannot leak it back through arithmetic."""
203
  import core.perm_scope as perm_scope
204
- return perm_scope.visible_fields(fields, agent_principal(agent), module)
205
 
206
 
207
  def agent_rows(agent, rows, module, fields, ctx=None, st=None):
@@ -218,15 +218,13 @@ def agent_rows(agent, rows, module, fields, ctx=None, st=None):
218
  default for the same reason `perm_scope.apply_row_scope`'s is: without one this is
219
  byte-identical to the wall that shipped before the argument existed.
220
 
221
- ⚠ `hidden_keys` DELIBERATELY DOES NOT TAKE IT IN THIS CHANGE. Its field-grant leg
222
- under-hides without a handle (see `perm_scope.visible_overlays`), which is a real and
223
- separate gap on this surface — but it is the FIELD wall, it moves what a channel receives
224
- rather than what I16 asked for, and widening two walls in one ticket is how a permission
225
- change stops being reviewable.
226
  """
227
  import core.perm_scope as perm_scope
228
  p = agent_principal(agent)
229
- hide = perm_scope.hidden_keys(p, module, fields)
230
  kept = perm_scope.apply_row_scope(rows, p, module, fields, ctx, st=st)
231
  # Both wires, never one: the field list and the row payload are separate, and stripping only
232
  # the first leaves the value sitting in the second where anyone can read it.
@@ -298,7 +296,8 @@ def channel_answer_context(rt, channel, principal=None, automation_id=None):
298
  if not isinstance(grants, list) or str(automation_id) not in {
299
  str(item) for item in grants if isinstance(item, str)}:
300
  return None
301
- return {"channel": str(agent.get("channel") or ""), "principal": expected,
 
302
  "writeApproval": agent.get("writeApproval", True) is not False,
303
  "automationId": str(automation_id or "")}
304
 
@@ -336,6 +335,53 @@ def _reply_text(text, provenance):
336
  return f"{body[:max(0, 4000 - len(footer) - 2)]}\n\n{footer}"
337
 
338
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
339
  def deliver_channel_reply(rt, channel, text, *, principal=None, automation_id=None,
340
  provenance=None, post=None):
341
  """Send only after one channel/principal binding and an optional automation grant validate."""
 
197
  return bool(perm_scope.may_access(agent_principal(agent), module))
198
 
199
 
200
+ def agent_visible_fields(agent, fields, module, st=None):
201
  """`fields` minus this channel's hidden closure — the SAME transitive closure a person gets,
202
  so a formula over a hidden column cannot leak it back through arithmetic."""
203
  import core.perm_scope as perm_scope
204
+ return perm_scope.visible_fields(fields, agent_principal(agent), module, st=st)
205
 
206
 
207
  def agent_rows(agent, rows, module, fields, ctx=None, st=None):
 
218
  default for the same reason `perm_scope.apply_row_scope`'s is: without one this is
219
  byte-identical to the wall that shipped before the argument existed.
220
 
221
+ The field projection takes the same runtime too. Without it a user-generated field can be
222
+ absent from the declared grant while its value survives in the row payload — precisely the
223
+ half-wall a customer-intelligence digest must not cross.
 
 
224
  """
225
  import core.perm_scope as perm_scope
226
  p = agent_principal(agent)
227
+ hide = perm_scope.hidden_keys(p, module, fields, st=st)
228
  kept = perm_scope.apply_row_scope(rows, p, module, fields, ctx, st=st)
229
  # Both wires, never one: the field list and the row payload are separate, and stripping only
230
  # the first leaves the value sitting in the second where anyone can read it.
 
296
  if not isinstance(grants, list) or str(automation_id) not in {
297
  str(item) for item in grants if isinstance(item, str)}:
298
  return None
299
+ return {"agentId": str(agent.get("id") or ""),
300
+ "channel": str(agent.get("channel") or ""), "principal": expected,
301
  "writeApproval": agent.get("writeApproval", True) is not False,
302
  "automationId": str(automation_id or "")}
303
 
 
335
  return f"{body[:max(0, 4000 - len(footer) - 2)]}\n\n{footer}"
336
 
337
 
338
+ def permission_safe_automation_digest(rt, context, *, module="", fields=None, rows=None,
339
+ summary="", links=None, limit=12):
340
+ """Compose a bounded digest only after applying the destination Agent's live read wall.
341
+
342
+ The caller supplies trusted object links, never cell/field values in a URL. For a database
343
+ digest both the schema and rows pass through the exact Slack-agent permission projection;
344
+ for a Sheet digest the body intentionally carries no cell values at all.
345
+ """
346
+ ctx = context if isinstance(context, dict) else {}
347
+ agent = _agents(rt).get(str(ctx.get("agentId") or ""))
348
+ if not isinstance(agent, dict) or agent.get("active", True) is False:
349
+ return None
350
+ try:
351
+ cap = max(1, min(int(limit), APPROVAL_MAX_ROWS))
352
+ except (TypeError, ValueError):
353
+ cap = APPROVAL_MAX_ROWS
354
+ visible, safe_rows, readable = [], [], True
355
+ if module:
356
+ readable = agent_may_read(agent, module)
357
+ if readable:
358
+ visible = agent_visible_fields(agent, fields or [], module, st=rt)
359
+ safe_rows = agent_rows(agent, rows or [], module, fields or [], st=rt)[:cap]
360
+ labels = {str(field.get("key") or ""): str(field.get("label") or field.get("key") or "")
361
+ for field in visible if isinstance(field, dict) and field.get("key")}
362
+ lines = [" ".join(str(summary or "Customer intelligence found a change.").split())[:500]]
363
+ if readable:
364
+ for row in safe_rows:
365
+ cells = []
366
+ for key in list(labels)[:8]:
367
+ value = (row or {}).get(key)
368
+ if value in (None, ""):
369
+ continue
370
+ cells.append(f"{labels[key]}: {' '.join(str(value).split())[:160]}")
371
+ if cells:
372
+ lines.append(" • " + "; ".join(cells))
373
+ trusted_links = [str(value).strip() for value in (links or [])
374
+ if str(value).strip().startswith("#/")][:cap]
375
+ base = (os.environ.get("AIOS_PUBLIC_BASE") or os.environ.get("APP_BASE_URL") or "").rstrip("/")
376
+ if base:
377
+ trusted_links = [base + "/" + value for value in trusted_links]
378
+ if trusted_links:
379
+ lines.append("Open in Loopable: " + " ".join(trusted_links))
380
+ else:
381
+ return None
382
+ return "\n".join(lines)[:3200]
383
+
384
+
385
  def deliver_channel_reply(rt, channel, text, *, principal=None, automation_id=None,
386
  provenance=None, post=None):
387
  """Send only after one channel/principal binding and an optional automation grant validate."""
api/routes_templates.py CHANGED
@@ -239,13 +239,14 @@ def _requirement(database, definitions, have, database_states, connector=""):
239
  else "ready" if switched_on else "off"}
240
 
241
 
242
- def _catalog_row(key, name, description, requires):
243
  # Prefer the missing connection even when an earlier database is merely off.
244
  blocked = next((row for row in requires if row["state"] == "disconnected"), None)
245
  blocked = blocked or next((row for row in requires if row["state"] != "ready"), None)
246
  return {"key": key, "name": str(name).replace(chr(0x2014), "-").replace(chr(0x2013), "-"),
247
  "description": str(description).replace(chr(0x2014), "-").replace(chr(0x2013), "-"),
248
  "requires": requires, "creatable": all(row["state"] == "ready" for row in requires),
 
249
  "action": None if blocked is None else {
250
  "kind": "connect" if blocked["state"] == "disconnected" else "switch_on",
251
  "connector": blocked["connector"], "database": blocked["database"]}}
@@ -269,6 +270,8 @@ def _module_templates(session):
269
  from routes_connectors import _entry_types
270
  from routes_keychain import env_odoo_available
271
  from routes_nav import application_templates
 
 
272
 
273
  definitions = user_tables.all_defs(session.runtime)
274
  have, _scopes, _failure = _entry_types(session.runtime, session.uname)
@@ -299,6 +302,9 @@ def _module_templates(session):
299
  # These App contracts map fields in the existing wizard and require no fixed connector.
300
  app = [_catalog_row(row["kind"], row["label"], "Choose a database and map its fields.", [])
301
  for row in application_templates(session=session)["templates"]]
 
 
 
302
  return {"sections": [{"module": "sheet", "templates": sheet},
303
  {"module": "data", "templates": data},
304
  {"module": "app", "templates": app}]}
@@ -406,6 +412,35 @@ def create_module_template(module: str, key: str, body: dict = Body(default=None
406
  from urllib.parse import quote
407
 
408
  body = body if isinstance(body, dict) else {}
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
409
  section = next((row for row in _module_templates(session)["sections"] if row["module"] == module), None)
410
  template = next((row for row in section["templates"] if row["key"] == key), None) if section else None
411
  if template is None:
 
239
  else "ready" if switched_on else "off"}
240
 
241
 
242
+ def _catalog_row(key, name, description, requires, *, creation_mode="wizard"):
243
  # Prefer the missing connection even when an earlier database is merely off.
244
  blocked = next((row for row in requires if row["state"] == "disconnected"), None)
245
  blocked = blocked or next((row for row in requires if row["state"] != "ready"), None)
246
  return {"key": key, "name": str(name).replace(chr(0x2014), "-").replace(chr(0x2013), "-"),
247
  "description": str(description).replace(chr(0x2014), "-").replace(chr(0x2013), "-"),
248
  "requires": requires, "creatable": all(row["state"] == "ready" for row in requires),
249
+ "creationMode": creation_mode,
250
  "action": None if blocked is None else {
251
  "kind": "connect" if blocked["state"] == "disconnected" else "switch_on",
252
  "connector": blocked["connector"], "database": blocked["database"]}}
 
270
  from routes_connectors import _entry_types
271
  from routes_keychain import env_odoo_available
272
  from routes_nav import application_templates
273
+ from routes_applications import _may_open
274
+ from modules.finance_apps import finance_templates
275
 
276
  definitions = user_tables.all_defs(session.runtime)
277
  have, _scopes, _failure = _entry_types(session.runtime, session.uname)
 
302
  # These App contracts map fields in the existing wizard and require no fixed connector.
303
  app = [_catalog_row(row["kind"], row["label"], "Choose a database and map its fields.", [])
304
  for row in application_templates(session=session)["templates"]]
305
+ app.extend(_catalog_row("finance:" + row["key"], row["label"], row["description"], [],
306
+ creation_mode="post")
307
+ for row in finance_templates(may_open=lambda permission: _may_open(session, permission)))
308
  return {"sections": [{"module": "sheet", "templates": sheet},
309
  {"module": "data", "templates": data},
310
  {"module": "app", "templates": app}]}
 
412
  from urllib.parse import quote
413
 
414
  body = body if isinstance(body, dict) else {}
415
+ if module == "app" and key.startswith("finance:"):
416
+ import application_registry
417
+ from modules.finance_apps import finance_application
418
+ from routes_applications import _may_open
419
+
420
+ finance_key = key.removeprefix("finance:")
421
+ try:
422
+ descriptor = finance_application(
423
+ finance_key, may_open=lambda permission: _may_open(session, permission))
424
+ except KeyError:
425
+ raise err(404, "unknown_template", "That template does not exist.")
426
+ except PermissionError:
427
+ raise err(403, "finance_forbidden", f"Access to Finance template {finance_key!r} is not granted.")
428
+ if not session.runtime.available():
429
+ raise err(503, "store_unavailable", "The tenant store is unavailable. Nothing was saved.")
430
+ try:
431
+ application = application_registry.create(
432
+ session.runtime, session.tenant, session.uname,
433
+ {"templateKind": application_registry.FINANCE_TEMPLATE_KIND,
434
+ "financeKey": finance_key, "name": body.get("name") or descriptor["label"],
435
+ "navPlacement": {"section": "app", "order": 0}},
436
+ allowed_databases=set())
437
+ except application_registry.ApplicationError as exc:
438
+ raise err(400, "bad_application", str(exc))
439
+ except PermissionError as exc:
440
+ raise err(403, "forbidden", str(exc))
441
+ except Exception:
442
+ raise err(503, "store_unavailable", "The Finance application was not saved.")
443
+ return {"address": "#" + application["deepLink"], "application": application}
444
  section = next((row for row in _module_templates(session)["sections"] if row["module"] == module), None)
445
  template = next((row for row in section["templates"] if row["key"] == key), None) if section else None
446
  if template is None:
api/test_s63_integration.py ADDED
@@ -0,0 +1,240 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Hermetic regression checks for the SDLC 63 cross-lane repairs."""
2
+ import copy
3
+ import json
4
+ from pathlib import Path
5
+ import subprocess
6
+ import sys
7
+ import unittest
8
+ from unittest.mock import patch
9
+
10
+ ROOT = Path(__file__).resolve().parents[2]
11
+ sys.path[:0] = [str(ROOT / "platform"), str(ROOT / "aios-web/api")]
12
+
13
+ import aios_grid
14
+ import odoo_relational as rel
15
+ from core import user_tables
16
+ from harness import filter_eval
17
+ from verify_odoo_relational import FixtureCursor, MemStore
18
+
19
+
20
+ class SourceCounts:
21
+ """Data-free Odoo double whose answers differ by domain, not merely by model."""
22
+ def __init__(self):
23
+ self.calls = []
24
+
25
+ def search_count(self, model, domain):
26
+ self.calls.append(("count", model, domain))
27
+ if model == "account.move":
28
+ move_types = next(value for field, op, value in domain if field == "move_type")
29
+ return 2 if "out_invoice" in move_types else 3
30
+ return {"sale.order": 4, "account.account": 5,
31
+ "sale.order.line": 6, "account.move.line": 7}[model]
32
+
33
+ def search_read(self, model, domain, fields, limit=None):
34
+ self.calls.append(("read", model, domain))
35
+ return [{"id": 1}, {"id": 2}]
36
+
37
+ def read_group(self, model, domain, fields, groupby, lazy=False):
38
+ self.calls.append(("group", model, domain))
39
+ if model == "account.invoice.line.agent":
40
+ return [{"agent_id": [2, "Two"]}, {"agent_id": [3, "Three"]}]
41
+ return [{"partner_id": [8, "Eight"]}, {"partner_id": [9, "Nine"]}]
42
+
43
+
44
+ class IntegrationTests(unittest.TestCase):
45
+ def test_archived_defaults_and_removable_filter(self):
46
+ fields = json.loads((ROOT / "platform/aios_grid_fields.json").read_text())["fields"]
47
+ config = aios_grid._default_view_config(fields)
48
+ self.assertEqual(config["filters"], [{"colId": "archived", "op": "neq", "value": "true"}])
49
+ tree = {"conj": "and", "nodes": config["filters"]}
50
+ self.assertTrue(filter_eval.evaluate(tree, {"archived": False}, fields))
51
+ self.assertFalse(filter_eval.evaluate(tree, {"archived": True}, fields))
52
+ self.assertTrue(filter_eval.matches({"conj": "and", "nodes": []}, {"archived": True}, fields))
53
+ self.assertEqual(aios_grid._default_view_config([
54
+ {"key": "archived", "type": "checkbox", "source": "overlay"}])["filters"], [])
55
+ script = r'''
56
+ const fs=require('fs'),ts=require('typescript'),vm=require('vm'),assert=require('assert/strict');
57
+ const text=fs.readFileSync('src/customer-grid/useGridColumns.ts','utf8');
58
+ const file=ts.createSourceFile('views.ts',text,ts.ScriptTarget.Latest,true,ts.ScriptKind.TS);
59
+ const wanted=new Set(['isDefaultVisible','defaultViewConfig']);
60
+ const declarations=file.statements.filter(node=>ts.isFunctionDeclaration(node)&&node.name&&wanted.has(node.name.text));
61
+ assert.equal(declarations.length,2);
62
+ const js=ts.transpileModule(declarations.map(node=>node.getText(file)).join('\n'),
63
+ {compilerOptions:{module:ts.ModuleKind.CommonJS,target:ts.ScriptTarget.ES2020}}).outputText;
64
+ const context={exports:{}};vm.runInNewContext(js,context);
65
+ const archived={key:'archived',type:'checkbox',source:'odoo'};
66
+ const ordinary={key:'name',type:'text',source:'odoo'};
67
+ const overlay={key:'archived',type:'checkbox',source:'overlay'};
68
+ console.log(JSON.stringify({odoo:context.exports.defaultViewConfig([ordinary,archived]).filters,
69
+ overlay:context.exports.defaultViewConfig([ordinary,overlay]).filters}));
70
+ '''
71
+ result = subprocess.run(["node", "-e", script], cwd=ROOT / "aios-web/web",
72
+ capture_output=True, text=True, encoding="utf-8", timeout=30)
73
+ self.assertEqual(result.returncode, 0, result.stdout + result.stderr)
74
+ client = json.loads(result.stdout)
75
+ self.assertEqual(client["odoo"], config["filters"])
76
+ self.assertEqual(client["overlay"], [])
77
+
78
+ def test_database_read_failure_preserves_rows_and_successful_sibling(self):
79
+ rt = MemStore()
80
+ with patch.object(user_tables, "row_limit", return_value=None), \
81
+ patch.object(rel, "is_read_through_key", return_value=False):
82
+ first = rel.refresh_curated(rt, FixtureCursor(), {"invoices", "bills"}, retire=False)
83
+ self.assertEqual(first[2], {})
84
+ before = copy.deepcopy(rt.data[user_tables.STORE_KEY][rel.INVOICES_KEY])
85
+ with patch.dict(rel._READERS, {"invoices": lambda *_: (_ for _ in ()).throw(
86
+ RuntimeError("private row must never enter receipt"))}):
87
+ built, written, errors, receipts, _ = rel.refresh_curated(
88
+ rt, FixtureCursor(), {"invoices", "bills"}, retire=False)
89
+ self.assertNotIn("invoices", built)
90
+ self.assertNotIn(rel.INVOICES_KEY, written)
91
+ self.assertIn(rel.BILLS_KEY, written)
92
+ self.assertEqual(set(errors), {rel.INVOICES_KEY})
93
+ self.assertEqual(rt.data[user_tables.STORE_KEY][rel.INVOICES_KEY], before)
94
+ self.assertNotIn("private row", json.dumps(receipts))
95
+ self.assertEqual(receipts[rel.BILLS_KEY]["state"], "succeeded")
96
+ self.assertIsNone(receipts[rel.BILLS_KEY]["sourceCount"])
97
+ self.assertEqual(receipts[rel.BILLS_KEY]["mirrorCount"], written[rel.BILLS_KEY]["rows"])
98
+
99
+ def test_duplicate_reader_ids_refuse_before_mutation(self):
100
+ rt = MemStore()
101
+ with patch.object(user_tables, "row_limit", return_value=None), \
102
+ patch.dict(rel._READERS, {"invoices": lambda *_: [{"_id": "1"}, {"_id": "1"}]}):
103
+ _, written, errors, _, _ = rel.refresh_curated(rt, FixtureCursor(), {"invoices"}, retire=False)
104
+ self.assertEqual(written, {})
105
+ self.assertEqual(set(errors), {rel.INVOICES_KEY})
106
+ self.assertEqual(rt.data, {})
107
+
108
+ def test_all_curated_receipts_are_database_and_domain_specific(self):
109
+ rt = MemStore()
110
+ source = SourceCounts()
111
+ expected = {"invoices": 2, "orders": 4, "agents": 3, "accounts": 5,
112
+ "vendors": 2, "bills": 3, "order_lines": 6, "gl_lines": 7}
113
+ by_bucket = {bucket: key for bucket, key, _label, _fields in rel.TABLES}
114
+
115
+ def fake_plan(_cur, rt=None, connection_id=None, buckets=None):
116
+ bucket = next(iter(buckets))
117
+ rows = ([] if bucket in {"order_lines", "gl_lines"} else
118
+ [{"_id": f"{bucket}-{index}"} for index in range(expected[bucket])])
119
+ return {bucket: rows, "problems": [], "limits": {}}
120
+
121
+ def fake_apply(_rt, planned, **_kwargs):
122
+ bucket = next(key for key in expected if key in planned)
123
+ key = by_bucket[bucket]
124
+ return {key: {"rows": len(planned[bucket])}}
125
+
126
+ def fake_population(key, **_kwargs):
127
+ bucket = next(name for name, table_key in by_bucket.items() if table_key == key)
128
+ return expected[bucket]
129
+
130
+ with patch.object(rel, "plan", side_effect=fake_plan), \
131
+ patch.object(rel, "apply_plan", side_effect=fake_apply), \
132
+ patch.object(user_tables, "row_limit", return_value=None), \
133
+ patch.object(rel, "is_read_through_key",
134
+ side_effect=lambda key: key in {rel.ORDER_LINES_KEY, rel.GL_LINES_KEY}), \
135
+ patch("routes_odoo_tables.population", side_effect=fake_population):
136
+ built, written, errors, receipts, _ = rel.refresh_curated(
137
+ rt, FixtureCursor(), set(expected), retire=False, source_client=source)
138
+
139
+ self.assertEqual(errors, {})
140
+ self.assertEqual(set(receipts), set(by_bucket.values()))
141
+ self.assertEqual(set(written), set(by_bucket.values()))
142
+ for bucket, key in by_bucket.items():
143
+ receipt = receipts[key]
144
+ self.assertEqual(receipt["schemaKey"], key)
145
+ self.assertEqual(receipt["domainKey"], rel.TABLE_SUBJECTS[key])
146
+ self.assertEqual(receipt["topicKey"], rel.TABLE_TOPICS[key])
147
+ self.assertEqual(receipt["selection"], {"bucket": bucket, "enabled": True})
148
+ self.assertEqual(receipt["sourceCount"], expected[bucket])
149
+ self.assertEqual(receipt["mirrorCount"], expected[bucket])
150
+ self.assertEqual(receipt["sourceReconciliation"], "matched")
151
+ self.assertEqual(receipt["freshness"]["state"], "current")
152
+ self.assertEqual(receipts[rel.INVOICES_KEY]["model"], "account.move")
153
+ self.assertEqual(receipts[rel.BILLS_KEY]["model"], "account.move")
154
+ self.assertNotEqual(receipts[rel.INVOICES_KEY]["domain"],
155
+ receipts[rel.BILLS_KEY]["domain"])
156
+ self.assertEqual(receipts[rel.VENDORS_KEY]["model"], "res.partner")
157
+ self.assertEqual(receipts[rel.VENDORS_KEY]["countModel"], "account.move")
158
+ self.assertEqual(receipts[rel.ORDER_LINES_KEY]["materializedCount"], 0)
159
+ self.assertEqual(receipts[rel.ORDER_LINES_KEY]["servingCount"], 6)
160
+ route = (ROOT / "aios-web/api/routes_odoo_tables.py").read_text(encoding="utf-8")
161
+ refresh_endpoint = route.split("def refresh_odoo_tables", 1)[1].split(
162
+ '@router.get("/odoo-tables/status")', 1)[0]
163
+ self.assertIn("all_connections=True", refresh_endpoint)
164
+
165
+ def test_browser_grid_header_and_cell_events_with_both_negative_controls(self):
166
+ script = r'''
167
+ const fs=require('fs'),ts=require('typescript'),assert=require('assert/strict');
168
+ const text=fs.readFileSync('src/sheet/SheetGrid.tsx','utf8');
169
+ const file=ts.createSourceFile('grid.tsx',text,ts.ScriptTarget.Latest,true,ts.ScriptKind.TSX);
170
+ const handlers=[];
171
+ function visit(node){
172
+ if(ts.isJsxAttribute(node)&&node.name.getText(file)==='onCellClicked') handlers.push(node.initializer.expression.getText(file));
173
+ ts.forEachChild(node,visit);
174
+ }
175
+ visit(file); assert.equal(handlers.length,2);
176
+ function compiled(handler){
177
+ return ts.transpileModule('globalThis.__sheetClick = '+handler,{compilerOptions:{target:ts.ScriptTarget.ES2020}}).outputText;
178
+ }
179
+ const controls=handlers.map(handler=>{
180
+ const sentinel=handler.indexOf('if (item[0] === -1)');
181
+ const signalAt=handler.indexOf('signal(NAV_MINIMIZE_EVENT);',sentinel);
182
+ assert.ok(sentinel>=0&&signalAt>sentinel,'production signal must follow the sentinel');
183
+ const signal='signal(NAV_MINIMIZE_EVENT);';
184
+ const moved=handler.slice(0,sentinel)+signal+'\n'+handler.slice(sentinel,signalAt)+
185
+ handler.slice(signalAt+signal.length);
186
+ return {positive:compiled(handler),negative:compiled(moved)};
187
+ });
188
+ console.log(JSON.stringify(controls));
189
+ '''
190
+ result = subprocess.run(["node", "-e", script], cwd=ROOT / "aios-web/web",
191
+ capture_output=True, text=True, encoding="utf-8", timeout=30)
192
+ self.assertEqual(result.returncode, 0, result.stdout + result.stderr)
193
+ controls = json.loads(result.stdout)
194
+
195
+ # This leg clicks real browser DOM targets wired to the production TypeScript handlers.
196
+ # Each pane also gets the inverse build (signal moved before the row-header sentinel), so
197
+ # the browser proof can demonstrate that it detects the exact regression it guards.
198
+ from playwright.sync_api import sync_playwright
199
+ with sync_playwright() as playwright:
200
+ browser = playwright.chromium.launch(headless=True)
201
+ page = browser.new_page()
202
+ page.set_content('<button id="header">row header</button><button id="cell">cell</button>')
203
+ for pane, pair in zip(("frozen", "unfrozen"), controls, strict=True):
204
+ def run(compiled):
205
+ page.evaluate("""() => {
206
+ globalThis.__signals = 0; globalThis.__rows = 0;
207
+ globalThis.signal = () => globalThis.__signals++;
208
+ globalThis.NAV_MINIMIZE_EVENT = 'aios:nav-minimize';
209
+ globalThis.fromBand = value => value;
210
+ globalThis.selectDocumentAxis = () => globalThis.__rows++;
211
+ globalThis.painter = false; globalThis.addrAt = () => 'A1';
212
+ globalThis.openViewHeader = () => false;
213
+ globalThis.validationOf = () => false;
214
+ globalThis.cellAt = () => null; globalThis.tab = {};
215
+ globalThis.followCellLink = () => false; globalThis.setAnchor = () => {};
216
+ }""")
217
+ page.evaluate("code => { (0, eval)(code); }", compiled)
218
+ page.evaluate("""() => {
219
+ document.querySelector('#header').onclick = () =>
220
+ globalThis.__sheetClick([-1, 0], {shiftKey: false, bounds: {}});
221
+ document.querySelector('#cell').onclick = () =>
222
+ globalThis.__sheetClick([0, 0], {shiftKey: false, bounds: {}});
223
+ }""")
224
+ page.click("#header")
225
+ after_header = page.evaluate("() => ({signals: __signals, rows: __rows})")
226
+ page.click("#cell")
227
+ after_cell = page.evaluate("() => ({signals: __signals, rows: __rows})")
228
+ return after_header, after_cell
229
+
230
+ header, cell = run(pair["positive"])
231
+ self.assertEqual(header, {"signals": 0, "rows": 1}, pane)
232
+ self.assertEqual(cell, {"signals": 1, "rows": 1}, pane)
233
+ broken_header, _ = run(pair["negative"])
234
+ self.assertEqual(broken_header, {"signals": 1, "rows": 1},
235
+ f"{pane} negative control did not expose the regression")
236
+ browser.close()
237
+
238
+
239
+ if __name__ == "__main__":
240
+ unittest.main()
api/test_sheet_commit_events.py ADDED
@@ -0,0 +1,162 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Exercise value-only workbook commit notifications without crossing cell contents."""
2
+ import copy
3
+ from pathlib import Path
4
+ import sys
5
+ import unittest
6
+ from unittest.mock import patch
7
+
8
+ ROOT = Path(__file__).resolve().parents[2]
9
+ sys.path[:0] = [str(ROOT / "platform"), str(ROOT / "aios-web/api")]
10
+ from core import sheets, sheet_compat, sheet_events
11
+
12
+
13
+ class MemoryStore:
14
+ key = "sheet-event-fixture"
15
+
16
+ def __init__(self):
17
+ self.data = {}
18
+ self.locked = False
19
+ self.fail_commit = False
20
+
21
+ def get(self, key, default=None):
22
+ return copy.deepcopy(self.data.get(key, default))
23
+
24
+ def update(self, key, fn, **kwargs):
25
+ self.locked = True
26
+ try:
27
+ value = fn(copy.deepcopy(self.data.get(key, {})))
28
+ if self.fail_commit:
29
+ raise RuntimeError("fixture commit failure")
30
+ self.data[key] = copy.deepcopy(value)
31
+ finally:
32
+ self.locked = False
33
+
34
+
35
+ class CommitEvents(unittest.TestCase):
36
+ def setUp(self):
37
+ self.st = MemoryStore()
38
+ self.book = sheets.create_workbook(self.st, name="Fixture", created_by="qa")
39
+ self.tab = self.book["sheets"][0]["id"]
40
+ self.events = []
41
+
42
+ def observe(st, events):
43
+ self.assertFalse(st.locked, "no Automation call while the workbook lock is held")
44
+ for event in events:
45
+ self.assertEqual(st.get(sheets.workbook_key(self.book["id"]))["rev"], event["revision"])
46
+ self.events.extend(events)
47
+
48
+ self.observer = patch.object(sheet_events, "COMMIT_OBSERVER", observe)
49
+ self.observer.start()
50
+ self.addCleanup(self.observer.stop)
51
+
52
+ def patch_cells(self, cells):
53
+ return sheets.patch_cells(self.st, self.book["id"], self.tab, cells, actor="qa")
54
+
55
+ def test_bulk_patch_is_one_committed_event_without_cell_values(self):
56
+ self.patch_cells({f"B{i}": {"kind": "literal", "value": i} for i in range(1, 20001)})
57
+ self.assertEqual(len(self.events), 1)
58
+ event = self.events[0]
59
+ self.assertEqual(event["ranges"], ["B1:B20000"])
60
+ self.assertEqual(event["recordCount"], 20000)
61
+ self.assertEqual(event["changeKinds"], ["value"])
62
+ self.assertTrue(event["committed"])
63
+ self.assertNotIn("cells", event)
64
+ self.assertNotIn("values", event)
65
+
66
+ def test_formatting_identical_values_and_refused_cells_do_not_emit(self):
67
+ self.patch_cells({"A1": {"kind": "literal", "value": 7}})
68
+ self.events.clear()
69
+ self.patch_cells({"A1": {"kind": "literal", "value": 7, "style": {"bold": True}}})
70
+ self.patch_cells({"A1": {"kind": "literal", "value": 7}, "B2": {"kind": "unknown"}})
71
+ self.assertEqual(self.events, [])
72
+
73
+ def test_failed_commit_and_stale_put_do_not_emit(self):
74
+ self.st.fail_commit = True
75
+ with self.assertRaises(RuntimeError):
76
+ self.patch_cells({"A1": {"kind": "literal", "value": 3}})
77
+ self.st.fail_commit = False
78
+ self.assertEqual(self.events, [])
79
+ self.patch_cells({"A1": {"kind": "literal", "value": 3}})
80
+ self.events.clear()
81
+ with self.assertRaises(sheets.SheetRefusal):
82
+ sheets.put_workbook(self.st, self.book["id"], self.book)
83
+ self.assertEqual(self.events, [])
84
+
85
+ def test_removal_and_sparse_changes_keep_exact_watched_ranges(self):
86
+ self.patch_cells({"A1": {"kind": "literal", "value": 1},
87
+ "A3": {"kind": "literal", "value": 3}})
88
+ self.assertEqual(self.events[-1]["ranges"], ["A1", "A3"])
89
+ self.patch_cells({"A1": None})
90
+ self.assertEqual(self.events[-1]["ranges"], ["A1"])
91
+
92
+ def test_refresh_and_compatibility_share_the_same_commit_boundary(self):
93
+ book = sheets.get_workbook(self.st, self.book["id"])
94
+ book["sheets"][0]["cells"] = {"A1": {"kind": "literal", "value": 4}}
95
+ saved = sheets.put_workbook(self.st, self.book["id"], book, lossless=True)
96
+ self.assertEqual(len(self.events), 1)
97
+ sheet_compat.execute(self.st, self.book["id"], "range.values.write",
98
+ {"sheet": self.tab, "range": "A1", "values": [[5]],
99
+ "expected_revision": saved["rev"]}, actor="qa")
100
+ self.assertEqual(len(self.events), 2)
101
+ self.assertNotEqual(self.events[0]["batchId"], self.events[1]["batchId"])
102
+
103
+ def test_store_retry_publishes_only_the_successful_mutation(self):
104
+ original = self.st.update
105
+
106
+ def retry(key, fn, **kwargs):
107
+ if key == sheets.workbook_key(self.book["id"]):
108
+ discarded = self.st.get(key)
109
+ discarded["rev"] = 900
110
+ fn(discarded)
111
+ return original(key, fn, **kwargs)
112
+
113
+ with patch.object(self.st, "update", retry):
114
+ self.patch_cells({"A1": {"kind": "literal", "value": 2}})
115
+ self.assertEqual(len(self.events), 1)
116
+ self.assertLess(self.events[0]["revision"], 900)
117
+
118
+ def test_new_tab_backfill_is_not_an_existing_watch_change(self):
119
+ book = sheets.get_workbook(self.st, self.book["id"])
120
+ book["sheets"].append({"id": "imported", "name": "Imported", "cells": {
121
+ "A1": {"kind": "literal", "value": 17}}})
122
+ sheets.put_workbook(self.st, self.book["id"], book)
123
+ self.assertEqual(self.events, [])
124
+
125
+ def test_binding_refresh_publishes_output_change_but_not_refresh_stamp(self):
126
+ cell = {"kind": "bound", "value": 10, "binding": {
127
+ "database": "customers", "aggregate": "balance", "id": "current_ytd"}}
128
+ # Drive the persistence seam with already-canonical stored cells to isolate refresh
129
+ # notification from the independently tested binding-schema cleaner.
130
+ key = sheets.workbook_key(self.book["id"])
131
+ self.st.data[key]["sheets"][0]["cells"]["A1"] = copy.deepcopy(cell)
132
+
133
+ def refresh(state):
134
+ state["rev"] += 1
135
+ state["sheets"][0]["cells"]["A1"].update(value=11, refreshed_at="fixture-1")
136
+ return state, state
137
+
138
+ sheets._persist(self.st, key, refresh)
139
+ self.assertEqual(self.events[-1]["changeKinds"], ["value", "binding"])
140
+ self.assertIn("current_ytd", self.events[-1]["bindings"])
141
+ self.assertIn("customers.balance", self.events[-1]["bindings"])
142
+ self.events.clear()
143
+ sheets._persist(self.st, key, refresh)
144
+ self.assertEqual(self.events, [])
145
+
146
+ def test_notification_failure_does_not_turn_committed_edit_into_a_refusal(self):
147
+ with patch.object(sheet_events, "COMMIT_OBSERVER", side_effect=RuntimeError("private detail")), \
148
+ self.assertLogs("core.sheet_events", level="ERROR") as logs:
149
+ result = self.patch_cells({"A1": {"kind": "literal", "value": 42}})
150
+ self.assertEqual(sheets.get_workbook(self.st, self.book["id"])["rev"], result["rev"])
151
+ self.assertNotIn("private detail", str(logs.output))
152
+
153
+ def test_negative_control_detects_disconnected_commit_publisher(self):
154
+ result = unittest.TestResult()
155
+ with patch.object(sheet_events, "publish", return_value=None):
156
+ CommitEvents("test_bulk_patch_is_one_committed_event_without_cell_values").run(result)
157
+ self.assertEqual(len(result.failures), 1)
158
+ self.assertEqual(result.errors, [])
159
+
160
+
161
+ if __name__ == "__main__":
162
+ unittest.main()
platform/aios_grid.py CHANGED
@@ -3577,10 +3577,13 @@ def _default_view_config(fields):
3577
  # MUST move together, which is the whole subject of the note above.
3578
  shown = [field["key"] for field in fields if _is_default_visible(field)]
3579
  hidden = [field["key"] for field in fields if field["key"] not in shown]
 
 
 
3580
  return {
3581
  # `filters` is the ROOT of the filter tree: leaf conditions and/or nested
3582
  # condition groups ({conj, children}); `filterConj` joins the root level.
3583
- "filters": [], "filterConj": "and",
3584
  "sorts": [], "groupBy": None, "colorBy": None,
3585
  "rowHeightMode": "short", "order": shown + hidden, "visible": shown,
3586
  "widths": {}, "memberPids": [],
@@ -3702,7 +3705,7 @@ def views_from_defs(defs, saved_views, fields, system_name="All customers", lock
3702
  op_map = {">=": "gte", ">": "gt", "<=": "lte", "<": "lt", "=": "eq",
3703
  "contains": "contains"}
3704
  for name, definition in (defs or {}).items():
3705
- filters = []
3706
  for rule in definition.get("rules") or []:
3707
  if rule.get("field") not in {field["key"] for field in fields}:
3708
  continue
 
3577
  # MUST move together, which is the whole subject of the note above.
3578
  shown = [field["key"] for field in fields if _is_default_visible(field)]
3579
  hidden = [field["key"] for field in fields if field["key"] not in shown]
3580
+ archive_filter = [{"colId": "archived", "op": "neq", "value": "true"}] if any(
3581
+ field.get("key") == "archived" and field.get("type") == "checkbox"
3582
+ and field.get("source") == "odoo" for field in fields) else []
3583
  return {
3584
  # `filters` is the ROOT of the filter tree: leaf conditions and/or nested
3585
  # condition groups ({conj, children}); `filterConj` joins the root level.
3586
+ "filters": archive_filter, "filterConj": "and",
3587
  "sorts": [], "groupBy": None, "colorBy": None,
3588
  "rowHeightMode": "short", "order": shown + hidden, "visible": shown,
3589
  "widths": {}, "memberPids": [],
 
3705
  op_map = {">=": "gte", ">": "gt", "<=": "lte", "<": "lt", "=": "eq",
3706
  "contains": "contains"}
3707
  for name, definition in (defs or {}).items():
3708
+ filters = [dict(rule) for rule in base["filters"]]
3709
  for rule in definition.get("rules") or []:
3710
  if rule.get("field") not in {field["key"] for field in fields}:
3711
  continue
platform/aios_grid_fields.json CHANGED
@@ -32,6 +32,14 @@
32
  ],
33
  "description": "Whether this customer still exists in Odoo. Archived means deleted there."
34
  },
 
 
 
 
 
 
 
 
35
  {
36
  "key": "agent",
37
  "label": "Agent",
 
32
  ],
33
  "description": "Whether this customer still exists in Odoo. Archived means deleted there."
34
  },
35
+ {
36
+ "key": "archived",
37
+ "label": "Archived",
38
+ "type": "checkbox",
39
+ "source": "odoo",
40
+ "default": false,
41
+ "description": "Whether this customer is archived in Odoo."
42
+ },
43
  {
44
  "key": "agent",
45
  "label": "Agent",
platform/core/audit.py CHANGED
@@ -12,6 +12,8 @@ AUDIT_KINDS = frozenset({
12
  "auth.mfa_failed", "auth.logout", "admin.grant", "credential.added",
13
  "credential.rotated", "credential.revoked", "amazon.sync", "amazon.throttled",
14
  "amazon.authorization_lost", "data.purge", "data.restore", "config.changed",
 
 
15
  })
16
  BUCKET = "security_events"
17
  MAX_EVENTS = 2000
@@ -33,6 +35,7 @@ SECRET_KEYS = frozenset(_key_name(key) for key in (
33
  ))
34
  _BEARER = re.compile(r"\bbearer\s+\S+", re.IGNORECASE)
35
  _JWT = re.compile(r"\beyJ[A-Za-z0-9_-]*\.")
 
36
 
37
 
38
  class AuditRefused(ValueError):
@@ -54,6 +57,8 @@ def _string(key, value):
54
  raise AuditRefused(key, "bearer token shape")
55
  if _JWT.search(value):
56
  raise AuditRefused(key, "JWT token shape")
 
 
57
 
58
 
59
  def _attribution(key, value):
@@ -174,3 +179,27 @@ def read(tenant, limit=200, now=None) -> list[dict]:
174
  doc = runtime.get_runtime(str(tenant)).get(BUCKET)
175
  rows = ordered(doc, tenant, limit)
176
  return [row for row in rows if row["ts"] >= now - RETENTION_SECONDS]
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
12
  "auth.mfa_failed", "auth.logout", "admin.grant", "credential.added",
13
  "credential.rotated", "credential.revoked", "amazon.sync", "amazon.throttled",
14
  "amazon.authorization_lost", "data.purge", "data.restore", "config.changed",
15
+ "external.query",
16
+ "external.write",
17
  })
18
  BUCKET = "security_events"
19
  MAX_EVENTS = 2000
 
35
  ))
36
  _BEARER = re.compile(r"\bbearer\s+\S+", re.IGNORECASE)
37
  _JWT = re.compile(r"\beyJ[A-Za-z0-9_-]*\.")
38
+ _LOOPABLE_CREDENTIAL = re.compile(r"\blp(?:b|o|r|c)1\.[A-Za-z0-9._~-]+")
39
 
40
 
41
  class AuditRefused(ValueError):
 
57
  raise AuditRefused(key, "bearer token shape")
58
  if _JWT.search(value):
59
  raise AuditRefused(key, "JWT token shape")
60
+ if _LOOPABLE_CREDENTIAL.search(value):
61
+ raise AuditRefused(key, "Loopable credential shape")
62
 
63
 
64
  def _attribution(key, value):
 
179
  doc = runtime.get_runtime(str(tenant)).get(BUCKET)
180
  rows = ordered(doc, tenant, limit)
181
  return [row for row in rows if row["ts"] >= now - RETENTION_SECONDS]
182
+
183
+
184
+ def external_write_receipt(tenant, principal, *, tool, auth_type, affected_count,
185
+ diff_digest, revision_before, revision_after,
186
+ confirmation_id) -> dict:
187
+ """Emit and return a value-free C7 write receipt.
188
+
189
+ Target ids, field names, record values and arguments deliberately do not belong here. The
190
+ receipt proves who confirmed which normalized diff and how many objects moved; the concrete
191
+ before/after stays in the short-lived preview response instead of the retained security log.
192
+ """
193
+ detail = {
194
+ "surface": "ai_api",
195
+ "tool": str(tool or "")[:128],
196
+ "authentication": str(auth_type or "")[:32],
197
+ "affected_count": max(0, int(affected_count or 0)),
198
+ "diff_digest": str(diff_digest or "")[:128],
199
+ "revision_before": str(revision_before or "")[:128],
200
+ "revision_after": str(revision_after or "")[:128],
201
+ "confirmation_id": str(confirmation_id or "")[:128],
202
+ }
203
+ item = event("external.write", tenant, principal, detail)
204
+ stored = record("external.write", tenant, principal, detail)
205
+ return {"event": item, "persisted": stored is not None}
platform/core/daily_sales_repair.py ADDED
@@ -0,0 +1,977 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """A refusal-first, exact-snapshot repair for a Daily Sales workbook the caller names.
2
+
3
+ The legacy source-oracle plan and the workbook-derived invoice summary plan share the same
4
+ backup, exact-write and readback guards. Canonical periods use core.windows.normalize; the
5
+ summary uses the workbook's own filter valueRefs instead of freezing its calendar in a period.
6
+ The authorized operator injects the independent invoice validator and the full joined partner set.
7
+ The module validates the complete projected document, writes and verifies a backup, then delegates
8
+ one atomic, preconditioned cell mutation to :mod:`core.sheets`.
9
+ """
10
+
11
+ from __future__ import annotations
12
+
13
+ import copy
14
+ import json
15
+ import math
16
+ import re
17
+ from datetime import date
18
+
19
+ from core import sheets
20
+ from core import windows as W
21
+
22
+
23
+ TARGET_WORKBOOK_ID = "b8edba3d3f47"
24
+ TARGET_TENANT = "royal-imports"
25
+ SUMMARY_TAB = "Summary (All)"
26
+ SUMMARY_TAB_ID = "c2545d26c42c"
27
+ SUMMARY_COLUMNS = tuple(chr(column) for column in range(ord("F"), ord("R") + 1))
28
+ TICKET_FORMULA_ERROR_CODES = frozenset({"#VALUE!", "#DIV/0!", "#REF!", "#N/A", "#NAME?"})
29
+ _EXCEL_ERROR_ALIASES = {
30
+ "bad_value": "#VALUE!", "div_zero": "#DIV/0!", "invalid_reference": "#REF!",
31
+ "external_reference": "#REF!", "no_match": "#N/A", "unknown_name": "#NAME?",
32
+ }
33
+ _SUMMARY_RATIO_ROOTS = {
34
+ "F69": "=(F75/F64) * (365 / (F$7 - DATE(YEAR(F$7),1,1) + 1))",
35
+ "F70": "=(F76/F67) * (365 / (F$7 - DATE(YEAR(F$7),1,1) + 1))",
36
+ "R70": "=(R76/R67) * (365 / (R$7 - DATE(YEAR(R$7),1,1) + 1))",
37
+ }
38
+ _SUMMARY_RATIO_CASCADES = {
39
+ "F71": "=F69-F70", "F72": "=F71/F70", "F73": "=G72-F72",
40
+ "R71": "=R69-R70", "R72": "=R71/R70", "R73": "=R72-Q72",
41
+ }
42
+
43
+
44
+ def operation_for(workbook_id):
45
+ """Bind a repair operation to the workbook the caller names."""
46
+ return "repair-daily-sales-" + workbook_id
47
+
48
+
49
+ OPERATION = operation_for(TARGET_WORKBOOK_ID)
50
+ YTD_NAMES = {"Current YTD": "ytd", "Previous YTD": "ytd_last_year"}
51
+ if not all(kind in W.WINDOW_KINDS for kind in YTD_NAMES.values()):
52
+ raise RuntimeError("C2 requires every YTD window kind to exist in core.windows.WINDOW_KINDS.")
53
+
54
+ _VISUAL_TAB_KEYS = (
55
+ "colWidths", "rowHeight", "rowHeights", "freeze", "rules", "hiddenRows", "hiddenCols",
56
+ "charts", "visibility", "view", "print", "excel",
57
+ )
58
+ _VISUAL_CELL_KEYS = ("style", "validation", "link", "note", "sparkline", "excel")
59
+ _FORBIDDEN_BINDING_KEYS = {"fallback", "static", "staticValue", "defaultValue"}
60
+
61
+
62
+ class RepairRefusal(sheets.SheetRefusal):
63
+ """A named refusal from the exceptional repair workflow."""
64
+
65
+
66
+ def _canonical(value):
67
+ return json.loads(json.dumps(value, sort_keys=True, default=str))
68
+
69
+
70
+ def _tab(book, name):
71
+ try:
72
+ return sheets.find_tab(book, name)
73
+ except sheets.SheetRefusal as exc:
74
+ raise RepairRefusal(exc.code, exc.message) from exc
75
+
76
+
77
+ def _addr(raw):
78
+ value = str(raw or "").strip().upper().replace("$", "")
79
+ match = re.fullmatch(r"([A-Z]{1,3})([1-9][0-9]{0,6})", value)
80
+ if not match:
81
+ raise RepairRefusal("repair_address_invalid", "Every repair source needs one A1 address.")
82
+ column = 0
83
+ for char in match.group(1):
84
+ column = column * 26 + ord(char) - ord("A") + 1
85
+ if column > sheets.DOCUMENT_MAX_COLS or int(match.group(2)) > sheets.DOCUMENT_MAX_ROWS:
86
+ raise RepairRefusal("repair_address_invalid",
87
+ "A repair source address is outside the workbook dimensions.")
88
+ return value
89
+
90
+
91
+ def _validate_marker(marker, expected_revision, workbook_id):
92
+ if not isinstance(marker, dict) or marker.get("direct_tenant_data_live") is not True:
93
+ raise RepairRefusal(
94
+ "repair_marker_required",
95
+ "This repair requires the direct tenant data live marker from the owner instruction.")
96
+ expected = {
97
+ "operation": operation_for(workbook_id),
98
+ "tenant": TARGET_TENANT,
99
+ "workbook_id": workbook_id,
100
+ "expected_revision": expected_revision,
101
+ }
102
+ if any(marker.get(key) != value for key, value in expected.items()):
103
+ raise RepairRefusal(
104
+ "repair_marker_mismatch",
105
+ "The direct tenant data marker does not exactly match this workbook and revision.")
106
+ if not str(marker.get("owner_instruction") or "").strip():
107
+ raise RepairRefusal(
108
+ "repair_owner_instruction_required",
109
+ "The direct tenant data marker must carry the owner's exact repair instruction.")
110
+
111
+
112
+ def _contains_forbidden_key(value):
113
+ if isinstance(value, dict):
114
+ return bool(_FORBIDDEN_BINDING_KEYS.intersection(value)) or any(
115
+ _contains_forbidden_key(child) for child in value.values())
116
+ if isinstance(value, list):
117
+ return any(_contains_forbidden_key(child) for child in value)
118
+ return False
119
+
120
+
121
+ def _validate_binding(name, binding, *, window_kind=None):
122
+ if not isinstance(binding, dict) or _contains_forbidden_key(binding):
123
+ raise RepairRefusal(
124
+ "repair_binding_static",
125
+ name + " must be a canonical database binding without a static fallback.")
126
+ if not str(binding.get("database") or "").strip():
127
+ raise RepairRefusal("repair_binding_invalid", name + " must name its canonical database.")
128
+ aggregate = binding.get("aggregate")
129
+ if not isinstance(aggregate, dict) or not (
130
+ aggregate.get("metric_key") or aggregate.get("fn")):
131
+ raise RepairRefusal("repair_binding_invalid", name + " must name a real aggregate.")
132
+ if not aggregate.get("metric_key") and (
133
+ aggregate.get("fn") not in {"sum", "count", "avg", "min", "max"}
134
+ or (aggregate["fn"] != "count" and not aggregate.get("field"))):
135
+ raise RepairRefusal("repair_binding_invalid", name + " must name a real column aggregate.")
136
+ scope = binding.get("scope")
137
+ if not isinstance(scope, dict):
138
+ scope = {}
139
+ has_view = bool(str(scope.get("view_id") or "").strip())
140
+ tree = scope.get("filter")
141
+ # Accept the same explicit, nonempty conjunction used by rows 108-113. Requiring
142
+ # both referenced date bounds keeps a missing period from becoming an unbounded sum.
143
+ nodes = tree.get("nodes") if isinstance(tree, dict) else None
144
+ has_filter = (isinstance(nodes, list) and bool(nodes) and tree.get("conj") == "and"
145
+ and all(isinstance(n, dict) and n.get("colId") and n.get("op")
146
+ and ("value" in n or isinstance(n.get("valueRef"), dict))
147
+ for n in nodes))
148
+ if not has_view and not has_filter:
149
+ raise RepairRefusal(
150
+ "repair_binding_scope_required",
151
+ name + " must use a saved View or an explicit nonempty filter scope.")
152
+ if "related" in binding:
153
+ from core.sheet_binding import BindingMetadataRefusal, clean_related
154
+ try:
155
+ clean_related(binding["related"])
156
+ except BindingMetadataRefusal as exc:
157
+ raise RepairRefusal(exc.code, exc.message) from exc
158
+ period = binding.get("period")
159
+ if period is None and has_filter and not has_view:
160
+ bounds = {}
161
+ for node in nodes:
162
+ slot = node.get("valueRef")
163
+ if slot is not None:
164
+ if not isinstance(slot, dict) or set(slot) != {"ref"}:
165
+ raise RepairRefusal("repair_binding_invalid", name + " has an invalid cell reference.")
166
+ _addr(slot["ref"])
167
+ if node["op"] in {"gte", "lte"}:
168
+ bounds.setdefault(node["colId"], set()).add(node["op"])
169
+ if any(ops == {"gte", "lte"} for ops in bounds.values()):
170
+ return
171
+ window = W.normalize(copy.deepcopy(period.get("window") if isinstance(period, dict) else None))
172
+ if window is None or (window_kind is not None and window["kind"] != window_kind):
173
+ expected_window = "the " + window_kind if window_kind is not None else "a valid"
174
+ raise RepairRefusal(
175
+ "repair_binding_period_invalid",
176
+ name + " must use " + expected_window + " canonical date window.")
177
+
178
+
179
+ def _preserve_visual(before, desired):
180
+ """Carry every T20 cell display property across a content-only repair."""
181
+ held = {key: copy.deepcopy((before or {}).get(key)) for key in _VISUAL_CELL_KEYS
182
+ if isinstance(before, dict) and key in before}
183
+ if desired is None:
184
+ return {"kind": "empty", "state": "typed", **held} if held else None
185
+ out = copy.deepcopy(desired)
186
+ for key, value in held.items():
187
+ out[key] = value
188
+ return out
189
+
190
+
191
+ def visual_metadata(book):
192
+ """The T20 visual strata, including per-cell display properties."""
193
+ tabs = []
194
+ for tab in book.get("sheets") or ():
195
+ if not isinstance(tab, dict):
196
+ continue
197
+ item = {"id": tab.get("id"), "name": tab.get("name")}
198
+ for key in _VISUAL_TAB_KEYS:
199
+ if key in tab:
200
+ item[key] = copy.deepcopy(tab[key])
201
+ cell_visuals = {}
202
+ for address, cell in (tab.get("cells") or {}).items():
203
+ kept = {key: copy.deepcopy(cell[key]) for key in _VISUAL_CELL_KEYS
204
+ if isinstance(cell, dict) and key in cell}
205
+ if kept:
206
+ cell_visuals[address] = kept
207
+ item["cells"] = cell_visuals
208
+ tabs.append(item)
209
+ return _canonical(tabs)
210
+
211
+
212
+ def formula_error_census(book):
213
+ """Return every stored formula that still carries a calculation refusal/error."""
214
+ errors = []
215
+ for tab in book.get("sheets") or ():
216
+ for address, cell in (tab.get("cells") or {}).items():
217
+ if not isinstance(cell, dict) or cell.get("kind") != "formula":
218
+ continue
219
+ error = cell.get("error") or (cell.get("refusal") if cell.get("state") == "refused" else None)
220
+ if error:
221
+ errors.append({"sheet": tab.get("name"), "address": address,
222
+ "code": error.get("code") if isinstance(error, dict) else str(error)})
223
+ return sorted(errors, key=lambda row: (str(row["sheet"]), row["address"]))
224
+
225
+
226
+ def excel_formula_error_census(book, *, sheet=SUMMARY_TAB):
227
+ """The ticket's five Excel codes, separate from the unchanged full error census."""
228
+ errors = []
229
+ for tab in book.get("sheets") or ():
230
+ if sheet is not None and tab.get("name") != sheet:
231
+ continue
232
+ for address, cell in (tab.get("cells") or {}).items():
233
+ if not isinstance(cell, dict) or cell.get("kind") != "formula":
234
+ continue
235
+ error = cell.get("error") or (cell.get("refusal") if cell.get("state") == "refused" else None)
236
+ if not error:
237
+ continue
238
+ code = error.get("code") if isinstance(error, dict) else str(error)
239
+ excel_code = error.get("excelCode") if isinstance(error, dict) else None
240
+ excel_code = excel_code or _EXCEL_ERROR_ALIASES.get(code, code)
241
+ if excel_code in TICKET_FORMULA_ERROR_CODES:
242
+ errors.append({"sheet": tab.get("name"), "address": address,
243
+ "code": code, "excelCode": excel_code})
244
+ return sorted(errors, key=lambda row: (str(row["sheet"]), row["address"]))
245
+
246
+
247
+ def check_summary_formula_errors(book):
248
+ """Count the real tab only after proving this same census finds planted errors.
249
+
250
+ The negative control is an isolated in-memory document, never a tenant edit. Exercise
251
+ both code spellings, excelCode precedence, and the refused-cell fallback on every call.
252
+ """
253
+ _tab(book, SUMMARY_TAB)
254
+ planted, expected = {}, {}
255
+ for internal, excel in (("bad_value", "#VALUE!"), ("div_zero", "#DIV/0!"),
256
+ ("invalid_reference", "#REF!"), ("no_match", "#N/A"),
257
+ ("unknown_name", "#NAME?")):
258
+ for field in ("error", "refusal"):
259
+ for payload in ({"code": internal, "excelCode": excel}, {"code": excel},
260
+ {"code": internal}, {"code": "probe", "excelCode": excel}):
261
+ address = "A" + str(len(planted) + 1)
262
+ planted[address] = {"kind": "formula", "state": "refused",
263
+ "formula": "=1/0", field: payload}
264
+ expected[address] = excel
265
+ for index, code in enumerate(("circular_reference", "blank", "#NUM!"), 1):
266
+ planted["B" + str(index)] = {"kind": "formula", "state": "refused",
267
+ "formula": "=1/0", "error": {"code": code}}
268
+ probe = {"sheets": [{"name": SUMMARY_TAB, "cells": planted},
269
+ {"name": "Summary (SR)", "cells": {"A1": planted["A1"]}}]}
270
+ found = excel_formula_error_census(probe)
271
+ if len(found) != len(expected) or {row["address"]: row["excelCode"] for row in found} != expected:
272
+ raise RepairRefusal("repair_formula_census_blind",
273
+ "The formula error census failed to find its planted errors.")
274
+ errors = excel_formula_error_census(book)
275
+ return {"sheet": SUMMARY_TAB, "codes": sorted(TICKET_FORMULA_ERROR_CODES),
276
+ "count": len(errors), "errors": errors,
277
+ "negative_control": {"planted": len(expected), "found": len(found)}}
278
+
279
+
280
+ def _formula_error_comparison(before, after, edits, reported_errors=()):
281
+ """Reject new errors anywhere and every residual error in explicitly planned cells.
282
+
283
+ Compare identities, not totals: fixing one cell cannot pay for breaking another. Code
284
+ aliases denote the same failure; unplanned cell changes still have their separate guard.
285
+ Opaque engine diagnostics cannot be matched to a pre-existing error and fail closed.
286
+ """
287
+ def identity(row):
288
+ if not isinstance(row, dict) or not row.get("sheet") or not row.get("address"):
289
+ return None
290
+ code = row.get("excelCode") or row.get("code")
291
+ return (str(row["sheet"]).casefold(), str(row["address"]).upper(),
292
+ _EXCEL_ERROR_ALIASES.get(code, code))
293
+
294
+ baseline = {identity(row) for row in formula_error_census(before)}
295
+ repaired = {(row["sheet"].casefold(), row["address"]) for row in edits}
296
+ introduced, remaining, unchanged, seen = [], [], [], set()
297
+ for row in [*formula_error_census(after), *reported_errors]:
298
+ key = identity(row)
299
+ if key is not None and key in seen:
300
+ continue
301
+ seen.add(key)
302
+ if key is None or key not in baseline:
303
+ introduced.append(copy.deepcopy(row))
304
+ if key is not None and key[:2] in repaired:
305
+ remaining.append(copy.deepcopy(row))
306
+ elif key is not None and key in baseline:
307
+ unchanged.append(copy.deepcopy(row))
308
+ return {"introduced": introduced, "remaining_in_repair": remaining,
309
+ "preexisting_elsewhere": unchanged}
310
+
311
+
312
+ def _project(book, edits):
313
+ projected = copy.deepcopy(book)
314
+ for edit in edits:
315
+ tab = _tab(projected, edit["sheet"])
316
+ cells = tab.setdefault("cells", {})
317
+ if edit["after"] is None:
318
+ cells.pop(edit["address"], None)
319
+ else:
320
+ cells[edit["address"]] = copy.deepcopy(edit["after"])
321
+ return projected
322
+
323
+
324
+ def _cell_map(book):
325
+ return {(str(tab.get("name") or "").casefold(), address): copy.deepcopy(cell)
326
+ for tab in book.get("sheets") or () if isinstance(tab, dict)
327
+ for address, cell in (tab.get("cells") or {}).items()}
328
+
329
+
330
+ def _checked_projection(projected, edits, formula_check):
331
+ """Make formula output part of the plan, never an unpersisted preflight side effect."""
332
+ if not callable(formula_check):
333
+ errors = formula_error_census(projected)
334
+ return projected, errors
335
+ result = formula_check(copy.deepcopy(projected))
336
+ if isinstance(result, dict) and isinstance(result.get("book"), dict):
337
+ checked = result["book"]
338
+ errors = list(result.get("errors") or ())
339
+ else:
340
+ checked = copy.deepcopy(projected)
341
+ errors = list(result or ())
342
+ before, after = _cell_map(projected), _cell_map(checked)
343
+ planned = {(row["sheet"].casefold(), row["address"]): row for row in edits}
344
+ sheet_names = {str(tab.get("name") or "").casefold(): tab.get("name")
345
+ for tab in projected.get("sheets") or () if isinstance(tab, dict)}
346
+ for slot in sorted(before.keys() | after.keys()):
347
+ if before.get(slot) == after.get(slot):
348
+ continue
349
+ row = planned.get(slot)
350
+ if row is None or row["kind"] == "recalculated":
351
+ previous, recalculated = before.get(slot), after.get(slot)
352
+ # Recalculation may refresh a healthy formula's result, but may not repair
353
+ # an unrelated refused formula or change authored content or presentation.
354
+ if not all(isinstance(cell, dict) and cell.get("kind") == "formula"
355
+ and isinstance(cell.get("formula"), str)
356
+ and not cell.get("error") and not cell.get("refusal")
357
+ and cell.get("state") != "refused"
358
+ for cell in (previous, recalculated)) \
359
+ or previous.get("formula") != recalculated.get("formula") \
360
+ or {key: previous[key] for key in _VISUAL_CELL_KEYS if key in previous} \
361
+ != {key: recalculated[key] for key in _VISUAL_CELL_KEYS if key in recalculated}:
362
+ raise RepairRefusal(
363
+ "repair_unplanned_formula_change",
364
+ "Formula validation would change a cell outside the supplied source diff.")
365
+ desired = sheets.clean_cell(_preserve_visual(previous, recalculated), slot[1])
366
+ if row is None:
367
+ sheet = sheet_names[slot[0]]
368
+ row = {"kind": "recalculated", "name": sheet + "!" + slot[1],
369
+ "sheet": sheet, "address": slot[1],
370
+ "before": copy.deepcopy(previous), "after": desired}
371
+ edits.append(row)
372
+ planned[slot] = row
373
+ else:
374
+ row["after"] = desired
375
+ continue
376
+ if row is None or row["kind"] not in {"source", "derived"} \
377
+ or not isinstance(row.get("after"), dict) \
378
+ or row["after"].get("kind") != "formula":
379
+ raise RepairRefusal(
380
+ "repair_unplanned_formula_change",
381
+ "Formula validation would change a cell outside the supplied source diff.")
382
+ recalculated = after.get(slot)
383
+ if not isinstance(recalculated, dict) \
384
+ or (row["kind"] == "source"
385
+ and recalculated.get("value") != row.get("cached_value")):
386
+ raise RepairRefusal(
387
+ "repair_formula_cache_mismatch",
388
+ row["name"] + " does not recalculate to its supplied cached Excel value.")
389
+ row["after"] = sheets.clean_cell(recalculated, row["address"])
390
+ return _project(projected, [
391
+ {**row, "before": before.get((row["sheet"].casefold(), row["address"]))}
392
+ for row in edits
393
+ if before.get((row["sheet"].casefold(), row["address"])) != row["after"]
394
+ ]), errors
395
+
396
+
397
+ def summary_block_snapshot(book):
398
+ """Read-only QI3 pin. The lead saves this as proto/before/QI3-summary-block.json.
399
+
400
+ This is the measured F31:R35 extent, not a search for a plausible summary. Labels and
401
+ units are included in the evidence but are never proposed as edits.
402
+ """
403
+ tab = _tab(book, SUMMARY_TAB)
404
+ cells = tab.get("cells") or {}
405
+ labels = {28: ("C", "Invoice Data Metrics"), 30: ("D", "Total Sales"),
406
+ 31: ("D", "Current YTD"), 32: ("D", "Previous YTD"),
407
+ 33: ("D", "YTD Growth"), 34: ("D", "YTD Growth"),
408
+ 35: ("D", "YTD improvement")}
409
+ if book.get("id") != TARGET_WORKBOOK_ID or tab.get("id") != SUMMARY_TAB_ID or any(
410
+ str((cells.get(col + str(row)) or {}).get("value") or "").strip() != label
411
+ for row, (col, label) in labels.items()):
412
+ raise RepairRefusal("repair_summary_extent_mismatch",
413
+ "The workbook does not match the pinned Invoice Data Metrics block.")
414
+ addresses = {col + str(row) for col in ("D", "E", *SUMMARY_COLUMNS)
415
+ for row in range(31, 36)} | {"C28", "D30", "D10"}
416
+ addresses |= {col + str(row) for col in SUMMARY_COLUMNS for row in (5, 6, 7)}
417
+ addresses |= {addr for addr in cells if re.fullmatch(r"[A-Z]+2", addr)}
418
+ return {"workbook_id": book["id"], "revision": book.get("rev"),
419
+ "workbook_digest": sheets.workbook_digest(book), "sheet": SUMMARY_TAB,
420
+ "sheet_id": tab["id"], "figure_extent": "F31:R35", "figure_count": 65,
421
+ "cells": {addr: copy.deepcopy(cells.get(addr)) for addr in sorted(addresses)}}
422
+
423
+
424
+ def _summary_value(book, address):
425
+ cell = (_tab(book, SUMMARY_TAB).get("cells") or {}).get(address) or {}
426
+ if cell.get("error") or cell.get("state") == "refused":
427
+ raise RepairRefusal("repair_summary_input_unreadable",
428
+ "Summary input " + address + " has a calculation error.")
429
+ return cell.get("value")
430
+
431
+
432
+ def _summary_date(value):
433
+ try:
434
+ return date.fromisoformat(str(value))
435
+ except ValueError as exc:
436
+ raise RepairRefusal("repair_summary_date_invalid",
437
+ "Every summary date must be a real ISO date from the sheet or mirror.") from exc
438
+
439
+
440
+ def _resolved(binding, book, resolve_binding):
441
+ if not callable(resolve_binding):
442
+ raise RepairRefusal("repair_binding_resolver_required", "The summary needs its scoped source resolver.")
443
+ result = resolve_binding(copy.deepcopy(binding), book, _tab(book, SUMMARY_TAB))
444
+ if not isinstance(result, dict) or "value" not in result or result.get("value") is None:
445
+ raise RepairRefusal("repair_binding_unreadable", "A summary source did not return a value.")
446
+ if result.get("limits"):
447
+ # The route wraps this exception, retaining its message but not custom attributes.
448
+ # Include each cause and fix there too; upstream clause separators become sentences.
449
+ details = []
450
+ for limit in result["limits"]:
451
+ if isinstance(limit, dict):
452
+ cause = str(limit.get("cause") or "The source reported an incomplete answer.")
453
+ fix = str(limit.get("recommendation") or "Repair the source scope and retry.")
454
+ details.append("Cause: " + cause + " Recommended fix: " + fix)
455
+ detail = " ".join(details).replace(chr(8212), ". ").replace(chr(8211), " to ")
456
+ exc = RepairRefusal("repair_binding_limited",
457
+ "A summary source could not apply its complete scope. " + detail)
458
+ exc.limits = copy.deepcopy(result["limits"])
459
+ raise exc
460
+ return result["value"]
461
+
462
+
463
+ def _summary_formula_projection(book, edits, formula_check):
464
+ if not callable(formula_check):
465
+ raise RepairRefusal("repair_formula_engine_required",
466
+ "The summary requires the real document formula engine.")
467
+ result = formula_check(copy.deepcopy(book))
468
+ if not isinstance(result, dict) or not isinstance(result.get("book"), dict):
469
+ raise RepairRefusal("repair_formula_engine_required",
470
+ "Formula validation must return the calculated document.")
471
+ # Only adopt planned formulas here. The final whole-document projection records healthy
472
+ # downstream recalculations as exact edits and refuses every other unplanned change.
473
+ for row in edits:
474
+ if row["kind"] == "derived":
475
+ cell = (_tab(result["book"], row["sheet"]).get("cells") or {}).get(row["address"])
476
+ if not isinstance(cell, dict) or cell.get("kind") != "formula" or cell.get("error") \
477
+ or cell.get("state") != "fresh":
478
+ raise RepairRefusal("repair_formula_errors",
479
+ "Summary formula " + row["address"] + " could not calculate.")
480
+ row["after"] = sheets.clean_cell(cell, row["address"])
481
+ return _project(book, edits)
482
+
483
+
484
+ def _append_summary_ratio_plan(current, edits, formula_check):
485
+ """Plan the measured roots and let the real engine decide cascade blank handling."""
486
+ if not callable(formula_check):
487
+ raise RepairRefusal("repair_formula_engine_required",
488
+ "The ratio repair requires the real document formula engine.")
489
+ cells = _tab(current, SUMMARY_TAB).get("cells") or {}
490
+ planned = {(row["sheet"].casefold(), row["address"]) for row in edits}
491
+ ratios = {}
492
+ for address, authored in {**_SUMMARY_RATIO_ROOTS, **_SUMMARY_RATIO_CASCADES}.items():
493
+ before = copy.deepcopy(cells.get(address))
494
+ guarded = '=IFERROR(' + authored[1:] + ',"")'
495
+ if not isinstance(before, dict) or before.get("kind") != "formula" \
496
+ or before.get("formula") not in (authored, guarded):
497
+ raise RepairRefusal("repair_ratio_formula_mismatch",
498
+ "The measured ratio formula has changed at " + address + ".")
499
+ if (SUMMARY_TAB.casefold(), address) in planned:
500
+ raise RepairRefusal("repair_edit_duplicate", "A repair cell may be named only once.")
501
+ desired = copy.deepcopy(before)
502
+ for key in ("value", "display", "error", "refusal"):
503
+ desired.pop(key, None)
504
+ desired.update(state="stale", formula=(guarded if address in _SUMMARY_RATIO_ROOTS
505
+ else before["formula"]))
506
+ row = {"kind": "derived", "name": SUMMARY_TAB + "!" + address,
507
+ "sheet": SUMMARY_TAB, "address": address, "before": before,
508
+ "after": sheets.clean_cell(_preserve_visual(before, desired), address)}
509
+ edits.append(row)
510
+ ratios[address] = row
511
+ result = formula_check(_project(current, edits))
512
+ if not isinstance(result, dict) or not isinstance(result.get("book"), dict):
513
+ raise RepairRefusal("repair_formula_engine_required",
514
+ "Formula validation must return the calculated document.")
515
+ calculated = _tab(result["book"], SUMMARY_TAB).get("cells") or {}
516
+ guarded_cascades = []
517
+ for address, authored in _SUMMARY_RATIO_CASCADES.items():
518
+ cell = calculated.get(address) or {}
519
+ error = cell.get("error") or (cell.get("refusal") if cell.get("state") == "refused" else None)
520
+ # The current engine refuses arithmetic over IFERROR's empty string as `blank`.
521
+ # Only that observed refusal authorizes wrapping a cascade; other failures remain
522
+ # visible to the existing projection refusal. Never replace a formula with a value.
523
+ if isinstance(error, dict) and error.get("code") == "blank":
524
+ ratios[address]["after"]["formula"] = '=IFERROR(' + authored[1:] + ',"")'
525
+ guarded_cascades.append(address)
526
+ return {"roots": list(_SUMMARY_RATIO_ROOTS), "guarded_cascades": guarded_cascades}
527
+
528
+
529
+ def _build_summary_plan(current, request, resolve_binding, formula_check):
530
+ """Build all 65 figures, date/scope helpers and guarded ratios, without tenant writes.
531
+
532
+ summary_block carries before=summary_block_snapshot(current), pricelist, the complete
533
+ partner_ids behind customer_link, and validate_invoice=modules.sheet.validate. The last
534
+ is a callable injected by the trusted operator (this workflow has no HTTP route).
535
+ No core-to-modules/API import or second source-query door is needed.
536
+ """
537
+ summary = request["summary_block"]
538
+ if not isinstance(summary, dict) or summary.get("before") != summary_block_snapshot(current):
539
+ raise RepairRefusal("repair_summary_pin_required",
540
+ "The summary requires the exact before snapshot recorded by the lane lead.")
541
+ if request.get("source_cells") or request.get("binding_cells"):
542
+ raise RepairRefusal("repair_summary_plan_conflict",
543
+ "Run the workbook-derived summary separately from an Excel source-oracle repair.")
544
+ if not str(request.get("refreshed_at") or "").strip():
545
+ raise RepairRefusal("repair_timestamp_required", "The summary needs an explicit refresh timestamp.")
546
+ cells = _tab(current, SUMMARY_TAB).get("cells") or {}
547
+ # Allocate four fully empty rows inside the existing sheet dimensions. Never overwrite
548
+ # authored content, add a tab, resize the sheet, or change hidden/print/visual metadata.
549
+ tab_rows = int(_tab(current, SUMMARY_TAB).get("rows") or 0)
550
+ helper_row = next((row for row in range(tab_rows - 3, 35, -1)
551
+ if all((cells.get(col + str(r)) or {}).get("kind", "empty") == "empty"
552
+ for r in range(row, row + 4) for col in SUMMARY_COLUMNS)), None)
553
+ if helper_row is None:
554
+ raise RepairRefusal("repair_summary_helper_space",
555
+ "The summary needs four empty rows in its existing sheet for date and scope formulas.")
556
+ edits = []
557
+
558
+ def add(address, desired, kind):
559
+ before = copy.deepcopy(cells.get(address))
560
+ edits.append({"kind": kind, "name": SUMMARY_TAB + "!" + address,
561
+ "sheet": SUMMARY_TAB, "address": address, "before": before,
562
+ "after": sheets.clean_cell(_preserve_visual(before, desired), address)})
563
+
564
+ def formula(address, text):
565
+ add(address, {"kind": "formula", "state": "stale", "formula": text}, "derived")
566
+
567
+ scope_ref = "F" + str(helper_row + 3)
568
+ formula(scope_ref, '=IF($D$10="RI","Royal 1 (USD)",IF($D$10="FFS","Fisch (USD)",NA()))')
569
+ windows = {}
570
+ for col in SUMMARY_COLUMNS:
571
+ end = _summary_date(_summary_value(current, col + "7"))
572
+ year_refs = [addr for addr, cell in cells.items() if re.fullmatch(r"[A-Z]+2", addr)
573
+ and str((cell or {}).get("value")) in {str(end.year), str(float(end.year))}]
574
+ # The year can be one merged/shared header. Prefer this column, otherwise require
575
+ # an unambiguous year cell; never guess between two independent calendar headers.
576
+ year_ref = col + "2" if col + "2" in year_refs else year_refs[0] if len(year_refs) == 1 else None
577
+ if not year_ref:
578
+ raise RepairRefusal("repair_summary_year_ambiguous",
579
+ "The summary needs an unambiguous year on row 2 for column " + col + ".")
580
+ first, previous_first, previous_end = (col + str(helper_row + offset) for offset in range(3))
581
+ # DATE/EDATE already return ISO dates. String concatenation preserves that value
582
+ # without introducing an inferred date style on a previously blank helper cell.
583
+ formula(first, '=DATE(' + year_ref + ',1,1)&""')
584
+ formula(previous_first, '=DATE(' + year_ref + '-1,1,1)&""')
585
+ formula(previous_end, '=EDATE(' + col + '7,-12)&""')
586
+ windows[col] = ((first, col + "7"), (previous_first, previous_end))
587
+ projected = _summary_formula_projection(_project(current, edits), edits, formula_check)
588
+ if _summary_value(projected, scope_ref) != summary.get("pricelist"):
589
+ raise RepairRefusal("repair_summary_partner_scope",
590
+ "The supplied partner set must use the pricelist computed from D10.")
591
+ partners = summary.get("partner_ids")
592
+ if not isinstance(partners, list) or any(type(pid) is not int or pid <= 0 for pid in partners) \
593
+ or len(set(partners)) != len(partners):
594
+ raise RepairRefusal("repair_summary_partner_scope",
595
+ "Supply every joined partner ID once, with no source row limit.")
596
+ if not callable(summary.get("validate_invoice")):
597
+ raise RepairRefusal("repair_summary_validator_required",
598
+ "Inject modules.sheet.validate for independent invoice reconciliation.")
599
+ as_of = _summary_date(_resolved({"database": "ut_odoo_invoices",
600
+ "aggregate": {"kind": "column", "fn": "max", "field": "invoice_date"},
601
+ "scope": {"filter": {"conj": "and", "nodes": []}}},
602
+ projected, resolve_binding))
603
+ for col, ranges in windows.items():
604
+ for row_number, (first, last) in zip((31, 32), ranges):
605
+ binding = {"database": "ut_odoo_invoices",
606
+ "aggregate": {"field": "amount_untaxed", "fn": "sum", "kind": "column"},
607
+ "scope": {"filter": {"conj": "and", "nodes": [
608
+ {"colId": "invoice_date", "op": "gte", "valueRef": {"ref": first}},
609
+ {"colId": "invoice_date", "op": "lte", "valueRef": {"ref": last}},
610
+ {"colId": "move_type", "op": "eq", "value": "out_invoice"}]}},
611
+ "related": {"via": "customer_link", "filter": {"conj": "and", "nodes": [
612
+ {"colId": "pricelist", "op": "eq", "valueRef": {"ref": scope_ref}}]}}}
613
+ _validate_binding(col + str(row_number), binding)
614
+ value = _resolved(binding, projected, resolve_binding)
615
+ if type(value) not in (int, float) or not math.isfinite(value):
616
+ raise RepairRefusal("repair_binding_unreadable", "An invoice total must be a finite number.")
617
+ add(col + str(row_number), {"kind": "bound", "state": "fresh", "binding": binding,
618
+ "value": value, "display": str(value),
619
+ "refreshed_at": request["refreshed_at"]}, "binding")
620
+ from openpyxl.formula.translate import Translator
621
+ for row_number in range(33, 36):
622
+ origin = "F" + str(row_number)
623
+ source = cells.get(origin) or {}
624
+ if source.get("kind") != "formula" or not str(source.get("formula") or "").startswith("="):
625
+ raise RepairRefusal("repair_summary_formula_missing",
626
+ "The summary requires its authored formula at " + origin + ".")
627
+ for col in SUMMARY_COLUMNS:
628
+ address = col + str(row_number)
629
+ try:
630
+ text = Translator(source["formula"], origin=origin).translate_formula(address)
631
+ except Exception as exc:
632
+ raise RepairRefusal("repair_summary_formula_translate",
633
+ "The authored summary formula could not be filled across.") from exc
634
+ formula(address, text)
635
+ ratio_evidence = _append_summary_ratio_plan(current, edits, formula_check)
636
+ projected = _summary_formula_projection(_project(current, edits), edits, formula_check)
637
+ checks = validate(projected, windows=windows, mirror_as_of=as_of.isoformat(),
638
+ partner_ids=partners, invoice_validate=summary["validate_invoice"],
639
+ formula_check=formula_check)
640
+ return edits, {"before": summary["before"], "helper_rows": list(range(helper_row, helper_row + 4)),
641
+ "pricelist_cell": scope_ref, "mirror_as_of": as_of.isoformat(),
642
+ "partner_count": len(partners), "checks": checks,
643
+ "ratio_repair": ratio_evidence}
644
+
645
+
646
+ def validate(book, *, windows, mirror_as_of, partner_ids, invoice_validate, formula_check):
647
+ """Reconcile 26 invoice figures and 39 formulas against independent Odoo results.
648
+
649
+ Windows are read from the planned sheet, with each upper bound capped at the mirror's
650
+ max(invoice_date). Re-evaluate the authored formulas over the independent amounts too.
651
+ Invoice-date bounding is not a historical snapshot; preserve validate()'s limitation rows.
652
+ """
653
+ proof, checks = copy.deepcopy(book), []
654
+ as_of = _summary_date(mirror_as_of)
655
+ for col in SUMMARY_COLUMNS:
656
+ for row_number, (first_ref, last_ref) in zip((31, 32), windows[col]):
657
+ first = _summary_date(_summary_value(book, first_ref))
658
+ last = min(_summary_date(_summary_value(book, last_ref)), as_of)
659
+ if first > last:
660
+ raise RepairRefusal("repair_summary_asof_before_window",
661
+ "The invoice mirror ends before a summary window begins. Refresh it first.")
662
+ address = col + str(row_number)
663
+ observed = _summary_value(book, address)
664
+ rows = invoice_validate(first.isoformat(), last.isoformat(), {"invoiced_sales": observed},
665
+ team_id=None, partner_ids=list(partner_ids),
666
+ basis="invoice", include_refunds=False)
667
+ if not isinstance(rows, list) or len(rows) != 1 or rows[0].get("check") != "invoiced_sales" \
668
+ or rows[0].get("ok") is not True or rows[0].get("a") != observed \
669
+ or type(rows[0].get("b")) not in (int, float) or not math.isfinite(rows[0]["b"]) \
670
+ or abs(observed - rows[0]["b"]) > 0.01:
671
+ exc = RepairRefusal("repair_summary_reconciliation",
672
+ "Independent Odoo reconciliation failed for " + address + ".")
673
+ exc.checks = copy.deepcopy(rows)
674
+ raise exc
675
+ checks.append({**copy.deepcopy(rows[0]), "address": address,
676
+ "date_from": first.isoformat(), "date_to": last.isoformat()})
677
+ _tab(proof, SUMMARY_TAB)["cells"][address]["value"] = rows[0]["b"]
678
+ result = formula_check(proof)
679
+ proof = result["book"]
680
+ for col in SUMMARY_COLUMNS:
681
+ for row_number in range(33, 36):
682
+ address = col + str(row_number)
683
+ actual, expected = _summary_value(book, address), _summary_value(proof, address)
684
+ ok = (type(actual) in (int, float) and type(expected) in (int, float)
685
+ and math.isfinite(actual) and math.isfinite(expected)
686
+ and abs(actual - expected) <= (0.01 if row_number == 33 else 1e-9))
687
+ checks.append({"check": "invoice_derived", "address": address, "a": actual,
688
+ "b": expected, "gap": actual - expected if ok else None, "ok": ok})
689
+ if not ok:
690
+ raise RepairRefusal("repair_summary_reconciliation",
691
+ "The summary formula did not reconcile for " + address + ".")
692
+ return checks
693
+
694
+
695
+ def _build_plan(current, request, resolve_binding):
696
+ if not isinstance(request.get("source_cells"), list) or not request["source_cells"]:
697
+ raise RepairRefusal(
698
+ "repair_source_required",
699
+ "The repair requires a one-to-one source cell oracle with cached Excel values.")
700
+ if not isinstance(request.get("binding_cells"), list):
701
+ raise RepairRefusal("repair_bindings_required", "The repair requires both YTD bindings.")
702
+
703
+ edits, names, slots = [], set(), set()
704
+ for row in request["source_cells"]:
705
+ if not isinstance(row, dict):
706
+ raise RepairRefusal("repair_source_invalid", "Every source cell must be an object.")
707
+ name = str(row.get("name") or "").strip()
708
+ tab_name, address = str(row.get("sheet") or "").strip(), _addr(row.get("address"))
709
+ if not name or name in names:
710
+ raise RepairRefusal("repair_source_name_invalid",
711
+ "Every source cell needs a unique report-cell name.")
712
+ names.add(name)
713
+ slot = (tab_name.casefold(), address)
714
+ if slot in slots:
715
+ raise RepairRefusal("repair_edit_duplicate", "A repair cell may be named only once.")
716
+ slots.add(slot)
717
+ tab = _tab(current, tab_name)
718
+ before = copy.deepcopy((tab.get("cells") or {}).get(address))
719
+ supplied = copy.deepcopy(row.get("cell"))
720
+ if isinstance(supplied, dict) and "imported_value" in supplied:
721
+ raise RepairRefusal("repair_cache_invalid",
722
+ name + " carries a temporary imported value instead of its cache.")
723
+ desired = _preserve_visual(before, supplied)
724
+ if desired is not None:
725
+ desired = sheets.clean_cell(desired, addr=address)
726
+ if desired.get("value") != row.get("cached_value"):
727
+ raise RepairRefusal("repair_cache_mismatch",
728
+ name + " does not match its supplied cached Excel value.")
729
+ elif row.get("cached_value") is not None:
730
+ raise RepairRefusal("repair_cache_mismatch", name + " is blank but its cache is not.")
731
+ edits.append({"kind": "source", "name": name, "sheet": tab_name,
732
+ "address": address, "before": before, "after": desired,
733
+ "cached_value": copy.deepcopy(row.get("cached_value"))})
734
+
735
+ e4 = [row for row in edits if row["address"] == "E4"]
736
+ if len(e4) != 1 or (e4[0]["after"] is not None
737
+ and e4[0]["after"].get("kind") != "empty"):
738
+ raise RepairRefusal(
739
+ "repair_e4_not_blank",
740
+ "The source oracle must name E4 exactly once as blank and non-formula.")
741
+ if "Beg Date" not in names:
742
+ raise RepairRefusal("repair_beg_date_missing", "The source oracle must name Beg Date.")
743
+ beg_date = next(row for row in edits if row["name"] == "Beg Date")
744
+ try:
745
+ parsed_beg_date = date.fromisoformat(str((beg_date.get("after") or {}).get("value")))
746
+ except (TypeError, ValueError):
747
+ parsed_beg_date = None
748
+ if parsed_beg_date is None or parsed_beg_date.isoformat() != beg_date["cached_value"]:
749
+ raise RepairRefusal("repair_beg_date_invalid",
750
+ "Beg Date must carry one real ISO date matching its Excel cache.")
751
+ expected_names = request.get("named_report_cells")
752
+ if not isinstance(expected_names, list) or any(
753
+ not isinstance(item, str) or not item.strip() for item in expected_names):
754
+ raise RepairRefusal(
755
+ "repair_source_manifest_required",
756
+ "The repair requires the complete named report cell manifest from the source.")
757
+ normalized_manifest = [item.strip() for item in expected_names]
758
+ if len(set(normalized_manifest)) != len(normalized_manifest) or set(normalized_manifest) != names:
759
+ raise RepairRefusal(
760
+ "repair_source_manifest_mismatch",
761
+ "The source diff must map one-to-one onto the complete named report cell manifest.")
762
+
763
+ binding_names = []
764
+ refreshed_at = str(request.get("refreshed_at") or "").strip()
765
+ if not refreshed_at:
766
+ raise RepairRefusal("repair_timestamp_required",
767
+ "The deterministic repair plan requires an explicit refresh timestamp.")
768
+ for row in request["binding_cells"]:
769
+ if not isinstance(row, dict):
770
+ raise RepairRefusal("repair_binding_invalid", "Every YTD binding must be an object.")
771
+ name = str(row.get("name") or "").strip()
772
+ # S64-T21 widened this loop to ANY named report cell, so a blank or repeated name is now
773
+ # reachable for an Orders row too. The sentence has to cover both, not just the YTD pair.
774
+ if not name or name in binding_names:
775
+ raise RepairRefusal("repair_bindings_required",
776
+ "Every repair binding needs its own report cell name, and the "
777
+ "repair requires one Current YTD and one Previous YTD among them.")
778
+ binding_names.append(name)
779
+ binding = copy.deepcopy(row.get("binding"))
780
+ _validate_binding(name, binding, window_kind=YTD_NAMES.get(name))
781
+ tab_name, address = str(row.get("sheet") or "").strip(), _addr(row.get("address"))
782
+ slot = (tab_name.casefold(), address)
783
+ if slot in slots:
784
+ raise RepairRefusal("repair_edit_duplicate", "A repair cell may be named only once.")
785
+ slots.add(slot)
786
+ tab = _tab(current, tab_name)
787
+ before = copy.deepcopy((tab.get("cells") or {}).get(address))
788
+ if not callable(resolve_binding):
789
+ raise RepairRefusal(
790
+ "repair_binding_resolver_required",
791
+ "Canonical data must resolve both YTD bindings before this repair can run.")
792
+ resolved = resolve_binding(binding, current, tab)
793
+ if not isinstance(resolved, dict) or "value" not in resolved:
794
+ raise RepairRefusal("repair_binding_unreadable", name + " did not resolve canonical data.")
795
+ canonical_binding = copy.deepcopy(resolved.get("binding") or binding)
796
+ _validate_binding(name, canonical_binding, window_kind=YTD_NAMES.get(name))
797
+ desired = _preserve_visual(before, {
798
+ "kind": "bound", "state": "fresh", "binding": canonical_binding,
799
+ "value": copy.deepcopy(resolved["value"]),
800
+ "display": "" if resolved["value"] is None else str(resolved["value"]),
801
+ "refreshed_at": refreshed_at,
802
+ })
803
+ desired = sheets.clean_cell(desired, addr=address)
804
+ edits.append({"kind": "binding", "name": name, "sheet": tab_name,
805
+ "address": address, "before": before, "after": desired})
806
+ if set(YTD_NAMES) - set(binding_names):
807
+ raise RepairRefusal("repair_bindings_required",
808
+ "The repair requires one Current YTD and one Previous YTD binding.")
809
+
810
+ edits.sort(key=lambda row: (row["sheet"].casefold(), row["address"]))
811
+ return edits
812
+
813
+
814
+ def _write_backup(backup_store, current, digest, diff_digest, workbook_id):
815
+ revision = int(current.get("rev") or 0)
816
+ key = "sheet_repair_backup_" + workbook_id + "_r" + str(revision) + "_" + digest[:16]
817
+ record = {"operation": operation_for(workbook_id), "workbook_id": workbook_id,
818
+ "revision": revision, "digest": digest, "diff_digest": diff_digest,
819
+ "workbook": copy.deepcopy(current)}
820
+
821
+ def apply(existing):
822
+ if existing and existing != record:
823
+ raise RepairRefusal("repair_backup_conflict",
824
+ "A different backup already occupies this repair key.")
825
+ return copy.deepcopy(record)
826
+
827
+ backup_store.update(key, apply, flush="sync")
828
+ readback = backup_store.get(key)
829
+ if readback != record or sheets.workbook_digest(readback.get("workbook") or {}) != digest:
830
+ raise RepairRefusal("repair_backup_unverified",
831
+ "The workbook backup could not be read back byte for byte.")
832
+ return key
833
+
834
+
835
+ def execute(st, backup_store, request, *, target_workbook_id=TARGET_WORKBOOK_ID,
836
+ resolve_binding, formula_check=None, actor="", actor_name=""):
837
+ """Validate, back up, repair and read back only the workbook this call is POINTED AT.
838
+
839
+ ⭐ `target_workbook_id` is what makes the repair general, and it is the one argument to read
840
+ twice. It is checked against THREE independent things before a byte moves: the request's own
841
+ `workbook_id`, the marker's (through `operation_for`, so a marker minted for one workbook
842
+ cannot authorise another), and the STORED document's identity. A caller pointed at a fixture
843
+ and handed a request for another workbook is refused, which is the whole point of S64-T21.
844
+
845
+ ⚠ THE DEFAULT IS THE OWNER'S WORKBOOK, DELIBERATELY, AND IT IS NOT A HOLE. It is the one
846
+ workbook this module was written for and the only one an unpointed call could ever have
847
+ reached; every other target must be NAMED. An EXPLICIT empty or non-string target still
848
+ refuses `repair_target_required`, so "pointed at nothing" and "pointed at the default" stay
849
+ different answers. Making the argument mandatory instead would have been a breaking change to
850
+ `verify_sheet.py`'s five S63-T21 call sites, a gate no lane on this board owns, and it would
851
+ have bought no safety the route does not already give: `routes_sheet.py` carries the same
852
+ default for the operator.
853
+ """
854
+ if not isinstance(target_workbook_id, str) or not target_workbook_id.strip():
855
+ raise RepairRefusal("repair_target_required",
856
+ "The repair requires an explicitly named target workbook.")
857
+ target_workbook_id = target_workbook_id.strip()
858
+ if backup_store is None or not callable(getattr(backup_store, "update", None)) \
859
+ or not callable(getattr(backup_store, "get", None)):
860
+ raise RepairRefusal("repair_backup_required",
861
+ "A readable, writable backup store is required before repair.")
862
+ if not isinstance(request, dict) or request.get("workbook_id") != target_workbook_id:
863
+ raise RepairRefusal("repair_workbook_mismatch",
864
+ "This operation may repair only workbook " + target_workbook_id + ".")
865
+ revision = request.get("expected_revision")
866
+ if isinstance(revision, bool) or not isinstance(revision, int) or revision < 0:
867
+ raise RepairRefusal("repair_revision_required",
868
+ "An exact expected workbook revision is required for repair.")
869
+ _validate_marker(request.get("marker"), revision, target_workbook_id)
870
+ try:
871
+ calculation_date = date.fromisoformat(str(request.get("calculation_date") or ""))
872
+ except ValueError as exc:
873
+ raise RepairRefusal("repair_calculation_date_required",
874
+ "The deterministic repair plan requires one ISO calculation date.") from exc
875
+ if calculation_date.isoformat() != request["calculation_date"]:
876
+ raise RepairRefusal("repair_calculation_date_required",
877
+ "The deterministic repair plan requires one ISO calculation date.")
878
+
879
+ current = sheets.workbook_snapshot(st, target_workbook_id)
880
+ if str(current.get("id") or "") != target_workbook_id:
881
+ raise RepairRefusal("repair_workbook_mismatch",
882
+ "The stored workbook identity does not match this repair target.")
883
+ if int(current.get("rev") or 0) != revision:
884
+ raise RepairRefusal("repair_revision_mismatch",
885
+ "The named workbook is not at the expected repair revision.")
886
+ digest = sheets.workbook_digest(current)
887
+ summary_evidence = None
888
+ summary_errors_before = None
889
+ if "summary_block" in request:
890
+ summary_errors_before = check_summary_formula_errors(current)
891
+ edits, summary_evidence = _build_summary_plan(current, request, resolve_binding, formula_check)
892
+ else:
893
+ edits = _build_plan(current, request, resolve_binding)
894
+ projected = _project(current, edits)
895
+ projected, projected_errors = _checked_projection(projected, edits, formula_check)
896
+ comparison = _formula_error_comparison(current, projected, edits, projected_errors)
897
+ summary_errors_after = (check_summary_formula_errors(projected)
898
+ if summary_evidence is not None else None)
899
+ if comparison["introduced"] or comparison["remaining_in_repair"] \
900
+ or (summary_errors_after is not None and summary_errors_after["count"]):
901
+ raise RepairRefusal(
902
+ "repair_formula_errors",
903
+ "The projected repair introduces a formula error or leaves one in its repair scope, "
904
+ "so no repair was applied.")
905
+ visual_before = visual_metadata(current)
906
+ if visual_metadata(projected) != visual_before:
907
+ raise RepairRefusal("repair_visual_metadata_changed",
908
+ "The projected repair changes Daily Sales visual metadata.")
909
+
910
+ exact_edits = [
911
+ {key: copy.deepcopy(row[key]) for key in ("sheet", "address", "before", "after")}
912
+ for row in edits]
913
+ diff_digest = sheets.workbook_digest({"edits": exact_edits})
914
+ backup_key = _write_backup(backup_store, current, digest, diff_digest, target_workbook_id)
915
+ write = sheets.apply_exact_cell_edits(
916
+ st, target_workbook_id, exact_edits,
917
+ revision, digest, actor=actor, actor_name=actor_name)
918
+ readback = sheets.workbook_snapshot(st, target_workbook_id)
919
+
920
+ if int(readback.get("rev") or 0) != revision + 1:
921
+ raise RepairRefusal("repair_readback_revision",
922
+ "The repaired workbook revision did not read back as expected.")
923
+ for row in edits:
924
+ actual = copy.deepcopy((_tab(readback, row["sheet"]).get("cells") or {}).get(row["address"]))
925
+ expected = None if row["after"] is None else sheets.clean_cell(row["after"], row["address"])
926
+ if actual != expected:
927
+ raise RepairRefusal("repair_readback_cell",
928
+ row["sheet"] + "!" + row["address"] + " did not read back exactly.")
929
+ checked_readback, readback_errors = _checked_projection(readback, copy.deepcopy(edits), formula_check)
930
+ readback_comparison = _formula_error_comparison(current, checked_readback, edits, readback_errors)
931
+ summary_errors_readback = (check_summary_formula_errors(checked_readback)
932
+ if summary_evidence is not None else None)
933
+ if readback_comparison["introduced"] or readback_comparison["remaining_in_repair"] \
934
+ or (summary_errors_readback is not None and summary_errors_readback["count"]) \
935
+ or _cell_map(checked_readback) != _cell_map(readback):
936
+ raise RepairRefusal("repair_readback_formula_state",
937
+ "The repaired workbook did not read back calculation-current.")
938
+ if visual_metadata(readback) != visual_before:
939
+ raise RepairRefusal("repair_readback_visual_metadata",
940
+ "Daily Sales visual metadata changed during repair.")
941
+ if summary_evidence is not None:
942
+ tab = _tab(readback, SUMMARY_TAB)
943
+ if any((tab.get("cells", {}).get(col + str(row)) or {}).get("kind")
944
+ != ("bound" if row in (31, 32) else "formula")
945
+ for col in SUMMARY_COLUMNS for row in range(31, 36)):
946
+ raise RepairRefusal("repair_summary_readback_kind",
947
+ "Every summary figure must read back as its planned binding or formula.")
948
+ summary_evidence["readback"] = summary_block_snapshot(readback)
949
+ summary_evidence["formula_error_check"] = {
950
+ "before": summary_errors_before, "projected": summary_errors_after,
951
+ "readback": summary_errors_readback,
952
+ }
953
+
954
+ source_readback = {row["name"]: copy.deepcopy(
955
+ (_tab(readback, row["sheet"]).get("cells") or {}).get(row["address"]))
956
+ for row in edits if row["kind"] == "source"}
957
+ bindings = {row["name"]: copy.deepcopy(
958
+ ((_tab(readback, row["sheet"]).get("cells") or {}).get(row["address"]) or {}).get("binding"))
959
+ for row in edits if row["kind"] == "binding"}
960
+ result = {
961
+ "workbook_id": target_workbook_id,
962
+ "revision_before": revision,
963
+ "revision_after": readback["rev"],
964
+ "backup_key": backup_key,
965
+ "backup_digest": digest,
966
+ "diff_digest": diff_digest,
967
+ "applied": write["edits"],
968
+ "paths": write["paths"],
969
+ "source_cells": source_readback,
970
+ "bindings": bindings,
971
+ "formula_errors": formula_error_census(readback),
972
+ "formula_error_comparison": readback_comparison,
973
+ "visual_metadata_preserved": True,
974
+ }
975
+ if summary_evidence is not None:
976
+ result["summary_block"] = summary_evidence
977
+ return result
platform/core/external_access.py CHANGED
@@ -8,6 +8,7 @@ credential, user, bucket, or tenant runtime.
8
  """
9
  import calendar
10
  from dataclasses import dataclass
 
11
  import hashlib
12
  import hmac
13
  import secrets
@@ -25,6 +26,7 @@ ALL_DATABASES = "all"
25
  MAX_SCOPE_DATABASES = 200
26
  ALGORITHM = "sha256-salt-v1"
27
  AUTH_ERROR = "invalid_external_credential"
 
28
  _DUMMY_SALT = bytes(32)
29
  _DUMMY_HASH = hashlib.sha256(_DUMMY_SALT + b"invalid").hexdigest()
30
  #: Seconds between two durable `last_used` stamps for ONE credential.
@@ -114,6 +116,28 @@ def _now() -> str:
114
  return time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
115
 
116
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
117
  def _stamp_is_due(last_used, now: float) -> bool:
118
  """Is this credential's `last_used` stale enough to be worth a durable write?
119
 
@@ -164,7 +188,7 @@ def _assert_bound_runtime(st, tenant: str) -> None:
164
  raise ExternalAccessError("external_identity_invalid")
165
 
166
 
167
- def _active_public_user(username: str, tenant: str) -> dict:
168
  try:
169
  registry = users.registry() or {}
170
  except Exception as exc:
@@ -180,6 +204,20 @@ def _active_public_user(username: str, tenant: str) -> dict:
180
  return public
181
 
182
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
183
  def _metadata(record: dict) -> dict:
184
  """Return list-safe metadata. Salt, digest, algorithm and owner never cross this seam."""
185
  access, databases = scope_of(record)
@@ -189,6 +227,8 @@ def _metadata(record: dict) -> dict:
189
  "created": record.get("created"),
190
  "last_used": record.get("last_used"),
191
  "revoked_at": record.get("revoked_at"),
 
 
192
  "access": access,
193
  "databases": ALL_DATABASES if databases == ALL_DATABASES else list(databases),
194
  }
@@ -205,16 +245,16 @@ def label_for(st, key_id: str) -> str:
205
  return label
206
 
207
 
208
- def mint(st, username: str, tenant: str, label: str, access=None, databases=None) -> tuple[dict, str]:
 
209
  """Create one credential and return ``(metadata, token)``; the token is never persisted."""
210
  access = normalise_access(access)
211
  databases = normalise_databases(databases)
212
  owner, tenant_key = _normalise_owner(username, tenant)
213
  _assert_bound_runtime(st, tenant_key)
214
  _active_public_user(owner, tenant_key)
215
- clean_label = str(label or "").strip()
216
- if not clean_label or len(clean_label) > 80:
217
- raise ExternalAccessError("external_label_invalid")
218
 
219
  secret = secrets.token_urlsafe(32)
220
  salt = secrets.token_bytes(32)
@@ -244,6 +284,8 @@ def mint(st, username: str, tenant: str, label: str, access=None, databases=None
244
  "created": created,
245
  "last_used": None,
246
  "revoked_at": None,
 
 
247
  }
248
  book[key_id] = record
249
  minted["record"] = record
@@ -262,6 +304,94 @@ def mint(st, username: str, tenant: str, label: str, access=None, databases=None
262
  return _metadata(record), token
263
 
264
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
265
  def list_for_user(st, username: str, tenant: str) -> list[dict]:
266
  """List only this user's active credential metadata, newest first."""
267
  owner, tenant_key = _normalise_owner(username, tenant)
@@ -355,6 +485,8 @@ def verify(token: str) -> VerifiedCredential:
355
  and record.get("tenant") == tenant_key
356
  and bool(record.get("username"))
357
  and not record.get("revoked_at")
 
 
358
  )
359
  if not record_ok or not matched:
360
  raise ExternalAuthError(AUTH_ERROR)
 
8
  """
9
  import calendar
10
  from dataclasses import dataclass
11
+ from datetime import datetime, timezone
12
  import hashlib
13
  import hmac
14
  import secrets
 
26
  MAX_SCOPE_DATABASES = 200
27
  ALGORITHM = "sha256-salt-v1"
28
  AUTH_ERROR = "invalid_external_credential"
29
+ _UNSET = object()
30
  _DUMMY_SALT = bytes(32)
31
  _DUMMY_HASH = hashlib.sha256(_DUMMY_SALT + b"invalid").hexdigest()
32
  #: Seconds between two durable `last_used` stamps for ONE credential.
 
116
  return time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
117
 
118
 
119
+ def _epoch(value: str | None) -> float | None:
120
+ if value is None:
121
+ return None
122
+ try:
123
+ return datetime.strptime(value, "%Y-%m-%dT%H:%M:%SZ").replace(
124
+ tzinfo=timezone.utc).timestamp()
125
+ except (TypeError, ValueError, OverflowError):
126
+ return None
127
+
128
+
129
+ def normalise_expiry(value, *, now: float | None = None) -> str | None:
130
+ """Return one canonical future UTC expiry, or ``None`` for no expiry."""
131
+ if value is None or value == "":
132
+ return None
133
+ if not isinstance(value, str):
134
+ raise ExternalAccessError("external_expiry_invalid")
135
+ parsed = _epoch(value.strip())
136
+ if parsed is None or parsed <= (time.time() if now is None else float(now)):
137
+ raise ExternalAccessError("external_expiry_invalid")
138
+ return time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime(parsed))
139
+
140
+
141
  def _stamp_is_due(last_used, now: float) -> bool:
142
  """Is this credential's `last_used` stale enough to be worth a durable write?
143
 
 
188
  raise ExternalAccessError("external_identity_invalid")
189
 
190
 
191
+ def resolve_active_user(username: str, tenant: str) -> dict:
192
  try:
193
  registry = users.registry() or {}
194
  except Exception as exc:
 
204
  return public
205
 
206
 
207
+ def _active_public_user(username: str, tenant: str) -> dict:
208
+ """Backward-compatible private spelling retained for the SDLC 62 seam."""
209
+ return resolve_active_user(username, tenant)
210
+
211
+
212
+ def _normalise_label(label) -> str:
213
+ clean = str(label or "").strip()
214
+ if (not clean or len(clean) > 80 or any(ord(char) < 32 for char in clean)
215
+ or "lpb1." in clean or "lpo1." in clean or "lpr1." in clean
216
+ or "lpc1." in clean):
217
+ raise ExternalAccessError("external_label_invalid")
218
+ return clean
219
+
220
+
221
  def _metadata(record: dict) -> dict:
222
  """Return list-safe metadata. Salt, digest, algorithm and owner never cross this seam."""
223
  access, databases = scope_of(record)
 
227
  "created": record.get("created"),
228
  "last_used": record.get("last_used"),
229
  "revoked_at": record.get("revoked_at"),
230
+ "rotated_at": record.get("rotated_at"),
231
+ "expires_at": record.get("expires_at"),
232
  "access": access,
233
  "databases": ALL_DATABASES if databases == ALL_DATABASES else list(databases),
234
  }
 
245
  return label
246
 
247
 
248
+ def mint(st, username: str, tenant: str, label: str, access=None, databases=None,
249
+ expires_at=None) -> tuple[dict, str]:
250
  """Create one credential and return ``(metadata, token)``; the token is never persisted."""
251
  access = normalise_access(access)
252
  databases = normalise_databases(databases)
253
  owner, tenant_key = _normalise_owner(username, tenant)
254
  _assert_bound_runtime(st, tenant_key)
255
  _active_public_user(owner, tenant_key)
256
+ clean_label = _normalise_label(label)
257
+ expiry = normalise_expiry(expires_at)
 
258
 
259
  secret = secrets.token_urlsafe(32)
260
  salt = secrets.token_bytes(32)
 
284
  "created": created,
285
  "last_used": None,
286
  "revoked_at": None,
287
+ "rotated_at": None,
288
+ "expires_at": expiry,
289
  }
290
  book[key_id] = record
291
  minted["record"] = record
 
304
  return _metadata(record), token
305
 
306
 
307
+ def edit(st, username: str, tenant: str, key_id: str, *, label=_UNSET, access=_UNSET,
308
+ databases=_UNSET, expires_at=_UNSET) -> dict:
309
+ """Edit metadata/scope in one store update; credential material is left untouched."""
310
+ owner, tenant_key = _normalise_owner(username, tenant)
311
+ _assert_bound_runtime(st, tenant_key)
312
+ resolve_active_user(owner, tenant_key)
313
+ target = str(key_id or "").strip()
314
+ if label is access is databases is expires_at is _UNSET:
315
+ raise ExternalAccessError("external_edit_empty")
316
+ clean_label = _normalise_label(label) if label is not _UNSET else _UNSET
317
+ clean_access = normalise_access(access) if access is not _UNSET else _UNSET
318
+ clean_databases = normalise_databases(databases) if databases is not _UNSET else _UNSET
319
+ clean_expiry = normalise_expiry(expires_at) if expires_at is not _UNSET else _UNSET
320
+ edited: dict = {}
321
+
322
+ def _set(value):
323
+ if not isinstance(value, dict):
324
+ raise ExternalAccessError("external_key_not_found")
325
+ book = dict(value)
326
+ current = book.get(target)
327
+ if (not isinstance(current, dict) or current.get("username") != owner
328
+ or current.get("tenant") != tenant_key or current.get("revoked_at")):
329
+ raise ExternalAccessError("external_key_not_found")
330
+ record = dict(current)
331
+ if clean_label is not _UNSET:
332
+ record["label"] = clean_label
333
+ if clean_access is not _UNSET:
334
+ record["access"] = clean_access
335
+ if clean_databases is not _UNSET:
336
+ record["databases"] = (ALL_DATABASES if clean_databases == ALL_DATABASES
337
+ else list(clean_databases))
338
+ if clean_expiry is not _UNSET:
339
+ record["expires_at"] = clean_expiry
340
+ book[target] = record
341
+ edited["record"] = record
342
+ return book
343
+
344
+ try:
345
+ st.update(BUCKET, _set, flush="sync")
346
+ except ExternalAccessError:
347
+ raise
348
+ except Exception as exc:
349
+ raise ExternalAccessError("external_access_unavailable") from exc
350
+ record = edited.get("record")
351
+ if not isinstance(record, dict):
352
+ raise ExternalAccessError("external_access_unavailable")
353
+ return _metadata(record)
354
+
355
+
356
+ def rotate(st, username: str, tenant: str, key_id: str) -> tuple[dict, str]:
357
+ """Replace one owned key's secret atomically and reveal the replacement exactly once."""
358
+ owner, tenant_key = _normalise_owner(username, tenant)
359
+ _assert_bound_runtime(st, tenant_key)
360
+ resolve_active_user(owner, tenant_key)
361
+ target = str(key_id or "").strip()
362
+ secret = secrets.token_urlsafe(32)
363
+ salt = secrets.token_bytes(32)
364
+ rotated: dict = {}
365
+
366
+ def _set(value):
367
+ if not isinstance(value, dict):
368
+ raise ExternalAccessError("external_key_not_found")
369
+ book = dict(value)
370
+ current = book.get(target)
371
+ if (not isinstance(current, dict) or current.get("username") != owner
372
+ or current.get("tenant") != tenant_key or current.get("revoked_at")):
373
+ raise ExternalAccessError("external_key_not_found")
374
+ record = dict(current)
375
+ record.update({
376
+ "salt": salt.hex(), "hash": _digest(salt, secret), "algo": ALGORITHM,
377
+ "rotated_at": _now(), "last_used": None,
378
+ })
379
+ book[target] = record
380
+ rotated["record"] = record
381
+ return book
382
+
383
+ try:
384
+ st.update(BUCKET, _set, flush="sync")
385
+ except ExternalAccessError:
386
+ raise
387
+ except Exception as exc:
388
+ raise ExternalAccessError("external_access_unavailable") from exc
389
+ record = rotated.get("record")
390
+ if not isinstance(record, dict):
391
+ raise ExternalAccessError("external_access_unavailable")
392
+ return _metadata(record), f"lpb1.{tenant_key}.{target}.{secret}"
393
+
394
+
395
  def list_for_user(st, username: str, tenant: str) -> list[dict]:
396
  """List only this user's active credential metadata, newest first."""
397
  owner, tenant_key = _normalise_owner(username, tenant)
 
485
  and record.get("tenant") == tenant_key
486
  and bool(record.get("username"))
487
  and not record.get("revoked_at")
488
+ and (_epoch(record.get("expires_at")) is None
489
+ if not record.get("expires_at") else _epoch(record.get("expires_at")) > time.time())
490
  )
491
  if not record_ok or not matched:
492
  raise ExternalAuthError(AUTH_ERROR)
platform/core/external_oauth.py ADDED
@@ -0,0 +1,723 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Hash-only OAuth 2.1-style authorization for the existing remote MCP resource.
2
+
3
+ The authorization endpoint is the only browser-session step. Code exchange, refresh, bearer
4
+ verification, and disconnect resolve from opaque tokens and fresh tenant state without a cookie.
5
+ Public clients use PKCE S256 and have no shared client secret to leak or impersonate a user.
6
+ """
7
+ from __future__ import annotations
8
+
9
+ import base64
10
+ from dataclasses import dataclass
11
+ import hashlib
12
+ import hmac
13
+ import secrets
14
+ import time
15
+ from urllib.parse import urlsplit, urlunsplit
16
+
17
+ from core import external_access
18
+
19
+
20
+ BUCKET = "external_oauth"
21
+ ALGORITHM = "sha256-salt-v1"
22
+ CODE_TTL_SECONDS = 5 * 60
23
+ ACCESS_TTL_SECONDS = 60 * 60
24
+ REFRESH_TTL_SECONDS = 30 * 24 * 60 * 60
25
+ MAX_REFRESH_ROTATIONS = 256
26
+ MAX_REDIRECT_URIS = 10
27
+ SCOPES = ("mcp:read", "mcp:write", "offline_access")
28
+ AUTH_ERROR = external_access.AUTH_ERROR
29
+ _STAMP_EVERY = 900.0
30
+ _DUMMY_SALT = bytes(32)
31
+ _DUMMY_HASH = hashlib.sha256(_DUMMY_SALT + b"invalid").hexdigest()
32
+ _UNSET = object()
33
+
34
+
35
+ class ExternalOAuthError(external_access.ExternalAccessError):
36
+ """Stable OAuth refusal; HTTP vocabulary belongs to the API route."""
37
+
38
+
39
+ class ExternalOAuthAuthError(ExternalOAuthError):
40
+ """Uniform bearer refusal with no tenant, client, or grant oracle."""
41
+
42
+
43
+ @dataclass(frozen=True)
44
+ class VerifiedOAuth:
45
+ tenant: str
46
+ grant_id: str
47
+ client_id: str
48
+ user: dict
49
+ runtime: object
50
+ access: str
51
+ databases: object
52
+ scopes: tuple[str, ...]
53
+ resource: str
54
+
55
+
56
+ @dataclass(frozen=True)
57
+ class DisconnectedOAuth:
58
+ """Safe attribution plus public metadata for one token-authenticated disconnect."""
59
+
60
+ tenant: str
61
+ username: str
62
+ connection: dict
63
+
64
+
65
+ def _now(now=None) -> float:
66
+ return time.time() if now is None else float(now)
67
+
68
+
69
+ def _iso(epoch: float | None) -> str | None:
70
+ if epoch is None:
71
+ return None
72
+ return time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime(epoch))
73
+
74
+
75
+ def _digest(salt: bytes, secret: str) -> str:
76
+ return hashlib.sha256(salt + secret.encode("utf-8")).hexdigest()
77
+
78
+
79
+ def _state_digest(state: str) -> str:
80
+ return hashlib.sha256(state.encode("utf-8")).hexdigest()
81
+
82
+
83
+ def _pkce_challenge(verifier: str) -> str:
84
+ return base64.urlsafe_b64encode(hashlib.sha256(verifier.encode("ascii")).digest()).rstrip(b"=").decode("ascii")
85
+
86
+
87
+ def _book(st, *, fresh=True) -> dict:
88
+ try:
89
+ value = st.get(BUCKET, fresh=fresh)
90
+ except Exception as exc:
91
+ raise ExternalOAuthError("oauth_unavailable") from exc
92
+ if value is None:
93
+ return {"clients": {}, "codes": {}, "states": {}, "grants": {}}
94
+ if not isinstance(value, dict):
95
+ raise ExternalOAuthError("oauth_unavailable")
96
+ out = {}
97
+ for key in ("clients", "codes", "states", "grants"):
98
+ child = value.get(key, {})
99
+ if not isinstance(child, dict):
100
+ raise ExternalOAuthError("oauth_unavailable")
101
+ out[key] = child
102
+ return out
103
+
104
+
105
+ def _normalise_redirect_uris(value) -> tuple[str, ...]:
106
+ if not isinstance(value, (list, tuple)) or not 1 <= len(value) <= MAX_REDIRECT_URIS:
107
+ raise ExternalOAuthError("oauth_redirect_uri_invalid")
108
+ clean = []
109
+ for raw in value:
110
+ if not isinstance(raw, str) or not raw.strip() or len(raw.strip()) > 1000:
111
+ raise ExternalOAuthError("oauth_redirect_uri_invalid")
112
+ uri = raw.strip()
113
+ parsed = urlsplit(uri)
114
+ loopback = parsed.scheme == "http" and (parsed.hostname or "").lower() in {
115
+ "localhost", "127.0.0.1", "::1",
116
+ }
117
+ if (parsed.fragment or parsed.username or parsed.password or not parsed.netloc
118
+ or (parsed.scheme != "https" and not loopback)):
119
+ raise ExternalOAuthError("oauth_redirect_uri_invalid")
120
+ clean.append(uri)
121
+ return tuple(dict.fromkeys(clean))
122
+
123
+
124
+ def _normalise_state(value) -> str:
125
+ if not isinstance(value, str) or not 16 <= len(value) <= 512 or any(ord(c) < 32 for c in value):
126
+ raise ExternalOAuthError("oauth_state_invalid")
127
+ return value
128
+
129
+
130
+ def _normalise_challenge(value, method) -> str:
131
+ if method != "S256" or not isinstance(value, str) or not 43 <= len(value) <= 128:
132
+ raise ExternalOAuthError("oauth_pkce_required")
133
+ allowed = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_"
134
+ if any(char not in allowed for char in value):
135
+ raise ExternalOAuthError("oauth_pkce_required")
136
+ return value
137
+
138
+
139
+ def _normalise_verifier(value) -> str:
140
+ if not isinstance(value, str) or not 43 <= len(value) <= 128:
141
+ raise ExternalOAuthError("oauth_invalid_grant")
142
+ allowed = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-._~"
143
+ if any(char not in allowed for char in value):
144
+ raise ExternalOAuthError("oauth_invalid_grant")
145
+ return value
146
+
147
+
148
+ def _normalise_scopes(value, access: str) -> tuple[str, ...]:
149
+ requested = str(value or "mcp:read offline_access").split()
150
+ requested = list(dict.fromkeys(requested))
151
+ if (not requested or any(scope not in SCOPES for scope in requested)
152
+ or "mcp:read" not in requested
153
+ or ("mcp:write" in requested and access != external_access.ACCESS_READWRITE)):
154
+ raise ExternalOAuthError("oauth_scope_invalid")
155
+ return tuple(requested)
156
+
157
+
158
+ def normalise_resource(value) -> str:
159
+ """Canonicalise one RFC 8707 audience URI while rejecting ambiguous targets."""
160
+ if not isinstance(value, str) or not value.strip() or len(value.strip()) > 1000:
161
+ raise ExternalOAuthError("oauth_resource_invalid")
162
+ parsed = urlsplit(value.strip())
163
+ if (parsed.scheme.lower() not in {"http", "https"} or not parsed.netloc
164
+ or parsed.fragment or parsed.username or parsed.password):
165
+ raise ExternalOAuthError("oauth_resource_invalid")
166
+ try:
167
+ port = parsed.port
168
+ except ValueError as exc:
169
+ raise ExternalOAuthError("oauth_resource_invalid") from exc
170
+ hostname = (parsed.hostname or "").lower()
171
+ if not hostname:
172
+ raise ExternalOAuthError("oauth_resource_invalid")
173
+ host = f"[{hostname}]" if ":" in hostname else hostname
174
+ netloc = host + (f":{port}" if port is not None else "")
175
+ return urlunsplit((parsed.scheme.lower(), netloc, parsed.path, parsed.query, ""))
176
+
177
+
178
+ def _comparison_material(record, prefix: str) -> tuple[bool, bytes, str]:
179
+ usable = isinstance(record, dict) and record.get("algo") == ALGORITHM
180
+ if usable:
181
+ try:
182
+ salt = bytes.fromhex(str(record.get(prefix + "_salt") or ""))
183
+ expected = str(record.get(prefix + "_hash") or "")
184
+ if len(salt) != 32 or len(expected) != 64:
185
+ raise ValueError("bad digest")
186
+ int(expected, 16)
187
+ return True, salt, expected
188
+ except (TypeError, ValueError):
189
+ pass
190
+ return False, _DUMMY_SALT, _DUMMY_HASH
191
+
192
+
193
+ def _used_refresh_history(record, secret: str) -> tuple[bool, list[dict], bool]:
194
+ """Validate and compare every consumed token retained for this live family."""
195
+ rows = record.get("used_refresh", ()) if isinstance(record, dict) else ()
196
+ if not isinstance(rows, list) or len(rows) > MAX_REFRESH_ROTATIONS:
197
+ return False, [], False
198
+ valid = True
199
+ matched = False
200
+ for item in rows:
201
+ usable, salt, expected = _comparison_material({
202
+ "algo": item.get("algo") if isinstance(item, dict) else None,
203
+ "used_salt": item.get("salt") if isinstance(item, dict) else None,
204
+ "used_hash": item.get("hash") if isinstance(item, dict) else None,
205
+ }, "used")
206
+ valid = valid and usable
207
+ matched = hmac.compare_digest(_digest(salt, secret), expected) and usable or matched
208
+ return valid, list(rows), matched
209
+
210
+
211
+ def _token(prefix: str, tenant: str, identifier: str, secret: str) -> str:
212
+ return f"{prefix}.{tenant}.{identifier}.{secret}"
213
+
214
+
215
+ def _parse_token(token, prefixes) -> tuple[str, str, str, str]:
216
+ parts = str(token or "").strip().split(".")
217
+ if (len(parts) != 4 or parts[0] not in prefixes or not all(parts[1:])
218
+ or len(parts[1]) > 128 or len(parts[2]) > 128 or len(parts[3]) > 256):
219
+ raise ExternalOAuthAuthError(AUTH_ERROR)
220
+ return parts[0], parts[1].lower(), parts[2], parts[3]
221
+
222
+
223
+ def _metadata(record: dict, client: dict | None = None) -> dict:
224
+ return {
225
+ "id": record.get("id"),
226
+ "client_id": record.get("client_id"),
227
+ "label": (client or {}).get("label"),
228
+ "created": _iso(record.get("created_at")),
229
+ "last_used": _iso(record.get("last_used_at")),
230
+ "expires_at": _iso(record.get("refresh_expires_at")),
231
+ "disconnected_at": _iso(record.get("disconnected_at")),
232
+ "access": record.get("access", external_access.ACCESS_READ),
233
+ "databases": (external_access.ALL_DATABASES if record.get("databases") == external_access.ALL_DATABASES
234
+ else list(record.get("databases") or [])),
235
+ "scopes": list(record.get("scopes") or ()),
236
+ }
237
+
238
+
239
+ def _client_metadata(record: dict) -> dict:
240
+ access, databases = external_access.scope_of(record)
241
+ return {
242
+ "id": record.get("id"), "label": record.get("label"),
243
+ "redirect_uris": list(record.get("redirect_uris") or []),
244
+ "created": _iso(record.get("created_at")), "revoked_at": _iso(record.get("revoked_at")),
245
+ "access": access,
246
+ "databases": (external_access.ALL_DATABASES if databases == external_access.ALL_DATABASES
247
+ else list(databases)),
248
+ "token_endpoint_auth_method": "none", "pkce": "S256",
249
+ }
250
+
251
+
252
+ def register_client(st, username: str, tenant: str, label, redirect_uris, *, access=None,
253
+ databases=None) -> dict:
254
+ owner, tenant_key = external_access._normalise_owner(username, tenant)
255
+ external_access._assert_bound_runtime(st, tenant_key)
256
+ external_access.resolve_active_user(owner, tenant_key)
257
+ clean_label = external_access._normalise_label(label)
258
+ redirects = _normalise_redirect_uris(redirect_uris)
259
+ clean_access = external_access.normalise_access(access)
260
+ clean_databases = external_access.normalise_databases(databases)
261
+ created = _now()
262
+ made = {}
263
+
264
+ def _set(value):
265
+ book = _book_value(value)
266
+ clients = dict(book["clients"])
267
+ client_id = "lpc_" + secrets.token_urlsafe(18)
268
+ while client_id in clients:
269
+ client_id = "lpc_" + secrets.token_urlsafe(18)
270
+ record = {
271
+ "id": client_id, "username": owner, "tenant": tenant_key, "label": clean_label,
272
+ "redirect_uris": list(redirects), "access": clean_access,
273
+ "databases": (external_access.ALL_DATABASES if clean_databases == external_access.ALL_DATABASES
274
+ else list(clean_databases)),
275
+ "created_at": created, "revoked_at": None,
276
+ }
277
+ clients[client_id] = record
278
+ book["clients"] = clients
279
+ made["record"] = record
280
+ return book
281
+
282
+ _update(st, _set)
283
+ return _client_metadata(made["record"])
284
+
285
+
286
+ def list_clients(st, username: str, tenant: str) -> list[dict]:
287
+ owner, tenant_key = external_access._normalise_owner(username, tenant)
288
+ external_access._assert_bound_runtime(st, tenant_key)
289
+ return sorted([
290
+ _client_metadata(row) for row in _book(st)["clients"].values()
291
+ if isinstance(row, dict) and row.get("username") == owner
292
+ and row.get("tenant") == tenant_key and not row.get("revoked_at")
293
+ ], key=lambda row: str(row.get("created") or ""), reverse=True)
294
+
295
+
296
+ def edit_client(st, username: str, tenant: str, client_id: str, *, label=_UNSET,
297
+ redirect_uris=_UNSET, access=_UNSET, databases=_UNSET) -> dict:
298
+ owner, tenant_key = external_access._normalise_owner(username, tenant)
299
+ external_access._assert_bound_runtime(st, tenant_key)
300
+ external_access.resolve_active_user(owner, tenant_key)
301
+ if label is redirect_uris is access is databases is _UNSET:
302
+ raise ExternalOAuthError("oauth_edit_empty")
303
+ clean_label = external_access._normalise_label(label) if label is not _UNSET else _UNSET
304
+ clean_redirects = _normalise_redirect_uris(redirect_uris) if redirect_uris is not _UNSET else _UNSET
305
+ clean_access = external_access.normalise_access(access) if access is not _UNSET else _UNSET
306
+ clean_databases = external_access.normalise_databases(databases) if databases is not _UNSET else _UNSET
307
+ edited = {}
308
+
309
+ def _set(value):
310
+ book = _book_value(value)
311
+ clients = dict(book["clients"])
312
+ current = clients.get(client_id)
313
+ if (not isinstance(current, dict) or current.get("username") != owner
314
+ or current.get("tenant") != tenant_key or current.get("revoked_at")):
315
+ raise ExternalOAuthError("oauth_client_not_found")
316
+ row = dict(current)
317
+ if clean_label is not _UNSET:
318
+ row["label"] = clean_label
319
+ if clean_redirects is not _UNSET:
320
+ row["redirect_uris"] = list(clean_redirects)
321
+ if clean_access is not _UNSET:
322
+ row["access"] = clean_access
323
+ if clean_databases is not _UNSET:
324
+ row["databases"] = (external_access.ALL_DATABASES
325
+ if clean_databases == external_access.ALL_DATABASES
326
+ else list(clean_databases))
327
+ clients[client_id] = row
328
+ book["clients"] = clients
329
+ edited["record"] = row
330
+ return book
331
+
332
+ _update(st, _set)
333
+ return _client_metadata(edited["record"])
334
+
335
+
336
+ def authorize(st, username: str, tenant: str, *, client_id, redirect_uri, state,
337
+ code_challenge, code_challenge_method, resource, scope=None, response_type="code",
338
+ now=None) -> dict:
339
+ owner, tenant_key = external_access._normalise_owner(username, tenant)
340
+ external_access._assert_bound_runtime(st, tenant_key)
341
+ external_access.resolve_active_user(owner, tenant_key)
342
+ if response_type != "code":
343
+ raise ExternalOAuthError("oauth_response_type_invalid")
344
+ clean_state = _normalise_state(state)
345
+ challenge = _normalise_challenge(code_challenge, code_challenge_method)
346
+ audience = normalise_resource(resource)
347
+ issued_at = _now(now)
348
+ code_id, secret, salt = secrets.token_urlsafe(18), secrets.token_urlsafe(32), secrets.token_bytes(32)
349
+ issued = {}
350
+
351
+ def _set(value):
352
+ book = _book_value(value)
353
+ client = book["clients"].get(client_id)
354
+ if (not isinstance(client, dict) or client.get("username") != owner
355
+ or client.get("tenant") != tenant_key or client.get("revoked_at")):
356
+ raise ExternalOAuthError("oauth_client_invalid")
357
+ if redirect_uri not in (client.get("redirect_uris") or []):
358
+ raise ExternalOAuthError("oauth_redirect_uri_invalid")
359
+ access, databases = external_access.scope_of(client)
360
+ scopes = _normalise_scopes(scope, access)
361
+ states = {key: row for key, row in book["states"].items()
362
+ if isinstance(row, dict) and float(row.get("expires_at") or 0) >= issued_at}
363
+ state_id = hashlib.sha256(f"{tenant_key}\0{owner}\0{client_id}\0{clean_state}".encode()).hexdigest()
364
+ if state_id in states:
365
+ raise ExternalOAuthError("oauth_state_replayed")
366
+ states[state_id] = {"expires_at": issued_at + CODE_TTL_SECONDS}
367
+ codes = {key: row for key, row in book["codes"].items()
368
+ if isinstance(row, dict) and float(row.get("expires_at") or 0) >= issued_at - CODE_TTL_SECONDS}
369
+ row = {
370
+ "id": code_id, "tenant": tenant_key, "username": owner, "client_id": client_id,
371
+ "redirect_uri": redirect_uri, "state_hash": _state_digest(clean_state),
372
+ "code_challenge": challenge, "resource": audience,
373
+ "scopes": list(scopes), "access": access,
374
+ "databases": (external_access.ALL_DATABASES if databases == external_access.ALL_DATABASES
375
+ else list(databases)),
376
+ "code_salt": salt.hex(), "code_hash": _digest(salt, secret), "algo": ALGORITHM,
377
+ "created_at": issued_at, "expires_at": issued_at + CODE_TTL_SECONDS,
378
+ "consumed_at": None,
379
+ }
380
+ codes[code_id] = row
381
+ book.update({"states": states, "codes": codes})
382
+ issued["record"] = row
383
+ return book
384
+
385
+ _update(st, _set)
386
+ return {"redirect_uri": redirect_uri, "state": clean_state,
387
+ "code": _token("lpc1", tenant_key, code_id, secret),
388
+ "expires_in": CODE_TTL_SECONDS}
389
+
390
+
391
+ def exchange_code(code, *, client_id, redirect_uri, code_verifier, resource,
392
+ runtime_resolver, now=None) -> dict:
393
+ _, tenant_key, code_id, secret = _parse_token(code, {"lpc1"})
394
+ verifier = _normalise_verifier(code_verifier)
395
+ audience = normalise_resource(resource)
396
+ try:
397
+ st = runtime_resolver(tenant_key)
398
+ external_access._assert_bound_runtime(st, tenant_key)
399
+ except (KeyError, external_access.ExternalAccessError) as exc:
400
+ raise ExternalOAuthError("oauth_invalid_grant") from exc
401
+ snapshot = _book(st)
402
+ candidate = snapshot["codes"].get(code_id)
403
+ if isinstance(candidate, dict):
404
+ try:
405
+ external_access.resolve_active_user(candidate.get("username"), tenant_key)
406
+ except external_access.ExternalAccessError as exc:
407
+ raise ExternalOAuthError("oauth_invalid_grant") from exc
408
+ issued_at = _now(now)
409
+ access_secret, refresh_secret = secrets.token_urlsafe(32), secrets.token_urlsafe(32)
410
+ access_salt, refresh_salt = secrets.token_bytes(32), secrets.token_bytes(32)
411
+ grant_id = secrets.token_urlsafe(18)
412
+ result = {}
413
+
414
+ def _set(value):
415
+ book = _book_value(value)
416
+ row = book["codes"].get(code_id)
417
+ usable, salt, expected = _comparison_material(row, "code")
418
+ matched = hmac.compare_digest(_digest(salt, secret), expected)
419
+ client = book["clients"].get(client_id)
420
+ if (not usable or not matched or not isinstance(row, dict) or row.get("consumed_at")
421
+ or float(row.get("expires_at") or 0) < issued_at
422
+ or row.get("tenant") != tenant_key or row.get("client_id") != client_id
423
+ or row.get("redirect_uri") != redirect_uri
424
+ or row.get("resource") != audience
425
+ or not isinstance(client, dict) or client.get("revoked_at")
426
+ or client.get("username") != row.get("username")
427
+ or not hmac.compare_digest(_pkce_challenge(verifier), str(row.get("code_challenge") or ""))):
428
+ raise ExternalOAuthError("oauth_invalid_grant")
429
+ grants = dict(book["grants"])
430
+ while grant_id in grants:
431
+ raise ExternalOAuthError("oauth_unavailable")
432
+ grant = {
433
+ "id": grant_id, "tenant": tenant_key, "username": row["username"],
434
+ "client_id": client_id, "resource": audience,
435
+ "access": row["access"], "databases": row["databases"],
436
+ "scopes": list(row["scopes"]), "algo": ALGORITHM,
437
+ "access_salt": access_salt.hex(), "access_hash": _digest(access_salt, access_secret),
438
+ "refresh_salt": refresh_salt.hex(), "refresh_hash": _digest(refresh_salt, refresh_secret),
439
+ "used_refresh": [], "refresh_generation": 0,
440
+ "created_at": issued_at, "access_expires_at": issued_at + ACCESS_TTL_SECONDS,
441
+ "refresh_expires_at": issued_at + REFRESH_TTL_SECONDS,
442
+ "last_used_at": None, "disconnected_at": None,
443
+ }
444
+ grants[grant_id] = grant
445
+ codes = dict(book["codes"])
446
+ consumed = dict(row)
447
+ consumed["consumed_at"] = issued_at
448
+ codes[code_id] = consumed
449
+ book.update({"codes": codes, "grants": grants})
450
+ result["grant"] = grant
451
+ return book
452
+
453
+ _update(st, _set)
454
+ grant = result["grant"]
455
+ return {
456
+ "token_type": "Bearer", "access_token": _token("lpo1", tenant_key, grant_id, access_secret),
457
+ "expires_in": ACCESS_TTL_SECONDS,
458
+ "refresh_token": _token("lpr1", tenant_key, grant_id, refresh_secret),
459
+ "scope": " ".join(grant["scopes"]),
460
+ }
461
+
462
+
463
+ def refresh(refresh_token, *, client_id, resource, runtime_resolver, now=None) -> dict:
464
+ _, tenant_key, grant_id, secret = _parse_token(refresh_token, {"lpr1"})
465
+ try:
466
+ st = runtime_resolver(tenant_key)
467
+ external_access._assert_bound_runtime(st, tenant_key)
468
+ except (KeyError, external_access.ExternalAccessError) as exc:
469
+ raise ExternalOAuthError("oauth_invalid_grant") from exc
470
+ snapshot = _book(st)
471
+ candidate = snapshot["grants"].get(grant_id)
472
+ if isinstance(candidate, dict):
473
+ try:
474
+ external_access.resolve_active_user(candidate.get("username"), tenant_key)
475
+ except external_access.ExternalAccessError as exc:
476
+ raise ExternalOAuthError("oauth_invalid_grant") from exc
477
+ issued_at = _now(now)
478
+ audience = normalise_resource(resource)
479
+ next_access, next_refresh = secrets.token_urlsafe(32), secrets.token_urlsafe(32)
480
+ next_access_salt, next_refresh_salt = secrets.token_bytes(32), secrets.token_bytes(32)
481
+ result = {}
482
+
483
+ def _set(value):
484
+ book = _book_value(value)
485
+ grants = dict(book["grants"])
486
+ row = grants.get(grant_id)
487
+ current_ok, current_salt, current_hash = _comparison_material(row, "refresh")
488
+ current_match = hmac.compare_digest(_digest(current_salt, secret), current_hash)
489
+ history_ok, used, used_match = _used_refresh_history(row, secret)
490
+ if (isinstance(row, dict) and not history_ok and not row.get("disconnected_at")):
491
+ corrupted = dict(row)
492
+ corrupted["disconnected_at"] = issued_at
493
+ grants[grant_id] = corrupted
494
+ book["grants"] = grants
495
+ result["error"] = "oauth_invalid_grant"
496
+ return book
497
+ if (isinstance(row, dict) and used_match and not row.get("disconnected_at")):
498
+ replayed = dict(row)
499
+ replayed["disconnected_at"] = issued_at
500
+ grants[grant_id] = replayed
501
+ book["grants"] = grants
502
+ result["error"] = "oauth_refresh_replayed"
503
+ return book
504
+ client = book["clients"].get(client_id)
505
+ if (not current_ok or not current_match or not isinstance(row, dict)
506
+ or row.get("disconnected_at") or row.get("tenant") != tenant_key
507
+ or row.get("client_id") != client_id
508
+ or row.get("resource") != audience
509
+ or float(row.get("refresh_expires_at") or 0) < issued_at
510
+ or not isinstance(client, dict) or client.get("revoked_at")
511
+ or client.get("username") != row.get("username")):
512
+ raise ExternalOAuthError("oauth_invalid_grant")
513
+ generation = int(row.get("refresh_generation") or 0)
514
+ if generation >= MAX_REFRESH_ROTATIONS or len(used) >= MAX_REFRESH_ROTATIONS:
515
+ exhausted = dict(row)
516
+ exhausted["disconnected_at"] = issued_at
517
+ grants[grant_id] = exhausted
518
+ book["grants"] = grants
519
+ result["error"] = "oauth_refresh_exhausted"
520
+ return book
521
+ used.append({"salt": row.get("refresh_salt"), "hash": row.get("refresh_hash"),
522
+ "algo": ALGORITHM})
523
+ rotated = dict(row)
524
+ rotated.update({
525
+ "used_refresh": used,
526
+ "refresh_generation": generation + 1,
527
+ "access_salt": next_access_salt.hex(),
528
+ "access_hash": _digest(next_access_salt, next_access),
529
+ "refresh_salt": next_refresh_salt.hex(),
530
+ "refresh_hash": _digest(next_refresh_salt, next_refresh),
531
+ "access_expires_at": issued_at + ACCESS_TTL_SECONDS,
532
+ "last_used_at": None,
533
+ })
534
+ grants[grant_id] = rotated
535
+ book["grants"] = grants
536
+ result["grant"] = rotated
537
+ return book
538
+
539
+ _update(st, _set)
540
+ if result.get("error"):
541
+ raise ExternalOAuthError(result["error"])
542
+ grant = result["grant"]
543
+ return {
544
+ "token_type": "Bearer", "access_token": _token("lpo1", tenant_key, grant_id, next_access),
545
+ "expires_in": ACCESS_TTL_SECONDS,
546
+ "refresh_token": _token("lpr1", tenant_key, grant_id, next_refresh),
547
+ "scope": " ".join(grant["scopes"]),
548
+ }
549
+
550
+
551
+ def _intersect_databases(left, right):
552
+ if left == external_access.ALL_DATABASES:
553
+ return right
554
+ if right == external_access.ALL_DATABASES:
555
+ return left
556
+ allowed = set(right or ())
557
+ return tuple(item for item in (left or ()) if item in allowed)
558
+
559
+
560
+ def verify_access(token: str, *, resource, runtime_resolver) -> VerifiedOAuth:
561
+ try:
562
+ _, tenant_key, grant_id, secret = _parse_token(token, {"lpo1"})
563
+ try:
564
+ st = runtime_resolver(tenant_key)
565
+ external_access._assert_bound_runtime(st, tenant_key)
566
+ except KeyError as exc:
567
+ raise ExternalOAuthAuthError(AUTH_ERROR) from exc
568
+ book = _book(st)
569
+ audience = normalise_resource(resource)
570
+ row = book["grants"].get(grant_id)
571
+ usable, salt, expected = _comparison_material(row, "access")
572
+ matched = hmac.compare_digest(_digest(salt, secret), expected)
573
+ client = book["clients"].get(row.get("client_id")) if isinstance(row, dict) else None
574
+ now = _now()
575
+ if (not usable or not matched or not isinstance(row, dict) or row.get("disconnected_at")
576
+ or row.get("tenant") != tenant_key or float(row.get("access_expires_at") or 0) < now
577
+ or row.get("resource") != audience
578
+ or not isinstance(client, dict) or client.get("revoked_at")
579
+ or client.get("username") != row.get("username")
580
+ or client.get("tenant") != tenant_key):
581
+ raise ExternalOAuthAuthError(AUTH_ERROR)
582
+ try:
583
+ public = external_access.resolve_active_user(row["username"], tenant_key)
584
+ except external_access.ExternalAccessError as exc:
585
+ raise ExternalOAuthAuthError(AUTH_ERROR) from exc
586
+ grant_access, grant_databases = external_access.scope_of(row)
587
+ client_access, client_databases = external_access.scope_of(client)
588
+ access = (external_access.ACCESS_READWRITE
589
+ if grant_access == client_access == external_access.ACCESS_READWRITE
590
+ and "mcp:write" in (row.get("scopes") or ())
591
+ else external_access.ACCESS_READ)
592
+ databases = _intersect_databases(grant_databases, client_databases)
593
+ if now - float(row.get("last_used_at") or 0) >= _STAMP_EVERY:
594
+ def _stamp(value):
595
+ latest = _book_value(value)
596
+ grants = dict(latest["grants"])
597
+ current = grants.get(grant_id)
598
+ if (not isinstance(current, dict) or current.get("disconnected_at")
599
+ or current.get("access_hash") != row.get("access_hash")):
600
+ raise ExternalOAuthAuthError(AUTH_ERROR)
601
+ current = dict(current)
602
+ current["last_used_at"] = now
603
+ grants[grant_id] = current
604
+ latest["grants"] = grants
605
+ return latest
606
+ _update(st, _stamp, auth=True)
607
+ return VerifiedOAuth(tenant_key, grant_id, row["client_id"], public, st, access,
608
+ databases, tuple(row.get("scopes") or ()), audience)
609
+ except ExternalOAuthAuthError:
610
+ raise
611
+ except Exception as exc:
612
+ raise ExternalOAuthAuthError(AUTH_ERROR) from exc
613
+
614
+
615
+ def disconnect(token: str, *, runtime_resolver, now=None) -> DisconnectedOAuth:
616
+ prefix, tenant_key, grant_id, secret = _parse_token(token, {"lpo1", "lpr1"})
617
+ try:
618
+ st = runtime_resolver(tenant_key)
619
+ external_access._assert_bound_runtime(st, tenant_key)
620
+ except (KeyError, external_access.ExternalAccessError) as exc:
621
+ raise ExternalOAuthAuthError(AUTH_ERROR) from exc
622
+ stopped = {}
623
+ at = _now(now)
624
+ material = "access" if prefix == "lpo1" else "refresh"
625
+
626
+ def _set(value):
627
+ book = _book_value(value)
628
+ grants = dict(book["grants"])
629
+ row = grants.get(grant_id)
630
+ usable, salt, expected = _comparison_material(row, material)
631
+ if (not usable or not hmac.compare_digest(_digest(salt, secret), expected)
632
+ or not isinstance(row, dict) or row.get("tenant") != tenant_key
633
+ or not isinstance(row.get("username"), str) or not row.get("username")):
634
+ raise ExternalOAuthAuthError(AUTH_ERROR)
635
+ disconnected = dict(row)
636
+ if not disconnected.get("disconnected_at"):
637
+ disconnected["disconnected_at"] = at
638
+ grants[grant_id] = disconnected
639
+ book["grants"] = grants
640
+ stopped["record"] = disconnected
641
+ stopped["client"] = book["clients"].get(disconnected.get("client_id"))
642
+ return book
643
+
644
+ _update(st, _set, auth=True)
645
+ record = stopped["record"]
646
+ return DisconnectedOAuth(tenant_key, record["username"],
647
+ _metadata(record, stopped.get("client")))
648
+
649
+
650
+ def disconnect_for_user(st, username: str, tenant: str, grant_id: str) -> dict:
651
+ owner, tenant_key = external_access._normalise_owner(username, tenant)
652
+ external_access._assert_bound_runtime(st, tenant_key)
653
+ stopped = {}
654
+ at = _now()
655
+
656
+ def _set(value):
657
+ book = _book_value(value)
658
+ grants = dict(book["grants"])
659
+ row = grants.get(grant_id)
660
+ if (not isinstance(row, dict) or row.get("username") != owner
661
+ or row.get("tenant") != tenant_key):
662
+ raise ExternalOAuthError("oauth_connection_not_found")
663
+ disconnected = dict(row)
664
+ if not disconnected.get("disconnected_at"):
665
+ disconnected["disconnected_at"] = at
666
+ grants[grant_id] = disconnected
667
+ book["grants"] = grants
668
+ stopped["record"] = disconnected
669
+ stopped["client"] = book["clients"].get(disconnected.get("client_id"))
670
+ return book
671
+
672
+ _update(st, _set)
673
+ return _metadata(stopped["record"], stopped.get("client"))
674
+
675
+
676
+ def list_connections(st, username: str, tenant: str) -> list[dict]:
677
+ owner, tenant_key = external_access._normalise_owner(username, tenant)
678
+ external_access._assert_bound_runtime(st, tenant_key)
679
+ book = _book(st)
680
+ return sorted([
681
+ _metadata(row, book["clients"].get(row.get("client_id")))
682
+ for row in book["grants"].values()
683
+ if isinstance(row, dict) and row.get("username") == owner
684
+ and row.get("tenant") == tenant_key and not row.get("disconnected_at")
685
+ ], key=lambda row: str(row.get("created") or ""), reverse=True)
686
+
687
+
688
+ def label_for(st, grant_id: str) -> str:
689
+ book = _book(st)
690
+ grant = book["grants"].get(grant_id)
691
+ client = book["clients"].get(grant.get("client_id")) if isinstance(grant, dict) else None
692
+ if (not isinstance(grant, dict) or grant.get("disconnected_at")
693
+ or not isinstance(client, dict) or client.get("revoked_at")):
694
+ raise ExternalOAuthAuthError(AUTH_ERROR)
695
+ label = client.get("label")
696
+ if not isinstance(label, str) or not label:
697
+ raise ExternalOAuthError("oauth_unavailable")
698
+ return label
699
+
700
+
701
+ def _book_value(value) -> dict:
702
+ if value is None:
703
+ return {"clients": {}, "codes": {}, "states": {}, "grants": {}}
704
+ if not isinstance(value, dict):
705
+ raise ExternalOAuthError("oauth_unavailable")
706
+ book = dict(value)
707
+ for key in ("clients", "codes", "states", "grants"):
708
+ child = book.get(key, {})
709
+ if not isinstance(child, dict):
710
+ raise ExternalOAuthError("oauth_unavailable")
711
+ book[key] = dict(child)
712
+ return book
713
+
714
+
715
+ def _update(st, callback, *, auth=False) -> None:
716
+ try:
717
+ st.update(BUCKET, callback, flush="sync")
718
+ except (ExternalOAuthError, ExternalOAuthAuthError):
719
+ raise
720
+ except Exception as exc:
721
+ if auth:
722
+ raise ExternalOAuthAuthError(AUTH_ERROR) from exc
723
+ raise ExternalOAuthError("oauth_unavailable") from exc
platform/core/external_tools.py CHANGED
@@ -6,11 +6,10 @@ Adding a tool means adding a registry entry; writes also require the router's li
6
  """
7
  import copy
8
  import json
9
- import uuid
10
  from dataclasses import dataclass
11
- from typing import Callable
12
 
13
- from core import sheets, sheet_structure, sheet_templates
14
 
15
 
16
  @dataclass(frozen=True)
@@ -24,9 +23,15 @@ class ToolContext:
24
  cache_binding: Callable
25
  list_databases: Callable
26
  describe_database: Callable
 
27
  view_event: Callable
 
28
  app_upsert: Callable
29
  app_delete: Callable
 
 
 
 
30
 
31
 
32
  def _object(properties=None, required=(), **extra):
@@ -156,7 +161,7 @@ def _add_tab(ctx, args):
156
  raise sheets.SheetRefusal(
157
  "tab_name_taken",
158
  "Tab " + wanted + " is already in this workbook. Choose another name.")
159
- tab = {"id": uuid.uuid4().hex[:12], "name": args["name"], "cells": {}}
160
  book["sheets"].append(tab)
161
  return _put(ctx, args, book, tab["id"])
162
 
@@ -252,7 +257,7 @@ def _chart(ctx, args):
252
  book = _book(ctx, args, write=True)
253
  tab = sheets.find_tab(book, args["tab"])
254
  chart = copy.deepcopy(args["chart"])
255
- chart.setdefault("id", uuid.uuid4().hex[:12])
256
  tab["charts"] = [row for row in tab.get("charts", []) if row["id"] != chart["id"]] + [chart]
257
  return {**_put(ctx, args, book, tab["id"]), "chart_id": chart["id"]}
258
 
@@ -269,6 +274,55 @@ def _tool(description, schema, handler, *, write=False, grants=("analyst", "shee
269
  "grants": grants, "handler": handler}
270
 
271
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
272
  TOOLS = {
273
  "sheet_create_workbook": _tool("Create a private workbook with one initial tab.",
274
  _object({"name": _STRING}, ("name",), additionalProperties=False), _create, write=True),
@@ -317,14 +371,44 @@ TOOLS = {
317
  "describe_database": _tool("Read a database's field schema through the strict Assistant scope wall.",
318
  _object({"database": _STRING}, ("database",), additionalProperties=False),
319
  lambda ctx, args: ctx.describe_database(args["database"]), grants=("analyst",)),
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
320
  "view_upsert": _tool(
321
  "Create or update a saved View using the grid's filters, sorts, groupBy, visible fields and display config.",
322
  _object({"database": _STRING, "view": _object({
323
  "id": {"type": "string", "minLength": 1, "maxLength": 120},
324
  "name": {"type": "string", "minLength": 1, "maxLength": 120},
325
- "config": _VIEW_CONFIG, "note": _STRING}, ("id", "name", "config"))},
 
326
  ("database", "view"), additionalProperties=False),
327
  lambda ctx, args: ctx.view_event("view_upsert", args), write=True, grants=("analyst",)),
 
 
 
 
 
 
 
 
 
 
328
  "view_delete": _tool("Delete a saved View through its ordinary owner and database permission walls.",
329
  _object({"database": _STRING, "view_id": _STRING}, ("database", "view_id"),
330
  additionalProperties=False),
@@ -339,9 +423,83 @@ TOOLS = {
339
  _object({"app_id": _STRING, "confirmName": _STRING}, ("app_id", "confirmName"),
340
  additionalProperties=False),
341
  lambda ctx, args: ctx.app_delete(args), write=True, grants=("analyst",)),
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
342
  }
343
 
344
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
345
  def dispatch(ctx, name, arguments):
346
  """Dispatch one declared tool. The API translates core and grant refusals to MCP content."""
347
  tool = TOOLS.get(name) if isinstance(name, str) else None
@@ -358,15 +516,20 @@ def dispatch(ctx, name, arguments):
358
  raise sheets.SheetRefusal("tool_arguments_invalid", "Tool arguments must match the published input schema.")
359
  # Validate only the transport shape here. Document semantics belong to the app writers.
360
  for key, value in arguments.items():
361
- kind = schema["properties"].get(key, {}).get("type")
 
362
  types = {"object": dict, "array": list, "string": str, "integer": int}
363
  if kind in types and (not isinstance(value, types[kind]) or (kind == "integer" and isinstance(value, bool))):
364
  raise sheets.SheetRefusal("tool_arguments_invalid", f"Tool argument {key} must be an {kind}.")
 
 
 
365
  # A dimension patch must never become an arbitrary tab-document replacement.
366
  if name == "sheet_set_dimensions":
367
  allowed = schema["properties"]["dimensions"]["properties"]
368
  if arguments["dimensions"].keys() - allowed.keys():
369
  raise sheets.SheetRefusal("tool_arguments_invalid", "Dimensions may contain only the published geometry fields.")
370
- if "lpb1." in json.dumps(arguments, ensure_ascii=False, allow_nan=False):
 
371
  raise sheets.SheetRefusal("credential_in_arguments", "Tool arguments must not contain credential values.")
372
  return tool["handler"](ctx, arguments)
 
6
  """
7
  import copy
8
  import json
 
9
  from dataclasses import dataclass
10
+ from typing import Callable, Optional
11
 
12
+ from core import sheets, sheet_structure, sheet_templates, write_execution
13
 
14
 
15
  @dataclass(frozen=True)
 
23
  cache_binding: Callable
24
  list_databases: Callable
25
  describe_database: Callable
26
+ query_database: Callable
27
  view_event: Callable
28
+ view_share_preview: Callable
29
  app_upsert: Callable
30
  app_delete: Callable
31
+ database_object: Optional[Callable] = None
32
+ sheet_object: Optional[Callable] = None
33
+ validated_batch: Optional[Callable] = None
34
+ confirm_change: Optional[Callable] = None
35
 
36
 
37
  def _object(properties=None, required=(), **extra):
 
161
  raise sheets.SheetRefusal(
162
  "tab_name_taken",
163
  "Tab " + wanted + " is already in this workbook. Choose another name.")
164
+ tab = {"id": write_execution.new_id()[:12], "name": args["name"], "cells": {}}
165
  book["sheets"].append(tab)
166
  return _put(ctx, args, book, tab["id"])
167
 
 
257
  book = _book(ctx, args, write=True)
258
  tab = sheets.find_tab(book, args["tab"])
259
  chart = copy.deepcopy(args["chart"])
260
+ chart.setdefault("id", write_execution.new_id()[:12])
261
  tab["charts"] = [row for row in tab.get("charts", []) if row["id"] != chart["id"]] + [chart]
262
  return {**_put(ctx, args, book, tab["id"]), "chart_id": chart["id"]}
263
 
 
274
  "grants": grants, "handler": handler}
275
 
276
 
277
+ def _callback(ctx, name, args):
278
+ callback = getattr(ctx, name, None)
279
+ if not callable(callback):
280
+ raise sheets.SheetRefusal(
281
+ "tool_capability_unsupported",
282
+ "This deployment does not publish that object operation in its capability manifest.")
283
+ return callback(args)
284
+
285
+
286
+ _DATABASE_ACTIONS = (
287
+ "list", "describe", "query", "record_create", "record_update", "record_delete",
288
+ "field_create", "field_update", "field_delete",
289
+ )
290
+ _VIEW_ACTIONS = ("upsert", "delete", "share_preview")
291
+ _SHEET_OBJECT_ACTIONS = {
292
+ "workbook_object": ("list", "get", "create", "named_ranges", "export"),
293
+ "worksheet_object": (
294
+ "list", "get", "add", "update", "delete", "geometry", "hidden_bands",
295
+ "freeze", "gridlines", "named_ranges",
296
+ ),
297
+ "range_object": (
298
+ "read", "write_values", "write_formulas", "write_styles", "number_format",
299
+ "fill_down", "clear",
300
+ ),
301
+ "calculation_object": ("calculate", "refresh"),
302
+ }
303
+
304
+
305
+ def _object_tool(name, actions):
306
+ target = {
307
+ "workbook_id": _STRING,
308
+ "worksheet": _STRING,
309
+ "range": _RANGE,
310
+ "revision": {"type": "integer"},
311
+ "arguments": {"type": "object"},
312
+ }
313
+ return _tool(
314
+ "Use the published Excel-shaped capability manifest for one "
315
+ + name.removesuffix("_object") + " operation.",
316
+ _object({"action": {"enum": list(actions)}, **target}, ("action",),
317
+ additionalProperties=False),
318
+ lambda ctx, args, object_name=name: _callback(
319
+ ctx, "sheet_object", {"object": object_name.removesuffix("_object"), **args}),
320
+ # The action classifier below distinguishes reads. Declaring the surface write-capable
321
+ # keeps the API's literal allow-list fail closed when a new action is added.
322
+ write=True,
323
+ )
324
+
325
+
326
  TOOLS = {
327
  "sheet_create_workbook": _tool("Create a private workbook with one initial tab.",
328
  _object({"name": _STRING}, ("name",), additionalProperties=False), _create, write=True),
 
371
  "describe_database": _tool("Read a database's field schema through the strict Assistant scope wall.",
372
  _object({"database": _STRING}, ("database",), additionalProperties=False),
373
  lambda ctx, args: ctx.describe_database(args["database"]), grants=("analyst",)),
374
+ "query_database": _tool(
375
+ "Query currently permitted records with bounded filters, stable sorts, cursor pagination and aggregates.",
376
+ _object({
377
+ "database": _STRING,
378
+ "fields": {"type": "array", "items": _STRING, "maxItems": 200},
379
+ "filters": {"oneOf": [_FILTER, {"type": "array", "items": {"type": "object"}}]},
380
+ "sorts": {"type": "array", "maxItems": 8, "items": _object({
381
+ "field": _STRING, "direction": {"enum": ["asc", "desc"]},
382
+ }, ("field", "direction"), additionalProperties=False)},
383
+ "limit": {"type": "integer", "minimum": 1, "maximum": 500},
384
+ "cursor": _STRING,
385
+ "aggregates": {"type": "array", "maxItems": 16, "items": _object({
386
+ "fn": {"enum": ["count", "count_distinct", "sum", "avg", "min", "max"]},
387
+ "field": _STRING,
388
+ }, ("fn",), additionalProperties=False)},
389
+ }, ("database",), additionalProperties=False),
390
+ lambda ctx, args: ctx.query_database(args["database"], {
391
+ key: value for key, value in args.items() if key != "database"
392
+ }), grants=("analyst",)),
393
  "view_upsert": _tool(
394
  "Create or update a saved View using the grid's filters, sorts, groupBy, visible fields and display config.",
395
  _object({"database": _STRING, "view": _object({
396
  "id": {"type": "string", "minLength": 1, "maxLength": 120},
397
  "name": {"type": "string", "minLength": 1, "maxLength": 120},
398
+ "config": _VIEW_CONFIG, "note": _STRING}, ("id", "name", "config"),
399
+ additionalProperties=False)},
400
  ("database", "view"), additionalProperties=False),
401
  lambda ctx, args: ctx.view_event("view_upsert", args), write=True, grants=("analyst",)),
402
+ "view_share_preview": _tool(
403
+ "Preview a personal draft View's C8 audience and field inheritance without sharing or mutating it.",
404
+ _object({
405
+ "database": _STRING,
406
+ "view_id": _STRING,
407
+ "entries": {"type": "array", "maxItems": 200, "items": _object({
408
+ "user": _STRING, "role": {"enum": ["view", "edit"]},
409
+ }, ("user", "role"), additionalProperties=False)},
410
+ }, ("database", "view_id", "entries"), additionalProperties=False),
411
+ lambda ctx, args: ctx.view_share_preview(args), grants=("analyst",)),
412
  "view_delete": _tool("Delete a saved View through its ordinary owner and database permission walls.",
413
  _object({"database": _STRING, "view_id": _STRING}, ("database", "view_id"),
414
  additionalProperties=False),
 
423
  _object({"app_id": _STRING, "confirmName": _STRING}, ("app_id", "confirmName"),
424
  additionalProperties=False),
425
  lambda ctx, args: ctx.app_delete(args), write=True, grants=("analyst",)),
426
+ "database_object": _tool(
427
+ "Discover/query a database or preview-confirm record and field CRUD through its live UI walls.",
428
+ _object({
429
+ "action": {"enum": list(_DATABASE_ACTIONS)},
430
+ "database": _STRING,
431
+ "record_id": {"oneOf": [_STRING, {"type": "integer"}]},
432
+ "values": {"type": "object"},
433
+ "field": {"type": "object"},
434
+ "field_key": _STRING,
435
+ "query": {"type": "object"},
436
+ }, ("action",), additionalProperties=False),
437
+ lambda ctx, args: _callback(ctx, "database_object", args), write=True,
438
+ grants=("analyst",)),
439
+ "view_object": _tool(
440
+ "Read or preview-confirm a personal View operation through the ordinary grid writer.",
441
+ _object({
442
+ "action": {"enum": list(_VIEW_ACTIONS)}, "database": _STRING,
443
+ "view": {"type": "object"}, "view_id": _STRING,
444
+ "entries": {"type": "array", "items": {"type": "object"}, "maxItems": 200},
445
+ }, ("action", "database"), additionalProperties=False),
446
+ lambda ctx, args: (
447
+ ctx.view_event("view_upsert", {"database": args["database"], "view": args["view"]})
448
+ if args["action"] == "upsert" else
449
+ ctx.view_event("view_delete", {"database": args["database"],
450
+ "view_id": args["view_id"]})
451
+ if args["action"] == "delete" else
452
+ ctx.view_share_preview({"database": args["database"],
453
+ "view_id": args["view_id"],
454
+ "entries": args.get("entries") or []})
455
+ ), write=True, grants=("analyst",)),
456
+ **{name: _object_tool(name, actions)
457
+ for name, actions in _SHEET_OBJECT_ACTIONS.items()},
458
+ "validated_batch": _tool(
459
+ "Validate a bounded object-operation batch; writes preview and confirm as one unit.",
460
+ _object({"operations": {"type": "array", "minItems": 1, "maxItems": 50,
461
+ "items": _object({"name": _STRING,
462
+ "arguments": {"type": "object"}},
463
+ ("name", "arguments"),
464
+ additionalProperties=False)}},
465
+ ("operations",), additionalProperties=False),
466
+ lambda ctx, args: _callback(ctx, "validated_batch", args), write=True,
467
+ grants=("analyst",)),
468
+ "confirm_change": _tool(
469
+ "Confirm one unexpired preview. Destructive/schema changes require its exact typed phrase.",
470
+ _object({"confirmation_token": _STRING, "confirmation_phrase": _STRING},
471
+ ("confirmation_token",), additionalProperties=False),
472
+ lambda ctx, args: _callback(ctx, "confirm_change", args), grants=("analyst",)),
473
  }
474
 
475
 
476
+ _OBJECT_READ_ACTIONS = {
477
+ "database_object": frozenset({"list", "describe", "query"}),
478
+ "view_object": frozenset({"share_preview"}),
479
+ "workbook_object": frozenset({"list", "get", "export"}),
480
+ "worksheet_object": frozenset({"list", "get"}),
481
+ "range_object": frozenset({"read"}),
482
+ "calculation_object": frozenset(),
483
+ }
484
+
485
+
486
+ def is_write(name, arguments) -> bool:
487
+ """Classify dynamic object tools without trusting an unvalidated action as a read."""
488
+ tool = TOOLS.get(name) if isinstance(name, str) else None
489
+ if tool is None or name == "confirm_change":
490
+ return False
491
+ if name == "validated_batch":
492
+ operations = arguments.get("operations") if isinstance(arguments, dict) else None
493
+ if not isinstance(operations, list):
494
+ return True
495
+ return any(is_write(item.get("name"), item.get("arguments"))
496
+ for item in operations if isinstance(item, dict))
497
+ if name in _OBJECT_READ_ACTIONS:
498
+ action = arguments.get("action") if isinstance(arguments, dict) else None
499
+ return action not in _OBJECT_READ_ACTIONS[name]
500
+ return bool(tool.get("write"))
501
+
502
+
503
  def dispatch(ctx, name, arguments):
504
  """Dispatch one declared tool. The API translates core and grant refusals to MCP content."""
505
  tool = TOOLS.get(name) if isinstance(name, str) else None
 
516
  raise sheets.SheetRefusal("tool_arguments_invalid", "Tool arguments must match the published input schema.")
517
  # Validate only the transport shape here. Document semantics belong to the app writers.
518
  for key, value in arguments.items():
519
+ property_schema = schema["properties"].get(key, {})
520
+ kind = property_schema.get("type")
521
  types = {"object": dict, "array": list, "string": str, "integer": int}
522
  if kind in types and (not isinstance(value, types[kind]) or (kind == "integer" and isinstance(value, bool))):
523
  raise sheets.SheetRefusal("tool_arguments_invalid", f"Tool argument {key} must be an {kind}.")
524
+ if "enum" in property_schema and value not in property_schema["enum"]:
525
+ raise sheets.SheetRefusal(
526
+ "tool_arguments_invalid", f"Tool argument {key} is not a published operation.")
527
  # A dimension patch must never become an arbitrary tab-document replacement.
528
  if name == "sheet_set_dimensions":
529
  allowed = schema["properties"]["dimensions"]["properties"]
530
  if arguments["dimensions"].keys() - allowed.keys():
531
  raise sheets.SheetRefusal("tool_arguments_invalid", "Dimensions may contain only the published geometry fields.")
532
+ encoded_arguments = json.dumps(arguments, ensure_ascii=False, allow_nan=False)
533
+ if any(prefix in encoded_arguments for prefix in ("lpb1.", "lpo1.", "lpr1.", "lpc1.")):
534
  raise sheets.SheetRefusal("credential_in_arguments", "Tool arguments must not contain credential values.")
535
  return tool["handler"](ctx, arguments)
platform/core/field_permissions.py CHANGED
@@ -6,6 +6,7 @@ that could never be true. This module is the single normalization point for the
6
  personal, collaborative, or specific users.
7
  """
8
 
 
9
  import time
10
 
11
  import core.shared_overlay as shared_overlay
@@ -26,6 +27,18 @@ FIELD_ORIGINS = ("preset", "user")
26
  FIELD_AUDIENCES = ("private", "users", "everyone")
27
  FIELD_VALUE_STRATA = ("shared", "personal")
28
 
 
 
 
 
 
 
 
 
 
 
 
 
29
  #: ⭐⭐ W42-T47 / RULING R15 / CONTRACT C10 — WHICH STRATUM A DEFINITION WAS READ FROM.
30
  #:
31
  #: C10: *"`field_classes` reports, beside `{origin, audience, sharedBy, values, owner}`, WHICH
@@ -102,6 +115,377 @@ def grant_entries(permissions):
102
  return []
103
 
104
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
105
  def permissions_from_grants(entries):
106
  """⭐⭐ W40-T03 / CONTRACT C4 — THE CLASSIFICATION IS DERIVED FROM THE REGISTRY, NEVER STAMPED.
107
 
 
6
  personal, collaborative, or specific users.
7
  """
8
 
9
+ import copy
10
  import time
11
 
12
  import core.shared_overlay as shared_overlay
 
27
  FIELD_AUDIENCES = ("private", "users", "everyone")
28
  FIELD_VALUE_STRATA = ("shared", "personal")
29
 
30
+ #: SDLC63 C8 — the View-to-field edges live beside the View grants, in the SAME tenant-scoped
31
+ #: document, but outside ``shares.KINDS``. A boundary names fields once; it never copies the
32
+ #: View's current people into N field grant records. Consequently a later audience edit takes
33
+ #: effect on the next permission read and removing one View cannot erase a direct or second-View
34
+ #: path. ``core.shares`` deliberately remains the direct-grant registry; callers that need to
35
+ #: explain provenance can therefore distinguish its record from this derived edge.
36
+ VIEW_FIELD_BOUNDARIES_KEY = "view_field_boundaries"
37
+ VIEW_FIELD_AUDIT_CAP = 200
38
+ VIEW_SHARE_DIAGNOSTICS_KEY = "view_share_diagnostics"
39
+ VIEW_SHARE_DIAGNOSTIC_CAP = 30
40
+ VIEW_SHARE_DIAGNOSTIC_FIELDS_CAP = 500
41
+
42
  #: ⭐⭐ W42-T47 / RULING R15 / CONTRACT C10 — WHICH STRATUM A DEFINITION WAS READ FROM.
43
  #:
44
  #: C10: *"`field_classes` reports, beside `{origin, audience, sharedBy, values, owner}`, WHICH
 
115
  return []
116
 
117
 
118
+ # ---------------------------------------------------------- SDLC63 / C8: View audience inheritance
119
+
120
+ def view_field_keys(view):
121
+ """Fields deliberately used by one View, in stable encounter order.
122
+
123
+ ``config.order`` is intentionally absent. Both the browser reconciler and the server save
124
+ door append every database field to that list, including columns the author never selected;
125
+ treating it as an inclusion list is the old over-share. Visible columns and fields used by a
126
+ View rule/display are deliberate. Hiding a plain column therefore removes that inherited
127
+ path, while a hidden filter/group/sort input stays readable because the View still depends on
128
+ it.
129
+ """
130
+ if not isinstance(view, dict):
131
+ return []
132
+ cfg = view.get("config") if isinstance(view.get("config"), dict) else view
133
+ keys, seen = [], set()
134
+
135
+ def add(raw):
136
+ key = str(raw or "").strip()
137
+ if key and key not in seen:
138
+ seen.add(key)
139
+ keys.append(key)
140
+
141
+ def walk_filter(node):
142
+ if isinstance(node, list):
143
+ for child in node:
144
+ walk_filter(child)
145
+ return
146
+ if not isinstance(node, dict):
147
+ return
148
+ walk_filter(node.get("children"))
149
+ add(node.get("colId"))
150
+ rhs = node.get("rhs")
151
+ if isinstance(rhs, dict) and rhs.get("kind") != "stat":
152
+ add(rhs.get("colId"))
153
+
154
+ for key in cfg.get("visible") or ():
155
+ add(key)
156
+ walk_filter(cfg.get("filters"))
157
+ add(cfg.get("groupBy"))
158
+ colour = cfg.get("colorBy")
159
+ add(colour.get("colId") if isinstance(colour, dict) else colour)
160
+ sorts = cfg.get("sorts")
161
+ if isinstance(sorts, dict):
162
+ sorts = [sorts]
163
+ for spec in sorts or ():
164
+ if isinstance(spec, dict):
165
+ add(spec.get("colId"))
166
+ for key in (cfg.get("summary") or {}) if isinstance(cfg.get("summary"), dict) else ():
167
+ add(key)
168
+ display = cfg.get("display")
169
+ if isinstance(display, dict):
170
+ for name, value in display.items():
171
+ if str(name).endswith("Field"):
172
+ add(value)
173
+ for chart in display.get("charts") or ():
174
+ if isinstance(chart, dict):
175
+ add(chart.get("x"))
176
+ add(chart.get("y"))
177
+ for metric in display.get("calendarMetrics") or ():
178
+ if isinstance(metric, dict):
179
+ add(metric.get("field"))
180
+ for key in display.get("fields") or ():
181
+ add(key)
182
+ return keys
183
+
184
+
185
+ def _st_for_boundaries(st=None):
186
+ """Use the same tenant handle ``core.shares`` uses without reaching into a second store."""
187
+ return st if st is not None else shares._st(None)
188
+
189
+
190
+ def sync_view_field_boundary(view, grant_topic, actor, st=None, *, audit_additions=True,
191
+ force=False):
192
+ """Persist one View's current field edge and return its delta.
193
+
194
+ The audience snapshot used by an addition audit is read inside the same ``object_shares``
195
+ update as the boundary change. It is evidence only; permission resolution always reads the
196
+ live View grant, so the snapshot can never become an access list.
197
+ """
198
+ view_id = str((view or {}).get("id") or "").strip()
199
+ topic = str(grant_topic or "").strip()
200
+ who = _uname(actor)
201
+ if not view_id or not topic:
202
+ return {"added": [], "removed": [], "audience": [], "owner": None}
203
+ wanted = view_field_keys(view)
204
+ result = {"added": [], "removed": [], "audience": [], "owner": None}
205
+ handle = _st_for_boundaries(st)
206
+
207
+ # A private View needs no second write on every filter/resize autosave. Its first share calls
208
+ # this helper with ``force=True`` after the View grant lands; from then on the existing edge
209
+ # makes field additions/removals write-bearing. This keeps C8 off the ordinary private-view
210
+ # hot path without opening a share-before-boundary race.
211
+ try:
212
+ current = handle.get(shares.SHARES_KEY) or {}
213
+ except Exception: # noqa: BLE001
214
+ current = {}
215
+ current_boundaries = current.get(VIEW_FIELD_BOUNDARIES_KEY) or {}
216
+ current_grant = (current.get("view") or {}).get(view_id)
217
+ current_grant = current_grant if isinstance(current_grant, dict) else {}
218
+ current_audience = shares._clean_entries(current_grant.get("entries"))
219
+ current_owner = _uname(current_grant.get("owner")) or None
220
+ prior = (current_boundaries.get(view_id)
221
+ if isinstance(current_boundaries, dict) else None)
222
+ if not force and not isinstance(prior, dict) and not current_audience:
223
+ return {"added": [], "removed": [], "audience": [], "owner": current_owner}
224
+ if isinstance(prior, dict):
225
+ before = [str(key) for key in (prior.get("fields") or ()) if str(key)]
226
+ if prior.get("topic") == topic and before == wanted:
227
+ return {"added": [], "removed": [],
228
+ "audience": current_audience, "owner": current_owner}
229
+
230
+ def _apply(data):
231
+ data = data if isinstance(data, dict) else {}
232
+ boundaries = dict(data.get(VIEW_FIELD_BOUNDARIES_KEY) or {})
233
+ prior = boundaries.get(view_id) if isinstance(boundaries.get(view_id), dict) else {}
234
+ before = [str(key) for key in (prior.get("fields") or ()) if str(key)]
235
+ before_set, wanted_set = set(before), set(wanted)
236
+ added = [key for key in wanted if key not in before_set]
237
+ removed = [key for key in before if key not in wanted_set]
238
+ grant = (data.get("view") or {}).get(view_id)
239
+ grant = grant if isinstance(grant, dict) else {}
240
+ audience = shares._clean_entries(grant.get("entries"))
241
+ owner = _uname(grant.get("owner")) or None
242
+ audit = [row for row in (prior.get("audit") or ()) if isinstance(row, dict)]
243
+ if audit_additions and audience:
244
+ stamp = time.time()
245
+ snapshot = [{"user": row["user"], "role": row["role"]} for row in audience]
246
+ for key in added:
247
+ audit.append({"ts": stamp, "actor": who or None, "field": key,
248
+ "view": view_id, "audience": snapshot})
249
+ boundaries[view_id] = {
250
+ "topic": topic,
251
+ "fields": wanted,
252
+ "updatedBy": who or None,
253
+ "audit": audit[-VIEW_FIELD_AUDIT_CAP:],
254
+ }
255
+ data[VIEW_FIELD_BOUNDARIES_KEY] = boundaries
256
+ result.update({"added": added, "removed": removed,
257
+ "audience": audience, "owner": owner})
258
+ return data
259
+
260
+ handle.update(shares.SHARES_KEY, _apply, flush="async")
261
+ return result
262
+
263
+
264
+ def set_view_grants_with_boundary(view, grant_topic, entries, *, owner, granter, expect,
265
+ st=None, _fail_after_prepare=False):
266
+ """Replace one View audience and its field edge in one store transaction.
267
+
268
+ The mutation is built on a detached copy. If comparison, normalization, a test-injected
269
+ mid-operation fault, or persistence fails, neither the audience nor its derived boundary is
270
+ visible. The callback writes one bounded View record plus one bounded list of field keys;
271
+ its store-call count is therefore constant for a View with one field or five hundred.
272
+
273
+ ``_fail_after_prepare`` is an internal verifier seam. It raises after both changes have
274
+ been prepared but before the detached document is returned, proving rollback at the only
275
+ point a half-applied mutation could otherwise escape.
276
+ """
277
+ view_id = str((view or {}).get("id") or "").strip()
278
+ topic = str(grant_topic or "").strip()
279
+ if not view_id or not topic:
280
+ raise ValueError("a View share must name its View and database")
281
+ clean = shares._clean_entries(entries)
282
+ wanted = view_field_keys(view)
283
+ owner_l = _uname(owner) or None
284
+ granter_l = _uname(granter) or None
285
+ expected = ([(row["user"], row["role"])
286
+ for row in shares._clean_entries(expect)]
287
+ if expect is not None else None)
288
+ answer = {}
289
+
290
+ def _apply(raw):
291
+ data = copy.deepcopy(raw) if isinstance(raw, dict) else {}
292
+ views = dict(data.get("view") or {})
293
+ prior = views.get(view_id) if isinstance(views.get(view_id), dict) else {}
294
+ prior_entries = shares._clean_entries(prior.get("entries"))
295
+ if expected is not None:
296
+ current = [(row["user"], row["role"]) for row in prior_entries]
297
+ if current != expected:
298
+ raise shares.GrantsChanged(
299
+ "this item was shared with somebody else while you were editing, so nothing "
300
+ "was saved. Reopen the sharing panel to see who has access now, then make "
301
+ "your change again.")
302
+
303
+ prior_by = {row["user"]: row.get("by") for row in prior_entries}
304
+ stamped = []
305
+ for row in clean:
306
+ saved = {"user": row["user"], "role": row["role"]}
307
+ by = prior_by[row["user"]] if row["user"] in prior_by else granter_l
308
+ if by:
309
+ saved["by"] = by
310
+ stamped.append(saved)
311
+ keep_owner = owner_l or _uname(prior.get("owner")) or None
312
+ if not stamped and not keep_owner:
313
+ views.pop(view_id, None)
314
+ else:
315
+ views[view_id] = {"owner": keep_owner, "entries": stamped}
316
+
317
+ boundaries = dict(data.get(VIEW_FIELD_BOUNDARIES_KEY) or {})
318
+ old_boundary = (boundaries.get(view_id)
319
+ if isinstance(boundaries.get(view_id), dict) else {})
320
+ boundaries[view_id] = {
321
+ "topic": topic,
322
+ "fields": list(wanted),
323
+ "updatedBy": granter_l,
324
+ "audit": [dict(row) for row in (old_boundary.get("audit") or ())
325
+ if isinstance(row, dict)][-VIEW_FIELD_AUDIT_CAP:],
326
+ }
327
+ data["view"] = views
328
+ data[VIEW_FIELD_BOUNDARIES_KEY] = boundaries
329
+ if _fail_after_prepare:
330
+ raise RuntimeError("forced View-share transaction failure")
331
+ answer.update({"owner": keep_owner, "entries": stamped,
332
+ "fieldCount": len(wanted)})
333
+ return data
334
+
335
+ _st_for_boundaries(st).update(shares.SHARES_KEY, _apply, flush="sync")
336
+ return dict(answer)
337
+
338
+
339
+ def record_view_share_diagnostic(reference, view_id, actor, stage, fields, cause, st=None):
340
+ """Retain bounded server-only failure detail under a short support reference."""
341
+ ref = str(reference or "").strip().upper()[:40]
342
+ if not ref:
343
+ return False
344
+ clean_fields = []
345
+ seen = set()
346
+ for raw in fields or ():
347
+ key = str(raw or "").strip()[:160]
348
+ if key and key not in seen:
349
+ seen.add(key)
350
+ clean_fields.append(key)
351
+ if len(clean_fields) >= VIEW_SHARE_DIAGNOSTIC_FIELDS_CAP:
352
+ break
353
+
354
+ def _apply(raw):
355
+ data = copy.deepcopy(raw) if isinstance(raw, dict) else {}
356
+ rows = [dict(row) for row in (data.get(VIEW_SHARE_DIAGNOSTICS_KEY) or ())
357
+ if isinstance(row, dict)]
358
+ rows.append({
359
+ "reference": ref,
360
+ "ts": time.time(),
361
+ "view": str(view_id or "").strip()[:160],
362
+ "actor": _uname(actor) or None,
363
+ "stage": str(stage or "transaction").strip()[:80],
364
+ "fields": clean_fields,
365
+ "cause": (str(cause or "view share failed").strip()[:500]
366
+ if isinstance(cause, str)
367
+ else cause.__class__.__name__),
368
+ })
369
+ data[VIEW_SHARE_DIAGNOSTICS_KEY] = rows[-VIEW_SHARE_DIAGNOSTIC_CAP:]
370
+ return data
371
+
372
+ _st_for_boundaries(st).update(shares.SHARES_KEY, _apply, flush="sync")
373
+ return True
374
+
375
+
376
+ def drop_view_field_boundary(view_id, st=None):
377
+ """Remove exactly one inherited source; direct and other-View grants remain untouched."""
378
+ view_id = str(view_id or "").strip()
379
+ if not view_id:
380
+ return False
381
+ removed = {"value": False}
382
+
383
+ def _apply(data):
384
+ boundaries = data.get(VIEW_FIELD_BOUNDARIES_KEY)
385
+ if isinstance(boundaries, dict) and view_id in boundaries:
386
+ boundaries.pop(view_id, None)
387
+ removed["value"] = True
388
+ return data
389
+
390
+ _st_for_boundaries(st).update(shares.SHARES_KEY, _apply, flush="async")
391
+ return removed["value"]
392
+
393
+
394
+ def _field_role_sources_from_data(topic, key, me, is_admin, data):
395
+ """Resolve one field from one already-read share document."""
396
+ oid = shares.field_oid(topic, key)
397
+ raw_direct = (data.get("field") or {}).get(oid)
398
+ raw_direct = raw_direct if isinstance(raw_direct, dict) else {}
399
+ direct_rec = {"owner": _uname(raw_direct.get("owner")) or None,
400
+ "entries": shares._clean_entries(raw_direct.get("entries"))}
401
+ direct = shares.role_from(direct_rec, me, is_admin=is_admin)
402
+ if direct == "owner" or is_admin:
403
+ return {"role": "owner", "direct": direct, "views": []}
404
+ view_ids = []
405
+ boundaries = data.get(VIEW_FIELD_BOUNDARIES_KEY) or {}
406
+ view_grants = data.get("view") or {}
407
+ if isinstance(boundaries, dict) and isinstance(view_grants, dict):
408
+ for view_id, boundary in boundaries.items():
409
+ if not isinstance(boundary, dict) or boundary.get("topic") != topic:
410
+ continue
411
+ if key not in {str(value) for value in (boundary.get("fields") or ())}:
412
+ continue
413
+ raw = view_grants.get(str(view_id))
414
+ if not isinstance(raw, dict):
415
+ continue
416
+ rec = {"owner": _uname(raw.get("owner")) or None,
417
+ "entries": shares._clean_entries(raw.get("entries"))}
418
+ if shares.role_from(rec, me, is_admin=False) is not None:
419
+ view_ids.append(str(view_id))
420
+ role = direct if direct in ("edit", "view") else ("view" if view_ids else None)
421
+ return {"role": role, "direct": direct, "views": sorted(view_ids)}
422
+
423
+
424
+ def field_roles_for_keys(grant_topic, field_keys, viewer, *, is_admin=False, st=None):
425
+ """Resolve many field roles from one share-document read."""
426
+ topic = str(grant_topic or "").strip()
427
+ me = _uname(viewer)
428
+ keys = []
429
+ seen = set()
430
+ for raw in field_keys or ():
431
+ key = str(raw or "").strip()
432
+ if key and key not in seen:
433
+ seen.add(key)
434
+ keys.append(key)
435
+ if not topic or not me:
436
+ return {key: {"role": None, "direct": None, "views": []} for key in keys}
437
+ try:
438
+ data = _st_for_boundaries(st).get(shares.SHARES_KEY) or {}
439
+ except Exception: # noqa: BLE001
440
+ data = {}
441
+ return {key: _field_role_sources_from_data(topic, key, me, is_admin, data)
442
+ for key in keys}
443
+
444
+
445
+ def field_role_sources(grant_topic, field_key, viewer, *, is_admin=False, st=None):
446
+ """Resolve one field role and its direct/View provenance.
447
+
448
+ A View role inherits READ access only. ``Can edit`` on a View means editing the View's
449
+ configuration, not taking ownership of or editing a colleague's field definition/value.
450
+ Direct field roles remain stronger and are returned unchanged.
451
+ """
452
+ key = str(field_key or "").strip()
453
+ if not key:
454
+ return {"role": None, "direct": None, "views": []}
455
+ return field_roles_for_keys(
456
+ grant_topic, [key], viewer, is_admin=is_admin, st=st).get(
457
+ key, {"role": None, "direct": None, "views": []})
458
+
459
+
460
+ # Save the unwrapped direct resolver once, including across module reloads in verifier processes.
461
+ _DIRECT_FIELD_ROLE_FOR = getattr(shares.role_for, "_direct_field_role_for", shares.role_for)
462
+
463
+
464
+ def install_view_field_resolver():
465
+ """Install C8 at the existing ``shares.role_for`` wall, once per process.
466
+
467
+ Lane V does not own ``core.shares`` or ``core.perm_scope``. The application imports its
468
+ share router at startup, and that router explicitly installs this owned resolver. Every
469
+ existing caller therefore keeps asking the same wall; only ``kind='field'`` gains the
470
+ View-boundary leg. The original callable remains attached for direct/provenance reads.
471
+ """
472
+ if getattr(shares.role_for, "_view_field_resolver", False):
473
+ return shares.role_for
474
+
475
+ def _role_for(kind, oid, user, is_admin=False, st=None):
476
+ if kind != "field":
477
+ return _DIRECT_FIELD_ROLE_FOR(kind, oid, user, is_admin=is_admin, st=st)
478
+ topic, key = shares.split_field_oid(oid)
479
+ if not topic or not key:
480
+ return None
481
+ return field_role_sources(topic, key, user, is_admin=is_admin, st=st)["role"]
482
+
483
+ _role_for._view_field_resolver = True
484
+ _role_for._direct_field_role_for = _DIRECT_FIELD_ROLE_FOR
485
+ shares.role_for = _role_for
486
+ return _role_for
487
+
488
+
489
  def permissions_from_grants(entries):
490
  """⭐⭐ W40-T03 / CONTRACT C4 — THE CLASSIFICATION IS DERIVED FROM THE REGISTRY, NEVER STAMPED.
491
 
platform/core/grid_events.py CHANGED
@@ -343,6 +343,93 @@ def _field_share_keys(ctx):
343
  return raw, raw
344
 
345
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
346
  def _field_share_role(ctx, key, definition):
347
  """Return the specific Share field role, if this definition is governed by it.
348
 
@@ -1706,6 +1793,7 @@ def handle_one(event, ctx):
1706
  )
1707
  view['name'] = _unique_name(view.get('name'), _view_names)
1708
  ws['views'][view_id] = view
 
1709
  name_changed = view.get('name') != requested_name
1710
  # PERSISTED, but deliberately NO rerun — see the return-value contract in
1711
  # this function's docstring. This is the hot path: every filter, sort,
@@ -1770,6 +1858,8 @@ def handle_one(event, ctx):
1770
  try:
1771
  import core.shares as _shares
1772
  _shares.drop_objects([('view', view_id)], st=getattr(_tops(ctx), 'st', None))
 
 
1773
  except Exception:
1774
  pass
1775
  return True
@@ -1779,11 +1869,18 @@ def handle_one(event, ctx):
1779
  try:
1780
  import core.shares as _shares
1781
  _shares.drop_objects([('view', view_id)], st=getattr(_tops(ctx), 'st', None))
 
 
1782
  except Exception:
1783
  pass
1784
  else:
1785
  _session_ready()
1786
  ws['views'].pop(view_id, None)
 
 
 
 
 
1787
  # Wave-6 item 3: the client removed the view optimistically; a deleted view simply
1788
  # stops being served, so the next render differs in nothing the client still shows.
1789
  return False
 
343
  return raw, raw
344
 
345
 
346
+ def _sync_view_field_inheritance(ctx, view):
347
+ """Apply C8 after a validated View save and notify the field owner/tenant admins.
348
+
349
+ ``ctx.fields`` is already the caller's permission-projected schema. A key the editor cannot
350
+ read therefore never reaches the saved config or this boundary. New personal overlay fields
351
+ are promoted without changing ``createdBy`` so the inherited READ role points at a definition
352
+ the audience can actually receive; database-owned ``ut_*`` fields already live tenant-wide.
353
+ """
354
+ from core import field_permissions as _fp
355
+
356
+ tops = _tops(ctx)
357
+ st = getattr(tops, 'st', None) or ctx.st
358
+ shared_key, grant_topic = _field_share_keys(ctx)
359
+ delta = _fp.sync_view_field_boundary(
360
+ view, grant_topic, ctx.uname, st=st, audit_additions=True)
361
+ added = list(delta.get('added') or ())
362
+ audience = list(delta.get('audience') or ())
363
+ if not audience:
364
+ return delta
365
+
366
+ by_key = {str(field.get('key')): field for field in (ctx.fields or ())
367
+ if isinstance(field, dict) and field.get('key')}
368
+ table_key = str(getattr(tops, 'table_key', '') or '')
369
+ # Check every included key, not only this save's delta. If a transient overlay failure left
370
+ # an edge pointing at a still-personal definition, the audience remains fail-closed and the
371
+ # next View save retries residency instead of treating the boundary as finished forever.
372
+ # Successful promotions are idempotent because the projected schema then reports shared=True.
373
+ included = _fp.view_field_keys(view)
374
+ if not table_key.startswith('ut_'):
375
+ for key in included:
376
+ field = by_key.get(key)
377
+ if not isinstance(field, dict) or field.get('shared') is True:
378
+ continue
379
+ owner = str(field.get('createdBy') or '').strip().lower()
380
+ if not owner or owner != str(ctx.uname or '').strip().lower():
381
+ continue
382
+ try:
383
+ promoted = _fp.promote_field(
384
+ table_key, shared_key, grant_topic, owner, field, st=st)
385
+ by_key[key] = promoted
386
+ except Exception as exc: # noqa: BLE001
387
+ # Fail closed: the boundary alone cannot surface a definition that stayed
388
+ # personal. Report the infrastructure failure without putting field values in
389
+ # logs; every later live-audience save retries the idempotent promotion.
390
+ _tel.error('grid_events:view-field-promotion', exc)
391
+
392
+ if not added:
393
+ return delta
394
+
395
+ # Addition notifications are working-inbox facts, not permission authority. Send one small
396
+ # row per field/recipient and never let an inbox outage roll back an already accepted View.
397
+ try:
398
+ import core.alerts as _alerts
399
+ import core.perms as _perms
400
+ import core.shares as _shares
401
+ import core.users as _users
402
+ registry = _users.registry() or {}
403
+ tenant = str(getattr(st, 'key', '') or '').strip().lower()
404
+ admins = {
405
+ str(username).strip().lower()
406
+ for username, record in registry.items()
407
+ if isinstance(record, dict) and _perms.is_admin(record)
408
+ and (not tenant or str(record.get('tenant') or 'royal-imports').strip().lower() == tenant)
409
+ }
410
+ view_id = str((view or {}).get('id') or '')
411
+ view_name = str((view or {}).get('name') or 'this View')[:160]
412
+ actor = str(ctx.uname or '').strip().lower()
413
+ for key in added:
414
+ field = by_key.get(key) or {}
415
+ direct = _shares.grants(
416
+ 'field', _shares.field_oid(grant_topic, key), st=st)
417
+ owner = str(direct.get('owner') or field.get('createdBy') or '').strip().lower()
418
+ recipients = set(admins)
419
+ if owner:
420
+ recipients.add(owner)
421
+ label = str(field.get('label') or key)[:160]
422
+ for recipient in sorted(recipients - {actor, ''}):
423
+ _alerts.notify(
424
+ recipient, 'Field added to a shared View', topic='share',
425
+ key=f'view-field:{view_id}:{key}', row_id=view_id,
426
+ detail=f'{label} was added to {view_name}. Its audience can now read it.',
427
+ actor=actor, st=st)
428
+ except Exception as exc: # noqa: BLE001
429
+ _tel.error('grid_events:view-field-notification', exc)
430
+ return delta
431
+
432
+
433
  def _field_share_role(ctx, key, definition):
434
  """Return the specific Share field role, if this definition is governed by it.
435
 
 
1793
  )
1794
  view['name'] = _unique_name(view.get('name'), _view_names)
1795
  ws['views'][view_id] = view
1796
+ _sync_view_field_inheritance(ctx, view)
1797
  name_changed = view.get('name') != requested_name
1798
  # PERSISTED, but deliberately NO rerun — see the return-value contract in
1799
  # this function's docstring. This is the hot path: every filter, sort,
 
1858
  try:
1859
  import core.shares as _shares
1860
  _shares.drop_objects([('view', view_id)], st=getattr(_tops(ctx), 'st', None))
1861
+ from core import field_permissions as _fp
1862
+ _fp.drop_view_field_boundary(view_id, st=getattr(_tops(ctx), 'st', None))
1863
  except Exception:
1864
  pass
1865
  return True
 
1869
  try:
1870
  import core.shares as _shares
1871
  _shares.drop_objects([('view', view_id)], st=getattr(_tops(ctx), 'st', None))
1872
+ from core import field_permissions as _fp
1873
+ _fp.drop_view_field_boundary(view_id, st=getattr(_tops(ctx), 'st', None))
1874
  except Exception:
1875
  pass
1876
  else:
1877
  _session_ready()
1878
  ws['views'].pop(view_id, None)
1879
+ try:
1880
+ from core import field_permissions as _fp
1881
+ _fp.drop_view_field_boundary(view_id, st=ctx.st)
1882
+ except Exception:
1883
+ pass
1884
  # Wave-6 item 3: the client removed the view optimistically; a deleted view simply
1885
  # stops being served, so the next render differs in nothing the client still shows.
1886
  return False
platform/core/registry.py CHANGED
@@ -229,6 +229,53 @@ REGISTRY = [
229
  BY_KEY = {m['key']: m for m in REGISTRY}
230
 
231
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
232
  def brand_filterable(key) -> bool:
233
  return bool(BY_KEY.get(key, {}).get('brand'))
234
 
 
229
  BY_KEY = {m['key']: m for m in REGISTRY}
230
 
231
 
232
+ # Source policies - the single source of truth for connector database set-up.
233
+ #
234
+ # The API resolves eligibility from these rows and each source's registered analysis. Adding a
235
+ # source = add one row here + its provider and analysis in the connector's app layer. This is
236
+ # DATA ONLY: core knows neither the mirror nor the API, and imports neither to answer a policy.
237
+ #
238
+ # Fields:
239
+ # key stable source id (matches its registered analysis)
240
+ # label human name
241
+ # prefixes tuple of owned table-key prefixes; the longest matching prefix wins
242
+ # max_rows materialised-home ceiling, or None to use the store default; never an ingest cap
243
+ # window_max whole-table window ceiling, or None to use the datastore default
244
+ # rank_ops "client" or "server": where set-level rank operators are evaluated
245
+ # candidates "declared": only provider-returned grains; "prefix": every owned database
246
+ #
247
+ # None preserves the existing ceilings. Declared candidates preserve the existing grain set;
248
+ # a prefix identifies the policy owner without making an undeclared database read-through.
249
+ SOURCE_POLICIES = [
250
+ {'key': 'odoo', 'label': 'Odoo', 'prefixes': ('ut_odoo_',),
251
+ 'max_rows': None, 'window_max': None, 'rank_ops': 'client', 'candidates': 'declared'},
252
+ {'key': 'amazon', 'label': 'Amazon', 'prefixes': ('ut_amazon_',),
253
+ 'max_rows': None, 'window_max': None, 'rank_ops': 'client', 'candidates': 'declared'},
254
+ {'key': 'meta_ads', 'label': 'Meta Ads', 'prefixes': ('ut_meta_',),
255
+ 'max_rows': None, 'window_max': None, 'rank_ops': 'client', 'candidates': 'declared'},
256
+ ]
257
+ DEFAULT_SOURCE_POLICY = {
258
+ 'key': 'default', 'label': 'Default', 'prefixes': (),
259
+ 'max_rows': None, 'window_max': None, 'rank_ops': 'client', 'candidates': 'declared',
260
+ }
261
+
262
+
263
+ def source_policy(table_key):
264
+ """The source row owning this key, or DEFAULT_SOURCE_POLICY when no prefix matches."""
265
+ key = str(table_key or '')
266
+ match, longest = DEFAULT_SOURCE_POLICY, -1
267
+ for row in SOURCE_POLICIES:
268
+ for prefix in row['prefixes']:
269
+ if key.startswith(prefix) and len(prefix) > longest:
270
+ match, longest = row, len(prefix)
271
+ return match
272
+
273
+
274
+ def source_policies():
275
+ """The declared source rows, in declaration order; the fallback row is separate."""
276
+ return tuple(SOURCE_POLICIES)
277
+
278
+
279
  def brand_filterable(key) -> bool:
280
  return bool(BY_KEY.get(key, {}).get('brand'))
281
 
platform/core/sheet_binding.py CHANGED
@@ -1,18 +1,134 @@
1
- """Pure document cleaners shared by Sheet storage and Sheet query resolution.
2
 
3
- These functions decide only what binding metadata may be stored. They do not resolve data,
4
- permissions, or SQL. Keeping them in core lets the workbook document validate view bindings
5
- without importing the higher ``modules.sheet`` layer.
6
  """
7
 
8
  from __future__ import annotations
9
 
10
  import datetime
 
 
11
  import re
 
12
 
13
  from core import windows as W
14
 
15
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
16
  PERIOD_UNITS = ("day", "week", "month", "quarter", "year")
17
  PERIOD_START = re.compile(r"^\d{4}-(0[1-9]|1[0-2])$")
18
  PERIOD_DAY = re.compile(r"^\d{4}-(0[1-9]|1[0-2])-(0[1-9]|[12]\d|3[01])$")
@@ -21,7 +137,9 @@ DRAG_ACROSS = ("period", "none")
21
  DRAG_DOWN_NONE = "none"
22
 
23
  _MESSAGES = {
 
24
  "period_invalid": "{subject} is not a period this cell can use. Choose a valid start for the selected period unit.",
 
25
  "ending_invalid": "Ending {subject} cannot be used. Choose a valid ending date.",
26
  "ending_not_supported": "{subject} does not support an Ending. Choose year to date, last year to date, this month or this quarter.",
27
  "ending_cell_unreadable": "Ending from cell {subject} cannot be read. Use a cell containing a date in YYYY-MM-DD format.",
@@ -44,6 +162,58 @@ class BindingMetadataRefusal(Exception):
44
  f"cannot be {verb}.", f"cannot be {verb} because {self.detail}")
45
 
46
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
47
  def clean_period(raw):
48
  """Return one canonical Sheet period, or ``None`` when no window was named."""
49
  if not isinstance(raw, dict):
@@ -135,7 +305,10 @@ def clean_period(raw):
135
  out["ref"] = ref
136
  if (raw.get("window") is not None and "unit" not in out and "start" not in out
137
  and not out.get("frozen")):
138
- out["window"] = raw["window"]
 
 
 
139
  if not ({"start", "window", "ref", "frozen"} & set(out)):
140
  return None
141
  return out
 
1
+ """Binding metadata and server-owned Refresh snapshots shared by Sheet storage and routes.
2
 
3
+ Cleaners remain independent of query resolution. Change-token readers fingerprint source
4
+ state without assembling a database or evaluating an individual cell binding.
 
5
  """
6
 
7
  from __future__ import annotations
8
 
9
  import datetime
10
+ import hashlib
11
+ import json
12
  import re
13
+ from copy import deepcopy
14
 
15
  from core import windows as W
16
 
17
 
18
+ def fingerprint(value):
19
+ return hashlib.sha256(json.dumps(value, sort_keys=True, default=str,
20
+ separators=(",", ":")).encode()).hexdigest()
21
+
22
+
23
+ def database_change_token(st, database):
24
+ """Read a database fingerprint, without assembling rows or resolving a binding.
25
+
26
+ Authored databases share one JSON document. Its bucket revision cannot distinguish
27
+ siblings, so Postgres hashes the selected member IN the database and returns only a
28
+ token and row-free definition. This also sees connector writers which bypass the
29
+ ordinary row/field API. The HF adapter hashes the same documents locally.
30
+ """
31
+ from core import connector_sync, schema_overlay, store_pg, user_tables
32
+ workspace = {"customer_data": "customer_table_workspace",
33
+ "product_data": "product_table_workspace"}.get(
34
+ database, database + "_table_workspace")
35
+ shared = database if database.startswith("ut_") else workspace
36
+ buckets = [workspace, shared + "__shared"]
37
+ handle = st._store() if callable(getattr(st, "_store", None)) else st
38
+ if isinstance(handle, store_pg.PgStore):
39
+ key = getattr(st, "store_key", str)
40
+ # Values stay on the server. No row JSON crosses the connection on this path.
41
+ with store_pg._pool().connection() as con:
42
+ row = con.execute(store_pg._q(
43
+ "WITH source AS (SELECT value -> %s AS d FROM {tbl} WHERE key = %s) "
44
+ "SELECT md5(COALESCE((SELECT d::text FROM source), 'null') || "
45
+ "COALESCE((SELECT jsonb_object_agg(key, rev ORDER BY key)::text "
46
+ "FROM {tbl} WHERE key = ANY(%s)), 'null') || "
47
+ "COALESCE((SELECT (value -> %s)::text FROM {tbl} WHERE key = %s), 'null') || "
48
+ "COALESCE((SELECT (value -> %s)::text FROM {tbl} WHERE key = %s), 'null') || "
49
+ "COALESCE((SELECT jsonb_object_agg(e.key, e.value ORDER BY e.key)::text "
50
+ "FROM {tbl} kv, jsonb_each(kv.value) e WHERE kv.key = %s "
51
+ "AND e.value ->> 'databaseKey' = %s), 'null') || "
52
+ "COALESCE((SELECT jsonb_path_query_array(value, "
53
+ "'$.*.* ? (@.databases[*] == $db)', jsonb_build_object('db', %s::text))::text "
54
+ "FROM {tbl} WHERE key = %s), 'null')), "
55
+ "(SELECT d - 'rows' FROM source)", "store_kv", handle.tenant_slug),
56
+ (database, key(user_tables.STORE_KEY), [key(b) for b in buckets],
57
+ database, key(user_tables.ZONE_STORE_KEY), database, key(schema_overlay.STORE_KEY),
58
+ key(connector_sync.JOBS_KEY), database,
59
+ database, key(connector_sync.STATE_KEY))).fetchone()
60
+ token, definition = row[0], row[1] or {}
61
+ else:
62
+ definition = (st.get(user_tables.STORE_KEY, fresh=True) or {}).get(database) or {}
63
+ jobs = st.get(connector_sync.JOBS_KEY, fresh=True) or {}
64
+ states = st.get(connector_sync.STATE_KEY, fresh=True) or {}
65
+ token = fingerprint([definition, *(st.get(b, fresh=True) for b in buckets),
66
+ (st.get(user_tables.ZONE_STORE_KEY, fresh=True) or {}).get(database),
67
+ (st.get(schema_overlay.STORE_KEY, fresh=True) or {}).get(database),
68
+ {k: v for k, v in jobs.items() if v.get("databaseKey") == database},
69
+ [v for provider in sorted(states) for _, v in sorted(states[provider].items())
70
+ if database in (v.get("databases") or [])]])
71
+ # Registry topics and read-through connectors live in the observation plane. Sync
72
+ # receipts have second precision, so also fingerprint source content. A file-wide
73
+ # mtime would invalidate every database when just one source changed.
74
+ mirror = not definition or not user_tables.materialises(database, defn=definition)
75
+ if mirror:
76
+ token = fingerprint([token, mirror_change_token(st, database)])
77
+ return token
78
+
79
+
80
+ #: ⛔⛔ THE MIRROR TOKEN IS PROVIDED FROM ABOVE, NOT IMPORTED FROM BELOW (SDLC 65 combine).
81
+ #: S65-T03 wrote `mirror_change_token` here and it reached for `harness.datastore` and
82
+ #: `harness.runtime` -- a NEW `core -> harness` edge, which `verify_layering.py` (D-460) caught as
83
+ #: the wave's own violation of ARCHITECTURE.md section 1. ⛔ It is FIXED rather than grandfathered:
84
+ #: `GRANDFATHERED_CORE` is the list of edges that PRE-DATE that gate, and adding this wave's own
85
+ #: edge to it would be granting an exemption to the wave that created it.
86
+ #: The body moved to `harness/mirror_tokens.py`, which registers itself here on import, so the
87
+ #: dependency points DOWN the way every other layer does.
88
+ _MIRROR_TOKEN_PROVIDER = None
89
+
90
+
91
+ def set_mirror_token_provider(fn):
92
+ """Register the mirror reader. Called by `harness.mirror_tokens` at import."""
93
+ global _MIRROR_TOKEN_PROVIDER
94
+ _MIRROR_TOKEN_PROVIDER = fn
95
+
96
+
97
+ def mirror_change_token(st, database):
98
+ """One SQL token read over source tables, delegated to the layer that owns the mirror.
99
+
100
+ ⚠ NO PROVIDER IS A RAISE, DELIBERATELY, and it is the safe direction: `database_change_token`'s
101
+ contract is that a token which cannot be READ resolves rather than reusing a value, so an
102
+ unregistered provider costs a recompute and can never serve a stale number.
103
+ """
104
+ if _MIRROR_TOKEN_PROVIDER is None:
105
+ raise RuntimeError("No mirror change-token provider is registered; import "
106
+ "harness.mirror_tokens before asking for one.")
107
+ return _MIRROR_TOKEN_PROVIDER(st, database)
108
+
109
+
110
+ def read_refresh_cache(st, workbook_id):
111
+ from core import sheets
112
+ return st.get(sheets._scope_cache_key(workbook_id), fresh=True) or {}
113
+
114
+
115
+ def save_refresh_cache(st, workbook_id, entries, databases):
116
+ """Server-only snapshots; ordinary workbook PUT/PATCH cannot manufacture these slots."""
117
+ from core import sheets
118
+
119
+ def apply(doc):
120
+ cells = doc.setdefault("refreshCells", {})
121
+ for cell_key, scope_key, entry in entries:
122
+ slots = cells.setdefault(cell_key, {})
123
+ if scope_key not in slots and len(slots) >= sheets._MAX_SCOPE_ENTRIES_PER_CELL:
124
+ slots.pop(next(iter(slots)))
125
+ slots[scope_key] = deepcopy(entry)
126
+ sheets._merge_data_origins(doc, databases=databases)
127
+ return doc, None
128
+
129
+ sheets._persist(st, sheets._scope_cache_key(workbook_id), apply)
130
+
131
+
132
  PERIOD_UNITS = ("day", "week", "month", "quarter", "year")
133
  PERIOD_START = re.compile(r"^\d{4}-(0[1-9]|1[0-2])$")
134
  PERIOD_DAY = re.compile(r"^\d{4}-(0[1-9]|1[0-2])-(0[1-9]|[12]\d|3[01])$")
 
137
  DRAG_DOWN_NONE = "none"
138
 
139
  _MESSAGES = {
140
+ "related_invalid": "Choose one declared relationship and a valid filter tree for this cell.",
141
  "period_invalid": "{subject} is not a period this cell can use. Choose a valid start for the selected period unit.",
142
+ "period_window_invalid": "This cell's lookback window is invalid. Choose a valid window in the period control.",
143
  "ending_invalid": "Ending {subject} cannot be used. Choose a valid ending date.",
144
  "ending_not_supported": "{subject} does not support an Ending. Choose year to date, last year to date, this month or this quarter.",
145
  "ending_cell_unreadable": "Ending from cell {subject} cannot be read. Use a cell containing a date in YYYY-MM-DD format.",
 
162
  f"cannot be {verb}.", f"cannot be {verb} because {self.detail}")
163
 
164
 
165
+ def clean_related(raw):
166
+ """One declared edge key and a structurally valid, losslessly copied filter tree.
167
+
168
+ Absence is handled by the caller. An explicitly supplied null is malformed, as are
169
+ extra relationship bags and nested relationships. Operator support belongs to the
170
+ query compiler, which reports unpushable conditions rather than dropping metadata.
171
+ """
172
+ def refuse():
173
+ raise BindingMetadataRefusal("related_invalid")
174
+
175
+ if not isinstance(raw, dict) or set(raw) != {"via", "filter"}:
176
+ refuse()
177
+ via = raw["via"]
178
+ if not isinstance(via, str) or not via.strip():
179
+ refuse()
180
+
181
+ def reject_nested(value):
182
+ if isinstance(value, dict):
183
+ if "related" in value:
184
+ refuse()
185
+ for child in value.values():
186
+ reject_nested(child)
187
+ elif isinstance(value, list):
188
+ for child in value:
189
+ reject_nested(child)
190
+
191
+ reject_nested(raw["filter"])
192
+
193
+ def nodes_ok(nodes):
194
+ if not isinstance(nodes, list):
195
+ refuse()
196
+ for node in nodes:
197
+ if not isinstance(node, dict) or "related" in node:
198
+ refuse()
199
+ if "children" in node:
200
+ if ("colId" in node or node.get("conj", "and") not in ("and", "or")):
201
+ refuse()
202
+ nodes_ok(node["children"])
203
+ elif (not isinstance(node.get("colId"), str) or not node["colId"].strip()
204
+ or not isinstance(node.get("op"), str) or not node["op"].strip()):
205
+ refuse()
206
+
207
+ tree = raw["filter"]
208
+ if isinstance(tree, dict):
209
+ if set(tree) - {"nodes", "conj"} or tree.get("conj", "and") not in ("and", "or"):
210
+ refuse()
211
+ nodes_ok(tree.get("nodes"))
212
+ else:
213
+ nodes_ok(tree)
214
+ return {"via": via.strip(), "filter": deepcopy(tree)}
215
+
216
+
217
  def clean_period(raw):
218
  """Return one canonical Sheet period, or ``None`` when no window was named."""
219
  if not isinstance(raw, dict):
 
305
  out["ref"] = ref
306
  if (raw.get("window") is not None and "unit" not in out and "start" not in out
307
  and not out.get("frozen")):
308
+ window = W.normalize(raw["window"])
309
+ if window is None:
310
+ raise BindingMetadataRefusal("period_window_invalid")
311
+ out["window"] = window
312
  if not ({"start", "window", "ref", "frozen"} & set(out)):
313
  return None
314
  return out
platform/core/sheet_compat.py ADDED
@@ -0,0 +1,875 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Excel-shaped automation compatibility for Loopable Sheet.
2
+
3
+ This module is the one capability declaration and the one operation planner used by both the
4
+ HTTP door and the Python SDK. It deliberately models the bounded workbook operations exercised
5
+ by the refresh-sales script and the refresh-fpa operating procedure. Excel process, window, VBA
6
+ and modal-UI controls are inventoried too, but always refuse before the tenant store callback.
7
+
8
+ Writes are operation-level compare-and-swap patches: every write needs the revision the caller
9
+ read, and only the addressed cells or worksheet/workbook property is changed. This is not a
10
+ second workbook persistence implementation; the final callback uses ``core.sheets``' store seam.
11
+ """
12
+
13
+ from __future__ import annotations
14
+
15
+ import base64
16
+ import copy
17
+ import json
18
+ import math
19
+ import os
20
+ import re
21
+ import tempfile
22
+ from core import write_execution
23
+
24
+ from core import sheets
25
+
26
+
27
+ MANIFEST_VERSION = "2026-09-19"
28
+
29
+
30
+ def _cap(name, supported, signature, semantics, evidence, *, access="read", reason=""):
31
+ row = {
32
+ "name": name,
33
+ "supported": bool(supported),
34
+ "access": access if supported else "unsupported",
35
+ "signature": signature,
36
+ "semantics": semantics if supported else "Refuses before any workbook store update.",
37
+ "evidence": list(evidence),
38
+ }
39
+ if not supported:
40
+ row["unsupportedReason"] = reason
41
+ return row
42
+
43
+
44
+ _SALES = ".agents/skills/refresh-sales/scripts/refresh_daily_sales_performance.py"
45
+ _FPA = ".agents/skills/refresh-fpa/SKILL.md"
46
+
47
+
48
+ # The evidence references are intentionally literal and machine-readable. Section 26 of the
49
+ # Sheet gate re-censuses the source operations and requires every token to land on one row here.
50
+ _OPERATIONS = (
51
+ _cap("workbooks.open", True, "Open(id_or_name, UpdateLinks=0, ReadOnly=False)",
52
+ "Looks up one existing workbook by stable id, then exact case-insensitive name.",
53
+ [f"{_SALES}:421", f"{_FPA}:153"]),
54
+ _cap("worksheets.item", True, "Worksheets(name_or_id)",
55
+ "Looks up one worksheet by id, exact name, then case-insensitive name.",
56
+ [f"{_SALES}:425-426"]),
57
+ _cap("worksheet.range", True, "Range(A1 | A1:B9 | Cell, Cell)",
58
+ "Builds one inclusive rectangular A1 range inside Excel worksheet limits.",
59
+ [f"{_SALES}:392", f"{_SALES}:621-624"]),
60
+ _cap("worksheet.cells", True, "Cells(row, column)",
61
+ "Addresses one one-based cell and composes with Range.",
62
+ [f"{_SALES}:438", f"{_SALES}:581"]),
63
+ _cap("range.values.read", True, "Range.Value | Range.Value2",
64
+ "Returns a scalar for one cell or a row-major matrix for a rectangle; formulas return cached values.",
65
+ [f"{_SALES}:466", f"{_SALES}:584"]),
66
+ _cap("range.values.write", True, "Range.Value = scalar | matrix",
67
+ "Writes only addressed cell contents, retaining each cell's style and unrelated metadata.",
68
+ [f"{_SALES}:438", f"{_SALES}:624"], access="write"),
69
+ _cap("range.formulas.read", True, "Range.Formula | Range.FormulaR1C1",
70
+ "Returns authored formulas; FormulaR1C1 returns the preserved R1C1 source when present.",
71
+ [f"{_SALES}:393"]),
72
+ _cap("range.formulas.write", True, "Range.Formula[ R1C1] = scalar | matrix",
73
+ "Writes formulas only in the rectangle; R1C1 references are translated at each destination.",
74
+ [f"{_SALES}:403", f"{_FPA}:177-179"], access="write"),
75
+ _cap("range.styles.read", True, "Range.Style",
76
+ "Returns each cell's canonical style bag without interpreting unrelated cell content.",
77
+ [f"{_SALES}:433-436", f"{_FPA}:112" ]),
78
+ _cap("range.styles.write", True, "Range.Style = style | matrix",
79
+ "Replaces only the addressed cells' canonical style bags.",
80
+ [f"{_SALES}:433-436", f"{_FPA}:112"], access="write"),
81
+ _cap("range.number_formats.read", True, "Range.NumberFormat",
82
+ "Returns Excel format-code strings from canonical cell styles.",
83
+ [f"{_SALES}:10", f"{_FPA}:177" ]),
84
+ _cap("range.number_formats.write", True, "Range.NumberFormat = code | matrix",
85
+ "Changes only style.format on addressed cells; General removes the explicit format.",
86
+ [f"{_SALES}:10", f"{_FPA}:177"], access="write"),
87
+ _cap("range.fill_down", True, "Range.FillDown()",
88
+ "Copies the top row down inside the rectangle and translates A1/R1C1 formulas per row.",
89
+ [f"{_SALES}:404", f"{_FPA}:177-179"], access="write"),
90
+ _cap("range.clear_contents", True, "Range.ClearContents()",
91
+ "Clears values and formulas in the rectangle while retaining styles and worksheet properties.",
92
+ [f"{_SALES}:433-436", f"{_FPA}:139-143"], access="write"),
93
+ _cap("range.replace", True, "Range.Replace(What, Replacement, LookAt=xlWhole, MatchCase=False)",
94
+ "Replaces exact stored literal values only; formulas and partial strings are never rewritten.",
95
+ [f"{_SALES}:672-680"], access="write"),
96
+ _cap("range.copy", True, "target.CopyFrom(source, mode='all'|'values'|'formulas')",
97
+ "Copies equal-sized rectangles and translates formulas; values mode hardcodes cached values.",
98
+ [f"{_FPA}:121-130", f"{_FPA}:177-179"], access="write"),
99
+ _cap("range.paste_values", True, "target.PasteSpecial(xlPasteValues, source=source)",
100
+ "Hardcodes cached source values into an equal-sized target without an ambient process clipboard.",
101
+ [f"{_FPA}:177-179", f"{_FPA}:194-199"], access="write"),
102
+ _cap("worksheet.used_range", True, "Worksheet.UsedRange",
103
+ "Returns the inclusive true used rectangle, or A1 for an empty sheet.",
104
+ [f"{_SALES}:466", f"{_SALES}:656"]),
105
+ _cap("range.end_up", True, "Cells(Rows.Count, column).End(xlUp)",
106
+ "Returns the last nonblank row in the selected column, or row 1 when empty.",
107
+ [f"{_SALES}:581", f"{_SALES}:617"]),
108
+ _cap("worksheet.count_a", True, "WorksheetFunction.CountA(range)",
109
+ "Counts nonblank stored values or formulas inside the rectangle.",
110
+ [f"{_SALES}:457-463", f"{_SALES}:630"]),
111
+ _cap("worksheet.count_if", True, "WorksheetFunction.CountIf(range, criterion)",
112
+ "Supports the workflow's exact-cell, case-insensitive criterion.",
113
+ [f"{_SALES}:656"]),
114
+ _cap("worksheet.geometry.read", True, "Range.ColumnWidth | Range.RowHeight",
115
+ "Reads canonical per-column widths and per-row heights with worksheet defaults.",
116
+ [f"{_SALES}:10", f"{_FPA}:121-130"]),
117
+ _cap("worksheet.geometry.write", True, "Range.ColumnWidth|RowHeight = number",
118
+ "Changes only columns or rows intersecting the addressed range.",
119
+ [f"{_SALES}:10", f"{_FPA}:121-130"], access="write"),
120
+ _cap("worksheet.hidden.read", True, "Range.EntireRow.Hidden | EntireColumn.Hidden",
121
+ "Reads hidden state for rows or columns intersecting the addressed range.",
122
+ [f"{_SALES}:10"]),
123
+ _cap("worksheet.hidden.write", True, "Range.EntireRow|EntireColumn.Hidden = bool",
124
+ "Changes hidden state only for rows or columns intersecting the addressed range.",
125
+ [f"{_SALES}:10"], access="write"),
126
+ _cap("worksheet.freeze.read", True, "Worksheet.FreezePanes",
127
+ "Returns canonical frozen row and column counts.",
128
+ [f"{_SALES}:932-935"]),
129
+ _cap("worksheet.freeze.write", True, "Worksheet.FreezePanes = {rows, cols}",
130
+ "Changes only the worksheet freeze counts.",
131
+ [f"{_SALES}:932-935"], access="write"),
132
+ _cap("worksheet.gridlines.read", True, "Worksheet.DisplayGridlines",
133
+ "Reads the worksheet view's gridline flag, defaulting to shown.",
134
+ [f"{_SALES}:932-935"]),
135
+ _cap("worksheet.gridlines.write", True, "Worksheet.DisplayGridlines = bool",
136
+ "Changes only the worksheet view's gridline flag.",
137
+ [f"{_SALES}:932-935"], access="write"),
138
+ _cap("application.calculation.read", True, "Application.Calculation",
139
+ "Reads manual or automatic mode from workbook calculation metadata.",
140
+ [f"{_SALES}:423", f"{_FPA}:146-153"]),
141
+ _cap("application.calculation.write", True, "Application.Calculation = manual|automatic",
142
+ "Changes only workbook calculation mode and requires expected revision.",
143
+ [f"{_SALES}:423", f"{_SALES}:444", f"{_FPA}:151-157"], access="write"),
144
+ _cap("workbook.calculate_full", True, "Application.CalculateFull()",
145
+ "Runs the host formula evaluator over the workbook and persists refreshed formula caches.",
146
+ [f"{_SALES}:447", f"{_FPA}:187", f"{_FPA}:216"], access="write"),
147
+ _cap("workbook.refresh_all", True, "Workbook.RefreshAll()",
148
+ "Runs the same bounded calculation callback and stamps refreshed_at.",
149
+ [f"{_SALES}:447", f"{_FPA}:233"], access="write"),
150
+ _cap("workbook.names.item", True, "Workbook.Names(name)",
151
+ "Reads one workbook named range without resolving external names.",
152
+ [f"{_SALES}:63-66", f"{_FPA}:177"]),
153
+ _cap("workbook.names.write", True, "Workbook.Names.Add(name, refers_to)",
154
+ "Upserts one workbook-scoped A1 named range and preserves every other name.",
155
+ [f"{_SALES}:63-66", f"{_FPA}:177"], access="write"),
156
+ _cap("workbook.export_xlsx", True, "Workbook.Export(path|bytes)",
157
+ "Exports a data-free xlsx using the existing lossless Sheet export seam; it never runs VBA.",
158
+ [f"{_SALES}:506-538", f"{_FPA}:233"]),
159
+ _cap("workbook.save", True, "Workbook.Save()",
160
+ "Returns the current revision because Loopable operation writes are already atomic and durable.",
161
+ [f"{_SALES}:469", f"{_FPA}:157", f"{_FPA}:233"]),
162
+ _cap("workbook.read_only", True, "Workbook.ReadOnly",
163
+ "Returns false after the route's editor wall; the facade never bypasses authorization.",
164
+ [f"{_FPA}:160"]),
165
+
166
+ _cap("application.dispatch_ex", False, "DispatchEx('Excel.Application')", "", [f"{_SALES}:412"],
167
+ reason="Loopable has no local Excel process; choose the backend at the client boundary."),
168
+ _cap("workbooks.add", False, "Workbooks.Add()", "", [f"{_FPA}:147-153"],
169
+ reason="An Excel scratch workbook exists only to bind process-global calculation mode."),
170
+ _cap("application.visible", False, "Application.Visible = bool", "", [f"{_SALES}:413", f"{_SALES}:935"],
171
+ reason="Browser visibility is not a workbook data operation."),
172
+ _cap("application.window_state", False, "Application.WindowState = value", "", [f"{_SALES}:936"],
173
+ reason="Window maximization is local UI orchestration."),
174
+ _cap("application.display_alerts", False, "Application.DisplayAlerts = bool", "", [f"{_SALES}:414", f"{_SALES}:937"],
175
+ reason="Loopable has no Excel modal-alert process state."),
176
+ _cap("application.screen_updating", False, "Application.ScreenUpdating = bool", "", [f"{_SALES}:415"],
177
+ reason="Loopable writes documents without an Excel repaint loop."),
178
+ _cap("application.enable_events", False, "Application.EnableEvents = bool", "", [f"{_SALES}:416", f"{_SALES}:938"],
179
+ reason="Excel process events are not Sheet document data."),
180
+ _cap("application.automation_security", False, "Application.AutomationSecurity = value", "", [f"{_SALES}:417", f"{_SALES}:940"],
181
+ reason="Loopable never executes workbook macros."),
182
+ _cap("workbook.autosave_on", False, "Workbook.AutoSaveOn = bool", "", [f"{_SALES}:341-351"],
183
+ reason="Loopable writes are atomic store operations, not OneDrive AutoSave sessions."),
184
+ _cap("workbook.vbproject", False, "Workbook.VBProject", "", [f"{_SALES}:898"],
185
+ reason="VBA projects are preserved by import metadata but never exposed for execution or editing."),
186
+ _cap("vba.code_module", False, "CodeModule.Lines|DeleteLines|AddFromString", "", [f"{_SALES}:904-918"],
187
+ reason="Editing executable VBA is outside the Sheet document capability boundary."),
188
+ _cap("application.run", False, "Application.Run(macro, *args)", "", [f"{_SALES}:963-965"],
189
+ reason="Loopable never runs VBA; use workbook.export_xlsx for the supported export."),
190
+ _cap("application.get_save_as_filename", False, "Application.GetSaveAsFilename(...) ", "", [f"{_SALES}:806-863"],
191
+ reason="A modal file picker is browser/UI orchestration."),
192
+ _cap("application.msg_box", False, "MsgBox(message)", "", [f"{_SALES}:865-885"],
193
+ reason="A modal VBA message box is not a workbook data operation."),
194
+ _cap("workbook.close", False, "Workbook.Close(SaveChanges=False)", "", [f"{_SALES}:477", f"{_FPA}:160"],
195
+ reason="Loopable workbooks are request-scoped resources, not local process handles."),
196
+ _cap("application.quit", False, "Application.Quit()", "", [f"{_SALES}:478"],
197
+ reason="Loopable has no Excel process to terminate."),
198
+ )
199
+
200
+
201
+ CAPABILITY_MANIFEST = {
202
+ "schema": 1,
203
+ "version": MANIFEST_VERSION,
204
+ "hierarchy": ["Application", "Workbooks", "Workbook", "Worksheets", "Worksheet", "Range", "Cells"],
205
+ "inventory": {
206
+ "method": "mechanical Python attribute-call census plus the refresh-fpa operating workflow evidence",
207
+ "sources": [_SALES, _FPA],
208
+ "supported": sum(1 for row in _OPERATIONS if row["supported"]),
209
+ "unsupported": sum(1 for row in _OPERATIONS if not row["supported"]),
210
+ },
211
+ "operations": list(_OPERATIONS),
212
+ }
213
+
214
+ CAPABILITY_MANIFEST_BYTES = json.dumps(
215
+ CAPABILITY_MANIFEST, sort_keys=True, separators=(",", ":"), ensure_ascii=False
216
+ ).encode("utf-8")
217
+
218
+ _BY_NAME = {row["name"]: row for row in _OPERATIONS}
219
+
220
+
221
+ class CompatibilityRefusal(sheets.SheetRefusal):
222
+ """A named compatibility refusal suitable for the existing Sheet HTTP envelope."""
223
+
224
+
225
+ def capability_manifest():
226
+ return copy.deepcopy(CAPABILITY_MANIFEST)
227
+
228
+
229
+ def capability_manifest_bytes():
230
+ return CAPABILITY_MANIFEST_BYTES
231
+
232
+
233
+ def operation_spec(name):
234
+ return copy.deepcopy(_BY_NAME.get(str(name or "")))
235
+
236
+
237
+ def require_supported(name):
238
+ key = str(name or "").strip()
239
+ spec = _BY_NAME.get(key)
240
+ if spec is None:
241
+ raise CompatibilityRefusal(
242
+ "sheet_compat_operation_unknown", "That Sheet automation operation is not in the capability manifest.")
243
+ if not spec["supported"]:
244
+ raise CompatibilityRefusal(
245
+ "sheet_compat_operation_unsupported",
246
+ key + " is unsupported: " + spec["unsupportedReason"])
247
+ return spec
248
+
249
+
250
+ _CELL = re.compile(r"^\$?([A-Z]{1,3})\$?([1-9][0-9]{0,6})$")
251
+ _RECT = re.compile(r"^\$?([A-Z]{1,3})\$?([1-9][0-9]{0,6})(?::\$?([A-Z]{1,3})\$?([1-9][0-9]{0,6}))?$")
252
+
253
+
254
+ def _col_number(letters):
255
+ value = 0
256
+ for letter in str(letters).upper():
257
+ value = value * 26 + ord(letter) - 64
258
+ return value
259
+
260
+
261
+ def _letters(number):
262
+ out = ""
263
+ value = int(number)
264
+ while value:
265
+ value, rem = divmod(value - 1, 26)
266
+ out = chr(65 + rem) + out
267
+ return out
268
+
269
+
270
+ def _addr(row, col):
271
+ return _letters(col) + str(row)
272
+
273
+
274
+ def _range(raw):
275
+ text = str(raw or "").strip().upper().replace(" ", "")
276
+ found = _RECT.fullmatch(text)
277
+ if not found:
278
+ raise CompatibilityRefusal("sheet_compat_range_invalid", "A Sheet automation range needs one rectangular A1 address.")
279
+ c0, r0 = _col_number(found.group(1)), int(found.group(2))
280
+ c1, r1 = _col_number(found.group(3) or found.group(1)), int(found.group(4) or found.group(2))
281
+ if c0 > sheets.DOCUMENT_MAX_COLS or c1 > sheets.DOCUMENT_MAX_COLS \
282
+ or r0 > sheets.DOCUMENT_MAX_ROWS or r1 > sheets.DOCUMENT_MAX_ROWS:
283
+ raise CompatibilityRefusal("sheet_compat_range_invalid", "That Sheet automation range is outside Excel's dimensions.")
284
+ c0, c1 = sorted((c0, c1))
285
+ r0, r1 = sorted((r0, r1))
286
+ if (c1 - c0 + 1) * (r1 - r0 + 1) > sheets._MAX_CELLS_PER_SHEET:
287
+ raise CompatibilityRefusal("sheet_too_many_cells", "That Sheet automation range is larger than one operation accepts.")
288
+ return r0, c0, r1, c1
289
+
290
+
291
+ def _range_text(rect):
292
+ r0, c0, r1, c1 = rect
293
+ first, last = _addr(r0, c0), _addr(r1, c1)
294
+ return first if first == last else first + ":" + last
295
+
296
+
297
+ def _addresses(rect):
298
+ r0, c0, r1, c1 = rect
299
+ return [[_addr(row, col) for col in range(c0, c1 + 1)] for row in range(r0, r1 + 1)]
300
+
301
+
302
+ def _matrix(value, height, width, what, *, scalar=True):
303
+ if scalar and not isinstance(value, (list, tuple)):
304
+ return [[copy.deepcopy(value) for _ in range(width)] for _ in range(height)]
305
+ if not isinstance(value, (list, tuple)):
306
+ raise CompatibilityRefusal("sheet_compat_matrix_invalid", what + " needs a row-major matrix.")
307
+ rows = list(value)
308
+ if height == 1 and len(rows) == width and (not rows or not isinstance(rows[0], (list, tuple))):
309
+ rows = [rows]
310
+ if len(rows) != height or any(not isinstance(row, (list, tuple)) or len(row) != width for row in rows):
311
+ raise CompatibilityRefusal(
312
+ "sheet_compat_matrix_shape", what + " must have exactly " + str(height) + " rows and " + str(width) + " columns.")
313
+ return [[copy.deepcopy(item) for item in row] for row in rows]
314
+
315
+
316
+ def _tab(book, ref):
317
+ return sheets.find_tab(book, ref)
318
+
319
+
320
+ def _cell_value(cell):
321
+ if not isinstance(cell, dict) or cell.get("kind") == "empty":
322
+ return None
323
+ return cell.get("value", cell.get("text"))
324
+
325
+
326
+ def _display_matrix(tab, rect, getter):
327
+ grid = []
328
+ cells = tab.get("cells") or {}
329
+ for row in _addresses(rect):
330
+ grid.append([copy.deepcopy(getter(cells.get(addr))) for addr in row])
331
+ return grid[0][0] if len(grid) == len(grid[0]) == 1 else grid
332
+
333
+
334
+ def _keep(cell, keys=("style", "validation", "note", "excel", "link", "sparkline")):
335
+ return {key: copy.deepcopy(cell[key]) for key in keys if isinstance(cell, dict) and key in cell}
336
+
337
+
338
+ def _literal(value, prior):
339
+ if value is None:
340
+ held = _keep(prior)
341
+ return {"kind": "empty", "state": "typed", **held} if held else None
342
+ cell = {"kind": "literal", "state": "typed", "value": copy.deepcopy(value), **_keep(prior)}
343
+ if isinstance(value, str):
344
+ cell["text"] = value
345
+ return sheets.clean_cell(cell)
346
+
347
+
348
+ _R1C1 = re.compile(r"(?<![A-Z0-9_])R(?:(\[[-+]?\d+\])|(\d+))?C(?:(\[[-+]?\d+\])|(\d+))?", re.I)
349
+
350
+
351
+ def _r1c1_to_a1(formula, row, col):
352
+ def replace(found):
353
+ rr = row + int(found.group(1)[1:-1]) if found.group(1) else int(found.group(2) or row)
354
+ cc = col + int(found.group(3)[1:-1]) if found.group(3) else int(found.group(4) or col)
355
+ if rr < 1 or cc < 1 or rr > sheets.DOCUMENT_MAX_ROWS or cc > sheets.DOCUMENT_MAX_COLS:
356
+ raise CompatibilityRefusal("sheet_compat_formula_reference", "That R1C1 formula points outside the worksheet.")
357
+ return _addr(rr, cc)
358
+ return _R1C1.sub(replace, str(formula or ""))
359
+
360
+
361
+ def _formula(value, prior, addr, *, r1c1=False):
362
+ if value in (None, ""):
363
+ return _literal(None, prior)
364
+ text = str(value)
365
+ if not text.startswith("="):
366
+ text = "=" + text
367
+ found = _CELL.fullmatch(addr)
368
+ authored = _r1c1_to_a1(text, int(found.group(2)), _col_number(found.group(1))) if r1c1 else text
369
+ cell = {"kind": "formula", "state": "stale", "formula": authored, "value": None, **_keep(prior)}
370
+ if r1c1:
371
+ excel = dict(cell.get("excel") or {})
372
+ excel["formulaR1C1"] = text
373
+ cell["excel"] = excel
374
+ return sheets.clean_cell(cell, addr=addr)
375
+
376
+
377
+ def _translate_formula(cell, origin, destination):
378
+ if not isinstance(cell, dict) or cell.get("kind") != "formula":
379
+ return copy.deepcopy(cell)
380
+ r1c1 = (cell.get("excel") or {}).get("formulaR1C1")
381
+ if isinstance(r1c1, str):
382
+ return _formula(r1c1, cell, destination, r1c1=True)
383
+ try:
384
+ from openpyxl.formula.translate import Translator
385
+ translated = Translator(str(cell.get("formula") or ""), origin=origin).translate_formula(destination)
386
+ except Exception:
387
+ translated = str(cell.get("formula") or "")
388
+ return _formula(translated, cell, destination)
389
+
390
+
391
+ def _used(tab):
392
+ rect = tab.get("usedExtent") or sheets._used_extent(
393
+ tab.get("cells") or {}, (tab.get("excel") or {}).get("blankStyles"))
394
+ if not rect:
395
+ return (1, 1, 1, 1)
396
+ return (int(rect["row0"]) + 1, int(rect["col0"]) + 1,
397
+ int(rect["row1"]) + 1, int(rect["col1"]) + 1)
398
+
399
+
400
+ def _write_required(args):
401
+ revision = args.get("expected_revision")
402
+ if isinstance(revision, bool) or not isinstance(revision, int) or revision < 0:
403
+ raise CompatibilityRefusal(
404
+ "sheet_compat_revision_required", "A Sheet automation write requires the exact workbook revision it read.")
405
+ return revision
406
+
407
+
408
+ def _preflight(name, args):
409
+ spec = require_supported(name)
410
+ if not isinstance(args, dict):
411
+ raise CompatibilityRefusal("sheet_compat_arguments_invalid", "Sheet automation arguments must be an object.")
412
+ if spec["access"] == "write":
413
+ _write_required(args)
414
+ range_ops = {key for key in _BY_NAME if key.startswith("range.")} | {
415
+ "worksheet.count_a", "worksheet.count_if", "worksheet.geometry.read",
416
+ "worksheet.geometry.write", "worksheet.hidden.read", "worksheet.hidden.write",
417
+ }
418
+ if name.startswith("worksheet.") or name.startswith("range."):
419
+ if not str(args.get("sheet") or "").strip():
420
+ raise CompatibilityRefusal("sheet_compat_sheet_required", "That Sheet automation operation needs one worksheet.")
421
+ if name in range_ops:
422
+ _range(args.get("range"))
423
+ return spec
424
+
425
+
426
+ def _read(book, name, args):
427
+ if name == "workbooks.open":
428
+ return {"id": book.get("id"), "name": book.get("name"), "rev": book.get("rev")}
429
+ if name == "worksheets.item":
430
+ tab = _tab(book, args.get("sheet"))
431
+ return {key: copy.deepcopy(tab.get(key)) for key in ("id", "name", "cols", "rows")}
432
+ if name in ("worksheet.range", "worksheet.cells"):
433
+ return {"sheet": args.get("sheet"), "range": _range_text(_range(args.get("range")))}
434
+ if name.startswith("range.") or name.startswith("worksheet."):
435
+ tab = _tab(book, args.get("sheet"))
436
+ rect = _range(args.get("range")) if args.get("range") else None
437
+ if name == "range.values.read":
438
+ return _display_matrix(tab, rect, _cell_value)
439
+ if name == "range.formulas.read":
440
+ mode = str(args.get("mode") or "a1").lower()
441
+ return _display_matrix(tab, rect, lambda cell: (
442
+ ((cell or {}).get("excel") or {}).get("formulaR1C1") if mode == "r1c1"
443
+ else (cell or {}).get("formula")))
444
+ if name == "range.styles.read":
445
+ return _display_matrix(tab, rect, lambda cell: (cell or {}).get("style"))
446
+ if name == "range.number_formats.read":
447
+ def number_format(cell):
448
+ fmt = ((cell or {}).get("style") or {}).get("format")
449
+ return _xlsx_number_format(fmt)
450
+ return _display_matrix(tab, rect, number_format)
451
+ if name == "worksheet.used_range":
452
+ return {"address": _range_text(_used(tab))}
453
+ if name == "range.end_up":
454
+ _r0, c0, _r1, _c1 = rect
455
+ occupied = [int(re.search(r"\d+$", addr).group()) for addr, cell in (tab.get("cells") or {}).items()
456
+ if _col_number(re.match(r"[A-Z]+", addr).group()) == c0 and _cell_value(cell) not in (None, "")]
457
+ row = max(occupied, default=1)
458
+ return {"address": _addr(row, c0), "row": row, "column": c0}
459
+ if name == "worksheet.count_a":
460
+ return sum(1 for row in _addresses(rect) for addr in row
461
+ if ((tab.get("cells") or {}).get(addr) or {}).get("kind") == "formula"
462
+ or _cell_value((tab.get("cells") or {}).get(addr)) not in (None, ""))
463
+ if name == "worksheet.count_if":
464
+ wanted = str(args.get("criterion") if args.get("criterion") is not None else "").casefold()
465
+ return sum(1 for row in _addresses(rect) for addr in row
466
+ if str(_cell_value((tab.get("cells") or {}).get(addr)) or "").casefold() == wanted)
467
+ if name == "worksheet.geometry.read":
468
+ r0, c0, r1, c1 = rect
469
+ return {"columns": {_letters(col): (tab.get("colWidths") or {}).get(_letters(col), sheets.DEFAULT_COL_WIDTH)
470
+ for col in range(c0, c1 + 1)},
471
+ "rows": {str(row): (tab.get("rowHeights") or {}).get(str(row), tab.get("rowHeight", 24))
472
+ for row in range(r0, r1 + 1)}}
473
+ if name == "worksheet.hidden.read":
474
+ r0, c0, r1, c1 = rect
475
+ hidden_rows, hidden_cols = set(tab.get("hiddenRows") or ()), set(tab.get("hiddenCols") or ())
476
+ return {"rows": {str(row): row in hidden_rows for row in range(r0, r1 + 1)},
477
+ "columns": {_letters(col): col in hidden_cols for col in range(c0, c1 + 1)}}
478
+ if name == "worksheet.freeze.read":
479
+ return copy.deepcopy(tab.get("freeze") or {"rows": 0, "cols": 0})
480
+ if name == "worksheet.gridlines.read":
481
+ return (tab.get("view") or {}).get("gridlines", True)
482
+ if name == "application.calculation.read":
483
+ return (book.get("calculation") or {}).get("mode", "automatic")
484
+ if name == "workbook.names.item":
485
+ wanted = str(args.get("name") or "").casefold()
486
+ rows = book.get("namedRanges") or []
487
+ if isinstance(rows, dict):
488
+ rows = [{"name": key, "ref": value} for key, value in rows.items()]
489
+ for row in rows:
490
+ if isinstance(row, dict) and str(row.get("name") or "").casefold() == wanted:
491
+ return copy.deepcopy(row)
492
+ raise CompatibilityRefusal("sheet_compat_name_missing", "That workbook named range does not exist.")
493
+ if name == "workbook.save":
494
+ return {"rev": int(book.get("rev") or 0), "saved": True}
495
+ if name == "workbook.read_only":
496
+ return False
497
+ if name == "workbook.export_xlsx":
498
+ from core import sheet_import
499
+ handle, path = tempfile.mkstemp(prefix="loopable-sheet-export-", suffix=".xlsx")
500
+ os.close(handle)
501
+ try:
502
+ sheet_import.export_imported_workbook(book, path)
503
+ _apply_canonical_export_styles(book, path, sheet_import)
504
+ with open(path, "rb") as source:
505
+ payload = base64.b64encode(source.read()).decode("ascii")
506
+ return {"contentType": "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
507
+ "filename": str(args.get("filename") or (book.get("name") or "workbook") + ".xlsx"),
508
+ "base64": payload}
509
+ finally:
510
+ try:
511
+ os.remove(path)
512
+ except OSError:
513
+ pass
514
+ raise CompatibilityRefusal("sheet_compat_operation_unimplemented", name + " is declared but has no planner.")
515
+
516
+
517
+ def _xlsx_colour(value, palette):
518
+ """One canonical colour token as openpyxl's opaque ARGB value."""
519
+ raw = str(palette.get(str(value or ""), value or "") or "").strip().lstrip("#")
520
+ if re.fullmatch(r"[0-9a-fA-F]{6}", raw):
521
+ return "FF" + raw.upper()
522
+ if re.fullmatch(r"[0-9a-fA-F]{8}", raw):
523
+ return raw.upper()
524
+ return None
525
+
526
+
527
+ def _xlsx_number_format(spec):
528
+ """Render the closed canonical number-format vocabulary without guessing custom codes."""
529
+ spec = spec if isinstance(spec, dict) else {}
530
+ kind = str(spec.get("kind") or "general")
531
+ if kind == "code":
532
+ return str(spec.get("code") or "General")
533
+ if kind == "date":
534
+ return "yyyy-mm-dd"
535
+ decimals = max(0, int(spec.get("decimals") or 0))
536
+ places = ("." + ("0" * decimals)) if decimals else ""
537
+ grouping = "#,##0" if spec.get("thousands") else "0"
538
+ if kind == "number":
539
+ return grouping + places
540
+ if kind == "percent":
541
+ return "0" + places + "%"
542
+ if kind == "currency":
543
+ symbols = {"USD": "$", "EUR": "€", "GBP": "£", "JPY": "¥"}
544
+ symbol = symbols.get(str(spec.get("currency") or "USD").upper(),
545
+ str(spec.get("currency") or "USD").upper() + " ")
546
+ return symbol + grouping + places
547
+ return "General"
548
+
549
+
550
+ def _apply_canonical_export_styles(book, path, sheet_import):
551
+ """Add canonical Sheet styles where the import exporter has no source style table.
552
+
553
+ ``sheet_import.export_imported_workbook`` owns exact imported OOXML and must stay untouched.
554
+ A workbook authored through the compatibility facade has no ``stylesXml``, however, so that
555
+ seam intentionally emits defaults. In that one bounded case this overlay projects the
556
+ canonical style bags into the generated xlsx. Imported workbooks keep the lossless source
557
+ style path byte-for-byte and never pass through openpyxl a second time.
558
+ """
559
+ document = sheets.migrate_workbook(book, lossless=True)
560
+ if ((document.get("excel") or {}).get("stylesXml") or ""):
561
+ return
562
+
563
+ import openpyxl
564
+ from openpyxl.styles import Side
565
+
566
+ output = openpyxl.load_workbook(path)
567
+ size_steps = {"s": 9, "m": sheets.DEFAULT_FONT_PT, "l": 12, "xl": 14}
568
+ for tab in document.get("sheets") or []:
569
+ ws = output[str(tab.get("name") or "Sheet")[:31]]
570
+ for address, source in (tab.get("cells") or {}).items():
571
+ style = (source or {}).get("style") if isinstance(source, dict) else None
572
+ if not isinstance(style, dict) or not style:
573
+ continue
574
+ target = ws[address]
575
+
576
+ font = copy.copy(target.font)
577
+ font.bold = bool(style.get("bold"))
578
+ font.italic = bool(style.get("italic"))
579
+ font.underline = "single" if style.get("underline") else None
580
+ if style.get("font"):
581
+ font.name = str(style["font"])
582
+ size = style.get("size")
583
+ if isinstance(size, str):
584
+ font.sz = size_steps.get(size, sheets.DEFAULT_FONT_PT)
585
+ elif not isinstance(size, bool) and isinstance(size, (int, float)):
586
+ font.sz = float(size)
587
+ colour = _xlsx_colour(style.get("color"), sheet_import.PALETTE)
588
+ if colour:
589
+ font.color = colour
590
+ target.font = font
591
+
592
+ fill = _xlsx_colour(style.get("fill"), sheet_import.PALETTE)
593
+ if fill:
594
+ painted = copy.copy(target.fill)
595
+ painted.fill_type = "solid"
596
+ painted.fgColor = fill
597
+ painted.patternType = "solid"
598
+ target.fill = painted
599
+
600
+ alignment = copy.copy(target.alignment)
601
+ if style.get("align"):
602
+ alignment.horizontal = style["align"]
603
+ if style.get("valign"):
604
+ alignment.vertical = {"middle": "center"}.get(style["valign"], style["valign"])
605
+ alignment.wrap_text = bool(style.get("wrap"))
606
+ alignment.indent = int(style.get("indent") or 0)
607
+ alignment.text_rotation = int(style.get("textRotation") or 0)
608
+ target.alignment = alignment
609
+
610
+ borders = style.get("border") or {}
611
+ if borders:
612
+ border = copy.copy(target.border)
613
+ line = Side(style="thin", color="FFCBD1DC")
614
+ for side in ("top", "right", "bottom", "left"):
615
+ if borders.get(side):
616
+ setattr(border, side, line)
617
+ target.border = border
618
+ if style.get("format"):
619
+ target.number_format = _xlsx_number_format(style["format"])
620
+ output.save(path)
621
+ output.close()
622
+
623
+
624
+ def _tab_property(tab, key, value):
625
+ candidate = copy.deepcopy(tab)
626
+ if value in (None, {}, []):
627
+ candidate.pop(key, None)
628
+ else:
629
+ candidate[key] = copy.deepcopy(value)
630
+ cleaned = sheets._clean_sheet(candidate, lossless=True)
631
+ if key in cleaned:
632
+ tab[key] = copy.deepcopy(cleaned[key])
633
+ else:
634
+ tab.pop(key, None)
635
+
636
+
637
+ def _mutate(book, name, args, calculator=None):
638
+ touched = {}
639
+ result = None
640
+ tab = _tab(book, args.get("sheet")) if args.get("sheet") else None
641
+ rect = _range(args.get("range")) if args.get("range") else None
642
+ cells = tab.setdefault("cells", {}) if tab is not None else None
643
+
644
+ def mark(address):
645
+ touched.setdefault(str(tab.get("id") or tab.get("name")), []).append(address)
646
+
647
+ if name in ("range.values.write", "range.formulas.write", "range.styles.write", "range.number_formats.write"):
648
+ addresses = _addresses(rect)
649
+ height, width = len(addresses), len(addresses[0])
650
+ values = _matrix(args.get("values"), height, width, name)
651
+ for row_at, row in enumerate(addresses):
652
+ for col_at, address in enumerate(row):
653
+ prior, value = cells.get(address), values[row_at][col_at]
654
+ if name == "range.values.write":
655
+ after = _literal(value, prior)
656
+ elif name == "range.formulas.write":
657
+ after = _formula(value, prior, address, r1c1=str(args.get("mode") or "").lower() == "r1c1")
658
+ elif name == "range.styles.write":
659
+ style = sheets._clean_style(value)
660
+ after = copy.deepcopy(prior) if isinstance(prior, dict) else {"kind": "empty", "state": "typed"}
661
+ if style:
662
+ after["style"] = style
663
+ else:
664
+ after.pop("style", None)
665
+ after = sheets.clean_cell(after, addr=address)
666
+ else:
667
+ fmt = None if value in (None, "", "General", "general") else {"kind": "code", "code": str(value)}
668
+ after = copy.deepcopy(prior) if isinstance(prior, dict) else {"kind": "empty", "state": "typed"}
669
+ style = dict(after.get("style") or {})
670
+ if fmt:
671
+ style["format"] = fmt
672
+ else:
673
+ style.pop("format", None)
674
+ if style:
675
+ after["style"] = style
676
+ else:
677
+ after.pop("style", None)
678
+ after = sheets.clean_cell(after, addr=address)
679
+ if after is None:
680
+ cells.pop(address, None)
681
+ else:
682
+ cells[address] = after
683
+ mark(address)
684
+ result = {"cells": height * width}
685
+ elif name == "range.clear_contents":
686
+ for row in _addresses(rect):
687
+ for address in row:
688
+ after = _literal(None, cells.get(address))
689
+ if after is None:
690
+ cells.pop(address, None)
691
+ else:
692
+ cells[address] = after
693
+ mark(address)
694
+ result = {"cleared": sum(len(row) for row in _addresses(rect))}
695
+ elif name == "range.fill_down":
696
+ addresses = _addresses(rect)
697
+ sources = {address: copy.deepcopy(cells.get(address)) for address in addresses[0]}
698
+ for row in addresses[1:]:
699
+ for column_at, address in enumerate(row):
700
+ origin = addresses[0][column_at]
701
+ after = _translate_formula(sources[origin], origin, address)
702
+ if after is None:
703
+ cells.pop(address, None)
704
+ else:
705
+ cells[address] = after
706
+ mark(address)
707
+ result = {"filled": max(0, len(addresses) - 1) * len(addresses[0])}
708
+ elif name == "range.replace":
709
+ if str(args.get("look_at") or "whole").lower() != "whole":
710
+ raise CompatibilityRefusal("sheet_compat_replace_mode", "Only exact whole-cell replacement is supported.")
711
+ wanted, replacement = args.get("what"), args.get("replacement")
712
+ match_case = bool(args.get("match_case"))
713
+ matched = 0
714
+ for row in _addresses(rect):
715
+ for address in row:
716
+ prior = cells.get(address)
717
+ if not isinstance(prior, dict) or prior.get("kind") == "formula":
718
+ continue
719
+ value = _cell_value(prior)
720
+ equal = value == wanted if match_case else str(value or "").casefold() == str(wanted or "").casefold()
721
+ if equal:
722
+ cells[address] = _literal(replacement, prior)
723
+ mark(address)
724
+ matched += 1
725
+ result = {"replaced": matched}
726
+ elif name in ("range.copy", "range.paste_values"):
727
+ source = _range(args.get("source_range"))
728
+ source_tab = _tab(book, args.get("source_sheet") or args.get("sheet"))
729
+ source_addresses, target_addresses = _addresses(source), _addresses(rect)
730
+ if (len(source_addresses), len(source_addresses[0])) != (len(target_addresses), len(target_addresses[0])):
731
+ raise CompatibilityRefusal("sheet_compat_copy_shape", "Source and target copy ranges must have the same shape.")
732
+ mode = "values" if name == "range.paste_values" else str(args.get("mode") or "all").lower()
733
+ if mode not in ("all", "values", "formulas"):
734
+ raise CompatibilityRefusal("sheet_compat_copy_mode", "Copy mode must be all, values or formulas.")
735
+ source_cells = copy.deepcopy(source_tab.get("cells") or {})
736
+ for rr, row in enumerate(target_addresses):
737
+ for cc, address in enumerate(row):
738
+ origin = source_addresses[rr][cc]
739
+ source_cell, prior = source_cells.get(origin), cells.get(address)
740
+ if mode == "values":
741
+ after = _literal(_cell_value(source_cell), prior)
742
+ elif mode == "formulas":
743
+ after = _translate_formula(source_cell, origin, address) if (source_cell or {}).get("kind") == "formula" else prior
744
+ else:
745
+ after = _translate_formula(source_cell, origin, address)
746
+ if after is None:
747
+ cells.pop(address, None)
748
+ else:
749
+ cells[address] = after
750
+ mark(address)
751
+ result = {"copied": sum(len(row) for row in target_addresses), "mode": mode}
752
+ elif name == "worksheet.geometry.write":
753
+ r0, c0, r1, c1 = rect
754
+ if "column_width" in args:
755
+ widths = dict(tab.get("colWidths") or {})
756
+ for col in range(c0, c1 + 1):
757
+ widths[_letters(col)] = args["column_width"]
758
+ _tab_property(tab, "colWidths", widths)
759
+ if "row_height" in args:
760
+ heights = dict(tab.get("rowHeights") or {})
761
+ for row in range(r0, r1 + 1):
762
+ heights[str(row)] = args["row_height"]
763
+ _tab_property(tab, "rowHeights", heights)
764
+ result = {"range": _range_text(rect)}
765
+ elif name == "worksheet.hidden.write":
766
+ r0, c0, r1, c1 = rect
767
+ axis = str(args.get("axis") or "rows").lower()
768
+ hidden = bool(args.get("hidden"))
769
+ if axis not in ("rows", "columns"):
770
+ raise CompatibilityRefusal("sheet_compat_hidden_axis", "Hidden state needs rows or columns.")
771
+ key, members = ("hiddenRows", range(r0, r1 + 1)) if axis == "rows" else ("hiddenCols", range(c0, c1 + 1))
772
+ values = set(tab.get(key) or ())
773
+ values = values.union(members) if hidden else values.difference(members)
774
+ _tab_property(tab, key, sorted(values))
775
+ result = {"axis": axis, "hidden": hidden}
776
+ elif name == "worksheet.freeze.write":
777
+ _tab_property(tab, "freeze", {"rows": args.get("rows", 0), "cols": args.get("cols", 0)})
778
+ result = copy.deepcopy(tab.get("freeze") or {"rows": 0, "cols": 0})
779
+ elif name == "worksheet.gridlines.write":
780
+ view = dict(tab.get("view") or {})
781
+ view["gridlines"] = bool(args.get("shown"))
782
+ _tab_property(tab, "view", view)
783
+ result = view["gridlines"]
784
+ elif name == "application.calculation.write":
785
+ mode = str(args.get("mode") or "").lower()
786
+ if mode not in ("manual", "automatic"):
787
+ raise CompatibilityRefusal("sheet_compat_calculation_mode", "Calculation mode must be manual or automatic.")
788
+ calculation = dict(book.get("calculation") or {})
789
+ calculation["mode"] = mode
790
+ book["calculation"] = calculation
791
+ result = mode
792
+ elif name in ("workbook.calculate_full", "workbook.refresh_all"):
793
+ if not callable(calculator):
794
+ raise CompatibilityRefusal("sheet_compat_calculator_required", "This Sheet host has no formula evaluator configured.")
795
+ calculator(book)
796
+ stamp = write_execution.utcnow().replace(microsecond=0).isoformat()
797
+ calculation = dict(book.get("calculation") or {})
798
+ calculation["lastCalculatedAt"] = stamp
799
+ book["calculation"] = calculation
800
+ if name == "workbook.refresh_all":
801
+ book["refreshed_at"] = stamp
802
+ result = {"calculatedAt": stamp}
803
+ elif name == "workbook.names.write":
804
+ named, ref = str(args.get("name") or "").strip(), str(args.get("refers_to") or "").strip()
805
+ if not named or not ref:
806
+ raise CompatibilityRefusal("sheet_compat_name_invalid", "A named range needs a name and refers_to address.")
807
+ rows = book.get("namedRanges") or []
808
+ if isinstance(rows, dict):
809
+ rows = [{"name": key, "ref": value} for key, value in rows.items()]
810
+ kept = [copy.deepcopy(row) for row in rows if isinstance(row, dict)
811
+ and str(row.get("name") or "").casefold() != named.casefold()]
812
+ kept.append({"name": named, "ref": ref})
813
+ book["namedRanges"] = kept
814
+ result = copy.deepcopy(kept[-1])
815
+ else:
816
+ raise CompatibilityRefusal("sheet_compat_operation_unimplemented", name + " is declared but has no patcher.")
817
+
818
+ if tab is not None:
819
+ tab["usedExtent"] = sheets._used_extent(cells or {}, (tab.get("excel") or {}).get("blankStyles"))
820
+ if tab["usedExtent"]:
821
+ tab["cols"] = max(int(tab.get("cols") or 1), int(tab["usedExtent"]["col1"]) + 1)
822
+ tab["rows"] = max(int(tab.get("rows") or 1), int(tab["usedExtent"]["row1"]) + 1)
823
+ return result, touched
824
+
825
+
826
+ def execute(st, workbook_id, operation, arguments=None, *, actor="", actor_name="", calculator=None):
827
+ """Execute one declared operation through the shared planner/patcher.
828
+
829
+ Unsupported and malformed operation-level requests are checked before ``st.update`` is ever
830
+ reachable. A supported write validates its expected revision again under the store lock.
831
+ """
832
+ if arguments is None:
833
+ args = {}
834
+ elif not isinstance(arguments, dict):
835
+ raise CompatibilityRefusal(
836
+ "sheet_compat_arguments_invalid", "Sheet automation arguments must be an object.")
837
+ else:
838
+ args = dict(arguments)
839
+ spec = _preflight(operation, args)
840
+ if spec["access"] != "write":
841
+ book = sheets.get_workbook(st, workbook_id)
842
+ return {"operation": operation, "rev": int(book.get("rev") or 0),
843
+ "result": _read(book, operation, args)}
844
+
845
+ expected = _write_required(args)
846
+ wid = sheets._slug_id(workbook_id)
847
+ saved, envelope = {}, {}
848
+
849
+ def apply(state):
850
+ if not isinstance(state, dict) or not state.get("id"):
851
+ raise sheets.SheetRefusal("workbook_missing", "That workbook no longer exists.")
852
+ current = int(state.get("rev") or 0)
853
+ if current != expected:
854
+ raise sheets.SheetStaleRefusal(
855
+ "workbook_revision_stale",
856
+ "That workbook changed somewhere else, so this automation operation was not applied.",
857
+ rev_current=current)
858
+ book = copy.deepcopy(state)
859
+ result, touched = _mutate(book, operation, args, calculator=calculator)
860
+ new_rev = current + 1
861
+ by_id = {str(tab.get("id")): tab for tab in book.get("sheets") or []}
862
+ for tab_id, addresses in touched.items():
863
+ if tab_id in by_id:
864
+ sheets._append_write_log(by_id[tab_id], new_rev, sorted(set(addresses)),
865
+ actor=actor, actor_name=actor_name)
866
+ book["rev"] = new_rev
867
+ book["updated_at"] = write_execution.utcnow().replace(microsecond=0).isoformat()
868
+ sheets._json_safe(book, "workbook")
869
+ saved.update(copy.deepcopy(book))
870
+ envelope.update({"operation": operation, "rev": new_rev, "result": copy.deepcopy(result)})
871
+ return book, envelope
872
+
873
+ out = sheets._persist(st, sheets.workbook_key(wid), apply)
874
+ sheets._index_upsert(st, saved)
875
+ return out
platform/core/sheet_events.py ADDED
@@ -0,0 +1,101 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Value-only workbook commit notifications; the API supplies the Automation consumer.
2
+
3
+ No row or cell values cross this boundary. Publication happens after the synchronous store
4
+ commit, outside its lock. A new workbook/tab is initial backfill, not a watched change.
5
+ """
6
+ from __future__ import annotations
7
+
8
+ import logging
9
+ import re
10
+
11
+ COMMIT_OBSERVER = None
12
+ _ADDRESS = re.compile(r"^([A-Z]+)([1-9][0-9]*)$")
13
+
14
+
15
+ def _value(cell):
16
+ cell = cell if isinstance(cell, dict) else {}
17
+ # Paint, formula text, refresh stamps and provenance do not change the computed insight.
18
+ return cell.get("value"), cell.get("error"), cell.get("refusal")
19
+
20
+
21
+ def _binding_keys(cell, address):
22
+ binding = (cell or {}).get("binding") or {}
23
+ if not binding:
24
+ return set()
25
+ keys = {address}
26
+ for key in ("id", "key", "name"):
27
+ if binding.get(key):
28
+ keys.add(str(binding[key]))
29
+ field = binding.get("field") or binding.get("aggregate")
30
+ if field:
31
+ keys.add(str(field))
32
+ if binding.get("database"):
33
+ keys.add(f'{binding["database"]}.{field}')
34
+ return keys
35
+
36
+
37
+ def _ranges(addresses):
38
+ """Compress adjacent rows only; sparse gaps must never trigger an unrelated watch."""
39
+ columns = {}
40
+ for address in addresses:
41
+ match = _ADDRESS.fullmatch(address)
42
+ if match:
43
+ columns.setdefault(match[1], []).append(int(match[2]))
44
+ ranges = []
45
+ for column, rows in sorted(columns.items()):
46
+ start = end = None
47
+ for row in sorted(rows):
48
+ if start is not None and row == end + 1:
49
+ end = row
50
+ continue
51
+ if start is not None:
52
+ ranges.append(f"{column}{start}" if start == end else f"{column}{start}:{column}{end}")
53
+ start = end = row
54
+ if start is not None:
55
+ ranges.append(f"{column}{start}" if start == end else f"{column}{start}:{column}{end}")
56
+ return ranges
57
+
58
+
59
+ def committed_changes(before, after):
60
+ """Derive at most one event per existing tab from the transaction's actual before/after."""
61
+ if not isinstance(before, dict) or not isinstance(after, dict):
62
+ return []
63
+ workbook = str(after.get("id") or "")
64
+ if not workbook or before.get("id") != workbook:
65
+ return []
66
+ prior = {str(tab.get("id")): tab for tab in before.get("sheets") or []}
67
+ revision = int(after.get("rev") or 0)
68
+ events = []
69
+ for tab in after.get("sheets") or []:
70
+ sheet = str(tab.get("id") or "")
71
+ if sheet not in prior:
72
+ continue
73
+ old, new = prior[sheet].get("cells") or {}, tab.get("cells") or {}
74
+ changed, bindings = set(), set()
75
+ for address in old.keys() | new.keys():
76
+ left, right = old.get(address) or {}, new.get(address) or {}
77
+ value_changed = _value(left) != _value(right)
78
+ binding_changed = (left.get("binding") or {}) != (right.get("binding") or {})
79
+ if value_changed:
80
+ changed.add(address)
81
+ if value_changed or binding_changed:
82
+ bindings.update(_binding_keys(left, address) | _binding_keys(right, address))
83
+ if not changed and not bindings:
84
+ continue
85
+ events.append({
86
+ "kind": "sheet_insight_changed", "workbookId": workbook, "sheetId": sheet,
87
+ "batchId": f"{workbook}:{revision}:{sheet}", "revision": revision,
88
+ "committed": True, "changeKinds": (["value"] if changed else []) + (["binding"] if bindings else []),
89
+ "ranges": _ranges(changed), "bindings": sorted(bindings), "recordCount": len(changed),
90
+ })
91
+ return events
92
+
93
+
94
+ def publish(st, events):
95
+ if events and callable(COMMIT_OBSERVER):
96
+ try:
97
+ COMMIT_OBSERVER(st, events)
98
+ except Exception:
99
+ # A consumer failure cannot turn a committed edit into a purported failed save.
100
+ # Never log cell contents, credentials, or the consumer's raw exception message.
101
+ logging.getLogger(__name__).error("Sheet commit notification failed after persistence")
platform/core/sheet_excel.py ADDED
@@ -0,0 +1,65 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Windows Excel adapter and shared spreadsheet-backend protocols.
2
+
3
+ Importing this module is safe on Linux. The optional ``win32com`` dependency is loaded only when
4
+ ``excel_application`` is selected at the client boundary. Excel itself already implements the
5
+ canonical object hierarchy, so the adapter returns the genuine COM Application rather than
6
+ maintaining a second imitation of it.
7
+ """
8
+
9
+ from __future__ import annotations
10
+
11
+ from typing import Any, Protocol, runtime_checkable
12
+
13
+
14
+ @runtime_checkable
15
+ class OperationBackend(Protocol):
16
+ """Transport used by the Loopable facade."""
17
+
18
+ def manifest_bytes(self) -> bytes: ...
19
+
20
+ def open_workbook(self, identifier: str) -> dict[str, Any]: ...
21
+
22
+ def execute(self, workbook_id: str, operation: str, arguments: dict[str, Any]) -> dict[str, Any]: ...
23
+
24
+
25
+ @runtime_checkable
26
+ class SpreadsheetApplication(Protocol):
27
+ """The common boundary accepted by data-only workbook workflow functions."""
28
+
29
+ Workbooks: Any
30
+ WorksheetFunction: Any
31
+
32
+ def CalculateFull(self) -> Any: ...
33
+
34
+
35
+ def excel_application(*, visible: bool | None = None, dispatch_ex: bool = True):
36
+ """Return a genuine Excel COM Application, loading pywin32 only on Windows.
37
+
38
+ Process/UI settings remain the Excel caller's responsibility. They are not projected onto
39
+ Loopable, where the public capability manifest names them as refusal-only operations.
40
+ """
41
+ try:
42
+ import win32com.client as win32
43
+ except (ImportError, ModuleNotFoundError) as exc: # pragma: no cover - Windows-only path
44
+ raise RuntimeError(
45
+ "The Excel backend needs Windows, desktop Excel and pywin32. Choose backend='loopable' on this host."
46
+ ) from exc
47
+ application = (win32.DispatchEx if dispatch_ex else win32.Dispatch)("Excel.Application")
48
+ if visible is not None:
49
+ application.Visible = bool(visible)
50
+ return application
51
+
52
+
53
+ def select_application(backend: str, **options):
54
+ """Select Excel or Loopable once, outside business workflow logic."""
55
+ kind = str(backend or "").strip().lower()
56
+ if kind == "excel":
57
+ return excel_application(
58
+ visible=options.get("visible"), dispatch_ex=options.get("dispatch_ex", True))
59
+ if kind == "loopable":
60
+ from sdk.loopable_sheet import Application
61
+ transport = options.get("transport")
62
+ if transport is None:
63
+ raise ValueError("The Loopable backend needs an operation transport.")
64
+ return Application(transport)
65
+ raise ValueError("Spreadsheet backend must be 'excel' or 'loopable'.")
platform/core/sheet_import.py CHANGED
@@ -44,6 +44,9 @@ import re
44
  import zipfile
45
  import hashlib
46
  import json
 
 
 
47
  import xml.etree.ElementTree as ET
48
  from datetime import date, datetime, time, timezone
49
 
@@ -55,6 +58,15 @@ from core import sheets
55
  #: parse in reasonable time.
56
  MAX_IMPORT_BYTES = 25 * 1024 * 1024
57
 
 
 
 
 
 
 
 
 
 
58
  #: C4's job states, in the order a job walks them. `refused` is the exit that replaces `done`.
59
  IMPORT_STATES = ("reading", "cells", "formats", "recalc", "done", "refused")
60
 
@@ -126,10 +138,6 @@ PALETTE = {
126
  DEFAULT_INK = "--lp-ink"
127
  DEFAULT_FILL = "--lp-surface"
128
 
129
- #: The four font steps A1 froze, in points. A size is a STEP and never a pixel count, for
130
- #: `_clean_style`'s reason: a stored pixel size is a layout decision that outlives the layout.
131
- _SIZE_STEPS = ((9.5, "s"), (12.5, "m"), (16.5, "l"))
132
-
133
  #: Excel's horizontal alignments that have a home in `CELL_ALIGNS`. Everything else is named in
134
  #: `unsupported` rather than guessed at: `justify` and `distributed` are not `left`.
135
  _ALIGNS = {"left": "left", "center": "center", "centre": "center", "right": "right"}
@@ -314,22 +322,13 @@ def classify_format(code):
314
 
315
  # ── one cell's style ─────────────────────────────────────────────────────────────────────────
316
 
317
- def _size_step(points):
318
- if not isinstance(points, (int, float)):
319
- return ""
320
- for limit, step in _SIZE_STEPS:
321
- if points < limit:
322
- return step
323
- return "xl"
324
-
325
-
326
  def style_of(cell, notes):
327
  """The C2 style for one openpyxl cell, or `{}` when nothing about it is worth storing.
328
 
329
- ⚠ FONT FAMILY IS DROPPED IN SILENCE AND THAT IS THE DESIGN, not an omission: the app is Inter
330
- everywhere (DESIGN.md), so carrying Arial would either be ignored or would put a second
331
- typeface on a screen the design forbids. It is still NAMED in the report, once, because a
332
- person who imported an Arial workbook should be told why it does not look like Arial.
333
  """
334
  out = {}
335
  font = getattr(cell, "font", None)
@@ -340,9 +339,17 @@ def style_of(cell, notes):
340
  out["italic"] = True
341
  if font.u:
342
  out["underline"] = True
343
- step = _size_step(font.sz)
344
- if step and step != "m":
345
- out["size"] = step
 
 
 
 
 
 
 
 
346
  argb = argb_of(font.color)
347
  if argb:
348
  token = quantize(argb)
@@ -902,6 +909,134 @@ def _geometry_of(ws, notes):
902
  return cols, rows, widths, freeze, sorted(set(hidden_rows)), sorted(set(hidden_cols)), heights
903
 
904
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
905
  def _print_of(ws):
906
  """Store print setup and inclusive print areas; export executes these settings later."""
907
  setup = {key: getattr(ws.page_setup, key) for key in ws.page_setup.__attrs__
@@ -977,8 +1112,6 @@ def import_workbook(path, *, source="", progress=None):
977
  ⚠ `progress(state, fraction)` IS CALLED FROM WHATEVER THREAD RUNS THIS. It must not raise and
978
  must not block; the job's registry entry is the only thing it touches.
979
  """
980
- import openpyxl
981
-
982
  def _say(state, fraction):
983
  if progress:
984
  try:
@@ -987,6 +1120,11 @@ def import_workbook(path, *, source="", progress=None):
987
  pass
988
 
989
  _say("reading", 0.02)
 
 
 
 
 
990
  try:
991
  formulas = openpyxl.load_workbook(path, data_only=False, read_only=False,
992
  keep_links=False)
@@ -1088,6 +1226,9 @@ def import_workbook(path, *, source="", progress=None):
1088
  tab["hiddenRows"] = hidden_rows
1089
  if hidden_cols:
1090
  tab["hiddenCols"] = hidden_cols
 
 
 
1091
  charts = _charts_of(ws, name, notes)
1092
  if name in chart_tabs:
1093
  tab["id"] = chart_tabs[name]["id"]
@@ -1103,11 +1244,6 @@ def import_workbook(path, *, source="", progress=None):
1103
  values.close()
1104
  except Exception: # noqa: BLE001
1105
  pass
1106
- #: ⭐ THE FONT NOTE IS WORKBOOK-WIDE AND UNCONDITIONAL, because the answer is the same on
1107
- #: every file: the app is one typeface, so no imported font survives and the reader is told
1108
- #: once rather than per tab.
1109
- book_notes.add("every font was replaced for display; original fonts and formats are kept for export")
1110
-
1111
  # Original styles are shared once, with integer references on cells, so exact formats,
1112
  # theme colours, border weights and blank-cell styles survive without repeating XML per cell.
1113
  original = {}
@@ -1129,6 +1265,132 @@ def import_workbook(path, *, source="", progress=None):
1129
  }
1130
 
1131
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1132
  def take_imported_values(book):
1133
  """Pull every `imported_value` off the document and return `{(tab, addr): cached}`.
1134
 
 
44
  import zipfile
45
  import hashlib
46
  import json
47
+ import math
48
+ import os
49
+ import tempfile
50
  import xml.etree.ElementTree as ET
51
  from datetime import date, datetime, time, timezone
52
 
 
58
  #: parse in reasonable time.
59
  MAX_IMPORT_BYTES = 25 * 1024 * 1024
60
 
61
+ #: An xlsx is a ZIP and the compressed upload limit is therefore not a parse budget. These
62
+ #: ceilings are checked from the central directory before openpyxl is imported or called. They
63
+ #: deliberately leave more room than the 64 MiB stored document: worksheet XML repeats tags and
64
+ #: shared strings before the importer turns them into the compact JSON document.
65
+ MAX_ARCHIVE_UNCOMPRESSED_BYTES = 256 * 1024 * 1024
66
+ MAX_ARCHIVE_MEMBER_BYTES = 128 * 1024 * 1024
67
+ MAX_ARCHIVE_MEMBERS = 10_000
68
+ MAX_ARCHIVE_COMPRESSION_RATIO = 250
69
+
70
  #: C4's job states, in the order a job walks them. `refused` is the exit that replaces `done`.
71
  IMPORT_STATES = ("reading", "cells", "formats", "recalc", "done", "refused")
72
 
 
138
  DEFAULT_INK = "--lp-ink"
139
  DEFAULT_FILL = "--lp-surface"
140
 
 
 
 
 
141
  #: Excel's horizontal alignments that have a home in `CELL_ALIGNS`. Everything else is named in
142
  #: `unsupported` rather than guessed at: `justify` and `distributed` are not `left`.
143
  _ALIGNS = {"left": "left", "center": "center", "centre": "center", "right": "right"}
 
322
 
323
  # ── one cell's style ─────────────────────────────────────────────────────────────────────────
324
 
 
 
 
 
 
 
 
 
 
325
  def style_of(cell, notes):
326
  """The C2 style for one openpyxl cell, or `{}` when nothing about it is worth storing.
327
 
328
+ Imported workbook styling is cell data, not product chrome. A supported font family and the
329
+ authored point size therefore enter the normal display model. Anything that model cannot
330
+ express still survives exactly through the cell's style id plus the workbook's original
331
+ styles XML; the import report names the display fallback instead of pretending it was exact.
332
  """
333
  out = {}
334
  font = getattr(cell, "font", None)
 
339
  out["italic"] = True
340
  if font.u:
341
  out["underline"] = True
342
+ size = getattr(font, "sz", None)
343
+ if not isinstance(size, bool) and isinstance(size, (int, float)) and math.isfinite(size):
344
+ points = max(sheets.MIN_FONT_PT, min(sheets.MAX_FONT_PT, float(size)))
345
+ if points != sheets.DEFAULT_FONT_PT:
346
+ out["size"] = points
347
+ name = str(getattr(font, "name", None) or "").strip()
348
+ if name in sheets.CELL_FONTS:
349
+ if name != sheets.DEFAULT_CELL_FONT:
350
+ out["font"] = name
351
+ elif name:
352
+ notes.add("the " + name[:60] + " font is kept for export but uses the default face on screen")
353
  argb = argb_of(font.color)
354
  if argb:
355
  token = quantize(argb)
 
909
  return cols, rows, widths, freeze, sorted(set(hidden_rows)), sorted(set(hidden_cols)), heights
910
 
911
 
912
+ def _view_of(ws):
913
+ """Explicit worksheet display state in the existing tab ``view`` vocabulary.
914
+
915
+ ``None`` means the source omitted the property and is intentionally different from an
916
+ explicit ``True`` or 100 percent. That distinction is what lets compact product defaults
917
+ apply only when the file said nothing while an authored value always wins.
918
+ """
919
+ source = getattr(ws, "sheet_view", None)
920
+ if source is None:
921
+ return {}
922
+ out = {}
923
+ if isinstance(getattr(source, "showGridLines", None), bool):
924
+ out["gridlines"] = source.showGridLines
925
+ if isinstance(getattr(source, "showRowColHeaders", None), bool):
926
+ out["headings"] = source.showRowColHeaders
927
+ zoom = getattr(source, "zoomScale", None)
928
+ if not isinstance(zoom, bool) and isinstance(zoom, (int, float)) and math.isfinite(zoom):
929
+ out["zoom"] = zoom
930
+ page_view = {"normal": "normal", "pageBreakPreview": "pageBreak",
931
+ "pageLayout": "pageLayout"}.get(getattr(source, "view", None))
932
+ if page_view:
933
+ out["pageView"] = page_view
934
+ return out
935
+
936
+
937
+ def _archive_refusal(code, message):
938
+ raise ImportRefusal(code, message)
939
+
940
+
941
+ def _address_in_document(ref):
942
+ point = _corner(ref)
943
+ return (point is not None and 0 <= point[0] < sheets.DOCUMENT_MAX_COLS
944
+ and 0 <= point[1] < sheets.DOCUMENT_MAX_ROWS)
945
+
946
+
947
+ def _guard_worksheet_xml(stream):
948
+ """Stream one worksheet and reject impossible dimensions/cell counts before openpyxl.
949
+
950
+ ElementTree sees only bounded, central-directory-admitted bytes. It is used here as an
951
+ admission parser, not as the workbook parser: no cell value, formula or style is interpreted.
952
+ """
953
+ dimension_seen = False
954
+ cells = 0
955
+ try:
956
+ for _event, element in ET.iterparse(stream, events=("end",)):
957
+ tag = str(element.tag).rsplit("}", 1)[-1]
958
+ if tag == "dimension":
959
+ ref = str(element.get("ref") or "").replace("$", "").upper()
960
+ parts = ref.split(":")
961
+ if (len(parts) not in (1, 2) or not all(_address_in_document(part) for part in parts)
962
+ or (len(parts) == 2 and rect_of(ref) is None)):
963
+ _archive_refusal(
964
+ "import_declared_dimension_invalid",
965
+ "A worksheet declares cells outside Excel's supported dimensions, so it cannot be imported.")
966
+ dimension_seen = True
967
+ elif tag == "c":
968
+ cells += 1
969
+ ref = str(element.get("r") or "").replace("$", "").upper()
970
+ if not _address_in_document(ref):
971
+ _archive_refusal(
972
+ "import_declared_dimension_invalid",
973
+ "A worksheet contains a cell outside Excel's supported dimensions, so it cannot be imported.")
974
+ if cells > sheets._MAX_CELLS_PER_SHEET:
975
+ _archive_refusal(
976
+ "sheet_too_many_cells",
977
+ "A worksheet declares more than " + str(sheets._MAX_CELLS_PER_SHEET)
978
+ + " populated cells, so it cannot be imported.")
979
+ element.clear()
980
+ except ImportRefusal:
981
+ raise
982
+ except ET.ParseError as exc:
983
+ raise ImportRefusal(
984
+ "import_unreadable",
985
+ "That file could not be read as a workbook. Save it again as .xlsx and try once more.") from exc
986
+ # A dimension element is optional in OOXML. When it is absent, the independently bounded
987
+ # cell-address walk above is the authority; absence must not invent a default declaration.
988
+ return dimension_seen
989
+
990
+
991
+ def guard_import_archive(path):
992
+ """Admit a bounded OOXML archive before any openpyxl workbook parse occurs."""
993
+ try:
994
+ byte_count = os.path.getsize(path)
995
+ except OSError as exc:
996
+ raise ImportRefusal("import_unreadable", "That workbook file could not be read.") from exc
997
+ if byte_count > MAX_IMPORT_BYTES:
998
+ raise ImportRefusal(
999
+ "import_too_large",
1000
+ "That file is larger than the " + str(MAX_IMPORT_BYTES // (1024 * 1024))
1001
+ + " MB import limit, so it cannot be read.")
1002
+ try:
1003
+ with zipfile.ZipFile(path) as archive:
1004
+ members = archive.infolist()
1005
+ if len(members) > MAX_ARCHIVE_MEMBERS:
1006
+ _archive_refusal("import_zip_bloat",
1007
+ "That workbook archive holds too many parts to import safely.")
1008
+ names = [entry.filename for entry in members]
1009
+ if len(set(names)) != len(names):
1010
+ _archive_refusal("import_zip_bloat",
1011
+ "That workbook archive repeats a part name, so it cannot be imported safely.")
1012
+ total = sum(max(0, int(entry.file_size)) for entry in members)
1013
+ if total > MAX_ARCHIVE_UNCOMPRESSED_BYTES:
1014
+ _archive_refusal("import_zip_bloat",
1015
+ "That workbook expands beyond the safe import limit.")
1016
+ for entry in members:
1017
+ if entry.flag_bits & 0x1:
1018
+ _archive_refusal("import_encrypted",
1019
+ "Password protected workbooks cannot be imported.")
1020
+ if entry.file_size > MAX_ARCHIVE_MEMBER_BYTES:
1021
+ _archive_refusal("import_zip_bloat",
1022
+ "One workbook part expands beyond the safe import limit.")
1023
+ ratio = entry.file_size / max(1, entry.compress_size)
1024
+ if entry.file_size and ratio > MAX_ARCHIVE_COMPRESSION_RATIO:
1025
+ _archive_refusal("import_zip_bloat",
1026
+ "That workbook archive is compressed too densely to import safely.")
1027
+ worksheets = [entry for entry in members
1028
+ if re.fullmatch(r"xl/worksheets/sheet[^/]*\.xml", entry.filename)]
1029
+ for entry in worksheets:
1030
+ with archive.open(entry) as stream:
1031
+ _guard_worksheet_xml(stream)
1032
+ except ImportRefusal:
1033
+ raise
1034
+ except (OSError, zipfile.BadZipFile, zipfile.LargeZipFile) as exc:
1035
+ raise ImportRefusal(
1036
+ "import_unreadable",
1037
+ "That file could not be read as a workbook. Save it again as .xlsx and try once more.") from exc
1038
+
1039
+
1040
  def _print_of(ws):
1041
  """Store print setup and inclusive print areas; export executes these settings later."""
1042
  setup = {key: getattr(ws.page_setup, key) for key in ws.page_setup.__attrs__
 
1112
  ⚠ `progress(state, fraction)` IS CALLED FROM WHATEVER THREAD RUNS THIS. It must not raise and
1113
  must not block; the job's registry entry is the only thing it touches.
1114
  """
 
 
1115
  def _say(state, fraction):
1116
  if progress:
1117
  try:
 
1120
  pass
1121
 
1122
  _say("reading", 0.02)
1123
+ # The guard is deliberately before both the import and the first load_workbook call. A
1124
+ # hostile declared dimension or ZIP bomb must not reach openpyxl's object allocator.
1125
+ guard_import_archive(path)
1126
+ import openpyxl
1127
+
1128
  try:
1129
  formulas = openpyxl.load_workbook(path, data_only=False, read_only=False,
1130
  keep_links=False)
 
1226
  tab["hiddenRows"] = hidden_rows
1227
  if hidden_cols:
1228
  tab["hiddenCols"] = hidden_cols
1229
+ view = _view_of(ws)
1230
+ if view:
1231
+ tab["view"] = view
1232
  charts = _charts_of(ws, name, notes)
1233
  if name in chart_tabs:
1234
  tab["id"] = chart_tabs[name]["id"]
 
1244
  values.close()
1245
  except Exception: # noqa: BLE001
1246
  pass
 
 
 
 
 
1247
  # Original styles are shared once, with integer references on cells, so exact formats,
1248
  # theme colours, border weights and blank-cell styles survive without repeating XML per cell.
1249
  original = {}
 
1265
  }
1266
 
1267
 
1268
+ def export_imported_workbook(book, path):
1269
+ """Export current cell values with an imported workbook's exact source styles.
1270
+
1271
+ This is the lossless import round-trip seam, not the general Sheet exporter: cells without an
1272
+ imported ``excel.styleId`` use the generated default style. The original style table is
1273
+ restored only after openpyxl has written a structurally valid workbook, so current values,
1274
+ formulas, dimensions, visibility, freeze and view state come from the cleaned document while
1275
+ source fills, fonts, alignments, borders and number formats keep their exact OOXML records.
1276
+ """
1277
+ import openpyxl
1278
+
1279
+ document = sheets.migrate_workbook(book, lossless=True)
1280
+ target = str(path)
1281
+ output = openpyxl.Workbook()
1282
+ output.remove(output.active)
1283
+ source_styles = ((document.get("excel") or {}).get("stylesXml") or "")
1284
+ source_theme = ((document.get("excel") or {}).get("themeXml") or "")
1285
+ style_ids = []
1286
+ for tab in document.get("sheets") or []:
1287
+ ws = output.create_sheet(str(tab.get("name") or "Sheet")[:31])
1288
+ ws.sheet_state = tab.get("visibility", "visible")
1289
+ by_addr = {}
1290
+ for addr, cell in (tab.get("cells") or {}).items():
1291
+ if not isinstance(cell, dict):
1292
+ continue
1293
+ target_cell = ws[addr]
1294
+ if cell.get("kind") == "formula" and isinstance(cell.get("formula"), str):
1295
+ target_cell.value = cell["formula"]
1296
+ elif cell.get("kind") != "empty":
1297
+ target_cell.value = cell.get("value", cell.get("text"))
1298
+ style_id = (cell.get("excel") or {}).get("styleId")
1299
+ if isinstance(style_id, int) and not isinstance(style_id, bool) and style_id >= 0:
1300
+ by_addr[addr] = style_id
1301
+ for addr, style_id in ((tab.get("excel") or {}).get("blankStyles") or {}).items():
1302
+ if isinstance(style_id, int) and not isinstance(style_id, bool) and style_id >= 0:
1303
+ ws[addr] # Ensure the style-only cell exists in sheetData before the OOXML patch.
1304
+ by_addr.setdefault(addr, style_id)
1305
+
1306
+ # Exact Excel dimension records win. The canonical pixel maps remain the fallback for a
1307
+ # document whose source metadata predates this import seam.
1308
+ column_records = (tab.get("excel") or {}).get("columnDimensions") or {}
1309
+ for key, record in column_records.items():
1310
+ if not isinstance(record, dict):
1311
+ continue
1312
+ dim = ws.column_dimensions[str(key)]
1313
+ if record.get("width") is not None:
1314
+ dim.width = float(record["width"])
1315
+ if record.get("hidden") is not None:
1316
+ dim.hidden = str(record["hidden"]).lower() in ("1", "true")
1317
+ for key, width in (tab.get("colWidths") or {}).items():
1318
+ if key not in column_records:
1319
+ ws.column_dimensions[key].width = 0 if width == 0 else max(0, (float(width) - 5) / 7)
1320
+ row_records = (tab.get("excel") or {}).get("rowDimensions") or {}
1321
+ for key, record in row_records.items():
1322
+ if not isinstance(record, dict) or not str(key).isdigit():
1323
+ continue
1324
+ dim = ws.row_dimensions[int(key)]
1325
+ if record.get("ht") is not None:
1326
+ dim.height = float(record["ht"])
1327
+ if record.get("hidden") is not None:
1328
+ dim.hidden = str(record["hidden"]).lower() in ("1", "true")
1329
+ for key, height in (tab.get("rowHeights") or {}).items():
1330
+ if key not in row_records:
1331
+ ws.row_dimensions[int(key)].height = float(height) * 3 / 4
1332
+ for number in tab.get("hiddenRows") or []:
1333
+ ws.row_dimensions[int(number)].hidden = True
1334
+ for number in tab.get("hiddenCols") or []:
1335
+ ws.column_dimensions[_addr(int(number) - 1, 0).rstrip("0123456789")].hidden = True
1336
+
1337
+ freeze = tab.get("freeze") or {}
1338
+ if freeze.get("rows") or freeze.get("cols"):
1339
+ ws.freeze_panes = _addr(int(freeze.get("cols") or 0), int(freeze.get("rows") or 0))
1340
+ view = tab.get("view") or {}
1341
+ if "gridlines" in view:
1342
+ ws.sheet_view.showGridLines = bool(view["gridlines"])
1343
+ if "headings" in view:
1344
+ ws.sheet_view.showRowColHeaders = bool(view["headings"])
1345
+ if "zoom" in view:
1346
+ ws.sheet_view.zoomScale = int(view["zoom"])
1347
+ if view.get("pageView") in ("normal", "pageBreak", "pageLayout"):
1348
+ ws.sheet_view.view = {"normal": "normal", "pageBreak": "pageBreakPreview",
1349
+ "pageLayout": "pageLayout"}[view["pageView"]]
1350
+ style_ids.append(by_addr)
1351
+
1352
+ output.save(target)
1353
+ output.close()
1354
+ if not source_styles:
1355
+ return target
1356
+
1357
+ namespace = "http://schemas.openxmlformats.org/spreadsheetml/2006/main"
1358
+ ET.register_namespace("", namespace)
1359
+ with zipfile.ZipFile(target) as source:
1360
+ parts = [(entry, source.read(entry.filename)) for entry in source.infolist()]
1361
+ patched = []
1362
+ for entry, blob in parts:
1363
+ match = re.fullmatch(r"xl/worksheets/sheet([0-9]+)\.xml", entry.filename)
1364
+ if match and int(match.group(1)) <= len(style_ids):
1365
+ wanted = style_ids[int(match.group(1)) - 1]
1366
+ if wanted:
1367
+ root = ET.fromstring(blob)
1368
+ for cell in root.iter("{" + namespace + "}c"):
1369
+ style_id = wanted.get(cell.get("r"))
1370
+ if style_id is not None:
1371
+ cell.set("s", str(style_id))
1372
+ blob = ET.tostring(root, encoding="utf-8", xml_declaration=True)
1373
+ elif entry.filename == "xl/styles.xml":
1374
+ blob = source_styles.encode("utf-8")
1375
+ elif entry.filename == "xl/theme/theme1.xml" and source_theme:
1376
+ blob = source_theme.encode("utf-8")
1377
+ patched.append((entry, blob))
1378
+ handle, replacement = tempfile.mkstemp(prefix="sheet-export-", suffix=".xlsx",
1379
+ dir=os.path.dirname(os.path.abspath(target)))
1380
+ os.close(handle)
1381
+ try:
1382
+ with zipfile.ZipFile(replacement, "w") as archive:
1383
+ for entry, blob in patched:
1384
+ archive.writestr(entry, blob)
1385
+ os.replace(replacement, target)
1386
+ finally:
1387
+ try:
1388
+ os.remove(replacement)
1389
+ except OSError:
1390
+ pass
1391
+ return target
1392
+
1393
+
1394
  def take_imported_values(book):
1395
  """Pull every `imported_value` off the document and return `{(tab, addr): cached}`.
1396
 
platform/core/sheet_templates.py CHANGED
@@ -9,7 +9,7 @@ from copy import deepcopy
9
  from datetime import date, datetime, timedelta, timezone
10
  import re
11
 
12
- from core import sheets, user_tables
13
 
14
 
15
  # Do not use connector_spec.connector_of: it only covers SPEC'd tables and calls
@@ -205,7 +205,7 @@ def save_sheet_template(st, workbook, key, *, name="", description="", created_b
205
  databases.add(database)
206
  record = {"key": key, "name": name.strip() or book.get("name") or key,
207
  "description": description, "databases": sorted(databases), "workbook": book,
208
- "created_by": created_by, "created_at": datetime.now(timezone.utc).isoformat(),
209
  "byName": actor_name or created_by}
210
 
211
  def save(state):
 
9
  from datetime import date, datetime, timedelta, timezone
10
  import re
11
 
12
+ from core import sheets, user_tables, write_execution
13
 
14
 
15
  # Do not use connector_spec.connector_of: it only covers SPEC'd tables and calls
 
205
  databases.add(database)
206
  record = {"key": key, "name": name.strip() or book.get("name") or key,
207
  "description": description, "databases": sorted(databases), "workbook": book,
208
+ "created_by": created_by, "created_at": write_execution.utcnow().isoformat(),
209
  "byName": actor_name or created_by}
210
 
211
  def save(state):
platform/core/sheets.py CHANGED
@@ -60,8 +60,7 @@ import hashlib
60
  import json
61
  import math
62
  import re
63
- import uuid
64
- from datetime import datetime, timezone
65
 
66
  from core.sheet_binding import (
67
  BindingMetadataRefusal,
@@ -247,7 +246,7 @@ SHEET_CHART_STACKING = ("clustered", "stacked", "percent")
247
  #:
248
  #: The order is the order a cleaned tab reads in: identity, then A2's geometry, then C2's strata.
249
  SHEET_TAB_KEYS = ("id", "name", "cols", "rows", "cells", "usedExtent", "colWidths", "freeze",
250
- "rules", "hiddenRows", "hiddenCols", "charts", "imported", "rowHeight", "rowHeights",
251
  "visibility", "view", "print", "excel")
252
 
253
  #: The keys above that `_clean_sheet` writes UNCONDITIONALLY. Every other member of
@@ -261,8 +260,17 @@ _MAX_HIDDEN = 5000
261
  _MAX_CHARTS = 50
262
  _MAX_UNSUPPORTED = 200
263
 
264
- #: A colour is a `--lp-*` TOKEN NAME, not a hue. Hex is allowed so a future palette door is not
265
- #: locked out, but nothing free form reaches a canvas `fillStyle` from a stored document.
 
 
 
 
 
 
 
 
 
266
  _COLOUR = re.compile(r"^[A-Za-z0-9#_-]{1,40}$")
267
 
268
  _MAX_DECIMALS = 10
@@ -280,7 +288,7 @@ _MAX_DECIMALS = 10
280
  #: sheet at the ceiling is still far above `_MAX_CELLS_PER_SHEET`, so content runs out first.
281
  DEFAULT_SHEET_COLS = 200
282
  DEFAULT_SHEET_ROWS = 10000
283
- DEFAULT_COL_WIDTH = 120
284
  MIN_SHEET_COLS = 1
285
  MIN_SHEET_ROWS = 1
286
  MAX_SHEET_COLS = 702
@@ -367,7 +375,7 @@ class SheetStaleRefusal(SheetRefusal):
367
 
368
 
369
  def _now():
370
- return datetime.now(timezone.utc).replace(microsecond=0).isoformat()
371
 
372
 
373
  def workbook_key(workbook_id):
@@ -399,6 +407,7 @@ def _read(st, key):
399
 
400
  def _persist(st, key, mutate, seed=None):
401
  """Read-modify-write under the store's own lock, mirroring `chart_system._persist`."""
 
402
  result = {}
403
 
404
  def apply(current):
@@ -408,9 +417,12 @@ def _persist(st, key, mutate, seed=None):
408
  state.setdefault(k, copy.deepcopy(v))
409
  next_state, value = mutate(state)
410
  result["value"] = copy.deepcopy(value)
 
 
411
  return next_state
412
 
413
  st.update(key, apply, flush="sync")
 
414
  return result.get("value")
415
 
416
 
@@ -530,6 +542,29 @@ def _clean_format(raw):
530
  return out
531
 
532
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
533
  def _clean_style(raw):
534
  """Normalize a cell's A1 style, or None when nothing survives.
535
 
@@ -552,8 +587,8 @@ def _clean_style(raw):
552
  out[mark] = True
553
 
554
  for key in ("color", "fill"):
555
- token = _text(raw.get(key), 40)
556
- if token and _COLOUR.match(token):
557
  out[key] = token
558
 
559
  align = _text(raw.get("align"), 12)
@@ -583,13 +618,16 @@ def _clean_style(raw):
583
  #: picked, gone on the next read with nothing to say so. The type is decided FIRST, and a step
584
  #: name still round trips exactly as it always did. A point size is CLAMPED, not refused, for
585
  #: the rotation's reason below, and the default is stored as an absence just as `m` is.
 
 
 
586
  size = raw.get("size")
587
  if isinstance(size, str):
588
  step = _text(size, 4)
589
  if step in CELL_SIZES and step != "m":
590
  out["size"] = step
591
- elif not isinstance(size, bool) and isinstance(size, (int, float)):
592
- points = max(MIN_FONT_PT, min(MAX_FONT_PT, int(round(size))))
593
  if points != DEFAULT_FONT_PT:
594
  out["size"] = points
595
 
@@ -1365,6 +1403,32 @@ def _clean_imported(raw):
1365
  return out if (out["at"] or out["source"] or notes) else None
1366
 
1367
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1368
  def _clean_sheet(raw, index=0, *, lossless=False):
1369
  """Normalize a tab's known strata and retain additive JSON metadata without interpretation.
1370
 
@@ -1404,6 +1468,7 @@ def _clean_sheet(raw, index=0, *, lossless=False):
1404
  "colWidths": _clean_col_widths(raw.get("colWidths"), cols, lossless=lossless),
1405
  "freeze": _clean_freeze(raw.get("freeze"), cols, rows, lossless=lossless),
1406
  "rules": _clean_rules(raw.get("rules"), lossless=lossless),
 
1407
  "hiddenRows": _clean_hidden(raw.get("hiddenRows"), "rows"),
1408
  "hiddenCols": _clean_hidden(raw.get("hiddenCols"), "columns"),
1409
  "charts": _clean_charts(raw.get("charts")),
@@ -1481,13 +1546,13 @@ def migrate_workbook(raw, *, lossless=True):
1481
 
1482
  def new_workbook(name="", created_by="", workbook_id=""):
1483
  """The in-memory shape of a fresh workbook with ONE tab (R7: a workbook has tabs)."""
1484
- wid = _slug_id(workbook_id) if workbook_id else uuid.uuid4().hex[:12]
1485
  stamp = _now()
1486
  return {
1487
  "id": wid,
1488
  "schemaVersion": SHEET_SCHEMA_VERSION,
1489
  "name": _text(name, _MAX_NAME) or "Untitled workbook",
1490
- "sheets": [{"id": uuid.uuid4().hex[:12], "name": "Sheet1",
1491
  "cols": DEFAULT_SHEET_COLS, "rows": DEFAULT_SHEET_ROWS, "cells": {},
1492
  "usedExtent": None, "writesFrom": 0}],
1493
  "rev": 1,
@@ -1764,6 +1829,132 @@ def find_tab(book, tab_ref):
1764
  return tabs[at]
1765
 
1766
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1767
  def workbook_meta(book):
1768
  """The workbook WITHOUT any cells, for C3's `?tabs=meta`.
1769
 
 
60
  import json
61
  import math
62
  import re
63
+ from core import write_execution
 
64
 
65
  from core.sheet_binding import (
66
  BindingMetadataRefusal,
 
246
  #:
247
  #: The order is the order a cleaned tab reads in: identity, then A2's geometry, then C2's strata.
248
  SHEET_TAB_KEYS = ("id", "name", "cols", "rows", "cells", "usedExtent", "colWidths", "freeze",
249
+ "rules", "rowGroups", "hiddenRows", "hiddenCols", "charts", "imported", "rowHeight", "rowHeights",
250
  "visibility", "view", "print", "excel")
251
 
252
  #: The keys above that `_clean_sheet` writes UNCONDITIONALLY. Every other member of
 
260
  _MAX_CHARTS = 50
261
  _MAX_UNSUPPORTED = 200
262
 
263
+ #: ⭐⭐ SDLC 64 R2 / CONTRACT C1 — THE ACCEPTED FORM OF A CELL COLOUR, AS ONE NAMED PATTERN.
264
+ #: Mirrored BYTE FOR BYTE in `aios-web/web/src/sheet/types.ts::CELL_COLOUR_PATTERN`, and
265
+ #: `verify_sheet.py`'s C1 leg compares the two source strings, so the client cannot start
266
+ #: writing a form this cleaner would drop.
267
+ CELL_COLOUR_PATTERN = r"^(--lp-[a-z0-9-]{1,40}|#[0-9a-f]{6})$"
268
+ _CELL_COLOUR = re.compile(CELL_COLOUR_PATTERN)
269
+
270
+ #: R2 / C1: CELL `color` / `fill` use CELL_COLOUR_PATTERN above. This older guard stays on
271
+ #: `_clean_rule_style` and `_clean_scale`: their RULE_FILLS / RULE_INKS galleries remain token
272
+ #: only. Its existing storage allowance is unchanged; a cell fill's custom door does not widen
273
+ #: the conditional-format palettes or make either guard dead.
274
  _COLOUR = re.compile(r"^[A-Za-z0-9#_-]{1,40}$")
275
 
276
  _MAX_DECIMALS = 10
 
288
  #: sheet at the ceiling is still far above `_MAX_CELLS_PER_SHEET`, so content runs out first.
289
  DEFAULT_SHEET_COLS = 200
290
  DEFAULT_SHEET_ROWS = 10000
291
+ DEFAULT_COL_WIDTH = 100
292
  MIN_SHEET_COLS = 1
293
  MIN_SHEET_ROWS = 1
294
  MAX_SHEET_COLS = 702
 
375
 
376
 
377
  def _now():
378
+ return write_execution.utcnow().replace(microsecond=0).isoformat()
379
 
380
 
381
  def workbook_key(workbook_id):
 
407
 
408
  def _persist(st, key, mutate, seed=None):
409
  """Read-modify-write under the store's own lock, mirroring `chart_system._persist`."""
410
+ from core import sheet_events
411
  result = {}
412
 
413
  def apply(current):
 
417
  state.setdefault(k, copy.deepcopy(v))
418
  next_state, value = mutate(state)
419
  result["value"] = copy.deepcopy(value)
420
+ # A store retry replaces this delta; no observer runs for a discarded transaction.
421
+ result["events"] = sheet_events.committed_changes(current, next_state)
422
  return next_state
423
 
424
  st.update(key, apply, flush="sync")
425
+ sheet_events.publish(st, result.get("events", []))
426
  return result.get("value")
427
 
428
 
 
542
  return out
543
 
544
 
545
+ def _normalise_colour(token):
546
+ """One cell colour in C1's accepted form, or "" when nothing survives.
547
+
548
+ ⭐⭐ C1's FORM IS LOWERCASE `#rrggbb`, AND THAT IS WHAT IS STORED, SO A RELOAD COMPARES
549
+ BYTE FOR BYTE EQUAL TO THE SAVE. A colour already in that form is returned unchanged:
550
+ normalisation is the identity on everything the client writes, which is the whole point.
551
+
552
+ ⚠ THE TWO COERCIONS ARE A RESCUE, NOT A CONVENIENCE. `#AABBCC` and `#abc` are forms an
553
+ IMPORTED workbook can carry, and this cleaner's house rule is to DROP what it does not
554
+ recognise, in silence. Dropping a colour somebody can see is the exact failure R2 named,
555
+ so a case difference and the three digit shorthand are widened onto the stored form
556
+ instead of being thrown away. Anything else is still dropped.
557
+ """
558
+ if not token:
559
+ return ""
560
+ if token.startswith("#"):
561
+ body = token[1:].lower()
562
+ if len(body) == 3 and all(c in "0123456789abcdef" for c in body):
563
+ body = body[0] * 2 + body[1] * 2 + body[2] * 2
564
+ token = "#" + body
565
+ return token if _CELL_COLOUR.match(token) else ""
566
+
567
+
568
  def _clean_style(raw):
569
  """Normalize a cell's A1 style, or None when nothing survives.
570
 
 
587
  out[mark] = True
588
 
589
  for key in ("color", "fill"):
590
+ token = _normalise_colour(_text(raw.get(key), 40))
591
+ if token:
592
  out[key] = token
593
 
594
  align = _text(raw.get("align"), 12)
 
618
  #: picked, gone on the next read with nothing to say so. The type is decided FIRST, and a step
619
  #: name still round trips exactly as it always did. A point size is CLAMPED, not refused, for
620
  #: the rotation's reason below, and the default is stored as an absence just as `m` is.
621
+ #: Imported fractional points remain fractional: 11.5pt is authored workbook data, while
622
+ #: ribbon steps happen to be integers. Rounding here made an import unequal to its source on
623
+ #: the first clean/save cycle.
624
  size = raw.get("size")
625
  if isinstance(size, str):
626
  step = _text(size, 4)
627
  if step in CELL_SIZES and step != "m":
628
  out["size"] = step
629
+ elif not isinstance(size, bool) and isinstance(size, (int, float)) and math.isfinite(size):
630
+ points = max(MIN_FONT_PT, min(MAX_FONT_PT, size))
631
  if points != DEFAULT_FONT_PT:
632
  out["size"] = points
633
 
 
1403
  return out if (out["at"] or out["source"] or notes) else None
1404
 
1405
 
1406
+ def _clean_row_groups(raw, rows):
1407
+ """Inclusive sheet ranges, with independent nested collapse state and no empty stratum."""
1408
+ if raw is None:
1409
+ return None
1410
+ if not isinstance(raw, list):
1411
+ raise SheetRefusal("sheet_row_groups_invalid", "Row groups must be a list.")
1412
+ out, ids = [], set()
1413
+ for item in raw:
1414
+ if not isinstance(item, dict):
1415
+ raise SheetRefusal("sheet_row_groups_invalid", "A row group must be an object.")
1416
+ gid, start, end, collapsed = (item.get(key) for key in ("id", "start", "end", "collapsed"))
1417
+ if not isinstance(gid, str) or not gid.strip() or len(gid) > _MAX_ID or gid in ids \
1418
+ or any(isinstance(n, bool) or not isinstance(n, int) for n in (start, end)) \
1419
+ or not 1 <= start <= end <= rows or not isinstance(collapsed, bool):
1420
+ raise SheetRefusal("sheet_row_groups_invalid", "A row group needs an id, a valid row range and a collapse state.")
1421
+ for prior in out:
1422
+ disjoint = end < prior["start"] or start > prior["end"]
1423
+ nested = (start >= prior["start"] and end <= prior["end"]) \
1424
+ or (start <= prior["start"] and end >= prior["end"])
1425
+ if (start, end) == (prior["start"], prior["end"]) or not (disjoint or nested):
1426
+ raise SheetRefusal("sheet_row_groups_overlap", "Groups can be nested, but cannot duplicate or partially overlap.")
1427
+ ids.add(gid)
1428
+ out.append({"id": gid, "start": start, "end": end, "collapsed": collapsed})
1429
+ return out or None
1430
+
1431
+
1432
  def _clean_sheet(raw, index=0, *, lossless=False):
1433
  """Normalize a tab's known strata and retain additive JSON metadata without interpretation.
1434
 
 
1468
  "colWidths": _clean_col_widths(raw.get("colWidths"), cols, lossless=lossless),
1469
  "freeze": _clean_freeze(raw.get("freeze"), cols, rows, lossless=lossless),
1470
  "rules": _clean_rules(raw.get("rules"), lossless=lossless),
1471
+ "rowGroups": _clean_row_groups(raw.get("rowGroups"), rows),
1472
  "hiddenRows": _clean_hidden(raw.get("hiddenRows"), "rows"),
1473
  "hiddenCols": _clean_hidden(raw.get("hiddenCols"), "columns"),
1474
  "charts": _clean_charts(raw.get("charts")),
 
1546
 
1547
  def new_workbook(name="", created_by="", workbook_id=""):
1548
  """The in-memory shape of a fresh workbook with ONE tab (R7: a workbook has tabs)."""
1549
+ wid = _slug_id(workbook_id) if workbook_id else write_execution.new_id()[:12]
1550
  stamp = _now()
1551
  return {
1552
  "id": wid,
1553
  "schemaVersion": SHEET_SCHEMA_VERSION,
1554
  "name": _text(name, _MAX_NAME) or "Untitled workbook",
1555
+ "sheets": [{"id": write_execution.new_id()[:12], "name": "Sheet1",
1556
  "cols": DEFAULT_SHEET_COLS, "rows": DEFAULT_SHEET_ROWS, "cells": {},
1557
  "usedExtent": None, "writesFrom": 0}],
1558
  "rev": 1,
 
1829
  return tabs[at]
1830
 
1831
 
1832
+ def workbook_snapshot(st, workbook_id):
1833
+ """Return the stored workbook bytes as JSON data, without migration or cleaning.
1834
+
1835
+ The ordinary read door intentionally migrates old workbooks. A narrowly scoped repair needs
1836
+ a different primitive: its backup and compare-and-swap digest must describe the document that
1837
+ is actually stored, or a migration performed only in memory could make the precondition look
1838
+ stronger than it is. The returned copy is detached from the store.
1839
+ """
1840
+ wid = _slug_id(workbook_id)
1841
+ doc = _read(st, workbook_key(wid))
1842
+ if not doc or not doc.get("id"):
1843
+ raise SheetRefusal("workbook_missing", "That workbook no longer exists.")
1844
+ return copy.deepcopy(doc)
1845
+
1846
+
1847
+ def workbook_digest(book):
1848
+ """A stable digest for an exact workbook snapshot."""
1849
+ if not isinstance(book, dict):
1850
+ raise SheetRefusal("workbook_not_an_object", "A workbook must be an object.")
1851
+ encoded = json.dumps(book, sort_keys=True, separators=(",", ":"), ensure_ascii=False,
1852
+ default=str).encode("utf-8")
1853
+ return hashlib.sha256(encoded).hexdigest()
1854
+
1855
+
1856
+ def apply_exact_cell_edits(st, workbook_id, edits, expect_rev, expect_digest, *,
1857
+ actor="", actor_name=""):
1858
+ """Apply a preconditioned list of cell edits under the store lock.
1859
+
1860
+ This is deliberately narrower than PUT and stricter than the collaborative range PATCH. Each
1861
+ edit names one tab/address and carries the exact value observed while the repair was planned.
1862
+ The workbook revision, whole-document digest and every cell precondition must still match.
1863
+ Nothing outside those cells, normal revision/write-audit fields and derived used extents is
1864
+ rewritten. It is the storage primitive for exceptional, owner-authorized workbook repair;
1865
+ normal Sheet editing continues to use ``put_workbook`` and ``patch_cells``.
1866
+ """
1867
+ wid = _slug_id(workbook_id)
1868
+ if isinstance(expect_rev, bool) or not isinstance(expect_rev, int) or expect_rev < 0:
1869
+ raise SheetRefusal("repair_revision_required",
1870
+ "An exact workbook revision is required for this repair.")
1871
+ if not isinstance(expect_digest, str) or not re.fullmatch(r"[0-9a-f]{64}", expect_digest):
1872
+ raise SheetRefusal("repair_digest_required",
1873
+ "An exact workbook snapshot digest is required for this repair.")
1874
+ if not isinstance(edits, list) or not edits:
1875
+ raise SheetRefusal("repair_edits_required", "This repair does not name any cell edits.")
1876
+
1877
+ prepared, seen = [], set()
1878
+ for raw in edits:
1879
+ if not isinstance(raw, dict):
1880
+ raise SheetRefusal("repair_edit_invalid", "Every repair edit must be an object.")
1881
+ tab_ref = str(raw.get("sheet") or "").strip()
1882
+ addr = str(raw.get("address") or "").strip().upper().replace("$", "")
1883
+ matched = _ADDR.fullmatch(addr)
1884
+ if (not tab_ref or not matched
1885
+ or _col_index(re.match(r"[A-Z]+", addr).group(0)) >= DOCUMENT_MAX_COLS
1886
+ or int(re.search(r"[0-9]+$", addr).group(0)) > DOCUMENT_MAX_ROWS):
1887
+ raise SheetRefusal("repair_edit_invalid",
1888
+ "Every repair edit must name one sheet and one A1 address.")
1889
+ slot = (tab_ref.casefold(), addr)
1890
+ if slot in seen:
1891
+ raise SheetRefusal("repair_edit_duplicate",
1892
+ "A workbook repair may name each cell only once.")
1893
+ seen.add(slot)
1894
+ after = raw.get("after")
1895
+ prepared.append({"sheet": tab_ref, "address": addr,
1896
+ "before": copy.deepcopy(raw.get("before")),
1897
+ "after": None if after is None else clean_cell(after, addr=addr)})
1898
+ prepared.sort(key=lambda row: (row["sheet"].casefold(), row["address"]))
1899
+
1900
+ saved = {}
1901
+
1902
+ def _apply(state):
1903
+ prior = state if state.get("id") else None
1904
+ if prior is None:
1905
+ raise SheetRefusal("workbook_missing", "That workbook no longer exists.")
1906
+ if _slug_id(prior.get("id")) != wid:
1907
+ raise SheetRefusal("repair_workbook_mismatch",
1908
+ "The stored workbook identity does not match this repair target.")
1909
+ current = int(prior.get("rev") or 0)
1910
+ if current != expect_rev:
1911
+ raise SheetStaleRefusal(
1912
+ "workbook_revision_stale",
1913
+ "That workbook changed after the repair was planned, so no repair was applied.",
1914
+ rev_current=current)
1915
+ if workbook_digest(prior) != expect_digest:
1916
+ raise SheetRefusal(
1917
+ "repair_snapshot_changed",
1918
+ "That workbook changed without a revision change, so no repair was applied.")
1919
+
1920
+ book = copy.deepcopy(prior)
1921
+ touched = {}
1922
+ for edit in prepared:
1923
+ at = _tab_index(book.get("sheets") or [], edit["sheet"])
1924
+ if at is None:
1925
+ raise SheetRefusal("tab_missing", "Tab " + edit["sheet"]
1926
+ + " is not in this workbook.")
1927
+ tab = book["sheets"][at]
1928
+ cells = tab.setdefault("cells", {})
1929
+ actual = copy.deepcopy(cells.get(edit["address"]))
1930
+ if actual != edit["before"]:
1931
+ raise SheetRefusal(
1932
+ "repair_cell_changed",
1933
+ "Cell " + edit["sheet"] + "!" + edit["address"]
1934
+ + " changed after the repair was planned, so no repair was applied.")
1935
+ if edit["after"] is None:
1936
+ cells.pop(edit["address"], None)
1937
+ else:
1938
+ cells[edit["address"]] = copy.deepcopy(edit["after"])
1939
+ touched.setdefault(at, []).append(edit["address"])
1940
+
1941
+ new_rev = current + 1
1942
+ for at, addrs in touched.items():
1943
+ tab = book["sheets"][at]
1944
+ tab["usedExtent"] = _used_extent(
1945
+ tab.get("cells") or {}, (tab.get("excel") or {}).get("blankStyles"))
1946
+ _append_write_log(tab, new_rev, addrs, actor=actor, actor_name=actor_name)
1947
+ book["rev"] = new_rev
1948
+ book["updated_at"] = _now()
1949
+ _json_safe(book, "workbook")
1950
+ saved.update(book)
1951
+ return book, {"rev": new_rev, "edits": len(prepared),
1952
+ "paths": [row["sheet"] + "!" + row["address"] for row in prepared]}
1953
+
1954
+ out = _persist(st, workbook_key(wid), _apply)
1955
+ return {**out, "digest": workbook_digest(saved)}
1956
+
1957
+
1958
  def workbook_meta(book):
1959
  """The workbook WITHOUT any cells, for C3's `?tabs=meta`.
1960
 
platform/core/table_store.py CHANGED
@@ -19,6 +19,7 @@ import time
19
 
20
  import core.store as store
21
  import core.view_audit as view_audit
 
22
 
23
  #: ``__shared__`` remains the tenant-wide FIELD-annotation stratum. Before Wave 48 it also
24
  #: carried a second, legacy home for shared Views. That view map is migrated away on the first
@@ -1727,7 +1728,7 @@ class TableStore:
1727
  data.pop(ASSIGNED_KEY, None)
1728
  if isinstance(before, dict):
1729
  bucket = data.setdefault(owner, {}).setdefault(TRASH_KEY, {})
1730
- stamp = time.time()
1731
  bucket[vid] = {'view': before, 'deleted_at': float(stamp),
1732
  'deleted_by': who, 'assignments': carried}
1733
  purge_trash(bucket, now=stamp)
@@ -1818,7 +1819,7 @@ class TableStore:
1818
  current.get(owner), dict) else None
1819
  if isinstance(live, dict):
1820
  live.pop(vid, None)
1821
- purge_trash(live, now=time.time())
1822
  if not live:
1823
  current[owner].pop(TRASH_KEY, None)
1824
  audit(action=None, owner=owner, view_id=vid)
@@ -1854,7 +1855,7 @@ class TableStore:
1854
  bucket = ((data.get(owner) or {}).get(TRASH_KEY) or {})
1855
  if not isinstance(bucket, dict) or not bucket:
1856
  return []
1857
- stamp = float(now if now is not None else time.time())
1858
  expired = [vid for vid, entry in bucket.items() if trash_expired(entry, now=stamp)]
1859
  # ⛔⛔ R11's ADOPTION RUNS BEFORE THE UPDATE AND OUTSIDE ITS CALLBACK (W51-T26).
1860
  # `sheets.py::save_folders` states the rule for this codebase in as many words: validation
@@ -1985,7 +1986,7 @@ class TableStore:
1985
  if isinstance(assignment, dict):
1986
  target[str(aid)] = dict(assignment)
1987
  bucket.pop(vid, None)
1988
- purge_trash(bucket, now=time.time())
1989
  if not bucket:
1990
  data[owner].pop(TRASH_KEY, None)
1991
  restored.clear()
 
19
 
20
  import core.store as store
21
  import core.view_audit as view_audit
22
+ from core import write_execution
23
 
24
  #: ``__shared__`` remains the tenant-wide FIELD-annotation stratum. Before Wave 48 it also
25
  #: carried a second, legacy home for shared Views. That view map is migrated away on the first
 
1728
  data.pop(ASSIGNED_KEY, None)
1729
  if isinstance(before, dict):
1730
  bucket = data.setdefault(owner, {}).setdefault(TRASH_KEY, {})
1731
+ stamp = write_execution.timestamp()
1732
  bucket[vid] = {'view': before, 'deleted_at': float(stamp),
1733
  'deleted_by': who, 'assignments': carried}
1734
  purge_trash(bucket, now=stamp)
 
1819
  current.get(owner), dict) else None
1820
  if isinstance(live, dict):
1821
  live.pop(vid, None)
1822
+ purge_trash(live, now=write_execution.timestamp())
1823
  if not live:
1824
  current[owner].pop(TRASH_KEY, None)
1825
  audit(action=None, owner=owner, view_id=vid)
 
1855
  bucket = ((data.get(owner) or {}).get(TRASH_KEY) or {})
1856
  if not isinstance(bucket, dict) or not bucket:
1857
  return []
1858
+ stamp = float(now if now is not None else write_execution.timestamp())
1859
  expired = [vid for vid, entry in bucket.items() if trash_expired(entry, now=stamp)]
1860
  # ⛔⛔ R11's ADOPTION RUNS BEFORE THE UPDATE AND OUTSIDE ITS CALLBACK (W51-T26).
1861
  # `sheets.py::save_folders` states the rule for this codebase in as many words: validation
 
1986
  if isinstance(assignment, dict):
1987
  target[str(aid)] = dict(assignment)
1988
  bucket.pop(vid, None)
1989
+ purge_trash(bucket, now=write_execution.timestamp())
1990
  if not bucket:
1991
  data[owner].pop(TRASH_KEY, None)
1992
  restored.clear()
platform/core/user_tables.py CHANGED
@@ -69,6 +69,22 @@ SHARED_NOTE_READS_KEY = 'shared_note_reads'
69
  ODOO_SCHEMA_SNAPSHOT_KEY = 'odoo_schema_snapshot'
70
  ODOO_SCHEMA_ENABLEMENTS_KEY = 'odoo_schema_enablements'
71
  ODOO_SYNC_STATUS_KEY = 'odoo_sync_status'
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
72
  # A reserved, derived multi-select field. It is table data backed by durable Zone membership,
73
  # not a user-editable column and therefore cannot collide with a field a user creates.
74
  ZONE_FIELD_KEY = '__zone__'
@@ -467,6 +483,74 @@ def connector_database_identity(table_key, rt=None, st=None, definition=None):
467
  declared.get('model') or (getattr(spec, 'native', '') if spec else ''))
468
 
469
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
470
  def credential_footprint(entry_type, connection_id, rt=None, st=None):
471
  """Disclose what revoking this ONE credential purges and what its policy keeps.
472
 
 
69
  ODOO_SCHEMA_SNAPSHOT_KEY = 'odoo_schema_snapshot'
70
  ODOO_SCHEMA_ENABLEMENTS_KEY = 'odoo_schema_enablements'
71
  ODOO_SYNC_STATUS_KEY = 'odoo_sync_status'
72
+ CONNECTOR_HEALTH_FIELD = 'connectorHealth'
73
+ CONNECTOR_HEALTH_STATES = (
74
+ 'never_run', 'queued', 'running', 'succeeded', 'partial', 'failed', 'disabled',
75
+ )
76
+ _CONNECTOR_SAFE_ERRORS = {
77
+ 'maps_credential_missing': 'Add the scraper credential in Keychain before choosing Run now.',
78
+ 'scraper_credential_missing': 'Add the scraper credential in Keychain before choosing Run now.',
79
+ 'maps_unbounded_request': 'Google Maps discovery requires an explicit bounded result cap.',
80
+ 'maps_invalid_search': 'Google Maps discovery needs a valid category or search.',
81
+ 'maps_vendor_error': 'The Maps provider could not complete this run.',
82
+ 'maps_schema_drift': 'The Maps provider returned an unreadable result shape.',
83
+ 'scraper_run_failed': 'The provider could not complete this run. No result was written.',
84
+ 'provider_result_failed': 'The provider returned no usable result for this database.',
85
+ 'scraper_result_not_saved': 'The provider answered, but the result could not be saved.',
86
+ 'sync_status_not_saved': 'The run did not start because its durable status could not be saved.',
87
+ }
88
  # A reserved, derived multi-select field. It is table data backed by durable Zone membership,
89
  # not a user-editable column and therefore cannot collide with a field a user creates.
90
  ZONE_FIELD_KEY = '__zone__'
 
483
  declared.get('model') or (getattr(spec, 'native', '') if spec else ''))
484
 
485
 
486
+ def connector_health_receipt(state, *, last_success=None, source_timestamp=None,
487
+ materialized_timestamp=None, row_count=None, error=None):
488
+ """Validate the durable, data-free connector database health document."""
489
+ def stamp(value):
490
+ text = str(value or '').strip()
491
+ if not text:
492
+ return None
493
+ try:
494
+ parsed = _dt.datetime.fromisoformat(text.replace('Z', '+00:00'))
495
+ except (TypeError, ValueError, OverflowError):
496
+ return None
497
+ return parsed.isoformat()
498
+
499
+ clean_state = str(state or '').strip()
500
+ if clean_state not in CONNECTOR_HEALTH_STATES:
501
+ raise ValueError('unknown connector database health state')
502
+ count = None
503
+ if row_count is not None:
504
+ if type(row_count) is not int or row_count < 0:
505
+ raise ValueError('row_count must be a non-negative integer or None')
506
+ count = row_count
507
+ clean_error = None
508
+ if error is not None:
509
+ raw = error if isinstance(error, dict) else {}
510
+ requested_code = str(raw.get('code') or '').strip()
511
+ code = requested_code if requested_code in _CONNECTOR_SAFE_ERRORS else 'connector_sync_failed'
512
+ # Never persist provider/exception prose. Only named classifications get specific copy.
513
+ message = _CONNECTOR_SAFE_ERRORS.get(code, 'This database could not be synced.')
514
+ clean_error = {'code': code, 'message': message}
515
+ return {
516
+ 'state': clean_state,
517
+ 'lastSuccess': stamp(last_success),
518
+ 'sourceTimestamp': stamp(source_timestamp),
519
+ 'materializedTimestamp': stamp(materialized_timestamp),
520
+ 'rowCount': count,
521
+ 'error': clean_error,
522
+ }
523
+
524
+
525
+ def record_connector_health(table_key, state, *, st=None, last_success=None,
526
+ source_timestamp=None, materialized_timestamp=None,
527
+ row_count=None, error=None):
528
+ """Persist one data-free database sync receipt beside its governed definition.
529
+
530
+ Provider runners supply already-sanitized error ``code``/``message`` values. This seam
531
+ deliberately accepts neither exceptions nor arbitrary provider payloads, so a failed run can
532
+ survive a process restart without turning the tenant store into a credential or row log.
533
+ """
534
+ key = str(table_key or '').strip()
535
+ if not key.startswith(KEY_PREFIX):
536
+ raise ValueError('a governed database key is required')
537
+ receipt = connector_health_receipt(
538
+ state, last_success=last_success, source_timestamp=source_timestamp,
539
+ materialized_timestamp=materialized_timestamp, row_count=row_count, error=error)
540
+
541
+ def _set(current):
542
+ document = dict(current or {})
543
+ definition = dict(document.get(key) or {})
544
+ if not definition:
545
+ raise KeyError(key)
546
+ definition[CONNECTOR_HEALTH_FIELD] = receipt
547
+ document[key] = definition
548
+ return document
549
+
550
+ landed = _st(st).update(STORE_KEY, _set, flush='sync')
551
+ return dict((landed.get(key) or {}).get(CONNECTOR_HEALTH_FIELD) or receipt)
552
+
553
+
554
  def credential_footprint(entry_type, connection_id, rt=None, st=None):
555
  """Disclose what revoking this ONE credential purges and what its policy keeps.
556
 
platform/core/write_execution.py ADDED
@@ -0,0 +1,45 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Request-local server metadata for replaying an explicitly confirmed write.
2
+
3
+ The gateway signs the seed and timestamp with the preview. Revalidation and the
4
+ ordinary writer reuse them so generated ids and timestamps cannot change the diff.
5
+ This does not replace document revision or permission checks. Unscoped UI writes
6
+ continue to use fresh ids and the wall clock.
7
+ """
8
+ from contextlib import contextmanager
9
+ from contextvars import ContextVar
10
+ from datetime import datetime, timezone
11
+ import time
12
+ import uuid
13
+
14
+
15
+ _execution = ContextVar('confirmed_write_execution', default=None)
16
+
17
+
18
+ @contextmanager
19
+ def replay(metadata):
20
+ state = {'seed': str(metadata['seed']), 'timestamp': float(metadata['timestamp']), 'index': 0}
21
+ token = _execution.set(state)
22
+ try:
23
+ yield
24
+ finally:
25
+ _execution.reset(token)
26
+
27
+
28
+ def new_id():
29
+ state = _execution.get()
30
+ if state is None:
31
+ return uuid.uuid4().hex
32
+ # Replace, never mutate, the context value: independently copied async contexts
33
+ # must not share a mutable sequence counter.
34
+ index = state['index']
35
+ _execution.set({**state, 'index': index + 1})
36
+ return uuid.uuid5(uuid.NAMESPACE_URL, f"loopable-write:{state['seed']}:{index}").hex
37
+
38
+
39
+ def timestamp():
40
+ state = _execution.get()
41
+ return time.time() if state is None else state['timestamp']
42
+
43
+
44
+ def utcnow():
45
+ return datetime.fromtimestamp(timestamp(), timezone.utc)
platform/harness/filter_sql.py CHANGED
@@ -157,6 +157,9 @@ _DATE_CMP = dict(_NUM_CMP)
157
  #: rather than WHERE; columns_used = colIds actually referenced in the emitted SQL (an
158
  #: always-true `1=1` branch references nothing, and must not force HAVING routing).
159
  Compiled = namedtuple('Compiled', 'sql params uses_aggregate columns_used')
 
 
 
160
 
161
 
162
  # --------------------------------------------------------------------------- value coercion
@@ -612,6 +615,10 @@ def _compile_cohort(rule, state):
612
 
613
  parts, params = [], []
614
  for ids in members:
 
 
 
 
615
  ids = [int(x) for x in ids]
616
  if not ids:
617
  parts.append('1=0')
 
157
  #: rather than WHERE; columns_used = colIds actually referenced in the emitted SQL (an
158
  #: always-true `1=1` branch references nothing, and must not force HAVING routing).
159
  Compiled = namedtuple('Compiled', 'sql params uses_aggregate columns_used')
160
+ #: HOTFIX 2026-09-23: a cohort set given as a SELECT of ids rather than a list. A server-side
161
+ #: rank answer (Top N, above average ...) over 256k rows is a subquery, never a 128k-id IN list.
162
+ SqlMembership = namedtuple('SqlMembership', 'sql params')
163
 
164
 
165
  # --------------------------------------------------------------------------- value coercion
 
615
 
616
  parts, params = [], []
617
  for ids in members:
618
+ if isinstance(ids, SqlMembership):
619
+ parts.append(f"COALESCE({id_sql} IN ({ids.sql}), FALSE)")
620
+ params.extend(ids.params)
621
+ continue
622
  ids = [int(x) for x in ids]
623
  if not ids:
624
  parts.append('1=0')
platform/harness/mirror_tokens.py ADDED
@@ -0,0 +1,64 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """The mirror's own change token, in the layer that owns the mirror (SDLC 65 combine).
2
+
3
+ ⛔⛔ WHY IT LIVES HERE AND NOT IN `core/sheet_binding.py`, WHERE S65-T03 WROTE IT.
4
+ It reads the analytical mirror: `harness.datastore.ENTITIES` and `harness.runtime.mirror_cursor`.
5
+ `core` never imports up (ARCHITECTURE.md section 1), and `verify_layering.py`'s D-460 leg caught
6
+ the new `core -> harness` edge as this wave's own violation. Moving the body up and registering it
7
+ back down is the fix the gate's first remedy names -- "move it down" read from the other end. The
8
+ alternative, a `GRANDFATHERED_CORE` entry, was REFUSED: that list is edges which PRE-DATE the gate,
9
+ and a wave adding its own edge to it exempts itself from the rule it just broke.
10
+
11
+ ⭐ Importing this module is what arms `core.sheet_binding.mirror_change_token`; `routes_sheet`
12
+ does it. Until then that function RAISES, which `database_change_token` already treats as "cannot
13
+ prove unchanged, therefore resolve" -- the safe direction.
14
+ """
15
+ from __future__ import annotations
16
+ import re
17
+
18
+ from core import sheet_binding
19
+ from core.sheet_binding import fingerprint
20
+
21
+
22
+ def mirror_change_token(st, database):
23
+ """One SQL token read over source tables, not N binding queries or Python row copies.
24
+
25
+ The current mirror has no per-table monotonic revision. Content fingerprints close
26
+ that gap without changing its writer or adding a row cap. Count, xor and sum include
27
+ entire records; schema and sync receipts cover field changes and no-op syncs too.
28
+ Registry metrics depend on the Odoo observation set, so their token includes it.
29
+ """
30
+ from core import connector_spec
31
+ from harness import datastore
32
+ from harness.runtime import mirror_cursor
33
+ spec = connector_spec.spec_for(database)
34
+ if database in ("customer_data", "product_data"):
35
+ sources = set(datastore.ENTITIES)
36
+ elif spec:
37
+ sources = {spec.source}
38
+ # A connector grid also reads joined label/identity columns declared by its spec.
39
+ sources.update(target.source for _, key, _, _ in spec.links
40
+ if (target := connector_spec.spec_for(key)) is not None)
41
+ else:
42
+ raise RuntimeError("The connector has no declared source for its change token.")
43
+ if not sources or any(not re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]*", s) for s in sources):
44
+ raise RuntimeError("The connector source identity is invalid.")
45
+ sources = sorted(sources)
46
+ receipt_sources = sorted(set(sources) | ({"meta"} if any(s.startswith("meta_") for s in sources) else set()))
47
+ clauses = [f"SELECT '{source}' AS source, CAST(count(*) AS VARCHAR) AS n, "
48
+ f"CAST(bit_xor(hash(t)) AS VARCHAR) AS x, "
49
+ f"CAST(sum(CAST(hash(t) AS HUGEINT)) AS VARCHAR) AS s FROM \"{source}\" t"
50
+ for source in sources]
51
+ placeholders = ",".join("?" for _ in sources)
52
+ clauses.append("SELECT 'schema', table_name, column_name, data_type FROM duckdb_columns() "
53
+ f"WHERE table_name IN ({placeholders})")
54
+ # _sync_state is the resident mirror writer's receipt, not the app's status cache.
55
+ clauses.append("SELECT 'sync', entity, phase, concat_ws('|', last_id, cursor_wd, rows, updated_at) "
56
+ "FROM _sync_state WHERE split_part(entity, '.', 1) IN ("
57
+ + ",".join("?" for _ in receipt_sources) + ")")
58
+ rows = mirror_cursor(st).execute(" UNION ALL ".join(clauses) + " ORDER BY 1,2,3,4",
59
+ [*sources, *receipt_sources]).fetchall()
60
+ if not any(row[0] == "sync" for row in rows):
61
+ raise RuntimeError("The connector has no sync receipt.")
62
+ return fingerprint(rows)
63
+
64
+ sheet_binding.set_mirror_token_provider(mirror_change_token)