fsanyoto commited on
Commit
c1fc133
·
verified ·
1 Parent(s): 367a695

Deploy AIOS web (React glide grid + FastAPI slice)

Browse files
This view is limited to 50 files because it contains too many changes.   See raw diff
Files changed (50) hide show
  1. RELEASES.json +7 -1
  2. VERSION +1 -1
  3. api/ai_review.py +3 -2
  4. api/automation_engine.py +68 -0
  5. api/connectors_amazon.py +132 -1
  6. api/main.py +175 -3
  7. api/odoo_relational.py +74 -5
  8. api/providers.py +2 -1
  9. api/routes_admin.py +6 -1
  10. api/routes_automation.py +121 -10
  11. api/routes_connectors.py +192 -25
  12. api/routes_customers.py +22 -4
  13. api/routes_dependents.py +19 -0
  14. api/routes_external.py +489 -26
  15. api/routes_grid.py +197 -22
  16. api/routes_keychain.py +8 -0
  17. api/routes_module_index.py +132 -0
  18. api/routes_nav.py +94 -20
  19. api/routes_odoo_tables.py +310 -306
  20. api/routes_products.py +23 -3
  21. api/routes_query.py +326 -29
  22. api/routes_sheet.py +717 -115
  23. api/routes_starred.py +196 -19
  24. api/routes_tables.py +99 -56
  25. api/routes_templates.py +179 -9
  26. api/semantic_registry.py +152 -14
  27. api/test_odoo_rank_refresh.py +65 -0
  28. platform/aios_grid.py +48 -0
  29. platform/core/connector_sync.py +133 -2
  30. platform/core/dependents.py +233 -40
  31. platform/core/external_access.py +80 -7
  32. platform/core/external_tools.py +372 -0
  33. platform/core/measure_resolve.py +4 -2
  34. platform/core/odoo.py +3 -1
  35. platform/core/perm_scope.py +17 -4
  36. platform/core/relations.py +33 -0
  37. platform/core/shares.py +51 -3
  38. platform/core/sheet_binding.py +180 -0
  39. platform/core/sheet_import.py +24 -0
  40. platform/core/sheet_structure.py +316 -0
  41. platform/core/sheet_templates.py +111 -5
  42. platform/core/sheets.py +543 -60
  43. platform/core/store_pg.py +69 -0
  44. platform/core/table_store.py +28 -0
  45. platform/core/user_tables.py +52 -26
  46. platform/core/windows.py +492 -0
  47. platform/harness/amazon_client.py +73 -1
  48. platform/harness/amazon_store.py +430 -21
  49. platform/harness/measure_filter.py +7 -2
  50. platform/harness/runtime.py +25 -0
RELEASES.json CHANGED
@@ -1,6 +1,12 @@
1
  {
2
- "current": "v74 (40dceba)",
3
  "releases": [
 
 
 
 
 
 
4
  {
5
  "version": "v74",
6
  "sha": "40dceba",
 
1
  {
2
+ "current": "v75 (2c51469)",
3
  "releases": [
4
+ {
5
+ "version": "v75",
6
+ "sha": "2c51469",
7
+ "date": "2026-09-19",
8
+ "subject": "release v75"
9
+ },
10
  {
11
  "version": "v74",
12
  "sha": "40dceba",
VERSION CHANGED
@@ -1 +1 @@
1
- v74 (40dceba)
 
1
+ v75 (2c51469)
api/ai_review.py CHANGED
@@ -174,8 +174,9 @@ def _call_openai(p, model, system, user, timeout):
174
 
175
 
176
  #: What transport the last Anthropic call actually used: `'sdk'`, `'http'`, or `''` before any.
177
- #: ⚠ PROCESS-LOCAL AND FOR REPORTING ONLY. It exists so `GET /meta` and `verify_web_agent` can say
178
- #: WHICH path ran rather than inferring it from a requirements file nobody in this lane can edit.
 
179
  LAST_ANTHROPIC_TRANSPORT = ""
180
 
181
 
 
174
 
175
 
176
  #: What transport the last Anthropic call actually used: `'sdk'`, `'http'`, or `''` before any.
177
+ #: ⚠ PROCESS-LOCAL AND FOR REPORTING ONLY. It exists so an administrator can ask WHICH path ran
178
+ #: rather than infer it from a requirements file: `GET /api/v1/automations/ai/transport`
179
+ #: (`routes_automation.ai_transport`, D-384) serves it beside the installed SDK version.
180
  LAST_ANTHROPIC_TRANSPORT = ""
181
 
182
 
api/automation_engine.py CHANGED
@@ -14553,6 +14553,74 @@ def relation_cells(rt, table_key, tables=None):
14553
  return changes, limits
14554
 
14555
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
14556
  def related_product_metric_values(rt, customer_id, products, fields, offer, *,
14557
  team_id=None, agent=None, today=None):
14558
  """Read-only customer × real product metrics; the caller supplies walled server rows.
 
14553
  return changes, limits
14554
 
14555
 
14556
+ def related_metric_values(rt, table_key, rows, fields, offer, related, *, team_id=None,
14557
+ today=None):
14558
+ """Read-only linked cells at customer, product or declared entity grain.
14559
+
14560
+ The host supplies the same resolved membership as its rows door and its walled catalogue.
14561
+ A request-local memo never survives a parent, account, permission or tenant change.
14562
+ """
14563
+ from core import measure_resolve, relations
14564
+ from harness import semantic as sem, windows
14565
+ import time
14566
+
14567
+ rt.assert_datastore_matches()
14568
+ today = today or time.strftime('%Y-%m-%d')
14569
+ by_key = {m['key']: m for m in offer}
14570
+ out = relations.MeasureAnswer({r['pid']: {f['key']: None for f in fields} for r in rows})
14571
+ memo = {}
14572
+ for field in fields:
14573
+ spec = field.get('measure') or {}
14574
+ key = spec.get('key')
14575
+ if key not in by_key:
14576
+ continue # The host removes unavailable catalogue entries before serialization.
14577
+ try:
14578
+ bounds = windows.resolve(spec.get('window'), today)
14579
+ if bounds is None:
14580
+ raise ValueError('The metric date window cannot be resolved. Choose a valid period.')
14581
+ if table_key == 'customer_data':
14582
+ errors = []
14583
+ values = measure_resolve.column_values(
14584
+ [field], team_id, [r['pid'] for r in rows], today, today, memo,
14585
+ on_error=lambda _where, exc: errors.append(exc), related=related)
14586
+ if errors:
14587
+ raise ValueError('This metric could not be read for the linked record. Try again after the database is ready.')
14588
+ out.refusals.extend(getattr(values, 'refusals', ()))
14589
+ for pid, cells in values.items():
14590
+ out[pid].update(cells)
14591
+ continue
14592
+ topic = sem.topic_for_grid(table_key)
14593
+ cache_key = (topic, key, bounds, team_id)
14594
+ if cache_key not in memo:
14595
+ memo[cache_key] = sem.entity_measure_values(
14596
+ topic, [key], date_from=bounds[0], date_to=bounds[1], team_id=team_id,
14597
+ offer=offer, related=related)
14598
+ values = memo[cache_key]
14599
+ markers = getattr(values, 'refusals', ())
14600
+ if markers:
14601
+ out.refusals.extend(dict(marker, field=field['key']) for marker in markers)
14602
+ continue # Never zero-fill a refused answer or expose its unscoped values.
14603
+ for row in rows:
14604
+ identity = row.get('code') if table_key == 'product_data' else row['pid']
14605
+ if identity in (None, ''):
14606
+ raise ValueError('The metric identity cannot be read for this record.')
14607
+ cell = sem.entity_zero_fill(values.get(identity), [key], offer)
14608
+ out[row['pid']][field['key']] = cell.get(key)
14609
+ except Exception as exc:
14610
+ message = (str(exc) if isinstance(exc, (relations.PinRefusal, sem._EntityMeasureError)) else
14611
+ 'This metric cannot be read for the linked record. Check its period and source binding, then try again.')
14612
+ out.refusals.append(dict(code='metric_scope_unavailable', message=message,
14613
+ path=related.path, field=field['key'], scoped=False))
14614
+ # Customer resolvers deliberately retain diagnostics even when their low-level pin refuses.
14615
+ for marker in out.refusals:
14616
+ marker['message'] = marker['message'].replace('Values are unscoped.', 'Values are unavailable.').replace(
14617
+ 'Metric values are unscoped.', 'Metric values are unavailable.')
14618
+ for cells in out.values():
14619
+ if marker.get('field') in cells:
14620
+ cells[marker['field']] = None
14621
+ return out
14622
+
14623
+
14624
  def related_product_metric_values(rt, customer_id, products, fields, offer, *,
14625
  team_id=None, agent=None, today=None):
14626
  """Read-only customer × real product metrics; the caller supplies walled server rows.
api/connectors_amazon.py CHANGED
@@ -94,6 +94,137 @@ def assert_safe_status(payload):
94
  return payload
95
 
96
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
97
  #: W55-T19 / R7 / C10 — AMAZON'S OWN ENABLEMENT DOCUMENT. This is deliberately not Odoo's
98
  #: model catalogue or connector config: Amazon publishes report schemas, not ``fields_get``, and
99
  #: its seven databases are enabled at the provider boundary rather than masquerading as Odoo
@@ -340,7 +471,7 @@ def main(argv):
340
  return 1
341
  print("\nreport types this connector will pull (owner ruling R4):")
342
  for table, spec in REPORT_TYPES.items():
343
- rep = spec.get("report") or "NO CONFIRMED REPORT FLAVOR (see O5)"
344
  print(f" {table:<16} {rep}")
345
 
346
  if "--probe-report" in argv:
 
94
  return payload
95
 
96
 
97
+ def first_load_status(tenant_key, rt=None):
98
+ """Writer-side progress helper; request status uses sync_state's read-only cursor."""
99
+ from harness import amazon_store
100
+ return assert_safe_status(amazon_store.first_load_status(tenant_key, rt=rt))
101
+
102
+
103
+ def sync_state(rt, tenant_key, *, cadence_seconds=None, now=None):
104
+ """Each Amazon table's sync state, or None when this mirror cannot be read.
105
+
106
+ Read only: never _connect_tenant_mirror or first_load_status, which issue DDL.
107
+ Missing bookkeeping is unknown, not a measured zero. First-load numbers describe the
108
+ whole connection, because that is the scope of the producer's durable progress marker.
109
+ """
110
+ import json
111
+ from datetime import datetime, timezone
112
+ import duckdb
113
+ from harness import amazon_store as az
114
+ from routes_odoo_tables import _sync_entities, stuck_verdict
115
+
116
+ def stamp(value):
117
+ if not value:
118
+ return None
119
+ try:
120
+ parsed = datetime.fromisoformat(str(value).replace("Z", "+00:00"))
121
+ return (parsed if parsed.tzinfo else parsed.replace(tzinfo=timezone.utc)).astimezone(timezone.utc)
122
+ except (ValueError, TypeError, OverflowError):
123
+ return None
124
+
125
+ def iso(value):
126
+ return value.isoformat().replace("+00:00", "Z") if value else None
127
+
128
+ cadence = cadence_seconds if cadence_seconds is not None else 1800
129
+ try:
130
+ path = az._tenant_mirror_path(tenant_key, rt=rt)
131
+ if not path.is_file():
132
+ return assert_safe_status(None)
133
+ # ⛔⛔ A SECOND `duckdb.connect(read_only=True)` ON A FILE THIS PROCESS ALREADY HOLDS RAISES
134
+ # `ConnectionException: Can't open a connection to same database file with a different
135
+ # configuration`. On the DEFAULT tenant layout the Amazon mirror IS the process-global
136
+ # analytical mirror (`datastore.DB_PATH`), so the swallowed refusal below turned this whole
137
+ # screen into a permanent "sync status is unknown" on exactly the deployment the ticket is
138
+ # for. Measured on staging 2026-09-14, W60-T35. `datastore.ro_con()` is the shipped
139
+ # per-thread READ cursor on that same instance, and it takes no lock against the writer.
140
+ from harness import datastore as _ds
141
+ if path.resolve() == _ds.DB_PATH.resolve():
142
+ # `ro_cursor()` is a cursor on the instance this process ALREADY holds: no second
143
+ # connection, no lock contention with the writer, and no DDL. Deliberately NOT
144
+ # `ro_con()`, which refuses until `ready()` and would report the whole screen as
145
+ # unknown whenever the Odoo backfill is mid-flight, and NOT `datastore.connect()`,
146
+ # which issues `CREATE TABLE IF NOT EXISTS`.
147
+ entities = _sync_entities(_ds.ro_cursor(), cadence)
148
+ else:
149
+ with duckdb.connect(str(path), read_only=True) as cur:
150
+ entities = _sync_entities(cur, cadence)
151
+ except Exception: # noqa: BLE001
152
+ # No exception text: paths, provider payloads and credentials never become status copy.
153
+ return assert_safe_status(None)
154
+ if entities is None:
155
+ return assert_safe_status(None)
156
+
157
+ clock = stamp(now) if now is not None else datetime.now(timezone.utc)
158
+ states, bad_ticks = {}, set()
159
+ for table in az.TABLES:
160
+ tick = entities.get(az.entity_key("sync_tick_" + table))
161
+ if tick:
162
+ updated = stamp(tick.get("updatedAt"))
163
+ if updated is None or tick.get("phase") not in ("attempted", "done", "running"):
164
+ bad_ticks.add(table)
165
+ else:
166
+ states[table] = {"state": tick["phase"], "updated_at": iso(updated)}
167
+ # An unreadable input cannot support a cadence promise for any table in the rotation.
168
+ plan = az.sync_plan(states, now=clock) if not bad_ticks and clock is not None else None
169
+ first_load, load_unknown = None, ""
170
+ load = entities.get(az.entity_key(az.FIRST_LOAD_STATUS_KEY))
171
+ if load:
172
+ try:
173
+ progress = json.loads(load.get("cursor") or "")
174
+ remaining, eta = progress["remaining"], progress["etaSeconds"]
175
+ if (type(remaining) is not int or not 0 <= remaining <= 2**53 - 1
176
+ or type(eta) is not int or not 0 <= eta <= 2**53 - 1
177
+ or (load.get("phase") == "done" and remaining > 0)):
178
+ raise ValueError("Invalid first-load progress")
179
+ if remaining > 0:
180
+ first_load = {"remaining": remaining, "etaSeconds": eta,
181
+ "cause": az.first_load_eta_cause(remaining),
182
+ "needsAttention": load.get("phase") == "failed"}
183
+ except (ValueError, TypeError, KeyError, OverflowError, RecursionError):
184
+ load_unknown = "First-load progress is unknown because its saved marker could not be read. Retry sync to refresh it."
185
+ elif (entities.get(az.entity_key(az.FIRST_LOAD_END_KEY)) or {}).get("phase") not in (None, "done"):
186
+ load_unknown = "First-load progress is not available yet. Keep the connection enabled and check again after this pass."
187
+
188
+ answer = {}
189
+ for table in az.TABLES:
190
+ landed = entities.get(az.entity_key(table)) or {}
191
+ tick = entities.get(az.entity_key("sync_tick_" + table)) or {}
192
+ watermark = entities.get(az.entity_key(az.WATERMARK_PREFIX + table)) or {}
193
+ rows = landed.get("rows")
194
+ if type(rows) is not int or rows < 0:
195
+ rows = None
196
+ successes = [when for marker in (landed, tick, watermark)
197
+ if marker.get("phase") == "done"
198
+ and (when := stamp(marker.get("updatedAt"))) is not None]
199
+ stuck, stuck_cause = tick.get("stuck"), tick.get("stuckCause") or ""
200
+ if tick.get("phase") == "attempted":
201
+ # The shared stall helper calls an active phase 'running'; Amazon calls it 'attempted'.
202
+ updated = stamp(tick.get("updatedAt"))
203
+ stuck, stuck_cause = stuck_verdict(
204
+ "running", iso(updated), cadence,
205
+ now=clock.replace(tzinfo=None) if clock else None)
206
+ reason = plan["deferred"].get(table) if plan else None
207
+ # Only the planner's explicit due stamp is a due date. Minimum stagger spacing is not
208
+ # a promise about when a later resident tick will select this table.
209
+ next_due = stamp(reason.partition("next due ")[2]) if reason else None
210
+ unknown = [load_unknown] if load_unknown else []
211
+ if rows is None:
212
+ unknown.append("This container has no readable row count for this table.")
213
+ if plan is None:
214
+ unknown.append("The sync schedule is unknown because a saved timestamp could not be read. Retry sync to refresh it.")
215
+ answer[table] = {
216
+ "rowsLoaded": rows,
217
+ "lastSuccess": iso(max(successes)) if successes else None,
218
+ "inProgress": tick.get("phase") == "attempted" and stuck is False,
219
+ "deferred": {"reason": reason} if reason else None,
220
+ "nextDue": iso(next_due),
221
+ "firstLoad": first_load,
222
+ "stuck": stuck, "stuckCause": stuck_cause,
223
+ "unknownCause": " ".join(unknown),
224
+ }
225
+ return assert_safe_status(answer)
226
+
227
+
228
  #: W55-T19 / R7 / C10 — AMAZON'S OWN ENABLEMENT DOCUMENT. This is deliberately not Odoo's
229
  #: model catalogue or connector config: Amazon publishes report schemas, not ``fields_get``, and
230
  #: its seven databases are enabled at the provider boundary rather than masquerading as Odoo
 
471
  return 1
472
  print("\nreport types this connector will pull (owner ruling R4):")
473
  for table, spec in REPORT_TYPES.items():
474
+ rep = spec.get("report") or spec.get("report_discover") or "NO CONFIRMED REPORT FLAVOR (see O5)"
475
  print(f" {table:<16} {rep}")
476
 
477
  if "--probe-report" in argv:
api/main.py CHANGED
@@ -20,6 +20,7 @@ session's tenant claim resolves through `harness.runtime.get_runtime`, which is
20
  is the rule the ~30–80 MB/tenant target depends on — EXIT-0 measured the alternative at a ~0.6 GB
21
  commit FLOOR per tenant, duplicated within 2% per tenant, nothing shared.
22
  """
 
23
  import os
24
  import sys
25
  from pathlib import Path
@@ -273,7 +274,7 @@ def _view_revision_unverifiable(request: Request, exc: view_audit.RevisionUnveri
273
 
274
 
275
  @app.get("/api/health")
276
- def health():
277
  """Unauthenticated LIVENESS only — it must answer before anyone can sign in, so it may not
278
  reveal anything about the deployment beyond "the process is up". No version, no tenant list,
279
  no config: a health endpoint is the one URL every scanner finds first.
@@ -282,7 +283,17 @@ def health():
282
  release and "what is LIVE running" needed an answer). A build identifier tells an unauthenticated
283
  caller exactly which commit's known issues apply. It rides `GET /api/v1/settings` instead, behind
284
  a session — and the authoritative copy is the `VERSION` file in the Space repo, which
285
- `deploy_web.space_version()` reads without needing the app to be up at all."""
 
 
 
 
 
 
 
 
 
 
286
  return {"ok": True}
287
 
288
 
@@ -398,6 +409,7 @@ app.include_router(routes_charts.router)
398
  # and that router, so this is the one mount in this block that crosses no fence. It is still
399
  # written HERE rather than at the end, for the reason the block below spells out at length.
400
  app.include_router(routes_sheet.router)
 
401
  # ⭐⭐ WAVE 32 (R1 / C5, cross-fence wiring 6) — THE QUERY MODULE. E's router, A's line, and the
402
  # FIFTH consecutive wave in which this exact line is the artifact the protocol nearly loses.
403
  # ⛔ MOUNTED HERE, IN THIS BLOCK, AND NOT AT THE END OF THE FILE — measured by SESSION E in its own
@@ -766,6 +778,158 @@ def _pull_amazon(why):
766
  return False
767
 
768
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
769
  def _rebuild_meta_relational(why, rt, previous_fingerprint=None, force_relations=False):
770
  """Spawn/refresh the `ut_meta_*` locked databases off the SAME mirror the Odoo half just used.
771
 
@@ -1320,6 +1484,7 @@ def _store_resync_loop():
1320
  Streamlit app piggybacks on page renders. Measure memos key on the pool stamp, so a
1321
  refreshed pool re-reads the freshly synced store."""
1322
  import time as _t
 
1323
  passes = 0
1324
  last_relational_success_at = None
1325
  last_meta_fingerprint = None
@@ -1388,10 +1553,13 @@ def _store_resync_loop():
1388
  # stored fields such as customer_rank can change without partner write_date advancing,
1389
  # so staging served a provably stale customer population for its first hour. Subsequent
1390
  # passes still sleep at the freshly resolved cadence, and manual still skips only Odoo.
 
1391
  if passes:
1392
- _t.sleep(_every)
 
1393
  passes += 1
1394
  RESYNC_STATE["passes"] = passes
 
1395
  sync_report, delete_report, _err = _resync_mirror_pass(_manual, passes)
1396
  RESYNC_STATE["lastPassAt"] = _t.strftime("%Y-%m-%dT%H:%M:%SZ", _t.gmtime())
1397
  RESYNC_STATE["lastPassOk"] = _err is None
@@ -1444,6 +1612,10 @@ def _store_resync_loop():
1444
  _amzn_previous = last_amzn_fingerprint if _meta_reason is None else None
1445
  _reason = _resync_rebuild_reason(sync_report, delete_report,
1446
  last_relational_success_at, _meta_now)
 
 
 
 
1447
  _meta_state = {}
1448
  if _reason is not None:
1449
  print(f"[aios-api] odoo relational rebuild required ({_reason})")
 
20
  is the rule the ~30–80 MB/tenant target depends on — EXIT-0 measured the alternative at a ~0.6 GB
21
  commit FLOOR per tenant, duplicated within 2% per tenant, nothing shared.
22
  """
23
+ import hmac
24
  import os
25
  import sys
26
  from pathlib import Path
 
274
 
275
 
276
  @app.get("/api/health")
277
+ def health(challenge: str = ""):
278
  """Unauthenticated LIVENESS only — it must answer before anyone can sign in, so it may not
279
  reveal anything about the deployment beyond "the process is up". No version, no tenant list,
280
  no config: a health endpoint is the one URL every scanner finds first.
 
283
  release and "what is LIVE running" needed an answer). A build identifier tells an unauthenticated
284
  caller exactly which commit's known issues apply. It rides `GET /api/v1/settings` instead, behind
285
  a session — and the authoritative copy is the `VERSION` file in the Space repo, which
286
+ `deploy_web.space_version()` reads without needing the app to be up at all.
287
+
288
+ W-265 adds a possession challenge for the deployer. It reveals only whether this process was
289
+ born after one high-entropy, single-deploy nonce was stamped; it never returns the nonce,
290
+ version, Space id, or configuration. A normal liveness request keeps the exact
291
+ ``{"ok": true}`` response shape.
292
+ """
293
+ if challenge:
294
+ expected = os.environ.get("AIOS_DEPLOY_NONCE") or ""
295
+ return {"ok": True,
296
+ "deployReady": bool(expected and hmac.compare_digest(challenge, expected))}
297
  return {"ok": True}
298
 
299
 
 
409
  # and that router, so this is the one mount in this block that crosses no fence. It is still
410
  # written HERE rather than at the end, for the reason the block below spells out at length.
411
  app.include_router(routes_sheet.router)
412
+ import routes_module_index; app.include_router(routes_module_index.router) # W61-T20 / C6: App and Agents folder index
413
  # ⭐⭐ WAVE 32 (R1 / C5, cross-fence wiring 6) — THE QUERY MODULE. E's router, A's line, and the
414
  # FIFTH consecutive wave in which this exact line is the artifact the protocol nearly loses.
415
  # ⛔ MOUNTED HERE, IN THIS BLOCK, AND NOT AT THE END OF THE FILE — measured by SESSION E in its own
 
778
  return False
779
 
780
 
781
+ class _SyncNowRefused(Exception):
782
+ """An application-authored, credential-free sentence for a Sync now receipt."""
783
+
784
+
785
+ def _sync_now_failure(cause):
786
+ """Classify actual provider failures without returning vendor text or exception reprs."""
787
+ import re
788
+ text = str(cause).lower()
789
+ causes = (
790
+ (("reduce the amount of data", "response too large"),
791
+ "The provider refused the pull because the response contains too much data."),
792
+ (("no ad accounts",), "The Meta token cannot access any ad accounts."),
793
+ (("max_pages", "page cap"), "The pull stopped at the provider page limit."),
794
+ (("sandbox",), "Amazon sync requires a production credential; this credential is for the sandbox."),
795
+ (("deletion", "retention"), "Amazon sync is blocked by an outstanding data deletion requirement."),
796
+ (("no stored", "no odoo credentials", "access_token is not set"),
797
+ "This tenant has no usable credential for the connector."),
798
+ (("enablement document", "capability unavailable"),
799
+ "The connector database settings could not be read."),
800
+ (("throttl", "rate limit", "too many requests"),
801
+ "The provider rate limit stopped the sync. Retry after the provider delay."),
802
+ (("timed out", "timeout"), "The sync failed because the provider request timed out."),
803
+ (("permission", "access denied", "unauthorized", "invalid oauth", "invalid token",
804
+ "expired", "authentication"), "The provider refused the connector credential or its permissions."),
805
+ (("duplicate key", "primary key", "constraint"),
806
+ "The mirror rejected rows because they violate a database constraint."),
807
+ (("connection", "network", "name resolution"),
808
+ "The sync could not connect to the provider or tenant database."),
809
+ # A-META / D-201: a Meta pull on a container whose mirror never held `meta_*` raises
810
+ # DuckDB's CatalogException. Standing rule 1 wants the CAUSE, not "unexpected error":
811
+ # "the loader is broken" and "this server has never held these tables" are different
812
+ # operator actions, and only the second one is true here.
813
+ (("does not exist", "no such table", "catalog error"),
814
+ "This server's mirror does not hold this connector's tables yet, so there was nothing "
815
+ "to update. Run the connector's first load on the server that owns this tenant."),
816
+ )
817
+ for needles, sentence in causes:
818
+ if any(needle in text for needle in needles):
819
+ return sentence
820
+ status = re.search(r"\bhttp\s+(\d{3})\b", text)
821
+ if status:
822
+ return f"The provider refused the sync request with HTTP status {status.group(1)}."
823
+ if isinstance(cause, TypeError):
824
+ return "The connector failed because its provider call used incompatible arguments."
825
+ if isinstance(cause, (KeyError, ValueError)):
826
+ return "The connector could not read an expected value in the sync response."
827
+ return "The connector encountered an unexpected error while processing the sync response."
828
+
829
+
830
+ def _run_sync_requests():
831
+ """Drain Sync now intents on the resident resync thread, the mirror's single writer.
832
+
833
+ D-29 / intake F6: HTTP handlers only queue intents. Provider calls stay here, before the
834
+ resident projection pass. A failed request cannot discard another tenant's queued work.
835
+ """
836
+ from core import connector_sync
837
+ odoo_requested = False
838
+ for receipt in connector_sync.take_sync_requests():
839
+ provider, tenant = receipt["provider"], receipt["tenant"]
840
+ ok, detail = False, ""
841
+ first_load_progress, continuing = None, False
842
+ try:
843
+ from harness import runtime, datastore
844
+ rt = runtime.get_runtime(tenant)
845
+ if rt.key != tenant:
846
+ raise _SyncNowRefused("The sync runtime does not match the requesting tenant.")
847
+ # These two loaders use a process-global mirror. Never repoint it under readers.
848
+ # Amazon's writer opens an explicit tenant file and does not need this restriction.
849
+ if provider in ("odoo", "meta_ads") and (
850
+ rt.datastore_path.resolve() != datastore.DB_PATH.resolve() or
851
+ datastore.path_for(tenant).resolve() != datastore.DB_PATH.resolve()):
852
+ raise _SyncNowRefused(
853
+ "This tenant's mirror is not resident on this server. Sync requires its tenant worker.")
854
+ if provider == "odoo":
855
+ import core.odoo as odoo
856
+ import odoo_relational as rel
857
+ if rt.odoo_paused() or rel.frozen(rt):
858
+ raise _SyncNowRefused("Odoo is paused or disconnected. Resume the connection before syncing.")
859
+ source = rt.odoo_source()
860
+ if source is None:
861
+ raise _SyncNowRefused("This tenant has no Odoo credential. Connect Odoo before syncing.")
862
+ odoo_requested = True
863
+ previous_client = getattr(odoo._tlocal, "client", None)
864
+ previous_context = getattr(odoo._tlocal, "tenant_ctx", None)
865
+ try:
866
+ # Raw mirror pulls use the resolved tenant credential and a dedicated
867
+ # transport. Do not borrow a request's cached XML-RPC client.
868
+ odoo._tlocal.client = odoo.OdooClient(creds=source._creds)
869
+ odoo._tlocal.tenant_ctx = None
870
+ report = datastore.sync_all(log=lambda *a, **k: None)
871
+ finally:
872
+ odoo._tlocal.client = previous_client
873
+ odoo._tlocal.tenant_ctx = previous_context
874
+ if not report:
875
+ raise _SyncNowRefused("Odoo did not run a sync. Check whether the connection is paused.")
876
+ if any(row.get("phase") != "live" for row in report.values()):
877
+ raise _SyncNowRefused("Odoo backfill is incomplete. Run Sync now again to continue it.")
878
+ problems = []
879
+ elif provider == "amazon_sp":
880
+ from harness import amazon_store as amazon
881
+ if not amazon.credential(tenant, rt=rt):
882
+ raise _SyncNowRefused("This tenant has no stored Amazon credential. Connect Amazon before syncing.")
883
+ first_load = amazon.needs_first_load(tenant, rt=rt)
884
+ options = {"first_load": True, "report_budget": amazon.FIRST_LOAD_REPORTS_PER_PASS} if first_load else {}
885
+ report = amazon.sync(tenant, rt=rt, scheduled=False, **options, log=lambda *a, **k: None)
886
+ from connectors_amazon import assert_safe_status
887
+ first_load_progress = assert_safe_status(report.get("firstLoad"))
888
+ continuing = report.get("state") == "partial"
889
+ problems = list(report.get("problems") or []) + list(report.get("audit_problems") or [])
890
+ tables = report.get("tables") or {}
891
+ if not problems and not tables and not continuing:
892
+ raise _SyncNowRefused("Amazon returned no table results, so the sync could not be confirmed.")
893
+ if not problems and not continuing and all(row.get("state") == "disabled" for row in tables.values()):
894
+ raise _SyncNowRefused("All Amazon databases are disabled. Enable a database before syncing.")
895
+ elif provider == "meta_ads":
896
+ from harness import meta_store as meta
897
+ if not meta.token(rt):
898
+ raise _SyncNowRefused("Meta Ads has no access token. Connect Meta Ads before syncing.")
899
+ report = meta.sync(tenant, log=lambda *a, **k: None, insights_days=7)
900
+ problems = list(report.get("problems") or [])
901
+ if any(row.get("capped") for row in (report.get("tables") or {}).values()):
902
+ problems.append("The pull hit the page cap.")
903
+ if not problems and not report.get("tables"):
904
+ raise _SyncNowRefused("Meta Ads returned no table results, so the sync could not be confirmed.")
905
+ else:
906
+ raise _SyncNowRefused("This connector has no Sync now runner.")
907
+ if continuing:
908
+ ok = True
909
+ detail = (f"The first load is continuing. {int(report['remaining'])} windows remain; "
910
+ f"Amazon report pacing needs at least {int(report['etaSeconds'])} seconds.")
911
+ if problems:
912
+ detail += " " + _sync_now_failure(problems[0])
913
+ elif problems:
914
+ detail = _sync_now_failure(problems[0])
915
+ else:
916
+ ok, detail = True, "Sync finished."
917
+ except _SyncNowRefused as exc:
918
+ detail = str(exc)
919
+ except Exception as exc: # noqa: BLE001
920
+ detail = _sync_now_failure(exc)
921
+ if first_load_progress is not None:
922
+ connector_sync.finish_sync_request(provider, tenant, ok=ok, detail=detail,
923
+ first_load=first_load_progress)
924
+ else:
925
+ connector_sync.finish_sync_request(provider, tenant, ok=ok, detail=detail)
926
+ if continuing and ok:
927
+ # Finish first: the desk refuses a second in-flight intent. Return to the resident
928
+ # loop so its Odoo mirror pass runs before this continuation is claimed.
929
+ connector_sync.request_sync("amazon_sp", tenant, requested_by="first load")
930
+ return odoo_requested
931
+
932
+
933
  def _rebuild_meta_relational(why, rt, previous_fingerprint=None, force_relations=False):
934
  """Spawn/refresh the `ut_meta_*` locked databases off the SAME mirror the Odoo half just used.
935
 
 
1484
  Streamlit app piggybacks on page renders. Measure memos key on the pool stamp, so a
1485
  refreshed pool re-reads the freshly synced store."""
1486
  import time as _t
1487
+ from core import connector_sync
1488
  passes = 0
1489
  last_relational_success_at = None
1490
  last_meta_fingerprint = None
 
1553
  # stored fields such as customer_rank can change without partner write_date advancing,
1554
  # so staging served a provably stale customer population for its first hour. Subsequent
1555
  # passes still sleep at the freshly resolved cadence, and manual still skips only Odoo.
1556
+ # The wait is now interruptible so an administrator's Sync now wakes this writer.
1557
  if passes:
1558
+ connector_sync.sync_wake_event().wait(_every)
1559
+ connector_sync.sync_wake_event().clear()
1560
  passes += 1
1561
  RESYNC_STATE["passes"] = passes
1562
+ _odoo_requested = _run_sync_requests()
1563
  sync_report, delete_report, _err = _resync_mirror_pass(_manual, passes)
1564
  RESYNC_STATE["lastPassAt"] = _t.strftime("%Y-%m-%dT%H:%M:%SZ", _t.gmtime())
1565
  RESYNC_STATE["lastPassOk"] = _err is None
 
1612
  _amzn_previous = last_amzn_fingerprint if _meta_reason is None else None
1613
  _reason = _resync_rebuild_reason(sync_report, delete_report,
1614
  last_relational_success_at, _meta_now)
1615
+ # The scheduled pass may now be quiet because Sync now just advanced this mirror.
1616
+ # Its zero-change report cannot suppress projection of the explicit Odoo pull.
1617
+ if _odoo_requested:
1618
+ _reason = "sync now"
1619
  _meta_state = {}
1620
  if _reason is not None:
1621
  print(f"[aios-api] odoo relational rebuild required ({_reason})")
api/odoo_relational.py CHANGED
@@ -1876,6 +1876,9 @@ def authoritative_schema_snapshot():
1876
  MODEL_CATALOG_KEY = "odoo_readable_model_catalog"
1877
  MODEL_CATALOG_VERSION = 1
1878
  MODEL_READ_PAGE = 2000
 
 
 
1879
  MODEL_CATALOG_TTL = _dt.timedelta(hours=6)
1880
  _MODEL_FIELD_TYPES = {
1881
  "boolean": "checkbox", "integer": "int", "float": "number", "monetary": "currency",
@@ -3776,7 +3779,7 @@ def _model_cell(value):
3776
  return value
3777
 
3778
 
3779
- def _read_model_rows(client, catalogue_row):
3780
  """Read one explicitly enabled model in stable pages; every call is a read method."""
3781
  model = str(catalogue_row.get("model") or "")
3782
  metadata = catalogue_row.get("fields") or {}
@@ -3792,19 +3795,76 @@ def _read_model_rows(client, catalogue_row):
3792
  continue
3793
  rows.append({"_id": str(rid),
3794
  **{name: _model_cell((item or {}).get(name)) for name in fields}})
3795
- if len(page or ()) < MODEL_READ_PAGE:
 
 
 
3796
  break
3797
  offset += len(page)
3798
  return rows
3799
 
3800
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
3801
  def sync_discovered_models(rt, source_connection_id=None, namespace=None,
3802
- username="automation", today=None, table_keys=None):
3803
- """Materialise only explicitly enabled long-tail models, atomically and tenant-bound.
3804
 
3805
  Disabled models are absent from this pass rather than represented by an empty row set. That
3806
  distinction is the preservation guarantee: no updater touches their definition, rows, fields,
3807
  grants, ownership, or tenant rename, and re-enabling resumes against the same stable key.
 
 
3808
  """
3809
  requested = set(table_keys) if table_keys is not None else None
3810
  if requested is not None and not requested:
@@ -3834,6 +3894,10 @@ def sync_discovered_models(rt, source_connection_id=None, namespace=None,
3834
  from core.odoo import get_odoo
3835
  with source._bound(): # tenant credential + scope
3836
  client = get_odoo()
 
 
 
 
3837
  for choice, key in zip(selected, keys):
3838
  model = str(choice["model"])
3839
  row = by_model[model]
@@ -3936,7 +4000,7 @@ def available(rt=None, tenant_key=None, connection_id=None):
3936
 
3937
 
3938
  def refresh(rt, tenant, username="automation", cur=None, today=None, connection_id=None,
3939
- all_connections=False, table_keys=None):
3940
  """THE entry point — the store-resync path and the route both call this.
3941
 
3942
  ⚠ It must be CALLED on resync by something outside this file. If it is not wired, every row
@@ -4046,6 +4110,11 @@ def refresh(rt, tenant, username="automation", cur=None, today=None, connection_
4046
  # Same rule as `skipped` below — R6's second sentence: what was deliberately not built (or no
4047
  # longer built) SAYS SO, with the keys.
4048
  plan_report = {}
 
 
 
 
 
4049
  written = apply_plan(rt, built, username=username, today=today, report=plan_report,
4050
  connection_id=ns, retire=requested is None)
4051
  # The fixed mirror roster above and the discovered long tail deliberately use separate
 
1876
  MODEL_CATALOG_KEY = "odoo_readable_model_catalog"
1877
  MODEL_CATALOG_VERSION = 1
1878
  MODEL_READ_PAGE = 2000
1879
+ # Inline rows rewrite the whole store document, so batch publications after the first page.
1880
+ MODEL_PUBLISH_PAGES = 5
1881
+ MODEL_PUBLISH_SECONDS = 10
1882
  MODEL_CATALOG_TTL = _dt.timedelta(hours=6)
1883
  _MODEL_FIELD_TYPES = {
1884
  "boolean": "checkbox", "integer": "int", "float": "number", "monetary": "currency",
 
3779
  return value
3780
 
3781
 
3782
+ def _read_model_rows(client, catalogue_row, *, on_page=None):
3783
  """Read one explicitly enabled model in stable pages; every call is a read method."""
3784
  model = str(catalogue_row.get("model") or "")
3785
  metadata = catalogue_row.get("fields") or {}
 
3795
  continue
3796
  rows.append({"_id": str(rid),
3797
  **{name: _model_cell((item or {}).get(name)) for name in fields}})
3798
+ final = len(page or ()) < MODEL_READ_PAGE
3799
+ if on_page is not None:
3800
+ on_page(rows, final)
3801
+ if final:
3802
  break
3803
  offset += len(page)
3804
  return rows
3805
 
3806
 
3807
+ def _publish_discovered_model(rt, client, choice, catalogue_row, *, username, today,
3808
+ on_progress):
3809
+ """Publish a single job's growing prefix; source reads and progress have no row ceiling."""
3810
+ ut = _ut()
3811
+ key, model = choice["key"], choice["model"]
3812
+ fields, stamp = _model_field_specs(catalogue_row), today or _iso_today()
3813
+ # A bounded census is never a denominator. Failure to count does not prevent row reads.
3814
+ try:
3815
+ total = client.search_count(model, [])
3816
+ except Exception: # noqa: BLE001
3817
+ total = None
3818
+ if isinstance(total, bool) or not isinstance(total, int) or total < 0:
3819
+ total = None
3820
+
3821
+ stored_n = len(((rt.get(ut.STORE_KEY) or {}).get(key) or {}).get("rows") or {})
3822
+ if total is not None and stored_n and total < stored_n * MAX_SHRINK:
3823
+ raise Refused(f"{model}: source count is {total} rows against {stored_n} stored. "
3824
+ "The prior database was preserved. Check Odoo read access and retry sync.")
3825
+
3826
+ written, published_rows, pages_since_publish = {}, 0, 0
3827
+ last_publish = _time.monotonic()
3828
+
3829
+ def on_page(rows, final):
3830
+ nonlocal published_rows, pages_since_publish, last_publish
3831
+ pages_since_publish += 1
3832
+ if (not written or final or pages_since_publish >= MODEL_PUBLISH_PAGES
3833
+ or _time.monotonic() - last_publish >= MODEL_PUBLISH_SECONDS):
3834
+ def apply(current):
3835
+ current = current if isinstance(current, dict) else {}
3836
+ written[key] = _ensure_table_inplace(
3837
+ current, key, choice["label"], fields, rows, username, stamp)
3838
+ from core import connector_sync
3839
+ current[key]["connector"] = {
3840
+ **connector_sync.database_identity(
3841
+ "odoo", connection_for_table_key(key, rt), key, model),
3842
+ "key": "odoo", "readOnly": True}
3843
+ return current
3844
+
3845
+ # The caller supplies its fenced runtime. A superseded write must propagate.
3846
+ rt.update(ut.STORE_KEY, apply, flush="sync")
3847
+ ut.register_connected(key)
3848
+ published_rows = written[key]["rows"]
3849
+ pages_since_publish, last_publish = 0, _time.monotonic()
3850
+ on_progress(published_rows, total)
3851
+
3852
+ values = _read_model_rows(client, catalogue_row, on_page=on_page)
3853
+ if total is not None and len(values) < total:
3854
+ raise Refused("Odoo returned fewer rows than its count. "
3855
+ "Retry sync to load the remaining rows.")
3856
+ return {"tables": written, "errors": []}
3857
+
3858
+
3859
  def sync_discovered_models(rt, source_connection_id=None, namespace=None,
3860
+ username="automation", today=None, table_keys=None, on_progress=None):
3861
+ """Materialise explicitly enabled long-tail models through the tenant's store.
3862
 
3863
  Disabled models are absent from this pass rather than represented by an empty row set. That
3864
  distinction is the preservation guarantee: no updater touches their definition, rows, fields,
3865
  grants, ownership, or tenant rename, and re-enabling resumes against the same stable key.
3866
+ A single-database job supplies on_progress and publishes growing prefixes. Other callers
3867
+ retain the all-model atomic publication and its shrink guards.
3868
  """
3869
  requested = set(table_keys) if table_keys is not None else None
3870
  if requested is not None and not requested:
 
3894
  from core.odoo import get_odoo
3895
  with source._bound(): # tenant credential + scope
3896
  client = get_odoo()
3897
+ if requested is not None and len(requested) == 1 and on_progress is not None:
3898
+ return _publish_discovered_model(
3899
+ rt, client, selected[0], by_model[str(selected[0]["model"])],
3900
+ username=username, today=today, on_progress=on_progress)
3901
  for choice, key in zip(selected, keys):
3902
  model = str(choice["model"])
3903
  row = by_model[model]
 
4000
 
4001
 
4002
  def refresh(rt, tenant, username="automation", cur=None, today=None, connection_id=None,
4003
+ all_connections=False, table_keys=None, on_apply=None):
4004
  """THE entry point — the store-resync path and the route both call this.
4005
 
4006
  ⚠ It must be CALLED on resync by something outside this file. If it is not wired, every row
 
4110
  # Same rule as `skipped` below — R6's second sentence: what was deliberately not built (or no
4111
  # longer built) SAYS SO, with the keys.
4112
  plan_report = {}
4113
+ if on_apply is not None:
4114
+ try:
4115
+ on_apply()
4116
+ except Exception: # noqa: BLE001
4117
+ pass # Optional status reporting cannot fail a valid plan's publication.
4118
  written = apply_plan(rt, built, username=username, today=today, report=plan_report,
4119
  connection_id=ns, retire=requested is None)
4120
  # The fixed mirror roster above and the discovered long tail deliberately use separate
api/providers.py CHANGED
@@ -641,7 +641,8 @@ def anthropic_send(req, timeout=None, use_sdk=None):
641
  ⭐ `transport` is `'sdk'` or `'http'` and it is RETURNED, not logged and forgotten: a reader
642
  who cannot tell which path ran cannot tell whether the requirements pin reached the container
643
  [[report-the-cause-before-you-fix-it]]. ⛔ A gate must assert on THIS RETURN VALUE, never on
644
- `ai_review.LAST_ANTHROPIC_TRANSPORT` — that global is a convenience for `GET /meta`, and a
 
645
  check keyed to it passes on a stale value written by an earlier check in the same process.
646
 
647
  `status` is an int and `body` is the parsed JSON dict in BOTH paths, including on an error —
 
641
  ⭐ `transport` is `'sdk'` or `'http'` and it is RETURNED, not logged and forgotten: a reader
642
  who cannot tell which path ran cannot tell whether the requirements pin reached the container
643
  [[report-the-cause-before-you-fix-it]]. ⛔ A gate must assert on THIS RETURN VALUE, never on
644
+ `ai_review.LAST_ANTHROPIC_TRANSPORT` — that global is a convenience for the admin view
645
+ `GET /api/v1/automations/ai/transport` (D-384), and a
646
  check keyed to it passes on a stale value written by an earlier check in the same process.
647
 
648
  `status` is an int and `body` is the parsed JSON dict in BOTH paths, including on an error —
api/routes_admin.py CHANGED
@@ -1956,10 +1956,15 @@ def _clean_perms(v, governed_keys=None, session=None, stored=None, stale_out=Non
1956
  # client, or a copy of a record written before this ticket, must not read as a
1957
  # revocation. `perm_scope.may_metrics` applies the identical rule on the read side, so
1958
  # the validator and the wall cannot disagree about what an absent key means.
 
 
 
 
 
1959
  out[key] = {"access": bool(raw.get("access", True)),
1960
  "filter": clean_tree,
1961
  "hiddenFields": sorted(submitted),
1962
- "metrics": bool(raw.get("metrics", True))}
1963
  _refuse_unshapeable_bu(out)
1964
  return out
1965
 
 
1956
  # client, or a copy of a record written before this ticket, must not read as a
1957
  # revocation. `perm_scope.may_metrics` applies the identical rule on the read side, so
1958
  # the validator and the wall cannot disagree about what an absent key means.
1959
+ # D-465: type checked like `hiddenFields`. `bool("false")` is True, so a coerced string
1960
+ # stored a GRANT the administrator meant as a denial. Absence still grants.
1961
+ metrics = raw.get("metrics", True)
1962
+ if not isinstance(metrics, bool):
1963
+ raise err(400, "bad_perms", f"{key}: metrics must be true or false")
1964
  out[key] = {"access": bool(raw.get("access", True)),
1965
  "filter": clean_tree,
1966
  "hiddenFields": sorted(submitted),
1967
+ "metrics": metrics}
1968
  _refuse_unshapeable_bu(out)
1969
  return out
1970
 
api/routes_automation.py CHANGED
@@ -126,10 +126,27 @@ def _voice_options(session):
126
  tables.append(option)
127
  except Exception: # noqa: BLE001 - permission/absence.
128
  pass
129
- # User tables are permissioned by their own object wall inside automation_tables().
130
- for table in automation_tables(session).get("tables") or []:
131
- option = _voice_option(table.get("key"), table.get("label") or table.get("key"),
132
- table.get("fields"), table.get("views"))
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
133
  if option:
134
  tables.append(option)
135
  return {"tables": tables}
@@ -210,8 +227,23 @@ def _voice_view_input(session, audience):
210
 
211
  @router.get("/voice-agents")
212
  def list_voice_agents(session: Session = Depends(_GATE)):
213
- options = _voice_options(session)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
214
  rows = voice.list_browser_dtos(session.runtime)
 
215
  for row in rows:
216
  row["audienceOptions"] = options
217
  return {"voiceAgents": rows}
@@ -1081,6 +1113,49 @@ def _discover_default(session, tables):
1081
  lambda: engine.discover_default_table(session.runtime))
1082
 
1083
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1084
  def warm_default(rt):
1085
  """⭐⭐ WAVE 30 · T12, THE COLD HALF. Elect the discovery picker's default at BOOT.
1086
 
@@ -1396,7 +1471,22 @@ def _table_views(session, key):
1396
 
1397
  @router.get("/automations/tables")
1398
  def automation_tables(session: Session = Depends(_GATE)):
1399
- """The blank databases an automation can target, with their fields.
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1400
 
1401
  ⚠ A READ-ONLY MIRROR of C3-UT's `GET /api/v1/tables`, under this router's own prefix so the
1402
  two can never collide. It exists because the automation editor needs the table+field list to
@@ -1442,11 +1532,12 @@ def automation_tables(session: Session = Depends(_GATE)):
1442
  "type": f.get("type") or "text",
1443
  "automation": f.get("automation") or None}
1444
  for f in (t.get("fields") or [])]}
1445
- views = _table_views(session, key)
1446
- if views is not None:
1447
- row["views"] = views # absent when the bucket did not answer — never []
 
1448
  out.append(row)
1449
- return {"tables": out}
1450
 
1451
 
1452
  @router.get("/automations/presets")
@@ -1953,6 +2044,26 @@ def agent_author_action(auto_id: str, body: dict = Body(default=None),
1953
  "actionId": action_id, "agent": agent_id, "notes": notes}
1954
 
1955
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1956
  @router.get("/automations/{auto_id}")
1957
  def get_automation(auto_id: str, session: Session = Depends(_GATE)):
1958
  """One automation, by id.
 
126
  tables.append(option)
127
  except Exception: # noqa: BLE001 - permission/absence.
128
  pass
129
+ # User tables are permissioned by their own object wall inside _automation_table_rows().
130
+ #
131
+ # ⭐⭐ W62-T17 — THE VIEW LIST IS BOUGHT PER DATABASE AND `_voice_option` THROWS MOST OF IT
132
+ # AWAY. That helper returns None unless a database has BOTH a phone-type field AND a saved
133
+ # View, so a database with no phone field is dropped whatever its Views say — and resolving
134
+ # them cost a `shares.grants` read plus a whole workspace-document read EACH. MEASURED on
135
+ # staging: `_table_views` x24 = 6,810 ms of this pass
136
+ # (`AutoWiki/waves/wave62/proto/T17-profile.md`). The phone test is free (the field list is
137
+ # already in hand), so it is asked FIRST and the view read is spent only where it can matter.
138
+ #
139
+ # ⚠ THE ANSWER IS IDENTICAL, not merely similar: the skipped databases are exactly the ones
140
+ # `_voice_option` returned None for, and `_table_views` is still the single source of the
141
+ # view list for every database that survives — ABSENT (None) still reads as "nobody looked"
142
+ # and still drops the option, exactly as it did when the list rode on the row.
143
+ for table in _automation_table_rows(session, with_views=False):
144
+ if not any(isinstance(field, dict) and field.get("key") and field.get("type") == "phone"
145
+ for field in (table.get("fields") or [])):
146
+ continue
147
+ key = table.get("key")
148
+ option = _voice_option(key, table.get("label") or key,
149
+ table.get("fields"), _table_views(session, key))
150
  if option:
151
  tables.append(option)
152
  return {"tables": tables}
 
227
 
228
  @router.get("/voice-agents")
229
  def list_voice_agents(session: Session = Depends(_GATE)):
230
+ """Every voice agent, each carrying the audience choices its editor offers.
231
+
232
+ ⭐⭐ W62-T17 (R12, D-739) — **THE OPTIONS ARE RESOLVED ONLY IF A ROW WILL CARRY THEM**, and on
233
+ the store that produced this ticket that is the whole fix. `_voice_options` is the most
234
+ expensive read pass in this file (MEASURED at 14,705 ms of this route's 14,748 ms on staging's
235
+ Royal Imports, `AutoWiki/waves/wave62/proto/T17-profile.md`), and Royal Imports has NO voice
236
+ agents, so every one of those seconds was assembled and then thrown away with the empty list.
237
+ The payload is unchanged by construction: with no rows the loop below never ran, so the
238
+ options never reached a caller.
239
+
240
+ ⛔ THE ORDER IS THE FIX, not a tidy-up — read the rows first, then decide. Reinstating the
241
+ eager call "because it is simpler" restores a 14 second answer to an 18 byte payload. This is
242
+ the same shape this file already carries twice (W29-T01 and W30-T12, both on `list_automations`
243
+ below): work whose result is discarded is still paid for.
244
+ """
245
  rows = voice.list_browser_dtos(session.runtime)
246
+ options = _voice_options(session) if rows else None
247
  for row in rows:
248
  row["audienceOptions"] = options
249
  return {"voiceAgents": rows}
 
1113
  lambda: engine.discover_default_table(session.runtime))
1114
 
1115
 
1116
+ def automation_ids(session):
1117
+ """Every automation id `GET /automations` would list, WITHOUT wiring a single DTO.
1118
+
1119
+ ⭐⭐ W62-T17 — THE ID-ONLY ACCESSOR `routes_module_index._readable_ids` ASKED FOR IN WRITING:
1120
+ *"all_definitions omits synthetic field/Odoo agents. Keep the complete automation list until it
1121
+ has an id-only accessor."* That route needs the id SET and nothing else — it uses it to drop a
1122
+ folder placement whose item the caller cannot read — and it was building the whole Agents rail
1123
+ payload to get it. MEASURED on staging: `GET /api/v1/module-index/agents` spent 1,101 ms of its
1124
+ 1,180 ms inside `list_automations`, whose `_wire` reads `agent_contracts` and
1125
+ `automation_agent_actions` once PER AUTOMATION (`AutoWiki/waves/wave62/proto/T17-profile.md`).
1126
+
1127
+ ⛔⛔ THE SET MUST BE `list_automations`' SET EXACTLY, BECAUSE IT IS A WALL. An id this omits is
1128
+ a tile that silently stops rendering — the worst failure shape on a folder home, because the
1129
+ placement is still stored and the page simply looks emptier. So all three strata stay: the
1130
+ stored definitions, the synthetic field agents (C3), and the derived Odoo sync row (C4). Add a
1131
+ stratum to `list_automations` and it belongs here in the same commit.
1132
+
1133
+ ⛔ AND THE STATEMENTS SEED STAYS, for a reason that is not obvious: `seed_statements_agent`
1134
+ MINTS a row that `all_definitions` then lists (W35-T37 / R10), once per tenant per process. If
1135
+ only `list_automations` seeded it, this set would depend on WHICH ROUTE the process happened to
1136
+ serve first — the Agents home would hide the statements agent's placement until somebody
1137
+ opened the rail. Same guard, same idempotency key, same never-take-the-rail-down posture.
1138
+
1139
+ ⚠ WHAT IT DELIBERATELY DOES NOT DO is the board-retirement scan. That is hygiene over stored
1140
+ state (`all_definitions` strips retired board state on every read anyway), it changes no id,
1141
+ and `list_automations` still runs it on its own first call per tenant.
1142
+ """
1143
+ if session.tenant not in _STATEMENTS_SEEDED:
1144
+ try:
1145
+ engine.seed_statements_agent(session.runtime, username=session.uname or "system")
1146
+ except Exception: # noqa: BLE001
1147
+ pass
1148
+ _STATEMENTS_SEEDED.add(session.tenant)
1149
+ ids = {str(defn.get("id")) for defn in engine.all_definitions(session.runtime).values()
1150
+ if isinstance(defn, dict) and defn.get("id")}
1151
+ ids |= {str(row.get("id")) for row in _field_agent_rows(session)
1152
+ if isinstance(row, dict) and row.get("id")}
1153
+ odoo = _odoo_sync_row(session)
1154
+ if isinstance(odoo, dict) and odoo.get("id"):
1155
+ ids.add(str(odoo["id"]))
1156
+ return ids
1157
+
1158
+
1159
  def warm_default(rt):
1160
  """⭐⭐ WAVE 30 · T12, THE COLD HALF. Elect the discovery picker's default at BOOT.
1161
 
 
1471
 
1472
  @router.get("/automations/tables")
1473
  def automation_tables(session: Session = Depends(_GATE)):
1474
+ """The blank databases an automation can target, with their fields. See the helper below."""
1475
+ return {"tables": _automation_table_rows(session)}
1476
+
1477
+
1478
+ def _automation_table_rows(session, with_views=True):
1479
+ """The blank databases an automation can target, with their fields — the rows behind
1480
+ `GET /automations/tables`, as a plain list.
1481
+
1482
+ ⭐⭐ W62-T17 — SPLIT OUT OF THE ROUTE SO ONE IN-PROCESS CALLER CAN DECLINE THE EXPENSIVE HALF.
1483
+ The route's payload is unchanged (`with_views=True` is the default and the route passes
1484
+ nothing else); `_voice_options` above is the caller that asks for `with_views=False`, because
1485
+ it filters on phone fields first and most databases never reach the view list at all. See that
1486
+ call site for the measurement.
1487
+
1488
+ ⛔ A PLAIN FUNCTION, NOT A SECOND ROUTE. The wall, the C8 migration and the lent document all
1489
+ stay exactly where they were; only the `views` read became optional.
1490
 
1491
  ⚠ A READ-ONLY MIRROR of C3-UT's `GET /api/v1/tables`, under this router's own prefix so the
1492
  two can never collide. It exists because the automation editor needs the table+field list to
 
1532
  "type": f.get("type") or "text",
1533
  "automation": f.get("automation") or None}
1534
  for f in (t.get("fields") or [])]}
1535
+ if with_views:
1536
+ views = _table_views(session, key)
1537
+ if views is not None:
1538
+ row["views"] = views # absent when the bucket did not answer — never []
1539
  out.append(row)
1540
+ return out
1541
 
1542
 
1543
  @router.get("/automations/presets")
 
2044
  "actionId": action_id, "agent": agent_id, "notes": notes}
2045
 
2046
 
2047
+ @router.get("/automations/ai/transport")
2048
+ def ai_transport(session: Session = Depends(_GATE)):
2049
+ """D-384: which Anthropic path THIS process ran, read-only, administrators only.
2050
+
2051
+ A verifier could prove the SDK path on its own box and never in the container, because the
2052
+ installed version and `ai_review.LAST_ANTHROPIC_TRANSPORT` had no reader. Two segments under
2053
+ `/automations/`, so the one-segment `/automations/{auto_id}` below cannot shadow it.
2054
+ ⚠ `anthropic` is imported HERE, lazily: `verify_no_streamlit.py` boots the API with optional
2055
+ packages blocked, and it drives this handler with the SDK blocked too."""
2056
+ if not session.admin:
2057
+ raise err(403, "admin_only", "AI transport diagnostics are an admin view")
2058
+ try:
2059
+ import anthropic # noqa: PLC0415
2060
+ version = str(getattr(anthropic, "__version__", "") or "")
2061
+ except ImportError:
2062
+ version = ""
2063
+ return {"anthropicVersion": version or None, "sdkInstalled": bool(version),
2064
+ "lastTransport": ai_review.LAST_ANTHROPIC_TRANSPORT or None}
2065
+
2066
+
2067
  @router.get("/automations/{auto_id}")
2068
  def get_automation(auto_id: str, session: Session = Depends(_GATE)):
2069
  """One automation, by id.
api/routes_connectors.py CHANGED
@@ -23,6 +23,8 @@ from __future__ import annotations
23
  import os
24
  import time
25
  import threading
 
 
26
  from datetime import datetime, timezone
27
 
28
  from fastapi import APIRouter, Body, Depends
@@ -1201,13 +1203,12 @@ def _sync_payload(connector_key, connection_id, row=None):
1201
  def _odoo_sync_runner(session, connection_id):
1202
  """Run Odoo through C6 without moving provider imports into the data layer."""
1203
  connector_sync.record(session.runtime, "odoo", connection_id, "syncing")
1204
- # The current Odoo entry point plans and applies in one call, so the closest truthful seam is
1205
- # immediately before it starts applying the tenant mirror into governed database definitions.
1206
- connector_sync.record(session.runtime, "odoo", connection_id, "materializing")
1207
  import odoo_relational
1208
  result = odoo_relational.refresh(session.runtime, session.tenant,
1209
  username=session.uname or "automation",
1210
- connection_id=connection_id)
 
 
1211
  tables = result.get("tables") if isinstance(result, dict) else {}
1212
  databases = sorted(str(key) for key in (tables or {}).keys())
1213
  sync_errors = []
@@ -1235,6 +1236,11 @@ def _odoo_sync_runner(session, connection_id):
1235
  _SYNC_RUNNERS = {"odoo": _odoo_sync_runner}
1236
 
1237
 
 
 
 
 
 
1238
  class _SupersededMaterialization(Exception):
1239
  pass
1240
 
@@ -1296,15 +1302,29 @@ class _MaterializationRuntime:
1296
  marker = [self.job["id"], self.job["leaseToken"]]
1297
  def apply(current):
1298
  self.valid()
1299
- if name == ut.STORE_KEY and (current.get(key) or {}).get("_connectorJob") != marker:
 
 
1300
  raise _SupersededMaterialization()
1301
  answer = fn(current)
 
 
 
 
1302
  # Lease expiry during a long merge cannot acknowledge a stale write either.
1303
  self.valid()
1304
  return answer
1305
  return self.rt.update(name, apply, flush=flush, **kwargs)
1306
 
1307
 
 
 
 
 
 
 
 
 
1308
  def _run_database_job(rt, candidate):
1309
  """Run one persisted receipt. Repeating a completed publication reconciles the same rows."""
1310
  import core.user_tables as ut
@@ -1312,13 +1332,14 @@ def _run_database_job(rt, candidate):
1312
  # A disconnected source is paused work, not a failed attempt. Preserve the receipt so
1313
  # reconnect can resume it; the publication fence still rechecks a mid-run disconnect.
1314
  if (rt.get(rel.CONFIG_KEY, fresh=True) or {}).get("frozen"):
 
1315
  return
1316
  job = connector_sync.claim(rt, candidate)
1317
  if job is None:
1318
  return
1319
  fenced = _MaterializationRuntime(rt, job)
1320
  key = job["databaseKey"]
1321
- def finish(success=False):
1322
  # The pre-claim check cannot cover a disconnect during source I/O or publication.
1323
  # Only the current lease may requeue; disable and a newer Save still win.
1324
  interrupted = bool((rt.get(rel.CONFIG_KEY, fresh=True) or {}).get("frozen"))
@@ -1326,7 +1347,13 @@ def _run_database_job(rt, candidate):
1326
  # A newer field Save can persist after rows commit but before this acknowledgement.
1327
  # The old rows remain valid history; they cannot establish the new settings ready.
1328
  fenced.valid()
1329
- return connector_sync.finish(rt, job, success=success, interrupted=interrupted)
 
 
 
 
 
 
1330
 
1331
  stopped = threading.Event()
1332
  def heartbeat():
@@ -1339,9 +1366,8 @@ def _run_database_job(rt, candidate):
1339
  try:
1340
  def install(current):
1341
  fenced.valid()
1342
- if key not in (current or {}):
1343
- raise _SupersededMaterialization()
1344
- current[key]["_connectorJob"] = [job["id"], job["leaseToken"]]
1345
  return current
1346
  rt.update(ut.STORE_KEY, install, flush="sync")
1347
  heartbeat_thread = threading.Thread(target=heartbeat, daemon=True,
@@ -1349,6 +1375,15 @@ def _run_database_job(rt, candidate):
1349
  heartbeat_thread.start()
1350
  cid = job["connectionId"]
1351
  source_id = None if cid == "default" else cid
 
 
 
 
 
 
 
 
 
1352
  if rel.base_table_key(key) in rel.ODOO_TABLE_MODELS:
1353
  result = rel.refresh(
1354
  fenced, getattr(rt, "tenant", None), username="automation",
@@ -1358,26 +1393,58 @@ def _run_database_job(rt, candidate):
1358
  _cid, namespace = rel.connection_namespace(rt, source_id)
1359
  result = {"discovered": rel.sync_discovered_models(
1360
  fenced, source_connection_id=source_id, namespace=namespace,
1361
- username="automation", table_keys={key})}
 
 
1362
  discovered = result.get("discovered") or {}
1363
  published = {**(result.get("tables") or {}), **(discovered.get("tables") or {})}
1364
  success = (key in published and not result.get("failed") and not result.get("frozen")
1365
  and not result.get("limits") and not discovered.get("errors"))
1366
- finish(success=success)
1367
  except _SupersededMaterialization:
1368
  # Disable/new Save already invalidated the table marker; never restore it here.
1369
  finish()
1370
- except Exception:
 
 
1371
  # Raw provider exceptions may carry credentials or source values. Store only safe copy.
1372
- finish()
 
 
 
 
 
1373
  finally:
1374
  stopped.set()
1375
  if heartbeat_thread:
1376
  heartbeat_thread.join(timeout=1)
1377
 
1378
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1379
  def _resume_database_jobs(rt):
1380
  """One serial pass, also usable by isolated synthetic fixtures."""
 
 
 
 
1381
  for candidate in connector_sync.jobs(rt):
1382
  if candidate.get("provider") == "odoo" and (
1383
  candidate.get("state") == "queued" or
@@ -1394,6 +1461,52 @@ _materialization_wake = threading.Event()
1394
  _materialization_stop = threading.Event()
1395
  _materialization_thread = None
1396
  _materialization_lifecycle = threading.Lock()
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1397
 
1398
 
1399
  @router.on_event("startup")
@@ -1417,7 +1530,7 @@ def _start_database_jobs():
1417
  if _materialization_stop.is_set():
1418
  break
1419
  try:
1420
- _resume_database_jobs(runtime.get_runtime(tenant))
1421
  except Exception:
1422
  continue # Durable queued/leased work is retried by the next scan.
1423
  except Exception:
@@ -1435,9 +1548,14 @@ def _start_database_jobs():
1435
 
1436
  @router.on_event("shutdown")
1437
  def _stop_database_jobs():
 
1438
  with _materialization_lifecycle:
1439
  _materialization_stop.set()
1440
  _materialization_wake.set()
 
 
 
 
1441
 
1442
 
1443
  def _sync_refusals():
@@ -1575,6 +1693,30 @@ def connector_sync_status(connector_key: str, connection_id: str = None,
1575
  return _sync_payload(connector_key, connection_id, row)
1576
 
1577
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1578
  @router.post("/connectors/{connector_key}/sync/retry")
1579
  def connector_sync_retry(connector_key: str, connection_id: str = None,
1580
  session: Session = Depends(require_session), database_key: str = None):
@@ -1809,6 +1951,7 @@ def _enable_odoo_database(table_key, session, connection_id=None):
1809
  # materialisation remains separate work.
1810
  if failure is not None:
1811
  raise err(*failure)
 
1812
  if not exists:
1813
  reason = _materialisation_failure_reason(result)
1814
  message = "The setting was enabled, but the database was not created."
@@ -1816,11 +1959,14 @@ def _enable_odoo_database(table_key, session, connection_id=None):
1816
  message = f"{message} {reason}"
1817
  else:
1818
  message = f"{message} The field catalogue could not be published."
1819
- raise err(409, "database_not_created", message)
 
 
 
 
1820
 
1821
  # Every permission-visible field is available from schema alone. Existing per-database
1822
  # choices survive a retry; population counts are optional background observations.
1823
- model = str(choice.get("model") or "").strip()
1824
  try:
1825
  schema_keys = {_odoo_field_key(session.runtime, key, model, name): True
1826
  for name in result["sourceFields"]}
@@ -1832,9 +1978,12 @@ def _enable_odoo_database(table_key, session, connection_id=None):
1832
  },
1833
  flush="sync")
1834
  except Exception: # noqa: BLE001
1835
- raise err(503, "field_auto_enable_failed",
1836
- "The database was created, but its field choices could not be saved. "
1837
- "Try again.")
 
 
 
1838
  _evict_odoo_product_pools(session.runtime)
1839
 
1840
  # W56-T36: there is deliberately no navigation projection eviction here. Data's database
@@ -2578,11 +2727,25 @@ def _detail_databases(rt, is_admin, connector_key, uname="", connection_id=None)
2578
  return _named_databases(rt, is_admin, uname, out)
2579
 
2580
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2581
  def _amazon_databases(rt, is_admin, spec=None):
2582
  """⭐⭐ W52 · R6 — THE SEVEN AMAZON DATABASES, DISCLOSED WITH WHERE EACH COLUMN CAME FROM.
2583
 
2584
- Fills the SAME `ConnectorDatabase` shape Odoo does, so the Databases tab needs no client
2585
- change: `key`, `label`, `enabled`, `fields`, `excluded`, `sharesFieldsWith`.
2586
 
2587
  ⛔⛔ **`excluded` CARRIES THE PROVENANCE, AND THAT IS THE WHOLE POINT OF THE SCREEN HERE.** Five
2588
  of the seven tables have NO published schema, because Amazon publishes artifacts only for its
@@ -2611,12 +2774,13 @@ def _amazon_databases(rt, is_admin, spec=None):
2611
  enabled = enablement.get("databases") or {}
2612
  cat = amazon_schema.catalogue()
2613
  tables = cat.get("tables") or {}
 
2614
 
2615
  # Which OTHER declared tables are filled from the SAME report document. Derived from the
2616
  # report map so a table added to `REPORT_TYPES` gets this disclosure for free.
2617
  by_report = {}
2618
  for name in amazon_schema.PROVENANCE:
2619
- rep = str((cx.REPORT_TYPES.get(name) or {}).get("report") or "")
2620
  if rep:
2621
  by_report.setdefault(rep, []).append("ut_amzn_" + name)
2622
 
@@ -2625,7 +2789,7 @@ def _amazon_databases(rt, is_admin, spec=None):
2625
  key = "ut_amzn_" + name
2626
  spec = tables.get(name) or {}
2627
  cols = spec.get("columns") or []
2628
- rep = str((cx.REPORT_TYPES.get(name) or {}).get("report") or "")
2629
  fields = [{"key": str(c.get("name") or ""),
2630
  "label": str(c.get("name") or "").replace("_", " ").capitalize(),
2631
  "type": "text" if str(c.get("type")) == "VARCHAR" else "number",
@@ -2656,6 +2820,7 @@ def _amazon_databases(rt, is_admin, spec=None):
2656
  # read from Odoo's grid config, and disabling it leaves every stored row and
2657
  # definition untouched: only this provider config document changes.
2658
  "enabled": enabled.get(key) is True,
 
2659
  "fields": fields,
2660
  "excluded": excluded,
2661
  "sharesFieldsWith": [k for k in by_report.get(rep, ()) if k != key]})
@@ -2876,9 +3041,11 @@ def connector_detail(rt, user, uname, is_admin, connector_key):
2876
  return _safe_connector_status(key, {"overview": overview,
2877
  "keychain": keychain,
2878
  "databases": databases,
 
 
2879
  "materialization": ({"provider": key, "statusPath": f"/connectors/{key}/jobs",
2880
  "retryPath": f"/connectors/{key}/sync/retry"}
2881
- if is_admin and key in connector_sync.providers() else None),
2882
  # W55-T19 / R7 / C10 — a provider-specific capability. The client neither infers it
2883
  # from ``introspectable`` nor sends Amazon through Odoo's schema/config endpoints.
2884
  "enablement": enablement,
 
23
  import os
24
  import time
25
  import threading
26
+ import xmlrpc.client
27
+ from concurrent.futures import ThreadPoolExecutor
28
  from datetime import datetime, timezone
29
 
30
  from fastapi import APIRouter, Body, Depends
 
1203
  def _odoo_sync_runner(session, connection_id):
1204
  """Run Odoo through C6 without moving provider imports into the data layer."""
1205
  connector_sync.record(session.runtime, "odoo", connection_id, "syncing")
 
 
 
1206
  import odoo_relational
1207
  result = odoo_relational.refresh(session.runtime, session.tenant,
1208
  username=session.uname or "automation",
1209
+ connection_id=connection_id,
1210
+ on_apply=lambda: connector_sync.record(
1211
+ session.runtime, "odoo", connection_id, "materializing"))
1212
  tables = result.get("tables") if isinstance(result, dict) else {}
1213
  databases = sorted(str(key) for key in (tables or {}).keys())
1214
  sync_errors = []
 
1236
  _SYNC_RUNNERS = {"odoo": _odoo_sync_runner}
1237
 
1238
 
1239
+ ODOO_DISCONNECTED_CAUSE = (
1240
+ "Odoo is disconnected, so rows cannot load. "
1241
+ "Reconnect Odoo in the Keychain and this resumes on its own.")
1242
+
1243
+
1244
  class _SupersededMaterialization(Exception):
1245
  pass
1246
 
 
1302
  marker = [self.job["id"], self.job["leaseToken"]]
1303
  def apply(current):
1304
  self.valid()
1305
+ missing = name == ut.STORE_KEY and key not in current
1306
+ if (name == ut.STORE_KEY and not missing
1307
+ and (current.get(key) or {}).get("_connectorJob") != marker):
1308
  raise _SupersededMaterialization()
1309
  answer = fn(current)
1310
+ if missing and key in answer:
1311
+ # A reconciled receipt may precede its definition. Its first schema write
1312
+ # installs the marker under the same lease and configuration fence.
1313
+ answer[key]["_connectorJob"] = marker
1314
  # Lease expiry during a long merge cannot acknowledge a stale write either.
1315
  self.valid()
1316
  return answer
1317
  return self.rt.update(name, apply, flush=flush, **kwargs)
1318
 
1319
 
1320
+ def _odoo_read_access_refused(exc):
1321
+ """Match only XML-RPC faults with explicit access/read refusal markers."""
1322
+ if not isinstance(exc, xmlrpc.client.Fault):
1323
+ return False
1324
+ message = str(exc).casefold()
1325
+ return "security restrictions" in message or "operation: read" in message
1326
+
1327
+
1328
  def _run_database_job(rt, candidate):
1329
  """Run one persisted receipt. Repeating a completed publication reconciles the same rows."""
1330
  import core.user_tables as ut
 
1332
  # A disconnected source is paused work, not a failed attempt. Preserve the receipt so
1333
  # reconnect can resume it; the publication fence still rechecks a mid-run disconnect.
1334
  if (rt.get(rel.CONFIG_KEY, fresh=True) or {}).get("frozen"):
1335
+ connector_sync.stall(rt, candidate, ODOO_DISCONNECTED_CAUSE)
1336
  return
1337
  job = connector_sync.claim(rt, candidate)
1338
  if job is None:
1339
  return
1340
  fenced = _MaterializationRuntime(rt, job)
1341
  key = job["databaseKey"]
1342
+ def finish(success=False, cause=""):
1343
  # The pre-claim check cannot cover a disconnect during source I/O or publication.
1344
  # Only the current lease may requeue; disable and a newer Save still win.
1345
  interrupted = bool((rt.get(rel.CONFIG_KEY, fresh=True) or {}).get("frozen"))
 
1347
  # A newer field Save can persist after rows commit but before this acknowledgement.
1348
  # The old rows remain valid history; they cannot establish the new settings ready.
1349
  fenced.valid()
1350
+ # Only provider-authored sentences reach this seam, never raw exception messages.
1351
+ cause = str(cause or "").replace("\u2014", ",").replace("\u2013", "-")
1352
+ result = connector_sync.finish(
1353
+ rt, job, success=success, interrupted=interrupted, cause=cause)
1354
+ if interrupted:
1355
+ connector_sync.stall(rt, job, ODOO_DISCONNECTED_CAUSE)
1356
+ return result
1357
 
1358
  stopped = threading.Event()
1359
  def heartbeat():
 
1366
  try:
1367
  def install(current):
1368
  fenced.valid()
1369
+ if key in (current or {}):
1370
+ current[key]["_connectorJob"] = [job["id"], job["leaseToken"]]
 
1371
  return current
1372
  rt.update(ut.STORE_KEY, install, flush="sync")
1373
  heartbeat_thread = threading.Thread(target=heartbeat, daemon=True,
 
1375
  heartbeat_thread.start()
1376
  cid = job["connectionId"]
1377
  source_id = None if cid == "default" else cid
1378
+ if key not in (rt.get(ut.STORE_KEY, fresh=True) or {}):
1379
+ schema = rel.publish_database_schema(fenced, key, connection_id=source_id,
1380
+ username="automation")
1381
+ if not job["model"]:
1382
+ # Reconciliation cannot infer a missing definition's model. Now the provider
1383
+ # has resolved it, queue its real field fingerprint before reading any rows.
1384
+ fenced.valid()
1385
+ _queue_odoo_database(rt, key, model=schema["identity"]["model"], changed=True)
1386
+ return
1387
  if rel.base_table_key(key) in rel.ODOO_TABLE_MODELS:
1388
  result = rel.refresh(
1389
  fenced, getattr(rt, "tenant", None), username="automation",
 
1393
  _cid, namespace = rel.connection_namespace(rt, source_id)
1394
  result = {"discovered": rel.sync_discovered_models(
1395
  fenced, source_connection_id=source_id, namespace=namespace,
1396
+ username="automation", table_keys={key},
1397
+ on_progress=lambda rows, total: connector_sync.progress(
1398
+ fenced, job, rows=rows, total=total))}
1399
  discovered = result.get("discovered") or {}
1400
  published = {**(result.get("tables") or {}), **(discovered.get("tables") or {})}
1401
  success = (key in published and not result.get("failed") and not result.get("frozen")
1402
  and not result.get("limits") and not discovered.get("errors"))
1403
+ finish(success=success, cause=result.get("note") or discovered.get("note") or "")
1404
  except _SupersededMaterialization:
1405
  # Disable/new Save already invalidated the table marker; never restore it here.
1406
  finish()
1407
+ except rel.Refused as exc:
1408
+ finish(cause=str(exc))
1409
+ except Exception as exc:
1410
  # Raw provider exceptions may carry credentials or source values. Store only safe copy.
1411
+ cause = ""
1412
+ if _odoo_read_access_refused(exc):
1413
+ cause = (f"Odoo refused to read {job['model']} because the connected Odoo user has no "
1414
+ "read access, so retrying cannot load rows. "
1415
+ "Give that user read access in Odoo, or switch this database off.")
1416
+ finish(cause=cause)
1417
  finally:
1418
  stopped.set()
1419
  if heartbeat_thread:
1420
  heartbeat_thread.join(timeout=1)
1421
 
1422
 
1423
+ def _reconcile_database_jobs(rt):
1424
+ """Repair enabled Odoo grids with no receipt using only tenant-local definitions."""
1425
+ import core.user_tables as ut
1426
+ import odoo_relational as rel
1427
+ grids = (rt.get(rel.CONFIG_KEY, fresh=True) or {}).get("grids") or {}
1428
+ receipts = {job["databaseKey"] for job in connector_sync.jobs(rt, "odoo")}
1429
+ missing = [key for key, enabled in grids.items() if enabled is True and key not in receipts]
1430
+ if not missing:
1431
+ return
1432
+ definitions = rt.get_projection(ut.STORE_KEY, drop=("rows",)) or {}
1433
+ for key in missing:
1434
+ try:
1435
+ model = str(((definitions.get(key) or {}).get("connector") or {}).get("model") or "")
1436
+ # The queue's identity validator also decides whether an unknown model is legal.
1437
+ _queue_odoo_database(rt, key, model=model)
1438
+ except Exception:
1439
+ continue # One unresolvable definition cannot starve other queued work.
1440
+
1441
+
1442
  def _resume_database_jobs(rt):
1443
  """One serial pass, also usable by isolated synthetic fixtures."""
1444
+ try:
1445
+ _reconcile_database_jobs(rt)
1446
+ except Exception:
1447
+ pass # A reconciliation failure must still fall through to the receipt scan.
1448
  for candidate in connector_sync.jobs(rt):
1449
  if candidate.get("provider") == "odoo" and (
1450
  candidate.get("state") == "queued" or
 
1461
  _materialization_stop = threading.Event()
1462
  _materialization_thread = None
1463
  _materialization_lifecycle = threading.Lock()
1464
+ # 3 is a starting point, not tuned: each worker holds an Odoo read and a growing store document in a small Space container.
1465
+ MATERIALIZATION_WORKERS = 3
1466
+ _materialization_pool = None
1467
+ _materialization_inflight = set()
1468
+ _materialization_inflight_lock = threading.Lock()
1469
+
1470
+
1471
+ def _release_database_job(key):
1472
+ # Completion and cancellation both release the reservation, including failed claims.
1473
+ with _materialization_inflight_lock:
1474
+ _materialization_inflight.discard(key)
1475
+
1476
+
1477
+ def _dispatch_database_jobs(rt):
1478
+ """Submit a resident scan without waiting for source reads or row publication."""
1479
+ global _materialization_pool
1480
+ try:
1481
+ _reconcile_database_jobs(rt)
1482
+ except Exception:
1483
+ pass # Match the serial fixture pass even when reconciliation is unavailable.
1484
+ for candidate in connector_sync.jobs(rt):
1485
+ if candidate.get("provider") == "odoo" and (
1486
+ candidate.get("state") == "queued" or
1487
+ candidate.get("state") == "syncing" and candidate.get("leaseUntil", 0) <= time.time()):
1488
+ key = (rt.key, candidate["provider"], candidate["connectionId"], candidate["databaseKey"])
1489
+ with _materialization_inflight_lock:
1490
+ if key in _materialization_inflight:
1491
+ continue
1492
+ _materialization_inflight.add(key)
1493
+ try:
1494
+ with _materialization_lifecycle:
1495
+ if _materialization_stop.is_set():
1496
+ _release_database_job(key)
1497
+ return
1498
+ if _materialization_pool is None:
1499
+ _materialization_pool = ThreadPoolExecutor(
1500
+ max_workers=MATERIALIZATION_WORKERS,
1501
+ thread_name_prefix="connector-materialization-job")
1502
+ pool = _materialization_pool
1503
+ future = pool.submit(_run_database_job, rt, candidate)
1504
+ # Register outside both locks: an already finished future calls back inline.
1505
+ future.add_done_callback(lambda _future, key=key: _release_database_job(key))
1506
+ except Exception:
1507
+ _release_database_job(key)
1508
+ # A concurrent shutdown or failed submit leaves the receipt for the next pass.
1509
+ continue
1510
 
1511
 
1512
  @router.on_event("startup")
 
1530
  if _materialization_stop.is_set():
1531
  break
1532
  try:
1533
+ _dispatch_database_jobs(runtime.get_runtime(tenant))
1534
  except Exception:
1535
  continue # Durable queued/leased work is retried by the next scan.
1536
  except Exception:
 
1548
 
1549
  @router.on_event("shutdown")
1550
  def _stop_database_jobs():
1551
+ global _materialization_pool
1552
  with _materialization_lifecycle:
1553
  _materialization_stop.set()
1554
  _materialization_wake.set()
1555
+ pool, _materialization_pool = _materialization_pool, None
1556
+ if pool is not None:
1557
+ # Running reads retain their reservation until done; lease expiry permits recovery.
1558
+ pool.shutdown(wait=False, cancel_futures=True)
1559
 
1560
 
1561
  def _sync_refusals():
 
1693
  return _sync_payload(connector_key, connection_id, row)
1694
 
1695
 
1696
+ @router.post("/connectors/{connector_key}/sync/now")
1697
+ def connector_sync_now(connector_key: str, session: Session = Depends(require_session)):
1698
+ _sync_admin(session)
1699
+ if connector_key not in connector_sync.providers():
1700
+ raise err(404, "sync_provider_unknown", "This connector is not registered for sync.")
1701
+ try:
1702
+ return connector_sync.request_sync(connector_key, session.runtime.key,
1703
+ requested_by=session.uname)
1704
+ except ValueError:
1705
+ raise err(404, "sync_provider_unknown", "This connector is not registered for sync.") from None
1706
+
1707
+
1708
+ @router.get("/connectors/{connector_key}/sync/now")
1709
+ def connector_sync_now_status(connector_key: str, session: Session = Depends(require_session)):
1710
+ _sync_admin(session)
1711
+ if connector_key not in connector_sync.providers():
1712
+ raise err(404, "sync_provider_unknown", "This connector is not registered for sync.")
1713
+ return connector_sync.sync_request(connector_key, session.runtime.key) or {
1714
+ "provider": connector_key, "tenant": session.runtime.key, "state": "idle",
1715
+ "requestedAt": None, "startedAt": None, "finishedAt": None,
1716
+ "requestedBy": "", "detail": "", "accepted": False,
1717
+ }
1718
+
1719
+
1720
  @router.post("/connectors/{connector_key}/sync/retry")
1721
  def connector_sync_retry(connector_key: str, connection_id: str = None,
1722
  session: Session = Depends(require_session), database_key: str = None):
 
1951
  # materialisation remains separate work.
1952
  if failure is not None:
1953
  raise err(*failure)
1954
+ model = str(choice.get("model") or "").strip()
1955
  if not exists:
1956
  reason = _materialisation_failure_reason(result)
1957
  message = "The setting was enabled, but the database was not created."
 
1959
  message = f"{message} {reason}"
1960
  else:
1961
  message = f"{message} The field catalogue could not be published."
1962
+ try:
1963
+ _queue_odoo_database(session.runtime, key, model=model)
1964
+ finally:
1965
+ # Preserve the original refusal envelope even if the queue store is unavailable.
1966
+ raise err(409, "database_not_created", message)
1967
 
1968
  # Every permission-visible field is available from schema alone. Existing per-database
1969
  # choices survive a retry; population counts are optional background observations.
 
1970
  try:
1971
  schema_keys = {_odoo_field_key(session.runtime, key, model, name): True
1972
  for name in result["sourceFields"]}
 
1978
  },
1979
  flush="sync")
1980
  except Exception: # noqa: BLE001
1981
+ try:
1982
+ _queue_odoo_database(session.runtime, key, model=model)
1983
+ finally:
1984
+ raise err(503, "field_auto_enable_failed",
1985
+ "The database was created, but its field choices could not be saved. "
1986
+ "Try again.")
1987
  _evict_odoo_product_pools(session.runtime)
1988
 
1989
  # W56-T36: there is deliberately no navigation projection eviction here. Data's database
 
2727
  return _named_databases(rt, is_admin, uname, out)
2728
 
2729
 
2730
+ def _amazon_report_name(table):
2731
+ """The report type this Amazon table reads, whichever key declares it.
2732
+
2733
+ ⛔ SETTLEMENTS DECLARE `report_discover`, NOT `report` (W60-T36), because Amazon issues them on
2734
+ its own schedule and refuses a createReport for that type: it is a getReports DISCOVERY filter
2735
+ and is deliberately absent from `_APPROVED_REPORT_TYPES`. Reading only `report` here printed an
2736
+ empty Source name for settlements on the connector screen, which reads as "this table has no
2737
+ report" when the truth is "this table is not requested, it is listed".
2738
+ """
2739
+ import connectors_amazon as cx # noqa: PLC0415 - provider owns state
2740
+ spec = cx.REPORT_TYPES.get(table) or {}
2741
+ return str(spec.get("report") or spec.get("report_discover") or "")
2742
+
2743
+
2744
  def _amazon_databases(rt, is_admin, spec=None):
2745
  """⭐⭐ W52 · R6 — THE SEVEN AMAZON DATABASES, DISCLOSED WITH WHERE EACH COLUMN CAME FROM.
2746
 
2747
+ Fills the shared `ConnectorDatabase` shape and adds an administrator-only Amazon `sync`
2748
+ block. Members retain the shared shape without the field catalogue or sync bookkeeping.
2749
 
2750
  ⛔⛔ **`excluded` CARRIES THE PROVENANCE, AND THAT IS THE WHOLE POINT OF THE SCREEN HERE.** Five
2751
  of the seven tables have NO published schema, because Amazon publishes artifacts only for its
 
2774
  enabled = enablement.get("databases") or {}
2775
  cat = amazon_schema.catalogue()
2776
  tables = cat.get("tables") or {}
2777
+ states = cx.sync_state(rt, getattr(rt, "key", "")) if is_admin else None
2778
 
2779
  # Which OTHER declared tables are filled from the SAME report document. Derived from the
2780
  # report map so a table added to `REPORT_TYPES` gets this disclosure for free.
2781
  by_report = {}
2782
  for name in amazon_schema.PROVENANCE:
2783
+ rep = _amazon_report_name(name)
2784
  if rep:
2785
  by_report.setdefault(rep, []).append("ut_amzn_" + name)
2786
 
 
2789
  key = "ut_amzn_" + name
2790
  spec = tables.get(name) or {}
2791
  cols = spec.get("columns") or []
2792
+ rep = _amazon_report_name(name)
2793
  fields = [{"key": str(c.get("name") or ""),
2794
  "label": str(c.get("name") or "").replace("_", " ").capitalize(),
2795
  "type": "text" if str(c.get("type")) == "VARCHAR" else "number",
 
2820
  # read from Odoo's grid config, and disabling it leaves every stored row and
2821
  # definition untouched: only this provider config document changes.
2822
  "enabled": enabled.get(key) is True,
2823
+ **({"sync": (states or {}).get(name)} if is_admin else {}),
2824
  "fields": fields,
2825
  "excluded": excluded,
2826
  "sharesFieldsWith": [k for k in by_report.get(rep, ()) if k != key]})
 
3041
  return _safe_connector_status(key, {"overview": overview,
3042
  "keychain": keychain,
3043
  "databases": databases,
3044
+ "syncNow": ({"provider": key, "path": f"/connectors/{key}/sync/now"}
3045
+ if is_admin and key in connector_sync.providers() else None),
3046
  "materialization": ({"provider": key, "statusPath": f"/connectors/{key}/jobs",
3047
  "retryPath": f"/connectors/{key}/sync/retry"}
3048
+ if is_admin and key == "odoo" else None),
3049
  # W55-T19 / R7 / C10 — a provider-specific capability. The client neither infers it
3050
  # from ``introspectable`` nor sends Amazon through Odoo's schema/config endpoints.
3051
  "enablement": enablement,
api/routes_customers.py CHANGED
@@ -513,6 +513,8 @@ def grid_assembly(session: Session, scope: str = "customer", storage_key: str =
513
  if _ov:
514
  ws["overlays"] = {pid: {k: v for k, v in (cells or {}).items() if k not in hidden}
515
  for pid, cells in _ov.items()}
 
 
516
 
517
  today = time.strftime("%Y-%m-%d")
518
  fields = aios_grid.project_date_recurrences(fields, today=today)
@@ -603,7 +605,7 @@ def grid_assembly(session: Session, scope: str = "customer", storage_key: str =
603
  "team_id": team_id, "routes": normalized_routes, "zones": normalized_zones}
604
 
605
 
606
- def _payload(session: Session):
607
  """`{fields, rows, today, docs, pulled_at}` — X2's shape, which `verify_fields_contract.py`
608
  referees. Rows are now built by `aios_grid.rows_from_pool` (embed == standalone by
609
  construction); see `grid_assembly` for what that fixed.
@@ -617,11 +619,18 @@ def _payload(session: Session):
617
  """
618
  import aios_grid
619
 
620
- g = grid_assembly(session)
621
  rows = aios_grid.rows_from_pool(
622
  g["rows_src"], g["fields"], g["ws"].get("overlays"), derived=g["derived"])
623
  from core import route_system
624
  rows = route_system.apply_customer_projections(rows, g["routes"], g["zones"])
 
 
 
 
 
 
 
625
  # ⭐ C4 / D-138 (W30-T37) — THE DOCUMENTS PRODUCER FOR THE CUSTOMER SCOPE. The write door
626
  # (`doc_add`/`doc_fetch`/`doc_delete`) never stopped working and every client half is
627
  # complete; what vanished with `app.py` at EXIT-6 was the only thing that ever set this key.
@@ -636,6 +645,8 @@ def _payload(session: Session):
636
  # "every document in the tenant" mode to reach for, deliberately.
637
  from core import grid_events as _ge
638
  return {"fields": g["fields"], "rows": rows,
 
 
639
  # `today` rides the payload because every relative date condition must resolve against
640
  # the TENANT's day, never the browser's — a client that falls back to its own clock
641
  # disagrees with the server for everyone west of it.
@@ -718,8 +729,15 @@ def allowed_pids(session: Session):
718
 
719
 
720
  @router.get("/customers")
721
- def customers(session: Session = Depends(module_gate(MODULE))):
722
- return _payload(session)
 
 
 
 
 
 
 
723
 
724
 
725
  @router.get("/customers/assigned/{view_id}/rows")
 
513
  if _ov:
514
  ws["overlays"] = {pid: {k: v for k, v in (cells or {}).items() if k not in hidden}
515
  for pid, cells in _ov.items()}
516
+ # D-464: and the saved views are a FOURTH wire, served by `/workspace` from `workspace`.
517
+ workspace["views"] = aios_grid.narrow_view_keys(workspace.get("views"), hidden)
518
 
519
  today = time.strftime("%Y-%m-%d")
520
  fields = aios_grid.project_date_recurrences(fields, today=today)
 
605
  "team_id": team_id, "routes": normalized_routes, "zones": normalized_zones}
606
 
607
 
608
+ def _payload(session: Session, related=None):
609
  """`{fields, rows, today, docs, pulled_at}` — X2's shape, which `verify_fields_contract.py`
610
  referees. Rows are now built by `aios_grid.rows_from_pool` (embed == standalone by
611
  construction); see `grid_assembly` for what that fixed.
 
619
  """
620
  import aios_grid
621
 
622
+ g = grid_assembly(session, consume_corrections=not bool(related))
623
  rows = aios_grid.rows_from_pool(
624
  g["rows_src"], g["fields"], g["ws"].get("overlays"), derived=g["derived"])
625
  from core import route_system
626
  rows = route_system.apply_customer_projections(rows, g["routes"], g["zones"])
627
+ from routes_odoo_tables import narrow_related_rows
628
+ rows = narrow_related_rows(session, MODULE, rows, related)
629
+ from routes_tables import scope_linked_metrics, linked_metric_limits
630
+ rows, g['fields'], metric_refusals = scope_linked_metrics(
631
+ session, MODULE, rows, g['fields'], g['measures'], related)
632
+ if related:
633
+ g["pids"] = frozenset(row["pid"] for row in rows)
634
  # ⭐ C4 / D-138 (W30-T37) — THE DOCUMENTS PRODUCER FOR THE CUSTOMER SCOPE. The write door
635
  # (`doc_add`/`doc_fetch`/`doc_delete`) never stopped working and every client half is
636
  # complete; what vanished with `app.py` at EXIT-6 was the only thing that ever set this key.
 
645
  # "every document in the tenant" mode to reach for, deliberately.
646
  from core import grid_events as _ge
647
  return {"fields": g["fields"], "rows": rows,
648
+ "refusals": metric_refusals,
649
+ "limits": linked_metric_limits(metric_refusals),
650
  # `today` rides the payload because every relative date condition must resolve against
651
  # the TENANT's day, never the browser's — a client that falls back to its own clock
652
  # disagrees with the server for everyone west of it.
 
729
 
730
 
731
  @router.get("/customers")
732
+ def customers(session: Session = Depends(module_gate(MODULE)), related: str = None):
733
+ return _payload(session, related=related)
734
+
735
+
736
+ @router.post("/customers/aggregates")
737
+ def customer_aggregates(body: dict = Body(default=None),
738
+ session: Session = Depends(module_gate(MODULE))):
739
+ from routes_odoo_tables import odoo_table_aggregates
740
+ return odoo_table_aggregates(MODULE, body=body, session=session)
741
 
742
 
743
  @router.get("/customers/assigned/{view_id}/rows")
api/routes_dependents.py CHANGED
@@ -42,6 +42,8 @@ docstring names the cost of publishing it. This door returns names and a count,
42
  "Shared with Anna and 2 others" consumes.
43
  """
44
 
 
 
45
  from fastapi import APIRouter, Depends, Query
46
 
47
  import core.dependents as dependents_registry
@@ -148,6 +150,23 @@ def read_dependents(kind: str = Query(...), key: str = Query(...),
148
 
149
  target = {"kind": target_kind, "key": target_key, "database": resolved}
150
  report = dependents_registry.dependents_report(session.runtime, target)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
151
  label = _label_for(session, resolved, target_kind, target_key)
152
  body = {
153
  "target": target,
 
42
  "Shared with Anna and 2 others" consumes.
43
  """
44
 
45
+ from urllib.parse import parse_qs
46
+
47
  from fastapi import APIRouter, Depends, Query
48
 
49
  import core.dependents as dependents_registry
 
150
 
151
  target = {"kind": target_kind, "key": target_key, "database": resolved}
152
  report = dependents_registry.dependents_report(session.runtime, target)
153
+ seen = {resolved: True}
154
+ for dependent in report['dependents']:
155
+ # An address is not an authorisation. Inbound fields can live on a different
156
+ # database, so reuse the same wall before offering either C3 address shape.
157
+ for item in [dependent, *dependent.get('items', [])]:
158
+ href = item.get('href')
159
+ if not isinstance(href, str):
160
+ continue
161
+ if not href.startswith(('#/data?', '#/sheet?')):
162
+ item['href'] = None
163
+ item['reason'] = 'This dependent has no supported address available.'
164
+ continue
165
+ destinations = parse_qs(href.partition('?')[2]).get('database', [])
166
+ if (href.startswith('#/data?') and not destinations) or any(
167
+ not _may_open(session, destination, seen) for destination in destinations):
168
+ item['href'] = None
169
+ item['reason'] = 'You do not have access to open this dependent\'s database.'
170
  label = _label_for(session, resolved, target_kind, target_key)
171
  body = {
172
  "target": target,
api/routes_external.py CHANGED
@@ -1,13 +1,16 @@
1
- """Per-user credential management and the bearer-authenticated external read door."""
2
 
 
3
  import json
 
4
  from urllib.parse import quote
5
 
6
  from fastapi import APIRouter, Body, Depends, Header, HTTPException, Request, Response
 
7
 
8
  import deps
9
  from deps import Session, err, require_session
10
- from core import external_access
11
 
12
 
13
  router = APIRouter(prefix="/api/v1/external", tags=["external"])
@@ -15,6 +18,53 @@ router = APIRouter(prefix="/api/v1/external", tags=["external"])
15
  _CREDENTIAL_PREFIX = "lpb1."
16
 
17
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
18
  def _auth_refusal():
19
  raise err(401, external_access.AUTH_ERROR, "The credential is invalid or revoked.")
20
 
@@ -40,6 +90,10 @@ def require_external_session(
40
  "u": verified.user["username"],
41
  "external": True,
42
  "external_key_id": verified.key_id,
 
 
 
 
43
  },
44
  runtime=verified.runtime,
45
  )
@@ -64,13 +118,17 @@ def _can_use_analyst(session: Session) -> bool:
64
 
65
 
66
  def _management_refusal(exc: external_access.ExternalAccessError):
 
 
 
 
67
  if exc.code == "external_label_invalid":
68
  raise err(400, exc.code, "Enter a label between 1 and 80 characters.")
69
  if exc.code == "external_key_not_found":
70
  raise err(404, exc.code, "That credential was not found.")
71
  if exc.code in {"external_identity_invalid", "external_account_unavailable"}:
72
  raise err(409, exc.code, "This account cannot create an external credential.")
73
- raise err(503, "external_access_unavailable", "External access is unavailable right now.")
74
 
75
 
76
  def _resolve_external_scope(
@@ -128,6 +186,8 @@ def _resolve_external_scope(
128
  else:
129
  row["label"] = fallback_label(database)
130
  row["fields"] = []
 
 
131
  resolved.append(row)
132
  return resolved
133
 
@@ -150,13 +210,14 @@ def _portable_skill_document(session: Session, base_url: str) -> str:
150
  lines = [
151
  "---",
152
  "name: loopable-external-data",
153
- "description: Read this user's governed Loopable data through the external API.",
154
  "---",
155
  "",
156
  "# Loopable external data",
157
  "",
158
- ("This surface is a read-only view of the Loopable data this user can already see. "
159
- "It applies the same database, field, and row permissions as the Loopable app."),
 
160
  "",
161
  "## Authentication",
162
  "",
@@ -268,12 +329,67 @@ def _portable_skill_document(session: Session, base_url: str) -> str:
268
  ])
269
  lines.append("")
270
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
271
  lines.extend([
272
  "## Limits",
273
  "",
274
- ("This surface performs no writes. It returns no data the user cannot already see in "
275
- "Loopable, and it provides no export beyond what the user could already export from "
276
- "the app."),
 
 
 
 
277
  "",
278
  ])
279
  document = "\n".join(lines)
@@ -287,8 +403,11 @@ def create_key(body=Body(default=None), session: Session = Depends(require_sessi
287
  """Mint a credential for the signed-in human. Bearer auth is not accepted here."""
288
  session.require("analyst")
289
  label = body.get("label") if isinstance(body, dict) else None
 
 
290
  try:
291
- key, token = external_access.mint(session.runtime, session.uname, session.tenant, label)
 
292
  except external_access.ExternalAccessError as exc:
293
  _management_refusal(exc)
294
  return {"key": key, "secret": token}
@@ -316,7 +435,7 @@ def revoke_key(key_id: str, session: Session = Depends(require_session)):
316
 
317
  @router.get("/skill.md")
318
  def portable_skill(request: Request, session: Session = Depends(require_session)):
319
- """Download the signed-in human's current governed external read instructions."""
320
  session.require("analyst")
321
  document = _portable_skill_document(session, str(request.base_url))
322
  return Response(
@@ -328,21 +447,39 @@ def portable_skill(request: Request, session: Session = Depends(require_session)
328
 
329
  @router.get("/whoami")
330
  def whoami(session: Session = Depends(require_external_analyst)):
331
- """The one observable bearer route in this ticket; later tickets add governed data reads."""
 
332
  return {
333
  "username": session.uname,
334
  "name": session.user.get("name") or session.uname,
335
  "tenant": session.tenant,
336
  "key_id": session.claims["external_key_id"],
337
- "read_only": True,
 
 
 
338
  }
339
 
340
 
 
 
 
 
 
 
 
 
 
 
 
341
  @router.get("/databases")
342
  def list_databases(session: Session = Depends(require_external_analyst)):
343
  """List only sources this user may read or already sees elsewhere in the product."""
344
  databases = []
 
345
  for row in _resolve_external_scope(session):
 
 
346
  if not (row.get("permitted") or row.get("visible")):
347
  continue
348
  public = {
@@ -353,12 +490,14 @@ def list_databases(session: Session = Depends(require_external_analyst)):
353
  if not public["answerable"]:
354
  public["reason"] = row.get("reason") or ""
355
  databases.append(public)
356
- return {"databases": databases, "read_only": True}
 
357
 
358
 
359
  @router.get("/databases/{database}/schema")
360
  def database_schema(database: str, session: Session = Depends(require_external_analyst)):
361
  """Return the field contract after resolving the ordinary strict Assistant read wall."""
 
362
  resolved = _resolve_external_scope(
363
  session, databases=[database], include_schema=True, strict_schema=True,
364
  )
@@ -372,6 +511,7 @@ def query_database(
372
  session: Session = Depends(require_external_analyst),
373
  ):
374
  """Read one detached Assistant snapshot; FastAPI rejects non-object request bodies."""
 
375
  payload = body or {}
376
  return deps.assistant_read_scope(
377
  session,
@@ -381,19 +521,342 @@ def query_database(
381
  )
382
 
383
 
384
- # C7 / R3: one import-time chokepoint makes the external bearer surface structurally read only.
385
- # POST remains available for bounded query bodies and for the cookie-only credential lifecycle
386
- # above. No bearer dependency appears on those lifecycle routes.
387
- _READ_ONLY_METHODS = frozenset({"GET", "POST", "HEAD", "OPTIONS"})
388
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
389
 
390
- def _assert_read_only_router() -> None:
391
- for route in router.routes:
392
- methods = set(getattr(route, "methods", set()) or set())
393
- refused = methods - _READ_ONLY_METHODS
394
- if refused:
395
- joined = ", ".join(sorted(refused))
396
- raise RuntimeError(f"external route {route.path} advertises refused method {joined}")
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
397
 
398
 
399
- _assert_read_only_router()
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Per-user credentials, governed reads and explicitly allowed bearer build tools."""
2
 
3
+ import copy
4
  import json
5
+ import uuid
6
  from urllib.parse import quote
7
 
8
  from fastapi import APIRouter, Body, Depends, Header, HTTPException, Request, Response
9
+ from starlette.concurrency import run_in_threadpool
10
 
11
  import deps
12
  from deps import Session, err, require_session
13
+ from core import external_access, external_tools, sheets
14
 
15
 
16
  router = APIRouter(prefix="/api/v1/external", tags=["external"])
 
18
  _CREDENTIAL_PREFIX = "lpb1."
19
 
20
 
21
+ def _key_scope(session):
22
+ """(access, databases) for THIS request. A session without the claims is an in-app human."""
23
+ claims = session.claims or {}
24
+ return external_access.scope_of({
25
+ "access": claims.get("external_access"),
26
+ "databases": claims.get("external_databases"),
27
+ })
28
+
29
+
30
+ def _assert_database_scope(session, database):
31
+ if external_access.database_allowed(_key_scope(session)[1], database):
32
+ return
33
+ # A binding can carry no database at all. Name what was asked for when there is a name, and
34
+ # never interpolate a bare None into a sentence a caller reads.
35
+ named = str(database or "").strip()
36
+ raise err(403, "key_database_out_of_scope",
37
+ f"Database {named} is outside this credential's scope." if named
38
+ else "This credential is limited to named databases, and none was named.")
39
+
40
+
41
+ def _assert_write_scope(session, tool_name):
42
+ if not external_access.may_write(_key_scope(session)[0]):
43
+ raise err(403, "key_read_only", f"{tool_name} writes, and this credential is read only.")
44
+
45
+
46
+ def _databases_in_arguments(value):
47
+ """Find database references, including bound cell documents, without unbounded recursion."""
48
+ found = []
49
+
50
+ def scan(node, depth):
51
+ if depth > 12:
52
+ return
53
+ if isinstance(node, dict):
54
+ for key, item in node.items():
55
+ if key in ("database", "sourceDatabase") and isinstance(item, str):
56
+ found.append(item)
57
+ elif key == "compatibleDatabases" and isinstance(item, list):
58
+ found.extend(database for database in item if isinstance(database, str))
59
+ scan(item, depth + 1)
60
+ elif isinstance(node, (list, tuple)):
61
+ for item in node:
62
+ scan(item, depth + 1)
63
+
64
+ scan(value, 0)
65
+ return found
66
+
67
+
68
  def _auth_refusal():
69
  raise err(401, external_access.AUTH_ERROR, "The credential is invalid or revoked.")
70
 
 
90
  "u": verified.user["username"],
91
  "external": True,
92
  "external_key_id": verified.key_id,
93
+ "external_access": verified.access,
94
+ "external_databases": (external_access.ALL_DATABASES
95
+ if verified.databases == external_access.ALL_DATABASES
96
+ else list(verified.databases)),
97
  },
98
  runtime=verified.runtime,
99
  )
 
118
 
119
 
120
  def _management_refusal(exc: external_access.ExternalAccessError):
121
+ if exc.code == "external_access_scope_invalid":
122
+ raise err(400, exc.code, "Choose Read only or Read and write.")
123
+ if exc.code == "external_databases_invalid":
124
+ raise err(400, exc.code, "Choose all databases, or name at least one database.")
125
  if exc.code == "external_label_invalid":
126
  raise err(400, exc.code, "Enter a label between 1 and 80 characters.")
127
  if exc.code == "external_key_not_found":
128
  raise err(404, exc.code, "That credential was not found.")
129
  if exc.code in {"external_identity_invalid", "external_account_unavailable"}:
130
  raise err(409, exc.code, "This account cannot create an external credential.")
131
+ raise err(503, "external_access_unavailable", "The AI API is unavailable right now.")
132
 
133
 
134
  def _resolve_external_scope(
 
186
  else:
187
  row["label"] = fallback_label(database)
188
  row["fields"] = []
189
+ else:
190
+ row["label"] = fallback_label(database)
191
  resolved.append(row)
192
  return resolved
193
 
 
210
  lines = [
211
  "---",
212
  "name: loopable-external-data",
213
+ "description: Read governed data and build Sheet workbooks, saved Views and Apps as the key's maker.",
214
  "---",
215
  "",
216
  "# Loopable external data",
217
  "",
218
+ ("This surface reads governed data and writes only through the named build tools, "
219
+ "as the key's maker with the maker's own grants. It applies the same database, "
220
+ "field, row and workbook permissions as the Loopable app."),
221
  "",
222
  "## Authentication",
223
  "",
 
329
  ])
330
  lines.append("")
331
 
332
+ example_call = {"jsonrpc": "2.0", "id": 1, "method": "tools/call", "params": {
333
+ "name": "sheet_create_workbook", "arguments": {"name": "Gateway workbook"}}}
334
+ lines.extend([
335
+ "## Remote MCP tools",
336
+ "",
337
+ "Endpoint: `POST /api/v1/external/mcp`",
338
+ "",
339
+ ("Send one JSON-RPC 2.0 object per request. Methods: initialize, tools/list, tools/call "
340
+ "and ping. Initialize echoes your protocolVersion and advertises tools. Requests "
341
+ "without an id, or with a null id, receive HTTP 204."),
342
+ "",
343
+ "```json",
344
+ json.dumps(example_call, indent=2),
345
+ "```",
346
+ "",
347
+ ("A tools/call response has jsonrpc, id and result. The result contains "
348
+ "content: [{type: 'text', text: '<JSON {ok, data or error}>'}] and isError. "
349
+ "A tool refusal uses HTTP 200 with ok: false and isError: true; its error names "
350
+ "the tool, code, status and reason. Invalid or revoked credentials receive HTTP 401."),
351
+ "",
352
+ "```sh",
353
+ "curl --request POST \\",
354
+ " --header 'Authorization: Bearer <your Loopable credential>' \\",
355
+ " --header 'Content-Type: application/json' \\",
356
+ f" --data '{json.dumps(example_call, separators=(',', ':'))}' \\",
357
+ f" '{surface_url}/mcp'",
358
+ "```",
359
+ "",
360
+ ("Sheet tools require analyst and sheets grants. Geometry patches carry the revision "
361
+ "read by the server; pass rev to also protect the revision you read. Cell writes "
362
+ "return accepted and refused addresses, retaining the app's partial-write behavior."),
363
+ "",
364
+ ("View tools use the grid's native config: filters, sorts, groupBy, visible and display. "
365
+ "Fields omitted from visible are hidden. The grid validates the config. App tools use "
366
+ "the Application registry's typed config. app_delete requires the exact current "
367
+ "App name in confirmName, including after a rename."),
368
+ "",
369
+ ])
370
+ for name, tool in external_tools.TOOLS.items():
371
+ lines.extend([
372
+ f"### `{name}`",
373
+ "",
374
+ _markdown_inline(tool["description"]),
375
+ "",
376
+ "Writes: " + ("yes" if tool["write"] else "no") + ".",
377
+ "",
378
+ "```json",
379
+ json.dumps(tool["inputSchema"], indent=2, ensure_ascii=True),
380
+ "```",
381
+ "",
382
+ ])
383
  lines.extend([
384
  "## Limits",
385
  "",
386
+ ("This surface writes only through the named tools, as the key's maker with the maker's "
387
+ "own grants. Workbook roles and database restrictions still apply. Each credential also "
388
+ "carries its own access level and database scope, set when it is created and shown in "
389
+ "Loopable Settings. It returns no data "
390
+ "the maker cannot already see in Loopable. Sheet cell history records the maker and "
391
+ "key label. Saved Views and Apps keep the UI's username attribution; their writers "
392
+ "have no author display-name field. No tool writes to Odoo."),
393
  "",
394
  ])
395
  document = "\n".join(lines)
 
403
  """Mint a credential for the signed-in human. Bearer auth is not accepted here."""
404
  session.require("analyst")
405
  label = body.get("label") if isinstance(body, dict) else None
406
+ access = body.get("access") if isinstance(body, dict) else None
407
+ databases = body.get("databases") if isinstance(body, dict) else None
408
  try:
409
+ key, token = external_access.mint(session.runtime, session.uname, session.tenant, label,
410
+ access=access, databases=databases)
411
  except external_access.ExternalAccessError as exc:
412
  _management_refusal(exc)
413
  return {"key": key, "secret": token}
 
435
 
436
  @router.get("/skill.md")
437
  def portable_skill(request: Request, session: Session = Depends(require_session)):
438
+ """Download current governed external read and build instructions."""
439
  session.require("analyst")
440
  document = _portable_skill_document(session, str(request.base_url))
441
  return Response(
 
447
 
448
  @router.get("/whoami")
449
  def whoami(session: Session = Depends(require_external_analyst)):
450
+ """Name the authenticated maker and the explicit limits on bearer writes."""
451
+ access, databases = _key_scope(session)
452
  return {
453
  "username": session.uname,
454
  "name": session.user.get("name") or session.uname,
455
  "tenant": session.tenant,
456
  "key_id": session.claims["external_key_id"],
457
+ "access": access,
458
+ "databases": (external_access.ALL_DATABASES
459
+ if databases == external_access.ALL_DATABASES else list(databases)),
460
+ "write_scope": "Named tools only, as the key's maker with the maker's own grants.",
461
  }
462
 
463
 
464
+ @router.get("/scope/databases")
465
+ def scope_databases(session: Session = Depends(require_session)):
466
+ """List the databases this signed-in maker could put inside a new credential's scope."""
467
+ session.require("analyst")
468
+ return {"databases": [
469
+ {key: row[key] for key in ("database", "label", "permitted", "answerable")}
470
+ for row in _resolve_external_scope(session)
471
+ if row.get("permitted") or row.get("visible")
472
+ ]}
473
+
474
+
475
  @router.get("/databases")
476
  def list_databases(session: Session = Depends(require_external_analyst)):
477
  """List only sources this user may read or already sees elsewhere in the product."""
478
  databases = []
479
+ key_databases = _key_scope(session)[1]
480
  for row in _resolve_external_scope(session):
481
+ if not external_access.database_allowed(key_databases, row["database"]):
482
+ continue
483
  if not (row.get("permitted") or row.get("visible")):
484
  continue
485
  public = {
 
490
  if not public["answerable"]:
491
  public["reason"] = row.get("reason") or ""
492
  databases.append(public)
493
+ return {"databases": databases,
494
+ "write_scope": "Named tools only, as the key's maker with the maker's own grants."}
495
 
496
 
497
  @router.get("/databases/{database}/schema")
498
  def database_schema(database: str, session: Session = Depends(require_external_analyst)):
499
  """Return the field contract after resolving the ordinary strict Assistant read wall."""
500
+ _assert_database_scope(session, database)
501
  resolved = _resolve_external_scope(
502
  session, databases=[database], include_schema=True, strict_schema=True,
503
  )
 
511
  session: Session = Depends(require_external_analyst),
512
  ):
513
  """Read one detached Assistant snapshot; FastAPI rejects non-object request bodies."""
514
+ _assert_database_scope(session, database)
515
  payload = body or {}
516
  return deps.assistant_read_scope(
517
  session,
 
521
  )
522
 
523
 
524
+ def _view_context(session, database, event):
525
+ import routes_grid
 
 
526
 
527
+ scope = routes_grid._scope_or_400(
528
+ {"customer_data": "customer", "product_data": "product"}.get(database, database))
529
+ try:
530
+ if scope.startswith("ut_"):
531
+ from routes_tables import ut_write_ctx
532
+
533
+ g = ut_write_ctx(session, scope)
534
+ if g.get("limits") and (event.get("pid") is not None or event.get("pids")):
535
+ limit = g["limits"][0]
536
+ raise err(409, "pid_scope_unresolved",
537
+ "This connector mirror's record scope is unresolved, so a change "
538
+ "addressed to particular records cannot be admitted. "
539
+ f"{limit.get('cause') or 'The row set is unresolved'}. "
540
+ f"{limit.get('recommendation') or ''}".strip())
541
+ elif scope == "product":
542
+ from routes_products import MODULE, product_assembly
543
+
544
+ session.require(MODULE)
545
+ g = product_assembly(session, consume_corrections=False)
546
+ else:
547
+ from routes_customers import grid_assembly
548
 
549
+ session.require(routes_grid.MODULE)
550
+ g = grid_assembly(session, scope=scope, consume_corrections=False)
551
+ except HTTPException as exc:
552
+ info = exc.detail.get("error", {}) if isinstance(exc.detail, dict) else {}
553
+ raise err(exc.status_code, info.get("code", "view_refused"),
554
+ f"Database {database}: {info.get('message') or 'The app refused access.'}") from exc
555
+ ctx = routes_grid._ctx(
556
+ session, g["fields"], g["pids"], scope_key=scope,
557
+ measure_keys=frozenset(m["key"] for m in g["measures"]),
558
+ resolved_ids=frozenset(g["measure_sets"]),
559
+ cohort_ids=frozenset(c["id"] for c in g["lists"]),
560
+ measure_offer=tuple(g["measures"]), visible_views=tuple(g["views"]),
561
+ recurrence_validator=lambda raw: routes_grid.govern_date_recurrence(
562
+ session, scope, g["fields"], raw))
563
+ if scope.startswith("ut_"):
564
+ ctx.hidden_keys = frozenset(g["hidden"])
565
+ return ctx
566
+
567
+
568
+ def _application_access(session, databases, *, strict=False):
569
+ import routes_assets
570
+
571
+ if not strict:
572
+ return routes_assets.application_access_context(session, databases)
573
+ allowed, records = set(), {}
574
+ for database in databases or ():
575
+ try:
576
+ permitted, semantic = routes_assets.application_access_context(
577
+ session, [database], strict=True)
578
+ except PermissionError as exc:
579
+ # Name only the submitted database, never a hidden registry object's source.
580
+ raise err(403, "forbidden", f"Database {database}: {exc}") from exc
581
+ allowed |= permitted
582
+ records.update(semantic)
583
+ return allowed, records
584
+
585
+
586
+ def _view_event(session, kind, args):
587
+ from core import grid_events
588
+
589
+ database = args["database"]
590
+ _assert_database_scope(session, database)
591
+ event = {"id": uuid.uuid4().hex, "type": kind}
592
+ if kind == "view_upsert":
593
+ raw = args["view"]
594
+ if (not isinstance(raw.get("config"), dict)
595
+ or not isinstance(raw.get("id"), str) or not raw["id"].strip()
596
+ or not isinstance(raw.get("name"), str) or not raw["name"].strip()):
597
+ raise err(400, "tool_arguments_invalid", "A View needs an id, a name and an object config.")
598
+ view_id = raw["id"]
599
+ event["view"] = raw
600
+ # A membership config also addresses records, even though the payload nests its ids.
601
+ if raw["config"].get("memberPids"):
602
+ event["pids"] = raw["config"]["memberPids"]
603
+ else:
604
+ view_id = args["view_id"]
605
+ event["viewId"] = view_id
606
+ if not view_id.strip() or len(view_id) > 120 or view_id != view_id.strip():
607
+ raise err(400, "tool_arguments_invalid", "A View id must have 1 to 120 characters without surrounding spaces.")
608
+ ctx = _view_context(session, database, event)
609
+ try:
610
+ before = ctx.table.find_view(view_id, prefer_owner=session.uname)
611
+ rerender = grid_events.handle_one(event, ctx)
612
+ except grid_events.StoreUnavailable as exc:
613
+ raise err(503, "store_unavailable", f"View {view_id}: the tenant store is unavailable.") from exc
614
+ if ctx.out.refusals:
615
+ refusal = ctx.out.refusals[0]
616
+ raise err(400, refusal.get("code", "view_refused"),
617
+ f"View {view_id}: {refusal.get('message') or 'The app refused this change.'}")
618
+ if ctx.out.toast:
619
+ raise err(403, "view_refused", f"View {view_id}: {ctx.out.toast}")
620
+ after = ctx.table.find_view(view_id, prefer_owner=session.uname)
621
+ changed = before != after
622
+ if not changed:
623
+ raise err(409, "view_not_changed", f"View {view_id}: nothing was changed.")
624
+ # ⛔⛔ `handle_one`'s BOOL IS A RERENDER FLAG, NOT A SUCCESS FLAG, and reading it as success is
625
+ # the mistake this comment exists to stop the next reader making. Its own contract block
626
+ # (`grid_events.py:1182`) says so in as many words: "view_delete - False (the deleted view
627
+ # simply stops being served). view_upsert - unchanged: False, except an unresolved measure
628
+ # condition." So an ordinary successful view write returns False, and refusing on it would
629
+ # refuse every save that worked.
630
+ # ⭐ THE SILENT NO-OP IS CAUGHT BY `changed` ABOVE INSTEAD, which is the scar this tool was
631
+ # written against: `view_upsert` answers 200 and writes nothing when the view id belongs to
632
+ # another user ([[a-view-id-another-user-holds-refuses-silently]]), and a store read before and
633
+ # after is the only thing that can tell that apart from a save that landed
634
+ # ([[lost-write-looks-like-failed-read]]). `rerender` rides out as information, never as the
635
+ # verdict.
636
+ return {"database": database, "view_id": view_id, "rerender": bool(rerender)}
637
+
638
+
639
+ def _application_upsert(session, args):
640
+ import application_registry as applications
641
+ import routes_assets
642
+
643
+ for database in args.get("compatibleDatabases") or []:
644
+ _assert_database_scope(session, database)
645
+ if args.get("sourceDatabase") is not None:
646
+ _assert_database_scope(session, args["sourceDatabase"])
647
+ payload = {key: value for key, value in args.items() if key != "app_id"}
648
+ app_id = args.get("app_id")
649
+ if "app_id" in args and not str(app_id or "").strip():
650
+ raise err(400, "bad_application", "app_id must name an App to update.")
651
+ if not session.runtime.available():
652
+ raise err(503, "store_unavailable", "The tenant store is unavailable. Nothing was saved.")
653
+ try:
654
+ if not app_id:
655
+ databases = payload.get("compatibleDatabases") or []
656
+ if payload.get("sourceDatabase"):
657
+ databases = [*databases, payload["sourceDatabase"]]
658
+ allowed, records = _application_access(session, databases, strict=True)
659
+ return {"application": applications.create(
660
+ session.runtime, session.tenant, session.uname, payload,
661
+ semantic_records=records, allowed_databases=allowed)}
662
+ current = routes_assets._application_databases(session.runtime, app_id)
663
+ allowed, _ = _application_access(session, current)
664
+ if not current or not allowed:
665
+ raise applications.ApplicationError("application does not exist")
666
+ proposed = payload.get("compatibleDatabases") if "compatibleDatabases" in payload else current
667
+ proposed_allowed, records = _application_access(session, proposed, strict=True)
668
+ return {"application": applications.mutate(
669
+ session.runtime, app_id, session.uname, session.admin, payload,
670
+ semantic_records=records, allowed_databases=allowed | proposed_allowed,
671
+ target_database=next((database for database in proposed
672
+ if database in proposed_allowed), None))}
673
+ except (applications.ApplicationError, PermissionError) as exc:
674
+ routes_assets._application_error(exc)
675
+
676
+
677
+ def _tool_context(session: Session) -> external_tools.ToolContext:
678
+ # Import within the API layer. Core receives callbacks, never an upward dependency.
679
+ import routes_sheet
680
+ import formula_runtime
681
+ import routes_assets
682
+ from modules import sheet as sheet_mod
683
+
684
+ def project(book):
685
+ served = routes_sheet._read_for(session, book)
686
+ databases = _key_scope(session)[1]
687
+ if databases != external_access.ALL_DATABASES:
688
+ # The owner projection can share cells with its input; key scope is response-only.
689
+ served = copy.deepcopy(served)
690
+ for tab in served.get("sheets", []):
691
+ for cell in tab.get("cells", {}).values():
692
+ if not isinstance(cell, dict) or cell.get("kind") != "bound":
693
+ continue
694
+ binding = cell.get("binding")
695
+ database = binding.get("database") if isinstance(binding, dict) else None
696
+ if not external_access.database_allowed(databases, database):
697
+ cell["value"] = None
698
+ cell["display"] = ""
699
+ routes_sheet._recompute_formulas(served, formula_runtime, sheet_mod)
700
+ return served
701
+
702
+ def resolve_binding(binding, book, tab):
703
+ if isinstance(binding, dict):
704
+ _assert_database_scope(session, binding.get("database"))
705
+ return routes_sheet._resolve_cell_binding(session, binding, book, tab)
706
+
707
+ def cache_binding(workbook_id, tab, addr, binding, resolved):
708
+ # Same best-effort cache fill as the UI's compute_cell. A fresh value must replace
709
+ # this scope's old slot before an editor's read projection can use that slot.
710
+ try:
711
+ sheets.fill_scope_cache(session.runtime, workbook_id, tab + "!" + addr,
712
+ routes_sheet._binding_scope_key(session, resolved["database"], binding),
713
+ resolved["value"], routes_sheet._now_iso(), resolved["database"])
714
+ except Exception:
715
+ pass
716
+
717
+ label = external_access.label_for(session.runtime, session.claims["external_key_id"])
718
+ maker = session.user.get("name") or session.uname
719
+ actor_name = f"{maker} via key {label}"
720
+ if _CREDENTIAL_PREFIX in actor_name:
721
+ raise sheets.SheetRefusal("credential_in_attribution", "Maker names and key labels must not contain credential values.")
722
+ return external_tools.ToolContext(
723
+ session=session, actor_name=actor_name,
724
+ require_reader=lambda book: routes_sheet._require_reader(session, book),
725
+ require_editor=lambda book: routes_sheet._require_editor(session, book),
726
+ project_book=project,
727
+ resolve_binding=resolve_binding,
728
+ cache_binding=cache_binding,
729
+ list_databases=lambda: list_databases(session),
730
+ describe_database=lambda database: database_schema(database, session),
731
+ view_event=lambda kind, args: _view_event(session, kind, args),
732
+ app_upsert=lambda args: _application_upsert(session, args),
733
+ # Reuse the UI's atomic NameConfirmedDelete wrapper as well as the registry delete.
734
+ app_delete=lambda args: routes_assets.delete_application(
735
+ args["app_id"], {"confirmName": args["confirmName"]}, session),
736
+ )
737
 
738
 
739
+ def _tool_refusal(name, code, message, status=400, **extra):
740
+ # Never echo arbitrary method/tool names or raw exception text across the bearer seam.
741
+ tool_name = name if isinstance(name, str) and name in external_tools.TOOLS else "tools/call"
742
+ sentence = f"{tool_name}: {_markdown_inline(message)}"
743
+ if _CREDENTIAL_PREFIX in sentence:
744
+ sentence = f"{tool_name}: The result contained a credential value and was withheld."
745
+ return {"ok": False, "error": {"tool": tool_name, "code": code,
746
+ "message": sentence, "status": status, **extra}}
747
+
748
+
749
+ def _mcp_content(rid, out):
750
+ encoded = json.dumps(out, default=str, ensure_ascii=True, allow_nan=False)
751
+ if _CREDENTIAL_PREFIX in encoded:
752
+ out = _tool_refusal(None, "credential_in_output", "The result contained a credential value and was withheld.")
753
+ encoded = json.dumps(out)
754
+ return {"jsonrpc": "2.0", "id": rid, "result": {
755
+ "content": [{"type": "text", "text": encoded}], "isError": not out.get("ok", False)}}
756
+
757
+
758
+ def _handle_mcp(req, session: Session):
759
+ """One HTTP request object; the stdio precedent's JSON-RPC and content envelopes."""
760
+ def protocol_error(rid, code, message):
761
+ return {"jsonrpc": "2.0", "id": rid, "error": {"code": code, "message": message}}
762
+
763
+ if not isinstance(req, dict):
764
+ return protocol_error(None, -32600, "An MCP request must be a JSON-RPC object.")
765
+ rid, method = req.get("id"), req.get("method")
766
+ if (rid is not None and (type(rid) not in (str, int) or _CREDENTIAL_PREFIX in str(rid))):
767
+ return protocol_error(None, -32600, "The request id must be a number or a safe string.")
768
+ if rid is None:
769
+ # Notifications never dispatch writes and never return a JSON-RPC body.
770
+ return None
771
+ if req.get("jsonrpc") != "2.0" or not isinstance(method, str):
772
+ return protocol_error(rid, -32600, "An MCP request needs jsonrpc 2.0 and a method.")
773
+ params = req.get("params", {})
774
+ if not isinstance(params, dict):
775
+ return protocol_error(rid, -32602, "MCP params must be an object.")
776
+ name = params.get("name")
777
+ try:
778
+ # Same Analyst wall as the read doors, inside the dispatcher so a grant refusal
779
+ # remains MCP content. Only the unchanged bearer dependency produces an HTTP 401.
780
+ require_external_analyst(session)
781
+ if method == "initialize":
782
+ protocol = params.get("protocolVersion") or "2025-06-18"
783
+ if not isinstance(protocol, str) or _CREDENTIAL_PREFIX in protocol:
784
+ return protocol_error(rid, -32602, "The protocol version must be a safe string.")
785
+ return {"jsonrpc": "2.0", "id": rid, "result": {
786
+ "protocolVersion": protocol, "capabilities": {"tools": {}},
787
+ "serverInfo": {"name": "loopable-external", "version": "1.0.0"}}}
788
+ if method == "ping":
789
+ return {"jsonrpc": "2.0", "id": rid, "result": {}}
790
+ if method == "tools/list":
791
+ return {"jsonrpc": "2.0", "id": rid, "result": {"tools": [
792
+ {"name": key, "description": tool["description"], "inputSchema": tool["inputSchema"]}
793
+ for key, tool in external_tools.TOOLS.items()]}}
794
+ if method != "tools/call":
795
+ return protocol_error(rid, -32601, "The requested MCP method was not found.")
796
+ if isinstance(name, str) and name in _WRITE_TOOLS:
797
+ _assert_write_scope(session, name)
798
+ for database in _databases_in_arguments(params.get("arguments", {})):
799
+ _assert_database_scope(session, database)
800
+ data = external_tools.dispatch(_tool_context(session), name, params.get("arguments", {}))
801
+ if isinstance(data, dict) and data.get("refused"):
802
+ first = data["refused"][0]
803
+ out = _tool_refusal(name, first["code"], first["sentence"])
804
+ out["error"]["data"] = data # Accepted edits remain visible even on partial refusal.
805
+ else:
806
+ out = {"ok": True, "data": data}
807
+ except external_access.ExternalAuthError:
808
+ _auth_refusal()
809
+ except external_access.ExternalAccessError:
810
+ out = _tool_refusal(name, "external_access_unavailable", "The AI API is unavailable right now.", 503)
811
+ except sheets.SheetRefusal as exc:
812
+ import routes_sheet
813
+ translated = routes_sheet._refuse(exc)
814
+ info = translated.detail["error"]
815
+ out = _tool_refusal(name, info["code"], info["message"], translated.status_code,
816
+ **{k: v for k, v in info.items() if k not in ("code", "message")})
817
+ except HTTPException as exc:
818
+ if exc.status_code == 401:
819
+ _auth_refusal()
820
+ info = exc.detail.get("error", {}) if isinstance(exc.detail, dict) else {}
821
+ out = _tool_refusal(name, info.get("code", "tool_refused"),
822
+ info.get("message", "The app refused this operation."), exc.status_code)
823
+ except (TypeError, ValueError, KeyError, AttributeError):
824
+ out = _tool_refusal(name, "tool_arguments_invalid", "The arguments could not be read. Use the published input schema.")
825
+ except Exception:
826
+ out = _tool_refusal(name, "tool_unavailable", "The tool could not complete this operation.", 503)
827
+ return _mcp_content(rid, out)
828
+
829
+
830
+ @router.post("/mcp")
831
+ async def external_mcp(request: Request, session: Session = Depends(require_external_session)):
832
+ """Remote MCP without a browser session, with the read doors' maker and Analyst wall."""
833
+ try:
834
+ req = await request.json()
835
+ except (ValueError, UnicodeError):
836
+ return {"jsonrpc": "2.0", "id": None,
837
+ "error": {"code": -32700, "message": "The request body must be valid JSON."}}
838
+ reply = await run_in_threadpool(_handle_mcp, req, session)
839
+ return Response(status_code=204) if reply is None else reply
840
+
841
+
842
+ # R8 (SDLC 61) revokes the read-only surface. Every write must be named here deliberately.
843
+ # A new write in the registry without a matching literal raises at import, before serving.
844
+ _WRITE_TOOLS = frozenset({
845
+ "sheet_create_workbook", "sheet_add_tab", "sheet_write_cells", "sheet_set_binding",
846
+ "sheet_set_styles", "sheet_set_dimensions", "sheet_set_validation", "sheet_set_freeze",
847
+ "sheet_add_rule", "sheet_set_chart", "sheet_structure",
848
+ "sheet_save_template",
849
+ "view_upsert", "view_delete", "app_upsert", "app_delete",
850
+ })
851
+
852
+
853
+ def _assert_write_allow_list() -> None:
854
+ declared = {name for name, tool in external_tools.TOOLS.items() if tool.get("write")}
855
+ undeclared = declared - _WRITE_TOOLS
856
+ stale = _WRITE_TOOLS - declared
857
+ if undeclared or stale:
858
+ raise RuntimeError("External write allow-list mismatch: undeclared="
859
+ + repr(sorted(undeclared)) + ", stale=" + repr(sorted(stale)))
860
+
861
+
862
+ _assert_write_allow_list()
api/routes_grid.py CHANGED
@@ -2491,9 +2491,11 @@ def _topic_rt(st, module):
2491
  return st
2492
 
2493
 
2494
- def _customer_rows(table_key, user, st):
2495
  """`(fields, rows)` for the customer topic — the SAME derivation `_team_agent` uses."""
2496
  import aios_grid
 
 
2497
  import core.perm_scope as perm_scope
2498
  from routes_customers import _pool_for
2499
 
@@ -2502,15 +2504,16 @@ def _customer_rows(table_key, user, st):
2502
  return list(aios_grid.FIELDS), _pool_for(rt, team_id, agent)
2503
 
2504
 
2505
- def _product_rows(table_key, user, st):
2506
  """`(fields, rows)` for the product topic. `consolidated=` follows the derived scope, so a
2507
  BU-pinned reader gets that BU's field contract rather than the consolidated one."""
2508
  import core.perm_scope as perm_scope
2509
  from routes_products import _pool_for, pd_fields
2510
 
2511
- rt = _topic_rt(st, table_key)
2512
  team_id, _agent = perm_scope.derive_pool_scope(user, table_key)
2513
- return pd_fields(consolidated=team_id is None, st=st), _pool_for(rt, team_id)
 
2514
 
2515
 
2516
  def _register_topic_rows():
@@ -2747,7 +2750,8 @@ def _may_open(session: Session, key: str, st=None, memo: dict = None):
2747
  return _walled_fields(session, key, st=st, memo=memo) is not None
2748
 
2749
 
2750
- def _source_fields(session: Session, table: str, st=None, memo: dict = None):
 
2751
  """The link-bearing field list for `table`, under this session's field wall.
2752
 
2753
  ⛔⛔ THE PRE-SET COLUMNS ARE NOT IN `scoped_fields` AND THAT IS THE TRAP THIS FUNCTION EXISTS
@@ -2769,7 +2773,8 @@ def _source_fields(session: Session, table: str, st=None, memo: dict = None):
2769
  import odoo_relational
2770
  fields = fields + list(odoo_relational.module_link_fields(table) or ())
2771
  except Exception: # noqa: BLE001
2772
- pass
 
2773
  return fields
2774
 
2775
 
@@ -2826,7 +2831,7 @@ def _ensure_odoo_spec():
2826
  return False
2827
 
2828
 
2829
- def _unwalled_fields(defs: dict, key: str):
2830
  """`key`'s field list WITHOUT this session's FIELD wall, for the W53-T18 diff pass only.
2831
 
2832
  ⛔⛔ IT IS NEVER SERVED AND NEVER WIDENS A TAB. Its ONE consumer is the unfiltered half of
@@ -2849,7 +2854,8 @@ def _unwalled_fields(defs: dict, key: str):
2849
  import odoo_relational
2850
  fields = fields + list(odoo_relational.module_link_fields(key) or ())
2851
  except Exception: # noqa: BLE001
2852
- pass
 
2853
  return fields
2854
 
2855
 
@@ -2884,7 +2890,7 @@ def _relation_context(session: Session, table: str, memo: dict = None):
2884
  than a silently empty grid — and `scoped_fields` is a `want_rows=False` door by construction.
2885
  Nothing here is handed to a mutation; `grid_related` is a GET.
2886
  ⛔ IT CANNOT WIDEN A WALL, and that was checked rather than assumed: the lent handle serves
2887
- only `user_tables` and (lazily, once) `object_shares`, every other store key falls through to
2888
  the real runtime, and `session.require` is not given it at all. MEASURED 2026-09-06 across all
2889
  ten of tenant #0's databases: `scoped_fields` under the lent handle returned a field list
2890
  EQUAL, key for key and value for value, to the one it returned under the bare runtime.
@@ -2895,16 +2901,19 @@ def _relation_context(session: Session, table: str, memo: dict = None):
2895
  """
2896
  import core.user_tables as user_tables
2897
 
 
2898
  try:
2899
  defs = dict(user_tables.all_defs(st=session.runtime) or {})
2900
  except Exception: # noqa: BLE001
2901
  defs = {}
 
2902
  try:
2903
  lent = user_tables.lend(session.runtime, **{user_tables.STORE_KEY: defs})
2904
  except Exception: # noqa: BLE001
2905
  # A store handle too old to lend is CORRECT-but-slow, exactly as `all_defs`' own fallback
2906
  # is: the pass simply goes back to paying a round trip per wall resolution.
2907
  lent = session.runtime
 
2908
 
2909
  # ⭐ ONE WALL RESOLUTION PER TABLE, SHARED BY BOTH CALLBACKS. `open_memo` and `field_memo`
2910
  # below still memoise their own composition (the grant gate, the pre-set link append); this is
@@ -2919,24 +2928,30 @@ def _relation_context(session: Session, table: str, memo: dict = None):
2919
  def may_open(key):
2920
  if key not in open_memo:
2921
  open_memo[key] = _may_open(session, key, st=lent, memo=wall_memo)
 
 
2922
  return open_memo[key]
2923
 
2924
  def fields_for(key):
2925
  if key not in field_memo:
2926
- field_memo[key] = _source_fields(session, key, st=lent, memo=wall_memo)
 
 
 
2927
  return field_memo[key]
2928
 
2929
  def open_fields_for(key):
2930
  if key not in wide_memo:
2931
  walled = list(fields_for(key) or ())
2932
  seen = {str(f.get("key")) for f in walled if isinstance(f, dict)}
2933
- extra = [f for f in _unwalled_fields(defs, key)
2934
  if isinstance(f, dict) and str(f.get("key")) not in seen]
2935
  wide_memo[key] = walled + extra
2936
  return wide_memo[key]
2937
 
2938
- return {"defs": defs, "may_open": may_open, "fields_for": fields_for,
2939
- "open_fields_for": open_fields_for}
 
2940
 
2941
 
2942
  def related_path_entries(session: Session, table: str, depth: int, ctx: dict = None):
@@ -2952,14 +2967,120 @@ def related_path_entries(session: Session, table: str, depth: int, ctx: dict = N
2952
  """
2953
  import core.relations as relations
2954
 
2955
- _ensure_odoo_spec()
2956
  table = str(table or "")
2957
  ctx = ctx or _relation_context(session, table)
 
 
2958
  return relations.related_paths(table, fields=ctx["fields_for"](table), defs=ctx["defs"],
2959
  depth=depth, may_open=ctx["may_open"],
2960
  fields_for=ctx["fields_for"])
2961
 
2962
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2963
  def related_refusals(session: Session, table: str, depth: int, entries: list, ctx: dict = None):
2964
  """W53-T18 / R13 / C3 — WHAT THE WALL REMOVED from `entries`, by name. The SIBLING of `related`.
2965
 
@@ -3031,6 +3152,33 @@ def _refusal_error(diag: dict):
3031
  return exc
3032
 
3033
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
3034
  @router.get("/grid/{table}/related")
3035
  def grid_related(table: str,
3036
  depth: int = Query(default=2),
@@ -3102,12 +3250,27 @@ def grid_related(table: str,
3102
  f"{relations.MAX_DEPTH} (those plus everything one further named hop away), and "
3103
  f"this request asked for {depth}. Ask for 1 or {relations.MAX_DEPTH}")
3104
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
3105
  ctx = _relation_context(session, table, memo=wall_memo)
3106
  entries = related_path_entries(session, table, depth, ctx)
3107
- if via:
3108
- wanted = [str(v) for v in via if str(v or "").strip()]
3109
- if len(wanted) > relations.MAX_DEPTH:
3110
- raise err(400, "over_depth", relations.over_depth_refusal(table, wanted))
3111
  hit = relations.pick_path(entries, wanted)
3112
  if hit is None:
3113
  # ⭐⭐ W53-T18 / R13 / C3 — THE ONE GENERIC `unknown_via` IS NOW ONE OF FOUR NAMED
@@ -3127,7 +3290,19 @@ def grid_related(table: str,
3127
  # ⚠ A RESOLVED PIN REPORTS NO REFUSALS. The caller named one path and got it; the other
3128
  # tabs the wall removed are the LISTING's answer, not this one, and repeating them here
3129
  # would attach an unasked-for permission report to every successful read.
3130
- return {"ok": True, "table": table, "depth": depth, "related": [hit], "refusals": []}
3131
- return {"ok": True, "table": table, "depth": depth, "related": entries,
3132
- "refusals": related_refusals(session, table, depth, entries, ctx)}
3133
-
 
 
 
 
 
 
 
 
 
 
 
 
 
2491
  return st
2492
 
2493
 
2494
+ def _customer_rows(table_key, user, st, want_rows=True):
2495
  """`(fields, rows)` for the customer topic — the SAME derivation `_team_agent` uses."""
2496
  import aios_grid
2497
+ if not want_rows:
2498
+ return list(aios_grid.FIELDS), []
2499
  import core.perm_scope as perm_scope
2500
  from routes_customers import _pool_for
2501
 
 
2504
  return list(aios_grid.FIELDS), _pool_for(rt, team_id, agent)
2505
 
2506
 
2507
+ def _product_rows(table_key, user, st, want_rows=True):
2508
  """`(fields, rows)` for the product topic. `consolidated=` follows the derived scope, so a
2509
  BU-pinned reader gets that BU's field contract rather than the consolidated one."""
2510
  import core.perm_scope as perm_scope
2511
  from routes_products import _pool_for, pd_fields
2512
 
2513
+ rt = _topic_rt(st, table_key) if want_rows else None
2514
  team_id, _agent = perm_scope.derive_pool_scope(user, table_key)
2515
+ return pd_fields(consolidated=team_id is None, st=st), (
2516
+ _pool_for(rt, team_id) if want_rows else [])
2517
 
2518
 
2519
  def _register_topic_rows():
 
2750
  return _walled_fields(session, key, st=st, memo=memo) is not None
2751
 
2752
 
2753
+ def _source_fields(session: Session, table: str, st=None, memo: dict = None,
2754
+ read_status: dict = None):
2755
  """The link-bearing field list for `table`, under this session's field wall.
2756
 
2757
  ⛔⛔ THE PRE-SET COLUMNS ARE NOT IN `scoped_fields` AND THAT IS THE TRAP THIS FUNCTION EXISTS
 
2773
  import odoo_relational
2774
  fields = fields + list(odoo_relational.module_link_fields(table) or ())
2775
  except Exception: # noqa: BLE001
2776
+ if read_status is not None:
2777
+ read_status["cacheable"] = False
2778
  return fields
2779
 
2780
 
 
2831
  return False
2832
 
2833
 
2834
+ def _unwalled_fields(defs: dict, key: str, read_status: dict = None):
2835
  """`key`'s field list WITHOUT this session's FIELD wall, for the W53-T18 diff pass only.
2836
 
2837
  ⛔⛔ IT IS NEVER SERVED AND NEVER WIDENS A TAB. Its ONE consumer is the unfiltered half of
 
2854
  import odoo_relational
2855
  fields = fields + list(odoo_relational.module_link_fields(key) or ())
2856
  except Exception: # noqa: BLE001
2857
+ if read_status is not None:
2858
+ read_status["cacheable"] = False
2859
  return fields
2860
 
2861
 
 
2890
  than a silently empty grid — and `scoped_fields` is a `want_rows=False` door by construction.
2891
  Nothing here is handed to a mutation; `grid_related` is a GET.
2892
  ⛔ IT CANNOT WIDEN A WALL, and that was checked rather than assumed: the lent handle serves
2893
+ `user_tables`, (lazily, once) `object_shares` and read-only `__shared` overlays; other keys fall through to
2894
  the real runtime, and `session.require` is not given it at all. MEASURED 2026-09-06 across all
2895
  ten of tenant #0's databases: `scoped_fields` under the lent handle returned a field list
2896
  EQUAL, key for key and value for value, to the one it returned under the bare runtime.
 
2901
  """
2902
  import core.user_tables as user_tables
2903
 
2904
+ read_status = {"cacheable": True}
2905
  try:
2906
  defs = dict(user_tables.all_defs(st=session.runtime) or {})
2907
  except Exception: # noqa: BLE001
2908
  defs = {}
2909
+ read_status["cacheable"] = False
2910
  try:
2911
  lent = user_tables.lend(session.runtime, **{user_tables.STORE_KEY: defs})
2912
  except Exception: # noqa: BLE001
2913
  # A store handle too old to lend is CORRECT-but-slow, exactly as `all_defs`' own fallback
2914
  # is: the pass simply goes back to paying a round trip per wall resolution.
2915
  lent = session.runtime
2916
+ read_status["cacheable"] = False
2917
 
2918
  # ⭐ ONE WALL RESOLUTION PER TABLE, SHARED BY BOTH CALLBACKS. `open_memo` and `field_memo`
2919
  # below still memoise their own composition (the grant gate, the pre-set link append); this is
 
2928
  def may_open(key):
2929
  if key not in open_memo:
2930
  open_memo[key] = _may_open(session, key, st=lent, memo=wall_memo)
2931
+ if not open_memo[key]:
2932
+ read_status["cacheable"] = False
2933
  return open_memo[key]
2934
 
2935
  def fields_for(key):
2936
  if key not in field_memo:
2937
+ field_memo[key] = _source_fields(session, key, st=lent, memo=wall_memo,
2938
+ read_status=read_status)
2939
+ if wall_memo.get(key) is None:
2940
+ read_status["cacheable"] = False
2941
  return field_memo[key]
2942
 
2943
  def open_fields_for(key):
2944
  if key not in wide_memo:
2945
  walled = list(fields_for(key) or ())
2946
  seen = {str(f.get("key")) for f in walled if isinstance(f, dict)}
2947
+ extra = [f for f in _unwalled_fields(defs, key, read_status=read_status)
2948
  if isinstance(f, dict) and str(f.get("key")) not in seen]
2949
  wide_memo[key] = walled + extra
2950
  return wide_memo[key]
2951
 
2952
+ read_status.update(defs=defs, may_open=may_open, fields_for=fields_for,
2953
+ open_fields_for=open_fields_for)
2954
+ return read_status
2955
 
2956
 
2957
  def related_path_entries(session: Session, table: str, depth: int, ctx: dict = None):
 
2967
  """
2968
  import core.relations as relations
2969
 
2970
+ spec_ready = _ensure_odoo_spec()
2971
  table = str(table or "")
2972
  ctx = ctx or _relation_context(session, table)
2973
+ if not spec_ready:
2974
+ ctx["cacheable"] = False
2975
  return relations.related_paths(table, fields=ctx["fields_for"](table), defs=ctx["defs"],
2976
  depth=depth, may_open=ctx["may_open"],
2977
  fields_for=ctx["fields_for"])
2978
 
2979
 
2980
+ def related_measure_pin(session, target, related, requested):
2981
+ """Compile metric membership with the rows door's exact path and leg resolvers.
2982
+
2983
+ SQL bindings are host declarations. No SQL, callbacks or row identities are trusted from
2984
+ the client beyond the origin id, named edge and requested subset of resolved target ids.
2985
+ Retain through rows as well as explicit hops so a metric cannot expand behind a row wall.
2986
+ """
2987
+ from core import perm_scope, relations
2988
+ import routes_odoo_tables as door
2989
+
2990
+ if isinstance(related, str):
2991
+ related = door._json_arg(related, 'related')
2992
+ allowed = door._relation_target_ids(session, target, related)
2993
+ if set(requested) - allowed:
2994
+ raise err(403, 'metric_record_refused', 'A requested record is outside this linked grid.')
2995
+ source, via = str(related['table']), related.get('via')
2996
+ path = door._via_path(via)
2997
+ entries = related_path_entries(session, source, len(path) if path else 1)
2998
+ entry = relations.pick_path(entries, path) if path else relations.pick(entries, via)
2999
+ legs = [(source, entry)]
3000
+ if len(entry.get('path') or ()) > 1:
3001
+ first, middle, last = door._related_path_legs(session, source, entry, entries)
3002
+ legs = [(source, first), (middle, last)]
3003
+
3004
+ bindings = {
3005
+ 'sale_order_line': [
3006
+ dict(table='customer_data', column='pid', sql='{alias}.order_partner_id'),
3007
+ dict(table='product_data', column='product_id', sql='{alias}.product_id'),
3008
+ dict(table='ut_odoo_orders', column='pid', sql='{alias}.order_id'),
3009
+ dict(table='ut_odoo_order_lines', column='pid', sql='{alias}.id'),
3010
+ dict(table='ut_odoo_agents', column='pid', sql='rp.agent_id'),
3011
+ ],
3012
+ 'sale_order': [
3013
+ dict(table='customer_data', column='pid', sql='{alias}.partner_id'),
3014
+ dict(table='ut_odoo_orders', column='pid', sql='{alias}.id'),
3015
+ dict(table='ut_odoo_agents', column='pid', sql='rp.agent_id'),
3016
+ dict(table='ut_odoo_order_lines', column='order_id', sql='{alias}.id'),
3017
+ ],
3018
+ }
3019
+ members = {}
3020
+ def read(table, column, values, where):
3021
+ columns = {'pid', column}
3022
+ if table == 'product_data':
3023
+ columns.add('product_id')
3024
+ if table == 'ut_odoo_order_lines':
3025
+ columns.add('order_id')
3026
+ return door._relation_rows(session, table, columns, match=(column, values), where=where)
3027
+
3028
+ def retain(table, ids):
3029
+ rows = read(table, 'pid', ids, None)
3030
+ if table in members:
3031
+ existing = {r['pid'] for r in members[table]}
3032
+ if {r['pid'] for r in rows} != existing:
3033
+ raise err(409, 'metric_scope_unsupported',
3034
+ 'This metric cannot distinguish repeated databases in the record path. '
3035
+ 'Use a direct relation or add a metric binding for these separate roles.')
3036
+ rows = [r for r in rows if r['pid'] in existing]
3037
+ members[table] = rows
3038
+
3039
+ with relations.measure_pin_context(read=read, bindings=bindings,
3040
+ may_open=lambda key: _may_open(session, key)):
3041
+ ids = {int(related['record'])}
3042
+ retain(source, ids)
3043
+ if not members[source]:
3044
+ raise err(403, 'metric_record_refused', 'The origin record is unavailable to this account.')
3045
+ for table, leg in legs:
3046
+ link, via = leg.get('link') or {}, leg.get('via') or {}
3047
+ through = link.get('through')
3048
+ if through and link.get('selectionMode') != 'manual':
3049
+ reverse = via.get('direction') == 'reverse'
3050
+ left = str(link['throughTo'] if reverse else link['throughOn'])
3051
+ frm = str(via.get('from') or 'pid')
3052
+ origins = door._relation_rows(session, table, [frm], ids=ids)
3053
+ bridge = read(str(through), left,
3054
+ [r[frm] for r in origins if r.get(frm) is not None], None)
3055
+ retain(str(through), {r['pid'] for r in bridge})
3056
+ ids = door._relation_leg_ids(session, table, ids, leg)
3057
+ retain(str(leg['target']), ids)
3058
+ required = tuple(members)
3059
+ # A manual link need not name the sales bridge. Its metrics still obey that
3060
+ # database's row wall, intersected with the already resolved record membership.
3061
+ if 'ut_odoo_order_lines' not in members:
3062
+ for table, column, join in [('customer_data', 'pid', 'partner_id'),
3063
+ ('product_data', 'product_id', 'product_id'),
3064
+ ('ut_odoo_orders', 'pid', 'order_id')]:
3065
+ if table in members:
3066
+ bridge = read('ut_odoo_order_lines', join,
3067
+ [r[column] for r in members[table]], None)
3068
+ # This extra wall belongs to line metrics. Header metrics use their own
3069
+ # traversed order membership, including legitimate orders with no lines.
3070
+ retain('ut_odoo_order_lines', {r['pid'] for r in bridge})
3071
+ bindings['sale_order'] = [binding for binding in bindings['sale_order']
3072
+ if binding['table'] != 'ut_odoo_order_lines']
3073
+ break
3074
+ # Required path tables must all bind. Binding only the destination would return globals.
3075
+ pin = relations.MeasurePin(entry['path'], members, bindings,
3076
+ required=required)
3077
+ scopes = [perm_scope.derive_pool_scope(session.user, table) for table in required]
3078
+ teams = {team for team, _agent in scopes if team is not None}
3079
+ if len(teams) > 1:
3080
+ raise err(403, 'metric_scope_conflict', 'The linked databases have incompatible business unit scopes.')
3081
+ return pin, next(iter(teams), None)
3082
+
3083
+
3084
  def related_refusals(session: Session, table: str, depth: int, entries: list, ctx: dict = None):
3085
  """W53-T18 / R13 / C3 — WHAT THE WALL REMOVED from `entries`, by name. The SIBLING of `related`.
3086
 
 
3152
  return exc
3153
 
3154
 
3155
+ _RELATED_CACHE_LIMIT = 128
3156
+
3157
+
3158
+ def _related_cache_key(session: Session, table: str, depth: int, via):
3159
+ """Names-only answers follow this user's schema, grant, sync and registrar revisions.
3160
+
3161
+ Tokens are read-through dict lookups, not document downloads. A runtime without revision
3162
+ support can still serve an uncached answer. Never reuse an answer with unknown revisions.
3163
+ """
3164
+ import core.store as store
3165
+ import core.shares as shares
3166
+ import core.user_tables as user_tables
3167
+ import odoo_relational
3168
+
3169
+ try:
3170
+ if not session.uname:
3171
+ return None
3172
+ return (session.uname, table, depth, via,
3173
+ session.runtime.revision(user_tables.STORE_KEY)["token"],
3174
+ session.runtime.revision(shares.SHARES_KEY)["token"],
3175
+ store.revision("users")["token"],
3176
+ session.runtime.revision(odoo_relational.CONFIG_KEY)["token"],
3177
+ user_tables.register_read_through(), user_tables.register_connected_prefix())
3178
+ except Exception: # noqa: BLE001
3179
+ return None
3180
+
3181
+
3182
  @router.get("/grid/{table}/related")
3183
  def grid_related(table: str,
3184
  depth: int = Query(default=2),
 
3250
  f"{relations.MAX_DEPTH} (those plus everything one further named hop away), and "
3251
  f"this request asked for {depth}. Ask for 1 or {relations.MAX_DEPTH}")
3252
 
3253
+ wanted = tuple(str(v).strip() for v in via if str(v or "").strip()) if via else None
3254
+ if wanted is not None and len(wanted) > relations.MAX_DEPTH:
3255
+ raise err(400, "over_depth", relations.over_depth_refusal(table, list(wanted)))
3256
+
3257
+ # This registration must run even on a hit; an import order repair cannot be skipped by
3258
+ # memoisation. Failed registration or reads are served as before, but never remembered.
3259
+ spec_ready = _ensure_odoo_spec()
3260
+ cache = getattr(session.runtime, "related_cache", None)
3261
+ cache_key = (_related_cache_key(session, table, depth, wanted)
3262
+ if spec_ready and cache is not None else None)
3263
+ if spec_ready and cache_key is not None:
3264
+ from copy import deepcopy
3265
+ with session.runtime._lock:
3266
+ cached = cache.get(cache_key)
3267
+ if cached is not None:
3268
+ return deepcopy(cached)
3269
+
3270
  ctx = _relation_context(session, table, memo=wall_memo)
3271
  entries = related_path_entries(session, table, depth, ctx)
3272
+ if wanted is not None:
3273
+ wanted = list(wanted)
 
 
3274
  hit = relations.pick_path(entries, wanted)
3275
  if hit is None:
3276
  # ⭐⭐ W53-T18 / R13 / C3 — THE ONE GENERIC `unknown_via` IS NOW ONE OF FOUR NAMED
 
3290
  # ⚠ A RESOLVED PIN REPORTS NO REFUSALS. The caller named one path and got it; the other
3291
  # tabs the wall removed are the LISTING's answer, not this one, and repeating them here
3292
  # would attach an unasked-for permission report to every successful read.
3293
+ answer = {"ok": True, "table": table, "depth": depth, "related": [hit], "refusals": []}
3294
+ else:
3295
+ answer = {"ok": True, "table": table, "depth": depth, "related": entries,
3296
+ "refusals": related_refusals(session, table, depth, entries, ctx)}
3297
+ if (spec_ready and cache_key is not None and ctx.get("cacheable", False) and ctx["defs"]
3298
+ and not answer["refusals"]
3299
+ and cache_key == _related_cache_key(session, table, depth, cache_key[3])):
3300
+ from copy import deepcopy
3301
+ # The Amazon purge's pool-cache tuple does not include this memo. That is safe because
3302
+ # every lookup checks the current tokens; old entries cannot match changed metadata.
3303
+ # Limit memory, not the number of related databases: eviction just rebuilds next open.
3304
+ with session.runtime._lock:
3305
+ if len(cache) >= _RELATED_CACHE_LIMIT:
3306
+ cache.clear()
3307
+ cache[cache_key] = deepcopy(answer)
3308
+ return answer
api/routes_keychain.py CHANGED
@@ -920,6 +920,14 @@ def add_key(body: dict = Body(default=None), session: Session = Depends(require_
920
  _rel_reconnect(session.runtime)
921
  except Exception: # noqa: BLE001
922
  pass
 
 
 
 
 
 
 
 
923
  if scope != DEFAULT_SCOPE:
924
  try:
925
  _write_scope(session.runtime, row["id"], scope, session.uname)
 
920
  _rel_reconnect(session.runtime)
921
  except Exception: # noqa: BLE001
922
  pass
923
+ elif etype == "amazon_sp":
924
+ # D-29 / intake F6: the HTTP thread only records intent. The resident resync thread
925
+ # remains the mirror's single writer. A queue failure must not undo a stored credential.
926
+ try:
927
+ from core import connector_sync
928
+ connector_sync.request_sync("amazon_sp", session.runtime.key, requested_by=session.uname)
929
+ except Exception: # noqa: BLE001
930
+ pass
931
  if scope != DEFAULT_SCOPE:
932
  try:
933
  _write_scope(session.runtime, row["id"], scope, session.uname)
api/routes_module_index.py ADDED
@@ -0,0 +1,132 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """W61-T20 / C6: personal App and Agents folder trees in the tenant store."""
2
+ import copy
3
+
4
+ from fastapi import APIRouter, Body, Depends, HTTPException
5
+
6
+ from core.sheets import SheetFolderRefusal, validate_folder_tree
7
+ from deps import Session, err, module_gate, require_session
8
+ import routes_assets
9
+ import routes_automation
10
+ import voice_control_plane as voice
11
+
12
+ router = APIRouter(prefix="/api/v1/module-index")
13
+ _AGENTS_GATE = module_gate("automation")
14
+
15
+
16
+ def _sentence(value):
17
+ return " ".join(str(value).replace("\u2014", "-").replace("\u2013", "-").split())
18
+
19
+
20
+ def _refuse(exc):
21
+ status = {"module_index_unknown": 404, "module_index_user_forbidden": 403,
22
+ "module_index_item_forbidden": 403, "index_revision_stale": 409}.get(exc.code, 400)
23
+ return err(status, exc.code, _sentence(exc.message))
24
+
25
+
26
+ def _access(module: str, session: Session = Depends(require_session)):
27
+ if module not in {"app", "agents"}:
28
+ raise _refuse(SheetFolderRefusal(
29
+ "module_index_unknown", f'Folder index "{module}" is not supported; use app or agents.'))
30
+ # App has the same session-only door as GET /applications; its rows have their own wall.
31
+ if module == "agents":
32
+ try:
33
+ _AGENTS_GATE(session)
34
+ except HTTPException as exc:
35
+ if exc.status_code != 403:
36
+ raise
37
+ raise err(403, "forbidden", "Your account or workspace does not have access to Agents (automation).")
38
+ return session
39
+
40
+
41
+ def _owner(session, user):
42
+ owner = user or session.uname
43
+ if owner != session.uname and not session.admin:
44
+ raise SheetFolderRefusal(
45
+ "module_index_user_forbidden", f'Only an administrator may read the folder tree for "{owner}".')
46
+ return owner
47
+
48
+
49
+ def _readable_ids(module, session):
50
+ # The projection is hygiene; the per-item list door is the wall. The client renders
51
+ # only listed items, so an unverifiable placement renders no tile. Use local voice
52
+ # records, never audience-option assembly, just to verify placement ids.
53
+ if module == "app":
54
+ return {row["id"] for row in routes_assets.application_index(session=session)["applications"]}
55
+ # ⭐⭐ W62-T17 — THE ID-ONLY ACCESSOR THIS COMMENT USED TO ASK FOR NOW EXISTS. It read
56
+ # *"all_definitions omits synthetic field/Odoo agents. Keep the complete automation list until
57
+ # it has an id-only accessor"*, and the whole Agents rail payload was being built to answer a
58
+ # membership question. `routes_automation.automation_ids` keeps all three strata (stored
59
+ # definitions, field agents, the derived Odoo sync row) and the statements seed, so the SET is
60
+ # unchanged; what it drops is `_wire` per automation, which read `agent_contracts` and
61
+ # `automation_agent_actions` once EACH TIME (MEASURED: 1,101 ms of this route's 1,180 ms on
62
+ # staging, `AutoWiki/waves/wave62/proto/T17-profile.md`).
63
+ return ({"agent:" + aid for aid in routes_automation.automation_ids(session)}
64
+ | {"voice:" + row["id"] for row in
65
+ voice._store(session.runtime)["voice_agent"].values() if isinstance(row, dict)})
66
+
67
+
68
+ def _project(bucket, ids):
69
+ folders, placed, _ = validate_folder_tree(bucket.get("folders"), bucket.get("itemFolders"))
70
+ return {"folders": folders, "itemFolders": {k: v for k, v in placed.items() if k in ids},
71
+ "rev": bucket.get("rev", 0)}
72
+
73
+
74
+ def _available(session):
75
+ if not session.runtime.available():
76
+ raise err(503, "store_unavailable", "The tenant store is unavailable. Reload folders to retry.")
77
+
78
+
79
+ @router.get("/{module}")
80
+ def read_module_index(module: str, user: str = "", session: Session = Depends(_access)):
81
+ try:
82
+ owner = _owner(session, user)
83
+ _available(session)
84
+ doc = session.runtime.get("module_index_" + module) or {}
85
+ bucket = (doc.get("users") or {}).get(owner) or {}
86
+ ids = _readable_ids(module, session) if bucket.get("itemFolders") else set()
87
+ return {"index": _project(bucket, ids)}
88
+ except SheetFolderRefusal as exc:
89
+ raise _refuse(exc)
90
+
91
+
92
+ @router.put("/{module}")
93
+ def write_module_index(module: str, body: dict = Body(default=None), user: str = "",
94
+ session: Session = Depends(_access)):
95
+ try:
96
+ owner = _owner(session, user)
97
+ # The admin override is for reading another user's tree, never for replacing it.
98
+ if owner != session.uname:
99
+ raise SheetFolderRefusal("module_index_user_forbidden",
100
+ f'The folder tree for "{owner}" can only be changed by its owner.')
101
+ _available(session)
102
+ raw = body or {}
103
+ expect_rev = raw.get("rev")
104
+ if type(expect_rev) is not int or expect_rev < 0:
105
+ raise SheetFolderRefusal("index_revision_required", "Reload folders before saving a folder change.")
106
+ placements = raw.get("itemFolders")
107
+ if not isinstance(raw.get("folders"), list) or not isinstance(placements, dict):
108
+ raise SheetFolderRefusal("module_index_invalid", "Send folders and itemFolders together to save the folder tree.")
109
+ ids = _readable_ids(module, session) if placements else set()
110
+ if any(item_id not in ids for item_id in placements):
111
+ raise SheetFolderRefusal("module_index_item_forbidden",
112
+ "A folder placement names an item you cannot read in this module.")
113
+ folders, placed, _ = validate_folder_tree(raw["folders"], placements)
114
+ saved = {}
115
+
116
+ def apply(current):
117
+ state = copy.deepcopy(current) if isinstance(current, dict) else {}
118
+ users = state.setdefault("users", {})
119
+ rev = (users.get(owner) or {}).get("rev", 0)
120
+ # Compare inside the store transaction, before touching any user's bucket.
121
+ if expect_rev != rev:
122
+ raise SheetFolderRefusal("index_revision_stale",
123
+ "Folders changed somewhere else. Reload folders and try again.")
124
+ saved.update(folders=folders, itemFolders=placed, rev=rev + 1)
125
+ users[owner] = dict(saved)
126
+ state["schema"] = 1
127
+ return state
128
+
129
+ session.runtime.update("module_index_" + module, apply, flush="sync")
130
+ return {"index": _project(saved, ids)}
131
+ except SheetFolderRefusal as exc:
132
+ raise _refuse(exc)
api/routes_nav.py CHANGED
@@ -81,6 +81,36 @@ def _now() -> int:
81
  return int(time.time())
82
 
83
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
84
  def _clean_nav_prefs(raw, page_keys, *, keep_unknown_ut=False):
85
  """Validated wholesale replacement, the `clean_folders` posture: prune, never invent.
86
 
@@ -118,7 +148,35 @@ def _clean_nav_prefs(raw, page_keys, *, keep_unknown_ut=False):
118
  placement[k] = v
119
  elif keep_unknown_ut and k.startswith("ut_"):
120
  placement[k] = v
121
- return {"folders": folders, "placement": placement}
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
122
 
123
 
124
  def _placeable_top_keys(session):
@@ -397,6 +455,28 @@ def _visible_database_keys(session):
397
  return keys, enumerated
398
 
399
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
400
  def _granted_view_ids(session):
401
  """Every view id a wave-21 NAMED-USER GRANT lets this caller see. Fail-closed to `set()`.
402
 
@@ -406,21 +486,13 @@ def _granted_view_ids(session):
406
  answer "which views has this person starred", so the derivation moves here beside the reader
407
  it feeds rather than being copied into a second file ([[one-evaluator-per-question]]).
408
 
409
- ⚠ THE READ-PREDICATE FILTER IS NOT BELT-AND-BRACES. `shared_with` answers "is there an entry
410
- naming me"; `may_see` answers whether that entry currently grants this caller visibility.
411
- Keeping those questions separate preserves dynamic all-eligible-user grants and fails closed
412
- for a stale or malformed grant. One tenant-wide bucket is read once per call.
413
  ⚠ Fail-closed: no grants is a NARROWER answer, never a wider one.
414
  """
415
- import core.shares as shares
416
- try:
417
- return {str(v) for v in
418
- ((shares.shared_with(session.uname, kind="view", st=session.runtime) or {})
419
- .get("view") or [])
420
- if shares.may_see("view", str(v), session.uname,
421
- st=session.runtime)}
422
- except Exception:
423
- return set()
424
 
425
 
426
  #: ⭐⭐ W54-T48 / OWNER RULING R4 / CONTRACT C4 — ONE CONNECTION, ONE FOLDER IN THE RAIL.
@@ -959,7 +1031,7 @@ def nav_prefs(session: Session = Depends(require_session)):
959
  try:
960
  stored = (session.runtime.get(_NAV_PREFS_KEY) or {}).get(session.uname) or {}
961
  except Exception:
962
- stored = {}
963
  keys, enumerated = _placeable_top_keys(session)
964
  return {"prefs": _clean_nav_prefs(stored, keys, keep_unknown_ut=not enumerated)}
965
 
@@ -967,17 +1039,19 @@ def nav_prefs(session: Session = Depends(require_session)):
967
  @router.post("/nav/prefs")
968
  def save_nav_prefs(body: dict = Body(default=None),
969
  session: Session = Depends(require_session)):
970
- """Wholesale replace, like the table folder stratum — the client sent its complete
971
- picture, validated here; a partial merge would resurrect deleted folders forever."""
972
  keys, enumerated = _placeable_top_keys(session)
973
- clean = _clean_nav_prefs(body or {}, keys, keep_unknown_ut=not enumerated)
974
  if not session.runtime.available():
975
  raise err(503, "store_unavailable",
976
  "the tenant store is unavailable. Nothing was saved.")
977
 
978
  def _up(data):
 
979
  data = data if isinstance(data, dict) else {}
980
- if clean["folders"] or clean["placement"]:
 
 
981
  data[session.uname] = clean
982
  else:
983
  data.pop(session.uname, None)
@@ -987,7 +1061,7 @@ def save_nav_prefs(body: dict = Body(default=None),
987
  session.runtime.update(_NAV_PREFS_KEY, _up)
988
  except Exception:
989
  raise err(503, "store_unavailable",
990
- "the folder change was not saved: the store refused the write.")
991
  return {"prefs": clean}
992
 
993
 
 
81
  return int(time.time())
82
 
83
 
84
+ def _clean_linked_layouts(raw):
85
+ """R8 personal presentation only. Never accept a SavedView or a field definition."""
86
+ if not isinstance(raw, dict):
87
+ return {}
88
+ layouts = {}
89
+ for key, value in raw.items():
90
+ if not isinstance(key, str) or not key or not isinstance(value, dict):
91
+ continue
92
+ layout = {}
93
+ for name in ("order", "visible"):
94
+ if isinstance(value.get(name), list):
95
+ layout[name] = list(dict.fromkeys(k for k in value[name] if isinstance(k, str)))
96
+ if isinstance(value.get("widths"), dict):
97
+ layout["widths"] = {k: w for k, w in value["widths"].items()
98
+ if isinstance(k, str) and type(w) in (int, float) and w > 0}
99
+ if isinstance(value.get("sorts"), list):
100
+ layout["sorts"] = [{"colId": s["colId"], "dir": s["dir"]}
101
+ for s in value["sorts"] if isinstance(s, dict)
102
+ and isinstance(s.get("colId"), str) and s.get("dir") in ("asc", "desc")]
103
+ if isinstance(value.get("filters"), list):
104
+ layout["filters"] = [n for n in value["filters"] if isinstance(n, dict)]
105
+ if value.get("filterConj") in ("and", "or"):
106
+ layout["filterConj"] = value["filterConj"]
107
+ if "groupBy" in value and (value["groupBy"] is None or isinstance(value["groupBy"], str)):
108
+ layout["groupBy"] = value["groupBy"]
109
+ if layout:
110
+ layouts[key] = layout
111
+ return layouts
112
+
113
+
114
  def _clean_nav_prefs(raw, page_keys, *, keep_unknown_ut=False):
115
  """Validated wholesale replacement, the `clean_folders` posture: prune, never invent.
116
 
 
148
  placement[k] = v
149
  elif keep_unknown_ut and k.startswith("ut_"):
150
  placement[k] = v
151
+ clean = {"folders": folders, "placement": placement}
152
+ layouts = _clean_linked_layouts(raw.get("linkedLayouts"))
153
+ if layouts:
154
+ clean["linkedLayouts"] = layouts
155
+ return clean
156
+
157
+
158
+ def _merge_nav_prefs(stored, body, page_keys, *, keep_unknown_ut=False):
159
+ """Replace navigation only when supplied; patch each personal link independently.
160
+
161
+ Called inside runtime.update so concurrent link and folder writes see the latest entry.
162
+ Folder clients may echo a stale linkedLayouts from GET; only a layout-only request owns it.
163
+ """
164
+ stored = stored if isinstance(stored, dict) else {}
165
+ body = body if isinstance(body, dict) else {}
166
+ navigation = "folders" in body or "placement" in body
167
+ merged = {**stored}
168
+ if navigation:
169
+ merged.update(folders=body.get("folders", []), placement=body.get("placement", {}))
170
+ else:
171
+ layouts = _clean_linked_layouts(stored.get("linkedLayouts"))
172
+ layouts.update(_clean_linked_layouts(body.get("linkedLayouts")))
173
+ merged["linkedLayouts"] = layouts
174
+ # A layout write does not own navigation pruning either.
175
+ clean = _clean_nav_prefs(merged, page_keys, keep_unknown_ut=keep_unknown_ut)
176
+ if not navigation:
177
+ for name, default in (("folders", []), ("placement", {})):
178
+ clean[name] = stored.get(name, default)
179
+ return clean
180
 
181
 
182
  def _placeable_top_keys(session):
 
455
  return keys, enumerated
456
 
457
 
458
+ def _granted_view_sources(session):
459
+ """`{view_id: owner}` for grants this caller may read, from one registry read.
460
+
461
+ `owner` may be ``None`` for a legacy ownerless grant. Callers preserve the old fail-closed
462
+ source lookup for that case; modern grants point directly at the canonical personal stratum.
463
+ """
464
+ import core.shares as shares
465
+ try:
466
+ out = {}
467
+ for raw_id, record in shares.grants_bulk("view", st=session.runtime).items():
468
+ owner = str(record.get("owner") or "").strip().lower() or None
469
+ if owner == session.uname:
470
+ continue
471
+ # Match the historical `shared_with` + `may_see` path: being an admin alone did not
472
+ # add a View to "shared with me"; a valid named or everyone grant did.
473
+ if shares.role_from(record, session.uname) is not None:
474
+ out[str(raw_id)] = owner
475
+ return out
476
+ except Exception:
477
+ return {}
478
+
479
+
480
  def _granted_view_ids(session):
481
  """Every view id a wave-21 NAMED-USER GRANT lets this caller see. Fail-closed to `set()`.
482
 
 
486
  answer "which views has this person starred", so the derivation moves here beside the reader
487
  it feeds rather than being copied into a second file ([[one-evaluator-per-question]]).
488
 
489
+ ⚠ THE READ PREDICATE STILL HAS ONE OWNER. `_granted_view_sources` obtains the registry once
490
+ through `grants_bulk` and delegates each held record to `shares.role_from`, the same pure
491
+ evaluator `may_see` uses. Named and dynamic-everyone grants therefore keep identical semantics
492
+ without one registry round trip per id.
493
  ⚠ Fail-closed: no grants is a NARROWER answer, never a wider one.
494
  """
495
+ return set(_granted_view_sources(session))
 
 
 
 
 
 
 
 
496
 
497
 
498
  #: ⭐⭐ W54-T48 / OWNER RULING R4 / CONTRACT C4 — ONE CONNECTION, ONE FOLDER IN THE RAIL.
 
1031
  try:
1032
  stored = (session.runtime.get(_NAV_PREFS_KEY) or {}).get(session.uname) or {}
1033
  except Exception:
1034
+ raise err(503, "store_unavailable", "Your saved preferences could not be loaded.")
1035
  keys, enumerated = _placeable_top_keys(session)
1036
  return {"prefs": _clean_nav_prefs(stored, keys, keep_unknown_ut=not enumerated)}
1037
 
 
1039
  @router.post("/nav/prefs")
1040
  def save_nav_prefs(body: dict = Body(default=None),
1041
  session: Session = Depends(require_session)):
1042
+ """Replace the supplied navigation picture or patch personal linked-grid layouts."""
 
1043
  keys, enumerated = _placeable_top_keys(session)
1044
+ clean = {}
1045
  if not session.runtime.available():
1046
  raise err(503, "store_unavailable",
1047
  "the tenant store is unavailable. Nothing was saved.")
1048
 
1049
  def _up(data):
1050
+ nonlocal clean
1051
  data = data if isinstance(data, dict) else {}
1052
+ clean = _merge_nav_prefs(data.get(session.uname), body, keys,
1053
+ keep_unknown_ut=not enumerated)
1054
+ if clean["folders"] or clean["placement"] or clean.get("linkedLayouts"):
1055
  data[session.uname] = clean
1056
  else:
1057
  data.pop(session.uname, None)
 
1061
  session.runtime.update(_NAV_PREFS_KEY, _up)
1062
  except Exception:
1063
  raise err(503, "store_unavailable",
1064
+ "the preference change was not saved: the store refused the write.")
1065
  return {"prefs": clean}
1066
 
1067
 
api/routes_odoo_tables.py CHANGED
@@ -23,6 +23,7 @@ import hashlib
23
  import hmac
24
  import json
25
  import time
 
26
 
27
  from fastapi import Body, Depends, HTTPException
28
  from fastapi import APIRouter, Query
@@ -1516,6 +1517,189 @@ def _related_source_value(session, entry, record):
1516
  return None
1517
 
1518
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1519
  def _cache_relation_plan(session, source, via, target):
1520
  """The offered module link and its explicit bridge, or a cause-specific refusal.
1521
 
@@ -1538,9 +1722,8 @@ def _cache_relation_plan(session, source, via, target):
1538
  raise err(404, "relation_target_missing",
1539
  f"'{target}' is not a current module/cache relation target in this workspace")
1540
  if not _may_open(session, source):
1541
- raise err(403, "not_permitted",
1542
- f"'{source}' could not be opened for your account, so this relation cannot "
1543
- "be followed")
1544
 
1545
  offered = next((entry for entry in related_entries(session, source)
1546
  if str(entry.get("key") or "") == via
@@ -1557,7 +1740,7 @@ def _cache_relation_plan(session, source, via, target):
1557
  if not isinstance(link, dict) or str(link.get("table") or "") != target:
1558
  raise err(404, "relation_target_missing",
1559
  f"the relation '{via}' no longer has a current target in this workspace")
1560
- through = str(link.get("through") or "")
1561
  through_on = str(link.get("throughOn") or "")
1562
  through_to = str(link.get("throughTo") or "")
1563
  on = str(link.get("on") or relations.IDENTITY_COL)
@@ -1567,13 +1750,11 @@ def _cache_relation_plan(session, source, via, target):
1567
  if relations.is_retired_table(through):
1568
  raise err(404, "relation_target_retired",
1569
  f"'{through}' is retired, so the relation to '{target}' is no longer available")
1570
- if not _may_open(session, through):
1571
- raise err(403, "forbidden_table",
1572
- f"the relation to '{target}' crosses '{through}', which your account cannot "
1573
- "open")
1574
  if not _may_open(session, target):
1575
  raise err(403, "forbidden_table",
1576
- f"the relation ends at '{target}', which your account cannot open")
1577
  return dict(offered, __source__=source), link, through, through_on, through_to, on
1578
 
1579
 
@@ -1601,15 +1782,23 @@ def _cache_relation_source_value(session, entry, record):
1601
 
1602
 
1603
  def _cache_relation_target_ids(session, source, record, via, target):
1604
- """Resolve one explicit source -> mirror bridge -> module-cache path to target row pids."""
 
 
 
 
 
1605
  entry, _link, through, through_on, through_to, on = _cache_relation_plan(
1606
- session, source, via, target)
1607
  source_value = _cache_relation_source_value(session, entry, record)
1608
  if source_value in (None, ""):
1609
  raise err(400, "unknown_relation",
1610
- f"record {record} has no readable source value for relation '{via}'")
1611
 
1612
- compiled = _compiled_read(through, session)
 
 
 
1613
  spec = compiled["spec"]
1614
  left = (spec.get("cols") or {}).get(through_on)
1615
  right = (spec.get("cols") or {}).get(through_to)
@@ -1658,7 +1847,11 @@ def odoo_relation_targets(source: str = Query(...), record: int = Query(..., ge=
1658
  via: str = Query(...), target: str = Query(...),
1659
  session: Session = Depends(require_session)):
1660
  """Target pids for a declared relation whose destination is a runtime-cache database."""
1661
- return _cache_relation_target_ids(session, source, record, via, target)
 
 
 
 
1662
 
1663
 
1664
  def _via_path(via):
@@ -1721,308 +1914,24 @@ def _related_path_legs(session, source, entry, entries):
1721
  return dict(first, __source__=source), mid_key, dict(last, __source__=mid_key)
1722
 
1723
 
1724
- def _related_hop_values(session, source, record, first, mid_key, last, notes):
1725
- """The values hop 2 joins on, resolved from the INTERMEDIATE database's own rows.
1726
-
1727
- ⭐⭐ W54-T40 — THIS IS THE HALF THAT MAKES A TWO-HOP TAB ANSWER ROWS INSTEAD OF A SENTENCE.
1728
- Hop 1 turns the source record into a set of intermediate rows; this returns the value each of
1729
- those rows carries in the column hop 2 is keyed on, and the caller pins THIS database by them.
1730
-
1731
- ⛔⛔ THE INTERMEDIATE IS READ THROUGH `_compiled_read`, WHICH IS THE SAME COMPILER THIS DOOR
1732
- USES FOR AN ORDINARY WINDOW, AND THAT IS THE WHOLE PERMISSION ARGUMENT. Its grant gate, its
1733
- administrator row wall and its hidden-field closure all run before a single id is produced, so
1734
- a traversal cannot be used to learn about rows of the middle database a reader may not see. A
1735
- second, lighter read here would be a second permission wall, and this file already carries the
1736
- rule that a second copy of that block does not drift towards refusing.
1737
- ⛔ AND AN INTERMEDIATE THIS DOOR CANNOT READ REFUSES BY NAME. `product_data` is a compiled
1738
- registry module with no read-through binding, so the one enumerated path that passes through it
1739
- answers a sentence carrying the cause and a way round it, rather than an empty grid.
1740
-
1741
- ⚠ NO CAP, AND THE COST IS REPORTED INSTEAD (standing rule 1). Every distinct value comes back,
1742
- however many there are, and the reader is told through `limits` that this tab resolves a second
1743
- database on every page.
1744
- """
1745
- import core.relations as relations
1746
- from fastapi import HTTPException # noqa: PLC0415
1747
-
1748
- value = _related_source_value(session, first, record)
1749
- if value in (None, ""):
1750
- raise err(400, "unknown_relation",
1751
- relations.unresolvable_source_refusal(source, first))
1752
- try:
1753
- mid = _compiled_read(mid_key, session)
1754
- except HTTPException as exc:
1755
- why = ""
1756
- if isinstance(getattr(exc, "detail", None), dict):
1757
- why = str((exc.detail.get("error") or {}).get("message") or "")
1758
- raise err(409, "relation_not_served",
1759
- f"this relation reaches '{last.get('target')}' through '{mid_key}', and that "
1760
- f"database could not be read here to follow it"
1761
- + (f" ({why})" if why else "")
1762
- + f". Open '{mid_key}' on its own, find the records this one is linked to, and "
1763
- f"follow the next relation from there")
1764
- mid_spec = mid["spec"]
1765
- on1 = str((first.get("via") or {}).get("on") or "")
1766
- step = (mid_spec.get("cols") or {}).get(on1)
1767
- if not step:
1768
- raise err(409, "relation_not_served",
1769
- f"'{mid_key}' has no live expression for '{on1}', which is the column the first "
1770
- f"hop of this relation joins on, so the traversal cannot be resolved. Open "
1771
- f"'{mid_key}' on its own, or ask for '{on1}' to be added to its live connection")
1772
- # ⭐ THE COLUMN HOP 2 IS KEYED ON, READ OFF THE INTERMEDIATE. `pid` is the row IDENTITY and is
1773
- # not a column of the mirror spec, so it resolves to the spec's own identity expression, which
1774
- # is the same one this route trusts everywhere else and is per table rather than a bare `id`.
1775
- frm2 = str((last.get("via") or {}).get("from") or relations.IDENTITY_COL)
1776
- take = (mid_spec["id"] if frm2 == relations.IDENTITY_COL
1777
- else ((mid_spec.get("cols") or {}).get(frm2) or (None,))[0])
1778
- if not take:
1779
- raise err(409, "relation_not_served",
1780
- f"'{mid_key}' has no live expression for '{frm2}', which is the value the second "
1781
- f"hop of this relation is keyed on, so the traversal cannot be resolved. Open "
1782
- f"'{mid_key}' on its own, or ask for '{frm2}' to be added to its live connection")
1783
- try:
1784
- value = step[1](value)
1785
- except Exception: # noqa: BLE001
1786
- value = str(value)
1787
- mid_from = mid_spec["from_sql"] if mid_spec.get("from_sql") else mid_spec["table"]
1788
- parts = [p for p in (mid["base_where"], first.get("where")) if p]
1789
- parts.append(f"{step[0]} = ?")
1790
- pred = " AND ".join(f"({p})" for p in parts)
1791
- rows = mid["cur"].execute(
1792
- f"SELECT DISTINCT {take} FROM {mid_from} WHERE {pred}",
1793
- list(mid["wall_params"]) + [value]).fetchall()
1794
- values = [r[0] for r in rows if r and r[0] is not None]
1795
- # ⭐⭐ STANDING RULE 1'S SECOND SENTENCE, AND `effect` IS CHOSEN SO THE READER GETS **THIS**
1796
- # SENTENCE RATHER THAN A NEIGHBOUR'S. `counts.ts::plainLimit` maps a known `effect` to one
1797
- # fixed client phrase and falls through to the SERVER's own first sentence for an effect it has
1798
- # not learned; `slow` maps to *"Rows this far down load slowly, each page is reached by walking
1799
- # every row above it"*, which is a true statement about a deep page and a FALSE one about a
1800
- # seven-row traversal. `traversal` is deliberately not in that table, so the strip prints the
1801
- # cause below, which is what actually happened.
1802
- # ⚠ THE FIRST CLAUSE IS THE ONE THAT REACHES THE STRIP: `firstSentence` cuts at the first full
1803
- # stop and truncates past 120 characters, so the mechanism is stated before the count.
1804
- notes.append({
1805
- "subject": "related", "effect": "traversal",
1806
- "cause": f"these rows are reached through '{mid_key}', so every page resolves that "
1807
- f"database first. This record matched {len(values):,} of its records, and this "
1808
- f"database is then narrowed to the rows those reach",
1809
- "recommendation": f"open '{mid_key}' on its own if you are paging through a large set, or "
1810
- f"filter this database beside the relation to ask a smaller question"})
1811
- return values
1812
 
1813
 
1814
  def _related_pin(table_key, session, spec, base_where, wall_params, cur, related):
1815
- """`(sql, params, notes)` for C6's pinned read, or `(None, [], [])` when nothing is pinned.
1816
-
1817
- ⛔⛔ WHY THIS DOES NOT CALL `compile_filter_tree(member_ids=)`, WHICH IS WHAT C6'S SENTENCE
1818
- SAYS. Measured against `platform/harness/filter_sql.py` on 2026-09-03, that call cannot express
1819
- a pinned read and the failure is silent in the widening direction, twice over:
1820
-
1821
- 1. `:583` returns None the moment the tree is INACTIVE, and the `member_ids` branch is BELOW
1822
- that return. A pin with no client filter is exactly the inactive case, so the ids would be
1823
- dropped and the whole table served.
1824
- 2. `:588` emits `(tree OR id_sql IN (...))`. Its own docstring: *"`member_ids` reproduces
1825
- `members.has(r.pid) || matchFilterTree(...)`"*. That is a UNION. ORing a pin onto a
1826
- filter WIDENS; a pinned read needs an INTERSECTION.
1827
-
1828
- So the tail is compiled by `core.relations.pin_sql`, in `filter_sql`'s OWN spelling (proven
1829
- byte for byte by `verify_pivot.py`, not by this comment), and ANDed in. `filter_sql.py` is lane
1830
- B's file this wave; a narrowing member mode there is an AMENDMENT and is booked in the handoff,
1831
- not edited in from here.
1832
-
1833
- ⭐ `id_sql` IS PASSED EXPLICITLY, AS `spec["id"]`, AND IS NEVER DEFAULTED. `harness/semantic.py`
1834
- defaults it to `f"{alias}.id"` inside its OWN compile calls; a call from the route layer never
1835
- reaches that line, and `filter_sql:586` RAISES without it. It matters that it is per table
1836
- rather than a literal `id`: `ut_odoo_order_lines` is join-shaped and its identity is `sol.id`,
1837
- so a bare `id` would be ambiguous across `sale_order_line` and `sale_order`. It is the same
1838
- expression this route already trusts as `tiebreak_sql` and as the fallback ORDER BY.
1839
-
1840
- ⭐⭐ W54-T40 / INSTRUCTION 22 / R12 — **A `via` MAY NOW BE A PATH, AND THE PATH IS SERVED.**
1841
- The owner's second quoted error was this function's own `bad_relation` sentence, and it fired
1842
- on 18 of the 25 rail tabs of `customer_data`: R12 says the client pins a traversal by naming
1843
- each edge, so a two-hop tab sends `via` as the ARRAY `["invoices","order_link"]`, and the
1844
- guard below accepted only the one-hop bag. MEASURED 2026-09-06, 6 of 25 tabs answered 200 and
1845
- 18 answered 400 on the first click.
1846
-
1847
- ⛔⛔ ACCEPTING THE ARRAY IS HALF A FIX AND WOULD HAVE MOVED THE ERROR RATHER THAN CLOSED IT.
1848
- `relations.pick` matches `via.direction` against `DIRECTIONS`, and a two-hop entry's direction
1849
- is the literal `path`, deliberately outside that tuple so nothing serves half a traversal. So
1850
- a path resolves through `relations.pick_path` against the DEPTH-AWARE listing, and the read is
1851
- then answered in TWO steps: hop 1 resolves the intermediate database's own rows to the values
1852
- hop 2 joins on, and hop 2 pins this database by them. Sending the object form alone answered
1853
- `400 unknown_relation` and changed nothing the reader could see.
1854
-
1855
- ⭐ EVERY HOP IS STILL PERMISSION CHECKED, AND AT PIN TIME, NOT ONLY AT LISTING TIME (R13). The
1856
- intermediate is resolved through `_compiled_read`, i.e. THE SAME compiler this door uses for
1857
- an ordinary read, so its grant, its row wall and its hidden-field closure all apply before a
1858
- single id crosses into this table's predicate. A second, laxer read of the intermediate would
1859
- be exactly the permission wall with a hole in it that `related_entries` exists to prevent.
1860
- ⛔ AND AN INTERMEDIATE THIS DOOR CANNOT READ IS A **REPORTED LIMIT**, never a silent empty:
1861
- `product_data` is a registry module with no read-through binding, so the one enumerated path
1862
- that passes through it refuses by name with a cause and a recommendation.
1863
- """
1864
  import core.relations as relations
1865
 
1866
  if isinstance(related, str):
1867
  related = _json_arg(related, "related")
1868
  if related in (None, "", {}):
1869
  return None, [], []
1870
- if not isinstance(related, dict):
1871
- raise err(400, "bad_relation",
1872
- "`related` must be an object naming the record to pin these rows to: "
1873
- "{table, record, via}")
1874
-
1875
- source = str(related.get("table") or "")
1876
- record = related.get("record")
1877
- via = related.get("via")
1878
- # ⭐⭐ W54-T40 — `via` IS EITHER ONE EDGE'S BAG OR A PATH OF EDGE KEYS, and `_via_path` is the
1879
- # one place that decides which. It answers None for the bag, which is what keeps the guard
1880
- # below spelling the SAME refusal for a genuinely incomplete pin.
1881
- path = _via_path(via)
1882
- if not source or record in (None, "") or (path is None and not isinstance(via, dict)):
1883
- raise err(400, "bad_relation",
1884
- "a pinned read needs the record's database, the record id and the relation to "
1885
- "follow, as {table, record, via}, where `via` is one relation or the array of "
1886
- "edge keys naming each hop. Ask GET /api/v1/grid/" + (source or "{table}")
1887
- + "/related for the relations this account may follow")
1888
-
1889
- from routes_grid import _may_open, related_entries, related_path_entries
1890
-
1891
- if not _may_open(session, source):
1892
- # ⚠ THE CAUSE IS NAMED AS A PAIR, ON PURPOSE. `_may_open` is fail-closed and answers False
1893
- # both for a genuine permission refusal and for a database it could not read at all, so a
1894
- # sentence claiming only the first would tell a reader their access was revoked when the
1895
- # store had merely hiccupped. Naming both is honest and still refuses.
1896
- raise err(403, "not_permitted",
1897
- f"'{source}' could not be opened for your account, either because it is not "
1898
- f"granted to you or because it could not be read just now, so rows here cannot "
1899
- f"be pinned to one of its records. Ask an administrator for access to that "
1900
- f"database, or try again")
1901
-
1902
- # ⛔ RESOLVED AGAINST THE **SCOPED** LISTING, which is what makes "an edge you cannot be
1903
- # offered is an edge you cannot pin" structural rather than remembered. `related_entries` is
1904
- # the same function `GET /grid/{table}/related` answers with; there is no second resolution.
1905
- # ⭐ W54-T40 — AND A PATH RESOLVES AGAINST THE DEPTH-AWARE LISTING, WHICH IS THE SAME RULE ONE
1906
- # LEVEL DEEPER: `related_path_entries` is what `GET /grid/{table}/related?depth=2` answers
1907
- # with, so a traversal this account is not offered is a traversal it cannot pin either.
1908
- # ⚠ THE DEPTH ASKED FOR IS THE LENGTH OF THE PATH, never a flat 2. A one-element path costs
1909
- # the depth-1 enumeration it needs and not the depth-2 one, which on `customer_data` is the
1910
- # difference between roughly half a second and roughly two.
1911
- if path is None:
1912
- entries = related_entries(session, source)
1913
- entry = relations.pick(entries, via)
1914
- else:
1915
- if len(path) > relations.MAX_DEPTH:
1916
- raise err(400, "over_depth", relations.over_depth_refusal(source, path))
1917
- entries = related_path_entries(session, source, len(path))
1918
- entry = relations.pick_path(entries, path)
1919
- if entry is None:
1920
- raise err(400, "unknown_relation", relations.unknown_via_refusal(source, via))
1921
- entry = dict(entry, __source__=source)
1922
-
1923
- if str(entry.get("target") or "") != str(table_key):
1924
- raise err(400, "unknown_relation",
1925
- f"the relation '{entry.get('key')}' on '{source}' points at "
1926
- f"'{entry.get('target')}', not at '{table_key}', so it cannot narrow this "
1927
- f"database. Read the database the relation names, or ask for the relation that "
1928
- f"reaches this one")
1929
- if not entry.get("served", True):
1930
- raise err(409, "relation_not_served", relations.unserved_target_refusal(entry))
1931
-
1932
- # ⭐⭐ W54-T40 — THE **LAST** HOP IS WHAT JOINS THIS DATABASE, and on a two-hop entry that is
1933
- # not `entry`. A composite carries `via = {direction: 'path', key, path}` and NO `on`, no
1934
- # `from` and no `where`: `relations._two_hop_entry` refuses to flatten the two hops' predicates
1935
- # onto the pair, because one of them may carry a predicate the other does not (the open-invoice
1936
- # bridge is exactly that edge). So the legs are resolved, each from its OWN table's scoped
1937
- # listing, and the last one is what this block below reads.
1938
  notes = []
1939
- first, mid_key, last = entry, "", entry
1940
- if len(entry.get("path") or ()) > 1:
1941
- first, mid_key, last = _related_path_legs(session, source, entry, entries)
1942
-
1943
- on = str((last.get("via") or {}).get("on") or "")
1944
- # ⚠ `spec["cols"]`, NOT the caller's narrowed `cols`. The join column is a JOIN, not a value
1945
- # this response returns: a relation must keep working when an administrator has hidden the id
1946
- # column it runs over, and no cell of it is ever projected. The hidden-field closure still
1947
- # governs everything that reaches the wire, one layer up.
1948
- # ⭐⭐ W54-T40 — AND THE ROW IDENTITY IS NOT ONE OF THOSE COLUMNS. `relations` defaults an edge
1949
- # with no declared `on` to `pid`, which is the wire's name for the row identity and is never a
1950
- # key of a mirror spec's `cols` (measured across all eight bound grains on 2026-09-06: not one
1951
- # carries it). Every such edge therefore refused with "has no live expression for 'pid'" while
1952
- # the expression was in hand the whole time, as `spec["id"]` - the same per-table identity this
1953
- # route already uses for the pin itself, for the tiebreak and for the fallback ORDER BY, and
1954
- # deliberately not a bare `id`, which is ambiguous on a join-shaped grain.
1955
- if on == relations.IDENTITY_COL:
1956
- binding = (spec["id"], int)
1957
- else:
1958
- binding = (spec.get("cols") or {}).get(on)
1959
- if not binding:
1960
- raise err(409, "relation_not_served",
1961
- f"'{table_key}' has no live expression for '{on}', which is the column this "
1962
- f"relation joins on, so the rows cannot be narrowed to one record here. Open "
1963
- f"this database directly, or ask for '{on}' to be added to its live connection")
1964
- on_expr, on_cast = binding[0], binding[1]
1965
-
1966
- if mid_key:
1967
- # ⭐ HOP 1, ANSWERED BY THE INTERMEDIATE DATABASE ITSELF, UNDER ITS OWN WALL. See
1968
- # `_related_hop_values`. An empty answer is an ANSWER (this record reaches nothing through
1969
- # that database) and compiles to `1=0` two blocks down, never to "no pin".
1970
- values = _related_hop_values(session, source, record, first, mid_key, last, notes)
1971
- else:
1972
- value = _related_source_value(session, entry, record)
1973
- if value in (None, ""):
1974
- raise err(400, "unknown_relation",
1975
- relations.unresolvable_source_refusal(source, entry))
1976
- values = [value]
1977
- cast = []
1978
- for one in values:
1979
- try:
1980
- cast.append(on_cast(one))
1981
- except Exception: # noqa: BLE001
1982
- cast.append(str(one))
1983
-
1984
- # ── the member ids, in ONE SQL over the bridge column (the ticket's `how:`) ────────────────
1985
- # ⛔ NOT THROUGH `datastore.window`: that clamps at `WINDOW_MAX` (5,000) rows, so a customer
1986
- # with more order lines than that would be pinned to the first 5,000 of them and the tab would
1987
- # show a smaller, entirely plausible number with nothing red anywhere. Standing rule 1: no cap
1988
- # on data from a connected source.
1989
- # ⚠ THE WALL RIDES ALONG. Resolving ids under `base_where` means the id SET a scoped reader can
1990
- # produce is already narrowed to rows they may see, so the pin cannot be used to learn how many
1991
- # rows exist beyond their boundary.
1992
- # ⭐ W54-T40 — `= ?` FOR ONE VALUE AND `IN (...)` FOR MANY, and the one-value spelling is kept
1993
- # BYTE FOR BYTE so a one-hop pin compiles exactly the SQL it compiled before this ticket. The
1994
- # many-value arm is only ever reached by the second hop of a traversal.
1995
- frm_body = spec["from_sql"] if spec.get("from_sql") else spec["table"]
1996
- ids = []
1997
- if cast:
1998
- parts = [p for p in (base_where, last.get("where")) if p]
1999
- parts.append(f"{on_expr} = ?" if len(cast) == 1
2000
- else f"{on_expr} IN ({','.join('?' for _ in cast)})")
2001
- pred = " AND ".join(f"({p})" for p in parts)
2002
- ids = [int(r[0]) for r in cur.execute(
2003
- f'SELECT DISTINCT {spec["id"]} FROM {frm_body} WHERE {pred}',
2004
- list(wall_params) + list(cast)).fetchall() if r and r[0] is not None]
2005
-
2006
- # ⭐ STANDING RULE 1'S SECOND SENTENCE, ON A COST RATHER THAN ON A CAP. Nothing is limited
2007
- # here; the reader is told that this particular relation is answered by reading the source
2008
- # database once per page, because it is keyed on a column rather than on the record id.
2009
- if str((first.get("via") or {}).get("from") or "") != relations.IDENTITY_COL:
2010
  notes.append({
2011
  "subject": "related", "effect": "slow",
2012
- "cause": f"this relation is keyed on the '{first['via']['from']}' column of "
2013
- f"'{source}' rather than on its record id, so every page of these rows is "
2014
- f"resolved by reading that database first",
2015
- "recommendation": "read this database on its own and filter it on "
2016
- f"'{on}' if you are paging through a large set"})
2017
- if len(ids) >= _PIN_REPORT_AT:
2018
- notes.append({
2019
- "subject": "related", "effect": "slow",
2020
- "cause": f"this record is linked to {len(ids):,} rows, and each page names every one "
2021
- f"of them in the query that narrows this database to them",
2022
- "recommendation": "add a filter beside the relation to ask a smaller question, or "
2023
- "read this database on its own"})
2024
-
2025
- sql, params = relations.pin_sql(spec["id"], ids)
2026
  return sql, params, notes
2027
 
2028
 
@@ -2300,7 +2209,7 @@ def _may_see_view(session, vid, store):
2300
 
2301
 
2302
  def _compiled_read(table_key, session, *, filters=None, filterConj="and", sorts=None,
2303
- search=None, tree=_TREE_UNSET, related=None):
2304
  """EVERYTHING THAT DECIDES **WHICH ROWS** A CONNECTED GRID MAY ANSWER OVER, compiled once.
2305
 
2306
  Returns `{defn, fields, cur, spec, cols, base_where, wall_params, where, params, limits,
@@ -2360,7 +2269,10 @@ def _compiled_read(table_key, session, *, filters=None, filterConj="and", sorts=
2360
  # tenant #0's eight databases is read-through. Left un-applied, an administrator could set a
2361
  # filter in Manage user, see it saved, and the scoped user would meet a 409 where their rows
2362
  # used to be. So the wall is APPLIED here, in SQL, three paragraphs down.
2363
- defn = _defn_or_refuse(session, table_key, defs_only=True, scope_applied=True)
 
 
 
2364
  fields = None
2365
  for _bucket, key, _label, mk_fields in rel.TABLES:
2366
  if key == table_key:
@@ -2615,6 +2527,87 @@ def _compiled_read(table_key, session, *, filters=None, filterConj="and", sorts=
2615
  "pred": pred}
2616
 
2617
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2618
  @router.get("/odoo-tables/{table_key}/rows")
2619
  def odoo_table_rows(table_key: str,
2620
  offset: int = Query(default=0, ge=0),
@@ -2652,6 +2645,10 @@ def odoo_table_rows(table_key: str,
2652
  import core.perm_scope as perm_scope
2653
  from harness import datastore
2654
 
 
 
 
 
2655
  # ⛔⛔ ONE COMPILER, AND THE AGGREGATES DOOR CALLS THE SAME ONE. The permission wall, the
2656
  # hidden-field closure, the user predicate and the search all live in `_compiled_read` above,
2657
  # because C1 promises that `/aggregates` answers over *the same compiled predicate this window
@@ -3041,6 +3038,13 @@ def odoo_table_aggregates(table_key: str, body: dict = Body(default=None),
3041
  if group_by is not None and not isinstance(group_by, str):
3042
  raise err(400, "bad_selection", "groupBy must be a column id")
3043
 
 
 
 
 
 
 
 
3044
  # ⛔⛔ THE PIN RIDES THE **SAME** COMPILER, AND THIS ONE LINE IS THE WHOLE OF CORRECTION (3).
3045
  # `/rows` takes `related` as a JSON query STRING and this door takes it as a BODY KEY; a worker
3046
  # who assumed both were bodies would have wired the rows and left the summaries answering over
 
23
  import hmac
24
  import json
25
  import time
26
+ from contextlib import contextmanager
27
 
28
  from fastapi import Body, Depends, HTTPException
29
  from fastapi import APIRouter, Query
 
1517
  return None
1518
 
1519
 
1520
+ def _materialised_table(table_key):
1521
+ """Storage dispatch, independent of the browser's fallible status roster."""
1522
+ return module_cache_source(table_key) is not None or table_key not in _sources()
1523
+
1524
+
1525
+ def _traversal_defn_or_refuse(session, table_key):
1526
+ """R50 / C4: admit an intermediate after the caller has established both end grants.
1527
+
1528
+ The session may already open both ends of a declared two-hop link. Its intermediate
1529
+ is read only to find target ids, and only join values leave this traversal. The compiled
1530
+ read still applies the row wall, hidden-field closure, BU scope and mirror guard.
1531
+ """
1532
+ ut = _ut()
1533
+ st = ut.lend_defs(session.runtime)
1534
+ defn = ut.get(table_key, st=st)
1535
+ if not defn:
1536
+ raise err(404, "unknown_table", "that database does not exist")
1537
+ return defn
1538
+
1539
+
1540
+ def _relation_rows(session, table, columns, *, ids=None, match=None, where=None,
1541
+ traversal_of=None):
1542
+ """Internal join values, after the ordinary row wall, with no row-count ceiling.
1543
+
1544
+ Join columns may be hidden on the wire. Only identities leave the resolver; the native
1545
+ payload builders still apply their complete hidden-field closure before returning cells.
1546
+ With traversal_of, the caller must already have established both ends of the declared
1547
+ edge named by that key. Only join values leave this intermediate read, never a cell.
1548
+ """
1549
+ import core.perm_scope as perm_scope
1550
+ import core.relations as relations
1551
+
1552
+ columns = set(columns) | {relations.IDENTITY_COL}
1553
+ if not _materialised_table(table):
1554
+ read = (_compiled_read(table, session, traversal_intermediate=True)
1555
+ if traversal_of is not None else _compiled_read(table, session))
1556
+ spec = read["spec"]
1557
+ bindings = dict(spec["cols"], pid=(spec["id"], int))
1558
+ if columns - set(bindings):
1559
+ raise err(409, "relation_target_unavailable", "A required relation column is unavailable.")
1560
+ parts = [p for p in (read["where"], where) if p]
1561
+ params = list(read["params"])
1562
+ for key, values in (("pid", ids), match or ("pid", None)):
1563
+ if values is None:
1564
+ continue
1565
+ expr, cast = bindings[key]
1566
+ values = list(values)
1567
+ parts.append(f"{expr} IN ({','.join('?' for _ in values)})" if values else "1=0")
1568
+ params.extend(cast(value) for value in values)
1569
+ names = sorted(columns)
1570
+ projection = ", ".join(bindings[key][0] for key in names)
1571
+ source = spec.get("from_sql") or spec["table"]
1572
+ predicate = " AND ".join(f"({part})" for part in parts) or "1=1"
1573
+ return [dict(zip(names, row)) for row in read["cur"].execute(
1574
+ f"SELECT {projection} FROM {source} WHERE {predicate}", params).fetchall()]
1575
+
1576
+ # This is an internal join, never a cell response. Apply the same permanent wall on the
1577
+ # complete contract before taking any join value, including a join on a hidden column.
1578
+ if traversal_of is None and not perm_scope.may_read(session.user, table, st=session.runtime):
1579
+ raise err(403, "forbidden_table", "This account cannot open the linked database.")
1580
+ if module_cache_source(table) is not None:
1581
+ session.require(table)
1582
+ fields, rows = perm_scope._read(table, session.user, session.runtime)
1583
+ rows = perm_scope.apply_row_scope(rows, session.user, table, fields, st=session.runtime)
1584
+ if where:
1585
+ raise err(409, "relation_target_unavailable", "This relation needs a connected database predicate.")
1586
+ if ids is not None:
1587
+ wanted = {str(value) for value in ids}
1588
+ rows = [row for row in rows if str(row.get("pid")) in wanted]
1589
+ if match is not None:
1590
+ key, values = match
1591
+ wanted = {str(value) for value in values}
1592
+ rows = [row for row in rows if str(row.get(key)) in wanted]
1593
+ return [{key: row.get(key) for key in columns} for row in rows]
1594
+
1595
+
1596
+ def _relation_cell_ids(value):
1597
+ """Stored link cells contain comma-separated identities, not a scalar join value."""
1598
+ parts = value if isinstance(value, (list, tuple, set)) else str(value or "").split(",")
1599
+ return {str(int(part)) for part in parts if str(part).strip().isdigit() and int(part) > 0}
1600
+
1601
+
1602
+ def _relation_leg_ids(session, source, source_ids, entry):
1603
+ """Follow one declared leg, including manual, inverse and through links."""
1604
+ import core.relations as relations
1605
+
1606
+ target = str(entry["target"])
1607
+ via = entry.get("via") or {}
1608
+ link = entry.get("link") or {}
1609
+ reverse = via.get("direction") == "reverse"
1610
+ frm, on = str(via.get("from") or "pid"), str(via.get("on") or "pid")
1611
+ field = str(via.get("field") or "")
1612
+ manual = bool(field and link and not link.get("inverse")
1613
+ and (link.get("selectionMode") == "manual" or not link.get("on")))
1614
+ inverse = str(link.get("inverse") or "")
1615
+ source_col, target_col = frm, on
1616
+ if manual:
1617
+ source_col, target_col = ("pid", field) if reverse else (field, "pid")
1618
+ elif inverse:
1619
+ source_col, target_col = (inverse, "pid") if reverse else ("pid", inverse)
1620
+ source_rows = _relation_rows(session, source, [source_col], ids=source_ids)
1621
+ values = {str(row[source_col]) for row in source_rows if row.get(source_col) not in (None, "")}
1622
+ # A column-keyed edge cannot fall back to the wire record id. In particular,
1623
+ # product_data.pid is a stable hash while sale_order_line.product_id is an Odoo
1624
+ # identity. Treating a missing/hidden/blank source value as an empty match would
1625
+ # turn "the join input is unavailable" into the business claim "no related rows".
1626
+ if source_col != relations.IDENTITY_COL and not values:
1627
+ raise err(400, "unknown_relation", relations.unresolvable_source_refusal(source, entry))
1628
+ if (manual and not reverse) or (inverse and reverse):
1629
+ values = set().union(*(_relation_cell_ids(row.get(source_col)) for row in source_rows))
1630
+ target_col = "pid"
1631
+ through = relations.declared_through(link) if not manual else None
1632
+ if through:
1633
+ left = str(link["throughTo"] if reverse else link["throughOn"])
1634
+ right = str(link["throughOn"] if reverse else link["throughTo"])
1635
+ middle = _relation_rows(session, str(through), [left, right], match=(left, values),
1636
+ traversal_of=str(entry.get("key") or via.get("key") or ""))
1637
+ values = {str(row[right]) for row in middle if row.get(right) not in (None, "")}
1638
+ if (manual and reverse) or (inverse and not reverse):
1639
+ rows = _relation_rows(session, target, [target_col])
1640
+ rows = [row for row in rows if _relation_cell_ids(row.get(target_col)) & values]
1641
+ else:
1642
+ rows = _relation_rows(session, target, [target_col], match=(target_col, values),
1643
+ where=entry.get("where"))
1644
+ return {int(row["pid"]) for row in rows}
1645
+
1646
+
1647
+ def _relation_label(key):
1648
+ """Prefer the database's display label in relation refusals."""
1649
+ source = module_cache_source(key)
1650
+ if source and source.get("label"):
1651
+ return source["label"]
1652
+ return next((label for _bucket, table, label, _fields in _rel().TABLES
1653
+ if table == key and label), str(key))
1654
+
1655
+
1656
+ def _relation_target_ids(session, target, related):
1657
+ """One pin resolver for SQL, pool and tenant rows, selected from the scoped path list."""
1658
+ import core.relations as relations
1659
+ from routes_grid import _may_open, related_path_entries
1660
+
1661
+ if isinstance(related, str):
1662
+ related = _json_arg(related, "related")
1663
+ if not isinstance(related, dict) or not related.get("table") or related.get("record") in (None, ""):
1664
+ raise err(400, "bad_relation", "A pinned read needs {table, record, via}.")
1665
+ source = str(related["table"])
1666
+ via = related.get("via")
1667
+ path = _via_path(via)
1668
+ if path and len(path) > relations.MAX_DEPTH:
1669
+ raise err(400, "over_depth", relations.over_depth_refusal(source, path))
1670
+ if not _may_open(session, source):
1671
+ raise err(403, "forbidden_table",
1672
+ f"the relation starts at '{_relation_label(source)}', which your account cannot open")
1673
+ entries = related_path_entries(session, source, len(path) if path else 1)
1674
+ entry = relations.pick_path(entries, path) if path else relations.pick(entries, via)
1675
+ if entry is None or str(entry.get("target")) != str(target):
1676
+ raise err(400, "unknown_relation", relations.unknown_via_refusal(source, via))
1677
+ try:
1678
+ ids = {int(related["record"])}
1679
+ except (TypeError, ValueError):
1680
+ raise err(400, "bad_relation", "The source record must have an integer identity.")
1681
+ legs = [(source, entry)]
1682
+ if len(entry.get("path") or ()) > 1:
1683
+ first, middle, last = _related_path_legs(session, source, entry, entries)
1684
+ legs = [(source, first), (middle, last)]
1685
+ for table, leg in legs:
1686
+ if not leg.get("served"):
1687
+ raise err(409, "relation_not_served", relations.unserved_target_refusal(leg))
1688
+ if not _may_open(session, leg["target"]):
1689
+ raise err(403, "forbidden_table",
1690
+ f"the relation ends at '{_relation_label(leg['target'])}', which your account cannot open")
1691
+ ids = _relation_leg_ids(session, table, ids, leg)
1692
+ return ids
1693
+
1694
+
1695
+ def narrow_related_rows(session, table_key, rows, related):
1696
+ """Intersect an already walled and stripped payload with one resolved identity set."""
1697
+ if related in (None, "", {}):
1698
+ return rows
1699
+ ids = _relation_target_ids(session, table_key, related)
1700
+ return [row for row in rows if int(row["pid"]) in ids]
1701
+
1702
+
1703
  def _cache_relation_plan(session, source, via, target):
1704
  """The offered module link and its explicit bridge, or a cause-specific refusal.
1705
 
 
1722
  raise err(404, "relation_target_missing",
1723
  f"'{target}' is not a current module/cache relation target in this workspace")
1724
  if not _may_open(session, source):
1725
+ raise err(403, "forbidden_table",
1726
+ f"the relation starts at '{_relation_label(source)}', which your account cannot open")
 
1727
 
1728
  offered = next((entry for entry in related_entries(session, source)
1729
  if str(entry.get("key") or "") == via
 
1740
  if not isinstance(link, dict) or str(link.get("table") or "") != target:
1741
  raise err(404, "relation_target_missing",
1742
  f"the relation '{via}' no longer has a current target in this workspace")
1743
+ through = relations.declared_through(link)
1744
  through_on = str(link.get("throughOn") or "")
1745
  through_to = str(link.get("throughTo") or "")
1746
  on = str(link.get("on") or relations.IDENTITY_COL)
 
1750
  if relations.is_retired_table(through):
1751
  raise err(404, "relation_target_retired",
1752
  f"'{through}' is retired, so the relation to '{target}' is no longer available")
1753
+ # R50: both end grants suffice. Traversal admission in _compiled_read replaces the
1754
+ # intermediate grant check and still applies the intermediate's row wall.
 
 
1755
  if not _may_open(session, target):
1756
  raise err(403, "forbidden_table",
1757
+ f"the relation ends at '{_relation_label(target)}', which your account cannot open")
1758
  return dict(offered, __source__=source), link, through, through_on, through_to, on
1759
 
1760
 
 
1782
 
1783
 
1784
  def _cache_relation_target_ids(session, source, record, via, target):
1785
+ """Resolve any offered path; retain the legacy Products ordered string transport."""
1786
+ via_key = str(via.get("key") or "") if isinstance(via, dict) else str(via or "")
1787
+ if not (source == "customer_data" and target == "product_data"
1788
+ and via_key == "products_ordered"):
1789
+ ids = _relation_target_ids(session, target, {"table": source, "record": record, "via": via})
1790
+ return {"ok": True, "target": str(target), "ids": sorted(ids)}
1791
  entry, _link, through, through_on, through_to, on = _cache_relation_plan(
1792
+ session, source, via_key, target)
1793
  source_value = _cache_relation_source_value(session, entry, record)
1794
  if source_value in (None, ""):
1795
  raise err(400, "unknown_relation",
1796
+ f"record {record} has no readable source value for relation '{via_key}'")
1797
 
1798
+ # R50 / C4: the intermediate of a DECLARED two-hop link is read to find target ids
1799
+ # only, so it is admitted on both ends' grants. The compiler still applies its row
1800
+ # wall, hidden-field closure and mirror guard to it.
1801
+ compiled = _compiled_read(through, session, traversal_intermediate=True)
1802
  spec = compiled["spec"]
1803
  left = (spec.get("cols") or {}).get(through_on)
1804
  right = (spec.get("cols") or {}).get(through_to)
 
1847
  via: str = Query(...), target: str = Query(...),
1848
  session: Session = Depends(require_session)):
1849
  """Target pids for a declared relation whose destination is a runtime-cache database."""
1850
+ parsed_via = _json_arg(via, "via") if via.startswith(("[", "{")) else via
1851
+ answer = _cache_relation_target_ids(session, source, record, parsed_via, target)
1852
+ # Internal cache diagnostics (sourceKind/path) stay internal; the public door has one stable
1853
+ # response shape for generic, legacy-string and structured-bag transports.
1854
+ return {"ok": True, "target": str(target), "ids": list(answer.get("ids") or ())}
1855
 
1856
 
1857
  def _via_path(via):
 
1914
  return dict(first, __source__=source), mid_key, dict(last, __source__=mid_key)
1915
 
1916
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1917
 
1918
 
1919
  def _related_pin(table_key, session, spec, base_where, wall_params, cur, related):
1920
+ """Compile the same resolved identity set on both mirror transports, as an intersection."""
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1921
  import core.relations as relations
1922
 
1923
  if isinstance(related, str):
1924
  related = _json_arg(related, "related")
1925
  if related in (None, "", {}):
1926
  return None, [], []
1927
+ ids = _relation_target_ids(session, table_key, related)
1928
+ sql, params = relations.pin_sql(spec["id"], sorted(ids))
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1929
  notes = []
1930
+ if len(ids) > _PIN_REPORT_AT:
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1931
  notes.append({
1932
  "subject": "related", "effect": "slow",
1933
+ "cause": f"This relation matched {len(ids):,} records, all of which are included.",
1934
+ "recommendation": "Add a filter to reduce the result while continuing to page through every match."})
 
 
 
 
 
 
 
 
 
 
 
 
1935
  return sql, params, notes
1936
 
1937
 
 
2209
 
2210
 
2211
  def _compiled_read(table_key, session, *, filters=None, filterConj="and", sorts=None,
2212
+ search=None, tree=_TREE_UNSET, related=None, traversal_intermediate=False):
2213
  """EVERYTHING THAT DECIDES **WHICH ROWS** A CONNECTED GRID MAY ANSWER OVER, compiled once.
2214
 
2215
  Returns `{defn, fields, cur, spec, cols, base_where, wall_params, where, params, limits,
 
2269
  # tenant #0's eight databases is read-through. Left un-applied, an administrator could set a
2270
  # filter in Manage user, see it saved, and the scoped user would meet a 409 where their rows
2271
  # used to be. So the wall is APPLIED here, in SQL, three paragraphs down.
2272
+ if traversal_intermediate:
2273
+ defn = _traversal_defn_or_refuse(session, table_key)
2274
+ else:
2275
+ defn = _defn_or_refuse(session, table_key, defs_only=True, scope_applied=True)
2276
  fields = None
2277
  for _bucket, key, _label, mk_fields in rel.TABLES:
2278
  if key == table_key:
 
2527
  "pred": pred}
2528
 
2529
 
2530
+ @contextmanager
2531
+ def _materialised_read(table_key, session, query, related):
2532
+ """One compiler for native rows and their aggregate sibling, over the walled payload.
2533
+
2534
+ The temporary connection contains only this request's already stripped, pinned rows and is
2535
+ closed before returning. It never registers data on the tenant's shared mirror connection.
2536
+ """
2537
+ import duckdb
2538
+
2539
+ if table_key == "customer_data":
2540
+ from routes_customers import _payload
2541
+ session.require(table_key)
2542
+ payload = _payload(session, related=related)
2543
+ elif table_key == "product_data":
2544
+ from routes_products import products
2545
+ session.require(table_key)
2546
+ payload = products(session=session, related=related)
2547
+ else:
2548
+ from routes_tables import table_rows
2549
+ payload = table_rows(table_key, session=session, related=related)
2550
+ fields = payload["fields"]
2551
+ with duckdb.connect(":memory:") as cur:
2552
+ cur.execute("CREATE TABLE pinned_rows AS SELECT value AS payload FROM json_each(?)",
2553
+ [json.dumps(payload["rows"], default=str)])
2554
+ bindings = {}
2555
+ for field in fields:
2556
+ key = str(field["key"])
2557
+ pointer = ("/" + key.replace("~", "~0").replace("/", "~1")).replace("'", "''")
2558
+ expr = f"json_extract_string(payload, '{pointer}')"
2559
+ numeric = field.get("type") in ("int", "currency", "pct", "float", "number")
2560
+ bindings[key] = (f"TRY_CAST({expr} AS DOUBLE)" if numeric else expr,
2561
+ float if numeric else str)
2562
+ spec = {"table": "pinned_rows", "id": "CAST(json_extract_string(payload, '/pid') AS BIGINT)",
2563
+ "cols": bindings, "where": None}
2564
+ cols = _filter_columns(spec, fields)
2565
+ limits = list(payload.get("limits") or [])
2566
+ tree, cohorts = _views_as_cohorts(query["filters"], table_key, session, spec, cols,
2567
+ None, [], cur, limits)
2568
+ missing = _leaf_cols(tree) - _PSEUDO_COLS - set(cols)
2569
+ if missing:
2570
+ raise err(400, "filter_unsupported", "A filter names a column unavailable to this account.")
2571
+ try:
2572
+ pred = _fs().compile_filter_tree(tree, conj=query["filterConj"], columns=cols,
2573
+ id_sql=spec["id"], today=payload.get("today") or _today(),
2574
+ cohort_sets=cohorts)
2575
+ except ValueError as exc:
2576
+ raise err(400, "filter_unsupported", str(exc))
2577
+ if pred is not None and pred.uses_aggregate:
2578
+ raise err(400, "filter_unsupported", "This condition cannot be evaluated for individual rows.")
2579
+ where, params = (pred.sql, list(pred.params)) if pred is not None else (None, [])
2580
+ search = _fs().compile_search(query["search"], cols) if query["search"] else None
2581
+ if search is not None:
2582
+ where = f"({where}) AND {search.sql}" if where else search.sql
2583
+ params.extend(search.params)
2584
+ yield {"cur": cur, "spec": spec, "cols": cols, "where": where, "params": params,
2585
+ "frm": {"table": "pinned_rows"}, "limits": limits, "payload": payload}
2586
+
2587
+
2588
+ def _materialised_window(table_key, session, offset, limit, filters, filterConj, sorts, search, related):
2589
+ from harness import datastore
2590
+
2591
+ query = _selection_query({"filters": _json_arg(filters, "filters"), "filterConj": filterConj,
2592
+ "sorts": _json_arg(sorts, "sorts"), "search": search})
2593
+ with _materialised_read(table_key, session, query, related) as read:
2594
+ spec = read["spec"]
2595
+ order = _fs().compile_order_by(query["sorts"], read["cols"], tiebreak_sql=spec["id"])
2596
+ win = datastore.window(select=f'{spec["id"]} AS pid', where=read["where"],
2597
+ params=tuple(read["params"]), order_by=order or spec["id"],
2598
+ offset=offset, limit=limit or None, cur=read["cur"], **read["frm"])
2599
+ payload = read["payload"]
2600
+ by_id = {int(row["pid"]): row for row in payload["rows"]}
2601
+ rows = [by_id[int(row[0])] for row in win["rows"]]
2602
+ if win["clamped"]:
2603
+ read["limits"].append({"subject": "limit", "effect": "window_clamped",
2604
+ "cause": f"One response carries at most {datastore.WINDOW_MAX} rows to bound response memory.",
2605
+ "recommendation": "Continue paging with offset. Every matching row remains available."})
2606
+ return {**payload, "ok": True, "rows": rows, "total": win["total"],
2607
+ "totalUnfiltered": len(by_id), "offset": win["offset"], "limit": win["limit"],
2608
+ "limits": read["limits"], "scope": {"table": table_key, "readThrough": True}}
2609
+
2610
+
2611
  @router.get("/odoo-tables/{table_key}/rows")
2612
  def odoo_table_rows(table_key: str,
2613
  offset: int = Query(default=0, ge=0),
 
2645
  import core.perm_scope as perm_scope
2646
  from harness import datastore
2647
 
2648
+ if _materialised_table(table_key):
2649
+ return _materialised_window(table_key, session, offset, limit, filters, filterConj,
2650
+ sorts, search, related)
2651
+
2652
  # ⛔⛔ ONE COMPILER, AND THE AGGREGATES DOOR CALLS THE SAME ONE. The permission wall, the
2653
  # hidden-field closure, the user predicate and the search all live in `_compiled_read` above,
2654
  # because C1 promises that `/aggregates` answers over *the same compiled predicate this window
 
3038
  if group_by is not None and not isinstance(group_by, str):
3039
  raise err(400, "bad_selection", "groupBy must be a column id")
3040
 
3041
+ if _materialised_table(table_key):
3042
+ with _materialised_read(table_key, session, query, payload.get("related")) as read:
3043
+ out = _aggregate_over(read["cur"], read["spec"], read["cols"], read["where"],
3044
+ read["params"], read["frm"], aggs, group_by=group_by,
3045
+ limits=read["limits"])
3046
+ return {"ok": True, **out, "scope": {"table": table_key, "readThrough": True}}
3047
+
3048
  # ⛔⛔ THE PIN RIDES THE **SAME** COMPILER, AND THIS ONE LINE IS THE WHOLE OF CORRECTION (3).
3049
  # `/rows` takes `related` as a JSON query STRING and this door takes it as a BODY KEY; a worker
3050
  # who assumed both were bodies would have wired the rows and left the summaries answering over
api/routes_products.py CHANGED
@@ -498,6 +498,8 @@ def product_assembly(session: Session, scope: str = "product", storage_key: str
498
  str(pid): {key: value for key, value in (row or {}).items() if key not in hidden}
499
  for pid, row in (ws.get("overlays") or {}).items()
500
  }
 
 
501
 
502
  today = time.strftime("%Y-%m-%d")
503
  fields = aios_grid.project_date_recurrences(fields, today=today)
@@ -561,7 +563,7 @@ def product_assembly(session: Session, scope: str = "product", storage_key: str
561
 
562
 
563
  @router.get("/products")
564
- def products(session: Session = Depends(module_gate(MODULE))):
565
  """The product table for this session's scope — the /customers envelope byte-for-byte
566
  (`{fields, rows, today, pulled_at}`) plus two additive keys (`identity`, `scope`).
567
 
@@ -573,13 +575,21 @@ def products(session: Session = Depends(module_gate(MODULE))):
573
  import aios_grid
574
  import modules.product_data as pd
575
 
576
- g = product_assembly(session)
577
  rows = aios_grid.rows_from_pool(
578
  g["rows_src"], g["fields"], g["ws"].get("overlays"), derived=g["derived"])
579
  rows = _seed_image(_seed_shared(rows, g["rows_src"], g["fields"]), g["fields"])
580
  from core import user_tables as _zones
581
  rows = _zones.zone_cells("product_data", rows, allowed_row_ids=g["pids"],
582
  st=session.runtime)
 
 
 
 
 
 
 
 
583
  # ⭐⭐ W41-T17 (owner instruction 25) — THE IDENTITY REPORT RIDES THE ENVELOPE, because a
584
  # report nothing reads is not a report. `product_data.identity_report` is standing rule 1's
585
  # second sentence as data for this grid's two identity columns: `product_id` (the Odoo
@@ -599,12 +609,21 @@ def products(session: Session = Depends(module_gate(MODULE))):
599
  # The envelope check beside it filters to the four `/customers` keys, so an additive key is
600
  # the shape this route already established with `identity` and `scope`.
601
  return {"fields": g["fields"], "rows": rows,
 
 
602
  "today": g["today"], "pulled_at": time.strftime("%Y-%m-%d %H:%M"),
603
  "identity": {"pid": "pid", "businessKey": "code"},
604
  "identity_report": pd.identity_report(g["rows_src"]),
605
  "scope": {"team_id": g["team_id"], "consolidated": g["team_id"] is None}}
606
 
607
 
 
 
 
 
 
 
 
608
  @router.patch("/products/{pid}")
609
  def patch_product(pid: int, body: dict = Body(default=None),
610
  session: Session = Depends(module_gate(MODULE))):
@@ -769,7 +788,8 @@ def pd_fields(consolidated=True, st=None):
769
  # `aios_grid_fields.json`. That file is the ODOO-SOURCED contract — every key in it is a
770
  # column `modules.product_data.pool()` reads off a SKU — and this one is neither read from
771
  # Odoo nor written to it. Injected after the JSON so the canonical file stays the answer to
772
- # "what does Odoo give us", which is the question `verify_fields_contract` referees.
 
773
  if st is not None:
774
  image = pd.image_field_definition(st=st)
775
  else:
 
498
  str(pid): {key: value for key, value in (row or {}).items() if key not in hidden}
499
  for pid, row in (ws.get("overlays") or {}).items()
500
  }
501
+ # D-464: the saved views' column keys are the next wire `/workspace` serves.
502
+ workspace["views"] = aios_grid.narrow_view_keys(workspace.get("views"), hidden)
503
 
504
  today = time.strftime("%Y-%m-%d")
505
  fields = aios_grid.project_date_recurrences(fields, today=today)
 
563
 
564
 
565
  @router.get("/products")
566
+ def products(session: Session = Depends(module_gate(MODULE)), related: str = None):
567
  """The product table for this session's scope — the /customers envelope byte-for-byte
568
  (`{fields, rows, today, pulled_at}`) plus two additive keys (`identity`, `scope`).
569
 
 
575
  import aios_grid
576
  import modules.product_data as pd
577
 
578
+ g = product_assembly(session, consume_corrections=not bool(related))
579
  rows = aios_grid.rows_from_pool(
580
  g["rows_src"], g["fields"], g["ws"].get("overlays"), derived=g["derived"])
581
  rows = _seed_image(_seed_shared(rows, g["rows_src"], g["fields"]), g["fields"])
582
  from core import user_tables as _zones
583
  rows = _zones.zone_cells("product_data", rows, allowed_row_ids=g["pids"],
584
  st=session.runtime)
585
+ from routes_odoo_tables import narrow_related_rows
586
+ rows = narrow_related_rows(session, MODULE, rows, related)
587
+ from routes_tables import scope_linked_metrics, linked_metric_limits
588
+ rows, g['fields'], metric_refusals = scope_linked_metrics(
589
+ session, MODULE, rows, g['fields'], g['measures'], related)
590
+ if related:
591
+ pinned = {row["pid"] for row in rows}
592
+ g["rows_src"] = [row for row in g["rows_src"] if row["pid"] in pinned]
593
  # ⭐⭐ W41-T17 (owner instruction 25) — THE IDENTITY REPORT RIDES THE ENVELOPE, because a
594
  # report nothing reads is not a report. `product_data.identity_report` is standing rule 1's
595
  # second sentence as data for this grid's two identity columns: `product_id` (the Odoo
 
609
  # The envelope check beside it filters to the four `/customers` keys, so an additive key is
610
  # the shape this route already established with `identity` and `scope`.
611
  return {"fields": g["fields"], "rows": rows,
612
+ "refusals": metric_refusals,
613
+ "limits": linked_metric_limits(metric_refusals),
614
  "today": g["today"], "pulled_at": time.strftime("%Y-%m-%d %H:%M"),
615
  "identity": {"pid": "pid", "businessKey": "code"},
616
  "identity_report": pd.identity_report(g["rows_src"]),
617
  "scope": {"team_id": g["team_id"], "consolidated": g["team_id"] is None}}
618
 
619
 
620
+ @router.post("/products/aggregates")
621
+ def product_aggregates(body: dict = Body(default=None),
622
+ session: Session = Depends(module_gate(MODULE))):
623
+ from routes_odoo_tables import odoo_table_aggregates
624
+ return odoo_table_aggregates(MODULE, body=body, session=session)
625
+
626
+
627
  @router.patch("/products/{pid}")
628
  def patch_product(pid: int, body: dict = Body(default=None),
629
  session: Session = Depends(module_gate(MODULE))):
 
788
  # `aios_grid_fields.json`. That file is the ODOO-SOURCED contract — every key in it is a
789
  # column `modules.product_data.pool()` reads off a SKU — and this one is neither read from
790
  # Odoo nor written to it. Injected after the JSON so the canonical file stays the answer to
791
+ # "what does Odoo give us". `verify_fields_contract` does NOT referee the product contract
792
+ # (D-400); `platform/verify_product_pool.py::section_product_field_contract` does.
793
  if st is not None:
794
  image = pd.image_field_definition(st=st)
795
  else:
api/routes_query.py CHANGED
@@ -74,6 +74,19 @@ MAX_HISTORY_CHARS = 6000
74
  MAX_RATING_REASON = 200
75
  QUERY_RATINGS = ("up", "down")
76
  MODEL_AUTO = "auto"
 
 
 
 
 
 
 
 
 
 
 
 
 
77
  NO_CITED_SOURCES_REASON = (
78
  "No authorized application data or retrieved knowledge was cited for this answer."
79
  )
@@ -166,6 +179,11 @@ def _answer_provenance(artifact=None):
166
  def _public_message(row, allowed_citations):
167
  """Project a stored answer through its current citation wall before it reaches the browser."""
168
  out = _safe(row) if isinstance(row, dict) else {}
 
 
 
 
 
169
  raw_citations = out.get("citationIds")
170
  citation_ids = [str(citation_id) for citation_id in (raw_citations if isinstance(raw_citations, list) else [])
171
  if str(citation_id) in allowed_citations]
@@ -1114,6 +1132,8 @@ def _public_state(state, session):
1114
  messages = sorted(state["messages"].values(), key=lambda row: row.get("createdAt", ""))
1115
  views = sorted((_public_view(row) for row in allowed_views), key=lambda row: row["createdAt"], reverse=True)
1116
  messages = [row for row in messages if not row.get("viewId") or row.get("viewId") in allowed_ids]
 
 
1117
  messages = [_public_message(row, allowed_citations) for row in messages]
1118
  return {"threads": _safe(threads), "messages": messages, "views": views,
1119
  "citations": _safe([row for key, row in state["citations"].items()
@@ -1344,14 +1364,21 @@ def _submit(body, session, chat=None):
1344
  raise err(400, "bad_request", "no question was asked")
1345
  if len(question) > MAX_QUESTION:
1346
  raise err(400, "question_too_long", f"a question must be under {MAX_QUESTION} characters")
1347
- if not target:
1348
- raise err(400, "bad_request", "one target database must be selected")
1349
  if selected_model not in model_choices():
1350
  raise err(400, "unknown_model", "that model is not available in Query")
1351
 
 
 
 
 
 
1352
  # This call happens before model selection and is the only permitted source-data read.
1353
- snapshot = assistant_read_scope(session, target, fields=None, filters=body.get("filters"))
1354
- sources = _sources(body, target, session)
 
 
 
 
1355
  state = _state(session)
1356
  thread_id = str(body.get("threadId") or "").strip()
1357
  if thread_id and thread_id not in state["threads"]:
@@ -1365,9 +1392,25 @@ def _submit(body, session, chat=None):
1365
  # turn's own messages exist, so the replay is strictly the prior conversation.
1366
  # ⭐ C7/T31: `session` rides along so the meter can attribute this call. See `_call_model`'s
1367
  # docstring for why it is an argument and not ambient state (PRD amendment A1).
1368
- spec, said, provider, reason = _call_model(question, snapshot, model=selected_model, chat=chat,
1369
- history=_history(state, thread_id), session=session)
1370
- view, refusal, refusal_code = _validate(spec, snapshot["fields"]) if spec is not None else (None, said, reason)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1371
 
1372
  artifact = None
1373
  citation = None
@@ -1404,12 +1447,17 @@ def _submit(body, session, chat=None):
1404
  assistant_message = {
1405
  "id": assistant_message_id, "threadId": thread_id, "role": "assistant", "createdAt": now,
1406
  "content": (said or _said_fallback(artifact)) if artifact else (refusal or "the assistant could not answer"),
 
1407
  "targetDatabase": target, "requestedModel": selected_model, "model": provider,
 
1408
  **_model_identity(provider),
1409
  "reason": refusal_code, "viewId": artifact["id"] if artifact else None,
1410
  "provenance": _answer_provenance(artifact),
1411
  "citationIds": artifact["citationIds"] if artifact else [], "numeric": artifact["numeric"] if artifact else None,
1412
  }
 
 
 
1413
  user_message = {"id": user_message_id, "threadId": thread_id, "role": "user", "createdAt": now,
1414
  "content": question, "sources": sources, "targetDatabase": target,
1415
  "requestedModel": selected_model}
@@ -1451,18 +1499,147 @@ def submit_chat(body: dict = Body(default=None), session: Session = Depends(requ
1451
  # Governed Query contract (Wave 45 C2). The existing chat workspace remains the presentation
1452
  # surface; this provider-neutral door is the shared execution seam for Assistant, Agents, voice,
1453
  # Slack and future adapters.
1454
- def _semantic_baseline(database):
1455
  from harness import semantic as sem
1456
  topic = next((row for row in sem.topics().values()
1457
  if isinstance(row, dict) and row.get("grid") == database), None)
1458
- if not topic:
1459
- raise err(404, "semantic_source_unknown",
1460
- "that database does not have a semantic contract")
1461
- return semantic_rt.topic_records(database, topic)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1462
 
1463
 
1464
  def _semantic_records(session, database):
1465
- return semantic_rt.load(session.runtime, _semantic_baseline(database))
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1466
 
1467
 
1468
  def _semantic_authorize(session, database, *, require_answerable=False):
@@ -1536,7 +1713,7 @@ def update_semantic_field(database: str, field: str, body: dict = Body(default=N
1536
  session: Session = Depends(require_session)):
1537
  _semantic_authorize(session, database)
1538
  try:
1539
- row = semantic_rt.update(session.runtime, _semantic_baseline(database), database, field,
1540
  body, actor=session.uname, is_admin=session.admin)
1541
  except semantic_rt.SemanticError as exc:
1542
  code = "forbidden" if "only a tenant admin" in str(exc) else "bad_semantic_contract"
@@ -1550,7 +1727,7 @@ def profile_semantic_field(database: str, field: str,
1550
  session: Session = Depends(require_session)):
1551
  """Profile values through Query's permissioned snapshot and persist only a bounded summary."""
1552
  _semantic_authorize(session, database, require_answerable=True)
1553
- baseline = _semantic_baseline(database)
1554
  records = semantic_rt.load(session.runtime, baseline)
1555
  try:
1556
  current = records[semantic_rt.field_key(database, field)]
@@ -1574,6 +1751,115 @@ def profile_semantic_field(database: str, field: str,
1574
  return {"field": row}
1575
 
1576
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1577
  def _resume_b64(raw):
1578
  return base64.urlsafe_b64encode(raw).decode("ascii").rstrip("=")
1579
 
@@ -1713,12 +1999,10 @@ def query_metrics(database: str = "", session: Session = Depends(require_session
1713
  _semantic_authorize_schema_only(session, database)
1714
  metrics, reason = _metric_catalogue(session, database)
1715
  # ⛔⛔ THE GOVERNED SOURCE WALL STAYS, BUT IT CANNOT BE THE FIRST THING ASKED.
1716
- # `_semantic_records` resolves `_semantic_baseline`, which raises 404 `semantic_source_unknown`
1717
- # for a database no topic declares — and `ut_odoo_order_lines` / `ut_odoo_gl_lines` are exactly
1718
- # that: read-through grains with no entity topic. A 404 carries no body the picker can read,
1719
- # so the reader was told nothing at all where C7 promises a sentence. The wall is therefore
1720
- # asked only once a topic is known to exist, which is the only case where it has a contract to
1721
- # enforce; a topic-less database answers 200 with `reason: no_topic`, and that IS its answer.
1722
  if reason != "no_topic":
1723
  _semantic_records(session, database)
1724
  return {"metrics": metrics, "reason": reason}
@@ -1838,22 +2122,34 @@ def governed_query(body: dict = Body(default=None),
1838
  body = body if isinstance(body, dict) else {}
1839
  database = str(body.get("database") or "").strip()
1840
  question = " ".join(str(body.get("prompt") or body.get("question") or "").split())
1841
- if not database or not question:
1842
- raise err(400, "bad_request", "a database and prompt are required")
1843
  if len(question) > MAX_QUESTION:
1844
  raise err(400, "question_too_long", f"a prompt must be under {MAX_QUESTION} characters")
1845
- _semantic_authorize(session, database, require_answerable=True)
 
 
 
 
 
 
 
 
1846
  records = _semantic_records(session, database)
 
 
 
 
 
1847
  approved = semantic_rt.approved_fields(records, database)
1848
  if not approved:
1849
- raise err(409, "query_semantics_unconfigured",
1850
- "this database has no approved semantic fields yet")
1851
  raw_requested = body.get("fieldKeys")
1852
  if raw_requested is not None and not isinstance(raw_requested, list):
1853
  raise err(400, "bad_request", "fieldKeys must be a list")
1854
  requested = list(dict.fromkeys(str(x) for x in (raw_requested or ()) if str(x)))
1855
  if not requested:
1856
- return _clarification(session, database, question, approved)
1857
  _verify_resume_token(body.get("resumeToken"), session, database, question,
1858
  approved, requested)
1859
  try:
@@ -1861,12 +2157,13 @@ def governed_query(body: dict = Body(default=None),
1861
  except semantic_rt.SemanticError as exc:
1862
  raise err(403, "semantic_field_unapproved", str(exc))
1863
  snapshot = assistant_read_scope(session, database, fields=requested, filters=None)
 
1864
  snapshot, formulas = _tool_formula_snapshot(body, snapshot, requested)
1865
  spec, planned_wording, provider = _tool_plan(question, body, snapshot, session)
1866
  if spec is None:
1867
  return {"status": "refused", "wording": planned_wording,
1868
  "speakable": planned_wording, "artifacts": None,
1869
- "model": _model_identity(provider)}
1870
  view, refusal, refusal_code = _validate(spec, snapshot.get("fields") or selected)
1871
  if view is None:
1872
  raise err(400, refusal_code or "bad_query_artifact", refusal or "that Query is invalid")
@@ -1891,7 +2188,7 @@ def governed_query(body: dict = Body(default=None),
1891
  "rows": [{key: row.get(key) for key in view["visible"]} for row in rows],
1892
  "groups": groups, "numeric": numeric,
1893
  "artifacts": artifacts, "semanticRouter": semantic_rt.router_projection(records, database),
1894
- "model": _model_identity(provider)}
1895
 
1896
 
1897
  @router.post("/query/formulas/parse")
 
74
  MAX_RATING_REASON = 200
75
  QUERY_RATINGS = ("up", "down")
76
  MODEL_AUTO = "auto"
77
+ QUERY_NO_SOURCE_MATCHED = (
78
+ "None of the databases you can read cover that question. Pick one from the list, "
79
+ "or ask about a database you can open."
80
+ )
81
+ QUERY_DRAFT_ADMIN = "This database has no approved fields yet. Pick the ones this answer should use and they will be approved for everyone."
82
+ QUERY_DRAFT_MEMBER = "This database has no approved fields yet. Pick the ones you need and your administrators will be asked to approve them."
83
+ QUERY_DRAFT_SENT = "Sent to your administrators. Ask again once one of them approves."
84
+
85
+
86
+ def _draft_approved_line(name, date):
87
+ return f"Approved by {_no_dashes(name)} on {str(date)[:10]}."
88
+
89
+
90
  NO_CITED_SOURCES_REASON = (
91
  "No authorized application data or retrieved knowledge was cited for this answer."
92
  )
 
179
  def _public_message(row, allowed_citations):
180
  """Project a stored answer through its current citation wall before it reaches the browser."""
181
  out = _safe(row) if isinstance(row, dict) else {}
182
+ if isinstance(out.get("draft"), dict):
183
+ # Historical cards cannot replay samples admitted under an older sensitivity or row
184
+ # grant. GET /semantics/{database}/draft computes their fresh permissioned profiles.
185
+ for field in out["draft"].get("fields") or ():
186
+ field["sample"] = semantic_rt.redacted_profile()
187
  raw_citations = out.get("citationIds")
188
  citation_ids = [str(citation_id) for citation_id in (raw_citations if isinstance(raw_citations, list) else [])
189
  if str(citation_id) in allowed_citations]
 
1132
  messages = sorted(state["messages"].values(), key=lambda row: row.get("createdAt", ""))
1133
  views = sorted((_public_view(row) for row in allowed_views), key=lambda row: row["createdAt"], reverse=True)
1134
  messages = [row for row in messages if not row.get("viewId") or row.get("viewId") in allowed_ids]
1135
+ messages = [row for row in messages if not isinstance(row.get("draft"), dict)
1136
+ or row["draft"].get("database") in permitted]
1137
  messages = [_public_message(row, allowed_citations) for row in messages]
1138
  return {"threads": _safe(threads), "messages": messages, "views": views,
1139
  "citations": _safe([row for key, row in state["citations"].items()
 
1364
  raise err(400, "bad_request", "no question was asked")
1365
  if len(question) > MAX_QUESTION:
1366
  raise err(400, "question_too_long", f"a question must be under {MAX_QUESTION} characters")
 
 
1367
  if selected_model not in model_choices():
1368
  raise err(400, "unknown_model", "that model is not available in Query")
1369
 
1370
+ routed = _route(session, question) if not target else {
1371
+ "database": target, "auto": False, "alternatives": []}
1372
+ target = routed["database"]
1373
+ route_refusal = (QUERY_NO_SOURCE_MATCHED if not target else routed.get("refusal"))
1374
+
1375
  # This call happens before model selection and is the only permitted source-data read.
1376
+ snapshot = (assistant_read_scope(session, target, fields=None, filters=body.get("filters"))
1377
+ if not route_refusal else None)
1378
+ sources = _sources(body, target, session) if not route_refusal else ([target] if target else [])
1379
+ if snapshot is not None and not routed["auto"]:
1380
+ routed.update({"label": _no_dashes(snapshot["label"]),
1381
+ "reason": f"You selected {_no_dashes(snapshot['label'])}."})
1382
  state = _state(session)
1383
  thread_id = str(body.get("threadId") or "").strip()
1384
  if thread_id and thread_id not in state["threads"]:
 
1392
  # turn's own messages exist, so the replay is strictly the prior conversation.
1393
  # ⭐ C7/T31: `session` rides along so the meter can attribute this call. See `_call_model`'s
1394
  # docstring for why it is an argument and not ambient state (PRD amendment A1).
1395
+ draft_card = None
1396
+ if not route_refusal:
1397
+ _semantic_authorize(session, target)
1398
+ records = _semantic_records(session, target)
1399
+ if not semantic_rt.approved_fields(records, target):
1400
+ draft_card = _draft_card(session, target, question, records)
1401
+ else:
1402
+ snapshot = _semantic_answer_snapshot(snapshot, records)
1403
+ if route_refusal:
1404
+ said, provider = route_refusal, None
1405
+ view, refusal, refusal_code = (None, route_refusal,
1406
+ "query_source_unavailable" if target else "query_no_source_matched")
1407
+ elif draft_card:
1408
+ said, provider = draft_card["alignmentQuestion"]["question"], None
1409
+ view, refusal, refusal_code = None, said, "query_semantics_drafting"
1410
+ else:
1411
+ spec, said, provider, reason = _call_model(question, snapshot, model=selected_model, chat=chat,
1412
+ history=_history(state, thread_id), session=session)
1413
+ view, refusal, refusal_code = _validate(spec, snapshot["fields"]) if spec is not None else (None, said, reason)
1414
 
1415
  artifact = None
1416
  citation = None
 
1447
  assistant_message = {
1448
  "id": assistant_message_id, "threadId": thread_id, "role": "assistant", "createdAt": now,
1449
  "content": (said or _said_fallback(artifact)) if artifact else (refusal or "the assistant could not answer"),
1450
+ # D-745: the Assistant restores routing from targetDatabase; keep it equal to routed["database"].
1451
  "targetDatabase": target, "requestedModel": selected_model, "model": provider,
1452
+ "routedTo": routed,
1453
  **_model_identity(provider),
1454
  "reason": refusal_code, "viewId": artifact["id"] if artifact else None,
1455
  "provenance": _answer_provenance(artifact),
1456
  "citationIds": artifact["citationIds"] if artifact else [], "numeric": artifact["numeric"] if artifact else None,
1457
  }
1458
+ if draft_card:
1459
+ assistant_message.update({key: draft_card[key] for key in
1460
+ ("status", "draft", "alignmentQuestion", "resumeToken")})
1461
  user_message = {"id": user_message_id, "threadId": thread_id, "role": "user", "createdAt": now,
1462
  "content": question, "sources": sources, "targetDatabase": target,
1463
  "requestedModel": selected_model}
 
1499
  # Governed Query contract (Wave 45 C2). The existing chat workspace remains the presentation
1500
  # surface; this provider-neutral door is the shared execution seam for Assistant, Agents, voice,
1501
  # Slack and future adapters.
1502
+ def _semantic_baseline(database, session=None):
1503
  from harness import semantic as sem
1504
  topic = next((row for row in sem.topics().values()
1505
  if isinstance(row, dict) and row.get("grid") == database), None)
1506
+ if topic:
1507
+ return semantic_rt.topic_records(database, topic)
1508
+ if session is not None:
1509
+ try:
1510
+ import odoo_relational
1511
+ base_key, connection_id = odoo_relational.split_table_key(database)
1512
+ if base_key is not None:
1513
+ namespace = (odoo_relational.connection_namespace(session.runtime, connection_id)[1]
1514
+ if connection_id else None)
1515
+ choice = next((row for row in odoo_relational.grid_choices(
1516
+ session.runtime, connection_id) if row.get("key") == database), None)
1517
+ model = (choice or {}).get("model")
1518
+ catalog = odoo_relational.readable_model_catalog(session.runtime, namespace)
1519
+ row = next((row for row in catalog.get("models") or ()
1520
+ if model and row.get("model") == model), None)
1521
+ if row is not None:
1522
+ return semantic_rt.generated_records(database, row, model=model)
1523
+ except Exception:
1524
+ # An unavailable cached catalogue is an unknown schema, not a server failure.
1525
+ pass
1526
+ # A custom database's own declared schema supplies draft rows. This is metadata only;
1527
+ # routing must not read records from every candidate just to recognize its label.
1528
+ projection = getattr(session.runtime, "get_projection", None)
1529
+ definitions = (projection("user_tables", drop=("rows",)) or {}) if projection else {}
1530
+ definition = definitions.get(database)
1531
+ if (database.startswith("ut_") and not database.startswith("ut_odoo_")
1532
+ and isinstance(definition, dict)):
1533
+ baseline = semantic_rt.topic_records(database, {"fields": definition.get("fields") or []})
1534
+ for field in definition.get("fields") or ():
1535
+ row = baseline.get(semantic_rt.field_key(database, field.get("key")))
1536
+ if row is None:
1537
+ continue
1538
+ row["relation"] = str((field.get("link") or {}).get("table") or "")
1539
+ if row["relation"] or row["dataType"] == "link":
1540
+ row["role"] = "link"
1541
+ elif row["dataType"] in ("date", "datetime"):
1542
+ row["role"] = "date"
1543
+ # Creating a source column does not itself grant tenant-wide approval.
1544
+ row.update({"state": "draft", "approvedBy": None, "approvedAt": None})
1545
+ return baseline
1546
+ raise err(404, "semantic_source_unknown",
1547
+ "that database does not have a semantic contract")
1548
 
1549
 
1550
  def _semantic_records(session, database):
1551
+ return semantic_rt.load(session.runtime, _semantic_baseline(database, session))
1552
+
1553
+
1554
+ def _route_candidates(session):
1555
+ """The status wall is the sole source of candidate keys; read semantic metadata only."""
1556
+ from harness import semantic as sem
1557
+ statuses = assistant_source_status(session, None)
1558
+ candidates = []
1559
+ definitions = None
1560
+ for database, status in statuses.items():
1561
+ if not status.get("permitted"):
1562
+ continue
1563
+ try:
1564
+ topic = next((row for row in sem.topics().values()
1565
+ if row.get("grid") == database), {})
1566
+ records = _semantic_records(session, database)
1567
+ approved = semantic_rt.approved_fields(records, database)
1568
+ if definitions is None:
1569
+ projection = getattr(session.runtime, "get_projection", None)
1570
+ definitions = (projection("user_tables", drop=("rows",)) or {}) if projection else {}
1571
+ definition = definitions.get(database) or {}
1572
+ label = _said(status.get("label") or definition.get("label") or
1573
+ topic.get("label") or database.removeprefix("ut_").replace("_", " ")) or database
1574
+ terms = []
1575
+
1576
+ def words(value):
1577
+ if isinstance(value, str) and value.strip():
1578
+ terms.append(value)
1579
+ terms.extend(re.findall(r"[^\W_]+", value))
1580
+
1581
+ words(label)
1582
+ words(topic.get("label"))
1583
+ words(topic.get("grain"))
1584
+ scope = topic.get("scope")
1585
+ for value in (scope.values() if isinstance(scope, dict) else [scope]):
1586
+ words(value)
1587
+ words((topic.get("semantic") or {}).get("scope"))
1588
+ # ⭐⭐ ONLY A CURATED TOPIC LENDS ITS FIELDS TO THE VOCABULARY, AND THIS WAS MEASURED
1589
+ # ON A RUNNING TENANT RATHER THAN REASONED ABOUT. W61-T28 gives every topic-less Odoo
1590
+ # database a pre-approved map generated from Odoo's own `fields_get`, so
1591
+ # `ut_odoo_order_lines` arrived carrying every raw field name `sale.order.line` has.
1592
+ # Asked "invoiced sales last week", the router picked ODOO ORDER LINES: raw names
1593
+ # matched both "invoiced" and "sale", and a generated map has dozens of fields where a
1594
+ # curated topic has a dozen, so the long tail outvoted all eight curated topics. A raw
1595
+ # column name is not a description of what a person is asking about. A generated
1596
+ # database therefore routes on WHAT IT IS (its label, below) and never on the field
1597
+ # names nobody wrote for this purpose; it stays fully readable once chosen, which is
1598
+ # what R27 pre-approved it for.
1599
+ if topic:
1600
+ for field in approved.values():
1601
+ words(field.get("label"))
1602
+ terms.extend(field.get("synonyms") or ())
1603
+ # Head words describe the field; explanatory tails often name OTHER databases.
1604
+ words(" ".join(str(field.get("purpose") or "").split()[:4]))
1605
+ candidates.append({"database": database, "label": label,
1606
+ "topic": topic.get("label") or "", "terms": terms,
1607
+ # What this database IS, as opposed to what it HOLDS. See
1608
+ # `route_question`: a subject word outweighs a field word, which is
1609
+ # what stops an orders question landing in the customer book.
1610
+ "subject": [label, topic.get("label") or ""],
1611
+ "curated": bool(topic),
1612
+ "answerable": bool(status.get("answerable")),
1613
+ "reason": _said(status.get("reason")) or ""})
1614
+ except Exception:
1615
+ # One unavailable semantic contract must not hide the other permitted databases.
1616
+ continue
1617
+ return candidates
1618
+
1619
+
1620
+ def _route(session, question):
1621
+ from harness import semantic as sem
1622
+ candidates = _route_candidates(session)
1623
+ result = sem.route_question(question, candidates)
1624
+ if not result["database"]:
1625
+ return {"database": "", "label": "", "reason": QUERY_NO_SOURCE_MATCHED,
1626
+ "auto": True, "alternatives": []}
1627
+ readable = sem.route_question(question, [row for row in candidates if row["answerable"]])
1628
+ # Availability breaks equally strong matches, never substitutes a worse semantic match.
1629
+ strength = lambda row: (row["score"], max(
1630
+ (len(" ".join(sem._route_tokens(term))) for term in row["matched"]), default=0))
1631
+ chosen = readable if readable["database"] and strength(readable) == strength(result) else result
1632
+ candidate = next(row for row in candidates if row["database"] == chosen["database"])
1633
+ matched = " and ".join(" ".join(re.findall(r"[^\W_]+", term))
1634
+ for term in chosen["matched"][:2])
1635
+ routed = {"database": candidate["database"], "label": candidate["label"],
1636
+ "reason": f"{candidate['label']} matched {matched}.", "auto": True,
1637
+ "alternatives": [{"database": row["database"], "label": _no_dashes(row["label"])}
1638
+ for row in result["ranked"] if row["database"] != candidate["database"]][:2]}
1639
+ if not candidate["answerable"]:
1640
+ cause = candidate["reason"] or "this database cannot answer Query yet"
1641
+ routed["refusal"] = f"{candidate['label']} cannot answer because {cause.rstrip('.')}."
1642
+ return routed
1643
 
1644
 
1645
  def _semantic_authorize(session, database, *, require_answerable=False):
 
1713
  session: Session = Depends(require_session)):
1714
  _semantic_authorize(session, database)
1715
  try:
1716
+ row = semantic_rt.update(session.runtime, _semantic_baseline(database, session), database, field,
1717
  body, actor=session.uname, is_admin=session.admin)
1718
  except semantic_rt.SemanticError as exc:
1719
  code = "forbidden" if "only a tenant admin" in str(exc) else "bad_semantic_contract"
 
1727
  session: Session = Depends(require_session)):
1728
  """Profile values through Query's permissioned snapshot and persist only a bounded summary."""
1729
  _semantic_authorize(session, database, require_answerable=True)
1730
+ baseline = _semantic_baseline(database, session)
1731
  records = semantic_rt.load(session.runtime, baseline)
1732
  try:
1733
  current = records[semantic_rt.field_key(database, field)]
 
1751
  return {"field": row}
1752
 
1753
 
1754
+ def _semantic_answer_snapshot(snapshot, records):
1755
+ """Custom approvals govern model input too; sensitive values never become answer rows."""
1756
+ database = snapshot.get("database") or ""
1757
+ custom = (database.startswith("ut_") and not database.startswith("ut_odoo_")
1758
+ and not any(row.get("topic") for row in records.values()))
1759
+ hidden = {row["field"] for row in records.values()
1760
+ if row.get("sensitivity") in ("sensitive", "restricted")}
1761
+ if not hidden and not custom:
1762
+ return snapshot
1763
+ out = copy.deepcopy(snapshot)
1764
+ if custom:
1765
+ approved = semantic_rt.approved_fields(records, database)
1766
+ out["fields"] = [field for field in out.get("fields") or () if field.get("key") in approved]
1767
+ out["records"] = [{key: value for key, value in row.items() if key == "pid" or key in approved}
1768
+ for row in out.get("records") or ()]
1769
+ for row in out.get("records") or ():
1770
+ for field in hidden.intersection(row):
1771
+ row[field] = None
1772
+ return out
1773
+
1774
+
1775
+ def _draft_card(session, database, question, records):
1776
+ snapshot = assistant_read_scope(session, database, fields=None, filters=None)
1777
+ fields = semantic_rt.draft_fields(records, snapshot)
1778
+ statuses = assistant_source_status(session, None)
1779
+ projection = getattr(session.runtime, "get_projection", None)
1780
+ definitions = (projection("user_tables", drop=("rows",)) or {}) if projection else {}
1781
+ from harness import semantic as sem
1782
+ topic_labels = {row.get("grid"): row.get("label") for row in sem.topics().values()}
1783
+ for field in fields:
1784
+ record = records[semantic_rt.field_key(database, field["field"])]
1785
+ relation = record.get("relation") or ""
1786
+ relation = {"customer": "customer_data", "product": "product_data"}.get(relation, relation)
1787
+ status = statuses.get(relation) or {}
1788
+ if status.get("permitted"):
1789
+ label = (status.get("label") or (definitions.get(relation) or {}).get("label")
1790
+ or topic_labels.get(relation))
1791
+ if label:
1792
+ field["why"] = field["why"].rstrip(".") + f", linked to {_no_dashes(label)}."
1793
+ field["label"] = _no_dashes(field["label"])
1794
+ # Profiles are projections for display, never an authority for field keys or approval.
1795
+ for key in ("values", "sample"):
1796
+ field["sample"][key] = [_no_dashes(value) for value in field["sample"].get(key) or ()]
1797
+ request = semantic_rt.draft_request(session.runtime, database) or {}
1798
+ approved = semantic_rt.approved_fields(records, database)
1799
+ approval = request if request.get("state") == "approved" else next(iter(approved.values()), {})
1800
+ state = "approved" if approved else "pending"
1801
+ draft = {"database": database, "label": _no_dashes(snapshot.get("label") or database),
1802
+ "canApprove": bool(session.admin), "fields": fields,
1803
+ "sentToAdmins": not session.admin, "state": state,
1804
+ "requestedBy": request.get("requestedBy") or session.uname,
1805
+ "approvedBy": approval.get("approvedBy"), "approvedAt": approval.get("approvedAt"),
1806
+ "sentMessage": QUERY_DRAFT_SENT}
1807
+ if draft["approvedBy"] and draft["approvedAt"]:
1808
+ draft["approvalLine"] = _draft_approved_line(draft["approvedBy"], draft["approvedAt"])
1809
+ return {"status": "clarification", "alignmentQuestion": {
1810
+ "id": "draft_fields", "question": QUERY_DRAFT_ADMIN if session.admin else QUERY_DRAFT_MEMBER,
1811
+ "choices": [{"field": field["field"], "label": field["label"]} for field in fields],
1812
+ "maxSelections": 8}, "draft": draft, "resumeToken": ""}
1813
+
1814
+
1815
+ @router.post("/query/semantics/{database}/approve")
1816
+ def approve_semantic_draft(database: str, body: dict = Body(default=None),
1817
+ session: Session = Depends(require_session)):
1818
+ _semantic_authorize(session, database)
1819
+ baseline = _semantic_baseline(database, session)
1820
+ body = body if isinstance(body, dict) else {}
1821
+ fields = body.get("fields")
1822
+ if (not isinstance(fields, list) or not fields or len(fields) > 8
1823
+ or any(not isinstance(field, str) for field in fields)):
1824
+ raise err(400, "bad_semantic_contract", "pick between one and eight database fields")
1825
+ fields = list(dict.fromkeys(fields))
1826
+ question = " ".join(str(body.get("question") or "").split())
1827
+ if not question or len(question) > MAX_QUESTION:
1828
+ raise err(400, "bad_request", "a question of up to 500 characters is required")
1829
+ try:
1830
+ # Validate the whole selection before the first write, including member requests.
1831
+ if any(semantic_rt.field_key(database, field) not in baseline for field in fields):
1832
+ raise semantic_rt.SemanticError("that field is not in the database contract")
1833
+ now = _dt.datetime.now(_dt.timezone.utc)
1834
+ if not session.admin:
1835
+ request = semantic_rt.record_draft_request(
1836
+ session.runtime, database, question, fields, actor=session.uname, now=now)
1837
+ return {"approved": [], "state": "pending", "sentToAdmins": True,
1838
+ "requestedBy": request["requestedBy"], "sentMessage": QUERY_DRAFT_SENT}
1839
+ for field in fields:
1840
+ semantic_rt.update(session.runtime, baseline, database, field, {"state": "approved"},
1841
+ actor=session.uname, is_admin=session.admin, now=now)
1842
+ if not semantic_rt.draft_request(session.runtime, database):
1843
+ semantic_rt.record_draft_request(
1844
+ session.runtime, database, question, fields, actor=session.uname, now=now)
1845
+ request = semantic_rt.approve_draft_request(
1846
+ session.runtime, database, fields, actor=session.uname, is_admin=session.admin, now=now)
1847
+ except semantic_rt.SemanticError as exc:
1848
+ raise err(400, "bad_semantic_contract", str(exc))
1849
+ return {"approved": fields, "approvedBy": request["approvedBy"],
1850
+ "approvedAt": request["approvedAt"], "state": "approved",
1851
+ "approvalLine": _draft_approved_line(request["approvedBy"], request["approvedAt"])}
1852
+
1853
+
1854
+ @router.get("/query/semantics/{database}/draft")
1855
+ def semantic_draft(database: str, session: Session = Depends(require_session)):
1856
+ _semantic_authorize(session, database)
1857
+ records = _semantic_records(session, database)
1858
+ request = semantic_rt.draft_request(session.runtime, database)
1859
+ return {**_draft_card(session, database, (request or {}).get("question") or "", records),
1860
+ "request": request}
1861
+
1862
+
1863
  def _resume_b64(raw):
1864
  return base64.urlsafe_b64encode(raw).decode("ascii").rstrip("=")
1865
 
 
1999
  _semantic_authorize_schema_only(session, database)
2000
  metrics, reason = _metric_catalogue(session, database)
2001
  # ⛔⛔ THE GOVERNED SOURCE WALL STAYS, BUT IT CANNOT BE THE FIRST THING ASKED.
2002
+ # Cached Odoo fields can now supply a baseline without an entity topic, including order and
2003
+ # GL lines. That schema does not create a validated entity-measure binding: keep returning
2004
+ # `reason: no_topic` for this picker until a topic owns the measure offer. Other callers of
2005
+ # `_semantic_records` still resolve the generated schema through their existing record wall.
 
 
2006
  if reason != "no_topic":
2007
  _semantic_records(session, database)
2008
  return {"metrics": metrics, "reason": reason}
 
2122
  body = body if isinstance(body, dict) else {}
2123
  database = str(body.get("database") or "").strip()
2124
  question = " ".join(str(body.get("prompt") or body.get("question") or "").split())
2125
+ if not question:
2126
+ raise err(400, "bad_request", "a prompt is required")
2127
  if len(question) > MAX_QUESTION:
2128
  raise err(400, "question_too_long", f"a prompt must be under {MAX_QUESTION} characters")
2129
+ routed = _route(session, question) if not database else {
2130
+ "database": database, "label": "", "reason": "", "auto": False, "alternatives": []}
2131
+ database = routed["database"]
2132
+ if database:
2133
+ _semantic_authorize(session, database, require_answerable=not routed["auto"])
2134
+ route_refusal = QUERY_NO_SOURCE_MATCHED if not database else routed.get("refusal")
2135
+ if route_refusal:
2136
+ return {"status": "refused", "wording": route_refusal, "speakable": route_refusal,
2137
+ "artifacts": None, "routedTo": routed, "model": _model_identity(None)}
2138
  records = _semantic_records(session, database)
2139
+ if not routed["auto"]:
2140
+ from harness import semantic as sem
2141
+ label = next((row.get("label") for row in sem.topics().values()
2142
+ if row.get("grid") == database), database)
2143
+ routed.update({"label": _no_dashes(label), "reason": f"You selected {_no_dashes(label)}."})
2144
  approved = semantic_rt.approved_fields(records, database)
2145
  if not approved:
2146
+ return {**_draft_card(session, database, question, records), "routedTo": routed}
 
2147
  raw_requested = body.get("fieldKeys")
2148
  if raw_requested is not None and not isinstance(raw_requested, list):
2149
  raise err(400, "bad_request", "fieldKeys must be a list")
2150
  requested = list(dict.fromkeys(str(x) for x in (raw_requested or ()) if str(x)))
2151
  if not requested:
2152
+ return {**_clarification(session, database, question, approved), "routedTo": routed}
2153
  _verify_resume_token(body.get("resumeToken"), session, database, question,
2154
  approved, requested)
2155
  try:
 
2157
  except semantic_rt.SemanticError as exc:
2158
  raise err(403, "semantic_field_unapproved", str(exc))
2159
  snapshot = assistant_read_scope(session, database, fields=requested, filters=None)
2160
+ snapshot = _semantic_answer_snapshot(snapshot, records)
2161
  snapshot, formulas = _tool_formula_snapshot(body, snapshot, requested)
2162
  spec, planned_wording, provider = _tool_plan(question, body, snapshot, session)
2163
  if spec is None:
2164
  return {"status": "refused", "wording": planned_wording,
2165
  "speakable": planned_wording, "artifacts": None,
2166
+ "model": _model_identity(provider), "routedTo": routed}
2167
  view, refusal, refusal_code = _validate(spec, snapshot.get("fields") or selected)
2168
  if view is None:
2169
  raise err(400, refusal_code or "bad_query_artifact", refusal or "that Query is invalid")
 
2188
  "rows": [{key: row.get(key) for key in view["visible"]} for row in rows],
2189
  "groups": groups, "numeric": numeric,
2190
  "artifacts": artifacts, "semanticRouter": semantic_rt.router_projection(records, database),
2191
+ "model": _model_identity(provider), "routedTo": routed}
2192
 
2193
 
2194
  @router.post("/query/formulas/parse")
api/routes_sheet.py CHANGED
@@ -221,6 +221,11 @@ def _registry_assembly(session: Session, table_key: str):
221
  if table_key in memo:
222
  return memo[table_key]
223
  try:
 
 
 
 
 
224
  if table_key == "product_data":
225
  from routes_products import product_assembly
226
  memo[table_key] = product_assembly(session, scope="product",
@@ -248,6 +253,159 @@ def _registry_assembly(session: Session, table_key: str):
248
  "shown rather than shown from an older set of columns. Try again in a moment.")
249
 
250
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
251
  def _registry_mirror(session: Session, cur, table_key: str, metric_window, needed=None):
252
  """`(cursor, spec, columns)` for one compiled registry grid, at the cell's own period.
253
 
@@ -616,7 +774,7 @@ _CAP_CODES = ("sheet_too_many_cells", "workbook_too_large")
616
 
617
  #: The refusals that mean "that thing is not here", answered 404 so a client can tell a missing
618
  #: workbook from a malformed one without reading the sentence.
619
- _MISSING_CODES = ("workbook_missing", "tab_missing")
620
 
621
  #: ⛔⛔ C5's TWO SHARING REFUSALS ANSWER 403, AND THE TUPLE EXISTS BECAUSE THE FALL-THROUGH BELOW
622
  #: IS 400 (wave 51, W51-T25). Without this branch `workbook_read_only` would arrive as a NAMED 400
@@ -642,10 +800,8 @@ def _refuse(exc):
642
  nothing else, so the 409 and the 413 are raised here directly rather than by bending `err`,
643
  which would change the error shape of every route in the app.
644
 
645
- ⚠ `workbook_revision_stale` (the PUT door's code) ALSO ENDS `_stale` and still lands here. It
646
- carries no `rev_current`, so its body is unchanged and the client's existing PUT branch keeps
647
- working. ⭐ D-639's `index_revision_stale` lands here too and DOES carry one, so the rail's
648
- new guard needed no branch of its own: matching the family's spelling was the whole cost.
649
  """
650
  if exc.code in _CAP_CODES:
651
  return HTTPException(status_code=413,
@@ -739,13 +895,23 @@ def _role_on_workbook(session: Session, book):
739
  round because a workbook that has never been shared has no registry record at all. Asking
740
  `shares.role_for` alone would answer `None` for the person who created it.
741
  """
742
- from core import shares
743
- if session.admin:
744
- return "owner"
745
- me = str(session.uname or "").strip().lower()
746
- if me and _workbook_owner(book) == me:
747
- return "owner"
748
- return shares.role_for("workbook", book.get("id"), me, st=_store(session))
 
 
 
 
 
 
 
 
 
 
749
 
750
 
751
  def _require_editor(session: Session, book):
@@ -778,6 +944,8 @@ def _wire_shares(rows):
778
  out = []
779
  for row in rows or ():
780
  row = dict(row)
 
 
781
  row["shares"] = [{"who": s.get("who"), "role": wire_role(s.get("role"))}
782
  for s in (row.get("shares") or [])
783
  if wire_role(s.get("role"))]
@@ -809,7 +977,24 @@ def _binding_scope_key(session: Session, database: str, binding: dict) -> str:
809
  return sheets.scope_hash(database, wall_tree, (binding or {}).get("scope") or {})
810
 
811
 
812
- def _reresolved_for_reader(session: Session, book):
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
813
  """A COPY of `book` whose every bound cell was recomputed under THIS reader's scope (C5).
814
 
815
  ⛔⛔ IN MEMORY, AND IT MUST NEVER PERSIST. `refresh_sheet` runs the same loop and ends with
@@ -850,6 +1035,8 @@ def _reresolved_for_reader(session: Session, book):
850
  for addr, cell in (tab.get("cells") or {}).items():
851
  if not isinstance(cell, dict) or cell.get("kind") != "bound":
852
  continue
 
 
853
  touched += 1
854
  binding = cell.get("binding") or {}
855
  db = str(binding.get("database") or "")
@@ -861,18 +1048,14 @@ def _reresolved_for_reader(session: Session, book):
861
  seen_db[db] = exc
862
  failure = seen_db.get(db)
863
  if failure is not None:
864
- cell["state"] = "refused"
865
- cell["value"] = None
866
- cell["display"] = ""
867
- cell["refusal"] = {"code": "database_unreadable",
868
- "message": f"You cannot read {db}."}
869
  cell.pop("limits", None)
870
  continue
871
 
872
  cell_key = tab_ref + "!" + addr
873
  scope_key = _binding_scope_key(session, db, binding)
874
  hit = None
875
- if wid and scope_key:
876
  try:
877
  hit = sheets.cached_scope_value(store, wid, cell_key, scope_key)
878
  except Exception: # noqa: BLE001
@@ -890,17 +1073,15 @@ def _reresolved_for_reader(session: Session, book):
890
  continue
891
 
892
  try:
893
- out = _resolve_binding(session, binding, view_memo=view_memo)
894
  except HTTPException as exc:
895
  detail = getattr(exc, "detail", None) or {}
896
  info = detail.get("error") if isinstance(detail, dict) else None
897
- cell["state"] = "refused"
898
- cell["value"] = None
899
- cell["display"] = ""
900
- cell["refusal"] = {
901
- "code": (info or {}).get("code", "sheet_unresolved"),
902
- "message": (info or {}).get("message", "This cell could not be refreshed."),
903
- }
904
  continue
905
  cell["state"] = "fresh"
906
  cell["value"] = out["value"]
@@ -932,15 +1113,24 @@ def _reresolved_for_reader(session: Session, book):
932
  return fresh
933
 
934
 
 
 
 
 
 
 
 
935
  def _read_for(session: Session, book):
936
  """The workbook this session is entitled to see: stored for the owner, re-resolved otherwise.
937
 
938
- ⚠ A STRANGER IS A NON-OWNER AND TAKES THE SAME PATH, deliberately. Nothing in this file
939
- refuses a stranger's READ today (see the module note the measurement is booked under), so the
940
- fail-closed answer to "what should they be shown" is: whatever THEIR OWN scope computes,
941
- never the owner's cached numbers.
942
  """
 
943
  if _role_on_workbook(session, book) == "owner":
 
 
 
944
  return book
945
  return _reresolved_for_reader(session, book)
946
 
@@ -951,8 +1141,7 @@ def list_sheets(session: Session = Depends(require_session)):
951
 
952
  ⚠ C5's `shares` rides here as well as on `/sheets/index`, because the client reads both.
953
  """
954
- from core import sheets
955
- rows = sheets.project_shares(_store(session), sheets.list_workbooks(_store(session)))
956
  return {"workbooks": _wire_shares(rows)}
957
 
958
 
@@ -964,10 +1153,33 @@ def create_sheet(body: dict = Body(default=None), session: Session = Depends(req
964
  try:
965
  book = sheets.create_workbook(_store(session),
966
  name=raw.get("name") or "",
967
- created_by=session.uname)
 
968
  except sheets.SheetRefusal as exc:
969
  raise _refuse(exc)
970
- return {"workbook": book}
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
971
 
972
 
973
  # ── the rail's own document (wave 50, W50-T17) ─────────────────────���─────────────────────────
@@ -982,17 +1194,10 @@ def create_sheet(body: dict = Body(default=None), session: Session = Depends(req
982
  # ⚠ `Depends(require_session)` ONLY, like every other door in this file, and that is correct: the
983
  # Sheet is granted to everyone, and the rail's folders name no database.
984
  #
985
- # ⛔⛔ MEASURED IN W51-T25 AND BOOKED, NOT FIXED: THAT SENTENCE MEANS EVERY SIGNED IN ACCOUNT CAN
986
- # ALREADY READ EVERY WORKBOOK. Nothing in `core.sheets.list_workbooks` or `::read_index` filters by
987
- # owner, and `GET /sheets/{id}` walls on the session alone, so an account holding NO share reads a
988
- # stranger's workbook with **200**. Driven in `verify_sheet.py` section 5c with a third identity.
989
- # ⇒ `sharedWithMe` is therefore a PARTITION of a list everyone can already see, and C5's
990
- # viewer/editor wall guards the WRITE doors rather than closing a read that was never shut.
991
- # Refusing a stranger's read is not in W51-T25's `done-when` and widening it here would change the
992
- # behaviour of a door six other tickets are built on, so it is an amendment finding for the root.
993
- # ⚠ WHAT THIS TICKET DOES GUARANTEE MEANWHILE is the half that leaks data rather than existence:
994
- # a non-owner never receives the owner's stored numbers, because every cell-bearing read
995
- # re-resolves under the caller's own scope first.
996
 
997
 
998
  @router.get("/sheets/index")
@@ -1007,14 +1212,13 @@ def read_sheet_index(session: Session = Depends(require_session)):
1007
  can reach: a client cannot send a revision it was never given, and a server that never checks
1008
  one is a client stamping a number nobody reads.
1009
  """
1010
- from core import sheets
1011
- return {"index": _wire_index(sheets.read_index(_store(session), viewer=session.uname))}
1012
 
1013
 
1014
  @router.put("/sheets/index")
1015
  def write_sheet_index(body: dict = Body(default=None),
1016
  session: Session = Depends(require_session)):
1017
- """Replace the folder, placement and star strata wholesale, or refuse by name.
1018
 
1019
  ⭐ `repairs` RIDES BACK OUT. `core.sheets.save_folders` records every change it made to the
1020
  payload, and the rail says so in a sentence rather than letting a workbook reappear at the
@@ -1029,13 +1233,102 @@ def write_sheet_index(body: dict = Body(default=None),
1029
  """
1030
  from core import sheets
1031
  raw = dict(body or {})
 
 
 
 
 
 
 
 
 
 
1032
  try:
1033
- out = sheets.save_folders(_store(session), raw.get("folders"),
1034
- raw.get("itemFolders"), starred=raw.get("starred"),
1035
- expect_rev=raw.get("rev"))
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1036
  except sheets.SheetRefusal as exc:
1037
  raise _refuse(exc)
1038
- return {"index": _wire_index(out)}
1039
 
1040
 
1041
  # ── C4: the .xlsx IMPORT, which is a JOB and not a request (wave 51, W51-T24) ────────────────
@@ -1131,9 +1424,11 @@ def _run_import(job_id, path, filename, runtime, uname):
1131
  _job_set(job_id, report=report)
1132
 
1133
  stamp = _now_iso()
1134
- made = sheets.create_workbook(runtime, name=book.get("name") or "", created_by=uname)
 
1135
  saved = sheets.put_workbook(runtime, made["id"],
1136
- dict(book, refreshed_at=stamp, rev=made["rev"]))
 
1137
  _job_set(job_id, state="done", progress=1.0, workbook_id=saved["id"],
1138
  sentence=_import_sentence(report))
1139
  except sheet_import.ImportRefusal as exc:
@@ -1258,6 +1553,7 @@ def read_sheet(workbook_id: str, tabs: str = "",
1258
  from core import sheets
1259
  try:
1260
  book = sheets.get_workbook(_store(session), workbook_id)
 
1261
  except sheets.SheetRefusal as exc:
1262
  raise _refuse(exc)
1263
  if str(tabs or "").strip().lower() == "meta":
@@ -1294,6 +1590,7 @@ def read_sheet_tab(workbook_id: str, tab: str,
1294
  from core import sheets
1295
  try:
1296
  book = sheets.get_workbook(_store(session), workbook_id)
 
1297
  served = _read_for(session, book)
1298
  found = sheets.find_tab(served, tab)
1299
  except sheets.SheetRefusal as exc:
@@ -1319,9 +1616,8 @@ def patch_sheet_cells(workbook_id: str, tab: str, body: dict = Body(default=None
1319
  same patch has landed. A CAP breach is the opposite: it is a whole request 413, because the
1320
  document as a whole cannot be stored and there is no partial version of that.
1321
 
1322
- ⚠ `rev` IN THE BODY MAKES THE WRITE CONDITIONAL, exactly as it does on the PUT door. Omit it
1323
- and the last writer wins; send the `rev` the tab read and a concurrent change answers 409
1324
- `sheet_revision_stale` carrying `rev_current`, which is the revision to refetch.
1325
 
1326
  ⛔⛔ C5's READ-ONLY CHECK RUNS BEFORE THAT COMPARISON AND THE ORDER IS THE CONTRACT. A viewer
1327
  is refused 403 `workbook_read_only`; a viewer holding a stale rev is ALSO refused
@@ -1347,7 +1643,8 @@ def patch_sheet_cells(workbook_id: str, tab: str, body: dict = Body(default=None
1347
  expect = raw.get("rev")
1348
  try:
1349
  out = sheets.patch_cells(_store(session), workbook_id, tab, raw.get("cells"),
1350
- expect_rev=expect)
 
1351
  except sheets.SheetRefusal as exc:
1352
  raise _refuse(exc)
1353
  accepted = out.pop("accepted", [])
@@ -1396,11 +1693,32 @@ def _recomputed_tab_cells(session: Session, workbook_id: str, tab_ref: str, *, c
1396
  (found.get("cells") or {}).get(str(addr).strip().upper()) for addr in changes}
1397
  _recompute_formulas(book, fr, sheet_mod,
1398
  cache_key=(session.tenant, session.uname, workbook_id), changes=edits)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1399
  found = sheets.find_tab(book, tab_ref)
1400
  out = {}
1401
  for addr, cell in (found.get("cells") or {}).items():
1402
  if not isinstance(cell, dict) or (cell.get("kind") != "formula"
1403
- and "spill" not in cell and "spillParent" not in cell):
 
1404
  continue
1405
  row = {"value": cell.get("value"),
1406
  "display": "" if cell.get("value") is None else str(cell.get("display") or ""),
@@ -1435,10 +1753,12 @@ def write_sheet(workbook_id: str, body: dict = Body(default=None),
1435
  # `rev` inside the body IS the expectation; `core.sheets.put_workbook` reads it. This door
1436
  # does not translate it into a second argument, because two ways to say the same thing is
1437
  # how one of them ends up ignored.
1438
- book = sheets.put_workbook(_store(session), workbook_id, raw)
 
 
1439
  except sheets.SheetRefusal as exc:
1440
  raise _refuse(exc)
1441
- return {"workbook": book}
1442
 
1443
 
1444
  @router.delete("/sheets/{workbook_id}")
@@ -1509,6 +1829,14 @@ def share_sheet(workbook_id: str, body: dict = Body(default=None),
1509
  rec = shares.set_grants("workbook", book["id"], entries,
1510
  owner=(None if held.get("owner") else _workbook_owner(book)),
1511
  granter=session.uname, st=_store(session), expect=prior)
 
 
 
 
 
 
 
 
1512
  except shares.GrantsChanged as exc:
1513
  # ⚠ `grants_changed`, THE SAME CODE `routes_shares.py` ALREADY ANSWERS, and a plain 409
1514
  # rather than the `_stale` shape: there is no revision to hand back here, and a
@@ -1601,6 +1929,8 @@ def refresh_sheet(workbook_id: str, session: Session = Depends(require_session))
1601
  for addr, cell in (tab.get("cells") or {}).items():
1602
  if not isinstance(cell, dict) or cell.get("kind") != "bound":
1603
  continue
 
 
1604
  binding = cell.get("binding") or {}
1605
  db = str(binding.get("database") or "")
1606
  if db not in seen_db:
@@ -1611,22 +1941,19 @@ def refresh_sheet(workbook_id: str, session: Session = Depends(require_session))
1611
  seen_db[db] = exc
1612
  failure = seen_db.get(db)
1613
  if failure is not None:
1614
- cell["state"] = "refused"
1615
- cell["value"] = None
1616
- cell["refusal"] = {"code": "database_unreadable",
1617
- "message": f"You cannot read {db}."}
1618
  continue
1619
  try:
1620
- out = _resolve_binding(session, binding, view_memo=view_memo)
1621
  except HTTPException as exc:
1622
  detail = getattr(exc, "detail", None) or {}
1623
  info = detail.get("error") if isinstance(detail, dict) else None
1624
- cell["state"] = "refused"
1625
- cell["value"] = None
1626
- cell["refusal"] = {
1627
- "code": (info or {}).get("code", "sheet_unresolved"),
1628
- "message": (info or {}).get("message", "This cell could not be refreshed."),
1629
- }
1630
  continue
1631
  cell["state"] = "fresh"
1632
  cell["value"] = out["value"]
@@ -1652,16 +1979,20 @@ def refresh_sheet(workbook_id: str, session: Session = Depends(require_session))
1652
  _recompute_formulas(book, fr, sheet_mod)
1653
  recalc_ms = round((perf_counter() - started) * 1000.0, 1)
1654
  try:
 
1655
  saved = sheets.put_workbook(_store(session), workbook_id,
1656
  {"name": book.get("name"), "sheets": book.get("sheets"),
1657
- "refreshed_at": stamp, "rev": book.get("rev")})
 
 
 
1658
  except sheets.SheetRefusal as exc:
1659
  raise _refuse(exc)
1660
  # ⭐⭐ R3 / W52-T12: REFRESH IS THE ONE EVENT ALLOWED TO INVALIDATE A READER'S CACHED SCOPE
1661
  # VALUE, so it is the one door that clears the whole per-scope cache. Every reader's next GET
1662
  # recomputes once, under their own scope, and fills its own slot again from there.
1663
  sheets.clear_scope_cache(_store(session), workbook_id)
1664
- return {"workbook": saved, "recalc_ms": recalc_ms}
1665
 
1666
 
1667
  def _now_iso():
@@ -1916,7 +2247,150 @@ def _metric_column_aggregate(session, database, aggregate, window):
1916
  return column(raw)
1917
 
1918
 
1919
- def _resolve_binding(session: Session, raw_binding, *, today=None, view_memo=None):
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1920
  """Evaluate ONE C2 binding and return `{value, limits, sql_columns}`.
1921
 
1922
  The order below is the contract, not a preference:
@@ -1944,7 +2418,10 @@ def _resolve_binding(session: Session, raw_binding, *, today=None, view_memo=Non
1944
  raise err(400, exc.code, exc.message)
1945
 
1946
  database = _require_database(session, binding["database"])
1947
- window = sheet_mod.period_window(binding.get("period"))
 
 
 
1948
  # W59-T14 / R10: the per-database wall is contract step 1. A metric key must not be looked
1949
  # up or acted on before the caller passes it; the ordinary column guards below still apply.
1950
  if binding["aggregate"].get("metric_key"):
@@ -1966,12 +2443,16 @@ def _resolve_binding(session: Session, raw_binding, *, today=None, view_memo=Non
1966
  scope.get("view_name", ""), memo=view_memo)
1967
  else:
1968
  scope_nodes, scope_conj = _fe_parts(scope.get("filter"))
 
 
1969
  needed = (sheet_mod.filter_columns_named(scope_nodes)
1970
  | sheet_mod.filter_columns_named(wall_nodes))
1971
  if binding["aggregate"].get("field"):
1972
  needed.add(str(binding["aggregate"]["field"]))
1973
 
1974
  cur, spec, cols = _mirror(session)(database, metric_window=window, needed=needed)
 
 
1975
 
1976
  # ⛔ THE COLUMN IS CHECKED BEFORE THE SQL IS BUILT (W53-T13). Now that every tenant column is a
1977
  # real column of the materialised table, a `sum` over a text column reaches DuckDB as a binder
@@ -2069,7 +2550,15 @@ def preview_cell(workbook_id: str, addr: str, body: dict = Body(default=None),
2069
  field through the database's ordinary field writer; reusing a column needs no field write.
2070
  """
2071
  raw = dict(body or {})
2072
- out = _resolve_binding(session, raw.get("binding"))
 
 
 
 
 
 
 
 
2073
  return {"value": out["value"], "display": "" if out["value"] is None else str(out["value"]),
2074
  "limits": out["limits"]}
2075
 
@@ -2196,6 +2685,11 @@ def compute_cell(workbook_id: str, addr: str, body: dict = Body(default=None),
2196
  raise err(404, "cell_missing", f"Cell {key} is not filled in.")
2197
  if cell.get("kind") != "bound":
2198
  raise err(409, "cell_not_bound", f"Cell {key} is not bound to a database.")
 
 
 
 
 
2199
 
2200
  tab_ref = str((tab or {}).get("id") or (tab or {}).get("name") or "")
2201
  binding = cell.get("binding") or {}
@@ -2203,20 +2697,17 @@ def compute_cell(workbook_id: str, addr: str, body: dict = Body(default=None),
2203
  saved = dict(cell)
2204
  limits, resolved = [], None
2205
  try:
2206
- resolved = _resolve_binding(session, binding)
2207
  except HTTPException as exc:
2208
  #: The same per-cell refusal shape `refresh_sheet` writes, so a cell refused by this door
2209
  #: and a cell refused by that one read identically. It is PERSISTED rather than raised:
2210
  #: a cell left `stale` would shimmer forever, and the reason belongs on the cell.
2211
  detail = getattr(exc, "detail", None) or {}
2212
  info = detail.get("error") if isinstance(detail, dict) else None
2213
- saved["state"] = "refused"
2214
- saved["value"] = None
2215
- saved["display"] = ""
2216
- saved["refusal"] = {
2217
- "code": (info or {}).get("code", "sheet_unresolved"),
2218
- "message": (info or {}).get("message", "This cell could not be refreshed."),
2219
- }
2220
  else:
2221
  limits = resolved["limits"]
2222
  saved["state"] = "fresh"
@@ -2226,7 +2717,9 @@ def compute_cell(workbook_id: str, addr: str, body: dict = Body(default=None),
2226
  saved.pop("refusal", None)
2227
 
2228
  try:
2229
- landed = sheets.patch_cells(_store(session), workbook_id, tab_ref, {key: saved})
 
 
2230
  except sheets.SheetRefusal as exc:
2231
  raise _refuse(exc)
2232
 
@@ -2250,6 +2743,104 @@ def compute_cell(workbook_id: str, addr: str, body: dict = Body(default=None),
2250
  return {"cell": wire, "rev": landed.get("rev"), "tab": tab_ref, "addr": key}
2251
 
2252
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2253
  @router.get("/sheets/{workbook_id}/cells/{addr}/drill")
2254
  def drill_cell(workbook_id: str, addr: str, session: Session = Depends(require_session)):
2255
  """`{database, view_id?, filter?, record_ids, count}` for `LinkGridModal` (R6, R13, R17).
@@ -2285,6 +2876,9 @@ def drill_cell(workbook_id: str, addr: str, session: Session = Depends(require_s
2285
  if cell.get("kind") != "bound":
2286
  raise err(409, "literal_no_drill",
2287
  "A hand typed value has no rows behind it.")
 
 
 
2288
 
2289
  try:
2290
  binding = sheet_mod.clean_binding(cell.get("binding"))
@@ -2317,14 +2911,24 @@ def drill_cell(workbook_id: str, addr: str, session: Session = Depends(require_s
2317
  # `_metric_derived` at a month window is 27 ms of an 8,437 ms drill on `customer_data`. So the
2318
  # window is kept UNCONDITIONALLY, exactly as W53-T13 wrote it, and the saving is taken from
2319
  # the column set instead, where the time actually is.
 
 
 
 
 
2320
  cur, spec, cols = _mirror(session)(
2321
- database, metric_window=sheet_mod.period_window(binding.get("period")),
2322
  needed=(sheet_mod.filter_columns_named(scope_nodes)
2323
  | sheet_mod.filter_columns_named(wall_nodes)))
 
 
2324
 
2325
- count_sql, count_params, limits = sheet_mod.build_aggregate_query(
2326
- spec, {"aggregate": {"fn": "count"}}, cols, scope_nodes=scope_nodes,
2327
- scope_conj=scope_conj, wall_nodes=wall_nodes, wall_conj=wall_conj)
 
 
 
2328
  total = (cur.execute(count_sql, count_params).fetchone() or [0])[0]
2329
 
2330
  ids_sql, ids_params, _ = sheet_mod.build_ids_query(
@@ -2388,7 +2992,7 @@ def _asked_key(asked, home, fr):
2388
  @router.post("/sheets/{workbook_id}/resolve-bindings")
2389
  def resolve_bindings(workbook_id: str, body: dict = Body(default=None),
2390
  session: Session = Depends(require_session)):
2391
- """`{addr: value}` for the NAMED bound cells only (contract C1, ruling R2).
2392
 
2393
  ⭐⭐ THIS IS THE DOOR THE COMMIT PATH USES, AND ITS WHOLE POINT IS THAT IT IS SMALL.
2394
  Under R2 a formula evaluates in the browser the instant it is committed, so the only thing the
@@ -2404,20 +3008,13 @@ def resolve_bindings(workbook_id: str, body: dict = Body(default=None),
2404
  by THE EXACT STRING THE CALLER ASKED WITH, so a client looks its answer up under the
2405
  reference it already holds and never has to re-derive this file's key spelling.
2406
 
2407
- ⛔⛔ A KEY IS ABSENT WHEN NO BOUND CELL LIVES AT THAT ADDRESS, AND THAT IS THE ONE
2408
- DISTINCTION THE SHAPE CAN CARRY. An address that is empty, holds a typed literal or holds a
2409
- formula is not this door's business - lane A's evaluator already has those values in hand -
2410
- so asking for one costs nothing and answers nothing.
2411
-
2412
- ⛔⛔ AND HERE IS WHAT THE SHAPE CANNOT CARRY, STATED RATHER THAN HIDDEN: a bound cell whose
2413
- binding was REFUSED just now (a database wall, a row wall, a scope condition this database
2414
- cannot compile) comes back as `null`, which is the same thing an aggregate over zero matching
2415
- rows returns. The cause is real and this door knows it; `{addr: value}` has nowhere to put it.
2416
- `POST /sheets/{id}/refresh` and `GET /sheets/{id}` both still carry the full
2417
- `{code, message}` refusal on the cell, so nothing is lost from the document - what is lost is
2418
- the client's ability to paint THIS cell's refusal at commit time. Raised as a dated PRD
2419
- amendment rather than fixed by widening the shape on one lane's authority, because lane A is
2420
- the consumer and a second shape agreed by nobody is worse than a known gap.
2421
 
2422
  ⛔ ONE BAD CELL NEVER FAILS THE BATCH, exactly as in `refresh_sheet`'s bound-cell loop: a
2423
  refusal is caught per cell and the siblings still answer.
@@ -2479,16 +3076,21 @@ def resolve_bindings(workbook_id: str, body: dict = Body(default=None),
2479
  if cell is None:
2480
  continue
2481
  binding = cell.get("binding") or {}
2482
- slot = json.dumps(binding, sort_keys=True, default=str)
 
2483
  if slot not in by_binding:
2484
  try:
2485
- by_binding[slot] = _resolve_binding(session, binding,
2486
- view_memo=view_memo)["value"]
2487
- except HTTPException:
2488
- # ⛔ SWALLOWED HERE AND NOWHERE ELSE. See the docstring: the flat body has no
2489
- # room for a cause, and failing the whole batch on one walled cell would take
2490
- # every sibling's number down with it. The same refusal still travels in full on
2491
- # the refresh and read doors, which is where the cell's own sentence comes from.
2492
- by_binding[slot] = None
 
 
 
 
2493
  answer[label] = by_binding[slot]
2494
  return answer
 
221
  if table_key in memo:
222
  return memo[table_key]
223
  try:
224
+ # W62 C2: a placed ut_* view needs its own field vocabulary, without fetching its rows.
225
+ if table_key.startswith("ut_"):
226
+ from routes_tables import ut_assembly
227
+ memo[table_key] = ut_assembly(session, table_key, consume_corrections=False, with_rows=False)
228
+ return memo[table_key]
229
  if table_key == "product_data":
230
  from routes_products import product_assembly
231
  memo[table_key] = product_assembly(session, scope="product",
 
253
  "shown rather than shown from an older set of columns. Try again in a moment.")
254
 
255
 
256
+ def _linked_customer_bridges(database):
257
+ """Only declared reverse customer edges with an unconditional, served relation."""
258
+ from core.relations import BRIDGES
259
+ return [bridge for bridge in BRIDGES if bridge.get("target") == database
260
+ and bridge.get("source") == "customer_data" and bridge.get("served")
261
+ and bridge.get("cardinality") == "one_to_many" and not bridge.get("where")]
262
+
263
+
264
+ def _linked_customer_fields(session, bridge):
265
+ from core import perm_scope
266
+ _require_database(session, bridge["source"])
267
+ g = _registry_assembly(session, bridge["source"])
268
+ fields = list(g.get("fields") or [])
269
+ hidden = perm_scope.hidden_keys(session.user, bridge["source"], fields, st=session.runtime)
270
+ # Only values the server can read per record. A browser formula or linked rollup needs
271
+ # another evaluator; offering it here would make a missing value look like a blank value.
272
+ fields = [f for f in fields if f.get("key") not in hidden
273
+ and f.get("type") not in {"formula", "link", "lookup", "rollup", "metric"}
274
+ and not f.get("measure")]
275
+ return g, fields
276
+
277
+
278
+ def _linked_relation_column(session, bridge, cur, spec, columns):
279
+ from core import perm_scope
280
+ from modules import sheet as sheet_mod
281
+ key = str(bridge.get("target_col") or "")
282
+ subject = bridge["key"] + ":" + (key or "(missing column)")
283
+ hidden = perm_scope.hidden_keys(session.user, bridge["target"], spec.get("fields") or [],
284
+ st=session.runtime)
285
+ if key in hidden:
286
+ raise sheet_mod.SheetRefusal("linked_filter_unreadable", subject,
287
+ "the relation column is hidden from this account.")
288
+ # The catalogue owns expressions such as sol.order_partner_id and aml.partner_id.
289
+ # The declaration supplies the key, and is a fallback only when the catalogue has none.
290
+ if (columns.get(key) or {}).get("sql"):
291
+ return key
292
+ if not re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]*", key):
293
+ raise sheet_mod.SheetRefusal("linked_filter_unreadable", subject,
294
+ "the link does not declare a readable relation column.")
295
+ source = spec.get("from_sql") or spec.get("table")
296
+ try:
297
+ cur.execute(f'SELECT {key} FROM {source} LIMIT 0')
298
+ except Exception:
299
+ raise sheet_mod.SheetRefusal("linked_filter_unreadable", subject,
300
+ "the declared relation column is unavailable on this database.")
301
+ columns[key] = {"sql": key, "type": "int", "aggregate": False}
302
+ return key
303
+
304
+
305
+ @router.get("/sheets/filter-fields/{database}")
306
+ def linked_filter_fields(database: str, session: Session = Depends(require_session)):
307
+ """The linked Filter offer, through the same field wall as the related grid."""
308
+ from modules import sheet as sheet_mod
309
+ database = _require_database(session, database)
310
+ bridges = _linked_customer_bridges(database)
311
+ if not bridges:
312
+ return {"fields": []}
313
+ cur, spec, columns = _mirror(session)(database)
314
+ columns = dict(columns)
315
+ offered = []
316
+ for bridge in bridges:
317
+ try:
318
+ _linked_relation_column(session, bridge, cur, spec, columns)
319
+ _g, fields = _linked_customer_fields(session, bridge)
320
+ except HTTPException as exc:
321
+ if exc.status_code == 403:
322
+ continue
323
+ raise
324
+ except sheet_mod.SheetRefusal:
325
+ continue
326
+ link_label = f'{bridge.get("source_label") or bridge["source"]} via {bridge.get("label") or bridge["key"]}'
327
+ for field in fields:
328
+ # Copy the existing field vocabulary, including options. No customer values or
329
+ # hidden definitions travel on the picker wire.
330
+ offered.append({**field,
331
+ "key": f'{sheet_mod.LINKED_FILTER_PREFIX}{bridge["key"]}:{field["key"]}',
332
+ "label": f'{link_label}: {field.get("label") or field["key"]}'})
333
+ return {"fields": offered}
334
+
335
+
336
+ def _linked_filter_scope(session, database, nodes, cur, spec, columns, *, today=None):
337
+ """Read linked values in Python, then give the relation-blind compiler flat predicates."""
338
+ from modules import sheet as sheet_mod
339
+ if not sheet_mod.linked_filter_ids(nodes):
340
+ return nodes
341
+ import aios_grid
342
+ from core import perm_scope, shared_overlay, view_templates
343
+ from harness import filter_eval, filter_sql
344
+
345
+ bridges = {b["key"]: b for b in _linked_customer_bridges(database)}
346
+ related = {}
347
+
348
+ def resolve(leaf):
349
+ subject = leaf["colId"]
350
+ bridge_key, field_key = sheet_mod.linked_field_parts(subject)
351
+ bridge = bridges.get(bridge_key)
352
+ if bridge is None:
353
+ raise sheet_mod.SheetRefusal("linked_filter_unreadable", subject,
354
+ "this database no longer has that available customer link.")
355
+ try:
356
+ relation_column = _linked_relation_column(session, bridge, cur, spec, columns)
357
+ if bridge_key not in related:
358
+ g, fields = _linked_customer_fields(session, bridge)
359
+ # Use the rows door's wall, including its shared-field enrichment. The pool's
360
+ # identity is pid; it must never be confused with the orders' partner_id.
361
+ rows = perm_scope.apply_row_scope(g["rows_src"], session.user, bridge["source"],
362
+ aios_grid.FIELDS, st=session.runtime)
363
+ related[bridge_key] = (g, fields, rows)
364
+ g, fields, source_rows = related[bridge_key]
365
+ except HTTPException as exc:
366
+ if exc.status_code != 403:
367
+ raise
368
+ raise sheet_mod.SheetRefusal("linked_filter_unreadable", subject,
369
+ "this account cannot read the linked customer database.")
370
+ field = next((f for f in fields if f.get("key") == field_key), None)
371
+ if field is None:
372
+ raise sheet_mod.SheetRefusal("linked_filter_unreadable", subject,
373
+ "the field is hidden, removed, or has no readable per-record value.")
374
+ local = {**leaf, "colId": field_key}
375
+ # Validate before evaluating even an empty pool, so malformed/stale leaves refuse
376
+ # rather than looking like a genuine zero. Cell slots were resolved by W60-T06 above.
377
+ probe_columns = {field_key: {"sql": "linked_value", "type": sheet_mod.mirror_field_type(field)}}
378
+ try:
379
+ compiled = sheet_mod.compile_scope([local], "and", probe_columns, today=today)
380
+ if (local.get("op") not in filter_sql.FILTER_OPS or compiled is None
381
+ or field_key not in compiled.columns_used):
382
+ raise ValueError("unusable linked condition")
383
+ except (ValueError, sheet_mod.SheetRefusal):
384
+ raise sheet_mod.SheetRefusal("linked_filter_unreadable", subject,
385
+ "its value or operator cannot be applied to this field.")
386
+ storage_key = view_templates.workspace_key(bridge["source"])
387
+ try:
388
+ definitions = shared_overlay.fields(storage_key, st=session.runtime)
389
+ rows = aios_grid.rows_from_pool(source_rows, fields, (g.get("ws") or {}).get("overlays"),
390
+ derived=g.get("derived"))
391
+ if field_key in definitions:
392
+ cells = shared_overlay.cells(storage_key, [r["pid"] for r in source_rows], st=session.runtime)
393
+ rows = [{**row, field_key: (cells.get(str(int(row["pid"]))) or {}).get(field_key, "")}
394
+ for row in rows]
395
+ ctx = filter_eval.EvalCtx(today=today)
396
+ ids = [row[bridge["source_col"]] for row in rows
397
+ if filter_eval.permits({"conj": "and", "nodes": [local]}, row, fields, ctx)]
398
+ return sheet_mod.linked_id_predicate(relation_column, ids)
399
+ except Exception:
400
+ raise sheet_mod.SheetRefusal("linked_filter_unreadable", subject,
401
+ "the linked customer values could not be read. Retry after the data is available.")
402
+
403
+ try:
404
+ return sheet_mod.rewrite_linked_filters(nodes, resolve)
405
+ except sheet_mod.SheetRefusal as exc:
406
+ raise err(409, exc.code, exc.message)
407
+
408
+
409
  def _registry_mirror(session: Session, cur, table_key: str, metric_window, needed=None):
410
  """`(cursor, spec, columns)` for one compiled registry grid, at the cell's own period.
411
 
 
774
 
775
  #: The refusals that mean "that thing is not here", answered 404 so a client can tell a missing
776
  #: workbook from a malformed one without reading the sentence.
777
+ _MISSING_CODES = ("workbook_missing", "folder_missing", "tab_missing", "unknown_sheet_template")
778
 
779
  #: ⛔⛔ C5's TWO SHARING REFUSALS ANSWER 403, AND THE TUPLE EXISTS BECAUSE THE FALL-THROUGH BELOW
780
  #: IS 400 (wave 51, W51-T25). Without this branch `workbook_read_only` would arrive as a NAMED 400
 
800
  nothing else, so the 409 and the 413 are raised here directly rather than by bending `err`,
801
  which would change the error shape of every route in the app.
802
 
803
+ Workbook, retained cell history and folder index refusals all carry the current revision
804
+ through the same existing envelope.
 
 
805
  """
806
  if exc.code in _CAP_CODES:
807
  return HTTPException(status_code=413,
 
895
  round because a workbook that has never been shared has no registry record at all. Asking
896
  `shares.role_for` alone would answer `None` for the person who created it.
897
  """
898
+ from core import sheets
899
+ return sheets.workbook_role(_store(session), book, session.uname, is_admin=session.admin)
900
+
901
+
902
+ def _require_reader(session: Session, book):
903
+ """R28: use the same predicate as the index; hidden and missing ids answer identically."""
904
+ from core import sheets
905
+ if not sheets.may_read_workbook(book, _role_on_workbook(session, book)):
906
+ raise _refuse(sheets.SheetRefusal("workbook_missing", "That workbook no longer exists."))
907
+
908
+
909
+ def _index_for(session: Session):
910
+ """C2: every list response uses the index's sole visibility filter and role helper."""
911
+ from core import sheets
912
+ # The index already owns the bulk grant join. Passing the per-object route helper here
913
+ # converts that one read back into N full registry reads and made 50-row lists miss budget.
914
+ return sheets.read_index(_store(session), viewer=session.uname, is_admin=session.admin)
915
 
916
 
917
  def _require_editor(session: Session, book):
 
944
  out = []
945
  for row in rows or ():
946
  row = dict(row)
947
+ if "role" in row:
948
+ row["role"] = "owner" if row["role"] == "owner" else wire_role(row["role"])
949
  row["shares"] = [{"who": s.get("who"), "role": wire_role(s.get("role"))}
950
  for s in (row.get("shares") or [])
951
  if wire_role(s.get("role"))]
 
977
  return sheets.scope_hash(database, wall_tree, (binding or {}).get("scope") or {})
978
 
979
 
980
+ def _has_ending(binding):
981
+ return (isinstance(binding, dict) and isinstance(binding.get("period"), dict)
982
+ and "ending" in binding["period"])
983
+
984
+
985
+ def _has_cell_inputs(binding):
986
+ from modules import sheet as sheet_mod
987
+ if _has_ending(binding):
988
+ return True
989
+ scope = binding.get("scope") if isinstance(binding, dict) else None
990
+ nodes, _ = _fe_parts(scope.get("filter") if isinstance(scope, dict) else None)
991
+ # Linked values and related-table grants can change independently of the orders' cache key.
992
+ if sheet_mod.linked_filter_ids(nodes):
993
+ return True
994
+ return any(sheet_mod.filter_cell_nodes(nodes))
995
+
996
+
997
+ def _reresolved_for_reader(session: Session, book, *, endings_only=False):
998
  """A COPY of `book` whose every bound cell was recomputed under THIS reader's scope (C5).
999
 
1000
  ⛔⛔ IN MEMORY, AND IT MUST NEVER PERSIST. `refresh_sheet` runs the same loop and ends with
 
1035
  for addr, cell in (tab.get("cells") or {}).items():
1036
  if not isinstance(cell, dict) or cell.get("kind") != "bound":
1037
  continue
1038
+ if endings_only and not _has_cell_inputs(cell.get("binding")):
1039
+ continue
1040
  touched += 1
1041
  binding = cell.get("binding") or {}
1042
  db = str(binding.get("database") or "")
 
1048
  seen_db[db] = exc
1049
  failure = seen_db.get(db)
1050
  if failure is not None:
1051
+ cell.update(_binding_refusal("database_unreadable", f"You cannot read {db}."))
 
 
 
 
1052
  cell.pop("limits", None)
1053
  continue
1054
 
1055
  cell_key = tab_ref + "!" + addr
1056
  scope_key = _binding_scope_key(session, db, binding)
1057
  hit = None
1058
+ if wid and scope_key and not _has_cell_inputs(binding):
1059
  try:
1060
  hit = sheets.cached_scope_value(store, wid, cell_key, scope_key)
1061
  except Exception: # noqa: BLE001
 
1073
  continue
1074
 
1075
  try:
1076
+ out = _resolve_cell_binding(session, binding, fresh, tab_ref, view_memo=view_memo)
1077
  except HTTPException as exc:
1078
  detail = getattr(exc, "detail", None) or {}
1079
  info = detail.get("error") if isinstance(detail, dict) else None
1080
+ cell.update(_binding_refusal(
1081
+ (info or {}).get("code", "sheet_unresolved"),
1082
+ (info or {}).get("message", "This cell could not be refreshed."),
1083
+ ))
1084
+ cell.pop("limits", None)
 
 
1085
  continue
1086
  cell["state"] = "fresh"
1087
  cell["value"] = out["value"]
 
1113
  return fresh
1114
 
1115
 
1116
+ def _response_workbook(book):
1117
+ """Omit private write history from response tabs without changing the stored document."""
1118
+ return {**book, "sheets": [{key: value for key, value in tab.items()
1119
+ if key not in ("writes", "writesFrom")}
1120
+ for tab in book.get("sheets") or []]}
1121
+
1122
+
1123
  def _read_for(session: Session, book):
1124
  """The workbook this session is entitled to see: stored for the owner, re-resolved otherwise.
1125
 
1126
+ W60-T09, 2026-09-14: R28 checks visibility before this projection. Admitted legacy readers
1127
+ and grantees still receive their own scoped numbers, never the owner's cached numbers.
 
 
1128
  """
1129
+ book = _response_workbook(book)
1130
  if _role_on_workbook(session, book) == "owner":
1131
+ if any(_has_cell_inputs(cell.get("binding"))
1132
+ for tab in book.get("sheets") or [] for cell in (tab.get("cells") or {}).values()):
1133
+ return _reresolved_for_reader(session, book, endings_only=True)
1134
  return book
1135
  return _reresolved_for_reader(session, book)
1136
 
 
1141
 
1142
  ⚠ C5's `shares` rides here as well as on `/sheets/index`, because the client reads both.
1143
  """
1144
+ rows = _index_for(session)["workbooks"]
 
1145
  return {"workbooks": _wire_shares(rows)}
1146
 
1147
 
 
1153
  try:
1154
  book = sheets.create_workbook(_store(session),
1155
  name=raw.get("name") or "",
1156
+ created_by=session.uname,
1157
+ on_duplicate="refuse")
1158
  except sheets.SheetRefusal as exc:
1159
  raise _refuse(exc)
1160
+ return {"workbook": _response_workbook(book)}
1161
+
1162
+
1163
+ @router.post("/sheets/from-template/{key}", status_code=201)
1164
+ def create_sheet_from_template(key: str, body: dict = Body(default=None),
1165
+ session: Session = Depends(require_session)):
1166
+ """Create a private workbook from a tenant-saved Sheet template."""
1167
+ from core import sheets, sheet_templates
1168
+ book = None
1169
+ try:
1170
+ record = sheet_templates.saved_template(_store(session), key)
1171
+ for database in (record or {}).get("databases", []):
1172
+ _require_database(session, database)
1173
+ book = sheet_templates.apply_saved_template(
1174
+ _store(session), key, name=(body or {}).get("name") or "", created_by=session.uname)
1175
+ # Owners read persisted results, so refresh under this maker before their first open.
1176
+ return {"workbook": _response_workbook(refresh_sheet(book["id"], session=session)["workbook"])}
1177
+ except Exception as exc:
1178
+ if book is not None:
1179
+ sheets.delete_workbook(_store(session), book["id"])
1180
+ if isinstance(exc, sheets.SheetRefusal):
1181
+ raise _refuse(exc)
1182
+ raise
1183
 
1184
 
1185
  # ── the rail's own document (wave 50, W50-T17) ─────────────────────���─────────────────────────
 
1194
  # ⚠ `Depends(require_session)` ONLY, like every other door in this file, and that is correct: the
1195
  # Sheet is granted to everyone, and the rail's folders name no database.
1196
  #
1197
+ # W60-T09, 2026-09-14: R28 supersedes the W51-T25 amendment finding. Both lists and all three
1198
+ # workbook read shapes use core.sheets.may_read_workbook with _role_on_workbook. New workbooks
1199
+ # are private to their creator and grantees; administrators see all. Unmarked or creatorless
1200
+ # documents retain legacy visibility. Admitted non-owners still re-resolve under their own scope.
 
 
 
 
 
 
 
1201
 
1202
 
1203
  @router.get("/sheets/index")
 
1212
  can reach: a client cannot send a revision it was never given, and a server that never checks
1213
  one is a client stamping a number nobody reads.
1214
  """
1215
+ return {"index": _wire_index(_index_for(session))}
 
1216
 
1217
 
1218
  @router.put("/sheets/index")
1219
  def write_sheet_index(body: dict = Body(default=None),
1220
  session: Session = Depends(require_session)):
1221
+ """D-639 / W62-T25: replace the rail wholesale or merge a placements-only delta.
1222
 
1223
  ⭐ `repairs` RIDES BACK OUT. `core.sheets.save_folders` records every change it made to the
1224
  payload, and the rail says so in a sentence rather than letting a workbook reappear at the
 
1233
  """
1234
  from core import sheets
1235
  raw = dict(body or {})
1236
+ # This document is tenant-shared while its workbook rows are object-scoped. A response-only
1237
+ # projection is not a write wall: without carrying these ids into the mutation, a stranger can
1238
+ # move a hidden workbook by guessing its id, or save their empty projection wholesale and wipe
1239
+ # the owner's folders/stars. Read the same projected index the GET returns, then make the core
1240
+ # transaction preserve every stratum outside that projection.
1241
+ visible = _index_for(session)
1242
+ visible_ids = {str(row.get("id")) for row in (visible.get("workbooks") or [])
1243
+ if isinstance(row, dict) and row.get("id")}
1244
+ visible_folder_ids = {str(row.get("id")) for row in (visible.get("folders") or [])
1245
+ if isinstance(row, dict) and row.get("id")}
1246
  try:
1247
+ if "placements" in raw:
1248
+ if set(raw) != {"placements"}:
1249
+ raise sheets.SheetRefusal(
1250
+ "index_request_mixed",
1251
+ "Send placements alone, or send the full Sheet rail with its revision.")
1252
+ if (not session.admin and isinstance(raw.get("placements"), dict)
1253
+ and any(str(wid) not in visible_ids for wid in raw["placements"])):
1254
+ # Hidden and nonexistent ids are deliberately identical: this write door is not
1255
+ # an oracle for private workbook existence.
1256
+ raise sheets.SheetRefusal("workbook_missing",
1257
+ "That workbook no longer exists.")
1258
+ out = sheets.merge_placements(_store(session), raw["placements"])
1259
+ else:
1260
+ folders = raw.get("folders")
1261
+ item_folders = raw.get("itemFolders")
1262
+ if not session.admin:
1263
+ submitted_workbooks = (set(str(wid) for wid in item_folders)
1264
+ if isinstance(item_folders, dict) else set())
1265
+ submitted_stars = (set(str(wid) for wid in raw.get("starred"))
1266
+ if isinstance(raw.get("starred"), list) else set())
1267
+ if (submitted_workbooks | submitted_stars) - visible_ids:
1268
+ raise sheets.SheetRefusal("workbook_missing",
1269
+ "That workbook no longer exists.")
1270
+ # A wholesale snapshot cannot otherwise distinguish "create this client id" from
1271
+ # "rename the private folder whose id I guessed". Current clients declare their
1272
+ # newly-created temporary ids. Every created id is replaced server-side, whether
1273
+ # it collides with a hidden folder or with nothing, so the response is not an
1274
+ # existence oracle.
1275
+ #
1276
+ # Older clients predate `createdFolderIds`. Preserve their one safe, observable
1277
+ # creation shape: at least one visible workbook is filed into the submitted new
1278
+ # tree. In that legacy shape every unknown submitted id is treated as temporary
1279
+ # and replaced, including empty siblings and ancestors. A bare undeclared id is
1280
+ # still refused identically whether it is hidden or nonexistent, so the repair does
1281
+ # not re-open the hidden-folder oracle while the old D-639 save door keeps working.
1282
+ declared_creates = "createdFolderIds" in raw
1283
+ created = raw.get("createdFolderIds", [])
1284
+ if (not isinstance(created, list)
1285
+ or any(not isinstance(fid, str) or not fid for fid in created)):
1286
+ raise sheets.SheetRefusal(
1287
+ "folder_create_ids_invalid",
1288
+ "New folders must declare their temporary identifiers.")
1289
+ created_ids = set(created)
1290
+ incoming = [dict(row) if isinstance(row, dict) else row
1291
+ for row in folders] if isinstance(folders, list) else []
1292
+ incoming_ids = {str(row.get("id")) for row in incoming
1293
+ if isinstance(row, dict) and row.get("id")}
1294
+ unknown = incoming_ids - visible_folder_ids
1295
+ if declared_creates:
1296
+ if unknown - created_ids or created_ids - unknown:
1297
+ raise sheets.SheetRefusal("folder_missing",
1298
+ "That folder no longer exists.")
1299
+ elif unknown:
1300
+ filed_into_unknown = {
1301
+ str(fid) for wid, fid in (item_folders.items()
1302
+ if isinstance(item_folders, dict) else [])
1303
+ if str(wid) in visible_ids and fid is not None and str(fid) in unknown
1304
+ }
1305
+ if not filed_into_unknown:
1306
+ raise sheets.SheetRefusal("folder_missing",
1307
+ "That folder no longer exists.")
1308
+ created_ids = unknown
1309
+ replacements = {fid: "f" + uuid.uuid4().hex for fid in created_ids}
1310
+ if replacements:
1311
+ for row in incoming:
1312
+ if not isinstance(row, dict):
1313
+ continue
1314
+ row["id"] = replacements.get(str(row.get("id")), row.get("id"))
1315
+ if row.get("parent") is not None:
1316
+ row["parent"] = replacements.get(str(row["parent"]), row["parent"])
1317
+ if isinstance(item_folders, dict):
1318
+ item_folders = {
1319
+ wid: replacements.get(str(fid), fid)
1320
+ for wid, fid in item_folders.items()
1321
+ }
1322
+ folders = incoming
1323
+ out = sheets.save_folders(_store(session), folders,
1324
+ item_folders, starred=raw.get("starred"),
1325
+ expect_rev=raw.get("rev"),
1326
+ writable_ids=None if session.admin else visible_ids,
1327
+ writable_folder_ids=(None if session.admin
1328
+ else visible_folder_ids))
1329
  except sheets.SheetRefusal as exc:
1330
  raise _refuse(exc)
1331
+ return {"index": _wire_index({**_index_for(session), "repairs": out.get("repairs", [])})}
1332
 
1333
 
1334
  # ── C4: the .xlsx IMPORT, which is a JOB and not a request (wave 51, W51-T24) ────────────────
 
1424
  _job_set(job_id, report=report)
1425
 
1426
  stamp = _now_iso()
1427
+ made = sheets.create_workbook(runtime, name=book.get("name") or "", created_by=uname,
1428
+ on_duplicate="suffix")
1429
  saved = sheets.put_workbook(runtime, made["id"],
1430
+ dict(book, refreshed_at=stamp, rev=made["rev"]), actor=uname,
1431
+ lossless=True)
1432
  _job_set(job_id, state="done", progress=1.0, workbook_id=saved["id"],
1433
  sentence=_import_sentence(report))
1434
  except sheet_import.ImportRefusal as exc:
 
1553
  from core import sheets
1554
  try:
1555
  book = sheets.get_workbook(_store(session), workbook_id)
1556
+ _require_reader(session, book)
1557
  except sheets.SheetRefusal as exc:
1558
  raise _refuse(exc)
1559
  if str(tabs or "").strip().lower() == "meta":
 
1590
  from core import sheets
1591
  try:
1592
  book = sheets.get_workbook(_store(session), workbook_id)
1593
+ _require_reader(session, book)
1594
  served = _read_for(session, book)
1595
  found = sheets.find_tab(served, tab)
1596
  except sheets.SheetRefusal as exc:
 
1616
  same patch has landed. A CAP breach is the opposite: it is a whole request 413, because the
1617
  document as a whole cannot be stored and there is no partial version of that.
1618
 
1619
+ With retained history, a stale revision merges by address and the later value wins;
1620
+ `conflicts` names other people's overlapping writes, and only missing history refuses.
 
1621
 
1622
  ⛔⛔ C5's READ-ONLY CHECK RUNS BEFORE THAT COMPARISON AND THE ORDER IS THE CONTRACT. A viewer
1623
  is refused 403 `workbook_read_only`; a viewer holding a stale rev is ALSO refused
 
1643
  expect = raw.get("rev")
1644
  try:
1645
  out = sheets.patch_cells(_store(session), workbook_id, tab, raw.get("cells"),
1646
+ expect_rev=expect, actor=session.uname,
1647
+ actor_name=session.user.get("name") or session.uname)
1648
  except sheets.SheetRefusal as exc:
1649
  raise _refuse(exc)
1650
  accepted = out.pop("accepted", [])
 
1693
  (found.get("cells") or {}).get(str(addr).strip().upper()) for addr in changes}
1694
  _recompute_formulas(book, fr, sheet_mod,
1695
  cache_key=(session.tenant, session.uname, workbook_id), changes=edits)
1696
+ # Date and filter-cell edits also change database totals. Recompute all readers, then formulas.
1697
+ ending_cells = set()
1698
+ for source in book.get("sheets") or []:
1699
+ for addr, cell in (source.get("cells") or {}).items():
1700
+ binding = cell.get("binding") or {}
1701
+ if cell.get("kind") != "bound" or not _has_cell_inputs(binding):
1702
+ continue
1703
+ try:
1704
+ result = _resolve_cell_binding(session, binding, book, source.get("id"))
1705
+ cell.update(value=result["value"], display="" if result["value"] is None else str(result["value"]), state="fresh")
1706
+ cell.pop("refusal", None)
1707
+ except HTTPException as exc:
1708
+ info = ((exc.detail or {}).get("error") or {})
1709
+ cell.update(_binding_refusal(
1710
+ info.get("code", "ending_cell_unreadable"),
1711
+ info.get("message", "The Ending date could not be read. Choose a valid date."),
1712
+ ))
1713
+ ending_cells.add((source.get("id"), addr))
1714
+ if ending_cells:
1715
+ _recompute_formulas(book, fr, sheet_mod)
1716
  found = sheets.find_tab(book, tab_ref)
1717
  out = {}
1718
  for addr, cell in (found.get("cells") or {}).items():
1719
  if not isinstance(cell, dict) or (cell.get("kind") != "formula"
1720
+ and "spill" not in cell and "spillParent" not in cell
1721
+ and (found.get("id"), addr) not in ending_cells):
1722
  continue
1723
  row = {"value": cell.get("value"),
1724
  "display": "" if cell.get("value") is None else str(cell.get("display") or ""),
 
1753
  # `rev` inside the body IS the expectation; `core.sheets.put_workbook` reads it. This door
1754
  # does not translate it into a second argument, because two ways to say the same thing is
1755
  # how one of them ends up ignored.
1756
+ book = sheets.put_workbook(_store(session), workbook_id, raw,
1757
+ actor=session.uname,
1758
+ actor_name=session.user.get("name") or session.uname)
1759
  except sheets.SheetRefusal as exc:
1760
  raise _refuse(exc)
1761
+ return {"workbook": _response_workbook(book)}
1762
 
1763
 
1764
  @router.delete("/sheets/{workbook_id}")
 
1829
  rec = shares.set_grants("workbook", book["id"], entries,
1830
  owner=(None if held.get("owner") else _workbook_owner(book)),
1831
  granter=session.uname, st=_store(session), expect=prior)
1832
+ # A successful explicit share narrows legacy visibility; ordinary saves never do.
1833
+ if "access" not in book:
1834
+ # W60-T09, 2026-09-14: the first share decides VISIBILITY, making the owner gate load bearing.
1835
+ # A non-owner must not reach this narrowing on an ownerless legacy workbook
1836
+ # and lock themselves out by sharing it.
1837
+ sheets.narrow_workbook(_store(session), book["id"])
1838
+ except sheets.SheetRefusal as exc:
1839
+ raise _refuse(exc)
1840
  except shares.GrantsChanged as exc:
1841
  # ⚠ `grants_changed`, THE SAME CODE `routes_shares.py` ALREADY ANSWERS, and a plain 409
1842
  # rather than the `_stale` shape: there is no revision to hand back here, and a
 
1929
  for addr, cell in (tab.get("cells") or {}).items():
1930
  if not isinstance(cell, dict) or cell.get("kind") != "bound":
1931
  continue
1932
+ if _is_view_binding(cell):
1933
+ continue # C2: a placed view refreshes through its own door
1934
  binding = cell.get("binding") or {}
1935
  db = str(binding.get("database") or "")
1936
  if db not in seen_db:
 
1941
  seen_db[db] = exc
1942
  failure = seen_db.get(db)
1943
  if failure is not None:
1944
+ cell.update(_binding_refusal("database_unreadable", f"You cannot read {db}."))
1945
+ cell.pop("limits", None)
 
 
1946
  continue
1947
  try:
1948
+ out = _resolve_cell_binding(session, binding, book, tab.get("id"), view_memo=view_memo)
1949
  except HTTPException as exc:
1950
  detail = getattr(exc, "detail", None) or {}
1951
  info = detail.get("error") if isinstance(detail, dict) else None
1952
+ cell.update(_binding_refusal(
1953
+ (info or {}).get("code", "sheet_unresolved"),
1954
+ (info or {}).get("message", "This cell could not be refreshed."),
1955
+ ))
1956
+ cell.pop("limits", None)
 
1957
  continue
1958
  cell["state"] = "fresh"
1959
  cell["value"] = out["value"]
 
1979
  _recompute_formulas(book, fr, sheet_mod)
1980
  recalc_ms = round((perf_counter() - started) * 1000.0, 1)
1981
  try:
1982
+ # R19: book came from get_workbook; clamping on refresh would destroy imported widths.
1983
  saved = sheets.put_workbook(_store(session), workbook_id,
1984
  {"name": book.get("name"), "sheets": book.get("sheets"),
1985
+ "refreshed_at": stamp, "rev": book.get("rev")},
1986
+ actor=session.uname,
1987
+ actor_name=session.user.get("name") or session.uname,
1988
+ lossless=True)
1989
  except sheets.SheetRefusal as exc:
1990
  raise _refuse(exc)
1991
  # ⭐⭐ R3 / W52-T12: REFRESH IS THE ONE EVENT ALLOWED TO INVALIDATE A READER'S CACHED SCOPE
1992
  # VALUE, so it is the one door that clears the whole per-scope cache. Every reader's next GET
1993
  # recomputes once, under their own scope, and fills its own slot again from there.
1994
  sheets.clear_scope_cache(_store(session), workbook_id)
1995
+ return {"workbook": _response_workbook(saved), "recalc_ms": recalc_ms}
1996
 
1997
 
1998
  def _now_iso():
 
2247
  return column(raw)
2248
 
2249
 
2250
+ def _ending_window(session, binding, *, book=None, tab=None, today=None, seen=()):
2251
+ """Resolve a period from authored cells. Submitted ending.date is never trusted for a ref."""
2252
+ from modules import sheet as sheet_mod
2253
+ from harness import windows as W
2254
+ period = binding.get("period")
2255
+ if not isinstance(period, dict) or (period.get("ending") or {}).get("kind") != "cell":
2256
+ return sheet_mod.period_window(period, today)
2257
+ ref = period.get("ref") or "(blank)"
2258
+ value = _read_binding_cell(session, ref, book=book, tab=tab, today=today, seen=seen)
2259
+ date = W.ending_cell_date(value, book.get("dateSystem", "1900"))
2260
+ if date is None:
2261
+ raise sheet_mod.SheetRefusal("ending_cell_unreadable", ref, "its value is not a real date.")
2262
+ return sheet_mod.period_window(period, today, cell_date=date)
2263
+
2264
+
2265
+ def _read_binding_cell(session, ref, *, book=None, tab=None, today=None, seen=(),
2266
+ code="ending_cell_unreadable"):
2267
+ """Shared cell read for Ending and Filter, including scoped bound and formula inputs."""
2268
+ from modules import sheet as sheet_mod
2269
+ import formula_runtime as fr
2270
+
2271
+ def refuse(reason):
2272
+ raise sheet_mod.SheetRefusal(code, ref or "(blank)", reason)
2273
+
2274
+ if not re.fullmatch(r"(?:(?:'([^']|'')+'|[^'!]+)!)?\$?[A-Za-z]{1,3}\$?[1-9][0-9]{0,6}", ref):
2275
+ refuse("its reference is invalid.")
2276
+ if book is None:
2277
+ refuse("the source workbook is unavailable.")
2278
+ tabs = book.get("sheets") or []
2279
+ names = _tab_scope_names(tabs)
2280
+ home = _resolve_home_tab(book, tabs, names, tab)
2281
+ key = _asked_key(ref, home, fr)
2282
+ if key in seen:
2283
+ refuse("the reference is circular.")
2284
+ cells = {fr.a1_key(addr, name): cell for source, name in zip(tabs, names)
2285
+ for addr, cell in (source.get("cells") or {}).items()}
2286
+ cell = cells.get(key)
2287
+ if not isinstance(cell, dict) or cell.get("kind") == "empty":
2288
+ refuse("the cell or its sheet is missing or blank.")
2289
+ if cell.get("kind") == "bound":
2290
+ try:
2291
+ value = _resolve_binding(session, cell.get("binding"), today=today,
2292
+ book=book, tab=key.rpartition("!")[0], ending_seen=(*seen, key))["value"]
2293
+ except HTTPException as exc:
2294
+ reason = ((exc.detail or {}).get("error") or {}).get("message", "its source is refused.")
2295
+ refuse(reason)
2296
+ elif cell.get("kind") == "formula":
2297
+ # The existing formula graph discovers actual inputs, including dynamic references.
2298
+ # A database input starts refused, so a cached value can never bypass this reader's wall.
2299
+ bound = {k: c for k, c in cells.items() if c.get("kind") == "bound"}
2300
+ specs = {k: ({"refused": "ending_input_pending"} if k in bound
2301
+ else fr._document_spec(c, k.rpartition("!")[0])) for k, c in cells.items()}
2302
+ engine = fr.SheetCalculation(specs, sheets=names, today=today,
2303
+ named_ranges=book.get("namedRanges"), date_system=book.get("dateSystem", "1900"))
2304
+ attempted, failures = set(), {}
2305
+ while True:
2306
+ dependencies, pending = set(), [key]
2307
+ while pending:
2308
+ dependency = pending.pop()
2309
+ if dependency in dependencies:
2310
+ continue
2311
+ dependencies.add(dependency)
2312
+ pending.extend(engine.consumed.get(dependency, ()))
2313
+ needed = (dependencies & set(bound)) - attempted
2314
+ if not needed:
2315
+ break
2316
+ changes = {}
2317
+ for dependency in needed:
2318
+ attempted.add(dependency)
2319
+ try:
2320
+ result = _resolve_binding(session, bound[dependency].get("binding"), today=today,
2321
+ book=book, tab=dependency.rpartition("!")[0], ending_seen=(*seen, key, dependency))
2322
+ changes[dependency] = {"value": result["value"]}
2323
+ except HTTPException as exc:
2324
+ info = ((exc.detail or {}).get("error") or {})
2325
+ failures[dependency] = info.get("message", "a database input is refused.")
2326
+ changes[dependency] = {"refused": info.get("code", "ending_cell_unreadable")}
2327
+ # Each pass resolves previously unread database cells, so the workbook bounds the loop.
2328
+ engine.update(changes, revision=engine.revision + 1, today=today)
2329
+ if engine.errors.get(key):
2330
+ error_code = engine.errors[key]
2331
+ reason = next((failures[k] for k in dependencies if k in failures), None)
2332
+ if not reason:
2333
+ reason = (sheet_mod.SheetRefusal(error_code, ref, "a formula input is refused.").message
2334
+ if error_code in sheet_mod.REFUSALS else fr._sheet_error_message(error_code))
2335
+ refuse(reason)
2336
+ value = engine.values.get(key)
2337
+ else:
2338
+ if cell.get("state") == "refused":
2339
+ refuse((cell.get("refusal") or {}).get("message") or "the cell is refused.")
2340
+ if cell.get("state") == "stale":
2341
+ refuse("the cell needs to be refreshed.")
2342
+ value = cell.get("value", cell.get("text"))
2343
+ return value
2344
+
2345
+
2346
+ def _filter_cell_values(session, nodes, *, book=None, tab=None, today=None, seen=()):
2347
+ from modules import sheet as sheet_mod
2348
+ try:
2349
+ return sheet_mod.resolve_filter_cells(nodes, lambda ref: _read_binding_cell(
2350
+ session, ref, book=book, tab=tab, today=today, seen=seen, code="filter_cell_unreadable"))
2351
+ except sheet_mod.SheetRefusal as exc:
2352
+ raise err(400, exc.code, exc.message)
2353
+
2354
+
2355
+ # ⛔⛔ W62-T12 / C2 / R29 — A `mode: "view"` BINDING IS SKIPPED BY EVERY METRIC RESOLVER, AND
2356
+ # SKIPPING IS NOT THE SAME AS REFUSING. A placed view has no scalar value: its rows are read through
2357
+ # `POST /sheets/{id}/view-rows` and PLACED by the client, because the placement is a structure op
2358
+ # only the client can take. Before this guard the refresh door handed a view binding to
2359
+ # `modules/sheet.py::clean_binding`, which REQUIRES an `aggregate`, so `Refresh All` marked every
2360
+ # placed view `refused` and painted "That cell's source is not set up correctly." over a view that
2361
+ # was perfectly well formed. Caught by this lane's own screenshot, not by a gate.
2362
+ def _is_view_binding(cell) -> bool:
2363
+ """True for a C2 placed view, which the metric resolvers must leave exactly as they found it."""
2364
+ binding = (cell or {}).get("binding") if isinstance(cell, dict) else None
2365
+ return isinstance(binding, dict) and binding.get("mode") == "view"
2366
+
2367
+
2368
+ # R48: the persisted code is short; the original cause stays intact in the document.
2369
+ _BINDING_REFUSAL_DISPLAY = {
2370
+ "database_unreadable": "#NO ACCESS",
2371
+ "field_unknown": "#FIELD?",
2372
+ }
2373
+
2374
+
2375
+ def _binding_refusal(code, message):
2376
+ """One refusal shape for compute, refresh and caller-scoped reads of metric cells."""
2377
+ return {
2378
+ "state": "refused",
2379
+ "value": None,
2380
+ "display": _BINDING_REFUSAL_DISPLAY.get(code, "#ERROR!"),
2381
+ "refusal": {"code": code, "message": message},
2382
+ }
2383
+
2384
+
2385
+ def _resolve_cell_binding(session, binding, book, tab, **kwargs):
2386
+ """Keep the legacy resolver calling convention for bindings without workbook inputs."""
2387
+ if _has_cell_inputs(binding):
2388
+ return _resolve_binding(session, binding, book=book, tab=tab, **kwargs)
2389
+ return _resolve_binding(session, binding, **kwargs)
2390
+
2391
+
2392
+ def _resolve_binding(session: Session, raw_binding, *, today=None, view_memo=None,
2393
+ book=None, tab=None, ending_seen=()):
2394
  """Evaluate ONE C2 binding and return `{value, limits, sql_columns}`.
2395
 
2396
  The order below is the contract, not a preference:
 
2418
  raise err(400, exc.code, exc.message)
2419
 
2420
  database = _require_database(session, binding["database"])
2421
+ try:
2422
+ window = _ending_window(session, binding, book=book, tab=tab, today=today, seen=ending_seen)
2423
+ except sheet_mod.SheetRefusal as exc:
2424
+ raise err(400, exc.code, exc.message)
2425
  # W59-T14 / R10: the per-database wall is contract step 1. A metric key must not be looked
2426
  # up or acted on before the caller passes it; the ordinary column guards below still apply.
2427
  if binding["aggregate"].get("metric_key"):
 
2443
  scope.get("view_name", ""), memo=view_memo)
2444
  else:
2445
  scope_nodes, scope_conj = _fe_parts(scope.get("filter"))
2446
+ scope_nodes = _filter_cell_values(session, scope_nodes, book=book, tab=tab,
2447
+ today=today, seen=ending_seen)
2448
  needed = (sheet_mod.filter_columns_named(scope_nodes)
2449
  | sheet_mod.filter_columns_named(wall_nodes))
2450
  if binding["aggregate"].get("field"):
2451
  needed.add(str(binding["aggregate"]["field"]))
2452
 
2453
  cur, spec, cols = _mirror(session)(database, metric_window=window, needed=needed)
2454
+ cols = dict(cols)
2455
+ scope_nodes = _linked_filter_scope(session, database, scope_nodes, cur, spec, cols, today=today)
2456
 
2457
  # ⛔ THE COLUMN IS CHECKED BEFORE THE SQL IS BUILT (W53-T13). Now that every tenant column is a
2458
  # real column of the materialised table, a `sum` over a text column reaches DuckDB as a binder
 
2550
  field through the database's ordinary field writer; reusing a column needs no field write.
2551
  """
2552
  raw = dict(body or {})
2553
+ binding = raw.get("binding") or {}
2554
+ book = None
2555
+ if _has_cell_inputs(binding):
2556
+ from core import sheets
2557
+ try:
2558
+ book = sheets.get_workbook(_store(session), workbook_id)
2559
+ except sheets.SheetRefusal as exc:
2560
+ raise _refuse(exc)
2561
+ out = _resolve_cell_binding(session, binding, book, raw.get("tab"))
2562
  return {"value": out["value"], "display": "" if out["value"] is None else str(out["value"]),
2563
  "limits": out["limits"]}
2564
 
 
2685
  raise err(404, "cell_missing", f"Cell {key} is not filled in.")
2686
  if cell.get("kind") != "bound":
2687
  raise err(409, "cell_not_bound", f"Cell {key} is not bound to a database.")
2688
+ if _is_view_binding(cell):
2689
+ # C2: a placed view has rows, not a value. Refusing BY NAME beats resolving it as a metric,
2690
+ # which is what painted "That cell's source is not set up correctly." over a good view.
2691
+ raise err(409, "cell_is_a_view",
2692
+ f"Cell {key} holds a view, so it is read through its own rows door.")
2693
 
2694
  tab_ref = str((tab or {}).get("id") or (tab or {}).get("name") or "")
2695
  binding = cell.get("binding") or {}
 
2697
  saved = dict(cell)
2698
  limits, resolved = [], None
2699
  try:
2700
+ resolved = _resolve_cell_binding(session, binding, book, tab_ref)
2701
  except HTTPException as exc:
2702
  #: The same per-cell refusal shape `refresh_sheet` writes, so a cell refused by this door
2703
  #: and a cell refused by that one read identically. It is PERSISTED rather than raised:
2704
  #: a cell left `stale` would shimmer forever, and the reason belongs on the cell.
2705
  detail = getattr(exc, "detail", None) or {}
2706
  info = detail.get("error") if isinstance(detail, dict) else None
2707
+ saved.update(_binding_refusal(
2708
+ (info or {}).get("code", "sheet_unresolved"),
2709
+ (info or {}).get("message", "This cell could not be refreshed."),
2710
+ ))
 
 
 
2711
  else:
2712
  limits = resolved["limits"]
2713
  saved["state"] = "fresh"
 
2717
  saved.pop("refusal", None)
2718
 
2719
  try:
2720
+ landed = sheets.patch_cells(_store(session), workbook_id, tab_ref, {key: saved},
2721
+ actor=session.uname,
2722
+ actor_name=session.user.get("name") or session.uname)
2723
  except sheets.SheetRefusal as exc:
2724
  raise _refuse(exc)
2725
 
 
2743
  return {"cell": wire, "rev": landed.get("rev"), "tab": tab_ref, "addr": key}
2744
 
2745
 
2746
+ @router.post("/sheets/{workbook_id}/view-rows")
2747
+ def view_rows(workbook_id: str, body: object = Body(default=None),
2748
+ session: Session = Depends(require_session)):
2749
+ """W62 C2 / R29: read the live rows of a placed view under this caller's walls."""
2750
+ from datetime import date
2751
+ from fastapi.responses import JSONResponse
2752
+ from core import perm_scope, sheets, user_tables
2753
+ from modules import sheet as sheet_mod
2754
+
2755
+ try:
2756
+ binding = sheets.clean_view_binding(body.get("binding") if isinstance(body, dict) else None)
2757
+ except sheets.SheetRefusal as exc:
2758
+ return JSONResponse(status_code=400, content={"error": exc.code, "message": exc.message})
2759
+ try:
2760
+ book = sheets.get_workbook(_store(session), workbook_id)
2761
+ except sheets.SheetRefusal as exc:
2762
+ raise _refuse(exc)
2763
+ _require_reader(session, book)
2764
+ database = _require_database(session, binding["database"])
2765
+ fields = list(_registry_assembly(session, database).get("fields") or [])
2766
+ hidden = perm_scope.hidden_keys(session.user, database, fields, st=session.runtime)
2767
+ visible = {field["key"]: field for field in fields if field.get("key") not in hidden}
2768
+ requested = binding["fields"]
2769
+ sorted_keys = {item["field"] for item in binding["sort"]}
2770
+ for key in dict.fromkeys([*requested, *sorted(sorted_keys)]):
2771
+ if key not in visible:
2772
+ return JSONResponse(status_code=400, content={
2773
+ "error": "view_field_unknown", "message": f"Field {key} is not available in this database."})
2774
+
2775
+ # W62 C2: "Filter and sort apply to ALL rows BEFORE limit. total counts every matching row."
2776
+ wall_tree = ((perm_scope.entry(session.user, database) or {}).get("filter")
2777
+ if perm_scope.row_scope_applies(session.user, database) else None)
2778
+ wall_nodes, wall_conj = _fe_parts(wall_tree)
2779
+ scope_nodes, scope_conj = _fe_parts(binding.get("filter"))
2780
+ # W62 C2: a filter-only field obeys the same grants; linked fields have their own resolver wall.
2781
+ for key in sorted(sheet_mod.filter_columns_named(scope_nodes)):
2782
+ if not key.startswith(sheet_mod.LINKED_FILTER_PREFIX) and key not in visible:
2783
+ return JSONResponse(status_code=400, content={
2784
+ "error": "view_field_unknown", "message": f"Field {key} is not available in this database."})
2785
+ today = date.today()
2786
+ try:
2787
+ scope_nodes = _filter_cell_values(session, scope_nodes, book=book, today=today)
2788
+ window = _ending_window(session, binding, book=book, today=today)
2789
+ needed = (set(requested) | sorted_keys | sheet_mod.filter_columns_named(scope_nodes)
2790
+ | sheet_mod.filter_columns_named(wall_nodes))
2791
+ cur, spec, cols = _mirror(session)(database, metric_window=window, needed=needed)
2792
+ cols = dict(cols)
2793
+ scope_nodes = _linked_filter_scope(session, database, scope_nodes, cur, spec, cols, today=today)
2794
+ for key in dict.fromkeys([*requested, *sorted(sorted_keys | sheet_mod.filter_columns_named(scope_nodes))]):
2795
+ if key in hidden or not (cols.get(key) or {}).get("sql"):
2796
+ return JSONResponse(status_code=400, content={
2797
+ "error": "view_field_unknown", "message": f"Field {key} is not available in this database."})
2798
+ # W62 C2: an unanswerable row wall must refuse before either SQL read can widen it.
2799
+ wall_compiled = sheet_mod.compile_scope(wall_nodes, wall_conj, cols, today=today)
2800
+ if sheet_mod.widen_limits(wall_nodes, cols, wall_compiled):
2801
+ raise ValueError("unapplied row wall")
2802
+ count_sql, params, limits = sheet_mod.build_aggregate_query(
2803
+ spec, {"aggregate": {"fn": "count"}}, cols, scope_nodes=scope_nodes,
2804
+ scope_conj=scope_conj, wall_nodes=wall_nodes, wall_conj=wall_conj, today=today)
2805
+ if limits:
2806
+ raise ValueError("unapplied view filter")
2807
+ except sheet_mod.SheetRefusal as exc:
2808
+ return JSONResponse(status_code=400, content={"error": exc.code, "message": exc.message})
2809
+ except ValueError:
2810
+ raise err(409, "scope_not_applicable",
2811
+ "This view uses a restriction that cannot be applied to this database's live connection.")
2812
+
2813
+ # W62 C2 / standing rule 1: "limit caps what is PLACED"; the matching count is unlimited.
2814
+ total = (cur.execute(count_sql, params).fetchone() or [0])[0]
2815
+ # ⭐⭐ W62-T14 — EVERY ROW CARRIES ITS RECORD ID, ALWAYS, AND IT IS NOT A PROJECTED FIELD.
2816
+ # A placed Grid over one of our own tables writes a record back, and the write door is Data's
2817
+ # own `PATCH /api/v1/tables/{database}/rows/{pid}`: without the id the client can only guess
2818
+ # from a row's POSITION in the sheet, which is wrong the moment anyone sorts, filters or
2819
+ # refreshes. So the id rides beside the values under the key `pid` while `fields` still names
2820
+ # only the columns the sheet paints. ⛔ It is the SAME `spec["id"]` expression `build_ids_query`
2821
+ # uses, so the identity here and the identity the drill lists can never be two different things.
2822
+ id_expr = str(spec.get("id") or "").strip()
2823
+ projection = ", ".join([*(cols[key]["sql"] for key in requested),
2824
+ *([id_expr] if id_expr else [])])
2825
+ rows_sql = count_sql.replace("SELECT COUNT(*) FROM", f"SELECT {projection} FROM", 1)
2826
+ if binding["sort"]:
2827
+ # W62 C2 / R30: expressions come only from resolved columns; directions are validated.
2828
+ order = ", ".join(cols[item["field"]]["sql"] + (" ASC" if item["dir"] == "asc" else " DESC")
2829
+ for item in binding["sort"])
2830
+ rows_sql += " ORDER BY " + order
2831
+ rows_sql += f" LIMIT {binding['limit']}"
2832
+ keys = [*requested, *(["pid"] if id_expr else [])]
2833
+ rows = [dict(zip(keys, row)) for row in cur.execute(rows_sql, params).fetchall()]
2834
+ # W62 C2 / R34: Data's _records_or_refuse uses database access (checked above) plus
2835
+ # W62 R34: records_mutable, with no separate edit-role grant; its row/field walls still apply.
2836
+ editable = user_tables.records_mutable(database, st=session.runtime)
2837
+ return {"fields": [{"key": key, "label": visible[key].get("label") or key,
2838
+ "kind": visible[key].get("type") or visible[key].get("kind") or "text"}
2839
+ for key in requested],
2840
+ "rows": rows, "total": int(total or 0), "limit": binding["limit"],
2841
+ "viewType": binding["viewType"], "editable": bool(editable)}
2842
+
2843
+
2844
  @router.get("/sheets/{workbook_id}/cells/{addr}/drill")
2845
  def drill_cell(workbook_id: str, addr: str, session: Session = Depends(require_session)):
2846
  """`{database, view_id?, filter?, record_ids, count}` for `LinkGridModal` (R6, R13, R17).
 
2876
  if cell.get("kind") != "bound":
2877
  raise err(409, "literal_no_drill",
2878
  "A hand typed value has no rows behind it.")
2879
+ if _is_view_binding(cell):
2880
+ raise err(409, "cell_is_a_view",
2881
+ f"Cell {key} holds a view, so its rows are already on the sheet.")
2882
 
2883
  try:
2884
  binding = sheet_mod.clean_binding(cell.get("binding"))
 
2911
  # `_metric_derived` at a month window is 27 ms of an 8,437 ms drill on `customer_data`. So the
2912
  # window is kept UNCONDITIONALLY, exactly as W53-T13 wrote it, and the saving is taken from
2913
  # the column set instead, where the time actually is.
2914
+ try:
2915
+ scope_nodes = _filter_cell_values(session, scope_nodes, book=book, tab=tab.get("id"))
2916
+ window = _ending_window(session, binding, book=book, tab=tab.get("id"))
2917
+ except sheet_mod.SheetRefusal as exc:
2918
+ raise err(400, exc.code, exc.message)
2919
  cur, spec, cols = _mirror(session)(
2920
+ database, metric_window=window,
2921
  needed=(sheet_mod.filter_columns_named(scope_nodes)
2922
  | sheet_mod.filter_columns_named(wall_nodes)))
2923
+ cols = dict(cols)
2924
+ scope_nodes = _linked_filter_scope(session, database, scope_nodes, cur, spec, cols)
2925
 
2926
+ try:
2927
+ count_sql, count_params, limits = sheet_mod.build_aggregate_query(
2928
+ spec, {"aggregate": {"fn": "count"}}, cols, scope_nodes=scope_nodes,
2929
+ scope_conj=scope_conj, wall_nodes=wall_nodes, wall_conj=wall_conj)
2930
+ except sheet_mod.SheetRefusal as exc:
2931
+ raise err(400, exc.code, exc.message)
2932
  total = (cur.execute(count_sql, count_params).fetchone() or [0])[0]
2933
 
2934
  ids_sql, ids_params, _ = sheet_mod.build_ids_query(
 
2992
  @router.post("/sheets/{workbook_id}/resolve-bindings")
2993
  def resolve_bindings(workbook_id: str, body: dict = Body(default=None),
2994
  session: Session = Depends(require_session)):
2995
+ """Values or named refusals for the NAMED bound cells only (contract C1, rulings R2 and R48).
2996
 
2997
  ⭐⭐ THIS IS THE DOOR THE COMMIT PATH USES, AND ITS WHOLE POINT IS THAT IT IS SMALL.
2998
  Under R2 a formula evaluates in the browser the instant it is committed, so the only thing the
 
3008
  by THE EXACT STRING THE CALLER ASKED WITH, so a client looks its answer up under the
3009
  reference it already holds and never has to re-derive this file's key spelling.
3010
 
3011
+ A KEY IS ABSENT WHEN NO BOUND CELL LIVES AT THAT ADDRESS. An address that is empty, holds a
3012
+ typed literal or holds a formula is not this door's business - lane A's evaluator already
3013
+ has those values in hand - so asking for one costs nothing and answers nothing.
3014
+
3015
+ R48: a refused binding answers `{value: null, refusal: {code, message}}`, so it cannot be
3016
+ mistaken for a legitimate empty aggregate. The flat shape is kept for every address that
3017
+ has an answer, including a genuine null, so no existing reader of those answers changes.
 
 
 
 
 
 
 
3018
 
3019
  ⛔ ONE BAD CELL NEVER FAILS THE BATCH, exactly as in `refresh_sheet`'s bound-cell loop: a
3020
  refusal is caught per cell and the siblings still answer.
 
3076
  if cell is None:
3077
  continue
3078
  binding = cell.get("binding") or {}
3079
+ own_tab = _asked_key(label, home, fr).rpartition("!")[0]
3080
+ slot = json.dumps([own_tab, binding], sort_keys=True, default=str)
3081
  if slot not in by_binding:
3082
  try:
3083
+ by_binding[slot] = _resolve_cell_binding(session, binding, book, own_tab,
3084
+ view_memo=view_memo)["value"]
3085
+ except HTTPException as exc:
3086
+ detail = getattr(exc, "detail", None) or {}
3087
+ info = detail.get("error") if isinstance(detail, dict) else None
3088
+ by_binding[slot] = {
3089
+ "value": None,
3090
+ "refusal": {
3091
+ "code": (info or {}).get("code", "sheet_unresolved"),
3092
+ "message": (info or {}).get("message", "This cell could not be refreshed."),
3093
+ },
3094
+ }
3095
  answer[label] = by_binding[slot]
3096
  return answer
api/routes_starred.py CHANGED
@@ -28,9 +28,11 @@ disagree — a view deleted, a view unmarked from inside the database, a store w
28
  and not the other — the product has two answers to one question and no way to tell which is right.
29
  The scan is the price of one flag, and it is paid off the critical path.
30
  """
 
31
  import hashlib
32
  import json
33
  import time
 
34
 
35
  from fastapi import APIRouter, Body, Depends
36
 
@@ -39,17 +41,17 @@ from deps import Session, err, require_session
39
 
40
  router = APIRouter(prefix="/api/v1")
41
 
42
- #: The per-user bucket holding the three ID-LIST kinds. `{username: {databases, agents, queries}}`
43
  #:
44
  #: ⚠ PER USER (R6), like `nav_recents` and unlike `nav_meta`: what I care about is not a fact about
45
  #: the object, so a tenant-wide star would put one person's shortlist on everybody's Home.
46
  _STARRED_KEY = "starred"
47
 
48
- #: The four kinds C2 declares. `view` is the odd one out — see the header.
49
- KINDS = ("database", "agent", "query", "view", "application")
50
 
51
- #: The three kinds this file's own bucket holds, and the payload key each answers under.
52
- _LIST_KINDS = {"database": "databases", "agent": "agents", "query": "queries"}
53
 
54
  #: A bound on each id list. ⛔ REPORTED, NEVER SILENT: a POST that would cross it is refused with
55
  #: the cause and the action (unstar something), rather than being accepted and truncated. A star
@@ -65,6 +67,88 @@ _MAX_ID = 120
65
  #: container starts; warm it is not reached.
66
  VIEW_SCAN_BUDGET_S = 2.5
67
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
68
 
69
  def _clean_id(raw):
70
  """One id, or "" — the only shape this file stores."""
@@ -90,7 +174,7 @@ def _clean_list(raw):
90
 
91
 
92
  def _read_lists(session):
93
- """This caller's three id lists. A store blip answers empty rather than taking the page down."""
94
  try:
95
  mine = (session.runtime.get(_STARRED_KEY) or {}).get(session.uname) or {}
96
  except Exception:
@@ -144,18 +228,56 @@ def _view_docs(session, order):
144
  import core.view_templates as view_templates
145
  docs, unread = {}, []
146
  started = time.perf_counter()
 
147
  for key in order:
148
  ws_key = view_templates.workspace_key(key)
149
  if not ws_key:
150
  continue # a module surface or a folder head, not a table at all
151
- if time.perf_counter() - started > VIEW_SCAN_BUDGET_S:
152
- unread.append(key)
153
- continue
 
 
 
 
 
 
 
154
  try:
155
- docs[key] = session.runtime.get_projection(ws_key,
156
- drop=("overlays", "fields")) or {}
 
 
 
 
 
157
  except Exception:
158
- unread.append(key)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
159
  return docs, unread
160
 
161
 
@@ -207,8 +329,25 @@ def _scan_starred_views(session, keys=None, cache=None):
207
  import core.table_store as table_store
208
  import core.view_templates as view_templates
209
 
210
- granted = routes_nav._granted_view_ids(session)
211
- out = []
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
212
  for key in order:
213
  doc = docs.get(key)
214
  if doc is None:
@@ -217,8 +356,43 @@ def _scan_starred_views(session, keys=None, cache=None):
217
  if not ws_key:
218
  continue
219
  views = table_store.make(ws_key, st=session.runtime)
220
- for vid, view in routes_nav._visible_views(doc, session.uname, session.admin,
221
- granted, views).items():
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
222
  cfg = view.get("config") or {}
223
  if cfg.get("important") is True:
224
  out.append({"id": vid, "database": key,
@@ -289,13 +463,14 @@ def _query_names(session, qids):
289
 
290
 
291
  def _payload(session, keys=None, cache=None):
292
- """The C2 answer: the three id lists, the starred views, and what could not be read.
293
 
294
  ⚠ `degraded` SHIPS AS `[]` RATHER THAN BEING OMITTED WHEN EMPTY — the same rule `/nav`'s
295
  `omitted`/`degraded` follow, for the same reason: a key a consumer has to test for is a key a
296
  consumer forgets to test for.
297
  ⚠ `cache` is a scan an earlier pass in THIS request already paid for — see `_scan_starred_views`.
298
  """
 
299
  lists = _read_lists(session)
300
  if keys is None:
301
  keys, _ = _database_keys(session)
@@ -307,6 +482,8 @@ def _payload(session, keys=None, cache=None):
307
  # validate, and pruning them here buys nothing: this list is the CALLER'S OWN writes, so it can
308
  # disclose nothing they did not already know, and B/C render only objects they can already see
309
  # (the way `groupRecents` drops an unreachable recent). Said out loud rather than left as a gap.
 
 
310
  lists["databases"] = [k for k in lists["databases"] if k in keys]
311
  # ⭐ ASK B-1 (2): the names, and — because the read that answers "what is it called" also answers
312
  # "does it still exist" — the same prune the databases get. ⛔ ONLY WHEN THE READ SUCCEEDED: a
@@ -327,7 +504,7 @@ def _payload(session, keys=None, cache=None):
327
  except Exception:
328
  application_rows = []
329
  out = {"databases": lists["databases"], "agents": lists["agents"],
330
- "queries": lists["queries"], "queryNames": qnames,
331
  "views": views, "applications": application_rows, "degraded": unread}
332
  if unread:
333
  # R6's second sentence, in the payload: the CAUSE and what it means, not a bare list.
@@ -503,7 +680,7 @@ def set_starred(body: dict = Body(default=None),
503
  session: Session = Depends(require_session)):
504
  """C2 — star or unstar ONE object, and answer with the caller's whole new list.
505
 
506
- Body: `{"kind": "database"|"agent"|"query"|"view", "id": str, "on": bool}`, plus an OPTIONAL
507
  `"database"` when `kind` is `view` (the fast path — see `_find_view`).
508
 
509
  ⚠ THE WHOLE LIST COMES BACK, not an ack, because the client is optimistic-then-reconciled
 
28
  and not the other — the product has two answers to one question and no way to tell which is right.
29
  The scan is the price of one flag, and it is paid off the critical path.
30
  """
31
+ import copy
32
  import hashlib
33
  import json
34
  import time
35
+ from concurrent.futures import ThreadPoolExecutor
36
 
37
  from fastapi import APIRouter, Body, Depends
38
 
 
41
 
42
  router = APIRouter(prefix="/api/v1")
43
 
44
+ #: The per-user bucket holding ID-LIST kinds. `{username: {databases, agents, queries, sheets}}`
45
  #:
46
  #: ⚠ PER USER (R6), like `nav_recents` and unlike `nav_meta`: what I care about is not a fact about
47
  #: the object, so a tenant-wide star would put one person's shortlist on everybody's Home.
48
  _STARRED_KEY = "starred"
49
 
50
+ #: Accepted star kinds. Views and applications keep their own record-backed write paths.
51
+ KINDS = ("database", "agent", "query", "view", "application", "sheet")
52
 
53
+ #: The kinds this file's own bucket holds, and the payload key each answers under.
54
+ _LIST_KINDS = {"database": "databases", "agent": "agents", "query": "queries", "sheet": "sheets"}
55
 
56
  #: A bound on each id list. ⛔ REPORTED, NEVER SILENT: a POST that would cross it is refused with
57
  #: the cause and the action (unstar something), rather than being accepted and truncated. A star
 
67
  #: container starts; warm it is not reached.
68
  VIEW_SCAN_BUDGET_S = 2.5
69
 
70
+ # Four projected reads fit beneath the store pool's eight-connection ceiling while removing the
71
+ # serial N-read wall from Home. This is deliberately bounded: a tenant with many databases must
72
+ # not turn one Starred request into an unbounded connection burst.
73
+ VIEW_SCAN_WORKERS = 4
74
+
75
+ # One bounded process-wide pool, rather than one executor per authenticated request. A timed-out
76
+ # Python thread cannot be killed safely; a per-request executor therefore let repeated slow store
77
+ # calls grow one live worker apiece until the backend returned. The shared pool keeps that failure
78
+ # bounded to VIEW_SCAN_WORKERS and `future.cancel()` removes work that has not started yet.
79
+ _BATCH_READ_POOL = ThreadPoolExecutor(
80
+ max_workers=VIEW_SCAN_WORKERS,
81
+ thread_name_prefix="starred-batch",
82
+ )
83
+
84
+
85
+ class _RequestReadMemo:
86
+ """Read-through snapshot for one Starred payload assembly, never a cross-request cache."""
87
+
88
+ def __init__(self, runtime):
89
+ self._runtime = runtime
90
+ self._docs = {}
91
+ self._projections = {}
92
+
93
+ def preload(self, names):
94
+ reader = getattr(self._runtime, "get_many", None)
95
+ if not callable(reader):
96
+ return
97
+ try:
98
+ answer = reader(names)
99
+ except Exception:
100
+ return
101
+ if isinstance(answer, dict):
102
+ self._docs.update(answer)
103
+
104
+ def get(self, name, fresh=False):
105
+ key = str(name)
106
+ if not fresh and key in self._docs:
107
+ return copy.deepcopy(self._docs[key])
108
+ answer = self._runtime.get(name, fresh=fresh)
109
+ if not fresh:
110
+ self._docs[key] = copy.deepcopy(answer)
111
+ return answer
112
+
113
+ def get_projection(self, name, drop=()):
114
+ stamp = (str(name), tuple(str(key) for key in drop))
115
+ if stamp not in self._projections:
116
+ self._projections[stamp] = self._runtime.get_projection(name, drop=drop)
117
+ return copy.deepcopy(self._projections[stamp])
118
+
119
+ def get_projections(self, names, drop=(), timeout_s=None):
120
+ keys = [str(name) for name in names or ()]
121
+ stamps = [(key, tuple(str(field) for field in drop)) for key in keys]
122
+ if all(stamp in self._projections for stamp in stamps):
123
+ return {key: copy.deepcopy(self._projections[stamp])
124
+ for key, stamp in zip(keys, stamps)}
125
+ reader = getattr(self._runtime, "get_projections", None)
126
+ if not callable(reader):
127
+ return None
128
+ answer = reader(keys, drop=drop, timeout_s=timeout_s)
129
+ if answer is None:
130
+ return None
131
+ for key, stamp in zip(keys, stamps):
132
+ self._projections[stamp] = copy.deepcopy(answer.get(key, {}))
133
+ return {key: copy.deepcopy(self._projections[stamp])
134
+ for key, stamp in zip(keys, stamps)}
135
+
136
+ def __getattr__(self, name):
137
+ return getattr(self._runtime, name)
138
+
139
+
140
+ def _memoized_session(session):
141
+ """Preload the four small registries Starred rereads while assembling one response."""
142
+ import application_registry
143
+ from core import shares
144
+
145
+ runtime = _RequestReadMemo(session.runtime)
146
+ runtime.preload((_STARRED_KEY, routes_nav._NAV_RECENTS_KEY,
147
+ shares.SHARES_KEY, application_registry._KEY))
148
+ wrapped = copy.copy(session)
149
+ wrapped.runtime = runtime
150
+ return wrapped
151
+
152
 
153
  def _clean_id(raw):
154
  """One id, or "" — the only shape this file stores."""
 
174
 
175
 
176
  def _read_lists(session):
177
+ """This caller's id lists. A store blip answers empty rather than taking the page down."""
178
  try:
179
  mine = (session.runtime.get(_STARRED_KEY) or {}).get(session.uname) or {}
180
  except Exception:
 
228
  import core.view_templates as view_templates
229
  docs, unread = {}, []
230
  started = time.perf_counter()
231
+ readable = []
232
  for key in order:
233
  ws_key = view_templates.workspace_key(key)
234
  if not ws_key:
235
  continue # a module surface or a folder head, not a table at all
236
+ readable.append((key, ws_key))
237
+
238
+ batch_reader = getattr(session.runtime, "get_projections", None)
239
+ if readable and callable(batch_reader):
240
+ # The Postgres implementation also installs a statement timeout, but the route keeps its
241
+ # own wall-clock guard: an optional store capability must not silently defeat the scan
242
+ # budget merely by ignoring that hint. The process-wide pool is deliberately bounded;
243
+ # creating one executor here would let repeated timeouts amplify live background workers.
244
+ remaining = max(0.001, VIEW_SCAN_BUDGET_S - (time.perf_counter() - started))
245
+ future = None
246
  try:
247
+ future = _BATCH_READ_POOL.submit(
248
+ batch_reader,
249
+ [ws_key for _key, ws_key in readable],
250
+ drop=("overlays", "fields"),
251
+ timeout_s=remaining,
252
+ )
253
+ batch = future.result(timeout=remaining)
254
  except Exception:
255
+ # One batch has one availability answer. Reporting every key as unread is honest;
256
+ # falling through to N retries would recreate the latency wall on an unhealthy store.
257
+ if future is not None:
258
+ future.cancel()
259
+ return {}, [key for key, _ws_key in readable]
260
+ if batch is not None:
261
+ for key, ws_key in readable:
262
+ docs[key] = batch.get(ws_key) or {}
263
+ return docs, unread
264
+
265
+ for offset in range(0, len(readable), VIEW_SCAN_WORKERS):
266
+ if time.perf_counter() - started > VIEW_SCAN_BUDGET_S:
267
+ unread.extend(key for key, _ in readable[offset:])
268
+ break
269
+ chunk = readable[offset:offset + VIEW_SCAN_WORKERS]
270
+ with ThreadPoolExecutor(max_workers=max(1, len(chunk))) as pool:
271
+ futures = [pool.submit(session.runtime.get_projection, ws_key,
272
+ drop=("overlays", "fields"))
273
+ for _key, ws_key in chunk]
274
+ # Consume in request order so recents-first remains the response order even when a
275
+ # later store read finishes first.
276
+ for (key, _ws_key), future in zip(chunk, futures):
277
+ try:
278
+ docs[key] = future.result() or {}
279
+ except Exception:
280
+ unread.append(key)
281
  return docs, unread
282
 
283
 
 
329
  import core.table_store as table_store
330
  import core.view_templates as view_templates
331
 
332
+ granted = routes_nav._granted_view_sources(session)
333
+ # ⭐⭐ W62-T17 (D-756) — EVERY GRANTED VIEW ID WAS RESOLVED ONCE PER DATABASE, and that loop is
334
+ # where `/starred`'s budget went. MEASURED on staging's Royal Imports, 26 databases
335
+ # (`AutoWiki/waves/wave62/proto/T17-profile.md`): 4,844 ms total, of which `_visible_views` was
336
+ # 3,075 ms — `object_shares` read 28 times (1,117 ms) and `TableStore.find_view` 26 times
337
+ # (2,072 ms), for ONE granted id. Neither figure is about a slow read: the grant registry is
338
+ # 8 KB and each read is a single ~40 ms round trip. They are about the same question being
339
+ # asked once per database.
340
+ #
341
+ # ⭐ A VIEW LIVES IN EXACTLY ONE DATABASE AND ITS GRANT NAMES ITS OWNER. The modern fast path
342
+ # resolves that owner directly in the projected document this scan already paid for; it never
343
+ # re-reads the grant registry or calls `find_view` for every database. `resolved` still removes
344
+ # a found id from every later database's lookup work.
345
+ #
346
+ # ⛔ THIS IS NOT A LENT `object_shares` SNAPSHOT. `TableStore.find_view` reaches the legacy
347
+ # migration, which WRITES grants and reads them back; lending that key across the migration
348
+ # would serve a pre-write value and raise. The branch below detects retired shapes in the
349
+ # document already held, runs the unchanged strict migration, and refreshes that one document.
350
+ resolved, out = set(), []
351
  for key in order:
352
  doc = docs.get(key)
353
  if doc is None:
 
356
  if not ws_key:
357
  continue
358
  views = table_store.make(ws_key, st=session.runtime)
359
+ pending = set(granted) - resolved
360
+
361
+ # The old path called `find_view` once for every pending grant in every database. Each
362
+ # call strictly re-read this same workspace document and the whole share registry. The
363
+ # projected document above already contains every personal View, so modern grants can
364
+ # resolve directly through their canonical owner. A legacy document still takes the old
365
+ # strict migration door and is re-read afterwards; the optimization never skips a write.
366
+ if pending and views.prepare_view_projection(doc):
367
+ doc = session.runtime.get_projection(ws_key, drop=("overlays", "fields")) or {}
368
+ docs[key] = doc
369
+
370
+ found = {}
371
+ own = doc.get(session.uname)
372
+ for vid, view in ((own.get("views") or {}) if isinstance(own, dict) else {}).items():
373
+ if isinstance(view, dict):
374
+ found[str(vid)] = view
375
+
376
+ for vid in pending:
377
+ owner = granted.get(vid)
378
+ view = None
379
+ if owner:
380
+ owner_ws = doc.get(owner)
381
+ view = ((owner_ws.get("views") or {}).get(vid)
382
+ if isinstance(owner_ws, dict) else None)
383
+ else:
384
+ # Preserve the historical first-source lookup for a legacy ownerless grant.
385
+ for candidate_owner, workspace in doc.items():
386
+ if candidate_owner in table_store.RESERVED_MEMBERS or not isinstance(workspace, dict):
387
+ continue
388
+ candidate = (workspace.get("views") or {}).get(vid)
389
+ if isinstance(candidate, dict):
390
+ view = candidate
391
+ break
392
+ if isinstance(view, dict):
393
+ found.setdefault(vid, view)
394
+ resolved.add(vid)
395
+ for vid, view in found.items():
396
  cfg = view.get("config") or {}
397
  if cfg.get("important") is True:
398
  out.append({"id": vid, "database": key,
 
463
 
464
 
465
  def _payload(session, keys=None, cache=None):
466
+ """The star answer: the caller's id lists, starred records, and what could not be read.
467
 
468
  ⚠ `degraded` SHIPS AS `[]` RATHER THAN BEING OMITTED WHEN EMPTY — the same rule `/nav`'s
469
  `omitted`/`degraded` follow, for the same reason: a key a consumer has to test for is a key a
470
  consumer forgets to test for.
471
  ⚠ `cache` is a scan an earlier pass in THIS request already paid for — see `_scan_starred_views`.
472
  """
473
+ session = _memoized_session(session)
474
  lists = _read_lists(session)
475
  if keys is None:
476
  keys, _ = _database_keys(session)
 
482
  # validate, and pruning them here buys nothing: this list is the CALLER'S OWN writes, so it can
483
  # disclose nothing they did not already know, and B/C render only objects they can already see
484
  # (the way `groupRecents` drops an unreachable recent). Said out loud rather than left as a gap.
485
+ # Sheets are also the CALLER'S OWN writes. R28 filters the sheet index; C2 forbids a second
486
+ # access check here, so sheet ids pass through unchanged.
487
  lists["databases"] = [k for k in lists["databases"] if k in keys]
488
  # ⭐ ASK B-1 (2): the names, and — because the read that answers "what is it called" also answers
489
  # "does it still exist" — the same prune the databases get. ⛔ ONLY WHEN THE READ SUCCEEDED: a
 
504
  except Exception:
505
  application_rows = []
506
  out = {"databases": lists["databases"], "agents": lists["agents"],
507
+ "queries": lists["queries"], "sheets": lists["sheets"], "queryNames": qnames,
508
  "views": views, "applications": application_rows, "degraded": unread}
509
  if unread:
510
  # R6's second sentence, in the payload: the CAUSE and what it means, not a bare list.
 
680
  session: Session = Depends(require_session)):
681
  """C2 — star or unstar ONE object, and answer with the caller's whole new list.
682
 
683
+ Body: `{"kind": "database"|"agent"|"query"|"view"|"application"|"sheet", "id": str, "on": bool}`, plus an OPTIONAL
684
  `"database"` when `kind` is `view` (the fast path — see `_find_view`).
685
 
686
  ⚠ THE WHOLE LIST COMES BACK, not an ack, because the client is optimistic-then-reconciled
api/routes_tables.py CHANGED
@@ -68,33 +68,45 @@ class _MetricReadRuntime:
68
  raise AttributeError(name)
69
 
70
 
71
- def _related_product_fields(session, rows):
72
  """Resolve authored Metric definitions through the existing workspace and field wall."""
73
  import aios_grid
74
  from core import grid_events, perm_scope, shared_overlay
75
  from modules import product_data as pd
 
 
 
 
 
 
 
 
 
 
 
 
 
76
 
77
  pids = frozenset(r["pid"] for r in rows)
78
  ctx = grid_events.EventCtx(uname=session.uname, allowed_pids=pids, fields=[],
79
  hidden_keys=set(), admin=session.admin, fallback_ws=None,
80
- seen_ids={}, st=session.runtime, scope_key="product",
81
- table=pd.table_ops(session.runtime))
82
  ws = grid_events.table_workspace(ctx, allowed_pids=pids, consume_corrections=False)
83
  _, fields, _, _ = aios_grid.workspace_wire(
84
- ws, session.uname, set(pids), defs={}, scope_key="product", with_cohorts=False,
85
- fields_base=aios_grid.product_fields(), st=session.runtime)
86
  have = {f["key"] for f in fields}
87
  fields = list(fields) + [dict(f, source="overlay", shared=True)
88
- for key, f in shared_overlay.fields(pd.TABLE_KEY, st=session.runtime).items()
89
  if key not in have]
90
- hidden = perm_scope.hidden_keys(session.user, "product_data", fields, st=session.runtime)
91
  return [f for f in fields if f.get("measure") and f["key"] not in hidden]
92
 
93
 
94
  @router.post("/tables/{table_key}/related-metrics")
95
  def related_metrics(table_key: str, body: dict = Body(default=None),
96
  session: Session = Depends(require_session)):
97
- """C6: a visible Products ordered edge, origin customer and real destination ids.
98
 
99
  The request supplies only grid identities and a named relation. Fields, periods, source
100
  identities and values are read from this tenant; nothing is authored by this endpoint.
@@ -106,66 +118,79 @@ def related_metrics(table_key: str, body: dict = Body(default=None),
106
  import automation_engine
107
 
108
  body = body if isinstance(body, dict) else {}
109
- pin = body.get("related") or {}
110
  pids = body.get("pids")
111
- if (table_key != "product_data" or not isinstance(pin, dict)
112
- or pin.get("table") != "customer_data"
113
- or type(pin.get("record")) is not int or pin["record"] <= 0
114
- or not isinstance(pids, list) or not pids
115
  or any(type(pid) is not int or pid <= 0 for pid in pids)):
116
- raise err(400, "bad_metric_scope", "Name the origin customer and related product records.")
117
  viewer = copy.copy(session)
118
  viewer.runtime = _MetricReadRuntime(session.runtime)
119
  try:
120
- # Both endpoints and the actual bridge are permission-bearing databases.
121
- for key in ("customer_data", "product_data", "ut_odoo_order_lines"):
122
- viewer.require(key)
123
- entries = routes_grid.related_path_entries(viewer, "customer_data", 1)
124
- via = pin.get("via")
125
- edge = relations.pick_path(entries, via) if isinstance(via, list) else relations.pick(entries, via)
126
- if (edge is None or edge.get("target") != "product_data"
127
- or edge.get("via", {}).get("field") != "products_ordered"):
128
- raise err(403, "metric_relation_refused", "This Products ordered relation is unavailable.")
129
- source_fields = routes_grid._source_fields(viewer, "customer_data")
130
- hidden = perm_scope.hidden_keys(viewer.user, "customer_data", source_fields, st=viewer.runtime)
131
- if "products_ordered" in hidden:
132
- raise err(403, "metric_relation_refused", "This Products ordered relation is unavailable.")
133
- customers = perm_scope.scoped_table(viewer.user, "customer_data", st=viewer.runtime)
134
- if not any(r.get("pid") == pin["record"] for r in customers):
135
- raise err(403, "metric_record_refused", "The origin record is not available to this account.")
136
- products = perm_scope.scoped_table(viewer.user, "product_data", st=viewer.runtime)
137
  requested = set(pids)
138
  selected = [r for r in products if r.get("pid") in requested]
139
- if (len(selected) != len(requested)
140
- or any(type(r.get("product_id")) is not int or r["product_id"] <= 0 for r in selected)):
141
  raise err(403, "metric_record_refused", "The destination records cannot be scoped for this account.")
142
- # A bridge restriction that cannot be translated must never be silently dropped.
143
- bridge_fields = perm_scope.scoped_fields(viewer.user, "ut_odoo_order_lines", st=viewer.runtime)
144
- declared = perm_scope.entry(viewer.user, "ut_odoo_order_lines") or {}
145
- if (perm_scope.row_scope_applies(viewer.user, "ut_odoo_order_lines")
146
- or (not viewer.admin and declared.get("hiddenFields"))
147
- or not {"partner_id", "product_id"}.issubset({f["key"] for f in bridge_fields})):
148
- raise err(409, "metric_scope_unsupported", "Order-line permission restrictions cannot be applied to these Metrics yet.")
149
- scopes = [perm_scope.derive_pool_scope(viewer.user, key)
150
- for key in ("customer_data", "product_data", "ut_odoo_order_lines")]
151
- teams = {team for team, _ in scopes if team is not None}
152
- agents = {agent for _, agent in scopes if agent is not None}
153
- if len(teams) > 1 or len(agents) > 1:
154
- raise err(403, "metric_scope_conflict", "The related databases have incompatible permission scopes.")
155
- fields = _related_product_fields(viewer, products)
156
- values, reasons = automation_engine.related_product_metric_values(
157
- viewer.runtime, pin["record"], selected, fields, routes_products.product_measures(viewer),
158
- team_id=next(iter(teams), None), agent=next(iter(agents), None))
159
  return {"ok": True, "values": values, "fields": [f["key"] for f in fields],
160
- "scope": {"customer": pin["record"],
161
- "products": {r["pid"]: r["product_id"] for r in selected}},
162
- "reason": " ".join(reasons)}
 
 
163
  except (perm_scope.Denied, perm_scope.UnknownTable):
164
  raise err(403, "metric_record_refused", "The related records are not available to this account.")
165
  except (perm_scope.Unresolvable, RuntimeError, ValueError):
166
  raise err(409, "metric_scope_unavailable", "Scoped Metrics could not be read. Reopen the related records after the database is ready.")
167
 
168
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
169
  def _ut():
170
  import core.user_tables as user_tables
171
  return user_tables
@@ -1537,6 +1562,9 @@ def ut_assembly(session: Session, table_key: str, storage_key: str = "",
1537
  # for both — the same sentence `routes_odoo_tables` writes over its own `overlays` dict.
1538
  shared_cells = {pid: {k: v for k, v in cells.items() if k not in hidden}
1539
  for pid, cells in shared_cells.items()}
 
 
 
1540
 
1541
  # ⭐⭐ W37-T12 / owner item 4 (R1) — `measures` IS NO LONGER UNCONDITIONALLY EMPTY. A database
1542
  # whose entity topic declares a `measures:` binding (today `ut_odoo_agents`) serves a real
@@ -2277,7 +2305,8 @@ def _thin_json(fields, merged, table_key=""):
2277
 
2278
 
2279
  @router.get("/tables/{table_key}/rows")
2280
- def table_rows(table_key: str, session: Session = Depends(require_session)):
 
2281
  """The `/customers`-shaped envelope for one user table: `{fields, rows, today, pulled_at,
2282
  identity}` — so the client's generic topic fetch consumes it with zero new parsing.
2283
 
@@ -2299,7 +2328,7 @@ def table_rows(table_key: str, session: Session = Depends(require_session)):
2299
  # PRE-WRITE snapshot, and handing one to a path that writes and then reads back would report the
2300
  # value it replaced (contract C5).
2301
  _lent = _ut().lend(session.runtime)
2302
- g = ut_assembly(session, table_key, st=_lent)
2303
  # ⛔ THE OVERLAY WAS NEVER ACTUALLY MERGED, and the docstring above has described the merge
2304
  # this line does not perform since the route was written (owner item 3, 2026-08-09:
2305
  # *"Using the swipe, fast doesn't register the CHANGE. I went back and it all got reseted"*).
@@ -2361,6 +2390,12 @@ def table_rows(table_key: str, session: Session = Depends(require_session)):
2361
  # ⭐⭐ WAVE-34 (R13) — the per-cell enrichment STATE rides the row, beside `_created`/`lat`/
2362
  # `lon`. See `_stamp_ai_states` for why it is a row key rather than a map beside `rows`.
2363
  _stamp_ai_states(table_key, g["fields"], rows, st=_lent)
 
 
 
 
 
 
2364
  # ⭐ R6's SECOND SENTENCE, ON THE WIRE (W30-T29). *"if there is lag or it can't be done, you
2365
  # need to explicitly tell me why and recommend a fix."* A ceiling that still applies to this
2366
  # database says so here, with its cause and the recommendation, rather than waiting to be
@@ -2382,16 +2417,24 @@ def table_rows(table_key: str, session: Session = Depends(require_session)):
2382
  if str(pid) in _zone_pids])
2383
  for zone in _ut().zones(table_key, st=_lent)]
2384
  return {"fields": _field_axes(g["fields"]), "rows": rows, "today": g["today"],
 
2385
  "zones": _zones,
2386
  "docs": _ge.docs_for(g["pids"], scope_key=table_key, uname=session.uname,
2387
  admin=session.admin, st=session.runtime),
2388
  "pulled_at": time.strftime("%Y-%m-%d %H:%M"),
2389
  "identity": {"pid": "pid"},
2390
  "scope": {"table": table_key, "rowCount": len(rows)},
2391
- "limits": [_report] if _report else [],
2392
  "recordsMutable": _ut().records_mutable(table_key, st=_lent)}
2393
 
2394
 
 
 
 
 
 
 
 
2395
  #: The per-cell provenance a row carries on the wire, one key per enrichment column.
2396
  #: ⛔ A ROW KEY RATHER THAN A SIBLING MAP, and the choice is load-bearing rather than cosmetic.
2397
  #: A `{colId: {pid: state}}` map beside `rows` would need a new PROP on `RecordDetail` and a new
 
68
  raise AttributeError(name)
69
 
70
 
71
+ def _related_product_fields(session, rows, table_key='product_data'):
72
  """Resolve authored Metric definitions through the existing workspace and field wall."""
73
  import aios_grid
74
  from core import grid_events, perm_scope, shared_overlay
75
  from modules import product_data as pd
76
+ import routes_customers
77
+
78
+ scope = {'product_data': 'product', 'customer_data': 'customer'}.get(table_key, table_key)
79
+ if table_key == 'product_data':
80
+ table, base = pd.table_ops(session.runtime), aios_grid.product_fields()
81
+ shared_key = pd.TABLE_KEY
82
+ elif table_key == 'customer_data':
83
+ table, base = routes_customers._customer_table(session), aios_grid.FIELDS
84
+ shared_key = routes_customers._shared_key()
85
+ else:
86
+ table = _ops(session, table_key)
87
+ base = perm_scope.scoped_fields(session.user, table_key, st=session.runtime)
88
+ shared_key = table_key
89
 
90
  pids = frozenset(r["pid"] for r in rows)
91
  ctx = grid_events.EventCtx(uname=session.uname, allowed_pids=pids, fields=[],
92
  hidden_keys=set(), admin=session.admin, fallback_ws=None,
93
+ seen_ids={}, st=session.runtime, scope_key=scope, table=table)
 
94
  ws = grid_events.table_workspace(ctx, allowed_pids=pids, consume_corrections=False)
95
  _, fields, _, _ = aios_grid.workspace_wire(
96
+ ws, session.uname, set(pids), defs={}, scope_key=scope, with_cohorts=False,
97
+ fields_base=base, st=session.runtime)
98
  have = {f["key"] for f in fields}
99
  fields = list(fields) + [dict(f, source="overlay", shared=True)
100
+ for key, f in shared_overlay.fields(shared_key, st=session.runtime).items()
101
  if key not in have]
102
+ hidden = perm_scope.hidden_keys(session.user, table_key, fields, st=session.runtime)
103
  return [f for f in fields if f.get("measure") and f["key"] not in hidden]
104
 
105
 
106
  @router.post("/tables/{table_key}/related-metrics")
107
  def related_metrics(table_key: str, body: dict = Body(default=None),
108
  session: Session = Depends(require_session)):
109
+ """R5: any offered relation, with the rows door's exact walled membership.
110
 
111
  The request supplies only grid identities and a named relation. Fields, periods, source
112
  identities and values are read from this tenant; nothing is authored by this endpoint.
 
118
  import automation_engine
119
 
120
  body = body if isinstance(body, dict) else {}
121
+ pin = body.get("related")
122
  pids = body.get("pids")
123
+ if (not isinstance(pin, dict) or not isinstance(pids, list)
 
 
 
124
  or any(type(pid) is not int or pid <= 0 for pid in pids)):
125
+ raise err(400, "bad_metric_scope", "Name the origin record, relation and linked records.")
126
  viewer = copy.copy(session)
127
  viewer.runtime = _MetricReadRuntime(session.runtime)
128
  try:
129
+ compiled, team_id = routes_grid.related_measure_pin(viewer, table_key, pin, pids)
130
+ # Read actual rows, not the fields-only door introduced by T24.
131
+ products = perm_scope.scoped_table(viewer.user, table_key, st=viewer.runtime)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
132
  requested = set(pids)
133
  selected = [r for r in products if r.get("pid") in requested]
134
+ if len(selected) != len(requested):
 
135
  raise err(403, "metric_record_refused", "The destination records cannot be scoped for this account.")
136
+ fields = _related_product_fields(viewer, products, table_key)
137
+ from core import measure_resolve
138
+ if not perm_scope.may_metrics(viewer.user, table_key):
139
+ offer = []
140
+ elif table_key == 'customer_data':
141
+ offer = measure_resolve.offer(team_id)
142
+ elif table_key == 'product_data':
143
+ offer = routes_products.product_measures(viewer)
144
+ else:
145
+ offer = ut_measures(table_key, viewer)
146
+ admitted = {m['key'] for m in offer}
147
+ fields = [f for f in fields if f['measure'].get('key') in admitted]
148
+ values = automation_engine.related_metric_values(
149
+ viewer.runtime, table_key, selected, fields, offer, compiled, team_id=team_id)
 
 
 
150
  return {"ok": True, "values": values, "fields": [f["key"] for f in fields],
151
+ "refusals": values.refusals,
152
+ "scope": {"table": table_key, "related": pin, "pids": sorted(requested),
153
+ "customer": pin.get('record'),
154
+ "products": {r['pid']: r.get('product_id') for r in selected}},
155
+ "reason": ' '.join(dict.fromkeys(r['message'] for r in values.refusals))}
156
  except (perm_scope.Denied, perm_scope.UnknownTable):
157
  raise err(403, "metric_record_refused", "The related records are not available to this account.")
158
  except (perm_scope.Unresolvable, RuntimeError, ValueError):
159
  raise err(409, "metric_scope_unavailable", "Scoped Metrics could not be read. Reopen the related records after the database is ready.")
160
 
161
 
162
+ def scope_linked_metrics(session, table_key, rows, fields, offer, related):
163
+ """Replace metric cells before a pinned payload can be filtered or aggregated."""
164
+ if not related or not any(f.get('measure') for f in fields):
165
+ return rows, fields, []
166
+ from core import perm_scope
167
+ import routes_grid
168
+ import automation_engine
169
+
170
+ admitted = {m['key'] for m in offer} if perm_scope.may_metrics(session.user, table_key) else set()
171
+ hidden = {f['key'] for f in fields if f.get('measure')
172
+ and f['measure'].get('key') not in admitted}
173
+ fields = [f for f in fields if f['key'] not in hidden]
174
+ rows = [{k: v for k, v in row.items() if k not in hidden} for row in rows]
175
+ metrics = [f for f in fields if f.get('measure')]
176
+ if not metrics:
177
+ return rows, fields, []
178
+ pin, team_id = routes_grid.related_measure_pin(session, table_key, related,
179
+ [row['pid'] for row in rows])
180
+ answer = automation_engine.related_metric_values(session.runtime, table_key, rows, metrics,
181
+ offer, pin, team_id=team_id)
182
+ return [{**row, **answer[row['pid']]} for row in rows], fields, answer.refusals
183
+
184
+
185
+ def linked_metric_limits(refusals):
186
+ """The same refusal accompanies rows, filters and aggregate siblings."""
187
+ if not refusals:
188
+ return []
189
+ return [dict(subject='Linked metrics',
190
+ cause=' '.join(dict.fromkeys(marker['message'] for marker in refusals)),
191
+ recommendation='Check the metric source binding and period, then reopen this tab.')]
192
+
193
+
194
  def _ut():
195
  import core.user_tables as user_tables
196
  return user_tables
 
1562
  # for both — the same sentence `routes_odoo_tables` writes over its own `overlays` dict.
1563
  shared_cells = {pid: {k: v for k, v in cells.items() if k not in hidden}
1564
  for pid, cells in shared_cells.items()}
1565
+ # D-464: the saved views' column keys, on the `/workspace` wire only; `views` stays whole
1566
+ # for `_ut_measure_sets` below.
1567
+ workspace["views"] = aios_grid.narrow_view_keys(workspace.get("views"), hidden)
1568
 
1569
  # ⭐⭐ W37-T12 / owner item 4 (R1) — `measures` IS NO LONGER UNCONDITIONALLY EMPTY. A database
1570
  # whose entity topic declares a `measures:` binding (today `ut_odoo_agents`) serves a real
 
2305
 
2306
 
2307
  @router.get("/tables/{table_key}/rows")
2308
+ def table_rows(table_key: str, session: Session = Depends(require_session),
2309
+ related: str = None):
2310
  """The `/customers`-shaped envelope for one user table: `{fields, rows, today, pulled_at,
2311
  identity}` — so the client's generic topic fetch consumes it with zero new parsing.
2312
 
 
2328
  # PRE-WRITE snapshot, and handing one to a path that writes and then reads back would report the
2329
  # value it replaced (contract C5).
2330
  _lent = _ut().lend(session.runtime)
2331
+ g = ut_assembly(session, table_key, st=_lent, consume_corrections=not bool(related))
2332
  # ⛔ THE OVERLAY WAS NEVER ACTUALLY MERGED, and the docstring above has described the merge
2333
  # this line does not perform since the route was written (owner item 3, 2026-08-09:
2334
  # *"Using the swipe, fast doesn't register the CHANGE. I went back and it all got reseted"*).
 
2390
  # ⭐⭐ WAVE-34 (R13) — the per-cell enrichment STATE rides the row, beside `_created`/`lat`/
2391
  # `lon`. See `_stamp_ai_states` for why it is a row key rather than a map beside `rows`.
2392
  _stamp_ai_states(table_key, g["fields"], rows, st=_lent)
2393
+ from routes_odoo_tables import narrow_related_rows
2394
+ rows = narrow_related_rows(session, table_key, rows, related)
2395
+ rows, g['fields'], metric_refusals = scope_linked_metrics(
2396
+ session, table_key, rows, g['fields'], g['measures'], related)
2397
+ if related:
2398
+ g["pids"] = frozenset(row["pid"] for row in rows)
2399
  # ⭐ R6's SECOND SENTENCE, ON THE WIRE (W30-T29). *"if there is lag or it can't be done, you
2400
  # need to explicitly tell me why and recommend a fix."* A ceiling that still applies to this
2401
  # database says so here, with its cause and the recommendation, rather than waiting to be
 
2417
  if str(pid) in _zone_pids])
2418
  for zone in _ut().zones(table_key, st=_lent)]
2419
  return {"fields": _field_axes(g["fields"]), "rows": rows, "today": g["today"],
2420
+ "refusals": metric_refusals,
2421
  "zones": _zones,
2422
  "docs": _ge.docs_for(g["pids"], scope_key=table_key, uname=session.uname,
2423
  admin=session.admin, st=session.runtime),
2424
  "pulled_at": time.strftime("%Y-%m-%d %H:%M"),
2425
  "identity": {"pid": "pid"},
2426
  "scope": {"table": table_key, "rowCount": len(rows)},
2427
+ "limits": ([_report] if _report else []) + linked_metric_limits(metric_refusals),
2428
  "recordsMutable": _ut().records_mutable(table_key, st=_lent)}
2429
 
2430
 
2431
+ @router.post("/tables/{table_key}/aggregates")
2432
+ def table_aggregates(table_key: str, body: dict = Body(default=None),
2433
+ session: Session = Depends(require_session)):
2434
+ from routes_odoo_tables import odoo_table_aggregates
2435
+ return odoo_table_aggregates(table_key, body=body, session=session)
2436
+
2437
+
2438
  #: The per-cell provenance a row carries on the wire, one key per enrichment column.
2439
  #: ⛔ A ROW KEY RATHER THAN A SIBLING MAP, and the choice is load-bearing rather than cosmetic.
2440
  #: A `{colId: {pid: state}}` map beside `rows` would need a new PROP on `RecordDetail` and a new
api/routes_templates.py CHANGED
@@ -72,13 +72,17 @@ def _target_or_refuse(session, table_key):
72
 
73
 
74
  @router.get("/templates")
75
- def list_templates(table: str = "", session: Session = Depends(require_session)):
76
  """What can be applied to THIS database. `{table, templates: [...]}`.
77
 
78
  Filtered by the target's real columns, so the picker cannot offer something the apply door
79
  would refuse — one predicate, two callers (`view_templates.offer` wraps
80
  `missing_columns`, which is the same function the POST below re-runs).
81
  """
 
 
 
 
82
  fields, source = _target_or_refuse(session, table)
83
  vt = _templates()
84
  return {"table": table, "templates": vt.offer(fields, source or None)}
@@ -172,6 +176,134 @@ def _store(session: Session):
172
  return session.runtime
173
 
174
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
175
  @router.get("/sheet-templates")
176
  def list_sheet_templates(session: Session = Depends(require_session)):
177
  """The workbook gallery, including sources this tenant has not connected."""
@@ -210,28 +342,35 @@ def apply_sheet_template(key: str, body: dict = Body(default=None),
210
  from routes_sheet import _require_database, refresh_sheet
211
 
212
  descriptor = sheet_templates.template(key)
213
- if descriptor is None:
 
 
214
  raise err(404, "unknown_sheet_template", "That sheet template does not exist.")
215
  body = body if isinstance(body, dict) else {}
216
  database_key = body.get("database")
217
  name = body.get("name", "")
218
- if not isinstance(database_key, str) or not database_key.strip():
219
  raise err(404, "unknown_database", "That database does not exist.")
220
  if not isinstance(name, str):
221
  raise err(400, "bad_request", "The workbook name must be text.")
222
- database_key = database_key.strip()
223
  st = _store(session)
224
- if database_key not in user_tables.all_defs(st):
225
  raise err(404, "unknown_database", "That database does not exist.")
226
- _require_database(session, database_key)
227
- if sheet_templates.source_of_database(database_key) != descriptor["source"]:
 
228
  raise err(400, "template_source_mismatch",
229
  "That database uses a different connector from this template.")
230
 
231
  book = None
232
  try:
233
- book = sheet_templates.apply_template(st, key, database_key, name=name,
234
- created_by=session.uname)
 
 
 
 
235
  # Owners read their persisted values. Use the same permission-scoped
236
  # refresh as Sheet so the first open already contains computed cells.
237
  refreshed = refresh_sheet(book["id"], session=session)["workbook"]
@@ -256,3 +395,34 @@ def apply_sheet_template(key: str, body: dict = Body(default=None),
256
  if isinstance(exc, (sheets.SheetRefusal, BindingRefusal)):
257
  raise err(409, exc.code, exc.message) from exc
258
  raise
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
72
 
73
 
74
  @router.get("/templates")
75
+ def list_templates(table: str = "", session: Session = Depends(require_session), module: str = ""):
76
  """What can be applied to THIS database. `{table, templates: [...]}`.
77
 
78
  Filtered by the target's real columns, so the picker cannot offer something the apply door
79
  would refuse — one predicate, two callers (`view_templates.offer` wraps
80
  `missing_columns`, which is the same function the POST below re-runs).
81
  """
82
+ if module:
83
+ if module not in ("sheet", "data", "app"):
84
+ raise err(400, "unknown_module", "Choose Sheet, Data or App.")
85
+ return _module_templates(session)
86
  fields, source = _target_or_refuse(session, table)
87
  vt = _templates()
88
  return {"table": table, "templates": vt.offer(fields, source or None)}
 
176
  return session.runtime
177
 
178
 
179
+ def _saved_sheet_templates(st):
180
+ """C4 metadata only. An absent or unrecognised document offers no saved templates."""
181
+ raw = st.get("sheet_templates_saved")
182
+ if not isinstance(raw, dict) or raw.get("schema") != 1:
183
+ return []
184
+ rows = raw.get("templates")
185
+ if not isinstance(rows, dict):
186
+ return []
187
+ return [row for key, row in rows.items()
188
+ if isinstance(row, dict) and row.get("key") == key
189
+ and isinstance(row.get("name"), str)
190
+ and isinstance(row.get("description"), str)
191
+ and isinstance(row.get("workbook"), dict)
192
+ and isinstance(row.get("databases"), list)
193
+ and all(isinstance(db, str) and db.strip() for db in row["databases"])]
194
+
195
+
196
+ def _database_states(session, definitions, databases):
197
+ """Resolve switches once per request without rereading the stored table document."""
198
+ from core import registry, user_tables
199
+
200
+ # With no arguments the registrar returns its current keys without changing them.
201
+ # Match is_connected's three legs using the definitions projection already in hand.
202
+ registered = user_tables.register_connected()
203
+ prefixes = user_tables.connected_prefixes()
204
+ states = {}
205
+ for database in set(databases):
206
+ key = str(database or "").strip()
207
+ definition = definitions.get(key) or {}
208
+ if registry.BY_KEY.get(database):
209
+ states[database] = perms.tenant_has_module(session.runtime, database)
210
+ else:
211
+ # Process-wide registration alone must not admit another tenant's database.
212
+ states[database] = database in definitions and (
213
+ key in registered or definition.get("connected") is True
214
+ or (any(key.startswith(prefix) for prefix in prefixes)
215
+ and definition.get("source") == user_tables.AUTOMATION_SOURCE
216
+ and definition.get("recordMode") == user_tables.AUTOMATION_RECORD_MODE))
217
+ return states
218
+
219
+
220
+ def _requirement(database, definitions, have, database_states, connector=""):
221
+ from core import registry
222
+ from routes_connectors import _module_connection_key, _table_connection_key
223
+
224
+ definition = definitions.get(database) or {}
225
+ module = registry.BY_KEY.get(database) or {}
226
+ connector = (connector or _table_connection_key(database, definition)
227
+ or _module_connection_key(module.get("subject")))
228
+ # These two compiled connector databases replaced the retired ut_odoo_* copies.
229
+ # Their switch is tenant admission; stored databases use the connector registrar.
230
+ switched_on = database_states[database] if module or connector else bool(definition)
231
+ label = definition.get("label") or module.get("label")
232
+ if not label:
233
+ parts = database.removeprefix("ut_").split("_")
234
+ brand = {"odoo": "Odoo", "amzn": "Amazon", "ig": "Instagram"}.get(parts[0], parts[0].capitalize())
235
+ label = brand + " " + " ".join(parts[1:])
236
+ return {"connector": connector, "database": database,
237
+ "label": str(label).replace(chr(0x2014), "-").replace(chr(0x2013), "-"),
238
+ "state": "disconnected" if connector and connector not in have
239
+ else "ready" if switched_on else "off"}
240
+
241
+
242
+ def _catalog_row(key, name, description, requires):
243
+ # Prefer the missing connection even when an earlier database is merely off.
244
+ blocked = next((row for row in requires if row["state"] == "disconnected"), None)
245
+ blocked = blocked or next((row for row in requires if row["state"] != "ready"), None)
246
+ return {"key": key, "name": str(name).replace(chr(0x2014), "-").replace(chr(0x2013), "-"),
247
+ "description": str(description).replace(chr(0x2014), "-").replace(chr(0x2013), "-"),
248
+ "requires": requires, "creatable": all(row["state"] == "ready" for row in requires),
249
+ "action": None if blocked is None else {
250
+ "kind": "connect" if blocked["state"] == "disconnected" else "switch_on",
251
+ "connector": blocked["connector"], "database": blocked["database"]}}
252
+
253
+
254
+ def _data_template_database(template, definitions):
255
+ fixed = {"odoo_customer": "customer_data", "odoo_product": "product_data"}
256
+ if template["source"] in fixed:
257
+ return fixed[template["source"]]
258
+ from routes_connectors import _table_connection_key
259
+ for key, definition in definitions.items():
260
+ fields = [field.get("key") for field in definition.get("fields", [])]
261
+ if (_table_connection_key(key, definition) == template["source"]
262
+ and not _templates().missing_columns(template, fields)):
263
+ return key
264
+ return ""
265
+
266
+
267
+ def _module_templates(session):
268
+ from core import sheet_templates, user_tables
269
+ from routes_connectors import _entry_types
270
+ from routes_keychain import env_odoo_available
271
+ from routes_nav import application_templates
272
+
273
+ definitions = user_tables.all_defs(session.runtime)
274
+ have, _scopes, _failure = _entry_types(session.runtime, session.uname)
275
+ have = set(have)
276
+ if env_odoo_available(session.runtime):
277
+ have.add("odoo")
278
+ saved = _saved_sheet_templates(session.runtime)
279
+ data_templates = [(row, _data_template_database(row, definitions))
280
+ for row in _templates().TEMPLATES]
281
+ databases = {row["database_hint"] for row in sheet_templates.TEMPLATES}
282
+ databases.update(db for row in saved for db in row["databases"])
283
+ databases.update(database for _row, database in data_templates if database)
284
+ database_states = _database_states(session, definitions, databases)
285
+ requirement = lambda database, connector="": _requirement(
286
+ database, definitions, have, database_states, connector)
287
+ sheet = [_catalog_row(row["key"], row["title"], row["answers"],
288
+ [requirement(row["database_hint"], row["source"])])
289
+ for row in sheet_templates.TEMPLATES]
290
+ sheet.extend(_catalog_row(row["key"], row["name"], row["description"],
291
+ [requirement(db) for db in row["databases"]])
292
+ for row in saved)
293
+ data = []
294
+ for row, database in data_templates:
295
+ if database:
296
+ connector = "odoo" if row["source"].startswith("odoo_") else row["source"]
297
+ data.append(_catalog_row(row["key"], row["label"], row["desc"],
298
+ [requirement(database, connector)]))
299
+ # These App contracts map fields in the existing wizard and require no fixed connector.
300
+ app = [_catalog_row(row["kind"], row["label"], "Choose a database and map its fields.", [])
301
+ for row in application_templates(session=session)["templates"]]
302
+ return {"sections": [{"module": "sheet", "templates": sheet},
303
+ {"module": "data", "templates": data},
304
+ {"module": "app", "templates": app}]}
305
+
306
+
307
  @router.get("/sheet-templates")
308
  def list_sheet_templates(session: Session = Depends(require_session)):
309
  """The workbook gallery, including sources this tenant has not connected."""
 
342
  from routes_sheet import _require_database, refresh_sheet
343
 
344
  descriptor = sheet_templates.template(key)
345
+ saved = (None if descriptor else
346
+ next((row for row in _saved_sheet_templates(_store(session)) if row["key"] == key), None))
347
+ if descriptor is None and saved is None:
348
  raise err(404, "unknown_sheet_template", "That sheet template does not exist.")
349
  body = body if isinstance(body, dict) else {}
350
  database_key = body.get("database")
351
  name = body.get("name", "")
352
+ if saved is None and (not isinstance(database_key, str) or not database_key.strip()):
353
  raise err(404, "unknown_database", "That database does not exist.")
354
  if not isinstance(name, str):
355
  raise err(400, "bad_request", "The workbook name must be text.")
356
+ database_key = database_key.strip() if isinstance(database_key, str) else ""
357
  st = _store(session)
358
+ if saved is None and database_key not in user_tables.all_defs(st):
359
  raise err(404, "unknown_database", "That database does not exist.")
360
+ for required in saved["databases"] if saved else [database_key]:
361
+ _require_database(session, required)
362
+ if descriptor and sheet_templates.source_of_database(database_key) != descriptor["source"]:
363
  raise err(400, "template_source_mismatch",
364
  "That database uses a different connector from this template.")
365
 
366
  book = None
367
  try:
368
+ if saved:
369
+ # Lane G owns C4 materialisation; never reconstruct its workbook here.
370
+ book = sheet_templates.apply_saved_template(st, key, name=name, created_by=session.uname)
371
+ else:
372
+ book = sheet_templates.apply_template(st, key, database_key, name=name,
373
+ created_by=session.uname)
374
  # Owners read their persisted values. Use the same permission-scoped
375
  # refresh as Sheet so the first open already contains computed cells.
376
  refreshed = refresh_sheet(book["id"], session=session)["workbook"]
 
395
  if isinstance(exc, (sheets.SheetRefusal, BindingRefusal)):
396
  raise err(409, exc.code, exc.message) from exc
397
  raise
398
+
399
+
400
+ # Declaration order is load bearing: keep this AFTER /templates/{key}/apply.
401
+ # Both paths have three segments; reversing them steals /templates/x/apply.
402
+ @router.post("/templates/{module}/{key}")
403
+ def create_module_template(module: str, key: str, body: dict = Body(default=None),
404
+ session: Session = Depends(require_session)):
405
+ from routes_nav import create_nav_application
406
+ from urllib.parse import quote
407
+
408
+ body = body if isinstance(body, dict) else {}
409
+ section = next((row for row in _module_templates(session)["sections"] if row["module"] == module), None)
410
+ template = next((row for row in section["templates"] if row["key"] == key), None) if section else None
411
+ if template is None:
412
+ raise err(404, "unknown_template", "That template does not exist.")
413
+ if not template["creatable"]:
414
+ raise err(409, "template_unavailable", "Connect the required source and switch on its databases.")
415
+ if module == "sheet":
416
+ database = template["requires"][0]["database"] if template["requires"] else ""
417
+ result = apply_sheet_template(key, {"database": database, "name": body.get("name", "")}, session)
418
+ return {"address": "#/sheet?wb=" + quote(result["workbook"]["id"], safe=""), **result}
419
+ if module == "data":
420
+ database = template["requires"][0]["database"]
421
+ result = apply_template(key, {"table": database}, session)
422
+ return {"address": "#/" + quote(database, safe=""), **result}
423
+ # The existing wizard still owns its POST-then-PATCH share sequence. API clients may
424
+ # dispatch creation here; the existing App writer retains schema and permission walls.
425
+ result = create_nav_application({**body, "templateKind": key}, session=session)
426
+ app = result["application"]
427
+ return {"address": "#/app?database=" + quote(app["sourceDatabase"], safe="") + "&application=" + quote(app["id"], safe=""),
428
+ **result}
api/semantic_registry.py CHANGED
@@ -9,6 +9,7 @@ import copy
9
  import hashlib
10
  import json
11
  from collections import Counter
 
12
 
13
 
14
  SEMANTIC_STATES = ("draft", "approved", "disabled")
@@ -17,6 +18,19 @@ VALUE_PROFILE_CAP = 50
17
  VALUE_SAMPLE_CAP = 12
18
  _KEY = "semantic_registry_v1"
19
 
 
 
 
 
 
 
 
 
 
 
 
 
 
20
 
21
  class SemanticError(ValueError):
22
  """A semantic contract was refused before persistence or retrieval."""
@@ -42,36 +56,48 @@ def field_key(database, field):
42
 
43
 
44
  def model_record(database, topic, field):
45
- """Build the governed baseline from a topic field declaration.
46
-
47
- Existing topic files predate approval. They therefore enter as ``draft`` unless the
48
- declaration explicitly says otherwise; merely having a label never grants retrieval.
49
- """
50
  field = field if isinstance(field, dict) else {}
51
- semantic = field.get("semantic") if isinstance(field.get("semantic"), dict) else {}
 
 
 
 
 
 
 
 
 
 
52
  state = str(semantic.get("state") or "draft").lower()
53
  sensitivity = str(semantic.get("sensitivity") or "internal").lower()
54
  if state not in SEMANTIC_STATES:
55
  raise SemanticError(f"unknown semantic state {state!r}")
56
  if sensitivity not in SENSITIVITY:
57
  raise SemanticError(f"unknown sensitivity {sensitivity!r}")
58
- return {
59
  "database": _text(database, 120),
60
  "topic": _text((topic or {}).get("key"), 120),
61
  "field": _text(field.get("key"), 120),
62
  "label": _text(field.get("label") or field.get("key"), 120),
63
- "purpose": _text(semantic.get("purpose") or field.get("means"), 500),
64
  "dataType": _text(field.get("type") or "text", 40),
65
  "role": _text(semantic.get("role") or "attribute", 60),
66
  "synonyms": _strings(semantic.get("synonyms")),
 
 
 
 
67
  "sensitivity": sensitivity,
68
  "state": state,
69
  "valueProfile": None,
70
  "revision": int(semantic.get("revision") or 1),
71
- "approvedBy": _text(semantic.get("approvedBy"), 120) or None,
 
 
72
  "disabledBy": _text(semantic.get("disabledBy"), 120) or None,
73
  "profiledBy": _text(semantic.get("profiledBy"), 120) or None,
74
- }
75
 
76
 
77
  def topic_records(database, topic):
@@ -82,7 +108,39 @@ def topic_records(database, topic):
82
  }
83
 
84
 
85
- def _clean_override(raw, baseline, *, actor, is_admin):
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
86
  if not is_admin:
87
  raise SemanticError("only a tenant admin can change semantic approval")
88
  raw = raw if isinstance(raw, dict) else {}
@@ -111,6 +169,7 @@ def _clean_override(raw, baseline, *, actor, is_admin):
111
  raise SemanticError("semantic state is not recognized")
112
  out["state"] = state
113
  out["approvedBy"] = _text(actor, 120) if state == "approved" else None
 
114
  out["disabledBy"] = _text(actor, 120) if state == "disabled" else None
115
  out["revision"] = int(baseline.get("revision") or 0) + 1
116
  return out
@@ -127,7 +186,7 @@ def merge_records(baseline, persisted):
127
  if key not in out or not isinstance(raw, dict):
128
  continue
129
  for name in ("purpose", "role", "synonyms", "sensitivity", "state", "valueProfile",
130
- "revision", "approvedBy", "disabledBy", "profiledBy"):
131
  if name in raw:
132
  out[key][name] = copy.deepcopy(raw[name])
133
  out[key] = enforce_redaction(out[key])
@@ -142,12 +201,12 @@ def load(runtime, baseline):
142
  return merge_records(baseline, raw)
143
 
144
 
145
- def update(runtime, baseline, database, field, patch, *, actor, is_admin):
146
  key = field_key(database, field)
147
  if key not in baseline:
148
  raise SemanticError("that field is not in the database contract")
149
  current = load(runtime, baseline)[key]
150
- changed = _clean_override(patch, current, actor=actor, is_admin=is_admin)
151
 
152
  def apply(raw):
153
  doc = copy.deepcopy(raw) if isinstance(raw, dict) else {}
@@ -240,6 +299,85 @@ def value_profile(values, *, sensitivity="internal", distinct_cap=VALUE_PROFILE_
240
  return {"redacted": False, "distinctCount": count, "values": [], "sample": sample}
241
 
242
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
243
  def generated_markdown(record):
244
  """A deterministic projection. Sensitive profiles carry no values by construction."""
245
  row = enforce_redaction(record)
 
9
  import hashlib
10
  import json
11
  from collections import Counter
12
+ from datetime import datetime, timezone
13
 
14
 
15
  SEMANTIC_STATES = ("draft", "approved", "disabled")
 
18
  VALUE_SAMPLE_CAP = 12
19
  _KEY = "semantic_registry_v1"
20
 
21
+ # The catalogue fields_get attribute list belongs to odoo_relational.py (lane N); add selection there to replace profile-derived choice values with Odoo choices.
22
+ ODOO_SEMANTIC_TYPES = {
23
+ "char": "text", "text": "text", "html": "text", "selection": "select",
24
+ "boolean": "checkbox", "integer": "int", "float": "number", "monetary": "currency",
25
+ "date": "date", "datetime": "datetime", "many2one": "link", "one2many": "link",
26
+ "many2many": "link", "binary": "text",
27
+ }
28
+ _ODOO_PII_NAMES = (
29
+ "email", "phone", "mobile", "fax", "vat", "street", "zip", "bank", "acc_number",
30
+ "iban", "password", "passwd", "token", "api_key", "secret", "signature", "ssn",
31
+ "birthday", "private",
32
+ )
33
+
34
 
35
  class SemanticError(ValueError):
36
  """A semantic contract was refused before persistence or retrieval."""
 
56
 
57
 
58
  def model_record(database, topic, field):
59
+ """Build a baseline from topic curation, with per-field declarations taking precedence."""
 
 
 
 
60
  field = field if isinstance(field, dict) else {}
61
+ key = field.get("key")
62
+ curated = (topic or {}).get("semantic") or {}
63
+ defaults = {
64
+ "state": "approved" if key in (curated.get("approved") or ()) else "draft",
65
+ "sensitivity": "sensitive" if key in (curated.get("sensitive") or ()) else "internal",
66
+ "purpose": (curated.get("purposes") or {}).get(key, field.get("means") or ""),
67
+ "synonyms": (curated.get("synonyms") or {}).get(key, []),
68
+ "choices": (curated.get("choices") or {}).get(key, []),
69
+ }
70
+ field_meta = field.get("semantic") if isinstance(field.get("semantic"), dict) else {}
71
+ semantic = {**defaults, **field_meta} # Per-field curation wins.
72
  state = str(semantic.get("state") or "draft").lower()
73
  sensitivity = str(semantic.get("sensitivity") or "internal").lower()
74
  if state not in SEMANTIC_STATES:
75
  raise SemanticError(f"unknown semantic state {state!r}")
76
  if sensitivity not in SENSITIVITY:
77
  raise SemanticError(f"unknown sensitivity {sensitivity!r}")
78
+ return enforce_redaction({
79
  "database": _text(database, 120),
80
  "topic": _text((topic or {}).get("key"), 120),
81
  "field": _text(field.get("key"), 120),
82
  "label": _text(field.get("label") or field.get("key"), 120),
83
+ "purpose": _text(semantic.get("purpose"), 500),
84
  "dataType": _text(field.get("type") or "text", 40),
85
  "role": _text(semantic.get("role") or "attribute", 60),
86
  "synonyms": _strings(semantic.get("synonyms")),
87
+ "relation": "",
88
+ "choices": copy.deepcopy(semantic.get("choices") or []),
89
+ "choicesFrom": "curated" if key in (curated.get("choices") or {}) or
90
+ "choices" in field_meta else ("profile" if field.get("type") == "select" else ""),
91
  "sensitivity": sensitivity,
92
  "state": state,
93
  "valueProfile": None,
94
  "revision": int(semantic.get("revision") or 1),
95
+ "approvedBy": (_text(semantic.get("approvedBy"), 120) or "platform baseline")
96
+ if state == "approved" else None,
97
+ "approvedAt": None,
98
  "disabledBy": _text(semantic.get("disabledBy"), 120) or None,
99
  "profiledBy": _text(semantic.get("profiledBy"), 120) or None,
100
+ })
101
 
102
 
103
  def topic_records(database, topic):
 
108
  }
109
 
110
 
111
+ def generated_records(database, model_row, *, model=""):
112
+ """Pre-approved Odoo schema from the cached model's own fields, never inferred fields."""
113
+ out = {}
114
+ model = model or model_row.get("model") or ""
115
+ for name, metadata in (model_row.get("fields") or {}).items():
116
+ if not isinstance(metadata, dict) or not name:
117
+ continue
118
+ odoo_type = metadata.get("type")
119
+ relation = str(metadata.get("relation") or "")
120
+ sensitive = any(part in str(name).casefold() for part in _ODOO_PII_NAMES)
121
+ choices = metadata.get("selection")
122
+ choices = copy.deepcopy(list(choices)) if isinstance(choices, (list, tuple)) else []
123
+ role = ("link" if relation else "measure" if odoo_type in ("monetary", "float") else
124
+ "date" if odoo_type in ("date", "datetime") else "attribute")
125
+ row = model_record(database, {}, {
126
+ "key": name, "label": metadata.get("string") or name,
127
+ "type": ODOO_SEMANTIC_TYPES.get(odoo_type, "text"),
128
+ "semantic": {"state": "approved", "approvedBy": "odoo connector baseline",
129
+ "sensitivity": "sensitive" if sensitive else "internal", "role": role,
130
+ "purpose": f"The {name} field on Odoo model {model}."},
131
+ })
132
+ row.update({"relation": relation, "choices": choices,
133
+ "choicesFrom": "odoo" if choices else
134
+ ("profile" if odoo_type == "selection" else "")})
135
+ out[field_key(database, name)] = row
136
+ return out
137
+
138
+
139
+ def _approval_time(now=None):
140
+ return (now or datetime.now(timezone.utc)).astimezone(timezone.utc).isoformat()
141
+
142
+
143
+ def _clean_override(raw, baseline, *, actor, is_admin, now=None):
144
  if not is_admin:
145
  raise SemanticError("only a tenant admin can change semantic approval")
146
  raw = raw if isinstance(raw, dict) else {}
 
169
  raise SemanticError("semantic state is not recognized")
170
  out["state"] = state
171
  out["approvedBy"] = _text(actor, 120) if state == "approved" else None
172
+ out["approvedAt"] = _approval_time(now) if state == "approved" else None
173
  out["disabledBy"] = _text(actor, 120) if state == "disabled" else None
174
  out["revision"] = int(baseline.get("revision") or 0) + 1
175
  return out
 
186
  if key not in out or not isinstance(raw, dict):
187
  continue
188
  for name in ("purpose", "role", "synonyms", "sensitivity", "state", "valueProfile",
189
+ "revision", "approvedBy", "approvedAt", "disabledBy", "profiledBy"):
190
  if name in raw:
191
  out[key][name] = copy.deepcopy(raw[name])
192
  out[key] = enforce_redaction(out[key])
 
201
  return merge_records(baseline, raw)
202
 
203
 
204
+ def update(runtime, baseline, database, field, patch, *, actor, is_admin, now=None):
205
  key = field_key(database, field)
206
  if key not in baseline:
207
  raise SemanticError("that field is not in the database contract")
208
  current = load(runtime, baseline)[key]
209
+ changed = _clean_override(patch, current, actor=actor, is_admin=is_admin, now=now)
210
 
211
  def apply(raw):
212
  doc = copy.deepcopy(raw) if isinstance(raw, dict) else {}
 
299
  return {"redacted": False, "distinctCount": count, "values": [], "sample": sample}
300
 
301
 
302
+ def draft_fields(records, snapshot, *, limit=8):
303
+ """Rank only source-declared, visible fields using the caller's real scoped rows."""
304
+ rows = [row for row in snapshot.get("records") or () if isinstance(row, dict)]
305
+ visible = {row["key"] for row in snapshot.get("fields") or () if row.get("key")}
306
+ ranked = []
307
+ for record in (records or {}).values():
308
+ field = record.get("field")
309
+ if (record.get("database") != snapshot.get("database") or field not in visible
310
+ or record.get("state") == "disabled"):
311
+ continue
312
+ values = [row.get(field) for row in rows]
313
+ populated = [value for value in values if value is not None and value != ""
314
+ and value != [] and value != {}]
315
+ role, kind = record.get("role") or "attribute", record.get("dataType") or "text"
316
+ linked = bool(record.get("relation")) or role == "link" or kind == "link"
317
+ dated = role == "date" or kind in ("date", "datetime", "created_time", "modified_time")
318
+ identifier = field.lower() in ("id", "pid", "uuid") or field.lower().endswith("_id")
319
+ evidence = f"Filled on {len(populated)} of {len(rows)} rows"
320
+ detail = (", and it links this database to related records" if linked else
321
+ ", and it is a date this database can be scoped by" if dated else
322
+ ", and it identifies a record" if identifier else "")
323
+ rank = len(populated) / max(1, len(rows)) + (0.4 if linked or dated else 0)
324
+ rank -= 1 if identifier and not linked else 0
325
+ sensitivity = record.get("sensitivity") or "internal"
326
+ ranked.append((rank, field, {
327
+ "field": field, "label": record.get("label") or field,
328
+ "why": evidence + detail + ".", "role": "link" if linked else "date" if dated else role,
329
+ "dataType": kind, "sensitivity": sensitivity,
330
+ "sample": value_profile(populated, sensitivity=sensitivity),
331
+ }))
332
+ ranked.sort(key=lambda item: (-item[0], item[1]))
333
+ return [item[2] for item in ranked[:max(0, min(int(limit), 8))]]
334
+
335
+
336
+ def pending_approval_key():
337
+ return "semantic_draft_requests_v1"
338
+
339
+
340
+ def record_draft_request(runtime, database, question, fields, *, actor, now=None):
341
+ """Queue field keys only; shared requests never persist samples from a member's rows."""
342
+ request = {"database": database, "question": _text(question, 500),
343
+ "fields": list(dict.fromkeys(fields))[:8], "requestedBy": _text(actor, 120),
344
+ "requestedAt": _approval_time(now), "state": "pending",
345
+ "approvedBy": None, "approvedAt": None}
346
+
347
+ def apply(raw):
348
+ doc = copy.deepcopy(raw) if isinstance(raw, dict) else {}
349
+ doc["schema"] = 1
350
+ doc.setdefault("requests", {})[database] = request
351
+ return doc
352
+
353
+ runtime.update(pending_approval_key(), apply)
354
+ return copy.deepcopy(request)
355
+
356
+
357
+ def draft_request(runtime, database):
358
+ raw = runtime.get(pending_approval_key()) or {}
359
+ row = (raw.get("requests") or {}).get(database)
360
+ return copy.deepcopy(row) if isinstance(row, dict) else None
361
+
362
+
363
+ def approve_draft_request(runtime, database, fields, *, actor, is_admin, now=None):
364
+ if not is_admin:
365
+ raise SemanticError("only a tenant admin can change semantic approval")
366
+ landed = {}
367
+
368
+ def apply(raw):
369
+ doc = copy.deepcopy(raw) if isinstance(raw, dict) else {"schema": 1, "requests": {}}
370
+ row = doc.setdefault("requests", {}).get(database)
371
+ if row:
372
+ row.update({"fields": list(dict.fromkeys(fields))[:8], "state": "approved",
373
+ "approvedBy": _text(actor, 120), "approvedAt": _approval_time(now)})
374
+ landed.update(row)
375
+ return doc
376
+
377
+ runtime.update(pending_approval_key(), apply)
378
+ return copy.deepcopy(landed)
379
+
380
+
381
  def generated_markdown(record):
382
  """A deterministic projection. Sensitive profiles carry no values by construction."""
383
  row = enforce_redaction(record)
api/test_odoo_rank_refresh.py CHANGED
@@ -5,7 +5,9 @@ import copy
5
  import io
6
  import os
7
  from pathlib import Path
 
8
  import socket
 
9
  import sys
10
  import tempfile
11
  from types import SimpleNamespace
@@ -17,6 +19,59 @@ import duckdb
17
  from harness import datastore as ds
18
 
19
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
20
  class RankRefreshTests(unittest.TestCase):
21
  def setUp(self):
22
  self.stack = ExitStack()
@@ -146,16 +201,26 @@ class RankRefreshTests(unittest.TestCase):
146
  # tests died on it -- in the MERGED tree only: lane G does not run this
147
  # gate and lane D's run predates lane G. The real `main.py` defines the
148
  # function at module scope and the deployed Space boots fine.
 
 
149
  _resync_mirror_pass=sync, _pull_meta=lambda *_: True,
150
  _pull_amazon=lambda *_: True,
151
  _rebuild_odoo_relational=rebuild,
152
  _rebuild_meta_relational=lambda *_: {},
153
  _rebuild_amazon_relational=lambda *_: {})
 
 
 
154
  exec(compile(ast.Module(body=nodes, type_ignores=[]),
155
  'main.py:resident-loop', 'exec'), ns)
156
  runtime = SimpleNamespace(get_runtime=lambda *_: object())
 
 
 
157
  with patch.dict(sys.modules, {'time': clock, 'harness.runtime': runtime}), \
158
  patch.object(harness, 'runtime', runtime, create=True), \
 
 
159
  redirect_stdout(io.StringIO()):
160
  with self.assertRaisesRegex(StopIteration, 'bounded synthetic'):
161
  ns['_store_resync_loop']()
 
5
  import io
6
  import os
7
  from pathlib import Path
8
+ import builtins
9
  import socket
10
+ import symtable
11
  import sys
12
  import tempfile
13
  from types import SimpleNamespace
 
19
  from harness import datastore as ds
20
 
21
 
22
+ class FixtureGap(AssertionError):
23
+ """The synthetic namespace is missing a collaborator. A fixture defect, never a loop failure."""
24
+
25
+
26
+ def _free_globals(node):
27
+ """Global names `node` reads, from Python's own symbol table rather than a name list."""
28
+ names = set()
29
+ def walk(table):
30
+ for sym in table.get_symbols():
31
+ if sym.is_referenced() and (
32
+ table.get_type() == 'module' and not (sym.is_assigned() or sym.is_imported())
33
+ or table.get_type() != 'module' and sym.is_global()):
34
+ names.add(sym.get_name())
35
+ for child in table.get_children():
36
+ walk(child)
37
+ walk(symtable.symtable(ast.unparse(node), 'main.py', 'exec'))
38
+ return names - set(dir(builtins))
39
+
40
+
41
+ def _with_collaborators(source, nodes, ns):
42
+ """`nodes` plus every top-level main.py def/class/assign they transitively call.
43
+
44
+ W-228: the namespace below was hand-maintained, so `_run_sync_requests` arrived as a bare
45
+ NameError that named the resident loop. Explicit stubs in `ns` still win. A name that is
46
+ neither stubbed nor a top-level def/class/assign (an import, say) raises a FixtureGap.
47
+ """
48
+ top = {}
49
+ for node in source.body:
50
+ if isinstance(node, (ast.FunctionDef, ast.ClassDef)):
51
+ top.setdefault(node.name, node)
52
+ elif isinstance(node, ast.Assign):
53
+ for target in node.targets:
54
+ if isinstance(target, ast.Name):
55
+ top.setdefault(target.id, node)
56
+ have = list(nodes)
57
+ defined = {name for name, node in top.items() if node in have}
58
+ todo = [name for node in have for name in _free_globals(node)]
59
+ while todo:
60
+ name = todo.pop()
61
+ if name in ns or name in defined:
62
+ continue
63
+ node = top.get(name)
64
+ if node is None:
65
+ raise FixtureGap(
66
+ f'FIXTURE GAP, not a resident-loop failure: main.py:_store_resync_loop reads '
67
+ f'{name!r}, which this test neither stubs nor can extract as a top-level '
68
+ f'def/class/assign of main.py. Stub it in observe_resident_recovery.')
69
+ have.append(node)
70
+ defined |= {n for n, candidate in top.items() if candidate is node}
71
+ todo.extend(_free_globals(node))
72
+ return have
73
+
74
+
75
  class RankRefreshTests(unittest.TestCase):
76
  def setUp(self):
77
  self.stack = ExitStack()
 
201
  # tests died on it -- in the MERGED tree only: lane G does not run this
202
  # gate and lane D's run predates lane G. The real `main.py` defines the
203
  # function at module scope and the deployed Space boots fine.
204
+ # W-228: no longer a bare NameError. Unstubbed collaborators are extracted
205
+ # from main.py by `_with_collaborators`, or fail as a named FixtureGap.
206
  _resync_mirror_pass=sync, _pull_meta=lambda *_: True,
207
  _pull_amazon=lambda *_: True,
208
  _rebuild_odoo_relational=rebuild,
209
  _rebuild_meta_relational=lambda *_: {},
210
  _rebuild_amazon_relational=lambda *_: {})
211
+ # W-228: the real helpers the loop calls (`_run_sync_requests` and what it needs) are
212
+ # extracted too; anything unresolvable fails as a named FixtureGap.
213
+ nodes = _with_collaborators(source, nodes, ns)
214
  exec(compile(ast.Module(body=nodes, type_ignores=[]),
215
  'main.py:resident-loop', 'exec'), ns)
216
  runtime = SimpleNamespace(get_runtime=lambda *_: object())
217
+ from core import connector_sync
218
+ # The cadence wait is now an interruptible event, so it is routed to the synthetic clock.
219
+ wake = SimpleNamespace(wait=sleep, clear=lambda: None)
220
  with patch.dict(sys.modules, {'time': clock, 'harness.runtime': runtime}), \
221
  patch.object(harness, 'runtime', runtime, create=True), \
222
+ patch.object(connector_sync, 'sync_wake_event', return_value=wake), \
223
+ patch.object(connector_sync, 'take_sync_requests', return_value=[]), \
224
  redirect_stdout(io.StringIO()):
225
  with self.assertRaisesRegex(StopIteration, 'bounded synthetic'):
226
  ns['_store_resync_loop']()
platform/aios_grid.py CHANGED
@@ -471,6 +471,11 @@ def _clean_permissions(raw):
471
  #: # $ % TEXT number formats already accepted by the client; `$` is also a regex anchor.
472
  #:
473
  #: `;` stays refused: no legal formula token needs a statement separator.
 
 
 
 
 
474
  _FORMULA_CHARS = re.compile(r"^[\w\s{}()+\-*/.,<>=!^&\"'\[\]|?:\\#$%@]*$")
475
  _FORMULA_REF = re.compile(r"\{([^{}]*)\}")
476
 
@@ -3783,6 +3788,49 @@ def views_from_defs(defs, saved_views, fields, system_name="All customers", lock
3783
  return views
3784
 
3785
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
3786
  def workspace_wire(ws, uname, pool_pids, defs=None, scope_key='customer', storage_key=None,
3787
  fields_base=None, with_cohorts=True, st=None):
3788
  """The client's `GridWorkspace` WIRE SHAPE from the stored table workspace — the ONE
 
471
  #: # $ % TEXT number formats already accepted by the client; `$` is also a regex anchor.
472
  #:
473
  #: `;` stays refused: no legal formula token needs a statement separator.
474
+ #:
475
+ #: ⛔ D-547: A CHARACTER FILTER, NOT A SAFETY BOUNDARY. `__import__("os")` passes it (word
476
+ #: characters, parens, quotes). Such a source is inert because the ENGINE refuses it: the server
477
+ #: parser door in `aios-web/api/formula_runtime.py` is an AST interpreter over a function allowlist
478
+ #: with no Python eval. `api/verify_formula_storage.py` records both halves; never read this as the wall.
479
  _FORMULA_CHARS = re.compile(r"^[\w\s{}()+\-*/.,<>=!^&\"'\[\]|?:\\#$%@]*$")
480
  _FORMULA_REF = re.compile(r"\{([^{}]*)\}")
481
 
 
3788
  return views
3789
 
3790
 
3791
+ def narrow_view_keys(views, hidden):
3792
+ """D-464: a WIRE COPY of `views` whose column-bearing config names no key in `hidden`.
3793
+
3794
+ `views_from_defs` merges saved configs verbatim and runs before the field wall exists, so a
3795
+ hidden field's KEY rode `/workspace` in `order`/`visible`/`widths`/`sorts`/`groupBy`/`colorBy`
3796
+ (and `stackField`, `summary`, `display.*Field`) to the very reader the wall targets. Each
3797
+ assembly calls this beside its overlay narrowing,
3798
+ on `workspace["views"]` only: `g["views"]` stays whole for server-side work (exports, measure
3799
+ sets), and a walled reader's own autosave is already cleaned against their walled keys.
3800
+ ⚠ FILTER LEAVES (and `secondaryView`) ARE LEFT ALONE: dropping a leaf would change which rows
3801
+ a view shows, so a hidden key named only in a filter still rides this wire."""
3802
+ hidden = set(hidden or ())
3803
+ if not hidden:
3804
+ return views
3805
+ out = []
3806
+ for view in views or []:
3807
+ cfg = view.get("config") if isinstance(view, dict) else None
3808
+ if not isinstance(cfg, dict):
3809
+ out.append(view)
3810
+ continue
3811
+ cfg = dict(cfg)
3812
+ for name in ("order", "visible"):
3813
+ if isinstance(cfg.get(name), list):
3814
+ cfg[name] = [key for key in cfg[name] if key not in hidden]
3815
+ if isinstance(cfg.get("widths"), dict):
3816
+ cfg["widths"] = {key: w for key, w in cfg["widths"].items() if key not in hidden}
3817
+ if isinstance(cfg.get("sorts"), list):
3818
+ cfg["sorts"] = [s for s in cfg["sorts"]
3819
+ if not (isinstance(s, dict) and s.get("colId") in hidden)]
3820
+ for name in ("groupBy", "colorBy", "stackField"):
3821
+ if isinstance(cfg.get(name), str) and cfg[name] in hidden:
3822
+ cfg[name] = None
3823
+ if isinstance(cfg.get("summary"), dict):
3824
+ cfg["summary"] = {key: s for key, s in cfg["summary"].items() if key not in hidden}
3825
+ if isinstance(cfg.get("display"), dict):
3826
+ # `DisplaySpec`'s `dateField`/`stackField`/`titleField`/`colorField`/... members.
3827
+ cfg["display"] = {name: value for name, value in cfg["display"].items()
3828
+ if not (name.endswith("Field") and isinstance(value, str)
3829
+ and value in hidden)}
3830
+ out.append(dict(view, config=cfg))
3831
+ return out
3832
+
3833
+
3834
  def workspace_wire(ws, uname, pool_pids, defs=None, scope_key='customer', storage_key=None,
3835
  fields_base=None, with_cohorts=True, st=None):
3836
  """The client's `GridWorkspace` WIRE SHAPE from the stored table workspace — the ONE
platform/core/connector_sync.py CHANGED
@@ -8,6 +8,7 @@ from __future__ import annotations
8
  import datetime as _dt
9
  import hashlib
10
  import json
 
11
  import time
12
  import uuid
13
 
@@ -20,6 +21,77 @@ _ERROR_LIMIT = 50
20
  _DEFAULT_CONNECTION = "default"
21
  _PROVIDERS = {}
22
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
23
  # One receipt per immutable database identity. No credentials, session, or source values.
24
  JOBS_KEY = "connector_materialization_jobs"
25
  MAX_ATTEMPTS = 3
@@ -34,6 +106,9 @@ def job_key(provider, connection_id, database_key):
34
 
35
  def _job_public(row):
36
  answer = {k: v for k, v in row.items() if k != "leaseToken"}
 
 
 
37
  answer["retryable"] = (row.get("state") == "failed"
38
  and int(row.get("attempts", 0)) < MAX_ATTEMPTS)
39
  answer["maxAttempts"] = MAX_ATTEMPTS
@@ -104,6 +179,36 @@ def enqueue(rt, provider, connection_id, database_key, *, model="", retry=False,
104
  return _job_public(landed[key])
105
 
106
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
107
  def claim(rt, candidate, *, now=None):
108
  """Atomically claim queued or abandoned work; replay callbacks have no side effects."""
109
  key = job_key(candidate["provider"], candidate["connectionId"], candidate["databaseKey"])
@@ -117,6 +222,7 @@ def claim(rt, candidate, *, now=None):
117
  if row.get("state") != "queued" and not (
118
  row.get("state") == "syncing" and row.get("leaseUntil", 0) <= stamp):
119
  return doc
 
120
  if int(row.get("attempts", 0)) >= MAX_ATTEMPTS:
121
  row.update(state="failed", at=stamp, leaseToken="", leaseUntil=0,
122
  error="Sync attempts exhausted. Check the connection and update its settings.")
@@ -139,7 +245,27 @@ def owns(rt, job, *, now=None):
139
  and row.get("leaseUntil", 0) > (time.time() if now is None else now))
140
 
141
 
142
- def finish(rt, job, *, success=False, renew=False, interrupted=False, now=None):
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
143
  """A stale worker cannot acknowledge, renew, or overwrite a newer receipt."""
144
  key = job_key(job["provider"], job["connectionId"], job["databaseKey"])
145
  stamp = time.time() if now is None else now
@@ -160,7 +286,8 @@ def finish(rt, job, *, success=False, renew=False, interrupted=False, now=None):
160
  else:
161
  row.update(state="ready" if success else "failed", at=stamp,
162
  leaseUntil=0, leaseToken="", workspaceChanged=bool(success),
163
- error="" if success else "Rows could not be synced. Check the connection and retry.")
 
164
  doc[key] = row
165
  return doc
166
 
@@ -178,6 +305,8 @@ def cancel(rt, provider, connection_id, database_key):
178
  doc[key] = {**doc[key], "state": "configured", "at": stamp,
179
  "leaseToken": "", "leaseUntil": 0, "workspaceChanged": False,
180
  "error": ""}
 
 
181
  return doc
182
 
183
  rt.update(JOBS_KEY, apply, flush="sync")
@@ -363,3 +492,5 @@ register_provider(
363
  label="Amazon",
364
  retry_hint="No sync runner is wired for this connector yet.",
365
  )
 
 
 
8
  import datetime as _dt
9
  import hashlib
10
  import json
11
+ import threading
12
  import time
13
  import uuid
14
 
 
21
  _DEFAULT_CONNECTION = "default"
22
  _PROVIDERS = {}
23
 
24
+ # One in-flight Sync now per (provider, tenant). Only the resident resync thread drains this
25
+ # intent desk. Nothing here imports a provider, calls a runner, or writes to the mirror/store.
26
+ _SYNC_REQUESTS = {}
27
+ _SYNC_REQUEST_LOCK = threading.Lock()
28
+ _SYNC_WAKE = threading.Event()
29
+
30
+
31
+ def request_sync(provider, tenant, *, requested_by="", now=None):
32
+ """Queue an intent. A duplicate queued/running request returns accepted=False."""
33
+ if provider not in _PROVIDERS:
34
+ raise ValueError("This connector is not registered for sync.")
35
+ tenant = str(tenant or "").strip()
36
+ if not tenant:
37
+ raise ValueError("A tenant is required for sync.")
38
+ with _SYNC_REQUEST_LOCK:
39
+ previous = _SYNC_REQUESTS.get((provider, tenant))
40
+ if previous and previous["state"] in ("queued", "running"):
41
+ return {**previous, "accepted": False}
42
+ receipt = {"provider": provider, "tenant": tenant, "state": "queued",
43
+ "requestedAt": time.time() if now is None else now,
44
+ "startedAt": None, "finishedAt": None, "detail": "",
45
+ "requestedBy": str(requested_by or ""), "accepted": True}
46
+ if requested_by == "first load" and previous and previous.get("firstLoad"):
47
+ # A continuation keeps the safe numeric progress visible while waiting for the writer.
48
+ receipt.update(detail=previous["detail"], firstLoad=dict(previous["firstLoad"]))
49
+ _SYNC_REQUESTS[(provider, tenant)] = receipt
50
+ _SYNC_WAKE.set()
51
+ return dict(receipt)
52
+
53
+
54
+ def sync_request(provider, tenant):
55
+ """Return a detached current receipt, or None."""
56
+ with _SYNC_REQUEST_LOCK:
57
+ receipt = _SYNC_REQUESTS.get((provider, tenant))
58
+ if receipt is None:
59
+ return None
60
+ result = {**receipt, "accepted": False}
61
+ if "firstLoad" in result:
62
+ result["firstLoad"] = dict(result["firstLoad"])
63
+ return result
64
+
65
+
66
+ def take_sync_requests():
67
+ """Claim every queued intent atomically, on the resident resync thread."""
68
+ with _SYNC_REQUEST_LOCK:
69
+ _SYNC_WAKE.clear()
70
+ claimed = []
71
+ for receipt in _SYNC_REQUESTS.values():
72
+ if receipt["state"] == "queued":
73
+ receipt.update(state="running", startedAt=time.time(), accepted=False)
74
+ claimed.append(dict(receipt))
75
+ return claimed
76
+
77
+
78
+ def finish_sync_request(provider, tenant, *, ok, detail="", now=None, first_load=None):
79
+ """Finish a claimed intent. The API runner supplies a safe cause sentence, never raw errors."""
80
+ with _SYNC_REQUEST_LOCK:
81
+ receipt = _SYNC_REQUESTS.get((provider, tenant))
82
+ if receipt is None or receipt["state"] != "running":
83
+ return None
84
+ receipt.update(state="done" if ok else "failed", detail=str(detail or "")[:300],
85
+ finishedAt=time.time() if now is None else now, accepted=False)
86
+ if first_load is not None:
87
+ receipt["firstLoad"] = dict(first_load)
88
+ return dict(receipt)
89
+
90
+
91
+ def sync_wake_event():
92
+ """The event the resident thread waits on between passes."""
93
+ return _SYNC_WAKE
94
+
95
  # One receipt per immutable database identity. No credentials, session, or source values.
96
  JOBS_KEY = "connector_materialization_jobs"
97
  MAX_ATTEMPTS = 3
 
106
 
107
  def _job_public(row):
108
  answer = {k: v for k, v in row.items() if k != "leaseToken"}
109
+ # Older status readers only know error. Keep one persisted cause and project it there.
110
+ if row.get("state") == "queued" and row.get("stall"):
111
+ answer["error"] = row["stall"]["cause"]
112
  answer["retryable"] = (row.get("state") == "failed"
113
  and int(row.get("attempts", 0)) < MAX_ATTEMPTS)
114
  answer["maxAttempts"] = MAX_ATTEMPTS
 
179
  return _job_public(landed[key])
180
 
181
 
182
+ def stall(rt, candidate, cause, *, now=None):
183
+ """Record WHY queued work cannot start, without consuming an attempt."""
184
+ key = job_key(candidate["provider"], candidate["connectionId"], candidate["databaseKey"])
185
+ stamp = time.time() if now is None else now
186
+ sentence = str(cause or "")[:300]
187
+
188
+ def eligible(row):
189
+ return (row.get("id") == candidate.get("id") and
190
+ (row.get("state") == "queued" or
191
+ row.get("state") == "syncing" and row.get("leaseUntil", 0) <= stamp))
192
+
193
+ previous = (rt.get(JOBS_KEY, fresh=True) or {}).get(key) or {}
194
+ # Avoid even submitting a store update on the unchanged 15-second drain pass.
195
+ if not eligible(previous) or (previous.get("stall") or {}).get("cause") == sentence:
196
+ return _job_public(previous)
197
+
198
+ def apply(current):
199
+ doc = dict(current or {})
200
+ row = dict(doc.get(key) or {})
201
+ if not eligible(row) or (row.get("stall") or {}).get("cause") == sentence:
202
+ return doc
203
+ row.update(state="queued", leaseUntil=0, leaseToken="",
204
+ stall={"at": stamp, "cause": sentence})
205
+ doc[key] = row
206
+ return doc
207
+
208
+ landed = rt.update(JOBS_KEY, apply, flush="sync")
209
+ return _job_public(landed.get(key) or {})
210
+
211
+
212
  def claim(rt, candidate, *, now=None):
213
  """Atomically claim queued or abandoned work; replay callbacks have no side effects."""
214
  key = job_key(candidate["provider"], candidate["connectionId"], candidate["databaseKey"])
 
222
  if row.get("state") != "queued" and not (
223
  row.get("state") == "syncing" and row.get("leaseUntil", 0) <= stamp):
224
  return doc
225
+ row.pop("stall", None)
226
  if int(row.get("attempts", 0)) >= MAX_ATTEMPTS:
227
  row.update(state="failed", at=stamp, leaseToken="", leaseUntil=0,
228
  error="Sync attempts exhausted. Check the connection and update its settings.")
 
245
  and row.get("leaseUntil", 0) > (time.time() if now is None else now))
246
 
247
 
248
+ def progress(rt, job, *, rows, total):
249
+ """Record published rows only while this worker still owns its unexpired lease."""
250
+ key = job_key(job["provider"], job["connectionId"], job["databaseKey"])
251
+ stamp = time.time()
252
+
253
+ def apply(current):
254
+ doc = dict(current or {})
255
+ row = dict(doc.get(key) or {})
256
+ if (row.get("id") != job.get("id") or row.get("state") != "syncing"
257
+ or row.get("leaseToken") != job.get("leaseToken")
258
+ or row.get("leaseUntil", 0) <= stamp):
259
+ return doc
260
+ row["progress"] = {"rows": rows, "total": total, "at": stamp}
261
+ doc[key] = row
262
+ return doc
263
+
264
+ landed = rt.update(JOBS_KEY, apply, flush="sync")
265
+ return _job_public(landed.get(key) or {})
266
+
267
+
268
+ def finish(rt, job, *, success=False, renew=False, interrupted=False, cause="", now=None):
269
  """A stale worker cannot acknowledge, renew, or overwrite a newer receipt."""
270
  key = job_key(job["provider"], job["connectionId"], job["databaseKey"])
271
  stamp = time.time() if now is None else now
 
286
  else:
287
  row.update(state="ready" if success else "failed", at=stamp,
288
  leaseUntil=0, leaseToken="", workspaceChanged=bool(success),
289
+ error="" if success else (str(cause or "")[:300] or
290
+ "Rows could not be synced. Check the connection and retry."))
291
  doc[key] = row
292
  return doc
293
 
 
305
  doc[key] = {**doc[key], "state": "configured", "at": stamp,
306
  "leaseToken": "", "leaseUntil": 0, "workspaceChanged": False,
307
  "error": ""}
308
+ doc[key].pop("stall", None)
309
+ doc[key].pop("progress", None)
310
  return doc
311
 
312
  rt.update(JOBS_KEY, apply, flush="sync")
 
492
  label="Amazon",
493
  retry_hint="No sync runner is wired for this connector yet.",
494
  )
495
+ register_provider("meta_ads", label="Meta Ads",
496
+ retry_hint="Use Sync now to retry the Meta Ads pull.")
platform/core/dependents.py CHANGED
@@ -47,7 +47,13 @@ walked directly instead, exactly as the shape allows.
47
  FIELD KEY (a string) of the link the rollup traverses. A rollup may also carry no `link` at all
48
  (the `{'source': {...}}` shape, which joins through a topic dim). Every walker below branches on
49
  `field['type']` first and on which key is present second.
 
 
 
 
50
  """
 
 
51
  import core.chart_system as chart_system
52
  import core.route_system as route_system
53
  import core.shares as shares
@@ -60,6 +66,13 @@ from core.table_store import ASSIGNED_KEY, SHARED_KEY
60
  #: happened to register first (W49-T11 registers `sheet_cell` after the merge).
61
  KIND_ORDER = ('view', 'route', 'link', 'rollup', 'grant', 'chart', 'sheet_cell')
62
 
 
 
 
 
 
 
 
63
  #: Singular / plural noun phrase per kind. ⛔ THE CASING IS R23's OWN — lowercase `views`, capital
64
  #: `Route`, capital `Sheet cells` — because the ruling quotes the sentence the user reads.
65
  _NOUNS = {
@@ -75,6 +88,36 @@ _NOUNS = {
75
  _PROVIDERS = {}
76
 
77
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
78
  def noun(kind, count):
79
  """The noun phrase for `count` of `kind`, agreeing in number."""
80
  pair = _NOUNS.get(str(kind or ''))
@@ -120,7 +163,7 @@ def clean_target(target):
120
  return {'kind': kind, 'key': key, 'database': database}
121
 
122
 
123
- def _dependent(kind, ids):
124
  """A Dependent, or None when nothing depends. ⛔ NONE IS THE ZERO CASE — never `count: 0`."""
125
  seen, out = set(), []
126
  for raw in ids or ():
@@ -130,7 +173,13 @@ def _dependent(kind, ids):
130
  out.append(value)
131
  if not out:
132
  return None
133
- return {'kind': kind, 'label': noun(kind, len(out)), 'count': len(out), 'ids': out}
 
 
 
 
 
 
134
 
135
 
136
  def dependents(st, target):
@@ -162,10 +211,36 @@ def dependents_report(st, target):
162
  count = int(item.get('count') or len(ids) or 0)
163
  if count <= 0:
164
  continue
165
- found.append({'kind': str(item.get('kind') or kind),
166
  'label': str(item.get('label') or noun(kind, count)),
167
  'count': count,
168
- 'ids': [str(i) for i in ids]})
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
169
  order = {kind: index for index, kind in enumerate(KIND_ORDER)}
170
  found.sort(key=lambda item: (order.get(item['kind'], len(order)), item['kind']))
171
  return {'dependents': found, 'errors': errors}
@@ -334,16 +409,27 @@ def view_dependents(st, target):
334
  """
335
  kind, key, database = target['kind'], target['key'], target['database']
336
  if kind == 'table':
337
- return _dependent('view', [vid for _owner, vid, _v in walk_views(st, key)])
 
 
338
  if kind == 'view':
339
  if not database:
340
  return None
341
- return _dependent('view', _assigned_ids(st, database, key))
342
- hits = []
 
 
 
 
 
 
 
 
343
  for _owner, view_id, view in walk_views(st, database):
344
- if _view_names_field(view, key):
345
- hits.append(view_id)
346
- return _dependent('view', hits)
 
347
 
348
 
349
  def _view_names_field(view, field_key):
@@ -383,21 +469,31 @@ def route_dependents(st, target):
383
  a `geographicField`, which is what makes them a FIELD dependent too.
384
  """
385
  kind, key, database = target['kind'], target['key'], target['database']
386
- routes, hits = _routes(st), []
387
  if kind == 'view':
388
- hits += [str(r.get('id')) for r in routes if str(r.get('sourceViewId') or '') == key]
389
- hits += [str(z.get('id')) for z in _zones(st, database or None)
390
- if str(z.get('sourceViewId') or '') == key]
391
- return _dependent('route', hits)
392
- if kind == 'table':
393
- view_ids = {vid for _owner, vid, _v in walk_views(st, key)}
394
- hits += [str(r.get('id')) for r in routes
395
- if str(r.get('sourceViewId') or '') in view_ids]
396
- hits += [str(z.get('id')) for z in _zones(st, key)]
397
- return _dependent('route', hits)
398
- hits += [str(z.get('id')) for z in _zones(st, database)
399
- if str(z.get('geographicField') or '') == key]
400
- return _dependent('route', hits)
 
 
 
 
 
 
 
 
 
 
401
 
402
 
403
  def link_dependents(st, target):
@@ -410,18 +506,18 @@ def link_dependents(st, target):
410
  kind, key = target['kind'], target['key']
411
  if kind == 'view':
412
  return None
413
- hits = []
414
  for table_key, definition in _defs(st).items():
415
  for field in _fields(definition):
416
  bag = _link_bag(field)
417
  if bag is None:
418
  continue
419
  if kind == 'table' and str(bag.get('table') or '') == key and str(table_key) != key:
420
- hits.append(f'{table_key}:{field.get("key")}')
421
  elif kind == 'field' and str(bag.get('inverse') or '') == key \
422
  and str(bag.get('table') or '') == str(target['database']):
423
- hits.append(f'{table_key}:{field.get("key")}')
424
- return _dependent('link', hits)
425
 
426
 
427
  def rollup_dependents(st, target):
@@ -435,7 +531,7 @@ def rollup_dependents(st, target):
435
  kind, key = target['kind'], target['key']
436
  if kind == 'view':
437
  return None
438
- hits = []
439
  for table_key, definition in _defs(st).items():
440
  fields = _fields(definition)
441
  by_key = {str(f.get('key') or ''): f for f in fields}
@@ -449,18 +545,18 @@ def rollup_dependents(st, target):
449
  hop_bag = _link_bag(hop_field) if hop_field else None
450
  if kind == 'table':
451
  if source and str(source.get('topic') or '') == key:
452
- hits.append(f'{table_key}:{field.get("key")}')
453
  elif hop_bag is not None and str(hop_bag.get('table') or '') == key:
454
- hits.append(f'{table_key}:{field.get("key")}')
455
  continue
456
  if str(target['database']) and str(table_key) != str(target['database']):
457
  continue
458
  if hop == key or str(bag.get('field') or '') == key:
459
- hits.append(f'{table_key}:{field.get("key")}')
460
  elif source and key in (str(source.get('on') or ''), str(source.get('groupBy') or ''),
461
  str(source.get('measure') or '')):
462
- hits.append(f'{table_key}:{field.get("key")}')
463
- return _dependent('rollup', hits)
464
 
465
 
466
  def grant_dependents(st, target):
@@ -471,13 +567,21 @@ def grant_dependents(st, target):
471
  * field -> the `field` grant on `<database>:<key>`
472
  """
473
  kind, key, database = target['kind'], target['key'], target['database']
 
 
 
 
 
 
 
 
474
  if kind == 'view':
475
- return _dependent('grant', share_audience(st, key)['names'])
476
  if kind == 'field':
477
  if not database:
478
  return None
479
  record = shares.grants('field', shares.field_oid(database, key), st=st)
480
- return _dependent('grant', [str(e.get('user') or '') for e in (record.get('entries') or ())])
481
  holders = [str(e.get('user') or '')
482
  for e in (shares.grants('database', key, st=st).get('entries') or ())]
483
  try:
@@ -490,21 +594,110 @@ def grant_dependents(st, target):
490
  continue
491
  for entry in (shares.grants('field', oid, st=st).get('entries') or ()):
492
  holders.append(str(entry.get('user') or ''))
493
- return _dependent('grant', holders)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
494
 
495
 
496
  def chart_dependents(st, target):
497
- """Saved charts filed against the target DATABASE.
498
 
499
  ⛔ A CHART NEVER DEPENDS ON A VIEW, and that is `chart_system`'s stated contract rather than an
500
  omission here: a chart's `config` is OPAQUE and this file must not start reading keys out of
501
  it to guess at a reference. Deleting a view cannot delete a chart, so it must not disclose one.
 
 
 
 
 
502
  """
503
  kind, key = target['kind'], target['key']
504
  if kind != 'table':
505
  return None
506
- return _dependent('chart', [str(c.get('id')) for c in _charts(st)
507
- if str(c.get('database') or '') == key])
 
 
 
 
 
 
 
 
 
 
 
 
508
 
509
 
510
  register('view', view_dependents)
 
47
  FIELD KEY (a string) of the link the rollup traverses. A rollup may also carry no `link` at all
48
  (the `{'source': {...}}` shape, which joins through a topic dim). Every walker below branches on
49
  `field['type']` first and on which key is present second.
50
+
51
+ W60-T31 / C3: grouped kind/label/count/ids retain their meanings. Additive `items` names each
52
+ id in order and gives its address or a reason it cannot be opened. Data addresses use only
53
+ the existing #/data? prefix; workbook addresses belong to the Sheet provider (C2).
54
  """
55
+ from urllib.parse import parse_qs, urlencode
56
+
57
  import core.chart_system as chart_system
58
  import core.route_system as route_system
59
  import core.shares as shares
 
66
  #: happened to register first (W49-T11 registers `sheet_cell` after the merge).
67
  KIND_ORDER = ('view', 'route', 'link', 'rollup', 'grant', 'chart', 'sheet_cell')
68
 
69
+ #: Built-in Default views are not dependents, including their saved owner copies. Their ids are
70
+ #: shared by EVERY database, so counting them charges one database's Default view to another.
71
+ #: `aios_grid.py` mints `all-customers` on every topic; its `IG_OVERVIEW_ID` is `tpl_overview`.
72
+ #: Hardcoded here because `core/` may not reach UP into `aios_grid`; `verify_shared_views.py`
73
+ #: pins this pair against that module so the declarations cannot drift.
74
+ PINNED_VIEW_IDS = frozenset({'all-customers', 'tpl_overview'})
75
+
76
  #: Singular / plural noun phrase per kind. ⛔ THE CASING IS R23's OWN — lowercase `views`, capital
77
  #: `Route`, capital `Sheet cells` — because the ruling quotes the sentence the user reads.
78
  _NOUNS = {
 
88
  _PROVIDERS = {}
89
 
90
 
91
+ DATA_HASH_PREFIX = '#/data?'
92
+
93
+
94
+ def data_hash(database, **params):
95
+ """Build a #/data?database=... address. The only hash builder in this module."""
96
+ database = str(database or '').strip()
97
+ if not database:
98
+ return None
99
+ query = {'database': database}
100
+ query.update({key: str(value) for key, value in params.items()
101
+ if key != 'database' and value is not None and str(value) != ''})
102
+ return DATA_HASH_PREFIX + urlencode(query)
103
+
104
+
105
+ def _dependent_item(item_id, label, href=None, reason=''):
106
+ """A named thing with an existing hash address or an explicit explanation."""
107
+ item_id = str(item_id or '')
108
+ label = str(label or item_id)
109
+ href = href if isinstance(href, str) and href.startswith((DATA_HASH_PREFIX, '#/sheet?')) else None
110
+ return {'id': item_id, 'label': label, 'href': href,
111
+ 'reason': str(reason or '') if href else
112
+ str(reason or f'{label} has no direct address available.')}
113
+
114
+
115
+ def _field_item(table_key, field):
116
+ key = str(field.get('key') or '')
117
+ return _dependent_item(f'{table_key}:{key}', field.get('label') or key,
118
+ data_hash(table_key, field=key) if key else None)
119
+
120
+
121
  def noun(kind, count):
122
  """The noun phrase for `count` of `kind`, agreeing in number."""
123
  pair = _NOUNS.get(str(kind or ''))
 
163
  return {'kind': kind, 'key': key, 'database': database}
164
 
165
 
166
+ def _dependent(kind, ids, items=None):
167
  """A Dependent, or None when nothing depends. ⛔ NONE IS THE ZERO CASE — never `count: 0`."""
168
  seen, out = set(), []
169
  for raw in ids or ():
 
173
  out.append(value)
174
  if not out:
175
  return None
176
+ result = {'kind': kind, 'label': noun(kind, len(out)), 'count': len(out), 'ids': out}
177
+ if items is not None:
178
+ by_id = {}
179
+ for item in items:
180
+ by_id.setdefault(str(item.get('id') or ''), item)
181
+ result['items'] = [by_id.get(value) or _dependent_item(value, value) for value in out]
182
+ return result
183
 
184
 
185
  def dependents(st, target):
 
211
  count = int(item.get('count') or len(ids) or 0)
212
  if count <= 0:
213
  continue
214
+ normalized = {'kind': str(item.get('kind') or kind),
215
  'label': str(item.get('label') or noun(kind, count)),
216
  'count': count,
217
+ 'ids': [str(i) for i in ids]}
218
+ # Lane A may supply a single Sheet address on the dependent itself. Preserve
219
+ # both that shape and per-id items; neither changes the legacy count or label.
220
+ if 'href' in item:
221
+ normalized['href'] = item['href'] if isinstance(item['href'], str) else None
222
+ if 'reason' in item:
223
+ normalized['reason'] = str(item.get('reason') or '')
224
+ # W60-T12 (combine-owned), 2026-09-14: `refusal` is the OTHER half of the same fact
225
+ # and was being dropped at this seam. A provider says "no address" with `href: None`
226
+ # AND says WHY in `refusal`; carrying the null while dropping the cause turns an
227
+ # explained dead end into an unexplained one, which is the defect R23's sentence
228
+ # exists to prevent. Both keys are optional and neither changes count or label.
229
+ if 'refusal' in item:
230
+ normalized['refusal'] = str(item.get('refusal') or '')
231
+ if isinstance(item.get('items'), list):
232
+ normalized['items'] = []
233
+ for entry in item['items']:
234
+ entry = entry if isinstance(entry, dict) else {'id': entry}
235
+ href = entry.get('href') if isinstance(entry.get('href'), str) else None
236
+ normalized['items'].append({
237
+ 'id': str(entry.get('id') or ''),
238
+ 'label': str(entry.get('label') or entry.get('id') or ''),
239
+ 'href': href,
240
+ 'reason': str(entry.get('reason') or ('' if href else
241
+ 'This dependent has no direct address available.')),
242
+ })
243
+ found.append(normalized)
244
  order = {kind: index for index, kind in enumerate(KIND_ORDER)}
245
  found.sort(key=lambda item: (order.get(item['kind'], len(order)), item['kind']))
246
  return {'dependents': found, 'errors': errors}
 
409
  """
410
  kind, key, database = target['kind'], target['key'], target['database']
411
  if kind == 'table':
412
+ items = [_dependent_item(vid, view.get('name') or vid, data_hash(key, view=vid))
413
+ for _owner, vid, view in walk_views(st, key) if vid not in PINNED_VIEW_IDS]
414
+ return _dependent('view', [item['id'] for item in items], items=items)
415
  if kind == 'view':
416
  if not database:
417
  return None
418
+ ids = _assigned_ids(st, database, key)
419
+ assignments = _table_doc(st, database).get(ASSIGNED_KEY) or {}
420
+ items = []
421
+ for item_id in ids:
422
+ recipient, _, view_id = item_id.partition(':')
423
+ view = assignments[recipient]['views'][view_id]
424
+ label = f'{recipient}: {view.get("name") or view_id}'
425
+ items.append(_dependent_item(item_id, label, data_hash(database, view=view_id)))
426
+ return _dependent('view', ids, items=items)
427
+ items = []
428
  for _owner, view_id, view in walk_views(st, database):
429
+ if view_id not in PINNED_VIEW_IDS and _view_names_field(view, key):
430
+ items.append(_dependent_item(view_id, view.get('name') or view_id,
431
+ data_hash(database, view=view_id)))
432
+ return _dependent('view', [item['id'] for item in items], items=items)
433
 
434
 
435
  def _view_names_field(view, field_key):
 
469
  a `geographicField`, which is what makes them a FIELD dependent too.
470
  """
471
  kind, key, database = target['kind'], target['key'], target['database']
472
+ routes, matched_routes, matched_zones = _routes(st), [], []
473
  if kind == 'view':
474
+ matched_routes = [r for r in routes if str(r.get('sourceViewId') or '') == key]
475
+ matched_zones = [z for z in _zones(st, database or None)
476
+ if str(z.get('sourceViewId') or '') == key]
477
+ elif kind == 'table':
478
+ view_ids = {vid for _owner, vid, _v in walk_views(st, key)
479
+ if vid not in PINNED_VIEW_IDS}
480
+ # A-QI7: Routes store no source database and are created only from customer_data.
481
+ # The database test must therefore use the target table key, not the tenant-wide Route
482
+ # list or a shared view id. Zones already carry their own table scope.
483
+ if str(key) == route_system.DEFAULT_ZONE_TABLE:
484
+ matched_routes = [r for r in routes if str(r.get('sourceViewId') or '') in view_ids]
485
+ matched_zones = _zones(st, key)
486
+ else:
487
+ matched_zones = [z for z in _zones(st, database)
488
+ if str(z.get('geographicField') or '') == key]
489
+ items = [_dependent_item(str(r.get('id')), r.get('name') or r.get('id'),
490
+ data_hash(route_system.DEFAULT_ZONE_TABLE, route=r.get('id')))
491
+ for r in matched_routes]
492
+ for zone in matched_zones:
493
+ name = zone.get('name') or zone.get('id')
494
+ items.append(_dependent_item(str(zone.get('id')), name, reason=
495
+ f'{name} has no direct address. Edit this zone in the Map view Zones menu.'))
496
+ return _dependent('route', [item['id'] for item in items], items=items)
497
 
498
 
499
  def link_dependents(st, target):
 
506
  kind, key = target['kind'], target['key']
507
  if kind == 'view':
508
  return None
509
+ items = []
510
  for table_key, definition in _defs(st).items():
511
  for field in _fields(definition):
512
  bag = _link_bag(field)
513
  if bag is None:
514
  continue
515
  if kind == 'table' and str(bag.get('table') or '') == key and str(table_key) != key:
516
+ items.append(_field_item(table_key, field))
517
  elif kind == 'field' and str(bag.get('inverse') or '') == key \
518
  and str(bag.get('table') or '') == str(target['database']):
519
+ items.append(_field_item(table_key, field))
520
+ return _dependent('link', [item['id'] for item in items], items=items)
521
 
522
 
523
  def rollup_dependents(st, target):
 
531
  kind, key = target['kind'], target['key']
532
  if kind == 'view':
533
  return None
534
+ items = []
535
  for table_key, definition in _defs(st).items():
536
  fields = _fields(definition)
537
  by_key = {str(f.get('key') or ''): f for f in fields}
 
545
  hop_bag = _link_bag(hop_field) if hop_field else None
546
  if kind == 'table':
547
  if source and str(source.get('topic') or '') == key:
548
+ items.append(_field_item(table_key, field))
549
  elif hop_bag is not None and str(hop_bag.get('table') or '') == key:
550
+ items.append(_field_item(table_key, field))
551
  continue
552
  if str(target['database']) and str(table_key) != str(target['database']):
553
  continue
554
  if hop == key or str(bag.get('field') or '') == key:
555
+ items.append(_field_item(table_key, field))
556
  elif source and key in (str(source.get('on') or ''), str(source.get('groupBy') or ''),
557
  str(source.get('measure') or '')):
558
+ items.append(_field_item(table_key, field))
559
+ return _dependent('rollup', [item['id'] for item in items], items=items)
560
 
561
 
562
  def grant_dependents(st, target):
 
567
  * field -> the `field` grant on `<database>:<key>`
568
  """
569
  kind, key, database = target['kind'], target['key'], target['database']
570
+
571
+ def named_grants(names):
572
+ subject = {'view': 'view', 'table': 'database', 'field': 'field'}[kind]
573
+ items = [_dependent_item(name, name, reason=
574
+ f'{name} has no direct access address. Change their access from this '
575
+ f'{subject}\'s share dialog.') for name in names]
576
+ return _dependent('grant', names, items=items)
577
+
578
  if kind == 'view':
579
+ return named_grants(share_audience(st, key)['names'])
580
  if kind == 'field':
581
  if not database:
582
  return None
583
  record = shares.grants('field', shares.field_oid(database, key), st=st)
584
+ return named_grants([str(e.get('user') or '') for e in (record.get('entries') or ())])
585
  holders = [str(e.get('user') or '')
586
  for e in (shares.grants('database', key, st=st).get('entries') or ())]
587
  try:
 
594
  continue
595
  for entry in (shares.grants('field', oid, st=st).get('entries') or ()):
596
  holders.append(str(entry.get('user') or ''))
597
+ return named_grants(holders)
598
+
599
+
600
+ def _sheet_chart_items(st, target):
601
+ """Sheet charts drawn over a cell that depends on `target` (W60-T12, contract C2).
602
+
603
+ ⛔⛔ THE DEPENDENCY DECISION IS NOT REMADE HERE, AND THAT IS THE WHOLE DESIGN. Whether a CELL
604
+ depends on the target is `modules/sheet.py`'s answer, and `core/` may not reach UP into
605
+ `modules/` to ask it directly -- the same layering rule `PINNED_VIEW_IDS` is hardcoded for. So
606
+ this reads the ALREADY-REGISTERED `sheet_cell` provider's own rows and takes the cells it
607
+ named. One source of truth for the predicate, so a chart can never disagree with the cell it
608
+ is drawn over; and if nothing registered `sheet_cell` there are no sheet charts to report and
609
+ this is silent, exactly as an absent provider is everywhere else in this file.
610
+
611
+ ⚠ A CELL WITH NO ADDRESS CONTRIBUTES NO CHART ADDRESS. `sheet_cell` answers `href: None` plus a
612
+ cause when the destination is hidden or unreadable, and a link built anyway would open some
613
+ OTHER cell -- which is the defect W60-T12 exists to close, not one to reintroduce one layer up.
614
+ """
615
+ provider = _PROVIDERS.get('sheet_cell')
616
+ if provider is None:
617
+ return []
618
+ try:
619
+ rows = provider(st, dict(target))
620
+ except Exception:
621
+ return [] # a provider that raises omits its kind; so does this
622
+ wanted = {}
623
+ for row in rows if isinstance(rows, (list, tuple)) else ():
624
+ if not isinstance(row, dict) or row.get('kind') != 'sheet_cell':
625
+ continue
626
+ href = row.get('href')
627
+ if not isinstance(href, str) or not href.startswith('#/sheet?'):
628
+ continue
629
+ params = parse_qs(href.split('?', 1)[1])
630
+ workbook_id = (params.get('wb') or [''])[0]
631
+ tab_id = (params.get('tab') or [''])[0]
632
+ addr = (params.get('cell') or [''])[0]
633
+ if workbook_id and tab_id and addr:
634
+ wanted.setdefault((workbook_id, tab_id), []).append(addr)
635
+ if not wanted:
636
+ return []
637
+
638
+ from core import sheets as sheet_store
639
+ items, seen = [], set()
640
+ for workbook_id in dict.fromkeys(book for book, _ in wanted):
641
+ try:
642
+ book = sheet_store.get_workbook(st, workbook_id)
643
+ except Exception:
644
+ continue # an unreadable workbook is `sheet_cell`'s to report
645
+ for tab in book.get('sheets') or ():
646
+ tab_id = str(tab.get('id') or '')
647
+ addrs = wanted.get((workbook_id, tab_id)) or []
648
+ if not addrs:
649
+ continue
650
+ # ⚠ THE ADDRESS MATHS IS `core/sheets.py`'s, not a second copy. `_clean_range` accepts
651
+ # both `"B4"` and the chart's `{col0,row0,col1,row1}` and answers the same zero-based
652
+ # rectangle for each, so a cell and the chart over it cannot drift apart.
653
+ cells = [rect for rect in (sheet_store._clean_range(a) for a in addrs) if rect]
654
+ for chart in tab.get('charts') or ():
655
+ if not isinstance(chart, dict):
656
+ continue
657
+ chart_id = str(chart.get('id') or '')
658
+ rect = sheet_store._clean_range(chart.get('range'))
659
+ if not chart_id or chart_id in seen or not rect:
660
+ continue
661
+ if not any(rect['col0'] <= c['col0'] <= rect['col1']
662
+ and rect['row0'] <= c['row0'] <= rect['row1'] for c in cells):
663
+ continue
664
+ seen.add(chart_id)
665
+ label = str(chart.get('name') or chart.get('title') or '').strip() or (
666
+ f"Chart in {book.get('name') or workbook_id} / {tab.get('name') or tab_id}")
667
+ items.append(_dependent_item(chart_id, label, '#/sheet?' + urlencode(
668
+ {'wb': workbook_id, 'tab': tab_id, 'chart': chart_id})))
669
+ return items
670
 
671
 
672
  def chart_dependents(st, target):
673
+ """Saved charts filed against the target DATABASE, and Sheet charts drawn over its cells.
674
 
675
  ⛔ A CHART NEVER DEPENDS ON A VIEW, and that is `chart_system`'s stated contract rather than an
676
  omission here: a chart's `config` is OPAQUE and this file must not start reading keys out of
677
  it to guess at a reference. Deleting a view cannot delete a chart, so it must not disclose one.
678
+
679
+ ⭐ W60-T12 (combine-owned) ADDED THE SECOND SOURCE. A Sheet chart is not reachable from
680
+ `sheet_dependents` at all -- it is a sibling of the cells, not one of them -- so before this a
681
+ database could be switched off while a chart still drew from it, and R23's refusal would not
682
+ have said so. Charts are not read from the sheet document here; see `_sheet_chart_items`.
683
  """
684
  kind, key = target['kind'], target['key']
685
  if kind != 'table':
686
  return None
687
+ items = [_dependent_item(str(c.get('id')), c.get('name') or c.get('id'), c.get('href'),
688
+ '' if c.get('href') else
689
+ 'This chart has no direct address. Open it from Charts.')
690
+ for c in _charts(st) if str(c.get('database') or '') == key]
691
+ items += _sheet_chart_items(st, target)
692
+ row = _dependent('chart', [item['id'] for item in items], items=items)
693
+ # ⭐ WHEN EXACTLY ONE CHART DEPENDS ON THE TARGET, THE ROW ADDRESSES ONE THING, so it carries
694
+ # that address itself -- the single-address-on-the-dependent shape `dependents_report` already
695
+ # preserves. ⛔ With several it carries none: a row-level link would have to pick one chart and
696
+ # would silently mean the others, which is the naming-by-display defect W60-T12 closed.
697
+ if row and len(items) == 1:
698
+ row['href'] = items[0]['href']
699
+ row['reason'] = items[0]['reason']
700
+ return row
701
 
702
 
703
  register('view', view_dependents)
platform/core/external_access.py CHANGED
@@ -1,8 +1,9 @@
1
- """Per-user credentials for the governed external read surface.
 
2
 
3
- The credential is authentication only. Verification reconstructs the same PUBLIC user record
4
- the normal API session carries, so every later database read still goes through ``perm_scope``
5
- and the existing Assistant mirror. This module never reads product rows and never caches a
6
  credential, user, bucket, or tenant runtime.
7
  """
8
  import calendar
@@ -17,6 +18,11 @@ from harness import runtime
17
 
18
 
19
  BUCKET = "external_access"
 
 
 
 
 
20
  ALGORITHM = "sha256-salt-v1"
21
  AUTH_ERROR = "invalid_external_credential"
22
  _DUMMY_SALT = bytes(32)
@@ -48,6 +54,50 @@ class ExternalAuthError(ExternalAccessError):
48
  """A uniform authentication refusal that reveals no credential or tenant detail."""
49
 
50
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
51
  @dataclass(frozen=True)
52
  class VerifiedCredential:
53
  """The authenticated public principal plus the request-scoped tenant runtime."""
@@ -56,6 +106,8 @@ class VerifiedCredential:
56
  key_id: str
57
  user: dict
58
  runtime: object
 
 
59
 
60
 
61
  def _now() -> str:
@@ -130,17 +182,33 @@ def _active_public_user(username: str, tenant: str) -> dict:
130
 
131
  def _metadata(record: dict) -> dict:
132
  """Return list-safe metadata. Salt, digest, algorithm and owner never cross this seam."""
 
133
  return {
134
  "id": record.get("id"),
135
  "label": record.get("label"),
136
  "created": record.get("created"),
137
  "last_used": record.get("last_used"),
138
  "revoked_at": record.get("revoked_at"),
 
 
139
  }
140
 
141
 
142
- def mint(st, username: str, tenant: str, label: str) -> tuple[dict, str]:
 
 
 
 
 
 
 
 
 
 
 
143
  """Create one credential and return ``(metadata, token)``; the token is never persisted."""
 
 
144
  owner, tenant_key = _normalise_owner(username, tenant)
145
  _assert_bound_runtime(st, tenant_key)
146
  _active_public_user(owner, tenant_key)
@@ -168,6 +236,8 @@ def mint(st, username: str, tenant: str, label: str) -> tuple[dict, str]:
168
  "username": owner,
169
  "tenant": tenant_key,
170
  "label": clean_label,
 
 
171
  "salt": salt.hex(),
172
  "hash": _digest(salt, secret),
173
  "algo": ALGORITHM,
@@ -295,8 +365,10 @@ def verify(token: str) -> VerifiedCredential:
295
  except ExternalAccessError as exc:
296
  raise ExternalAuthError(AUTH_ERROR) from exc
297
 
 
298
  if not _stamp_is_due(record.get("last_used"), time.time()):
299
- return VerifiedCredential(tenant=tenant_key, key_id=key_id, user=public, runtime=st)
 
300
 
301
  used_at = _now()
302
 
@@ -321,7 +393,8 @@ def verify(token: str) -> VerifiedCredential:
321
  raise
322
  except Exception as exc:
323
  raise ExternalAuthError(AUTH_ERROR) from exc
324
- return VerifiedCredential(tenant=tenant_key, key_id=key_id, user=public, runtime=st)
 
325
  except ExternalAuthError:
326
  raise
327
  except Exception as exc:
 
1
+ """Per-user credentials carrying an access level and database subset for governed reads and
2
+ builds, with the maker's live grants still applying on top.
3
 
4
+ Verification reconstructs the same PUBLIC user record the normal API session carries, so every
5
+ later database read still goes through ``perm_scope`` and the existing Assistant mirror.
6
+ This module never reads product rows and never caches a
7
  credential, user, bucket, or tenant runtime.
8
  """
9
  import calendar
 
18
 
19
 
20
  BUCKET = "external_access"
21
+ ACCESS_READ = "read"
22
+ ACCESS_READWRITE = "readwrite"
23
+ ACCESS_LEVELS = (ACCESS_READ, ACCESS_READWRITE)
24
+ ALL_DATABASES = "all"
25
+ MAX_SCOPE_DATABASES = 200
26
  ALGORITHM = "sha256-salt-v1"
27
  AUTH_ERROR = "invalid_external_credential"
28
  _DUMMY_SALT = bytes(32)
 
54
  """A uniform authentication refusal that reveals no credential or tenant detail."""
55
 
56
 
57
+ def normalise_access(value) -> str:
58
+ if value is None or value == "":
59
+ return ACCESS_READWRITE
60
+ if value in ACCESS_LEVELS:
61
+ return value
62
+ raise ExternalAccessError("external_access_scope_invalid")
63
+
64
+
65
+ def normalise_databases(value):
66
+ if value is None or (isinstance(value, str) and value.strip().lower() == ALL_DATABASES):
67
+ return ALL_DATABASES
68
+ if not isinstance(value, (list, tuple)) or not 1 <= len(value) <= MAX_SCOPE_DATABASES:
69
+ raise ExternalAccessError("external_databases_invalid")
70
+ if any(not isinstance(item, str) or not item.strip() or len(item.strip()) > 200
71
+ for item in value):
72
+ raise ExternalAccessError("external_databases_invalid")
73
+ return tuple(dict.fromkeys(item.strip() for item in value))
74
+
75
+
76
+ def scope_of(record):
77
+ """Read legacy defaults, but preserve a stored empty subset as no database access."""
78
+ record = record if isinstance(record, dict) else {}
79
+ access = record.get("access")
80
+ if access not in ACCESS_LEVELS:
81
+ access = ACCESS_READWRITE
82
+ databases = record.get("databases")
83
+ if isinstance(databases, (list, tuple)):
84
+ databases = tuple(item.strip() for item in databases
85
+ if isinstance(item, str) and item.strip())
86
+ else:
87
+ databases = ALL_DATABASES
88
+ return access, databases
89
+
90
+
91
+ def may_write(access) -> bool:
92
+ return access == ACCESS_READWRITE
93
+
94
+
95
+ def database_allowed(databases, database) -> bool:
96
+ if databases == ALL_DATABASES:
97
+ return True
98
+ return isinstance(database, str) and bool(database.strip()) and database.strip() in databases
99
+
100
+
101
  @dataclass(frozen=True)
102
  class VerifiedCredential:
103
  """The authenticated public principal plus the request-scoped tenant runtime."""
 
106
  key_id: str
107
  user: dict
108
  runtime: object
109
+ access: str = ACCESS_READWRITE
110
+ databases: object = ALL_DATABASES
111
 
112
 
113
  def _now() -> str:
 
182
 
183
  def _metadata(record: dict) -> dict:
184
  """Return list-safe metadata. Salt, digest, algorithm and owner never cross this seam."""
185
+ access, databases = scope_of(record)
186
  return {
187
  "id": record.get("id"),
188
  "label": record.get("label"),
189
  "created": record.get("created"),
190
  "last_used": record.get("last_used"),
191
  "revoked_at": record.get("revoked_at"),
192
+ "access": access,
193
+ "databases": ALL_DATABASES if databases == ALL_DATABASES else list(databases),
194
  }
195
 
196
 
197
+ def label_for(st, key_id: str) -> str:
198
+ """Read only a verified key's label from its tenant runtime, never credential material."""
199
+ record = _book(st).get(key_id)
200
+ if not isinstance(record, dict) or record.get("revoked_at"):
201
+ raise ExternalAuthError(AUTH_ERROR)
202
+ label = record.get("label")
203
+ if not isinstance(label, str) or not label:
204
+ raise ExternalAccessError("external_access_unavailable")
205
+ return label
206
+
207
+
208
+ def mint(st, username: str, tenant: str, label: str, access=None, databases=None) -> tuple[dict, str]:
209
  """Create one credential and return ``(metadata, token)``; the token is never persisted."""
210
+ access = normalise_access(access)
211
+ databases = normalise_databases(databases)
212
  owner, tenant_key = _normalise_owner(username, tenant)
213
  _assert_bound_runtime(st, tenant_key)
214
  _active_public_user(owner, tenant_key)
 
236
  "username": owner,
237
  "tenant": tenant_key,
238
  "label": clean_label,
239
+ "access": access,
240
+ "databases": ALL_DATABASES if databases == ALL_DATABASES else list(databases),
241
  "salt": salt.hex(),
242
  "hash": _digest(salt, secret),
243
  "algo": ALGORITHM,
 
365
  except ExternalAccessError as exc:
366
  raise ExternalAuthError(AUTH_ERROR) from exc
367
 
368
+ access, databases = scope_of(record)
369
  if not _stamp_is_due(record.get("last_used"), time.time()):
370
+ return VerifiedCredential(tenant=tenant_key, key_id=key_id, user=public, runtime=st,
371
+ access=access, databases=databases)
372
 
373
  used_at = _now()
374
 
 
393
  raise
394
  except Exception as exc:
395
  raise ExternalAuthError(AUTH_ERROR) from exc
396
+ return VerifiedCredential(tenant=tenant_key, key_id=key_id, user=public, runtime=st,
397
+ access=access, databases=databases)
398
  except ExternalAuthError:
399
  raise
400
  except Exception as exc:
platform/core/external_tools.py ADDED
@@ -0,0 +1,372 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """External build tools over the app's Sheet, saved View and Application writers.
2
+
3
+ The API supplies its existing permission, read-projection and binding-resolution functions as
4
+ request-scoped callbacks. Core never imports API modules or chooses another tenant runtime.
5
+ Adding a tool means adding a registry entry; writes also require the router's literal allow-list.
6
+ """
7
+ import copy
8
+ import json
9
+ import uuid
10
+ from dataclasses import dataclass
11
+ from typing import Callable
12
+
13
+ from core import sheets, sheet_structure, sheet_templates
14
+
15
+
16
+ @dataclass(frozen=True)
17
+ class ToolContext:
18
+ session: object
19
+ actor_name: str
20
+ require_reader: Callable
21
+ require_editor: Callable
22
+ project_book: Callable
23
+ resolve_binding: Callable
24
+ cache_binding: Callable
25
+ list_databases: Callable
26
+ describe_database: Callable
27
+ view_event: Callable
28
+ app_upsert: Callable
29
+ app_delete: Callable
30
+
31
+
32
+ def _object(properties=None, required=(), **extra):
33
+ return {"type": "object", "properties": properties or {}, "required": list(required), **extra}
34
+
35
+
36
+ _STRING = {"type": "string"}
37
+ _ADDRESS = {"type": "string", "description": "An A1 cell address on the named tab."}
38
+ _FORMAT = _object({"kind": _STRING, "code": _STRING, "decimals": {"type": "integer"},
39
+ "currency": _STRING, "thousands": {"type": "boolean"}}, ("kind",))
40
+ _STYLE = _object({
41
+ **{key: {"type": "boolean"} for key in ("bold", "italic", "underline", "strike", "wrap")},
42
+ **{key: _STRING for key in ("fill", "color", "font", "align", "valign")},
43
+ "size": {"type": ["number", "string"]}, "indent": {"type": "integer"},
44
+ "textRotation": {"type": "number"}, "format": _FORMAT,
45
+ "border": _object({side: {"type": "boolean"} for side in ("top", "right", "bottom", "left")}),
46
+ })
47
+ _VALIDATION = _object({"list": {"type": "array", "items": _STRING}}, ("list",))
48
+ _FILTER = _object({"conj": {"enum": ["and", "or"]}, "nodes": {"type": "array", "items": _object({
49
+ "colId": _STRING, "op": _STRING, "value": {},
50
+ "valueRef": _object({"ref": _ADDRESS}, ("ref",)),
51
+ "conj": _STRING, "children": {"type": "array", "items": {"type": "object"}},
52
+ })}})
53
+ _BINDING = _object({
54
+ "database": _STRING,
55
+ "aggregate": _object({"fn": _STRING, "field": _STRING, "metric_key": _STRING}),
56
+ "scope": _object({"filter": _FILTER, "view_id": _STRING, "view_name": _STRING}),
57
+ "period": _object({"unit": _STRING, "start": _STRING}),
58
+ "ending": {"type": "object"},
59
+ }, ("database", "aggregate", "scope"))
60
+ _CELL = _object({"kind": {"enum": ["literal", "formula", "bound", "empty"]},
61
+ "value": {}, "text": _STRING, "formula": _STRING, "state": _STRING,
62
+ "style": _STYLE, "validation": _VALIDATION, "binding": _BINDING}, ("kind",))
63
+ _RANGE = {"oneOf": [_STRING, _object({key: {"type": "integer", "minimum": 0}
64
+ for key in ("col0", "row0", "col1", "row1")},
65
+ ("col0", "row0", "col1", "row1"))]}
66
+ _RULE = _object({"id": _STRING, "range": _RANGE, "kind": _STRING, "op": _STRING,
67
+ "value": {}, "value2": {}, "formula": _STRING, "style": _STYLE,
68
+ "scale": {"type": "object"}}, ("range", "op"))
69
+ _CHART = _object({"id": _STRING, "kind": _STRING, "range": _RANGE, "title": _STRING,
70
+ "stacking": _STRING, "source": {"type": "object"},
71
+ "series": {"type": "array", "items": {"type": "object"}},
72
+ "geometry": {"type": "object"}, "anchor": {"type": "object"},
73
+ "settings": {"type": "object"}}, ("kind", "range"))
74
+ _TARGET = {"workbook_id": _STRING,
75
+ "tab": {"type": "string", "description": "Tab id or tab name."},
76
+ "rev": {"type": "integer", "description": "Optional expected workbook revision."}}
77
+ _VIEW_CONFIG = _object({
78
+ "filters": {"oneOf": [_FILTER, {"type": "array", "items": {"type": "object"}}]},
79
+ "filterConj": {"enum": ["and", "or"]},
80
+ "sorts": {"type": "array", "items": _object({
81
+ "colId": _STRING, "dir": {"enum": ["asc", "desc"]},
82
+ "locked": {"type": "boolean"}}, ("colId", "dir"))},
83
+ "groupBy": {"type": ["string", "null"]},
84
+ "visible": {"type": "array", "items": _STRING,
85
+ "description": "Fields to show. Fields omitted from this list are hidden."},
86
+ "order": {"type": "array", "items": _STRING},
87
+ "display": _object({"mode": _STRING}, ("mode",)),
88
+ }, description="The grid's native config, passed unchanged to its saved View validator.")
89
+ _APPLICATION = {
90
+ "app_id": {"type": "string", "minLength": 1,
91
+ "description": "Omit to create. Supply an id to update mutable properties."},
92
+ "name": _STRING,
93
+ "type": {"enum": ["chart", "route", "map", "timeseries"]},
94
+ "compatibleDatabases": {"type": "array", "items": _STRING},
95
+ "sourceDatabase": {"type": "string", "description": "Source for a template Application only."},
96
+ "templateKind": {"enum": ["route", "catalog"]},
97
+ "fieldMap": {"type": "object", "additionalProperties": _STRING},
98
+ "navPlacement": _object({"section": {"const": "app"},
99
+ "order": {"type": "integer", "minimum": 0, "maximum": 999}},
100
+ ("section", "order"), additionalProperties=False),
101
+ "semanticRoleBindings": {"type": "object"},
102
+ "config": _object({"schemaVersion": {"const": 1},
103
+ "display": _object({"mode": _STRING}, ("mode",))},
104
+ ("schemaVersion", "display"), additionalProperties=False),
105
+ "share": _object({"mode": {"enum": ["private", "users", "tenant"]},
106
+ "users": {"type": "array", "items": _STRING}}, ("mode",)),
107
+ "starred": {"type": "boolean"},
108
+ }
109
+
110
+
111
+ def _schema(properties, required=(), *, tab=True):
112
+ return _object({**_TARGET, **properties},
113
+ ("workbook_id", *(("tab",) if tab else ()), *required),
114
+ additionalProperties=False)
115
+
116
+
117
+ def _book(ctx, args, *, write=False):
118
+ book = sheets.get_workbook(ctx.session.runtime, args["workbook_id"])
119
+ (ctx.require_editor if write else ctx.require_reader)(book)
120
+ return book
121
+
122
+
123
+ def _saved(book, tab=None):
124
+ out = {"id": book["id"], "rev": book["rev"],
125
+ "tabs": [{"id": row["id"], "name": row["name"]} for row in book["sheets"]]}
126
+ if tab is not None:
127
+ out["tab"] = tab
128
+ return out
129
+
130
+
131
+ def _put(ctx, args, book, tab=None):
132
+ # Carry the revision read for this document patch even when the client omitted one.
133
+ # Never overwrite an intervening UI save with the older snapshot we just patched.
134
+ if "rev" in args:
135
+ book["rev"] = args["rev"]
136
+ return _saved(sheets.put_workbook(ctx.session.runtime, book["id"], book,
137
+ actor=ctx.session.uname, actor_name=ctx.actor_name), tab)
138
+
139
+
140
+ def _create(ctx, args):
141
+ return _saved(sheets.create_workbook(ctx.session.runtime, name=args["name"],
142
+ created_by=ctx.session.uname,
143
+ on_duplicate="refuse"))
144
+
145
+
146
+ def _add_tab(ctx, args):
147
+ book = _book(ctx, args, write=True)
148
+ #: ⛔ A DUPLICATE TAB NAME IS REFUSED, because every other tool addresses a tab by `tab`,
149
+ #: which `sheets.find_tab` resolves BY NAME to the FIRST match. Two tabs sharing a name make
150
+ #: every later write land on one of them and the other unreachable, with nothing reported.
151
+ #: Measured at SDLC 61's C7 build: a retried `sheet_add_tab` produced a second
152
+ #: `Summary (All)` and the workbook's cells split silently across the pair.
153
+ wanted = str(args["name"]).strip()
154
+ if any(str(sheet.get("name") or "").strip().casefold() == wanted.casefold()
155
+ for sheet in (book.get("sheets") or ())):
156
+ raise sheets.SheetRefusal(
157
+ "tab_name_taken",
158
+ "Tab " + wanted + " is already in this workbook. Choose another name.")
159
+ tab = {"id": uuid.uuid4().hex[:12], "name": args["name"], "cells": {}}
160
+ book["sheets"].append(tab)
161
+ return _put(ctx, args, book, tab["id"])
162
+
163
+
164
+ def _save_template(ctx, args):
165
+ book = ctx.project_book(_book(ctx, args))
166
+ record = sheet_templates.save_sheet_template(
167
+ ctx.session.runtime, book, args["key"], name=args["name"],
168
+ description=args["description"], created_by=ctx.session.uname, actor_name=ctx.actor_name)
169
+ return {"template": record}
170
+
171
+
172
+ def _read_cells(ctx, args):
173
+ book = ctx.project_book(_book(ctx, args))
174
+ tab = sheets.find_tab(book, args["tab"])
175
+ if "addresses" in args:
176
+ wanted = {addr.strip().upper() for addr in args["addresses"]}
177
+ tab = {**tab, "cells": {addr: cell for addr, cell in tab["cells"].items() if addr in wanted}}
178
+ return {"id": book["id"], "rev": book["rev"], "tab": tab}
179
+
180
+
181
+ def _patch(ctx, args, cells, *, resolved=None):
182
+ out = sheets.patch_cells(ctx.session.runtime, args["workbook_id"], args["tab"], cells,
183
+ expect_rev=args.get("rev"), actor=ctx.session.uname,
184
+ actor_name=ctx.actor_name)
185
+ if resolved is not None:
186
+ for addr in out["accepted"]:
187
+ cell = cells[addr]
188
+ ctx.cache_binding(args["workbook_id"], out["tab"], addr, cell["binding"], resolved)
189
+ # The UI computes after its patch too. Project before computing so a shared workbook
190
+ # never returns its owner's cached database numbers through an editor's write response.
191
+ book = ctx.project_book(_book(ctx, args))
192
+ tab = sheets.find_tab(book, args["tab"])
193
+ out["computed"] = {addr: cell for addr, cell in tab["cells"].items()
194
+ if cell.get("kind") in ("formula", "bound") or "spillParent" in cell}
195
+ return out
196
+
197
+
198
+ def _write_cells(ctx, args):
199
+ _book(ctx, args, write=True)
200
+ return _patch(ctx, args, args["cells"])
201
+
202
+
203
+ def _set_binding(ctx, args):
204
+ book = _book(ctx, args, write=True)
205
+ tab = sheets.find_tab(book, args["tab"])
206
+ resolved = ctx.resolve_binding(args["binding"], book, tab["id"])
207
+ addr = args["addr"].strip().upper()
208
+ before = tab["cells"].get(addr) or {}
209
+ cell = {key: copy.deepcopy(before[key]) for key in ("style", "validation", "link", "note")
210
+ if key in before}
211
+ cell.update(kind="bound", binding=copy.deepcopy(args["binding"]), state="fresh",
212
+ value=resolved["value"],
213
+ display="" if resolved["value"] is None else str(resolved["value"]))
214
+ out = _patch(ctx, args, {addr: cell}, resolved=resolved)
215
+ out["limits"] = resolved.get("limits") or []
216
+ return out
217
+
218
+
219
+ def _cell_property(ctx, args, property_name, values):
220
+ book = _book(ctx, args, write=True)
221
+ tab = sheets.find_tab(book, args["tab"])
222
+ cells = {}
223
+ for addr, value in values.items():
224
+ cell = copy.deepcopy(tab["cells"].get(addr.strip().upper()) or {"kind": "empty"})
225
+ cell[property_name] = value
226
+ cells[addr] = cell
227
+ return _patch(ctx, args, cells)
228
+
229
+
230
+ def _dimensions(ctx, args):
231
+ book = _book(ctx, args, write=True)
232
+ tab = sheets.find_tab(book, args["tab"])
233
+ tab.update(copy.deepcopy(args["dimensions"]))
234
+ return _put(ctx, args, book, tab["id"])
235
+
236
+
237
+ def _freeze(ctx, args):
238
+ book = _book(ctx, args, write=True)
239
+ tab = sheets.find_tab(book, args["tab"])
240
+ tab["freeze"] = args["freeze"]
241
+ return _put(ctx, args, book, tab["id"])
242
+
243
+
244
+ def _rule(ctx, args):
245
+ book = _book(ctx, args, write=True)
246
+ tab = sheets.find_tab(book, args["tab"])
247
+ tab["rules"] = [*(tab.get("rules") or []), args["rule"]]
248
+ return _put(ctx, args, book, tab["id"])
249
+
250
+
251
+ def _chart(ctx, args):
252
+ book = _book(ctx, args, write=True)
253
+ tab = sheets.find_tab(book, args["tab"])
254
+ chart = copy.deepcopy(args["chart"])
255
+ chart.setdefault("id", uuid.uuid4().hex[:12])
256
+ tab["charts"] = [row for row in tab.get("charts", []) if row["id"] != chart["id"]] + [chart]
257
+ return {**_put(ctx, args, book, tab["id"]), "chart_id": chart["id"]}
258
+
259
+
260
+ def _structure(ctx, args):
261
+ book = _book(ctx, args, write=True)
262
+ tab = sheets.find_tab(book, args["tab"])
263
+ shifted = sheet_structure.apply_structure(book, tab["id"], args["op"])
264
+ return _put(ctx, args, shifted, tab["id"])
265
+
266
+
267
+ def _tool(description, schema, handler, *, write=False, grants=("analyst", "sheets")):
268
+ return {"description": description, "inputSchema": schema, "write": write,
269
+ "grants": grants, "handler": handler}
270
+
271
+
272
+ TOOLS = {
273
+ "sheet_create_workbook": _tool("Create a private workbook with one initial tab.",
274
+ _object({"name": _STRING}, ("name",), additionalProperties=False), _create, write=True),
275
+ "sheet_add_tab": _tool("Add a blank tab to a workbook.",
276
+ _schema({"name": _STRING}, ("name",), tab=False), _add_tab, write=True),
277
+ "sheet_read_cells": _tool("Read a tab with styles, geometry and figures under the maker's grants.",
278
+ _schema({"addresses": {"type": "array", "items": _ADDRESS}}), _read_cells),
279
+ "sheet_write_cells": _tool("Write cell documents with values or formulas; null deletes a cell.",
280
+ _schema({"cells": {"type": "object", "additionalProperties": {"anyOf": [_CELL, {"type": "null"}]}}},
281
+ ("cells",)), _write_cells, write=True),
282
+ "sheet_set_binding": _tool("Set and resolve a Cell database binding, including filter valueRef cells.",
283
+ _schema({"addr": _ADDRESS, "binding": _BINDING}, ("addr", "binding")), _set_binding, write=True),
284
+ "sheet_set_styles": _tool("Replace styles by address, preserving cell content; null clears a style.",
285
+ _schema({"styles": {"type": "object", "additionalProperties": {"anyOf": [_STYLE, {"type": "null"}]}}},
286
+ ("styles",)), lambda ctx, args: _cell_property(ctx, args, "style", args["styles"]), write=True),
287
+ "sheet_set_dimensions": _tool("Replace supplied geometry fields; widths use letters and row indices start at one.",
288
+ _schema({"dimensions": _object({
289
+ "cols": {"type": "integer"}, "rows": {"type": "integer"},
290
+ "colWidths": {"type": "object", "additionalProperties": {"type": "number"}},
291
+ "rowHeight": {"type": "number"},
292
+ "rowHeights": {"type": "object", "additionalProperties": {"type": "number"}},
293
+ "hiddenRows": {"type": "array", "items": {"type": "integer"}},
294
+ "hiddenCols": {"type": "array", "items": {"type": "integer"}},
295
+ }, additionalProperties=False)}, ("dimensions",)), _dimensions, write=True),
296
+ "sheet_set_validation": _tool("Replace dropdown lists by address, preserving content; null clears validation.",
297
+ _schema({"validations": {"type": "object", "additionalProperties": {
298
+ "anyOf": [_VALIDATION, {"type": "null"}]}}}, ("validations",)),
299
+ lambda ctx, args: _cell_property(ctx, args, "validation", args["validations"]), write=True),
300
+ "sheet_set_freeze": _tool("Set frozen row and column counts, using zero to unfreeze.",
301
+ _schema({"freeze": _object({"rows": {"type": "integer"}, "cols": {"type": "integer"}})},
302
+ ("freeze",)), _freeze, write=True),
303
+ "sheet_add_rule": _tool("Append a conditional format rule accepted by the Sheet document cleaner.",
304
+ _schema({"rule": _RULE}, ("rule",)), _rule, write=True),
305
+ "sheet_set_chart": _tool("Add a chart or replace a chart with the same id using the app's chart document.",
306
+ _schema({"chart": _CHART}, ("chart",)), _chart, write=True),
307
+ "sheet_structure": _tool("Insert or delete rows or columns and reanchor references; positive delta inserts.",
308
+ _schema({"op": _object({"axis": {"enum": ["row", "col"]},
309
+ "at": {"type": "integer", "minimum": 1}, "delta": {"type": "integer"}},
310
+ ("axis", "at", "delta"))}, ("op",)), _structure, write=True),
311
+ "sheet_save_template": _tool("Save a readable workbook as a tenant Sheet template with its Cell databases.",
312
+ _object({key: _STRING for key in ("workbook_id", "key", "name", "description")},
313
+ ("workbook_id", "key", "name", "description"), additionalProperties=False),
314
+ _save_template, write=True),
315
+ "list_databases": _tool("List the databases permitted or visible to the key's maker.",
316
+ _object(additionalProperties=False), lambda ctx, args: ctx.list_databases(), grants=("analyst",)),
317
+ "describe_database": _tool("Read a database's field schema through the strict Assistant scope wall.",
318
+ _object({"database": _STRING}, ("database",), additionalProperties=False),
319
+ lambda ctx, args: ctx.describe_database(args["database"]), grants=("analyst",)),
320
+ "view_upsert": _tool(
321
+ "Create or update a saved View using the grid's filters, sorts, groupBy, visible fields and display config.",
322
+ _object({"database": _STRING, "view": _object({
323
+ "id": {"type": "string", "minLength": 1, "maxLength": 120},
324
+ "name": {"type": "string", "minLength": 1, "maxLength": 120},
325
+ "config": _VIEW_CONFIG, "note": _STRING}, ("id", "name", "config"))},
326
+ ("database", "view"), additionalProperties=False),
327
+ lambda ctx, args: ctx.view_event("view_upsert", args), write=True, grants=("analyst",)),
328
+ "view_delete": _tool("Delete a saved View through its ordinary owner and database permission walls.",
329
+ _object({"database": _STRING, "view_id": _STRING}, ("database", "view_id"),
330
+ additionalProperties=False),
331
+ lambda ctx, args: ctx.view_event("view_delete", args), write=True, grants=("analyst",)),
332
+ "app_upsert": _tool(
333
+ "Create an App with name, type, compatibleDatabases and config, or update it by app_id. Identity fields are immutable.",
334
+ _object(_APPLICATION, additionalProperties=False,
335
+ anyOf=[{"required": ["app_id"]}, {"required": ["name", "type", "config"]},
336
+ {"required": ["name", "templateKind", "sourceDatabase", "fieldMap", "navPlacement"]}]),
337
+ lambda ctx, args: ctx.app_upsert(args), write=True, grants=("analyst",)),
338
+ "app_delete": _tool("Delete an App. confirmName must match its current name exactly.",
339
+ _object({"app_id": _STRING, "confirmName": _STRING}, ("app_id", "confirmName"),
340
+ additionalProperties=False),
341
+ lambda ctx, args: ctx.app_delete(args), write=True, grants=("analyst",)),
342
+ }
343
+
344
+
345
+ def dispatch(ctx, name, arguments):
346
+ """Dispatch one declared tool. The API translates core and grant refusals to MCP content."""
347
+ tool = TOOLS.get(name) if isinstance(name, str) else None
348
+ if tool is None:
349
+ raise sheets.SheetRefusal("tool_not_found", "The requested tool is not available.")
350
+ for grant in tool["grants"]:
351
+ ctx.session.require(grant)
352
+ schema = tool["inputSchema"]
353
+ if not isinstance(arguments, dict):
354
+ raise sheets.SheetRefusal("tool_arguments_invalid", "Tool arguments must be an object.")
355
+ missing = set(schema["required"]) - arguments.keys()
356
+ unknown = arguments.keys() - schema["properties"].keys()
357
+ if missing or (unknown and schema.get("additionalProperties") is False):
358
+ raise sheets.SheetRefusal("tool_arguments_invalid", "Tool arguments must match the published input schema.")
359
+ # Validate only the transport shape here. Document semantics belong to the app writers.
360
+ for key, value in arguments.items():
361
+ kind = schema["properties"].get(key, {}).get("type")
362
+ types = {"object": dict, "array": list, "string": str, "integer": int}
363
+ if kind in types and (not isinstance(value, types[kind]) or (kind == "integer" and isinstance(value, bool))):
364
+ raise sheets.SheetRefusal("tool_arguments_invalid", f"Tool argument {key} must be an {kind}.")
365
+ # A dimension patch must never become an arbitrary tab-document replacement.
366
+ if name == "sheet_set_dimensions":
367
+ allowed = schema["properties"]["dimensions"]["properties"]
368
+ if arguments["dimensions"].keys() - allowed.keys():
369
+ raise sheets.SheetRefusal("tool_arguments_invalid", "Dimensions may contain only the published geometry fields.")
370
+ if "lpb1." in json.dumps(arguments, ensure_ascii=False, allow_nan=False):
371
+ raise sheets.SheetRefusal("credential_in_arguments", "Tool arguments must not contain credential values.")
372
+ return tool["handler"](ctx, arguments)
platform/core/measure_resolve.py CHANGED
@@ -152,8 +152,8 @@ def column_values(fields, team_id, allowed_pids, today, stamp, memo, on_error=No
152
 
153
  `related` is the row-window {table, record, via} pin. The host binds its scoped readers
154
  with relations.measure_pin_context and publishes the returned mapping's `refusals` beside
155
- its cells. A refused pin retains unscoped values plus a per-field named marker, on memo
156
- hits too. Every measure and its conditions use the same query scope and caller's pool.
157
  """
158
  mfields = [f for f in fields or [] if isinstance(f.get('measure'), dict)]
159
  if not mfields:
@@ -197,6 +197,8 @@ def column_values(fields, team_id, allowed_pids, today, stamp, memo, on_error=No
197
  if related is not None:
198
  out.refusals.extend(dict(marker, field=fkey)
199
  for marker in getattr(vals, 'refusals', ()))
 
 
200
  for pid, v in vals.items():
201
  out.setdefault(pid, {})[fkey] = v
202
  if len(memo) > 100:
 
152
 
153
  `related` is the row-window {table, record, via} pin. The host binds its scoped readers
154
  with relations.measure_pin_context and publishes the returned mapping's `refusals` beside
155
+ its cells. A refused pin publishes a per-field marker and no values, on memo hits too.
156
+ Every measure and its conditions use the same query scope and caller's pool.
157
  """
158
  mfields = [f for f in fields or [] if isinstance(f.get('measure'), dict)]
159
  if not mfields:
 
197
  if related is not None:
198
  out.refusals.extend(dict(marker, field=fkey)
199
  for marker in getattr(vals, 'refusals', ()))
200
+ if getattr(vals, 'refusals', ()):
201
+ continue
202
  for pid, v in vals.items():
203
  out.setdefault(pid, {})[fkey] = v
204
  if len(memo) > 100:
platform/core/odoo.py CHANGED
@@ -103,7 +103,9 @@ def tenant_scope(slug, creds=None, team_ids=None, team_names=None, exclude_partn
103
  Unbound (the default) = tenant #0 via env, which is every existing Streamlit/API caller.
104
  The connector layer (harness/connectors/odoo.py) wraps each query in this, so a second
105
  tenant's keychain credentials reach the wire without any module knowing. Scope constants
106
- are PUSHED here from Tenant.config because core/ may not import harness/ (layering)."""
 
 
107
  prev = getattr(_tlocal, 'tenant_ctx', None)
108
  _tlocal.tenant_ctx = {
109
  'slug': str(slug or _DEFAULT_SLUG),
 
103
  Unbound (the default) = tenant #0 via env, which is every existing Streamlit/API caller.
104
  The connector layer (harness/connectors/odoo.py) wraps each query in this, so a second
105
  tenant's keychain credentials reach the wire without any module knowing. Scope constants
106
+ are PUSHED here from Tenant.config because core/ may not import harness/ (layering; enforced
107
+ forward by platform/verify_layering.py, which grandfathers the older core -> harness edges
108
+ by name, D-460)."""
109
  prev = getattr(_tlocal, 'tenant_ctx', None)
110
  _tlocal.tenant_ctx = {
111
  'slug': str(slug or _DEFAULT_SLUG),
platform/core/perm_scope.py CHANGED
@@ -40,6 +40,7 @@ twice in this codebase (`modules: []` and `bus: [<unknown id>]` both read as UNR
40
  perms block; without this clause an explicit-marker scheme locks the owner out of their own
41
  product at exactly the moment the store is broken.
42
  """
 
43
  import re
44
 
45
  import core.perms as perms
@@ -1400,7 +1401,7 @@ class Unresolvable(RuntimeError):
1400
 
1401
 
1402
  #: Row readers DECLARED by the app layer, keyed by EXACT database key.
1403
- #: `reader(table_key, user, st) -> (fields, rows)`.
1404
  #:
1405
  #: ⛔ WHY A REGISTRY AND NOT AN IMPORT. `core` never imports up (`platform/ARCHITECTURE.md`), and
1406
  #: a registry TOPIC's rows are built by `modules/` + `aios_grid` behind an API-layer pool cache
@@ -1409,6 +1410,7 @@ class Unresolvable(RuntimeError):
1409
  #: `ROW_HOOKS`: *"`core` never imports up, so the app tells this layer rather than being
1410
  #: interrogated by it."*
1411
  _ROW_SOURCES = {}
 
1412
 
1413
  #: THE reader for a read-through `ut_*` grid — one reader, because there is one mirror.
1414
  #: `reader(table_key, field_keys, st) -> rows`.
@@ -1423,10 +1425,18 @@ def register_rows(reader, *table_keys):
1423
  says so ([[reachable-is-not-the-same-as-built]]). Routing the read door through the write
1424
  door's return keeps one construction site of the set instead of two.
1425
  """
 
 
 
 
 
 
 
1426
  for key in table_keys:
1427
  k = str(key or '').strip()
1428
  if k:
1429
  _ROW_SOURCES[k] = reader
 
1430
  return frozenset(_ROW_SOURCES)
1431
 
1432
 
@@ -1568,8 +1578,8 @@ def scoped_fields(user, table_key, st=None):
1568
  second path to the store (W1). So both come from here, off one wall.
1569
 
1570
  ⚠ On a `ut_*` database this reads the DEFINITION only — the projection, no rows (D-213). On a
1571
- registry topic it goes through the registered reader, which builds that topic's pool; the
1572
- pool is cached per scope on the tenant runtime, so it is a cache hit next to `scoped_table`.
1573
  """
1574
  fields, _rows = _scoped(user, table_key, st=st, ctx=None, want_rows=False)
1575
  return fields
@@ -1613,7 +1623,10 @@ def _read(table_key, user, st, want_rows=True):
1613
  """`(fields, rows)` BEFORE the wall — the app layer's reader, or core's own for a `ut_*`."""
1614
  reader = _ROW_SOURCES.get(table_key)
1615
  if reader is not None:
1616
- fields, rows = reader(table_key, user, st)
 
 
 
1617
  return list(fields or ()), list(rows or ())
1618
  ut = _ut()
1619
  if not table_key.startswith(ut.KEY_PREFIX):
 
40
  perms block; without this clause an explicit-marker scheme locks the owner out of their own
41
  product at exactly the moment the store is broken.
42
  """
43
+ import inspect
44
  import re
45
 
46
  import core.perms as perms
 
1401
 
1402
 
1403
  #: Row readers DECLARED by the app layer, keyed by EXACT database key.
1404
+ #: `reader(table_key, user, st[, want_rows=True]) -> (fields, rows)`.
1405
  #:
1406
  #: ⛔ WHY A REGISTRY AND NOT AN IMPORT. `core` never imports up (`platform/ARCHITECTURE.md`), and
1407
  #: a registry TOPIC's rows are built by `modules/` + `aios_grid` behind an API-layer pool cache
 
1410
  #: `ROW_HOOKS`: *"`core` never imports up, so the app tells this layer rather than being
1411
  #: interrogated by it."*
1412
  _ROW_SOURCES = {}
1413
+ _ROW_SOURCE_WANT_ROWS = {}
1414
 
1415
  #: THE reader for a read-through `ut_*` grid — one reader, because there is one mirror.
1416
  #: `reader(table_key, field_keys, st) -> rows`.
 
1425
  says so ([[reachable-is-not-the-same-as-built]]). Routing the read door through the write
1426
  door's return keeps one construction site of the set instead of two.
1427
  """
1428
+ # Decide the calling convention once. Never retry a reader's own TypeError as an old
1429
+ # signature: doing so can silently replace a failed read with a different answer.
1430
+ parameters = inspect.signature(reader).parameters
1431
+ accepts_want_rows = any(p.kind == inspect.Parameter.VAR_KEYWORD
1432
+ for p in parameters.values()) or (
1433
+ 'want_rows' in parameters and parameters['want_rows'].kind in (
1434
+ inspect.Parameter.POSITIONAL_OR_KEYWORD, inspect.Parameter.KEYWORD_ONLY))
1435
  for key in table_keys:
1436
  k = str(key or '').strip()
1437
  if k:
1438
  _ROW_SOURCES[k] = reader
1439
+ _ROW_SOURCE_WANT_ROWS[k] = accepts_want_rows
1440
  return frozenset(_ROW_SOURCES)
1441
 
1442
 
 
1578
  second path to the store (W1). So both come from here, off one wall.
1579
 
1580
  ⚠ On a `ut_*` database this reads the DEFINITION only — the projection, no rows (D-213). On a
1581
+ registry topic it asks the registered reader for fields only when supported. Legacy
1582
+ three-argument readers retain their existing call contract.
1583
  """
1584
  fields, _rows = _scoped(user, table_key, st=st, ctx=None, want_rows=False)
1585
  return fields
 
1623
  """`(fields, rows)` BEFORE the wall — the app layer's reader, or core's own for a `ut_*`."""
1624
  reader = _ROW_SOURCES.get(table_key)
1625
  if reader is not None:
1626
+ if _ROW_SOURCE_WANT_ROWS.get(table_key, False):
1627
+ fields, rows = reader(table_key, user, st, want_rows=want_rows)
1628
+ else:
1629
+ fields, rows = reader(table_key, user, st)
1630
  return list(fields or ()), list(rows or ())
1631
  ut = _ut()
1632
  if not table_key.startswith(ut.KEY_PREFIX):
platform/core/relations.py CHANGED
@@ -265,6 +265,20 @@ def _link_bag(field):
265
  return bag if isinstance(bag, dict) else None
266
 
267
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
268
  def _entry(key, label, target, direction, field=None, on=None, frm=None, served=True,
269
  where=None, cardinality=None):
270
  """ONE `/related` entry, in C6's shape.
@@ -745,6 +759,10 @@ def measure_pin_key(related):
745
  if related is None:
746
  return None
747
  ctx = _MEASURE_PIN_CONTEXT.get() or {}
 
 
 
 
748
  return json.dumps(related, sort_keys=True, separators=(',', ':')), ctx.get('cache_key')
749
 
750
 
@@ -754,11 +772,17 @@ class MeasurePin:
754
  rows: dict
755
  bindings: dict
756
  refusal: dict = None
 
757
 
758
  def predicate(self, store):
759
  """AND every declared, traversed member set into this metric's raw rows."""
760
  if self.refusal is not None:
761
  return None, []
 
 
 
 
 
762
  parts, params = [], []
763
  for binding in self.bindings.get(store['table'], ()):
764
  table, column = binding['table'], binding['column']
@@ -1126,6 +1150,15 @@ def one_hop(table, fields=None, defs=None, include_retired=False):
1126
  out = []
1127
  for entry in base:
1128
  entry = dict(entry)
 
 
 
 
 
 
 
 
 
1129
  got = cards.get(entry['key']) or bridge_cards.get(entry['key'])
1130
  entry['cardinality'] = str(got or 'many_to_many')
1131
  out.append(entry)
 
265
  return bag if isinstance(bag, dict) else None
266
 
267
 
268
+ def declared_through(link):
269
+ """R50 / C4: return the intermediate of a complete declared two-hop link, or ''.
270
+
271
+ The intermediate is a join, not a destination. R50 says grants on both ends suffice;
272
+ intermediate fields are never projected, returned or rendered. An incomplete through
273
+ bag is not a declared two-hop link and earns no traversal admission. Manual links do
274
+ not declare a traversal either.
275
+ """
276
+ if (isinstance(link, dict) and link.get('through') and link.get('throughOn')
277
+ and link.get('throughTo') and link.get('selectionMode') != 'manual'):
278
+ return str(link['through'])
279
+ return ''
280
+
281
+
282
  def _entry(key, label, target, direction, field=None, on=None, frm=None, served=True,
283
  where=None, cardinality=None):
284
  """ONE `/related` entry, in C6's shape.
 
759
  if related is None:
760
  return None
761
  ctx = _MEASURE_PIN_CONTEXT.get() or {}
762
+ if isinstance(related, MeasurePin):
763
+ # Compiled pins contain the server's walled membership, not just a wire path.
764
+ related = dict(path=related.path, rows=related.rows, bindings=related.bindings,
765
+ refusal=related.refusal, required=related.required)
766
  return json.dumps(related, sort_keys=True, separators=(',', ':')), ctx.get('cache_key')
767
 
768
 
 
772
  rows: dict
773
  bindings: dict
774
  refusal: dict = None
775
+ required: tuple = ()
776
 
777
  def predicate(self, store):
778
  """AND every declared, traversed member set into this metric's raw rows."""
779
  if self.refusal is not None:
780
  return None, []
781
+ bound = {b['table'] for b in self.bindings.get(store['table'], ())}
782
+ if set(self.required) - bound:
783
+ raise PinRefusal('relation_binding_unavailable',
784
+ 'This metric cannot be restricted to every database in this record path. '
785
+ 'Ask an administrator to add the missing metric binding.', self.path)
786
  parts, params = [], []
787
  for binding in self.bindings.get(store['table'], ()):
788
  table, column = binding['table'], binding['column']
 
1150
  out = []
1151
  for entry in base:
1152
  entry = dict(entry)
1153
+ # Keep related()'s legacy shape. Depth-aware pins also need the declared manual,
1154
+ # inverse or through configuration, which remains schema-only cache input.
1155
+ via = entry.get('via') or {}
1156
+ owner_fields = (((defs or {}).get(entry['target']) or {}).get('fields') or []
1157
+ if via.get('direction') == 'reverse' else fields or [])
1158
+ declared = next((field for field in owner_fields
1159
+ if field.get('key') == via.get('field')), None)
1160
+ if declared and _link_bag(declared):
1161
+ entry['link'] = dict(_link_bag(declared))
1162
  got = cards.get(entry['key']) or bridge_cards.get(entry['key'])
1163
  entry['cardinality'] = str(got or 'many_to_many')
1164
  out.append(entry)
platform/core/shares.py CHANGED
@@ -553,11 +553,28 @@ def role_for(kind, oid, user, is_admin=False, st=None):
553
  user = str(user or '').strip().lower()
554
  if not user:
555
  return None
556
- rec = grants(kind, oid, st=st)
557
- if is_admin or (rec['owner'] and rec['owner'] == user):
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
558
  return 'owner'
559
  best = None
560
- for e in rec['entries']:
561
  if e['user'] == user or e['user'] == EVERYONE:
562
  # The STRONGER of the two wins when both a personal and an everyone grant exist:
563
  # naming somebody explicitly is how you RAISE them above the room, so an
@@ -568,6 +585,37 @@ def role_for(kind, oid, user, is_admin=False, st=None):
568
  return best
569
 
570
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
571
  def may_see(kind, oid, user, is_admin=False, st=None):
572
  return role_for(kind, oid, user, is_admin=is_admin, st=st) is not None
573
 
 
553
  user = str(user or '').strip().lower()
554
  if not user:
555
  return None
556
+ return role_from(grants(kind, oid, st=st), user, is_admin=is_admin)
557
+
558
+
559
+ def role_from(rec, user, is_admin=False):
560
+ """The SAME answer as `role_for`, from a record the caller already holds. PURE.
561
+
562
+ ⭐⭐ W61-T17 — ONE EVALUATOR, TWO READ PATHS. `grants_bulk` exists so a list of N objects
563
+ costs ONE bucket read instead of N, and the moment a batched caller re-implements "which role
564
+ does this grant record give?" the two answers can drift — and the one that drifts is a
565
+ PERMISSION answer. So the loop lives here, `role_for` is now a thin read-then-call, and a
566
+ batched caller passes the record it already has. [[one-evaluator-per-question]]
567
+
568
+ ⚠ `rec` may be None (an object with no grant record at all), which reads as no role.
569
+ """
570
+ user = str(user or '').strip().lower()
571
+ if not user:
572
+ return None
573
+ rec = rec or {'owner': None, 'entries': []}
574
+ if is_admin or (rec.get('owner') and rec['owner'] == user):
575
  return 'owner'
576
  best = None
577
+ for e in rec.get('entries') or ():
578
  if e['user'] == user or e['user'] == EVERYONE:
579
  # The STRONGER of the two wins when both a personal and an everyone grant exist:
580
  # naming somebody explicitly is how you RAISE them above the room, so an
 
585
  return best
586
 
587
 
588
+ def grants_bulk(kind, st=None):
589
+ """`{oid: {'owner', 'entries'}}` for every object of one `kind` — in ONE bucket read.
590
+
591
+ ⭐⭐ W61-T17 — THE SAME SHAPE `granted_oids` EXISTS FOR, AND THE SAME REASON, stated there in
592
+ full: `grants()` re-reads and re-copies the WHOLE registry on every call, so a list of N
593
+ objects that asks per object pays N full reads of the tenant's entire grant registry. This
594
+ reads it once. `granted_oids` answers "does it have any grants"; this answers "what are they",
595
+ which is what a list joining `shares` onto rows actually needs.
596
+
597
+ ⛔ IT NORMALISES THROUGH `_clean_entries`, exactly as `grants` does, so a junk entry cannot
598
+ reach a caller by the batched door that the single door would have dropped. An owner husk left
599
+ by a full revoke keeps its owner and an EMPTY entry list, which is what `grants` answers too.
600
+ ⛔ FAIL-CLOSED: an unreadable bucket answers `{}` — no roles, no shares — never a default.
601
+ An unknown KIND still RAISES, as everywhere else in this module.
602
+ """
603
+ kind = _check_kind(kind)
604
+ try:
605
+ by_kind = (_st(st).get(SHARES_KEY) or {}).get(kind) or {}
606
+ rows = list(by_kind.items())
607
+ except Exception:
608
+ return {}
609
+ out = {}
610
+ for oid, rec in rows:
611
+ if not isinstance(rec, dict):
612
+ continue
613
+ out[str(oid)] = {'owner': (str(rec.get('owner')).strip().lower()
614
+ if rec.get('owner') else None),
615
+ 'entries': _clean_entries(rec.get('entries'))}
616
+ return out
617
+
618
+
619
  def may_see(kind, oid, user, is_admin=False, st=None):
620
  return role_for(kind, oid, user, is_admin=is_admin, st=st) is not None
621
 
platform/core/sheet_binding.py ADDED
@@ -0,0 +1,180 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Pure document cleaners shared by Sheet storage and Sheet query resolution.
2
+
3
+ These functions decide only what binding metadata may be stored. They do not resolve data,
4
+ permissions, or SQL. Keeping them in core lets the workbook document validate view bindings
5
+ without importing the higher ``modules.sheet`` layer.
6
+ """
7
+
8
+ from __future__ import annotations
9
+
10
+ import datetime
11
+ import re
12
+
13
+ from core import windows as W
14
+
15
+
16
+ PERIOD_UNITS = ("day", "week", "month", "quarter", "year")
17
+ PERIOD_START = re.compile(r"^\d{4}-(0[1-9]|1[0-2])$")
18
+ PERIOD_DAY = re.compile(r"^\d{4}-(0[1-9]|1[0-2])-(0[1-9]|[12]\d|3[01])$")
19
+ PERIOD_REF = re.compile(r"^[A-Z]{1,3}[1-9][0-9]{0,6}$")
20
+ DRAG_ACROSS = ("period", "none")
21
+ DRAG_DOWN_NONE = "none"
22
+
23
+ _MESSAGES = {
24
+ "period_invalid": "{subject} is not a period this cell can use. Choose a valid start for the selected period unit.",
25
+ "ending_invalid": "Ending {subject} cannot be used. Choose a valid ending date.",
26
+ "ending_not_supported": "{subject} does not support an Ending. Choose year to date, last year to date, this month or this quarter.",
27
+ "ending_cell_unreadable": "Ending from cell {subject} cannot be read. Use a cell containing a date in YYYY-MM-DD format.",
28
+ }
29
+
30
+
31
+ class BindingMetadataRefusal(Exception):
32
+ """A structured metadata refusal which each caller translates to its route exception."""
33
+
34
+ def __init__(self, code: str, subject: str = "", detail: str = ""):
35
+ super().__init__(code)
36
+ self.code = code
37
+ self.subject = str(subject or "")
38
+ self.detail = str(detail or "")
39
+ self.message = _MESSAGES.get(code, "{subject} could not be answered.").format(
40
+ subject=self.subject or "this database")
41
+ if code in ("ending_invalid", "ending_cell_unreadable") and self.detail:
42
+ verb = "used" if code == "ending_invalid" else "read"
43
+ self.message = self.message.replace(
44
+ f"cannot be {verb}.", f"cannot be {verb} because {self.detail}")
45
+
46
+
47
+ def clean_period(raw):
48
+ """Return one canonical Sheet period, or ``None`` when no window was named."""
49
+ if not isinstance(raw, dict):
50
+ return None
51
+ out = {}
52
+
53
+ if "ending" in raw:
54
+ window = W.normalize(raw.get("window"))
55
+ kind = (window or {}).get("kind")
56
+ if kind not in W.TO_DATE_KINDS:
57
+ raise BindingMetadataRefusal("ending_not_supported", kind or "This period")
58
+ ending = raw["ending"]
59
+ mode = ending.get("kind") if isinstance(ending, dict) else None
60
+ if mode not in W.ENDING_KINDS:
61
+ raise BindingMetadataRefusal("ending_invalid", str(mode or "selection"),
62
+ "the choice is unknown.")
63
+ step = raw.get("endingStep", W.ending_step_default(kind))
64
+ offset = raw.get("endingOffset", 0)
65
+ if (step not in W.ENDING_STEPS or isinstance(offset, bool)
66
+ or not isinstance(offset, int) or offset < 0):
67
+ raise BindingMetadataRefusal("ending_invalid", "step",
68
+ "its size or copy offset is invalid.")
69
+ out = {"window": window, "ending": {"kind": mode}, "endingStep": step}
70
+ if offset:
71
+ out["endingOffset"] = offset
72
+ if raw.get("locked") is True:
73
+ out["locked"] = True
74
+ if mode == "fixed":
75
+ day = W.ending_date(ending.get("date"))
76
+ if day is None:
77
+ raise BindingMetadataRefusal("ending_invalid", "fixed date",
78
+ "it is not a real ISO date.")
79
+ out["ending"]["date"] = day
80
+ if mode == "n_before_today":
81
+ n, unit = ending.get("n"), ending.get("unit")
82
+ if isinstance(n, bool) or not isinstance(n, int) or not 1 <= n <= W.MAX_N:
83
+ raise BindingMetadataRefusal(
84
+ "ending_invalid", "number", f"it must be a whole number from 1 to {W.MAX_N}.")
85
+ if unit not in W.ENDING_STEPS:
86
+ raise BindingMetadataRefusal("ending_invalid", "unit", "the unit is unknown.")
87
+ out["ending"].update({"n": n, "unit": unit})
88
+ if mode == "cell":
89
+ ref = str(raw.get("ref") or "").strip()
90
+ if not re.fullmatch(
91
+ r"(?:(?:'([^']|'')+'|[^'!]+)!)?\$?[A-Za-z]{1,3}\$?[1-9][0-9]{0,6}", ref):
92
+ raise BindingMetadataRefusal("ending_cell_unreadable", ref or "(blank)",
93
+ "its reference is invalid.")
94
+ out["ref"] = ref
95
+ return out
96
+
97
+ unit = str(raw.get("unit") or "").strip().lower()
98
+ if unit:
99
+ if unit not in PERIOD_UNITS:
100
+ raise BindingMetadataRefusal("period_invalid", unit)
101
+ out["unit"] = unit
102
+
103
+ start = str(raw.get("start") or "").strip()
104
+ if start:
105
+ pattern = PERIOD_DAY if unit in ("day", "week") else PERIOD_START
106
+ if not pattern.fullmatch(start):
107
+ raise BindingMetadataRefusal("period_invalid", start)
108
+ if unit in ("day", "week"):
109
+ try:
110
+ day = datetime.date.fromisoformat(start)
111
+ except ValueError as exc:
112
+ raise BindingMetadataRefusal("period_invalid", start) from exc
113
+ if unit == "week":
114
+ start = (day - datetime.timedelta(days=day.weekday())).isoformat()
115
+ out["start"] = start
116
+
117
+ if raw.get("frozen") is True:
118
+ dates = {}
119
+ for key in ("from", "to"):
120
+ value = str(raw.get(key) or "").strip()
121
+ if not PERIOD_DAY.fullmatch(value):
122
+ raise BindingMetadataRefusal("period_invalid", value or key)
123
+ dates[key] = value
124
+ if dates["to"] < dates["from"]:
125
+ raise BindingMetadataRefusal("period_invalid", dates["to"])
126
+ out.update({"frozen": True, **dates})
127
+
128
+ if raw.get("locked") is True:
129
+ out["locked"] = True
130
+ ref = str(raw.get("ref") or "").strip().upper().replace("$", "")
131
+ if ref:
132
+ if not PERIOD_REF.match(ref):
133
+ raise BindingMetadataRefusal("period_invalid", ref)
134
+ if not out.get("frozen"):
135
+ out["ref"] = ref
136
+ if (raw.get("window") is not None and "unit" not in out and "start" not in out
137
+ and not out.get("frozen")):
138
+ out["window"] = raw["window"]
139
+ if not ({"start", "window", "ref", "frozen"} & set(out)):
140
+ return None
141
+ return out
142
+
143
+
144
+ def clean_on_drag(raw):
145
+ """Return canonical two-axis drag metadata, or ``None`` when absent."""
146
+ if not isinstance(raw, dict):
147
+ return None
148
+ across = str(raw.get("across") or "").strip().lower()
149
+ if across not in DRAG_ACROSS:
150
+ across = "period"
151
+ down = raw.get("down")
152
+ if isinstance(down, dict) and str(down.get("field") or "").strip():
153
+ out_down = {"field": str(down["field"]).strip()}
154
+ if down.get("mode") in ("value", "presence", "member"):
155
+ out_down["mode"] = down["mode"]
156
+ else:
157
+ out_down = DRAG_DOWN_NONE
158
+ if isinstance(raw.get("steps"), list):
159
+ steps = []
160
+ for step in raw["steps"]:
161
+ if not isinstance(step, dict):
162
+ continue
163
+ field = step.get("field")
164
+ if not isinstance(field, str) or not field.strip() or step.get("axis") not in ("across", "down"):
165
+ continue
166
+ if "size" in step and step["size"] not in ("day", "week", "month", "quarter", "year"):
167
+ continue
168
+ cleaned = {"field": field.strip(), "axis": step["axis"]}
169
+ if step.get("mode") in ("value", "presence", "member"):
170
+ cleaned["mode"] = step["mode"]
171
+ if "size" in step:
172
+ cleaned["size"] = step["size"]
173
+ steps.append(cleaned)
174
+ first_down = next((step for step in steps if step["axis"] == "down"), None)
175
+ out_down = ({key: first_down[key] for key in ("field", "mode") if key in first_down}
176
+ if first_down else DRAG_DOWN_NONE)
177
+ across = ("period" if any(step["axis"] == "across" and step["field"]
178
+ in ("__period", "__ending") for step in steps) else "none")
179
+ return {"across": across, "down": out_down, "steps": steps}
180
+ return {"across": across, "down": out_down}
platform/core/sheet_import.py CHANGED
@@ -71,6 +71,13 @@ PALETTE = {
71
  "--lp-ink": "#202433",
72
  "--lp-muted": "#6b7280",
73
  "--lp-line": "#e6e9ef",
 
 
 
 
 
 
 
74
  "--lp-wash": "#fafbfc",
75
  "--lp-blue": "#9dbff2",
76
  "--lp-blue-tint": "#edf3fd",
@@ -89,6 +96,23 @@ PALETTE = {
89
  "--lp-purple": "#c3b3ee",
90
  "--lp-purple-tint": "#f1eefb",
91
  "--lp-purple-deep": "#6b57a8",
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
92
  "--lp-brand": "#6e56cf",
93
  "--lp-primary": "#5433cc",
94
  "--lp-primary-hover": "#472bad",
 
71
  "--lp-ink": "#202433",
72
  "--lp-muted": "#6b7280",
73
  "--lp-line": "#e6e9ef",
74
+ # ⭐ ADDED AT THE WAVE-62 COMBINE. W62-T31 / D-684 declares `--lp-line-strong` in
75
+ # `index.css` as a darker step than `--lp-line` for stronger separators, and
76
+ # `verify_sheet.py` C4's other direction went red the moment lane H's branch merged:
77
+ # a colour `:root` declares that the importer cannot quantize onto is a colour an
78
+ # imported cell can never keep. `--lp-line` was already here, so a line colour belongs
79
+ # in this table by the same rule; the size tokens the head note excludes are unaffected.
80
+ "--lp-line-strong": "#cbd1dc",
81
  "--lp-wash": "#fafbfc",
82
  "--lp-blue": "#9dbff2",
83
  "--lp-blue-tint": "#edf3fd",
 
96
  "--lp-purple": "#c3b3ee",
97
  "--lp-purple-tint": "#f1eefb",
98
  "--lp-purple-deep": "#6b57a8",
99
+ # ⭐ W61-T08 / THE COMBINE — THE THIRD SEAM. W61-T08 gives chart series eight distinct brand
100
+ # hues and declares each in the two seams its `done-when` names, `index.css` `:root` and
101
+ # `customer-grid/theme.ts`. This copy is a THIRD, and nothing pointed lane A at it: it lives in
102
+ # `platform/core`, which may not read the web tree at all, so only `verify_sheet.py` section 4
103
+ # relates the two — and it is an API gate, not one of that ticket's. It caught the drift on the
104
+ # merged tree exactly as designed ("a colour ADDED to :root cannot sit unreachable to an
105
+ # import"), which is why these are ADDED here rather than the leg being narrowed: an .xlsx
106
+ # painted one of the new hues must quantize onto it, not onto the nearest older colour.
107
+ "--lp-orange": "#edbf91",
108
+ "--lp-orange-tint": "#fcf1e6",
109
+ "--lp-orange-deep": "#ba6c27",
110
+ "--lp-pink": "#e6afcc",
111
+ "--lp-pink-tint": "#f9edf4",
112
+ "--lp-pink-deep": "#a54c78",
113
+ "--lp-teal": "#9bcfce",
114
+ "--lp-teal-tint": "#eaf5f4",
115
+ "--lp-teal-deep": "#00938d",
116
  "--lp-brand": "#6e56cf",
117
  "--lp-primary": "#5433cc",
118
  "--lp-primary-hover": "#472bad",
platform/core/sheet_structure.py ADDED
@@ -0,0 +1,316 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Pure workbook structure edits using Sheet's A1 scanner and the Wave 61 C1 rules.
2
+
3
+ Addresses, hidden lines and row-height keys are 1-based; rectangles are 0-based.
4
+ Structural edits move absolute references too. The dollar marks only pin a copy/fill.
5
+ The caller owns document validation, persistence and recalculation.
6
+ """
7
+ from __future__ import annotations
8
+
9
+ import copy
10
+ import math
11
+ import re
12
+ from collections.abc import Callable
13
+
14
+ from core.sheets import SheetRefusal
15
+
16
+
17
+ REF_SOURCE = r"(?:(?:'([^']{1,64})'|([A-Za-z_][A-Za-z0-9_]{0,63}))!)?(\$?)([A-Za-z]{1,3})(\$?)([1-9][0-9]{0,6})"
18
+ _REF = re.compile(REF_SOURCE)
19
+ _BARE_SHEET = re.compile(r"[A-Za-z_][A-Za-z0-9_]{0,63}")
20
+ _PREVIOUS = re.compile(r"[A-Za-z0-9_$]")
21
+ _START = re.compile(r"[A-Za-z$']")
22
+ _FUNCTION = re.compile(r"\s*\(")
23
+ BINDING_REF_KEYS = frozenset({"ref", "valueRef", "cellRef", "endingRef"})
24
+
25
+
26
+ def col_to_index(letters: str) -> int:
27
+ """Spreadsheet base-26: A -> 1, Z -> 26, AA -> 27."""
28
+ index = 0
29
+ for letter in letters.upper():
30
+ index = index * 26 + ord(letter) - 64
31
+ return index
32
+
33
+
34
+ def index_to_col(index: int) -> str:
35
+ """The inverse of col_to_index; an index below 1 yields an empty string."""
36
+ index = math.floor(index)
37
+ letters = []
38
+ while index > 0:
39
+ index, remainder = divmod(index - 1, 26)
40
+ letters.append(chr(65 + remainder))
41
+ return "".join(reversed(letters))
42
+
43
+
44
+ def parse_ref(text: str) -> dict | None:
45
+ """Parse one A1 reference, retaining the client's CellRef dictionary keys."""
46
+ match = _REF.fullmatch(str(text or "").strip())
47
+ if not match:
48
+ return None
49
+ sheet_quoted, sheet_bare, col_abs, col, row_abs, row = match.groups()
50
+ return {"sheet": sheet_quoted or sheet_bare or "", "colAbs": col_abs == "$",
51
+ "col": col.upper(), "rowAbs": row_abs == "$", "row": int(row)}
52
+
53
+
54
+ def format_ref(ref: dict | str) -> str:
55
+ """Format a CellRef, preserving dollar marks and quoting non-bare sheet names."""
56
+ if isinstance(ref, str):
57
+ return ref
58
+ body = (f"{'$' if ref['colAbs'] else ''}{ref['col']}"
59
+ f"{'$' if ref['rowAbs'] else ''}{ref['row']}")
60
+ sheet = ref["sheet"]
61
+ if not sheet:
62
+ return body
63
+ name = sheet if _BARE_SHEET.fullmatch(sheet) else "'" + sheet.replace("'", "") + "'"
64
+ return name + "!" + body
65
+
66
+
67
+ def rewrite_refs(text: str, rewrite: Callable[[dict], dict | str],
68
+ rewrite_range: Callable[[dict, dict, str], str] | None = None) -> str:
69
+ """Walk text exactly as a1.ts::rewriteRefs; strings and function names survive."""
70
+ src = str(text or "")
71
+ out = []
72
+ i = 0
73
+ quote = None
74
+ while i < len(src):
75
+ char = src[i]
76
+ if quote:
77
+ out.append(char)
78
+ if char == quote:
79
+ quote = None
80
+ i += 1
81
+ continue
82
+ if char in ('"', "'"):
83
+ close = src.find(char, i + 1)
84
+ is_sheet = char == "'" and close > i and src[close + 1:close + 2] == "!"
85
+ if not is_sheet:
86
+ quote = char
87
+ out.append(char)
88
+ i += 1
89
+ continue
90
+ previous = src[i - 1] if i else ""
91
+ if not _PREVIOUS.search(previous) and _START.fullmatch(char):
92
+ match = _REF.match(src, i)
93
+ if match and not _FUNCTION.match(src, match.end()):
94
+ ref = parse_ref(match[0])
95
+ if ref:
96
+ separator = re.match(r"\s*:\s*", src[match.end():]) if rewrite_range else None
97
+ if separator:
98
+ end = _REF.match(src, match.end() + separator.end())
99
+ if end and not _FUNCTION.match(src, end.end()):
100
+ last = parse_ref(end[0])
101
+ if last:
102
+ out.append(rewrite_range(ref, last, separator[0]))
103
+ i = end.end()
104
+ continue
105
+ out.append(format_ref(rewrite(ref)))
106
+ i = match.end()
107
+ continue
108
+ out.append(char)
109
+ i += 1
110
+ return "".join(out)
111
+
112
+
113
+ def shift_index(value: int, at: int, delta: int) -> int | None:
114
+ """R49 retires the clamp: both grammars already parse #REF! as an error token.
115
+
116
+ The feared bad_formula cannot occur. Deleted indices return None; only a range
117
+ with surviving cells may clamp its deleted endpoint and shrink.
118
+ """
119
+ return _moved_index(value, at, delta)
120
+
121
+
122
+ def _same_name(left: str, right: str) -> bool:
123
+ return str(left or "").strip().lower() == str(right or "").strip().lower()
124
+
125
+
126
+ def shift_for_structure(text: str, shift: dict, own_sheet: str, *, bare: bool = False) -> str:
127
+ """Re-anchor formula text or, with bare=True, a single reference field.
128
+
129
+ shift names the edited sheet and contains axis (row/col), at and delta.
130
+ A bare reference belongs to own_sheet. Invalid bare fields are left verbatim.
131
+ """
132
+ if not shift["delta"] or (bare and parse_ref(text) is None):
133
+ return str(text or "")
134
+
135
+ def index(ref):
136
+ return ref["row"] if shift["axis"] == "row" else col_to_index(ref["col"])
137
+
138
+ def at_index(ref, value):
139
+ return {**ref, shift["axis"]: value if shift["axis"] == "row" else index_to_col(value)}
140
+
141
+ def rewrite(ref):
142
+ if not _same_name(ref["sheet"] or own_sheet, shift["sheet"]):
143
+ return ref
144
+ moved = shift_index(index(ref), shift["at"], shift["delta"])
145
+ return "#REF!" if moved is None else at_index(ref, moved)
146
+
147
+ def rewrite_range(first, last, separator):
148
+ sheet = first["sheet"] or own_sheet
149
+ if last["sheet"] and not _same_name(sheet, last["sheet"]):
150
+ return format_ref(rewrite(first)) + separator + format_ref(rewrite(last))
151
+ # A bare right endpoint inherits the left endpoint's sheet, not the formula's tab.
152
+ if not _same_name(sheet, shift["sheet"]):
153
+ return format_ref(first) + separator + format_ref(last)
154
+ start, end = index(first), index(last)
155
+ low, high = min(start, end), max(start, end)
156
+ if low >= shift["at"] and high < shift["at"] - shift["delta"]:
157
+ return "#REF!"
158
+
159
+ def move(value):
160
+ moved = shift_index(value, shift["at"], shift["delta"])
161
+ return moved if moved is not None else shift["at"] if value == low else shift["at"] - 1
162
+
163
+ return format_ref(at_index(first, move(start))) + separator + format_ref(at_index(last, move(end)))
164
+
165
+ # Insertions keep the existing per-reference scan; range handling is deletion-only.
166
+ return rewrite_refs(text, rewrite, rewrite_range if shift["delta"] < 0 else None)
167
+
168
+
169
+ def _moved_index(value: int, at: int, delta: int) -> int | None:
170
+ if value < at:
171
+ return value
172
+ if delta > 0:
173
+ return value + delta
174
+ return None if value < at - delta else value + delta
175
+
176
+
177
+ def _reanchor_binding(value, shift: dict, own_sheet: str) -> bool:
178
+ """Rewrite only parseable ref-bearing fields in the copied binding's dict/list tree."""
179
+ changed = False
180
+ if isinstance(value, list):
181
+ for child in value:
182
+ changed = _reanchor_binding(child, shift, own_sheet) or changed
183
+ elif isinstance(value, dict):
184
+ for key, child in value.items():
185
+ if key in BINDING_REF_KEYS and isinstance(child, str) and parse_ref(child):
186
+ value[key] = shift_for_structure(child, shift, own_sheet, bare=True)
187
+ changed = value[key] != child or changed
188
+ elif isinstance(child, (dict, list)):
189
+ changed = _reanchor_binding(child, shift, own_sheet) or changed
190
+ return changed
191
+
192
+
193
+ def _reanchor_cells(tab: dict, shift: dict) -> None:
194
+ for cell in tab.get("cells", {}).values():
195
+ changed = False
196
+ if cell.get("kind") == "formula" and isinstance(cell.get("formula"), str):
197
+ previous = cell["formula"]
198
+ cell["formula"] = shift_for_structure(previous, shift, tab["name"])
199
+ changed = cell["formula"] != previous
200
+ if changed and cell.get("display") == previous:
201
+ cell["display"] = cell["formula"]
202
+ if cell.get("kind") == "bound":
203
+ changed = _reanchor_binding(cell.get("binding"), shift, tab["name"]) or changed
204
+ link = cell.get("link")
205
+ if isinstance(link, dict) and link.get("kind") in ("cell", "sheet"):
206
+ href = link.get("href")
207
+ if isinstance(href, str) and parse_ref(href):
208
+ link["href"] = shift_for_structure(href, shift, tab["name"], bare=True)
209
+ changed = link["href"] != href or changed
210
+ if changed and cell.get("kind") in ("formula", "bound"):
211
+ cell["state"] = "stale"
212
+
213
+
214
+ def _move_range(rect: dict, op: dict, *, drop_deleted: bool = False) -> dict | None:
215
+ """Keep surviving rectangle cells; collapse a fully deleted chart at the gap."""
216
+ start, end = op["axis"] + "0", op["axis"] + "1"
217
+ lo, hi = rect[start], rect[end]
218
+ point, delta = op["at"] - 1, op["delta"]
219
+ if delta > 0:
220
+ lo += delta if point <= lo else 0
221
+ hi += delta if point <= hi else 0
222
+ elif delta < 0:
223
+ past = point - delta
224
+ before, after = lo < point, hi >= past
225
+ if not before and not after and drop_deleted:
226
+ return None
227
+ lo = lo if before else max(lo, past) + delta if after else point
228
+ hi = hi + delta if after else min(hi, point - 1) if before else point
229
+ return {**rect, start: lo, end: hi}
230
+
231
+
232
+ def _move_tab(tab: dict, op: dict) -> None:
233
+ axis, at, delta = op["axis"], op["at"], op["delta"]
234
+ cells = {}
235
+ for address, cell in tab.get("cells", {}).items():
236
+ ref = parse_ref(address)
237
+ if not ref:
238
+ cells[address] = cell
239
+ continue
240
+ moved = _moved_index(ref["row"] if axis == "row" else col_to_index(ref["col"]), at, delta)
241
+ if moved is not None:
242
+ ref[axis] = moved if axis == "row" else index_to_col(moved)
243
+ cells[format_ref(ref)] = cell
244
+ tab["cells"] = cells
245
+ dimension = "rowHeights" if axis == "row" else "colWidths"
246
+ if dimension in tab:
247
+ sizes = {}
248
+ for key, size in tab[dimension].items():
249
+ moved = _moved_index(int(key) if axis == "row" else col_to_index(key), at, delta)
250
+ if moved is not None:
251
+ sizes[str(moved) if axis == "row" else index_to_col(moved)] = size
252
+ tab[dimension] = sizes
253
+ hidden = "hiddenRows" if axis == "row" else "hiddenCols"
254
+ if hidden in tab:
255
+ tab[hidden] = [moved for value in tab[hidden]
256
+ if (moved := _moved_index(value, at, delta)) is not None]
257
+ frozen = tab.get("freeze", {})
258
+ key = "rows" if axis == "row" else "cols"
259
+ if key in frozen and at <= frozen[key]:
260
+ # Only deleted frozen lines shrink the count, even when the band extends beyond it.
261
+ frozen[key] = max(0, frozen[key] + (delta if delta > 0 else -min(-delta, frozen[key] - at + 1)))
262
+ if "rules" in tab:
263
+ rules = []
264
+ for rule in tab["rules"]:
265
+ rect = _move_range(rule["range"], op, drop_deleted=True)
266
+ if rect is not None:
267
+ rules.append({**rule, "range": rect})
268
+ tab["rules"] = rules
269
+
270
+
271
+ def _move_charts(tab: dict, tab_id: str, op: dict) -> None:
272
+ for chart in tab.get("charts", []):
273
+ source = chart.get("source") or {}
274
+ if source.get("sheetId", tab["id"]) == tab_id:
275
+ chart["range"] = _move_range(chart["range"], op)
276
+ if "ranges" in source:
277
+ source["ranges"] = [_move_range(rect, op) for rect in source["ranges"]]
278
+ if tab["id"] == tab_id:
279
+ # Stored geometry anchors are zero-based too; pixel offsets and sizes stay intact.
280
+ for key in ("geometry", "anchor"):
281
+ geometry = chart.get(key)
282
+ if isinstance(geometry, dict) and op["axis"] in geometry:
283
+ moved = shift_index(geometry[op["axis"]] + 1, op["at"], op["delta"])
284
+ geometry[op["axis"]] = (op["at"] if moved is None else moved) - 1
285
+
286
+
287
+ def apply_structure(book: dict, tab_id: str, op: dict) -> dict:
288
+ """Return a new document after an insert/delete on tab_id; never access persistence.
289
+
290
+ op requires axis (row/col), a positive integer at and an integer delta. Zero delta
291
+ returns an independent unchanged copy. An optional sheet name must match tab_id.
292
+ Sheet dimensions are viewport extents, retained as in the client applyStructure.
293
+ """
294
+ if not isinstance(op, dict) or op.get("axis") not in ("row", "col"):
295
+ raise SheetRefusal("sheet_structure_invalid", "A structure change needs a row or column axis.")
296
+ if type(op.get("at")) is not int or op["at"] < 1:
297
+ raise SheetRefusal("sheet_structure_invalid", "A structure change needs a positive whole starting index.")
298
+ if type(op.get("delta")) is not int:
299
+ raise SheetRefusal("sheet_structure_invalid", "A structure change needs a whole insert or delete count.")
300
+ if not isinstance(book, dict) or not isinstance(book.get("sheets"), list):
301
+ raise SheetRefusal("sheet_structure_invalid", "A structure change needs a workbook with sheets.")
302
+ target = next((tab for tab in book["sheets"] if tab.get("id") == tab_id), None)
303
+ if target is None:
304
+ raise SheetRefusal("sheet_structure_tab_missing", "The sheet to insert or delete on does not exist.")
305
+ if "sheet" in op and not _same_name(op["sheet"], target["name"]):
306
+ raise SheetRefusal("sheet_structure_invalid", "The structure change names a different sheet.")
307
+ result = copy.deepcopy(book)
308
+ if not op["delta"]:
309
+ return result
310
+ shift = {**op, "sheet": target["name"]}
311
+ for tab in result["sheets"]:
312
+ _reanchor_cells(tab, shift)
313
+ _move_charts(tab, tab_id, op)
314
+ if tab["id"] == tab_id:
315
+ _move_tab(tab, op)
316
+ return result
platform/core/sheet_templates.py CHANGED
@@ -1,4 +1,4 @@
1
- """Loopable's prebuilt connector workbooks (W59-T29, contract C4).
2
 
3
  Descriptors are data; applying them uses the existing workbook persistence engine.
4
  The API owns permission checks and calculation under the calling user's row scope.
@@ -6,7 +6,7 @@ The API owns permission checks and calculation under the calling user's row scop
6
  from __future__ import annotations
7
 
8
  from copy import deepcopy
9
- from datetime import date, timedelta
10
  import re
11
 
12
  from core import sheets, user_tables
@@ -154,6 +154,103 @@ def templates_for_sources(sources):
154
  return [row for row in TEMPLATES if row["source"] in allowed]
155
 
156
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
157
  class TemplateRefusal(Exception):
158
  def __init__(self, code, *, missing=(), detail=""):
159
  self.code = code
@@ -205,8 +302,15 @@ def build_cells(template, resolved, database_key):
205
 
206
  for entry in template["bindings"]:
207
  cells, addr = target(entry)
 
 
 
 
 
 
 
208
  binding = {"database": database_key,
209
- "aggregate": {"fn": entry["fn"], "field": resolved[entry["field"]]},
210
  "scope": _resolved_scope(entry.get("scope", {"filter": []}), resolved)}
211
  if "period" in entry:
212
  binding["period"] = deepcopy(entry["period"])
@@ -233,12 +337,14 @@ def apply_template(st, key, database_key, *, name="", created_by=""):
233
  raise TemplateRefusal("unknown_database", detail="That database does not exist.")
234
  if source_of_database(database_key) != descriptor["source"]:
235
  raise TemplateRefusal("template_source_mismatch", detail="That database uses a different connector from this template.")
236
- resolved = resolve_fields(descriptor, (field["key"] for field in definition.get("fields", [])))
 
237
  prepared = build_cells(descriptor, resolved, database_key)
238
  # Validate tab names, extents and every cell before the first store write.
239
  checked = sheets.migrate_workbook({"sheets": [
240
  {"name": tab_name, "cells": cells} for _, tab_name, cells in prepared]})
241
- book = sheets.create_workbook(st, name=name or descriptor["title"], created_by=created_by)
 
242
  try:
243
  tabs = [{**tab, "cells": {}} for tab in checked["sheets"]]
244
  book = sheets.put_workbook(st, book["id"], {"name": book["name"], "sheets": tabs, "rev": book["rev"]})
 
1
+ """Loopable's built-in and tenant-saved connector workbooks.
2
 
3
  Descriptors are data; applying them uses the existing workbook persistence engine.
4
  The API owns permission checks and calculation under the calling user's row scope.
 
6
  from __future__ import annotations
7
 
8
  from copy import deepcopy
9
+ from datetime import date, datetime, timedelta, timezone
10
  import re
11
 
12
  from core import sheets, user_tables
 
154
  return [row for row in TEMPLATES if row["source"] in allowed]
155
 
156
 
157
+ SAVED_TEMPLATES_KEY = "sheet_templates_saved"
158
+
159
+
160
+ def saved_templates(st):
161
+ """Pure, detached C4 document reader; absent storage is an empty schema-1 document."""
162
+ return deepcopy(st.get(SAVED_TEMPLATES_KEY) or {"schema": 1, "templates": {}})
163
+
164
+
165
+ def saved_template(st, key):
166
+ """One detached saved record, or None. Built-in descriptors remain separate."""
167
+ return saved_templates(st)["templates"].get(key)
168
+
169
+
170
+ def _template_workbook(book):
171
+ clean = deepcopy(book)
172
+ for field in ("id", "rev", "writes", "writesFrom"):
173
+ clean.pop(field, None)
174
+ for tab in clean.get("sheets", []):
175
+ tab.pop("writes", None)
176
+ tab.pop("writesFrom", None)
177
+ return clean
178
+
179
+
180
+ def save_sheet_template(st, workbook, key, *, name="", description="", created_by="", actor_name=""):
181
+ """Save a workbook already admitted by the caller's reader wall, using C4's tenant document."""
182
+ if not isinstance(key, str) or not re.fullmatch(r"[A-Za-z0-9_-]+", key):
183
+ raise sheets.SheetRefusal("sheet_template_key_invalid",
184
+ "A Sheet template key must use letters, numbers, underscores or hyphens.")
185
+ if template(key) is not None:
186
+ raise sheets.SheetRefusal("sheet_template_key_exists",
187
+ f"Sheet template {key} already names a built-in template.")
188
+ if saved_template(st, key) is not None:
189
+ raise sheets.SheetRefusal("sheet_template_key_exists",
190
+ f"Sheet template {key} already exists. Choose another key.")
191
+ if not isinstance(name, str) or not isinstance(description, str):
192
+ raise sheets.SheetRefusal("sheet_template_text_invalid",
193
+ f"Sheet template {key} needs a text name and description.")
194
+ checked = sheets.migrate_workbook(workbook)
195
+ book = _template_workbook(checked)
196
+ databases = set()
197
+ for tab in book["sheets"]:
198
+ for addr, cell in tab["cells"].items():
199
+ if cell.get("kind") != "bound":
200
+ continue
201
+ database = (cell.get("binding") or {}).get("database")
202
+ if not isinstance(database, str) or not database.strip():
203
+ raise sheets.SheetRefusal("sheet_template_database_missing",
204
+ f"Sheet template {key} has no Cell database at {addr}.")
205
+ databases.add(database)
206
+ record = {"key": key, "name": name.strip() or book.get("name") or key,
207
+ "description": description, "databases": sorted(databases), "workbook": book,
208
+ "created_by": created_by, "created_at": datetime.now(timezone.utc).isoformat(),
209
+ "byName": actor_name or created_by}
210
+
211
+ def save(state):
212
+ document = deepcopy(state or {"schema": 1, "templates": {}})
213
+ if document.get("schema") != 1 or not isinstance(document.get("templates"), dict):
214
+ raise sheets.SheetRefusal("sheet_templates_schema_unknown",
215
+ "Saved Sheet templates use a document version this app cannot save.")
216
+ if key in document["templates"]:
217
+ raise sheets.SheetRefusal("sheet_template_key_exists",
218
+ f"Sheet template {key} already exists. Choose another key.")
219
+ document["templates"][key] = deepcopy(record)
220
+ return document
221
+
222
+ st.update(SAVED_TEMPLATES_KEY, save, flush="sync")
223
+ return deepcopy(record)
224
+
225
+
226
+ def apply_saved_template(st, key, *, name="", created_by=""):
227
+ """Materialise a saved workbook through the same create/put/patch engine as built-ins."""
228
+ record = saved_template(st, key)
229
+ if record is None:
230
+ # Keys from a URL can contain user text; refusal prose follows standing rule 2.
231
+ label = str(key).replace(chr(0x2014), "-").replace(chr(0x2013), "-")
232
+ raise sheets.SheetRefusal("unknown_sheet_template", f"Sheet template {label} does not exist.")
233
+ # Validate the whole document before the first write, including tabs after the first.
234
+ checked = sheets.migrate_workbook(_template_workbook(record["workbook"]))
235
+ book = sheets.create_workbook(st, name=name or record["name"], created_by=created_by,
236
+ on_duplicate="suffix")
237
+ try:
238
+ tabs = [{**tab, "cells": {}} for tab in checked["sheets"]]
239
+ book = sheets.put_workbook(st, book["id"], {
240
+ **checked, "name": book["name"], "sheets": tabs, "rev": book["rev"]}, actor=created_by)
241
+ for index, tab in enumerate(checked["sheets"]):
242
+ result = sheets.patch_cells(st, book["id"], book["sheets"][index]["id"],
243
+ tab["cells"], actor=created_by)
244
+ if result.get("refused"):
245
+ addresses = [str(item["addr"]) for item in result["refused"]]
246
+ raise sheets.SheetRefusal("template_cells_refused",
247
+ "The template could not apply cells: " + ", ".join(addresses) + ".")
248
+ return sheets.get_workbook(st, book["id"])
249
+ except Exception:
250
+ sheets.delete_workbook(st, book["id"])
251
+ raise
252
+
253
+
254
  class TemplateRefusal(Exception):
255
  def __init__(self, code, *, missing=(), detail=""):
256
  self.code = code
 
302
 
303
  for entry in template["bindings"]:
304
  cells, addr = target(entry)
305
+ if set(entry) - {"sheet", "addr", "fn", "field", "metric_key", "scope", "period"}:
306
+ raise TemplateRefusal("template_invalid", detail="The template binding has an unknown setting.")
307
+ aggregate = ({"metric_key": entry["metric_key"]} if "metric_key" in entry else
308
+ {"fn": entry["fn"], "field": resolved[entry["field"]]})
309
+ if "metric_key" in entry and (not entry["metric_key"] or "field" in entry or "fn" in entry
310
+ or not entry.get("period")):
311
+ raise TemplateRefusal("template_invalid", detail="A template metric needs a period and no field aggregate.")
312
  binding = {"database": database_key,
313
+ "aggregate": aggregate,
314
  "scope": _resolved_scope(entry.get("scope", {"filter": []}), resolved)}
315
  if "period" in entry:
316
  binding["period"] = deepcopy(entry["period"])
 
337
  raise TemplateRefusal("unknown_database", detail="That database does not exist.")
338
  if source_of_database(database_key) != descriptor["source"]:
339
  raise TemplateRefusal("template_source_mismatch", detail="That database uses a different connector from this template.")
340
+ field_keys = [field["key"] for field in definition.get("fields", [])]
341
+ resolved = resolve_fields(descriptor, field_keys)
342
  prepared = build_cells(descriptor, resolved, database_key)
343
  # Validate tab names, extents and every cell before the first store write.
344
  checked = sheets.migrate_workbook({"sheets": [
345
  {"name": tab_name, "cells": cells} for _, tab_name, cells in prepared]})
346
+ book = sheets.create_workbook(st, name=name or descriptor["title"], created_by=created_by,
347
+ on_duplicate="suffix")
348
  try:
349
  tabs = [{**tab, "cells": {}} for tab in checked["sheets"]]
350
  book = sheets.put_workbook(st, book["id"], {"name": book["name"], "sheets": tabs, "rev": book["rev"]})
platform/core/sheets.py CHANGED
@@ -63,6 +63,12 @@ import re
63
  import uuid
64
  from datetime import datetime, timezone
65
 
 
 
 
 
 
 
66
  #: C1's document key. `<id>` is the workbook id.
67
  WORKBOOK_KEY_PREFIX = "sheet_workbook_"
68
 
@@ -81,6 +87,7 @@ AMAZON_DATA_ORIGIN_NAME = "Amazon Services API"
81
  SHEET_SCHEMA_VERSION = 1
82
  DOCUMENT_MAX_COLS = 16384
83
  DOCUMENT_MAX_ROWS = 1048576
 
84
 
85
  #: ⛔ THE FROZEN VOCABULARY (contract C1, amended by C2). `verify_api.py` and `verify_sheet.py`
86
  #: assert `web/src/sheet/types.ts` carries exactly these, name for name.
@@ -92,6 +99,11 @@ DOCUMENT_MAX_ROWS = 1048576
92
  CELL_KINDS = ("literal", "formula", "bound", "empty")
93
  CELL_STATES = ("fresh", "stale", "typed", "refused")
94
 
 
 
 
 
 
95
  #: A literal is ALWAYS this state (R19). Stated once, here, so no caller has to remember it.
96
  #: ⚠ AN `empty` CELL TAKES THE SAME STATE, deliberately: it holds no binding, so `stale` would say
97
  #: a refresh is owed for a value that does not exist, and a reader counting stale cells would
@@ -192,7 +204,9 @@ SHEET_RULE_OPS = ("gt", "lt", "gte", "lte", "eq", "neq", "between", "contains",
192
  #: ⭐ `cell` IS STORED AS AN ABSENCE, exactly as `wrap: false` and `indent: 0` are: it is what
193
  #: every rule written before this ticket already is, so writing the word onto them would make a
194
  #: reload compare unequal to the save. Mirrored in `web/src/sheet/types.ts::SHEET_RULE_KINDS`.
195
- SHEET_RULE_KINDS = ("cell", "scale", "bar")
 
 
196
 
197
  #: The style keys a conditional format rule may repaint. A rule is a PAINT, never a format: it
198
  #: cannot change how a number reads, so it carries no `format`.
@@ -212,7 +226,11 @@ _RULE_STYLE_COLOURS = ("fill", "color")
212
  #: but not stored here, is the defect, and it is what widening this tuple prevents.
213
  SHEET_CHART_KINDS = ("bar", "line", "area", "donut", "column", "pie", "scatter", "combo",
214
  "waterfall", "histogram", "bubble", "bullet", "funnel", "gauge",
215
- "polar", "radar", "treemap", "stock", "gantt")
 
 
 
 
216
 
217
  #: ⛔⛔ D-640's MECHANISM, STATED. THIS IS EVERY KEY A CLEANED TAB MAY CARRY, and a key that is
218
  #: not named here is DELETED on the next save, in silence, with nothing going red: `_clean_sheet`
@@ -230,7 +248,7 @@ SHEET_CHART_KINDS = ("bar", "line", "area", "donut", "column", "pie", "scatter",
230
  #: The order is the order a cleaned tab reads in: identity, then A2's geometry, then C2's strata.
231
  SHEET_TAB_KEYS = ("id", "name", "cols", "rows", "cells", "usedExtent", "colWidths", "freeze",
232
  "rules", "hiddenRows", "hiddenCols", "charts", "imported", "rowHeight", "rowHeights",
233
- "visibility", "print", "excel")
234
 
235
  #: The keys above that `_clean_sheet` writes UNCONDITIONALLY. Every other member of
236
  #: `SHEET_TAB_KEYS` is written only when it holds something, for `_clean_style`'s reason: a tab
@@ -254,6 +272,7 @@ _MAX_DECIMALS = 10
254
  #: constants, which is why nothing could grow a sheet or remember a dragged column: there was
255
  #: nowhere to put the answer. These are the DEFAULTS a sheet is created with and the ceiling it may
256
  #: grow to, and `web/src/sheet/types.ts` mirrors the three defaults.
 
257
  #:
258
  #: ⚠ THE CEILING IS A RENDERING BUDGET, NOT A GRAMMAR LIMIT. `_ADDR` below already admits three
259
  #: column letters and seven row digits, and `a1.ts` agrees with it; 702 is the last TWO letter
@@ -266,7 +285,7 @@ MIN_SHEET_COLS = 1
266
  MIN_SHEET_ROWS = 1
267
  MAX_SHEET_COLS = 702
268
  MAX_SHEET_ROWS = 100000
269
- MIN_COL_WIDTH = 40
270
  MAX_COL_WIDTH = 1000
271
 
272
  #: A column key in `colWidths`. Letters only: a WIDTH belongs to a column, and a row number there
@@ -683,6 +702,54 @@ def _clean_sparkline(raw):
683
  return {"kind": kind, "range": rect}
684
 
685
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
686
  def clean_cell(raw, addr=""):
687
  """Normalize ONE cell, or raise `SheetRefusal` with a named code.
688
 
@@ -728,7 +795,17 @@ def clean_cell(raw, addr=""):
728
  if meta and meta.get("kind", "normal") not in ("normal", "array", "shared", "dataTable"):
729
  raise SheetRefusal("cell_formula_kind_unknown", "That formula representation is not supported.")
730
  binding = raw.get("binding")
731
- if isinstance(binding, dict):
 
 
 
 
 
 
 
 
 
 
732
  for tag in ("kind", "mode"):
733
  if tag in binding and binding[tag] not in ("number", "column"):
734
  raise SheetRefusal("cell_binding_kind_unknown", "A cell binding must choose Number or Column.")
@@ -778,7 +855,7 @@ def clean_cell(raw, addr=""):
778
  cell["value"] = _json_safe(raw.get("value"), "cell value")
779
 
780
  for key in ("binding", "source", "refusal"):
781
- blob = raw.get(key)
782
  if blob is None:
783
  continue
784
  if not isinstance(blob, dict):
@@ -881,12 +958,19 @@ def _clean_col_widths(raw, cols, *, lossless=False):
881
  "Column " + (letters or str(key)) + " is not a column letter.")
882
  if isinstance(value, bool) or not isinstance(value, (int, float)):
883
  if lossless:
884
- raise SheetRefusal("col_width_invalid", "A column width must be a finite positive size.")
885
  continue
886
  if not math.isfinite(value):
887
  raise SheetRefusal("col_width_invalid", "A column width must be a finite size.")
888
- if lossless and value <= 0:
889
- raise SheetRefusal("col_width_invalid", "A column width must be a finite positive size.")
 
 
 
 
 
 
 
890
  width = value if lossless else max(MIN_COL_WIDTH, min(MAX_COL_WIDTH, int(value)))
891
  if lossless or width != DEFAULT_COL_WIDTH:
892
  out[letters] = width
@@ -1045,6 +1129,18 @@ def _clean_rules(raw, *, lossless=False):
1045
  raise SheetRefusal(
1046
  "sheet_rule_kind_unknown",
1047
  "A sheet rule kind must be one of " + ", ".join(SHEET_RULE_KINDS) + ".")
 
 
 
 
 
 
 
 
 
 
 
 
1048
  rect = _clean_range(row.get("range"))
1049
  if rect is None:
1050
  continue
@@ -1113,6 +1209,12 @@ def _clean_charts(raw):
1113
  rect = _clean_range(row.get("range"))
1114
  if kind not in SHEET_CHART_KINDS:
1115
  raise SheetRefusal("sheet_chart_kind_unknown", "That chart kind is not supported.")
 
 
 
 
 
 
1116
  if rect is None:
1117
  raise SheetRefusal("sheet_chart_range_invalid", "A chart needs a valid source range.")
1118
  # Settings and source/series bags are authored data. Validate the discriminants we
@@ -1120,6 +1222,11 @@ def _clean_charts(raw):
1120
  chart = _document_object(row, "sheet_chart")
1121
  chart.update({"id": _text(row.get("id"), _MAX_ID) or _migration_id("chart", index, row),
1122
  "range": rect, "kind": kind})
 
 
 
 
 
1123
  for key in ("geometry", "anchor"):
1124
  if chart.get(key) is not None:
1125
  box = _document_object(chart[key], "sheet_chart_" + key)
@@ -1174,15 +1281,18 @@ def _clean_base_row_height(raw):
1174
  return raw
1175
 
1176
 
1177
- def _clean_row_heights(raw):
1178
  if raw is None:
1179
  return {}
1180
  out = _document_object(raw, "sheet_row_heights")
1181
  for row, height in out.items():
1182
  if not re.fullmatch(r"[1-9][0-9]*", str(row)) or int(row) > DOCUMENT_MAX_ROWS \
1183
  or isinstance(height, bool) or not isinstance(height, (int, float)) \
1184
- or not math.isfinite(height) or height <= 0:
1185
  raise SheetRefusal("sheet_row_height_invalid", "A row height needs a row number and a positive size.")
 
 
 
1186
  return out
1187
 
1188
 
@@ -1197,6 +1307,38 @@ def _clean_print(raw):
1197
  return out
1198
 
1199
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1200
  def _clean_imported(raw):
1201
  """C4's import record: when, from what, and WHAT DID NOT SURVIVE.
1202
 
@@ -1267,8 +1409,9 @@ def _clean_sheet(raw, index=0, *, lossless=False):
1267
  "charts": _clean_charts(raw.get("charts")),
1268
  "imported": _clean_imported(raw.get("imported")),
1269
  "rowHeight": _clean_base_row_height(raw.get("rowHeight")),
1270
- "rowHeights": _clean_row_heights(raw.get("rowHeights")),
1271
  "visibility": visibility,
 
1272
  "print": _clean_print(raw.get("print")),
1273
  "excel": excel,
1274
  }
@@ -1287,7 +1430,7 @@ def _clean_sheet(raw, index=0, *, lossless=False):
1287
  if value:
1288
  sheet[key] = value
1289
  extensions = {key: copy.deepcopy(value) for key, value in raw.items()
1290
- if key not in set(optional) | set(SHEET_TAB_ALWAYS_KEYS) | {"cellCount"}}
1291
  sheet.update(_json_safe(extensions, "sheet extensions"))
1292
  if lossless:
1293
  sheet["cols"] = max(cols, (sheet.get("freeze") or {}).get("cols", 0))
@@ -1308,7 +1451,7 @@ def _clean_sheets(raw, *, lossless=False):
1308
 
1309
  # ── the workbook ─────────────────────────────────────────────────────────────────────────────
1310
 
1311
- def migrate_workbook(raw):
1312
  """Pure unversioned/v0 -> v1 migration; validate current documents without recalculating.
1313
 
1314
  A read never writes or changes `rev`, timestamps or cached results. Unknown versions
@@ -1332,7 +1475,7 @@ def migrate_workbook(raw):
1332
  if book.get("print") is not None:
1333
  book["print"] = _clean_print(book["print"])
1334
  book["schemaVersion"] = SHEET_SCHEMA_VERSION
1335
- book["sheets"] = _clean_sheets(raw.get("sheets"), lossless=True)
1336
  return _json_safe(book, "workbook")
1337
 
1338
 
@@ -1346,10 +1489,11 @@ def new_workbook(name="", created_by="", workbook_id=""):
1346
  "name": _text(name, _MAX_NAME) or "Untitled workbook",
1347
  "sheets": [{"id": uuid.uuid4().hex[:12], "name": "Sheet1",
1348
  "cols": DEFAULT_SHEET_COLS, "rows": DEFAULT_SHEET_ROWS, "cells": {},
1349
- "usedExtent": None}],
1350
  "rev": 1,
1351
  "refreshed_at": None,
1352
  "created_by": _text(created_by, _MAX_ID),
 
1353
  "created_at": stamp,
1354
  "updated_at": stamp,
1355
  }
@@ -1416,8 +1560,45 @@ def _merge_data_origins(doc, *, sheets=(), databases=()):
1416
  return origins
1417
 
1418
 
1419
- def create_workbook(st, name="", created_by=""):
1420
- """Write a new `sheet_workbook_<id>` document and register it in the index."""
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1421
  book = new_workbook(name=name, created_by=created_by)
1422
  _json_safe(book, "workbook")
1423
 
@@ -1429,6 +1610,30 @@ def create_workbook(st, name="", created_by=""):
1429
  return book
1430
 
1431
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1432
  def get_workbook(st, workbook_id):
1433
  """One workbook, or a named refusal when the document is not there."""
1434
  wid = _slug_id(workbook_id)
@@ -1436,13 +1641,13 @@ def get_workbook(st, workbook_id):
1436
  if not doc or not doc.get("id"):
1437
  raise SheetRefusal("workbook_missing",
1438
  "That workbook no longer exists.")
1439
- return migrate_workbook(doc)
1440
 
1441
 
1442
- def put_workbook(st, workbook_id, raw, expect_rev=None):
1443
  """Replace a workbook's name and sheets, bumping `rev`.
1444
 
1445
- ⛔ `id`, `created_by` and `created_at` are PRIOR-WINS: a client may not reassign a workbook by
1446
  sending a different id, and a client that never read the document cannot blank its provenance.
1447
 
1448
  ⚠ `expect_rev` makes the write CONDITIONAL, and when it is not passed explicitly the `rev`
@@ -1461,7 +1666,8 @@ def put_workbook(st, workbook_id, raw, expect_rev=None):
1461
  if expect_rev is False:
1462
  expect_rev = None
1463
 
1464
- incoming = migrate_workbook(raw)
 
1465
  sheets = incoming["sheets"]
1466
  name = _text(raw.get("name"), _MAX_NAME)
1467
  refreshed_at = raw.get("refreshed_at")
@@ -1472,9 +1678,10 @@ def put_workbook(st, workbook_id, raw, expect_rev=None):
1472
  if prior is None:
1473
  raise SheetRefusal("workbook_missing", "That workbook no longer exists.")
1474
  if expect_rev is not None and int(prior.get("rev") or 0) != int(expect_rev):
1475
- raise SheetRefusal(
1476
  "workbook_revision_stale",
1477
- "That workbook changed somewhere else, so this save was not applied.")
 
1478
  book = migrate_workbook(prior)
1479
  # Immutable ownership and server read projections cannot be reassigned by a PUT.
1480
  # ``dataOrigins`` is SERVER-MAINTAINED (DPP 1.8) and therefore protected: a PUT body may
@@ -1483,16 +1690,24 @@ def put_workbook(st, workbook_id, raw, expect_rev=None):
1483
  # ``[]`` erases a genuine Amazon origin, and sending a well-formed tag plants one on an
1484
  # unrelated workbook. The origin list is rebuilt from the PRIOR document plus the cleaned
1485
  # cells by ``_merge_data_origins`` immediately below.
1486
- protected = {"id", "created_by", "created_at", "updated_at", "rev", "refreshed_at",
1487
  "sheets", "name", "refreshedFor", "shares", "role", "permissions",
1488
  "dataOrigins"}
1489
  book.update({key: value for key, value in incoming.items() if key not in protected})
1490
  book["name"] = name or _text(prior.get("name"), _MAX_NAME) or "Untitled workbook"
1491
- book["sheets"] = sheets
 
1492
  # DPP 1.8: merge from the PRIOR document and the cleaned cells. ``raw.dataOrigins`` is
1493
  # ignored, so a client cannot erase or replace the server-maintained attribution list.
1494
  _merge_data_origins(book, sheets=sheets)
1495
  book["rev"] = int(prior.get("rev") or 0) + 1
 
 
 
 
 
 
 
1496
  if refreshed_at is not None:
1497
  book["refreshed_at"] = _text(refreshed_at, 120) or None
1498
  book["updated_at"] = _now()
@@ -1574,7 +1789,7 @@ def workbook_meta(book):
1574
  return meta
1575
 
1576
 
1577
- def patch_cells(st, workbook_id, tab_ref, raw_cells, expect_rev=None):
1578
  """Merge a RANGE of cells into ONE tab. Returns `{rev, tab, refused}` (contract C3).
1579
 
1580
  ⭐⭐ ONE BAD CELL DOES NOT THROW THE COMMIT AWAY. Every address is cleaned on its own and a
@@ -1640,13 +1855,7 @@ def patch_cells(st, workbook_id, tab_ref, raw_cells, expect_rev=None):
1640
  if prior is None:
1641
  raise SheetRefusal("workbook_missing", "That workbook no longer exists.")
1642
  current = int(prior.get("rev") or 0)
1643
- if expect_rev is not None and current != int(expect_rev):
1644
- raise SheetStaleRefusal(
1645
- "sheet_revision_stale",
1646
- "This sheet changed somewhere else while you had it open, so this save was not "
1647
- "applied. Reload the tab and try again.",
1648
- rev_current=current)
1649
- book = migrate_workbook(prior)
1650
  tabs = book.get("sheets") or []
1651
  at = _tab_index(tabs, tab_ref)
1652
  if at is None:
@@ -1654,6 +1863,21 @@ def patch_cells(st, workbook_id, tab_ref, raw_cells, expect_rev=None):
1654
  "tab_missing",
1655
  "Tab " + (str(tab_ref) or "(unnamed)") + " is not in this workbook.")
1656
  tab = tabs[at]
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1657
  cells = dict(tab.get("cells") or {})
1658
  for key in deleted:
1659
  cells.pop(key, None)
@@ -1670,6 +1894,7 @@ def patch_cells(st, workbook_id, tab_ref, raw_cells, expect_rev=None):
1670
  book["sheets"] = tabs
1671
  _merge_data_origins(book, sheets=tabs)
1672
  book["rev"] = current + 1
 
1673
  book["updated_at"] = _now()
1674
  _json_safe(book, "workbook")
1675
  out["rev"] = book["rev"]
@@ -1677,6 +1902,7 @@ def patch_cells(st, workbook_id, tab_ref, raw_cells, expect_rev=None):
1677
  out["cells"] = len(cells)
1678
  out["schemaVersion"] = SHEET_SCHEMA_VERSION
1679
  out["usedExtent"] = tab["usedExtent"]
 
1680
  # Server-cleaned keys only. The route consumes this delta for calculation context.
1681
  out["accepted"] = list(dict.fromkeys([*merged, *deleted]))
1682
  saved.update(book)
@@ -1832,6 +2058,7 @@ def _index_row(book):
1832
  "id": book.get("id") or "",
1833
  "name": book.get("name") or "",
1834
  "created_by": book.get("created_by") or "",
 
1835
  "created_at": book.get("created_at") or "",
1836
  "updated_at": book.get("updated_at") or "",
1837
  "rev": int(book.get("rev") or 0),
@@ -1883,6 +2110,21 @@ def _index_upsert(st, book):
1883
  #: registry's words, so a second spelling cannot be introduced here by accident.
1884
 
1885
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1886
  def workbook_shares(st, workbook_id):
1887
  """C5's `shares` list for ONE workbook, computed from `core.shares` and never stored.
1888
 
@@ -1901,24 +2143,112 @@ def workbook_shares(st, workbook_id):
1901
  rec = shares.grants("workbook", str(workbook_id), st=st)
1902
  except Exception:
1903
  return []
1904
- return [{"who": e["user"], "role": e["role"]} for e in (rec.get("entries") or [])]
 
1905
 
 
 
1906
 
1907
- def project_shares(st, rows):
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1908
  """Join `shares` onto a list of index rows. Returns NEW rows; the stored ones are untouched.
1909
 
1910
- ⚠ Called by `read_index`, `save_folders` and `routes_sheet.list_sheets` — the three doors a
1911
- client reads the rail through. `list_workbooks` itself stays a bare pointer list on purpose:
 
1912
  `iter_workbooks` walks it to answer W49-T10's dependents question, and paying a grant-registry
1913
  read per workbook there would put a permission lookup inside a scan that asks no permission
1914
  question at all.
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1915
  """
1916
  out = []
 
 
 
 
 
 
 
 
1917
  for row in rows or ():
1918
  if not isinstance(row, dict) or not row.get("id"):
1919
  continue
1920
  joined = dict(row)
1921
- joined["shares"] = workbook_shares(st, row["id"])
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1922
  out.append(joined)
1923
  return out
1924
 
@@ -2374,7 +2704,31 @@ def _index_rev(doc):
2374
  return 0
2375
 
2376
 
2377
- def read_index(st, viewer=None):
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2378
  """The whole rail document: `{workbooks, folders, itemFolders, starred, rev}`, plus
2379
  `sharedWithMe` when a `viewer` is named.
2380
 
@@ -2388,8 +2742,8 @@ def read_index(st, viewer=None):
2388
  so a `rev` here without the check there (or the check without this) is a guard nothing can
2389
  reach ([[a-rev-guard-inert-on-the-events-channel]]). The pair lands together or not at all.
2390
 
2391
- ⚠ WHICH WRITES MOVE IT IS PART OF THE CONTRACT: `save_folders` and `_index_drop`, which are
2392
- the two that rewrite the folder, placement and star strata this guard protects.
2393
  `_index_upsert` deliberately does NOT, because it is called by `create_workbook`,
2394
  `put_workbook` AND `patch_cells` — bumping there would make every cell edit 409 a folder drag
2395
  that touched nothing the guard is about.
@@ -2399,6 +2753,20 @@ def read_index(st, viewer=None):
2399
  doc = _read(st, INDEX_KEY)
2400
  rows = [dict(r) for r in (doc.get("workbooks") or [])
2401
  if isinstance(r, dict) and r.get("id")]
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2402
  ids = {r["id"] for r in rows}
2403
  placements = doc.get("itemFolders")
2404
  try:
@@ -2407,37 +2775,45 @@ def read_index(st, viewer=None):
2407
  flat = [dict(f, parent=None) for f in (doc.get("folders") or [])
2408
  if isinstance(f, dict) and f.get("id")]
2409
  folders, placed, _repairs = validate_folder_tree(flat, placements)
 
 
 
2410
  out = {
2411
- "workbooks": project_shares(st, rows),
2412
  "folders": folders,
2413
  "itemFolders": {k: v for k, v in placed.items() if k in ids},
2414
- "starred": _clean_starred(doc.get("starred"), ids, []),
2415
  "rev": _index_rev(doc),
2416
  }
2417
  if viewer:
2418
- # C5's `sharedWithMe`. ⚠ FREE, and that is the point of using the registry: `shared_with`
2419
- # already answers "what was granted TO me" across kinds and already EXCLUDES what the
2420
- # caller owns, so the "Shared with me" partition of the rail needs no second store and no
2421
- # second definition of what being shared means.
2422
  try:
2423
  import core.shares as shares
2424
- granted = (shares.shared_with(viewer, kind="workbook", st=st) or {}).get("workbook")
 
 
 
 
 
2425
  except Exception:
2426
  granted = []
2427
  out["sharedWithMe"] = [w for w in (granted or []) if w in ids]
2428
  return out
2429
 
2430
 
2431
- def save_folders(st, folders, item_folders, starred=None, expect_rev=None):
 
2432
  """Replace the folder, placement and star strata WHOLESALE, once the tree is proven legal.
2433
 
2434
  Wholesale rather than per folder because the three are interdependent: a placement is only
2435
  legal while its folder exists, so committing them separately would leave a window where a
2436
  reader sees a workbook filed into a folder that is not there yet. One write, one state.
2437
 
2438
- ⛔ VALIDATION RUNS BEFORE THE UPDATE, NEVER INSIDE ITS CALLBACK. A raise from inside the
2439
- callback would abandon a half touched transaction; raising first means a refused tree leaves
2440
- the store exactly as it was. The callback below only PRUNES, and cannot raise.
2441
 
2442
  RETURNS `{workbooks, folders, itemFolders, starred, rev, repairs}`. RAISES
2443
  `SheetFolderRefusal` on a cycle or an over deep chain, having written nothing.
@@ -2455,9 +2831,16 @@ def save_folders(st, folders, item_folders, starred=None, expect_rev=None):
2455
  impossible.
2456
  """
2457
  tree, placed, repairs = validate_folder_tree(folders, item_folders)
 
 
 
2458
  out = {}
2459
 
2460
  def _apply(state):
 
 
 
 
2461
  if expect_rev is not None and _index_rev(state) != int(expect_rev):
2462
  raise SheetStaleRefusal(
2463
  "index_revision_stale",
@@ -2467,25 +2850,76 @@ def save_folders(st, folders, item_folders, starred=None, expect_rev=None):
2467
  rows = [r for r in (state.get("workbooks") or [])
2468
  if isinstance(r, dict) and r.get("id")]
2469
  ids = {r["id"] for r in rows}
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2470
  kept = {}
2471
- for item_id, fid in placed.items():
2472
  if item_id in ids:
2473
  kept[item_id] = fid
2474
  else:
2475
- repairs.append({"reason": "placement_missing_workbook", "item_id": item_id})
2476
- state["folders"] = tree
2477
  state["itemFolders"] = kept
2478
- if starred is not None:
2479
- state["starred"] = _clean_starred(starred, ids, repairs)
2480
  # D-639: this write IS the one the guard protects, so it moves the revision.
2481
  state["rev"] = _index_rev(state) + 1
2482
  out["value"] = {
2483
  "workbooks": [dict(r) for r in rows],
2484
- "folders": [dict(f) for f in tree],
2485
  "itemFolders": dict(kept),
2486
  "starred": list(state.get("starred") or []),
2487
  "rev": _index_rev(state),
2488
  }
 
2489
  return state, None
2490
 
2491
  _persist(st, INDEX_KEY, _apply, seed={"workbooks": []})
@@ -2495,5 +2929,54 @@ def save_folders(st, folders, item_folders, starred=None, expect_rev=None):
2495
  # store, and reading one document from inside another document's update callback puts a second
2496
  # store call under a lock this one already holds.
2497
  result["workbooks"] = project_shares(st, result.get("workbooks"))
2498
- result["repairs"] = repairs
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2499
  return result
 
63
  import uuid
64
  from datetime import datetime, timezone
65
 
66
+ from core.sheet_binding import (
67
+ BindingMetadataRefusal,
68
+ clean_on_drag as clean_binding_on_drag,
69
+ clean_period as clean_binding_period,
70
+ )
71
+
72
  #: C1's document key. `<id>` is the workbook id.
73
  WORKBOOK_KEY_PREFIX = "sheet_workbook_"
74
 
 
87
  SHEET_SCHEMA_VERSION = 1
88
  DOCUMENT_MAX_COLS = 16384
89
  DOCUMENT_MAX_ROWS = 1048576
90
+ WRITE_LOG_REVS = 200
91
 
92
  #: ⛔ THE FROZEN VOCABULARY (contract C1, amended by C2). `verify_api.py` and `verify_sheet.py`
93
  #: assert `web/src/sheet/types.ts` carries exactly these, name for name.
 
99
  CELL_KINDS = ("literal", "formula", "bound", "empty")
100
  CELL_STATES = ("fresh", "stale", "typed", "refused")
101
 
102
+ # W62 C2 / R31: "every view type can be placed" inside the binding.
103
+ VIEW_BINDING_TYPES = ("grid", "kanban", "map", "gallery", "timeline", "list", "calendar", "catalog", "swipe")
104
+ VIEW_BINDING_LIMIT_DEFAULT = 100
105
+ VIEW_BINDING_LIMIT_MAX = 10000
106
+
107
  #: A literal is ALWAYS this state (R19). Stated once, here, so no caller has to remember it.
108
  #: ⚠ AN `empty` CELL TAKES THE SAME STATE, deliberately: it holds no binding, so `stale` would say
109
  #: a refresh is owed for a value that does not exist, and a reader counting stale cells would
 
204
  #: ⭐ `cell` IS STORED AS AN ABSENCE, exactly as `wrap: false` and `indent: 0` are: it is what
205
  #: every rule written before this ticket already is, so writing the word onto them would make a
206
  #: reload compare unequal to the save. Mirrored in `web/src/sheet/types.ts::SHEET_RULE_KINDS`.
207
+ SHEET_RULE_KINDS = ("cell", "scale", "bar", "formula")
208
+ # The client compiler accepts 500 characters per condition.
209
+ MAX_RULE_FORMULA_LENGTH = 500
210
 
211
  #: The style keys a conditional format rule may repaint. A rule is a PAINT, never a format: it
212
  #: cannot change how a number reads, so it carries no `format`.
 
226
  #: but not stored here, is the defect, and it is what widening this tuple prevents.
227
  SHEET_CHART_KINDS = ("bar", "line", "area", "donut", "column", "pie", "scatter", "combo",
228
  "waterfall", "histogram", "bubble", "bullet", "funnel", "gauge",
229
+ "polar", "radar", "treemap", "stock", "gantt", "sunburst", "boxWhisker")
230
+
231
+ # C5 / R21: a spec-level sub-type, never a presentation setting or another chart kind.
232
+ SHEET_CHART_STACKING_KINDS = ("column", "bar", "line", "area")
233
+ SHEET_CHART_STACKING = ("clustered", "stacked", "percent")
234
 
235
  #: ⛔⛔ D-640's MECHANISM, STATED. THIS IS EVERY KEY A CLEANED TAB MAY CARRY, and a key that is
236
  #: not named here is DELETED on the next save, in silence, with nothing going red: `_clean_sheet`
 
248
  #: The order is the order a cleaned tab reads in: identity, then A2's geometry, then C2's strata.
249
  SHEET_TAB_KEYS = ("id", "name", "cols", "rows", "cells", "usedExtent", "colWidths", "freeze",
250
  "rules", "hiddenRows", "hiddenCols", "charts", "imported", "rowHeight", "rowHeights",
251
+ "visibility", "view", "print", "excel")
252
 
253
  #: The keys above that `_clean_sheet` writes UNCONDITIONALLY. Every other member of
254
  #: `SHEET_TAB_KEYS` is written only when it holds something, for `_clean_style`'s reason: a tab
 
272
  #: constants, which is why nothing could grow a sheet or remember a dragged column: there was
273
  #: nowhere to put the answer. These are the DEFAULTS a sheet is created with and the ceiling it may
274
  #: grow to, and `web/src/sheet/types.ts` mirrors the three defaults.
275
+ #: W62 C1 / R14 also mirrors the width bounds there; the client half lands in W62-T05.
276
  #:
277
  #: ⚠ THE CEILING IS A RENDERING BUDGET, NOT A GRAMMAR LIMIT. `_ADDR` below already admits three
278
  #: column letters and seven row digits, and `a1.ts` agrees with it; 702 is the last TWO letter
 
285
  MIN_SHEET_ROWS = 1
286
  MAX_SHEET_COLS = 702
287
  MAX_SHEET_ROWS = 100000
288
+ MIN_COL_WIDTH = 0
289
  MAX_COL_WIDTH = 1000
290
 
291
  #: A column key in `colWidths`. Letters only: a WIDTH belongs to a column, and a row number there
 
702
  return {"kind": kind, "range": rect}
703
 
704
 
705
+ def clean_view_binding(raw):
706
+ """W62-T10 / C2: normalize a PLACED VIEW binding, or raise `SheetRefusal` with a named code.
707
+
708
+ `{database, mode: "view", viewType, fields: [key], filter?, sort: [{field, dir}], period?,`
709
+ `limit (1..10000, default 100), onDrag?, block?: {cols, rows}}`.
710
+
711
+ ⛔ IT IS A SEPARATE VALIDATOR FROM `modules/sheet.py::clean_binding` RATHER THAN A WIDENING OF
712
+ IT. That one REQUIRES an `aggregate`, because a metric binding IS an aggregate; a view has
713
+ none. Widening it would make `aggregate` optional for every caller, and the day a metric
714
+ binding arrives without one it would resolve to nothing instead of refusing by name.
715
+ """
716
+ if not isinstance(raw, dict) or not isinstance(raw.get("database"), str) or not raw["database"].strip():
717
+ raise SheetRefusal("view_binding_invalid", "A view binding must be an object naming a database.")
718
+ if raw.get("mode", raw.get("kind")) != "view":
719
+ raise SheetRefusal("view_binding_mode_unknown", "A view binding must choose view mode.")
720
+ if raw.get("viewType") not in VIEW_BINDING_TYPES:
721
+ raise SheetRefusal("view_binding_type_unknown", "That view display type is not supported.")
722
+ fields = raw.get("fields")
723
+ if not isinstance(fields, list) or not fields or any(
724
+ not isinstance(key, str) or not key.strip() for key in fields):
725
+ raise SheetRefusal("view_binding_fields_invalid", "A view needs a non-empty list of field keys.")
726
+ sort = raw.get("sort", [])
727
+ if not isinstance(sort, list) or any(
728
+ not isinstance(item, dict) or not isinstance(item.get("field"), str)
729
+ or not item["field"].strip() or item.get("dir") not in ("asc", "desc") for item in sort):
730
+ raise SheetRefusal("view_binding_sort_invalid", "A view sort needs a field key and asc or desc direction.")
731
+ limit = raw.get("limit", VIEW_BINDING_LIMIT_DEFAULT)
732
+ if isinstance(limit, bool) or not isinstance(limit, int) or not 1 <= limit <= VIEW_BINDING_LIMIT_MAX:
733
+ raise SheetRefusal("view_binding_limit_invalid", "A view limit must be a whole number from 1 to 10000.")
734
+ out = {"database": raw["database"].strip(), "mode": "view", "viewType": raw["viewType"],
735
+ "fields": list(fields), "sort": [{"field": item["field"], "dir": item["dir"]} for item in sort],
736
+ "limit": limit}
737
+ for key in ("filter", "block"):
738
+ if key in raw:
739
+ out[key] = _document_object(raw[key], "view_binding_" + key)
740
+ # W62 C2: one lower-layer metadata cleaner serves both view and metric bindings.
741
+ if "period" in raw or "onDrag" in raw:
742
+ for key, cleaner in (("period", clean_binding_period), ("onDrag", clean_binding_on_drag)):
743
+ if key in raw:
744
+ try:
745
+ value = cleaner(_json_safe(copy.deepcopy(raw[key]), "view binding " + key))
746
+ except BindingMetadataRefusal as exc:
747
+ raise SheetRefusal(exc.code, exc.message) from exc
748
+ if value is not None:
749
+ out[key] = value
750
+ return out
751
+
752
+
753
  def clean_cell(raw, addr=""):
754
  """Normalize ONE cell, or raise `SheetRefusal` with a named code.
755
 
 
795
  if meta and meta.get("kind", "normal") not in ("normal", "array", "shared", "dataTable"):
796
  raise SheetRefusal("cell_formula_kind_unknown", "That formula representation is not supported.")
797
  binding = raw.get("binding")
798
+ # ⭐⭐ W62-T10 / C2 — A PLACED VIEW LIVES INSIDE `kind: "bound"`, AND `CELL_KINDS` STAYS
799
+ # FROZEN. Wave 49's cell-kind contract is the reason: the vocabulary is pinned here and two
800
+ # gates prove the client agrees with it, so a fifth kind would be a change to a frozen
801
+ # contract for a feature that does not need one. The discriminator is `binding.mode == "view"`,
802
+ # and a view binding carries no `aggregate` at all, which is why it cannot go through the
803
+ # metric validator below.
804
+ if isinstance(binding, dict) and (binding.get("mode") == "view" or binding.get("kind") == "view"):
805
+ if kind != "bound":
806
+ raise SheetRefusal("view_binding_invalid", "A view binding needs a bound cell.")
807
+ binding = clean_view_binding(binding)
808
+ elif isinstance(binding, dict):
809
  for tag in ("kind", "mode"):
810
  if tag in binding and binding[tag] not in ("number", "column"):
811
  raise SheetRefusal("cell_binding_kind_unknown", "A cell binding must choose Number or Column.")
 
855
  cell["value"] = _json_safe(raw.get("value"), "cell value")
856
 
857
  for key in ("binding", "source", "refusal"):
858
+ blob = binding if key == "binding" else raw.get(key)
859
  if blob is None:
860
  continue
861
  if not isinstance(blob, dict):
 
958
  "Column " + (letters or str(key)) + " is not a column letter.")
959
  if isinstance(value, bool) or not isinstance(value, (int, float)):
960
  if lossless:
961
+ raise SheetRefusal("col_width_invalid", "A column width must be a finite size.")
962
  continue
963
  if not math.isfinite(value):
964
  raise SheetRefusal("col_width_invalid", "A column width must be a finite size.")
965
+ # ⭐ W62-T04 / R14 — A ZERO IS NOW A WIDTH, AND A NEGATIVE ONE STILL IS NOT. Before this
966
+ # ticket the lossless door refused every `value <= 0`, which is the same test for both; R14
967
+ # made 0 Excel's own hidden column, so an xlsx carrying one must import rather than refuse.
968
+ # A negative width is still nothing a file or a person can mean, so it keeps its refusal on
969
+ # the lossless door and is clamped to the floor on a live save.
970
+ if lossless and value < 0:
971
+ raise SheetRefusal("col_width_invalid", "A column width cannot be negative.")
972
+ #: W62 C1 / R14 / R19: reads retain hidden zero widths and exact imported values;
973
+ #: only a live save applies the bounds a person can set.
974
  width = value if lossless else max(MIN_COL_WIDTH, min(MAX_COL_WIDTH, int(value)))
975
  if lossless or width != DEFAULT_COL_WIDTH:
976
  out[letters] = width
 
1129
  raise SheetRefusal(
1130
  "sheet_rule_kind_unknown",
1131
  "A sheet rule kind must be one of " + ", ".join(SHEET_RULE_KINDS) + ".")
1132
+ if kind == "formula":
1133
+ condition = row.get("value")
1134
+ if not isinstance(condition, str) or not condition.lstrip("= \t\r\n"):
1135
+ raise SheetRefusal("sheet_rule_formula_missing",
1136
+ "A formula rule needs a condition.")
1137
+ if len(condition) > MAX_RULE_FORMULA_LENGTH:
1138
+ raise SheetRefusal("sheet_rule_formula_too_long",
1139
+ "A formula rule condition can hold at most "
1140
+ + str(MAX_RULE_FORMULA_LENGTH) + " characters.")
1141
+ # A formula has one operand. Normalize even an imported empty/notEmpty op so
1142
+ # the existing value persistence path cannot silently drop its condition.
1143
+ op = "eq"
1144
  rect = _clean_range(row.get("range"))
1145
  if rect is None:
1146
  continue
 
1209
  rect = _clean_range(row.get("range"))
1210
  if kind not in SHEET_CHART_KINDS:
1211
  raise SheetRefusal("sheet_chart_kind_unknown", "That chart kind is not supported.")
1212
+ if "stacking" in row:
1213
+ if row["stacking"] not in SHEET_CHART_STACKING:
1214
+ raise SheetRefusal("sheet_chart_stacking_unknown", "That chart sub-type is not supported.")
1215
+ if kind not in SHEET_CHART_STACKING_KINDS: # Includes sunburst and boxWhisker (C5).
1216
+ raise SheetRefusal("sheet_chart_stacking_kind_invalid",
1217
+ "Chart sub-types need a column, bar, line or area chart.")
1218
  if rect is None:
1219
  raise SheetRefusal("sheet_chart_range_invalid", "A chart needs a valid source range.")
1220
  # Settings and source/series bags are authored data. Validate the discriminants we
 
1222
  chart = _document_object(row, "sheet_chart")
1223
  chart.update({"id": _text(row.get("id"), _MAX_ID) or _migration_id("chart", index, row),
1224
  "range": rect, "kind": kind})
1225
+ # Keep the C5 spec field explicitly across cleaning (D-640). Clustered is an absence,
1226
+ # including when an external writer sends the default by name.
1227
+ chart.pop("stacking", None)
1228
+ if row.get("stacking") in ("stacked", "percent"):
1229
+ chart["stacking"] = row["stacking"]
1230
  for key in ("geometry", "anchor"):
1231
  if chart.get(key) is not None:
1232
  box = _document_object(chart[key], "sheet_chart_" + key)
 
1281
  return raw
1282
 
1283
 
1284
+ def _clean_row_heights(raw, *, lossless=False):
1285
  if raw is None:
1286
  return {}
1287
  out = _document_object(raw, "sheet_row_heights")
1288
  for row, height in out.items():
1289
  if not re.fullmatch(r"[1-9][0-9]*", str(row)) or int(row) > DOCUMENT_MAX_ROWS \
1290
  or isinstance(height, bool) or not isinstance(height, (int, float)) \
1291
+ or not math.isfinite(height) or (lossless and height <= 0):
1292
  raise SheetRefusal("sheet_row_height_invalid", "A row height needs a row number and a positive size.")
1293
+ #: W62 C1 / R19: a live size is clamped, while imported positive sizes stay exact.
1294
+ if not lossless:
1295
+ out[row] = max(18, min(200, height))
1296
  return out
1297
 
1298
 
 
1307
  return out
1308
 
1309
 
1310
+ #: Wave 60 base kit: `SheetTabView` in `sheet/types.ts`. Display settings, so a bad value is
1311
+ #: DROPPED rather than refused: a sheet must never fail to save over a zoom level.
1312
+ _VIEW_ZOOM_MIN, _VIEW_ZOOM_MAX = 10, 400
1313
+ _VIEW_PAGE_VIEWS = ("normal", "pageBreak", "pageLayout")
1314
+
1315
+
1316
+ def _clean_view(raw):
1317
+ """A tab's display settings: gridlines, headings, zoom, pageView and split, nothing else.
1318
+
1319
+ Booleans stay booleans, zoom clamps to 10..400 (a non-number is dropped), pageView keeps only
1320
+ the three words, split keeps non-negative integer rows and cols or is dropped. An empty
1321
+ result is None, so the tab stores no empty stratum.
1322
+ """
1323
+ if not isinstance(raw, dict):
1324
+ return None
1325
+ out = {}
1326
+ for key in ("gridlines", "headings"):
1327
+ if isinstance(raw.get(key), bool):
1328
+ out[key] = raw[key]
1329
+ zoom = raw.get("zoom")
1330
+ if not isinstance(zoom, bool) and isinstance(zoom, (int, float)) and math.isfinite(zoom):
1331
+ out["zoom"] = max(_VIEW_ZOOM_MIN, min(_VIEW_ZOOM_MAX, zoom))
1332
+ if raw.get("pageView") in _VIEW_PAGE_VIEWS:
1333
+ out["pageView"] = raw["pageView"]
1334
+ split = raw.get("split")
1335
+ if isinstance(split, dict):
1336
+ parts = [split.get("rows"), split.get("cols")]
1337
+ if all(isinstance(n, int) and not isinstance(n, bool) and n >= 0 for n in parts):
1338
+ out["split"] = {"rows": parts[0], "cols": parts[1]}
1339
+ return out or None
1340
+
1341
+
1342
  def _clean_imported(raw):
1343
  """C4's import record: when, from what, and WHAT DID NOT SURVIVE.
1344
 
 
1409
  "charts": _clean_charts(raw.get("charts")),
1410
  "imported": _clean_imported(raw.get("imported")),
1411
  "rowHeight": _clean_base_row_height(raw.get("rowHeight")),
1412
+ "rowHeights": _clean_row_heights(raw.get("rowHeights"), lossless=lossless),
1413
  "visibility": visibility,
1414
+ "view": _clean_view(raw.get("view")),
1415
  "print": _clean_print(raw.get("print")),
1416
  "excel": excel,
1417
  }
 
1430
  if value:
1431
  sheet[key] = value
1432
  extensions = {key: copy.deepcopy(value) for key, value in raw.items()
1433
+ if key not in set(optional) | set(SHEET_TAB_ALWAYS_KEYS) | {"cellCount", "writes", "writesFrom"}}
1434
  sheet.update(_json_safe(extensions, "sheet extensions"))
1435
  if lossless:
1436
  sheet["cols"] = max(cols, (sheet.get("freeze") or {}).get("cols", 0))
 
1451
 
1452
  # ── the workbook ─────────────────────────────────────────────────────────────────────────────
1453
 
1454
+ def migrate_workbook(raw, *, lossless=True):
1455
  """Pure unversioned/v0 -> v1 migration; validate current documents without recalculating.
1456
 
1457
  A read never writes or changes `rev`, timestamps or cached results. Unknown versions
 
1475
  if book.get("print") is not None:
1476
  book["print"] = _clean_print(book["print"])
1477
  book["schemaVersion"] = SHEET_SCHEMA_VERSION
1478
+ book["sheets"] = _clean_sheets(raw.get("sheets"), lossless=lossless)
1479
  return _json_safe(book, "workbook")
1480
 
1481
 
 
1489
  "name": _text(name, _MAX_NAME) or "Untitled workbook",
1490
  "sheets": [{"id": uuid.uuid4().hex[:12], "name": "Sheet1",
1491
  "cols": DEFAULT_SHEET_COLS, "rows": DEFAULT_SHEET_ROWS, "cells": {},
1492
+ "usedExtent": None, "writesFrom": 0}],
1493
  "rev": 1,
1494
  "refreshed_at": None,
1495
  "created_by": _text(created_by, _MAX_ID),
1496
+ "access": "private",
1497
  "created_at": stamp,
1498
  "updated_at": stamp,
1499
  }
 
1560
  return origins
1561
 
1562
 
1563
+ def create_workbook(st, name="", created_by="", on_duplicate="allow"):
1564
+ """Write a new `sheet_workbook_<id>` document and register it in the index.
1565
+
1566
+ `on_duplicate` decides what a name already in this workspace means:
1567
+ `allow` (the default) writes it anyway, `refuse` raises `workbook_name_taken`, and `suffix`
1568
+ writes `Name (2)`, `Name (3)` instead.
1569
+
1570
+ ⛔ THE DUPLICATE-NAME RULE BELONGS TO THE DOORS, NOT TO THIS MECHANISM (owner ruling
1571
+ 2026-09-16, shaped by what the gate then measured). A workbook's identity is its id, so
1572
+ duplicates were always legal and nothing ever checked; the rail shows N identical rows with no
1573
+ date, size or owner and no reader can tell which is theirs. SDLC 61's C7 build left five
1574
+ workbooks named `Daily Sales Performance`, which is what made it visible.
1575
+ ⚠ **Where the rule lives matters.** A person names a workbook at exactly two doors, and both
1576
+ pass `refuse`: `POST /sheets` (Create new) and the gateway's `sheet_create_workbook`. Every
1577
+ other caller creates a workbook as a MECHANISM, not as a naming act: applying a template and
1578
+ importing a file both INHERIT a name, so they pass `suffix` and a template stays usable more
1579
+ than once. Putting `refuse` on this function instead made a template single use, made an
1580
+ import single use, and broke thirteen gate fixtures whose names are incidental. A rule that
1581
+ fires on a mechanism fires on everything that uses it.
1582
+ """
1583
+ wanted = str(name or "").strip()
1584
+ if wanted and on_duplicate != "allow":
1585
+ taken = {str((row or {}).get("name") or "").strip().casefold()
1586
+ for row in list_workbooks(st)}
1587
+ if wanted.casefold() in taken:
1588
+ if on_duplicate != "suffix":
1589
+ raise SheetRefusal(
1590
+ "workbook_name_taken",
1591
+ "A workbook named " + wanted + " is already in this workspace. "
1592
+ "Choose another name.")
1593
+ for nth in range(2, 1000):
1594
+ candidate = wanted + " (" + str(nth) + ")"
1595
+ if candidate.casefold() not in taken:
1596
+ name = candidate
1597
+ break
1598
+ else:
1599
+ raise SheetRefusal(
1600
+ "workbook_name_taken",
1601
+ "Too many workbooks are already named " + wanted + ".")
1602
  book = new_workbook(name=name, created_by=created_by)
1603
  _json_safe(book, "workbook")
1604
 
 
1610
  return book
1611
 
1612
 
1613
+ def _carry_write_logs(book, prior):
1614
+ """Only stored tabs may supply write history; client metadata never owns it."""
1615
+ held = {tab.get("id"): tab for tab in prior.get("sheets") or []}
1616
+ for tab in book.get("sheets") or []:
1617
+ source = held.get(tab.get("id")) or {}
1618
+ tab.pop("writes", None)
1619
+ tab["writesFrom"] = int(source.get("writesFrom") or 0)
1620
+ if "writes" in source:
1621
+ tab["writes"] = copy.deepcopy(source["writes"])
1622
+ return book
1623
+
1624
+
1625
+ def _append_write_log(tab, rev, addrs, *, actor="", actor_name=""):
1626
+ """Record accepted addresses and the oldest revision with complete retained history."""
1627
+ if not addrs:
1628
+ return
1629
+ writes = [*(tab.get("writes") or []), {"rev": rev, "by": actor,
1630
+ "byName": actor_name or actor, "addrs": list(addrs)}]
1631
+ dropped = writes[:-WRITE_LOG_REVS]
1632
+ if dropped:
1633
+ tab["writesFrom"] = max(int(tab.get("writesFrom") or 0), int(dropped[-1]["rev"]))
1634
+ tab["writes"] = writes[-WRITE_LOG_REVS:]
1635
+
1636
+
1637
  def get_workbook(st, workbook_id):
1638
  """One workbook, or a named refusal when the document is not there."""
1639
  wid = _slug_id(workbook_id)
 
1641
  if not doc or not doc.get("id"):
1642
  raise SheetRefusal("workbook_missing",
1643
  "That workbook no longer exists.")
1644
+ return _carry_write_logs(migrate_workbook(doc), doc)
1645
 
1646
 
1647
+ def put_workbook(st, workbook_id, raw, expect_rev=None, *, actor="", actor_name="", lossless=False):
1648
  """Replace a workbook's name and sheets, bumping `rev`.
1649
 
1650
+ ⛔ `id`, `created_by`, `access` and `created_at` are PRIOR-WINS: a client may not reassign a workbook by
1651
  sending a different id, and a client that never read the document cannot blank its provenance.
1652
 
1653
  ⚠ `expect_rev` makes the write CONDITIONAL, and when it is not passed explicitly the `rev`
 
1666
  if expect_rev is False:
1667
  expect_rev = None
1668
 
1669
+ #: W62 C1 / R19: app and AI saves share bounded defaults; import and refresh opt out.
1670
+ incoming = migrate_workbook(raw, lossless=lossless)
1671
  sheets = incoming["sheets"]
1672
  name = _text(raw.get("name"), _MAX_NAME)
1673
  refreshed_at = raw.get("refreshed_at")
 
1678
  if prior is None:
1679
  raise SheetRefusal("workbook_missing", "That workbook no longer exists.")
1680
  if expect_rev is not None and int(prior.get("rev") or 0) != int(expect_rev):
1681
+ raise SheetStaleRefusal(
1682
  "workbook_revision_stale",
1683
+ "That workbook changed somewhere else, so this save was not applied.",
1684
+ rev_current=int(prior.get("rev") or 0))
1685
  book = migrate_workbook(prior)
1686
  # Immutable ownership and server read projections cannot be reassigned by a PUT.
1687
  # ``dataOrigins`` is SERVER-MAINTAINED (DPP 1.8) and therefore protected: a PUT body may
 
1690
  # ``[]`` erases a genuine Amazon origin, and sending a well-formed tag plants one on an
1691
  # unrelated workbook. The origin list is rebuilt from the PRIOR document plus the cleaned
1692
  # cells by ``_merge_data_origins`` immediately below.
1693
+ protected = {"id", "created_by", "access", "created_at", "updated_at", "rev", "refreshed_at",
1694
  "sheets", "name", "refreshedFor", "shares", "role", "permissions",
1695
  "dataOrigins"}
1696
  book.update({key: value for key, value in incoming.items() if key not in protected})
1697
  book["name"] = name or _text(prior.get("name"), _MAX_NAME) or "Untitled workbook"
1698
+ book["sheets"] = copy.deepcopy(sheets)
1699
+ _carry_write_logs(book, prior)
1700
  # DPP 1.8: merge from the PRIOR document and the cleaned cells. ``raw.dataOrigins`` is
1701
  # ignored, so a client cannot erase or replace the server-maintained attribution list.
1702
  _merge_data_origins(book, sheets=sheets)
1703
  book["rev"] = int(prior.get("rev") or 0) + 1
1704
+ prior_tabs = {tab.get("id"): tab for tab in prior.get("sheets") or []}
1705
+ for tab in book["sheets"]:
1706
+ before = (prior_tabs.get(tab.get("id")) or {}).get("cells") or {}
1707
+ after = tab.get("cells") or {}
1708
+ changed = sorted(addr for addr in before.keys() | after.keys()
1709
+ if before.get(addr) != after.get(addr))
1710
+ _append_write_log(tab, book["rev"], changed, actor=actor, actor_name=actor_name)
1711
  if refreshed_at is not None:
1712
  book["refreshed_at"] = _text(refreshed_at, 120) or None
1713
  book["updated_at"] = _now()
 
1789
  return meta
1790
 
1791
 
1792
+ def patch_cells(st, workbook_id, tab_ref, raw_cells, expect_rev=None, *, actor="", actor_name=""):
1793
  """Merge a RANGE of cells into ONE tab. Returns `{rev, tab, refused}` (contract C3).
1794
 
1795
  ⭐⭐ ONE BAD CELL DOES NOT THROW THE COMMIT AWAY. Every address is cleaned on its own and a
 
1855
  if prior is None:
1856
  raise SheetRefusal("workbook_missing", "That workbook no longer exists.")
1857
  current = int(prior.get("rev") or 0)
1858
+ book = _carry_write_logs(migrate_workbook(prior), prior)
 
 
 
 
 
 
1859
  tabs = book.get("sheets") or []
1860
  at = _tab_index(tabs, tab_ref)
1861
  if at is None:
 
1863
  "tab_missing",
1864
  "Tab " + (str(tab_ref) or "(unnamed)") + " is not in this workbook.")
1865
  tab = tabs[at]
1866
+ writes = tab.get("writes") or []
1867
+ touched = list(dict.fromkeys([*merged, *deleted]))
1868
+ conflicts = {}
1869
+ if touched and expect_rev is not None and int(expect_rev) < current:
1870
+ revision = int(expect_rev)
1871
+ if revision < int(tab.get("writesFrom") or 0):
1872
+ raise SheetStaleRefusal(
1873
+ "sheet_revision_stale",
1874
+ "This tab's retained change history was truncated after your revision. Reload the tab and try again.",
1875
+ rev_current=current)
1876
+ for write in writes:
1877
+ if int(write["rev"]) > revision and write["by"] != actor:
1878
+ for addr in sorted(set(touched).intersection(write["addrs"])):
1879
+ conflicts[addr] = {"addr": addr, "by": write["by"],
1880
+ "byName": write.get("byName") or write["by"]}
1881
  cells = dict(tab.get("cells") or {})
1882
  for key in deleted:
1883
  cells.pop(key, None)
 
1894
  book["sheets"] = tabs
1895
  _merge_data_origins(book, sheets=tabs)
1896
  book["rev"] = current + 1
1897
+ _append_write_log(tab, book["rev"], touched, actor=actor, actor_name=actor_name)
1898
  book["updated_at"] = _now()
1899
  _json_safe(book, "workbook")
1900
  out["rev"] = book["rev"]
 
1902
  out["cells"] = len(cells)
1903
  out["schemaVersion"] = SHEET_SCHEMA_VERSION
1904
  out["usedExtent"] = tab["usedExtent"]
1905
+ out["conflicts"] = list(conflicts.values())
1906
  # Server-cleaned keys only. The route consumes this delta for calculation context.
1907
  out["accepted"] = list(dict.fromkeys([*merged, *deleted]))
1908
  saved.update(book)
 
2058
  "id": book.get("id") or "",
2059
  "name": book.get("name") or "",
2060
  "created_by": book.get("created_by") or "",
2061
+ "has_access": "access" in book,
2062
  "created_at": book.get("created_at") or "",
2063
  "updated_at": book.get("updated_at") or "",
2064
  "rev": int(book.get("rev") or 0),
 
2110
  #: registry's words, so a second spelling cannot be introduced here by accident.
2111
 
2112
 
2113
+ def narrow_workbook(st, workbook_id):
2114
+ """Persist explicit sharing's one way visibility decision on the latest document."""
2115
+ wid = _slug_id(workbook_id)
2116
+
2117
+ def _apply(state):
2118
+ if not state.get("id"):
2119
+ raise SheetRefusal("workbook_missing", "That workbook no longer exists.")
2120
+ state.setdefault("access", "private")
2121
+ return state, state
2122
+
2123
+ # R28 / W62-T25: explicit sharing changes the fact used by the index's visibility filter.
2124
+ book = _persist(st, workbook_key(wid), _apply)
2125
+ _index_upsert(st, book)
2126
+
2127
+
2128
  def workbook_shares(st, workbook_id):
2129
  """C5's `shares` list for ONE workbook, computed from `core.shares` and never stored.
2130
 
 
2143
  rec = shares.grants("workbook", str(workbook_id), st=st)
2144
  except Exception:
2145
  return []
2146
+ return shares_from_record(rec)
2147
+
2148
 
2149
+ def shares_from_record(rec):
2150
+ """C5's `shares` list from a grant record the caller already read. PURE.
2151
 
2152
+ W61-T17: `project_shares` reads the whole registry once and calls this per row, so the list
2153
+ a batched caller builds and the list `workbook_shares` builds are the same list by
2154
+ construction rather than by two copies of one comprehension.
2155
+ """
2156
+ return [{"who": e["user"], "role": e["role"]} for e in ((rec or {}).get("entries") or [])]
2157
+
2158
+
2159
+ def workbook_role(st, book, viewer, *, is_admin=False, rec=None):
2160
+ """Document ownership plus the existing grant reader, shared with the route's role helper.
2161
+
2162
+ ⭐ W61-T17 — `book` NEED ONLY ANSWER `id` AND `created_by`, and that is what lets the index
2163
+ stop reading whole workbook documents to build a list. An index row already carries both
2164
+ (`_index_row` rebuilds them off the document on every save), so the caller may pass the ROW.
2165
+ ⭐ `rec` is an already-read grant record, from `shares.grants_bulk`. Passing it skips a
2166
+ registry read; omitting it reads exactly as before. The role logic itself is
2167
+ `shares.role_from` either way, so the two paths cannot answer differently.
2168
+ """
2169
+ from core import shares
2170
+ me = str(viewer or "").strip().lower()
2171
+ if me and str(book.get("created_by") or "").strip().lower() == me:
2172
+ return "owner"
2173
+ if rec is not None:
2174
+ return shares.role_from(rec, me, is_admin=is_admin)
2175
+ return shares.role_for("workbook", book.get("id"), me, is_admin=is_admin, st=st)
2176
+
2177
+
2178
+ def may_read_workbook(book, role):
2179
+ """R28: a role admits a reader; unmarked or creatorless documents remain legacy-visible.
2180
+
2181
+ R28 / W62-T25: creation and explicit sharing stamp access. The index records its presence;
2182
+ workbook reads still use the document, and PUT must preserve marker absence too.
2183
+ Legacy visibility grants no ownership or editing role.
2184
+ """
2185
+ return bool(role) or "access" not in book or not book.get("created_by")
2186
+
2187
+
2188
+ def project_shares(st, rows, viewer=None, *, is_admin=False, role_on_workbook=None,
2189
+ grant_records=None):
2190
  """Join `shares` onto a list of index rows. Returns NEW rows; the stored ones are untouched.
2191
 
2192
+ W62-T25: a named viewer filters rows using document facts and receives their role.
2193
+ The route supplies its existing role helper; core callers use workbook_role directly.
2194
+ `list_workbooks` itself stays a bare pointer list on purpose:
2195
  `iter_workbooks` walks it to answer W49-T10's dependents question, and paying a grant-registry
2196
  read per workbook there would put a permission lookup inside a scan that asks no permission
2197
  question at all.
2198
+
2199
+ ⭐⭐ W61-T17 — IT NO LONGER READS EVERY WORKBOOK DOCUMENT TO BUILD A LIST, and that was the
2200
+ whole cost of opening the Sheet home. Measured on staging's Royal Imports with 50 workbooks:
2201
+ 12,485 ms and 269,946,170 characters read, to produce a 12 KB answer. Every one of those
2202
+ documents was pulled in full — all its tabs, all its cells — so that three fields could be
2203
+ consulted, and the list got slower with every cell anybody typed.
2204
+
2205
+ ⭐ THE ROW ALREADY ANSWERS TWO OF THE THREE. `_index_row` rebuilds `id` and `created_by` off
2206
+ the document on every save, and `workbook_role` reads nothing else; so does the route's
2207
+ injected helper. The third is `may_read_workbook`'s legacy-visibility clause, `"access" not in
2208
+ book`, and it is consulted ONLY when the viewer has no role at all AND the row names a
2209
+ creator - the rows that are otherwise about to be REFUSED. W62-T25 records that third fact
2210
+ as `has_access`; only rows written before that fact existed still read their document.
2211
+
2212
+ R28 / W62-T25: admission uses the document facts rebuilt on save and explicit sharing.
2213
+ `_index_drop` removes a deleted document's row. Older rows still check document existence
2214
+ when taking the fallback; opening any workbook always reads its document for real.
2215
  """
2216
  out = []
2217
+ bulk = grant_records
2218
+ if bulk is None and rows:
2219
+ try:
2220
+ from core import shares as _shares
2221
+ bulk = _shares.grants_bulk("workbook", st=st)
2222
+ except Exception:
2223
+ bulk = {}
2224
+ bulk = bulk or {}
2225
  for row in rows or ():
2226
  if not isinstance(row, dict) or not row.get("id"):
2227
  continue
2228
  joined = dict(row)
2229
+ # ⛔ NEVER None: an object with NO grants has no record in the bucket, and passing
2230
+ # None to `workbook_role` would read "caller supplied nothing" and send it back to
2231
+ # `shares.role_for` for a per-row read of the whole registry — which is one of the
2232
+ # two costs this change exists to remove. An empty record IS the answer.
2233
+ rec = bulk.get(str(row["id"])) or {"owner": None, "entries": []}
2234
+ if viewer:
2235
+ # The two fields the role question actually asks of a "book".
2236
+ light = {"id": row["id"], "created_by": row.get("created_by") or ""}
2237
+ role = (role_on_workbook(light) if role_on_workbook is not None
2238
+ else workbook_role(st, light, viewer, is_admin=is_admin, rec=rec))
2239
+ if not role and light["created_by"]:
2240
+ if "has_access" in row:
2241
+ if row["has_access"]:
2242
+ continue
2243
+ else:
2244
+ # R28 / W62-T25: old rows lack this fact, so retain their document check.
2245
+ # R28: this fallback set is expected to be empty after every workbook
2246
+ # has been saved once; guessing would change R28's legacy admission.
2247
+ book = _read(st, workbook_key(row["id"]))
2248
+ if not book or not book.get("id") or not may_read_workbook(book, role):
2249
+ continue
2250
+ joined["role"] = role or "view"
2251
+ joined["shares"] = shares_from_record(rec)
2252
  out.append(joined)
2253
  return out
2254
 
 
2704
  return 0
2705
 
2706
 
2707
+ def _visible_rail(folders, placed, starred, ids):
2708
+ """Prune hidden placements, stars and folders occupied only by hidden workbooks.
2709
+
2710
+ Keep visible folders' ancestors and intentionally empty folders. Otherwise creating an
2711
+ empty folder would immediately hide it from the person who needs to put a workbook in it.
2712
+ """
2713
+ parents = {f["id"]: f.get("parent") for f in folders}
2714
+
2715
+ def ancestors(folder_ids):
2716
+ found = set()
2717
+ for fid in folder_ids:
2718
+ while fid in parents and fid not in found:
2719
+ found.add(fid)
2720
+ fid = parents[fid]
2721
+ return found
2722
+
2723
+ visible = {wid: fid for wid, fid in placed.items() if wid in ids}
2724
+ occupied = ancestors(placed.values())
2725
+ needed = ancestors(set(visible.values()) | (parents.keys() - occupied))
2726
+ hidden_only = occupied - needed
2727
+ return ([f for f in folders if f["id"] not in hidden_only], visible,
2728
+ _clean_starred(starred, ids, []))
2729
+
2730
+
2731
+ def read_index(st, viewer=None, *, is_admin=False, role_on_workbook=None):
2732
  """The whole rail document: `{workbooks, folders, itemFolders, starred, rev}`, plus
2733
  `sharedWithMe` when a `viewer` is named.
2734
 
 
2742
  so a `rev` here without the check there (or the check without this) is a guard nothing can
2743
  reach ([[a-rev-guard-inert-on-the-events-channel]]). The pair lands together or not at all.
2744
 
2745
+ D-639 / W62-T25: `save_folders`, `merge_placements` and `_index_drop` move the revision
2746
+ because they write the folder, placement and star strata this guard protects.
2747
  `_index_upsert` deliberately does NOT, because it is called by `create_workbook`,
2748
  `put_workbook` AND `patch_cells` — bumping there would make every cell edit 409 a folder drag
2749
  that touched nothing the guard is about.
 
2753
  doc = _read(st, INDEX_KEY)
2754
  rows = [dict(r) for r in (doc.get("workbooks") or [])
2755
  if isinstance(r, dict) and r.get("id")]
2756
+ # W62 release repair: both the role join and Shared with me ask the SAME grant-registry
2757
+ # question. Read it once and hand the normalized records to both consumers; otherwise a
2758
+ # 50-row index pays one bulk read here, N more through an injected route helper, then one
2759
+ # final read through `shared_with`.
2760
+ grant_records = {}
2761
+ if rows:
2762
+ try:
2763
+ from core import shares as _shares
2764
+ grant_records = _shares.grants_bulk("workbook", st=st)
2765
+ except Exception:
2766
+ grant_records = {}
2767
+ rows = project_shares(st, rows, viewer, is_admin=is_admin,
2768
+ role_on_workbook=role_on_workbook,
2769
+ grant_records=grant_records)
2770
  ids = {r["id"] for r in rows}
2771
  placements = doc.get("itemFolders")
2772
  try:
 
2775
  flat = [dict(f, parent=None) for f in (doc.get("folders") or [])
2776
  if isinstance(f, dict) and f.get("id")]
2777
  folders, placed, _repairs = validate_folder_tree(flat, placements)
2778
+ starred = _clean_starred(doc.get("starred"), ids, [])
2779
+ if viewer:
2780
+ folders, placed, starred = _visible_rail(folders, placed, doc.get("starred"), ids)
2781
  out = {
2782
+ "workbooks": rows,
2783
  "folders": folders,
2784
  "itemFolders": {k: v for k, v in placed.items() if k in ids},
2785
+ "starred": starred,
2786
  "rev": _index_rev(doc),
2787
  }
2788
  if viewer:
2789
+ # C5's `sharedWithMe`, derived from the SAME normalized grant records as the role join.
2790
+ # Match `shares.shared_with`: exclude owned objects, admit personal/everyone grants, sort,
2791
+ # and finally intersect with the already-visible index ids. No second store read.
 
2792
  try:
2793
  import core.shares as shares
2794
+ me = str(viewer or "").strip().lower()
2795
+ granted = sorted(
2796
+ oid for oid, rec in grant_records.items()
2797
+ if str(rec.get("owner") or "").strip().lower() != me
2798
+ and shares.role_from(rec, me) is not None
2799
+ )
2800
  except Exception:
2801
  granted = []
2802
  out["sharedWithMe"] = [w for w in (granted or []) if w in ids]
2803
  return out
2804
 
2805
 
2806
+ def save_folders(st, folders, item_folders, starred=None, expect_rev=None, *,
2807
+ writable_ids=None, writable_folder_ids=None):
2808
  """Replace the folder, placement and star strata WHOLESALE, once the tree is proven legal.
2809
 
2810
  Wholesale rather than per folder because the three are interdependent: a placement is only
2811
  legal while its folder exists, so committing them separately would leave a window where a
2812
  reader sees a workbook filed into a folder that is not there yet. One write, one state.
2813
 
2814
+ ⛔ STRUCTURAL VALIDATION RUNS BEFORE THE UPDATE. A permission-scoped save also reconciles
2815
+ the submitted projection with hidden stored strata inside the callback, under the store lock;
2816
+ any refusal still abandons the callback atomically, so no half-touched rail can persist.
2817
 
2818
  RETURNS `{workbooks, folders, itemFolders, starred, rev, repairs}`. RAISES
2819
  `SheetFolderRefusal` on a cycle or an over deep chain, having written nothing.
 
2831
  impossible.
2832
  """
2833
  tree, placed, repairs = validate_folder_tree(folders, item_folders)
2834
+ writable = None if writable_ids is None else {str(wid) for wid in writable_ids}
2835
+ writable_folders = (None if writable_folder_ids is None
2836
+ else {str(fid) for fid in writable_folder_ids})
2837
  out = {}
2838
 
2839
  def _apply(state):
2840
+ tree_now = [dict(row) for row in tree]
2841
+ placed_now = dict(placed)
2842
+ repairs_now = list(repairs)
2843
+ starred_now = starred
2844
  if expect_rev is not None and _index_rev(state) != int(expect_rev):
2845
  raise SheetStaleRefusal(
2846
  "index_revision_stale",
 
2850
  rows = [r for r in (state.get("workbooks") or [])
2851
  if isinstance(r, dict) and r.get("id")]
2852
  ids = {r["id"] for r in rows}
2853
+ saved_tree, saved_placed, _saved_repairs = validate_folder_tree(
2854
+ state.get("folders"), state.get("itemFolders"))
2855
+
2856
+ if writable is not None:
2857
+ # An id absent from this caller's projection is indistinguishable from a missing one.
2858
+ # Refuse explicit guesses, then preserve every hidden placement/star against omission
2859
+ # from the caller's necessarily partial document.
2860
+ if any(item_id not in writable for item_id in placed_now):
2861
+ raise SheetRefusal("workbook_missing", "That workbook no longer exists.")
2862
+ incoming_stars = (_clean_starred(starred_now, None, [])
2863
+ if starred_now is not None else [])
2864
+ if any(item_id not in writable for item_id in incoming_stars):
2865
+ raise SheetRefusal("workbook_missing", "That workbook no longer exists.")
2866
+
2867
+ saved_by_id = {row["id"]: row for row in saved_tree}
2868
+ incoming_ids = {row["id"] for row in tree_now}
2869
+ if writable_folders is not None:
2870
+ hidden_folder_ids = set(saved_by_id) - writable_folders
2871
+ if incoming_ids & hidden_folder_ids:
2872
+ raise SheetRefusal("folder_missing", "That folder no longer exists.")
2873
+
2874
+ hidden_ids = ids - writable
2875
+ hidden_placed = {wid: fid for wid, fid in saved_placed.items()
2876
+ if wid in hidden_ids}
2877
+ parents = {row["id"]: row.get("parent") for row in saved_tree}
2878
+ required = set()
2879
+ for fid in hidden_placed.values():
2880
+ while fid in parents and fid not in required:
2881
+ required.add(fid)
2882
+ fid = parents[fid]
2883
+ # Missing hidden-only branches are restored from the stored tree. A shared folder the
2884
+ # caller edited keeps the submitted row; deleting one that still holds a hidden book
2885
+ # preserves the stored row only for that hidden stratum.
2886
+ combined_tree = list(tree_now)
2887
+ # Preserve the stored order of every restored hidden branch. Iterating the set
2888
+ # difference made an honest partial save reorder another user's hidden folders by
2889
+ # Python hash seed: the contents survived, but their rail order did not.
2890
+ combined_tree.extend(
2891
+ dict(row) for row in saved_tree
2892
+ if row["id"] in required and row["id"] not in incoming_ids)
2893
+ combined_placed = {**hidden_placed, **placed_now}
2894
+ tree_now, placed_now, combined_repairs = validate_folder_tree(
2895
+ combined_tree, combined_placed)
2896
+ repairs_now.extend(combined_repairs)
2897
+
2898
+ if starred_now is not None:
2899
+ hidden_stars = [wid for wid in _clean_starred(state.get("starred"), ids, [])
2900
+ if wid in hidden_ids]
2901
+ starred_now = hidden_stars + [wid for wid in incoming_stars if wid in writable]
2902
+
2903
  kept = {}
2904
+ for item_id, fid in placed_now.items():
2905
  if item_id in ids:
2906
  kept[item_id] = fid
2907
  else:
2908
+ repairs_now.append({"reason": "placement_missing_workbook", "item_id": item_id})
2909
+ state["folders"] = tree_now
2910
  state["itemFolders"] = kept
2911
+ if starred_now is not None:
2912
+ state["starred"] = _clean_starred(starred_now, ids, repairs_now)
2913
  # D-639: this write IS the one the guard protects, so it moves the revision.
2914
  state["rev"] = _index_rev(state) + 1
2915
  out["value"] = {
2916
  "workbooks": [dict(r) for r in rows],
2917
+ "folders": [dict(f) for f in tree_now],
2918
  "itemFolders": dict(kept),
2919
  "starred": list(state.get("starred") or []),
2920
  "rev": _index_rev(state),
2921
  }
2922
+ out["repairs"] = repairs_now
2923
  return state, None
2924
 
2925
  _persist(st, INDEX_KEY, _apply, seed={"workbooks": []})
 
2929
  # store, and reading one document from inside another document's update callback puts a second
2930
  # store call under a lock this one already holds.
2931
  result["workbooks"] = project_shares(st, result.get("workbooks"))
2932
+ result["repairs"] = list(out.get("repairs") or repairs)
2933
+ return result
2934
+
2935
+
2936
+ def merge_placements(st, placements, *, starred=None):
2937
+ """D-639 / W62-T25: apply only named moves to the current rail under the store's lock.
2938
+
2939
+ None unfiles a workbook. Disjoint moves need no expected revision; every accepted merge
2940
+ advances it so a stale wholesale save still refuses. Stars change only if explicitly supplied.
2941
+ """
2942
+ if not isinstance(placements, dict):
2943
+ raise SheetRefusal("placements_not_a_map", "Sheet placements must be a map of workbook ids.")
2944
+
2945
+ def _apply(state):
2946
+ rows = [r for r in (state.get("workbooks") or [])
2947
+ if isinstance(r, dict) and r.get("id")]
2948
+ ids = {r["id"] for r in rows}
2949
+ merged = dict(state.get("itemFolders") or {})
2950
+ for wid, fid in placements.items():
2951
+ if wid not in ids:
2952
+ raise SheetRefusal("placement_missing_workbook",
2953
+ 'The workbook "' + str(wid) + '" no longer exists in the Sheet rail.')
2954
+ if fid is None:
2955
+ merged.pop(wid, None)
2956
+ else:
2957
+ merged[wid] = fid
2958
+ # D-639: validate the merged tree inside the same update, before changing any stratum.
2959
+ # D-639: use wholesale repair names, but refuse a malformed delta instead of losing it.
2960
+ _tree, placed, repairs = validate_folder_tree(state.get("folders"), merged)
2961
+ if repairs:
2962
+ repair = repairs[0]
2963
+ raise SheetFolderRefusal(
2964
+ repair["reason"],
2965
+ 'The placement for workbook "' + str(repair.get("item_id") or "")
2966
+ + '" cannot be saved. Use an existing folder id or null to unfile it.',
2967
+ folder_id=str(repair.get("folder_id") or ""), reason=repair["reason"])
2968
+ for wid, fid in placements.items():
2969
+ if fid is not None:
2970
+ merged[wid] = placed[wid]
2971
+ state["itemFolders"] = merged
2972
+ if starred is not None:
2973
+ state["starred"] = _clean_starred(starred, ids, repairs)
2974
+ state["rev"] = _index_rev(state) + 1
2975
+ return state, {"workbooks": rows, "folders": state.get("folders") or [],
2976
+ "itemFolders": merged, "starred": state.get("starred") or [],
2977
+ "rev": _index_rev(state), "repairs": repairs}
2978
+
2979
+ result = _persist(st, INDEX_KEY, _apply, seed={"workbooks": []})
2980
+ # D-639: grant reads happen after the store releases its update lock, as in save_folders.
2981
+ result["workbooks"] = project_shares(st, result["workbooks"])
2982
  return result
platform/core/store_pg.py CHANGED
@@ -347,6 +347,66 @@ def get_projection(name, drop=(), tenant_slug=None):
347
  return _hf._project(data, frozenset(drops))
348
 
349
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
350
  def exists(name, tenant_slug=None):
351
  """True/False — and unlike the HF backend this is NEVER "True on uncertainty".
352
 
@@ -610,6 +670,15 @@ class PgStore:
610
  """
611
  return get_projection(name, drop=drop, tenant_slug=self.tenant_slug)
612
 
 
 
 
 
 
 
 
 
 
613
  def _read_strict(self, name):
614
  """Interface parity with the HF Store. There, `get` is lenient (swallows a transient
615
  failure) and `_read_strict` opts out so a failed read ABORTS a read-modify-write instead
 
347
  return _hf._project(data, frozenset(drops))
348
 
349
 
350
+ def _get_projections(names, drop=(), tenant_slug=None, timeout_s=None):
351
+ """Several projected documents in one round trip, keyed by their requested store key.
352
+
353
+ Private on purpose: the public module interface is the backend-parity surface, while this is
354
+ an optimization exposed only by a bound ``PgStore`` handle. Callers must retain a per-key
355
+ fallback for stores that do not offer it.
356
+ """
357
+ import core.store as _hf # noqa: PLC0415 — projection shape lives with Store
358
+
359
+ keys = list(dict.fromkeys(str(name) for name in (names or ()) if str(name)))
360
+ if not keys:
361
+ return {}
362
+ drops = tuple(str(d) for d in drop)
363
+ with _pool().connection() as con:
364
+ if timeout_s is not None:
365
+ timeout_ms = max(1, int(float(timeout_s) * 1000))
366
+ # Transaction-local: a pooled connection must not lend this caller's budget to the
367
+ # next request. PgStore connections are non-autocommit, so this setting covers the
368
+ # following projection and is discarded when the context commits or rolls back.
369
+ con.execute("SELECT set_config('statement_timeout', %s, true)",
370
+ (str(timeout_ms),))
371
+ rows = con.execute(
372
+ _q('SELECT kv.key, CASE '
373
+ "WHEN jsonb_typeof(kv.value) = 'object' THEN "
374
+ "COALESCE((SELECT jsonb_object_agg(e.entry_key, "
375
+ "CASE WHEN jsonb_typeof(e.entry_value) = 'object' "
376
+ 'THEN e.entry_value - %s::text[] ELSE e.entry_value END) '
377
+ 'FROM jsonb_each(kv.value) AS e(entry_key, entry_value)), '
378
+ "'{{}}'::jsonb) ELSE kv.value END FROM {tbl} AS kv "
379
+ 'WHERE kv.key = ANY(%s::text[])', 'store_kv', tenant_slug),
380
+ (list(drops), keys)).fetchall()
381
+ found = {str(key): value for key, value in rows}
382
+ dropped = frozenset(drops)
383
+ return {
384
+ key: _hf._project(
385
+ dict(found[key]) if isinstance(found.get(key), dict)
386
+ else (found[key] if key in found else {}),
387
+ dropped)
388
+ for key in keys
389
+ }
390
+
391
+
392
+ def _get_many(names, tenant_slug=None):
393
+ """Several ordinary documents in one round trip, keyed by requested store key."""
394
+ keys = list(dict.fromkeys(str(name) for name in (names or ()) if str(name)))
395
+ if not keys:
396
+ return {}
397
+ with _pool().connection() as con:
398
+ rows = con.execute(
399
+ _q('SELECT key, value FROM {tbl} WHERE key = ANY(%s::text[])',
400
+ 'store_kv', tenant_slug),
401
+ (keys,)).fetchall()
402
+ found = {str(key): value for key, value in rows}
403
+ return {
404
+ key: (dict(found[key]) if isinstance(found.get(key), dict)
405
+ else (found[key] if key in found else {}))
406
+ for key in keys
407
+ }
408
+
409
+
410
  def exists(name, tenant_slug=None):
411
  """True/False — and unlike the HF backend this is NEVER "True on uncertainty".
412
 
 
670
  """
671
  return get_projection(name, drop=drop, tenant_slug=self.tenant_slug)
672
 
673
+ def get_projections(self, names, drop=(), timeout_s=None):
674
+ """Bound batch projection; absent keys keep the single-read empty-document shape."""
675
+ return _get_projections(names, drop=drop, tenant_slug=self.tenant_slug,
676
+ timeout_s=timeout_s)
677
+
678
+ def get_many(self, names):
679
+ """Bound multi-key read used by request-scoped read snapshots."""
680
+ return _get_many(names, tenant_slug=self.tenant_slug)
681
+
682
  def _read_strict(self, name):
683
  """Interface parity with the HF Store. There, `get` is lenient (swallows a transient
684
  failure) and `_read_strict` opts out so a failed read ABORTS a read-modify-write instead
platform/core/table_store.py CHANGED
@@ -1331,6 +1331,34 @@ class TableStore:
1331
  })
1332
  return plans
1333
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1334
  def migrate_legacy_shared_views(self):
1335
  """Convert ``__shared__.views`` to personal source records plus explicit grants.
1336
 
 
1331
  })
1332
  return plans
1333
 
1334
+ def legacy_view_migration_needed(self, source=None):
1335
+ """Whether a preloaded workspace document still carries either retired View shape.
1336
+
1337
+ This is the read-only half of :meth:`migrate_legacy_shared_views`. A caller that already
1338
+ paid for a rows-free workspace projection can avoid a second store read for the common
1339
+ clean document while still routing every legacy document through the full strict,
1340
+ write-and-read-back migration below. Malformed legacy data raises here exactly as it does
1341
+ there; it is never interpreted as clean.
1342
+ """
1343
+ data = self._strict_document() if source is None else source
1344
+ if not isinstance(data, dict):
1345
+ raise LegacySharedViewMigrationError(
1346
+ 'legacy shared-view migration found a non-object workspace document')
1347
+ has_blank_legacy, _blank = self._blank_legacy_view_bucket(data)
1348
+ return bool(has_blank_legacy or self._legacy_shared_plans(data)
1349
+ or self._personal_permission_plans(data))
1350
+
1351
+ def prepare_view_projection(self, source):
1352
+ """Migrate a preloaded projection when needed; return whether its caller must refresh.
1353
+
1354
+ Keeping this decision beside the migration prevents read routes from acquiring a second,
1355
+ partial definition of which retired shapes are write-bearing.
1356
+ """
1357
+ if not self.legacy_view_migration_needed(source):
1358
+ return False
1359
+ self.migrate_legacy_shared_views()
1360
+ return True
1361
+
1362
  def migrate_legacy_shared_views(self):
1363
  """Convert ``__shared__.views`` to personal source records plus explicit grants.
1364
 
platform/core/user_tables.py CHANGED
@@ -218,7 +218,20 @@ def is_connected(table_key, st=None):
218
  key = str(table_key or '').strip()
219
  if key in _CONNECTED:
220
  return True
221
- table = get(key, st) or {}
 
 
 
 
 
 
 
 
 
 
 
 
 
222
  if not table:
223
  return False
224
  if table.get('connected') is True:
@@ -927,7 +940,7 @@ def _slug(label):
927
  return (s or 'table')[:40]
928
 
929
 
930
- def _ag_formula(raw):
931
  """`aios_grid._clean_formula`, reached the way this module reaches everything one layer up.
932
 
933
  ⚠ ONE call site for the function-local import, shared by BOTH field doors (`clean_fields` and
@@ -936,12 +949,11 @@ def _ag_formula(raw):
936
  cost twice. A third and fourth copy of `import aios_grid as _agX` is how one of them ends up
937
  calling a different validator.
938
 
939
- ⚠ `valid_keys` IS DELIBERATELY NOT PASSED. See the note at `_clean_field`'s formula branch:
940
- neither door has reliable sibling-field context, and a second, weaker copy of the ref rule is
941
- worse than leaving refs to the read-time behaviour `_clean_formula` already documents.
942
  """
943
  import aios_grid as _agf
944
- return _agf._clean_formula(raw)
945
 
946
 
947
  def _ag_geocode(raw):
@@ -2874,9 +2886,13 @@ def note_human_edit(table_key, values, row_id, st=None):
2874
  return sorted(touched)
2875
 
2876
 
2877
- def _clean_field(raw, previous=None, recurrence_validator=None):
2878
  """One field dict → the stored shape, or None. The single validator for create AND patch, so
2879
- a column cannot be typed one way on the way in and another on the way back."""
 
 
 
 
2880
  prev = previous or {}
2881
  raw = raw if isinstance(raw, dict) else {}
2882
  label = ' '.join(str(raw.get('label') or prev.get('label') or '').split())[:80]
@@ -3149,21 +3165,21 @@ def _clean_field(raw, previous=None, recurrence_validator=None):
3149
  # formula column with no expression is a column that can never show a value, so the pairing is
3150
  # enforced BOTH WAYS and either half alone refuses the field.
3151
  #
3152
- # ⚠ `valid_keys` IS DELIBERATELY OMITTED, and it is the one real asymmetry with the overlay
3153
- # door. `grid_events` passes the workspace's own key set, so a `{ref}` naming no column is
3154
- # refused at write time there. This function is handed ONE field with no sibling context —
3155
- # `add_field`/`patch_field` could supply the table's keys, but `clean_fields` validates a list
3156
- # mid-construction where half the referenced columns do not exist yet, and `clean_machine_
3157
- # fields` judges a list that has no table at all. A second, weaker copy of the ref rule in one
3158
- # of the three would be the divergence this module keeps paying for. STRUCTURE is checked here
3159
- # (charset, length, balanced parens and quotes, well-formed refs); an unresolvable ref blanks
3160
- # the CELL client-side, which is `_clean_formula`'s own documented read-time behaviour and the
3161
- # same answer a later-deleted column already produces.
3162
  # ⚠ Reached through `_ag_formula` so BOTH doors call one validator through one import.
3163
  formula_raw = (raw.get('formula') if 'formula' in raw
3164
  else (prev.get('formula') if prev_type == ftype else None))
3165
  if formula_raw is not None:
3166
- fx = _ag_formula(formula_raw)
3167
  if fx is None or ftype != 'formula':
3168
  return None
3169
  out['formula'] = fx
@@ -3780,7 +3796,8 @@ def add_field(table_key, raw, st=None, recurrence_validator=None):
3780
  have = [f for f in (defn.get('fields') or [])]
3781
  if len(have) >= MAX_FIELDS:
3782
  return None
3783
- field = _clean_field(raw, recurrence_validator=recurrence_validator)
 
3784
  if not field or any(f.get('key') == field['key'] for f in have):
3785
  return None
3786
  if not dynamic_link_filter_valid(field, have):
@@ -3988,7 +4005,13 @@ def patch_field(table_key, fkey, raw, st=None, recurrence_validator=None):
3988
  if 'description' in merged and isinstance(fields[idx].get('automation'), dict) \
3989
  and fields[idx]['automation'].get('preset') is True:
3990
  merged.pop('description')
3991
- field = _clean_field(merged, fields[idx], recurrence_validator=recurrence_validator)
 
 
 
 
 
 
3992
  if not field:
3993
  return None
3994
  if not dynamic_link_filter_valid(field, fields):
@@ -4473,10 +4496,13 @@ def may_edit_database(table_key, viewer, is_admin=False, st=None):
4473
  #:
4474
  #: ⚠ THE ALLOW-LIST IS THE SAFETY ARGUMENT, not the read-only-ness. Any other key passes through
4475
  #: to the real runtime uncached, `fresh=True` always bypasses, and every WRITE verb reaches the
4476
- #: runtime through `__getattr__` — so this can serve a stale value for exactly two keys, for
 
4477
  #: exactly one pass, to code proven not to mutate them (`all_tables` shallow-copies the top level
4478
  #: before anybody touches it; `grants`/`_clean_entries` build new objects; `may_open` only READS
4479
- #: `createdBy`).
 
 
4480
  _LENDABLE = (STORE_KEY, 'object_shares')
4481
 
4482
 
@@ -4487,7 +4513,7 @@ class _Lent:
4487
  self._runtime, self._docs = runtime, dict(docs or {})
4488
 
4489
  def get(self, name, fresh=False):
4490
- if fresh or name not in _LENDABLE:
4491
  return self._runtime.get(name, fresh) if fresh else self._runtime.get(name)
4492
  if name not in self._docs:
4493
  self._docs[name] = self._runtime.get(name) # lazily, once, for this pass
@@ -4507,7 +4533,7 @@ class _Lent:
4507
  A re-projection would cost a second copy to convert the first case and change nothing
4508
  about the second.
4509
  """
4510
- if name not in _LENDABLE:
4511
  return self._runtime.get_projection(name, drop=drop)
4512
  if name not in self._docs:
4513
  self._docs[name] = self._runtime.get_projection(name, drop=drop)
@@ -4518,7 +4544,7 @@ class _Lent:
4518
 
4519
 
4520
  def lend(st=None, **docs):
4521
- """A handle that serves `docs` for `_LENDABLE` keys and defers everything else to `st`.
4522
 
4523
  `lend(rt, user_tables=doc)` — keyword names are STORE KEYS. Call it once per read pass and
4524
  hand the result to every predicate in that pass; see the note above for what makes it safe.
 
218
  key = str(table_key or '').strip()
219
  if key in _CONNECTED:
220
  return True
221
+ # ⭐⭐ W62-T17 (D-739) — `all_defs`, NOT `get`, AND THE THREE KEYS BELOW ARE THE WHOLE ARGUMENT.
222
+ # This function asks `connected`, `source` and `recordMode`. Every one of them is a DEFINITION
223
+ # key; not one of them is a row. `get` reaches `all_tables`, which is the whole tenant
224
+ # document — MEASURED at 97,859,052 bytes and 2,274-2,816 ms PER CALL on staging's Royal
225
+ # Imports (`AutoWiki/waves/wave62/proto/T17-profile.md`), against 106,779 bytes for the
226
+ # rows-free projection that answers identically. D-739 called this "a platform-wide hazard,
227
+ # not a templates one: anything that calls it in a loop has this defect", and the loop callers
228
+ # were never the only victims — one call was already over the whole 2,000 ms module budget.
229
+ #
230
+ # ⚠ THE PROJECTION IS SAFE HERE FOR THE SAME REASON IT IS SAFE IN `/nav`: `all_defs` drops
231
+ # exactly `rows`, raises loudly (or falls back to the whole read) rather than answering empty,
232
+ # and a lent handle already serves whichever document the pass is holding. A caller that lends
233
+ # the WHOLE document gets MORE than asked for, which is the one direction this may be wrong in.
234
+ table = all_defs(st).get(key) or {}
235
  if not table:
236
  return False
237
  if table.get('connected') is True:
 
940
  return (s or 'table')[:40]
941
 
942
 
943
+ def _ag_formula(raw, valid_keys=None):
944
  """`aios_grid._clean_formula`, reached the way this module reaches everything one layer up.
945
 
946
  ⚠ ONE call site for the function-local import, shared by BOTH field doors (`clean_fields` and
 
949
  cost twice. A third and fourth copy of `import aios_grid as _agX` is how one of them ends up
950
  calling a different validator.
951
 
952
+ ⚠ `valid_keys` is forwarded as given, `None` by default. D-540: only `add_field`/`patch_field`
953
+ supply it; see the note at `_clean_field`'s formula branch.
 
954
  """
955
  import aios_grid as _agf
956
+ return _agf._clean_formula(raw, valid_keys)
957
 
958
 
959
  def _ag_geocode(raw):
 
2886
  return sorted(touched)
2887
 
2888
 
2889
+ def _clean_field(raw, previous=None, recurrence_validator=None, valid_keys=None):
2890
  """One field dict → the stored shape, or None. The single validator for create AND patch, so
2891
+ a column cannot be typed one way on the way in and another on the way back.
2892
+
2893
+ `valid_keys` (D-540): the table's other column keys, handed in ONLY by the two write doors
2894
+ that have table context (`add_field`, and `patch_field` when the caller sends a formula).
2895
+ Every other caller keeps `None`, which is the read-time/list-construction posture."""
2896
  prev = previous or {}
2897
  raw = raw if isinstance(raw, dict) else {}
2898
  label = ' '.join(str(raw.get('label') or prev.get('label') or '').split())[:80]
 
3165
  # formula column with no expression is a column that can never show a value, so the pairing is
3166
  # enforced BOTH WAYS and either half alone refuses the field.
3167
  #
3168
+ # ⛔ D-540 (2026-09-17): `valid_keys` IS PASSED WHEN THE CALLER HAS TABLE CONTEXT, and only
3169
+ # then. `grid_events` passes the workspace's own key set, and `formulaEngine.ts::
3170
+ # validateFormula` refuses a ref outside `knownKeys`, so the `ut_*` write door was the one
3171
+ # door that stored `{notacolumn}`. `add_field` and `patch_field` (when the caller sends a
3172
+ # formula) now hand in the table's other column keys and an unknown ref is refused there.
3173
+ # `clean_fields` (a list mid-construction, where half the referenced columns do not exist
3174
+ # yet), `clean_machine_fields` (a list with no table at all) and the link reciprocal keep the
3175
+ # `None` default: STRUCTURE only, and an unresolvable ref blanks the CELL client-side, which
3176
+ # is `_clean_formula`'s documented read-time behaviour. A patch that does not send a formula
3177
+ # keeps its stored one unjudged, so deleting a referenced column never vaporises a column.
3178
  # ⚠ Reached through `_ag_formula` so BOTH doors call one validator through one import.
3179
  formula_raw = (raw.get('formula') if 'formula' in raw
3180
  else (prev.get('formula') if prev_type == ftype else None))
3181
  if formula_raw is not None:
3182
+ fx = _ag_formula(formula_raw, valid_keys)
3183
  if fx is None or ftype != 'formula':
3184
  return None
3185
  out['formula'] = fx
 
3796
  have = [f for f in (defn.get('fields') or [])]
3797
  if len(have) >= MAX_FIELDS:
3798
  return None
3799
+ field = _clean_field(raw, recurrence_validator=recurrence_validator,
3800
+ valid_keys={str(f.get('key')) for f in have if f.get('key')})
3801
  if not field or any(f.get('key') == field['key'] for f in have):
3802
  return None
3803
  if not dynamic_link_filter_valid(field, have):
 
4005
  if 'description' in merged and isinstance(fields[idx].get('automation'), dict) \
4006
  and fields[idx]['automation'].get('preset') is True:
4007
  merged.pop('description')
4008
+ # D-540: a patch that SENDS a formula is judged against the table's other columns; one that
4009
+ # does not keeps its stored formula unjudged (a later-deleted ref blanks cells, not the column).
4010
+ field = _clean_field(
4011
+ merged, fields[idx], recurrence_validator=recurrence_validator,
4012
+ valid_keys=({str(f.get('key')) for f in fields
4013
+ if f.get('key') and str(f.get('key')) != str(fkey)}
4014
+ if isinstance(raw, dict) and 'formula' in raw else None))
4015
  if not field:
4016
  return None
4017
  if not dynamic_link_filter_valid(field, fields):
 
4496
  #:
4497
  #: ⚠ THE ALLOW-LIST IS THE SAFETY ARGUMENT, not the read-only-ness. Any other key passes through
4498
  #: to the real runtime uncached, `fresh=True` always bypasses, and every WRITE verb reaches the
4499
+ #: runtime through `__getattr__` — so this can serve a stale value for the two named keys and
4500
+ #: dynamically named `<key>__shared` overlay buckets, for
4501
  #: exactly one pass, to code proven not to mutate them (`all_tables` shallow-copies the top level
4502
  #: before anybody touches it; `grants`/`_clean_entries` build new objects; `may_open` only READS
4503
+ #: `createdBy`). Shared overlays are also read-only on this path: `_shared_value_keys` only
4504
+ #: reads their field keys, never mutates the document. Their first read is lent for one pass;
4505
+ #: `fresh=True` and write verbs still reach the runtime, never the snapshot.
4506
  _LENDABLE = (STORE_KEY, 'object_shares')
4507
 
4508
 
 
4513
  self._runtime, self._docs = runtime, dict(docs or {})
4514
 
4515
  def get(self, name, fresh=False):
4516
+ if fresh or (name not in _LENDABLE and not str(name).endswith('__shared')):
4517
  return self._runtime.get(name, fresh) if fresh else self._runtime.get(name)
4518
  if name not in self._docs:
4519
  self._docs[name] = self._runtime.get(name) # lazily, once, for this pass
 
4533
  A re-projection would cost a second copy to convert the first case and change nothing
4534
  about the second.
4535
  """
4536
+ if name not in _LENDABLE and not str(name).endswith('__shared'):
4537
  return self._runtime.get_projection(name, drop=drop)
4538
  if name not in self._docs:
4539
  self._docs[name] = self._runtime.get_projection(name, drop=drop)
 
4544
 
4545
 
4546
  def lend(st=None, **docs):
4547
+ """Lend `_LENDABLE` keys and `__shared` suffix buckets; defer other keys to `st`.
4548
 
4549
  `lend(rt, user_tables=doc)` — keyword names are STORE KEYS. Call it once per read pass and
4550
  hand the result to every predicate in that pass; see the note above for what makes it safe.
platform/core/windows.py ADDED
@@ -0,0 +1,492 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """core/windows.py — the DATE WINDOW vocabulary (CG-7, owner item 2, 2026-07-26).
2
+
3
+ A filter condition on a measure reads:
4
+
5
+ Where [Sales] [in the last 90 days] [>] [5,000]
6
+
7
+ The middle bracket is this module. It turns a window SPEC — a small JSON object a saved view can
8
+ persist — into a concrete `(date_from, date_to)` pair of ISO dates.
9
+
10
+ WHY IT IS ITS OWN MODULE, AND WHY IT TAKES `today` AS AN ARGUMENT
11
+ -----------------------------------------------------------------------------------------------
12
+ Two engines must agree on what "this quarter" means: this one (which compiles the SQL) and
13
+ `customer-grid/windows.ts` (which renders the label the user reads and the bounds the client
14
+ engine would use). If they disagree by one day, the grid shows a number the label denies, and
15
+ nothing errors. `aios-web/verify_windows.py` holds them in lock-step over a fixed set of probe
16
+ dates, exactly as verify_filter_engine.py does for the filter tree.
17
+
18
+ `today` is a PARAMETER, never `date.today()` read inside. Three reasons, all learned:
19
+ - a gate cannot compare two engines on "now" — the two runs are milliseconds apart and land on
20
+ different sides of midnight roughly once every few thousand runs, which is the worst kind of
21
+ flake because it looks like a real divergence
22
+ - a window resolved during a request must not shift between the count query and the row query
23
+ - the tenant's day boundary is the tenant's, not the server's
24
+
25
+ DESIGN NOTES THAT ARE EASY TO GET WRONG
26
+ -----------------------------------------------------------------------------------------------
27
+ - Bounds are INCLUSIVE at both ends and are DATES, not timestamps. `store_query` casts both
28
+ sides to TIMESTAMP and appends 23:59:59 to the upper bound, so a bare date is correct there;
29
+ the client compares the 10-char ISO prefix, so a bare date is correct there too.
30
+ - "Last N days" INCLUDES today. Airtable's "the past week" means the last 7 days up to and
31
+ including now, not the 7 days before yesterday. Off-by-one here silently drops today's
32
+ orders from every "recent" filter — invisible until someone asks why a sale they just
33
+ entered is missing.
34
+ - Week starts MONDAY (ISO 8601). Stated rather than defaulted: Python's weekday() is
35
+ Monday=0 and JavaScript's getDay() is Sunday=0, so the two engines disagree unless one of
36
+ them is explicitly corrected. That correction is the single most likely divergence in this
37
+ file, and the gate probes a Sunday and a Monday for exactly that reason.
38
+ - A `custom` window with only one bound is legal and means open-ended on the other side.
39
+ - An UNRESOLVABLE window returns None rather than a default. A window that quietly becomes
40
+ "all time" would widen a filter while still reporting an authoritative count.
41
+ The historical ``harness.windows`` import remains a compatibility re-export. The vocabulary
42
+ lives in core because workbook document validation also consumes it and core must never import
43
+ upward from harness.
44
+ """
45
+ import datetime as _dt
46
+ import re
47
+ import math
48
+
49
+ #: Every window kind the UI may offer. Mirrors WINDOW_KINDS in customer-grid/windows.ts;
50
+ #: verify_windows.py asserts the two lists are identical, because a kind the client can emit and
51
+ #: the server cannot resolve is a filter that silently stops narrowing.
52
+ WINDOW_KINDS = (
53
+ "all_time",
54
+ "today",
55
+ "yesterday",
56
+ "this_week",
57
+ "last_week",
58
+ "this_month",
59
+ "last_month",
60
+ "this_quarter",
61
+ "last_quarter",
62
+ "this_year",
63
+ "last_year",
64
+ "ytd",
65
+ "ytd_last_year",
66
+ "ltm",
67
+ "past_week",
68
+ "past_month",
69
+ "past_year",
70
+ "last_n_days",
71
+ "next_n_days",
72
+ "custom",
73
+ )
74
+
75
+ #: Kinds that carry an integer `n`.
76
+ N_KINDS = frozenset({"last_n_days", "next_n_days"})
77
+
78
+ #: How each kind reads in a sentence. The UI renders these; they are here so the label and the
79
+ #: arithmetic cannot drift apart in the one place a user would never think to check.
80
+ WINDOW_LABELS = {
81
+ "all_time": "all time",
82
+ "today": "today",
83
+ "yesterday": "yesterday",
84
+ "this_week": "this week",
85
+ "last_week": "last week",
86
+ "this_month": "this month",
87
+ "last_month": "last month",
88
+ "this_quarter": "this quarter",
89
+ "last_quarter": "last quarter",
90
+ "this_year": "this year",
91
+ "last_year": "last year",
92
+ "ytd": "year to date",
93
+ # "last year to date" (LYTD), not "year to date, last year": the longer phrasing CLIPPED to
94
+ # "year to date, last ye" in the 146px window select — and this vocabulary is CLOSED, so
95
+ # the rule for it is that it stays readable rather than that it fits. Seen in a live
96
+ # screenshot of the owner's own comparison; every assertion in that run was green.
97
+ "ytd_last_year": "last year to date",
98
+ "ltm": "the last 12 months",
99
+ "past_week": "the past week",
100
+ "past_month": "the past month",
101
+ "past_year": "the past year",
102
+ "last_n_days": "the last {n} days",
103
+ "next_n_days": "the next {n} days",
104
+ "custom": "a custom range",
105
+ }
106
+
107
+ MAX_N = 3650 # ten years; a bound, not a business rule
108
+
109
+ # --- the DATE-VALUE anchors (owner item 3, 2026-07-26) ----------------------------------------
110
+ # The second half of a date CONDITION, as distinct from a measure's window:
111
+ #
112
+ # Where [Last order] [is before] [one month ago]
113
+ # ^ op ^ THIS
114
+ #
115
+ # A window answers "over what period do I sum"; an anchor answers "which single date am I
116
+ # comparing against". Both live in this module for one reason: they are the same two-engine
117
+ # contract, resolved from the same `today`, and a gate that holds one in lock-step and not the
118
+ # other would leave half the sentence free to drift.
119
+
120
+ #: Anchor modes, in the order the picker offers them. `exact` is the historical behaviour — a
121
+ #: rule with NO mode is an `exact` rule whose value is an ISO date, which is what every view
122
+ #: saved before this change carries.
123
+ ANCHOR_MODES = (
124
+ "today",
125
+ "yesterday",
126
+ "one_week_ago",
127
+ "one_month_ago",
128
+ "n_days_ago",
129
+ "exact",
130
+ )
131
+
132
+ #: Modes that carry NO value. ⚠ This set is load-bearing far outside this module: a rule whose
133
+ #: value is blank normally reads as INACTIVE, and an inactive rule is IGNORED — which WIDENS the
134
+ #: result under a count nobody would doubt. Every activeness check (`isRuleActive` in TS,
135
+ #: `filter_sql.is_rule_active`, and the validator's value handling) has to know these four are
136
+ #: active with an empty value.
137
+ ANCHOR_VALUE_FREE = frozenset({"today", "yesterday", "one_week_ago", "one_month_ago"})
138
+
139
+ ANCHOR_LABELS = {
140
+ "today": "today",
141
+ "yesterday": "yesterday",
142
+ "one_week_ago": "one week ago",
143
+ "one_month_ago": "one month ago",
144
+ "n_days_ago": "{n} days ago",
145
+ "exact": "an exact date",
146
+ }
147
+
148
+
149
+ def _iso(d):
150
+ return d.isoformat()
151
+
152
+
153
+ def _month_start(d):
154
+ return d.replace(day=1)
155
+
156
+
157
+ def _month_end(d):
158
+ return _next_month(d.replace(day=1)) - _dt.timedelta(days=1)
159
+
160
+
161
+ def _next_month(d):
162
+ return (d.replace(day=28) + _dt.timedelta(days=4)).replace(day=1)
163
+
164
+
165
+ def _quarter_start(d):
166
+ return _dt.date(d.year, 3 * ((d.month - 1) // 3) + 1, 1)
167
+
168
+
169
+ def _days_in_month(y, m):
170
+ return (_dt.date(y + (m == 12), 1 if m == 12 else m + 1, 1) - _dt.timedelta(days=1)).day
171
+
172
+
173
+ def _shift_months(d, n):
174
+ """`d` moved `n` months, CLAMPING the day to the target month's length.
175
+
176
+ Jan 31 back one month is Dec 31, but Mar 31 back one month is Feb 28 (or 29) — there is no
177
+ Feb 31 to land on. Both engines must clamp identically or "the past month" differs by up to
178
+ three days for a third of the calendar; the gate probes a 31st and a leap day for exactly it.
179
+ """
180
+ total = (d.year * 12 + (d.month - 1)) + n
181
+ y, m = divmod(total, 12)
182
+ m += 1
183
+ return _dt.date(y, m, min(d.day, _days_in_month(y, m)))
184
+
185
+
186
+ def _parse_date(v):
187
+ """Accept 'YYYY-MM-DD' (and tolerate a longer ISO timestamp by taking its date part)."""
188
+ if isinstance(v, _dt.date):
189
+ return v
190
+ s = str(v or "").strip()[:10]
191
+ if not s:
192
+ return None
193
+ try:
194
+ return _dt.date.fromisoformat(s)
195
+ except ValueError:
196
+ return None
197
+
198
+
199
+ # Sheet Ending metadata belongs to the binding period, never to a window spec.
200
+ ENDING_KINDS = ("today", "yesterday", "last_week", "last_month", "last_quarter",
201
+ "last_year", "fixed", "cell", "n_before_today")
202
+ ENDING_LABELS = dict(zip(ENDING_KINDS, ("Today (live)", "Yesterday", "End of last week",
203
+ "End of last month", "End of last quarter", "End of last year", "A fixed date", "From a cell",
204
+ "N periods before today")))
205
+ TO_DATE_KINDS = ("ytd", "ytd_last_year", "this_month", "this_quarter")
206
+ ENDING_STEPS = ("day", "week", "month", "quarter", "year")
207
+
208
+
209
+ def ending_step_default(kind):
210
+ return "quarter" if kind == "this_quarter" else "month"
211
+
212
+
213
+ def ending_date(value):
214
+ """A real ISO day only. No truncation, calendar rollover, or blank fallback."""
215
+ if not isinstance(value, str) or not re.fullmatch(r"[0-9]{4}-[0-9]{2}-[0-9]{2}", value.strip()):
216
+ return None
217
+ d = _parse_date(value)
218
+ return _iso(d) if d else None
219
+
220
+
221
+ def resolve_ending(mode, value, today, step="month", offset=0, n=0, unit="month"):
222
+ """Inclusive ending date; cell values are supplied by the caller's trusted read."""
223
+ d = _parse_date(today)
224
+ if d is None:
225
+ raise ValueError("Ending needs a valid today date")
226
+ if mode not in ENDING_KINDS or step not in ENDING_STEPS:
227
+ return None
228
+ if isinstance(offset, bool) or not isinstance(offset, int) or offset < 0:
229
+ return None
230
+ if mode in ("fixed", "cell"):
231
+ parsed = ending_date(value)
232
+ if parsed is None:
233
+ return None
234
+ end = _parse_date(parsed)
235
+ elif mode == "n_before_today":
236
+ if (isinstance(n, bool) or not isinstance(n, int) or not 1 <= n <= MAX_N
237
+ or unit not in ENDING_STEPS):
238
+ return None
239
+ try:
240
+ if unit in ("day", "week"):
241
+ end = d - _dt.timedelta(days=n * (7 if unit == "week" else 1))
242
+ else:
243
+ end = _shift_months(d, -n * {"month": 1, "quarter": 3, "year": 12}[unit])
244
+ except (ValueError, OverflowError):
245
+ return None
246
+ elif mode == "today":
247
+ end = d
248
+ elif mode == "yesterday":
249
+ end = d - _dt.timedelta(days=1)
250
+ elif mode == "last_week":
251
+ end = d - _dt.timedelta(days=d.weekday() + 1)
252
+ elif mode == "last_month":
253
+ end = _month_start(d) - _dt.timedelta(days=1)
254
+ elif mode == "last_quarter":
255
+ end = _quarter_start(d) - _dt.timedelta(days=1)
256
+ else:
257
+ end = _dt.date(d.year - 1, 12, 31)
258
+ try:
259
+ if step in ("day", "week"):
260
+ end += _dt.timedelta(days=offset * (7 if step == "week" else 1))
261
+ else:
262
+ end = _shift_months(end, offset * {"month": 1, "quarter": 3, "year": 12}[step])
263
+ return _iso(end)
264
+ except (ValueError, OverflowError):
265
+ return None
266
+
267
+
268
+ def ending_cell_date(value, date_system="1900"):
269
+ """Sheet dates may be ISO days or Excel serials in the workbook's declared epoch."""
270
+ if not isinstance(value, (int, float)) or isinstance(value, bool):
271
+ return ending_date(value)
272
+ if not math.isfinite(value) or value < (0 if date_system == "1904" else 1):
273
+ return None
274
+ days = math.floor(value)
275
+ if date_system != "1904" and days == 60:
276
+ return None # Excel's fictional February 29, 1900 is not a calendar date.
277
+ epoch = (_dt.date(1904, 1, 1) if date_system == "1904" else
278
+ _dt.date(1899, 12, 31) if days < 60 else _dt.date(1899, 12, 30))
279
+ try:
280
+ return _iso(epoch + _dt.timedelta(days=days))
281
+ except (ValueError, OverflowError):
282
+ return None
283
+
284
+
285
+ def ending_bounds(kind, mode, value, today, step="month", offset=0, n=0, unit="month"):
286
+ """Keep the kind's own start. An ending before it refuses instead of reversing bounds."""
287
+ if kind not in TO_DATE_KINDS:
288
+ return None
289
+ end = resolve_ending(mode, value, today, step, offset, n, unit)
290
+ if end is None:
291
+ return None
292
+ start = resolve({"kind": kind}, today)[0]
293
+ return (start, end) if start <= end else None
294
+
295
+
296
+ def normalize(spec):
297
+ """Coerce an untrusted window spec to `{kind, n?, from?, to?}` or None.
298
+
299
+ Fail-closed on the KIND (an unknown kind is not a window), tolerant on the rest — the same
300
+ split `clean_filter_tree` uses, so one malformed window cannot cost a user their saved view.
301
+ """
302
+ if not isinstance(spec, dict):
303
+ return None
304
+ kind = spec.get("kind")
305
+ if kind not in WINDOW_KINDS:
306
+ return None
307
+ out = {"kind": kind}
308
+ if kind in N_KINDS:
309
+ try:
310
+ n = int(spec.get("n"))
311
+ except (TypeError, ValueError):
312
+ return None
313
+ if n < 1 or n > MAX_N:
314
+ return None
315
+ out["n"] = n
316
+ if kind == "custom":
317
+ f, t = _parse_date(spec.get("from")), _parse_date(spec.get("to"))
318
+ if f is None and t is None:
319
+ return None # a custom range with no bounds is not a window
320
+ if f is not None and t is not None and f > t:
321
+ f, t = t, f # the builder cannot enforce order; the engine can
322
+ if f is not None:
323
+ out["from"] = _iso(f)
324
+ if t is not None:
325
+ out["to"] = _iso(t)
326
+ return out
327
+
328
+
329
+ def resolve(spec, today):
330
+ """Window spec + the tenant's today -> `(date_from, date_to)`, both inclusive ISO dates.
331
+
332
+ Either side may be None, meaning open-ended. Returns None when the spec is not a window at
333
+ all — callers MUST treat that as "this condition cannot be evaluated" and refuse, never as
334
+ "no window", which would silently widen the result to all time.
335
+ """
336
+ spec = normalize(spec)
337
+ if spec is None:
338
+ return None
339
+ kind = spec["kind"]
340
+ d = _parse_date(today)
341
+ if d is None:
342
+ raise ValueError("resolve() needs an explicit `today`; see this module's docstring")
343
+
344
+ if kind == "all_time":
345
+ return (None, None)
346
+ if kind == "today":
347
+ return (_iso(d), _iso(d))
348
+ if kind == "yesterday":
349
+ y = d - _dt.timedelta(days=1)
350
+ return (_iso(y), _iso(y))
351
+
352
+ # ISO 8601: the week starts MONDAY. Python's weekday() is already Monday=0; the TS mirror
353
+ # has to correct getDay(), which is Sunday=0. The gate probes both a Sunday and a Monday.
354
+ if kind == "this_week":
355
+ start = d - _dt.timedelta(days=d.weekday())
356
+ return (_iso(start), _iso(start + _dt.timedelta(days=6)))
357
+ if kind == "last_week":
358
+ start = d - _dt.timedelta(days=d.weekday() + 7)
359
+ return (_iso(start), _iso(start + _dt.timedelta(days=6)))
360
+
361
+ if kind == "this_month":
362
+ return (_iso(_month_start(d)), _iso(_month_end(d)))
363
+ if kind == "last_month":
364
+ prev = _month_start(d) - _dt.timedelta(days=1)
365
+ return (_iso(_month_start(prev)), _iso(prev))
366
+
367
+ if kind == "this_quarter":
368
+ qs = _quarter_start(d)
369
+ qe = _month_end(_dt.date(qs.year, qs.month + 2, 1))
370
+ return (_iso(qs), _iso(qe))
371
+ if kind == "last_quarter":
372
+ prev_end = _quarter_start(d) - _dt.timedelta(days=1)
373
+ qs = _quarter_start(prev_end)
374
+ return (_iso(qs), _iso(prev_end))
375
+
376
+ if kind == "this_year":
377
+ return (_iso(_dt.date(d.year, 1, 1)), _iso(_dt.date(d.year, 12, 31)))
378
+ if kind == "last_year":
379
+ return (_iso(_dt.date(d.year - 1, 1, 1)), _iso(_dt.date(d.year - 1, 12, 31)))
380
+ if kind == "ytd":
381
+ # year-to-date ENDS TODAY, unlike this_year which runs to Dec 31. The distinction is the
382
+ # whole reason both exist: comparing "this year" against last year double-counts the
383
+ # months that have not happened yet.
384
+ return (_iso(_dt.date(d.year, 1, 1)), _iso(d))
385
+ if kind == "ytd_last_year":
386
+ # SAME PERIOD last year — Jan 1 LY through today's month/day LY. This is the honest
387
+ # partner of `ytd`, and it is not a nicety: it mirrors `core.periods.ytd_last_year`
388
+ # EXACTLY (including the Feb 29 -> Feb 28 clamp), which is what `pool()` computes the
389
+ # `revenue_ly` column from. `Sales[ytd] < Sales[ytd_last_year]` therefore reproduces the
390
+ # retired `at_risk > 0` condition rather than approximating it. Comparing `ytd` against
391
+ # `last_year` instead would pit seven months against twelve.
392
+ return (_iso(_dt.date(d.year - 1, 1, 1)), _iso(_shift_months(d, -12)))
393
+ if kind == "ltm":
394
+ # the last twelve months INCLUDING today: 365 days back, both ends inclusive
395
+ return (_iso(d - _dt.timedelta(days=364)), _iso(d))
396
+
397
+ # "The past week/month/year" are ROLLING and end TODAY — Airtable's wording, and NOT the
398
+ # calendar kinds above: `last_week` is the previous Monday-Sunday, `past_week` is the seven
399
+ # days ending today. Both are offered because a user means different things by them, and
400
+ # picking one to serve both would silently answer the other question.
401
+ # past_week = the same rule as last_n_days(7) — one period back, PLUS ONE DAY, so today
402
+ # is included and the span is exactly 7 days rather than 8.
403
+ # past_year agrees with `ltm` except across a leap day, where the calendar shift keeps the
404
+ # same month/day and the 364-day subtraction cannot.
405
+ if kind == "past_week":
406
+ return (_iso(d - _dt.timedelta(days=6)), _iso(d))
407
+ if kind == "past_month":
408
+ return (_iso(_shift_months(d, -1) + _dt.timedelta(days=1)), _iso(d))
409
+ if kind == "past_year":
410
+ return (_iso(_shift_months(d, -12) + _dt.timedelta(days=1)), _iso(d))
411
+
412
+ if kind == "last_n_days":
413
+ # INCLUSIVE of today — "the last 7 days" is today and the 6 before it, not the 7 before
414
+ # yesterday. Getting this wrong drops today's orders from every recent-activity filter.
415
+ return (_iso(d - _dt.timedelta(days=spec["n"] - 1)), _iso(d))
416
+ if kind == "next_n_days":
417
+ return (_iso(d), _iso(d + _dt.timedelta(days=spec["n"] - 1)))
418
+
419
+ if kind == "custom":
420
+ return (spec.get("from"), spec.get("to"))
421
+ raise AssertionError(f"unhandled window kind {kind!r}") # unreachable; WINDOW_KINDS is closed
422
+
423
+
424
+ def label(spec):
425
+ """How a window reads in the condition sentence ('in the last 90 days')."""
426
+ spec = normalize(spec)
427
+ if spec is None:
428
+ return "an invalid range"
429
+ return WINDOW_LABELS[spec["kind"]].format(n=spec.get("n"))
430
+
431
+
432
+ def resolve_anchor(mode, value, today):
433
+ """A date condition's right-hand side -> ONE inclusive ISO date, or None.
434
+
435
+ `mode` is an entry of ANCHOR_MODES; None/'' means `exact`, which is what every view saved
436
+ before anchors existed carries (op + an ISO date in `value`). `value` supplies the number for
437
+ `n_days_ago` and the date for `exact`, and is ignored by the four value-free modes.
438
+
439
+ Returns None when the anchor cannot be resolved — an unknown mode, a non-numeric N, a value
440
+ that is not a date. The caller must then treat the CONDITION as unanswerable and match
441
+ nothing. It must NOT fall through to "no condition": that widens the result while the count
442
+ beside it still looks authoritative, which is the whole reason this returns None rather than
443
+ a best guess (same contract as `resolve()` above).
444
+
445
+ `n_days_ago` accepts n = 0 where a WINDOW requires n >= 1. A zero-day window is empty and
446
+ could only be a mistake; "0 days ago" is today, which is a date a person can mean.
447
+ """
448
+ m = mode or "exact"
449
+ if m not in ANCHOR_MODES:
450
+ return None
451
+ d = _parse_date(today)
452
+ if d is None:
453
+ raise ValueError("resolve_anchor() needs an explicit `today`; see this module's "
454
+ "docstring on why the clock is never read here")
455
+ if m == "today":
456
+ return _iso(d)
457
+ if m == "yesterday":
458
+ return _iso(d - _dt.timedelta(days=1))
459
+ if m == "one_week_ago":
460
+ # A week ago is a DATE (today minus 7), not the past-week RANGE. The two read almost
461
+ # identically in English and mean different things in a comparison.
462
+ return _iso(d - _dt.timedelta(days=7))
463
+ if m == "one_month_ago":
464
+ return _iso(_shift_months(d, -1))
465
+ if m == "n_days_ago":
466
+ s = str("" if value is None else value).strip()
467
+ # `[0-9]` not `\d`: a Python \d matches every Unicode decimal digit, so a fullwidth
468
+ # '30' would resolve here and be rejected by the TS mirror's Number() — the exact
469
+ # two-engine divergence filter_sql.to_num was rewritten to avoid.
470
+ if not s or not all("0" <= ch <= "9" for ch in s):
471
+ return None
472
+ n = int(s)
473
+ # No `n < 0` guard: the digit test above already excludes a sign, so it would be
474
+ # unreachable. The negative control proved that — removing it changed nothing, which is
475
+ # how dead code hides in a defensive-looking line. The CAP is the live guard.
476
+ if n > MAX_N:
477
+ return None
478
+ return _iso(d - _dt.timedelta(days=n))
479
+ parsed = _parse_date(value) # exact
480
+ return None if parsed is None else _iso(parsed)
481
+
482
+
483
+ def _anchor_label(mode, value):
484
+ """How an anchor reads in the condition sentence ('one month ago', '30 days ago')."""
485
+ m = mode or "exact"
486
+ if m not in ANCHOR_MODES:
487
+ return "an invalid date"
488
+ if m == "n_days_ago":
489
+ return ANCHOR_LABELS[m].format(n=str("" if value is None else value).strip() or "N")
490
+ if m == "exact":
491
+ return str("" if value is None else value).strip() or ANCHOR_LABELS[m]
492
+ return ANCHOR_LABELS[m]
platform/harness/amazon_client.py CHANGED
@@ -245,8 +245,10 @@ REPORT_TYPES = {
245
  # reads whatever V2 actually sends and nothing here needs to know the column list -- which is
246
  # exactly the property that makes swapping the report type a one-line change.
247
  # · https://developer-docs.amazon.com/sp-api/docs/report-type-values-settlement
 
 
248
  "settlements": {
249
- "report": "GET_V2_SETTLEMENT_REPORT_DATA_FLAT_FILE_V2",
250
  "validates_against": "settlement summary vs settlement lines",
251
  "role": "Finance and Accounting",
252
  },
@@ -324,6 +326,13 @@ OPERATIONS = {
324
  "name": "createReport", "roles": _BULK_REPORT_ROLES, "method": "POST",
325
  "path": "reports/2021-06-30/reports", "report_types": _APPROVED_REPORT_TYPES,
326
  },
 
 
 
 
 
 
 
327
  "getReport": {
328
  "name": "getReport", "roles": _BULK_REPORT_ROLES, "method": "GET",
329
  "path": "reports/2021-06-30/reports/{reportId}",
@@ -967,6 +976,69 @@ def report_document(fields, report_type, start=None, end=None, options=None,
967
  "compression": meta.get("compressionAlgorithm")}
968
 
969
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
970
  def _download_document(url, compression="", timeout=120):
971
  """Fetch one presigned report document and decode it. -> text.
972
 
 
245
  # reads whatever V2 actually sends and nothing here needs to know the column list -- which is
246
  # exactly the property that makes swapping the report type a one-line change.
247
  # · https://developer-docs.amazon.com/sp-api/docs/report-type-values-settlement
248
+ # Amazon creates settlements on its own schedule and refuses createReport for them.
249
+ # This type is a getReports discovery filter, never a requestable report flavor.
250
  "settlements": {
251
+ "report_discover": "GET_V2_SETTLEMENT_REPORT_DATA_FLAT_FILE_V2",
252
  "validates_against": "settlement summary vs settlement lines",
253
  "role": "Finance and Accounting",
254
  },
 
326
  "name": "createReport", "roles": _BULK_REPORT_ROLES, "method": "POST",
327
  "path": "reports/2021-06-30/reports", "report_types": _APPROVED_REPORT_TYPES,
328
  },
329
+ # Discovery is approved only for Amazon-issued settlements and their finance role.
330
+ # Sharing the Reports path grants neither the other bulk types nor their roles.
331
+ "getReports": {
332
+ "name": "getReports", "roles": ("Finance and Accounting",), "method": "GET",
333
+ "path": "reports/2021-06-30/reports",
334
+ "report_types": (REPORT_TYPES["settlements"]["report_discover"],),
335
+ },
336
  "getReport": {
337
  "name": "getReport", "roles": _BULK_REPORT_ROLES, "method": "GET",
338
  "path": "reports/2021-06-30/reports/{reportId}",
 
976
  "compression": meta.get("compressionAlgorithm")}
977
 
978
 
979
+ SETTLEMENT_REPORT_PAGE_CAP = 1000
980
+
981
+
982
+ def settlement_reports(fields, *, since=None, timeout=_TIMEOUT, log=None):
983
+ """List settlements Amazon has already created, oldest first.
984
+
985
+ Amazon issues these on its own fortnightly cycle, so this lists rather than requests.
986
+ Only safe, named metadata leaves this function; vendor extras and tokens stay here.
987
+ """
988
+ require_operation("GET", _REPORTS_CREATE_PATH,
989
+ report_type=REPORT_TYPES["settlements"]["report_discover"])
990
+ import re
991
+
992
+ rtype = REPORT_TYPES["settlements"]["report_discover"]
993
+ token = access_token(fields, timeout=timeout)
994
+ region = str((fields or {}).get("region") or DEFAULT_REGION)
995
+ params = {"reportTypes": rtype, "processingStatuses": "DONE", "pageSize": 100}
996
+ if since is not None:
997
+ params["createdSince"] = str(since)
998
+ reports = []
999
+ for _page in range(SETTLEMENT_REPORT_PAGE_CAP):
1000
+ # _request owns the same _retry policy as the existing report/document reads.
1001
+ got = _request(_REPORTS_CREATE_PATH, token, region=region, timeout=timeout,
1002
+ sandbox=is_sandbox(fields), **params)
1003
+ for raw in got.get("reports") or ():
1004
+ row = {key: str(raw[key]) if raw.get(key) is not None else "" for key in (
1005
+ "reportId", "reportDocumentId", "reportType", "dataStartTime",
1006
+ "dataEndTime", "createdTime")}
1007
+ # Match the API status door without importing upward into the API layer.
1008
+ # Refuse unsafe values with a fixed sentence that cannot echo the credential.
1009
+ if any(len(value) > 512 or "Atzr|" in value
1010
+ or re.search(r"\bbearer\s+\S+", value, flags=re.IGNORECASE)
1011
+ or any(ch in value for ch in ("\u2014", "\u2013"))
1012
+ for value in row.values()):
1013
+ raise AmazonError("Settlement discovery returned unsafe metadata; no status was emitted.")
1014
+ if row["reportType"] != rtype or not row["reportDocumentId"]:
1015
+ raise AmazonError("Settlement discovery returned an unexpected type or no document ID.")
1016
+ reports.append(row)
1017
+ next_token = got.get("nextToken")
1018
+ if not next_token:
1019
+ if log:
1020
+ log(f"getReports discovered {len(reports)} settlement reports")
1021
+ return sorted(reports, key=lambda row: (row["createdTime"], row["reportId"]))
1022
+ # Amazon continuation tokens replace the original filters, which the token carries.
1023
+ params = {"nextToken": str(next_token)}
1024
+ raise AmazonError(
1025
+ f"Settlement discovery hit the {SETTLEMENT_REPORT_PAGE_CAP}-page cap because Amazon kept "
1026
+ "returning nextToken; retry with a narrower createdSince window.")
1027
+
1028
+
1029
+ def settlement_document(fields, document_id, *, timeout=_TIMEOUT):
1030
+ """One settlement document's text through getReportDocument and the presigned URL."""
1031
+ require_operation("GET", f"reports/2021-06-30/documents/{document_id}")
1032
+ token = access_token(fields, timeout=timeout)
1033
+ region = str((fields or {}).get("region") or DEFAULT_REGION)
1034
+ meta = _request(f"reports/2021-06-30/documents/{document_id}", token, region=region,
1035
+ timeout=timeout, sandbox=is_sandbox(fields))
1036
+ url = str(meta.get("url") or "")
1037
+ if not url:
1038
+ raise AmazonError("The settlement document carried no download URL.")
1039
+ return _download_document(url, str(meta.get("compressionAlgorithm") or ""))
1040
+
1041
+
1042
  def _download_document(url, compression="", timeout=120):
1043
  """Fetch one presigned report document and decode it. -> text.
1044
 
platform/harness/amazon_store.py CHANGED
@@ -70,6 +70,9 @@ PREFIX = "amzn_"
70
  TABLES = ("orders", "order_items", "listings", "settlements", "inventory",
71
  "pricing", "brand_analytics")
72
 
 
 
 
73
  # C1 is a tenant-store document, not an API dependency. verify_amazon gates reader parity.
74
  ENABLEMENT_KEY = "amazon_connector_enablement"
75
  # Settlements are issued roughly fortnightly; spending a report on every hourly tick adds nothing.
@@ -149,8 +152,20 @@ def entity_key(table):
149
  #: How far back a first sync reaches when the caller names no window. Amazon's flat-file order
150
  #: reports accept at most ~31 days per request, so a backfill is sliced (`_windows`).
151
  DEFAULT_BACKFILL_DAYS = int(os.environ.get("AMAZON_BACKFILL_DAYS") or 730)
 
 
152
  #: ⚠ AMAZON'S OWN CEILING, NOT OURS, and it is why a backfill is many reports rather than one.
153
  MAX_WINDOW_DAYS = 30
 
 
 
 
 
 
 
 
 
 
154
 
155
  # Connector metadata, NOT measured Amazon fields. TSV normalization strips leading underscores;
156
  # JSON flattening rejects these exact reserved keys. Neither path can overwrite metadata.
@@ -447,7 +462,7 @@ def _cell(value, kind):
447
  return None
448
 
449
 
450
- def _ensure(con, table, cols, rows=None, declared=None):
451
  """Create or widen the mirror table. A column whose type is now wrong is REBUILT, not patched.
452
 
453
  ⚠ Same ruling as `meta_store._ensure`: DuckDB cannot retype in place, and this is derived data
@@ -479,7 +494,15 @@ def _ensure(con, table, cols, rows=None, declared=None):
479
  clash = [c for c, ty in want.items()
480
  if c in have and have[c] != ty
481
  and not (have[c].startswith("VARCHAR") and ty == "VARCHAR")]
482
- if clash:
 
 
 
 
 
 
 
 
483
  con.execute(f"DROP TABLE {table}")
484
  have = {}
485
  if not have:
@@ -508,6 +531,47 @@ def _replace(con, table, cols, types, rows):
508
  return len(rows)
509
 
510
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
511
  def _windows(start, end, size=MAX_WINDOW_DAYS):
512
  """Slice a backfill into report-sized windows. Amazon's ceiling, reported not hidden."""
513
  out, cur = [], start
@@ -518,6 +582,49 @@ def _windows(start, end, size=MAX_WINDOW_DAYS):
518
  return out
519
 
520
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
521
  def _iso(dt):
522
  return dt.astimezone(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
523
 
@@ -1071,6 +1178,94 @@ def _read_state(con, key):
1071
  return {"state": row[0], "rows": row[1], "updated_at": row[2], "detail": row[3]}
1072
 
1073
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1074
  def _sync_event(kind, tenant, table, detail, rt, report, log):
1075
  """Sync is nondestructive: preserve C2's loud failure, but continue the data pull."""
1076
  try:
@@ -1108,7 +1303,7 @@ def _sync_problem(exc, table, operation, tenant, rt, report, log):
1108
 
1109
 
1110
  def sync(tenant_key="royal-imports", tables=None, since=None, log=print, rt=None,
1111
- max_wait=900, *, scheduled=False):
1112
  """Pull every declared table into the tenant's mirror. -> a report dict.
1113
 
1114
  Idempotent: a re-sync replaces each window rather than appending. Refuses, loudly and with the
@@ -1142,7 +1337,6 @@ def sync(tenant_key="royal-imports", tables=None, since=None, log=print, rt=None
1142
 
1143
  want = [t for t in (TABLES if tables is None else tables) if t in TABLES]
1144
  end = datetime.now(timezone.utc)
1145
- start = since or (end - timedelta(days=DEFAULT_BACKFILL_DAYS))
1146
  report = {"tenant": tenant_key, "tables": {}, "started": _iso(end),
1147
  "problems": [], "audit_problems": []}
1148
  enablement = read_database_enablement(rt)
@@ -1171,6 +1365,16 @@ def sync(tenant_key="royal-imports", tables=None, since=None, log=print, rt=None
1171
  if obligation["earliestTriggerAt"]:
1172
  con.close()
1173
  raise RuntimeError("Amazon deletion is outstanding; sync cannot repopulate purged data")
 
 
 
 
 
 
 
 
 
 
1174
  if scheduled:
1175
  plan = sync_plan({table: _read_state(con, entity_key("sync_tick_" + table))
1176
  for table in TABLES}, want, now=end)
@@ -1179,9 +1383,60 @@ def sync(tenant_key="royal-imports", tables=None, since=None, log=print, rt=None
1179
  report["tables"][table] = {"state": "deferred", "reason": reason}
1180
  log(f"{table}: DEFERRED: {reason}")
1181
 
1182
- for table in want:
1183
- spec = cx.REPORT_TYPES.get(table) or {}
1184
- rtype = spec.get("report")
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1185
  key, mirror = entity_key(table), table_name(table)
1186
  if not rtype:
1187
  report["tables"][table] = {"rows": 0, "state": "no confirmed bulk report"}
@@ -1189,15 +1444,89 @@ def sync(tenant_key="royal-imports", tables=None, since=None, log=print, rt=None
1189
  continue
1190
  prov = amazon_schema.PROVENANCE.get(table, "measured_at_sync")
1191
  root = (amazon_schema.ARTIFACTS.get(table) or (None, None))[1]
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1192
  try:
1193
  # Attempt first, so a refusal or process interruption cannot starve the other tables.
1194
- _record_state(con, entity_key("sync_tick_" + table), "attempted")
 
 
 
 
1195
  written, cols, cleared = 0, [], False
1196
  pricing_asins = []
1197
- for w0, w1 in _windows(start, end):
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1198
  # W59-T43 / C2 has no amazon.access kind. Record each credential-backed
1199
  # vendor attempt as a sync ACCESS phase, distinct from the completion event.
1200
  # Lane F/combine own the vocabulary decision; no credential or row enters C2.
 
1201
  _sync_event("amazon.sync", tenant_key, table,
1202
  {"state": "access", "operation": "report_document"}, rt, report, log)
1203
  text, meta = cx.report_document(
@@ -1237,7 +1566,25 @@ def sync(tenant_key="royal-imports", tables=None, since=None, log=print, rt=None
1237
  text, document_id=meta.get("reportDocumentId"), report_type=rtype)
1238
  declared = {}
1239
  if not cols:
1240
- continue
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1241
  if table == "orders":
1242
  rows = _dedupe_orders(rows)
1243
  if table == "inventory":
@@ -1246,16 +1593,32 @@ def sync(tenant_key="royal-imports", tables=None, since=None, log=print, rt=None
1246
  cols.append("inventory_source")
1247
  if table == "pricing":
1248
  pricing_asins.extend(_pricing_asins(rows))
1249
- types = _ensure(con, mirror, cols, rows, declared=declared)
1250
- # Clear once, after the table exists. The old multi-window branch deleted nothing,
1251
- # so every backfill appended a second copy of every historical row.
1252
- if not cleared:
1253
  con.execute(f"DELETE FROM {mirror} WHERE 1=1")
1254
  cleared = True
1255
- written += _replace(con, mirror, cols, types, rows)
 
 
 
 
 
 
 
 
 
 
 
1256
 
1257
- operations = {}
1258
  if table == "pricing":
 
 
 
 
 
 
1259
  state_key = ENTITY_STATE_KEYS[table]
1260
  if not pricing_asins:
1261
  _record_state(con, state_key, "unproven", 0,
@@ -1291,11 +1654,11 @@ def sync(tenant_key="royal-imports", tables=None, since=None, log=print, rt=None
1291
  awd_rows = _awd_rows(inventory)
1292
  if awd_rows:
1293
  awd_cols = sorted({c for row in awd_rows for c in row})
1294
- awd_types = _ensure(con, mirror, awd_cols, awd_rows)
1295
- if not cleared:
1296
  con.execute(f"DELETE FROM {mirror} WHERE 1=1")
1297
  cleared = True
1298
- written += _replace(con, mirror, awd_cols, awd_types, awd_rows)
1299
  _record_state(con, state_key, "done", len(awd_rows))
1300
  operations["listInventory"] = {
1301
  "state": "done", "rows": len(awd_rows)}
@@ -1312,21 +1675,67 @@ def sync(tenant_key="royal-imports", tables=None, since=None, log=print, rt=None
1312
  column_count, total_rows = len(cols), written
1313
  report["tables"][table] = {
1314
  "rows": total_rows, "columns": column_count, "provenance": prov,
1315
- "report": rtype, "operations": operations}
 
1316
  _record_state(con, key, "done", total_rows)
1317
  _record_state(con, entity_key("sync_tick_" + table), "done", total_rows)
 
 
1318
  _sync_event("amazon.sync", tenant_key, table,
1319
  {"state": "done", "rows": total_rows,
1320
  "operations_ok": all(v["state"] == "done" for v in operations.values())},
1321
  rt, report, log)
1322
  log(f" {table:<16} {total_rows:>7,} rows {column_count:>3} cols {prov}")
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1323
  except Exception as e: # noqa: BLE001
 
1324
  # ⚠ ONE TABLE'S REFUSAL IS NOT THE SYNC'S. A 403 on brand_analytics is R5's UNPROVEN
1325
  # ticket answering itself, and it must not cost the six tables that did work.
1326
- problem = _sync_problem(e, table, "report_document", tenant_key, rt, report, log)
 
1327
  report["tables"][table] = {"rows": 0, **problem}
1328
  _sync_event("amazon.sync", tenant_key, table,
1329
  {"state": problem["state"], "rows": 0}, rt, report, log)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1330
  report["finished"] = _iso(datetime.now(timezone.utc))
1331
  con.close()
1332
  return report
 
70
  TABLES = ("orders", "order_items", "listings", "settlements", "inventory",
71
  "pricing", "brand_analytics")
72
 
73
+ #: Head -> lines: both tables read the same document; a second report buys no new rows.
74
+ SHARED_REPORT_SIBLINGS = {"orders": "order_items"}
75
+
76
  # C1 is a tenant-store document, not an API dependency. verify_amazon gates reader parity.
77
  ENABLEMENT_KEY = "amazon_connector_enablement"
78
  # Settlements are issued roughly fortnightly; spending a report on every hourly tick adds nothing.
 
152
  #: How far back a first sync reaches when the caller names no window. Amazon's flat-file order
153
  #: reports accept at most ~31 days per request, so a backfill is sliced (`_windows`).
154
  DEFAULT_BACKFILL_DAYS = int(os.environ.get("AMAZON_BACKFILL_DAYS") or 730)
155
+ #: Per-table marker whose detail is the ISO UTC end of the last successfully landed window.
156
+ WATERMARK_PREFIX = "sync_mark_"
157
  #: ⚠ AMAZON'S OWN CEILING, NOT OURS, and it is why a backfill is many reports rather than one.
158
  MAX_WINDOW_DAYS = 30
159
+ #: R18: the most recent month lands before any history, so a newly connected account has usable
160
+ #: rows in minutes instead of hours. One window, because Amazon's own ceiling is 30 days.
161
+ FIRST_LOAD_RECENT_DAYS = MAX_WINDOW_DAYS
162
+ #: F4: Amazon paces createReport (burst 15, one per 60 s, 5 s gap), so 7 tables x 25 windows is
163
+ #: about 2.7 h. A first load therefore runs in BOUNDED passes, or it would hold the resident
164
+ #: thread for hours and the Odoo resync behind it would never run.
165
+ FIRST_LOAD_REPORTS_PER_PASS = 15
166
+ BACKFILL_FLOOR_PREFIX = "sync_floor_"
167
+ FIRST_LOAD_END_KEY = "sync_first_load_end"
168
+ FIRST_LOAD_STATUS_KEY = "sync_first_load_status"
169
 
170
  # Connector metadata, NOT measured Amazon fields. TSV normalization strips leading underscores;
171
  # JSON flattening rejects these exact reserved keys. Neither path can overwrite metadata.
 
462
  return None
463
 
464
 
465
+ def _ensure(con, table, cols, rows=None, declared=None, *, preserve_rows=False):
466
  """Create or widen the mirror table. A column whose type is now wrong is REBUILT, not patched.
467
 
468
  ⚠ Same ruling as `meta_store._ensure`: DuckDB cannot retype in place, and this is derived data
 
494
  clash = [c for c, ty in want.items()
495
  if c in have and have[c] != ty
496
  and not (have[c].startswith("VARCHAR") and ty == "VARCHAR")]
497
+ if clash and preserve_rows:
498
+ # Settlement document markers preclude replay: widening must retain prior lines.
499
+ # The measured types are VARCHAR, BIGINT and DOUBLE; never narrow historical data.
500
+ for c in clash:
501
+ widened = "VARCHAR" if "VARCHAR" in (have[c], want[c]) else "DOUBLE"
502
+ if have[c] != widened:
503
+ con.execute(f'ALTER TABLE "{table}" ALTER COLUMN "{c}" TYPE {widened}')
504
+ have[c] = want[c] = widened
505
+ elif clash:
506
  con.execute(f"DROP TABLE {table}")
507
  have = {}
508
  if not have:
 
531
  return len(rows)
532
 
533
 
534
+ def replace_keys_for(table):
535
+ """Replace on declared identity excluding lineage, which changes on every fetch."""
536
+ return tuple(c for c in (amazon_schema.TABLE_SPECS.get(table) or {}).get("row_identity", ())
537
+ if c not in _ORIGIN_TYPES)
538
+
539
+
540
+ def replace_window(con, table, cols, types, rows, keys):
541
+ """Replace incoming key groups atomically, or report why only insertion is possible."""
542
+ missing = [c for c in keys if c not in cols]
543
+ if missing:
544
+ note = (f"{table}: window replacement unavailable; missing identity columns: "
545
+ f"{', '.join(missing)}; rows inserted; use a report carrying these columns")
546
+ return _replace(con, table, cols, types, rows), note
547
+ if not keys or not rows:
548
+ return _replace(con, table, cols, types, rows), None
549
+ quote = lambda name: '"' + str(name).replace('"', '""') + '"'
550
+ target = quote(table)
551
+ temporary = quote("amazon_window_keys_" + uuid.uuid4().hex)
552
+ columns = ", ".join(quote(c) for c in keys)
553
+ # Use the mirror's types and the insert's cell conversion so NULL and numeric keys agree.
554
+ incoming = list(dict.fromkeys(tuple(_cell(row.get(c), types.get(c, "VARCHAR"))
555
+ for c in keys) for row in rows))
556
+ con.execute(f"CREATE TEMP TABLE {temporary} AS SELECT {columns} FROM {target} WHERE FALSE")
557
+ try:
558
+ con.executemany(f"INSERT INTO {temporary} VALUES ({', '.join('?' for _ in keys)})",
559
+ incoming)
560
+ predicate = " AND ".join(
561
+ f'{target}.{quote(c)} IS NOT DISTINCT FROM "k".{quote(c)}' for c in keys)
562
+ con.execute("BEGIN TRANSACTION")
563
+ try:
564
+ con.execute(f'DELETE FROM {target} WHERE EXISTS (SELECT 1 FROM {temporary} AS "k" WHERE {predicate})')
565
+ count = _replace(con, table, cols, types, rows)
566
+ con.execute("COMMIT")
567
+ except Exception:
568
+ con.execute("ROLLBACK")
569
+ raise
570
+ return count, None
571
+ finally:
572
+ con.execute(f"DROP TABLE {temporary}")
573
+
574
+
575
  def _windows(start, end, size=MAX_WINDOW_DAYS):
576
  """Slice a backfill into report-sized windows. Amazon's ceiling, reported not hidden."""
577
  out, cur = [], start
 
582
  return out
583
 
584
 
585
+ def first_load_plan(tables, start, end, *, recent_days=None, size=MAX_WINDOW_DAYS):
586
+ """R18's request order: every recent month, then each table's history, oldest last.
587
+
588
+ Settlements have one discovery slot (None, None), never a createReport window.
589
+ This planner is pure; durable floors filter its fixed windows when a pass resumes.
590
+ """
591
+ start, end = _utc(start), _utc(end)
592
+ if start >= end:
593
+ return []
594
+ size = min(float(size), MAX_WINDOW_DAYS)
595
+ recent_days = FIRST_LOAD_RECENT_DAYS if recent_days is None else recent_days
596
+ recent_days = min(float(recent_days), size)
597
+ if size <= 0 or recent_days <= 0:
598
+ raise ValueError("Amazon report windows must have a positive size.")
599
+ recent_start = max(start, end - timedelta(days=recent_days))
600
+ tables = list(dict.fromkeys(tables))
601
+ recent = [(table, None, None) if table == "settlements" else (table, recent_start, end)
602
+ for table in tables]
603
+ history = [(table, w0, w1) for table in tables if table != "settlements"
604
+ for w0, w1 in reversed(_windows(start, recent_start, size))]
605
+ return recent + history
606
+
607
+
608
+ def first_load_eta(report_count, *, stagger_seconds=None, burst=None, per_minute=None):
609
+ """Amazon createReport pacing floor in seconds; polling and other work can add time."""
610
+ from harness.amazon_client import REPORT_STAGGER_SECONDS, REPORT_BURST, REPORT_SUSTAINED_SECONDS
611
+ if report_count <= 0:
612
+ return 0
613
+ stagger = REPORT_STAGGER_SECONDS if stagger_seconds is None else stagger_seconds
614
+ tokens = REPORT_BURST if burst is None else burst
615
+ rate = 60 / REPORT_SUSTAINED_SECONDS if per_minute is None else per_minute
616
+ seconds = max(0, (report_count - 1) * stagger, (report_count - tokens) * 60 / rate)
617
+ return int(seconds) + (seconds > int(seconds))
618
+
619
+
620
+ def first_load_eta_cause(report_count):
621
+ """Screen-safe explanation of the vendor ceiling and the automatic continuation."""
622
+ from harness.amazon_client import REPORT_BURST, REPORT_SUSTAINED_SECONDS
623
+ return (f"Amazon's report rate allows a burst of {REPORT_BURST}, then one report every "
624
+ f"{REPORT_SUSTAINED_SECONDS:g} seconds. Keep the connection enabled while history "
625
+ "loads automatically. Report processing and other syncs can add time.")
626
+
627
+
628
  def _iso(dt):
629
  return dt.astimezone(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
630
 
 
1178
  return {"state": row[0], "rows": row[1], "updated_at": row[2], "detail": row[3]}
1179
 
1180
 
1181
+ def settlement_is_new(con, document_id):
1182
+ """A completed document is durable even when it contained no mirror lines."""
1183
+ state = _read_state(con, entity_key("settlement_doc_" + str(document_id)))
1184
+ return not state or state["state"] != "done"
1185
+
1186
+
1187
+ def read_watermark(con, table):
1188
+ """Missing or malformed detail means no successful window, never the current time."""
1189
+ state = _read_state(con, entity_key(WATERMARK_PREFIX + table))
1190
+ detail = (state or {}).get("detail")
1191
+ if not detail:
1192
+ return None
1193
+ try:
1194
+ return _utc(detail)
1195
+ except (ValueError, TypeError, AttributeError, OverflowError):
1196
+ return None
1197
+
1198
+
1199
+ def record_watermark(con, table, when, rows=0):
1200
+ """Persist only after this table's pass succeeds."""
1201
+ _record_state(con, entity_key(WATERMARK_PREFIX + table), "done", rows, _iso(_utc(when)))
1202
+
1203
+
1204
+ def read_backfill_floor(con, table):
1205
+ """Oldest contiguous landed window; missing or corrupt markers restart safely."""
1206
+ state = _read_state(con, entity_key(BACKFILL_FLOOR_PREFIX + table))
1207
+ try:
1208
+ return _utc(state["detail"]) if state and state.get("detail") else None
1209
+ except (ValueError, TypeError, AttributeError, OverflowError):
1210
+ return None
1211
+
1212
+
1213
+ def record_backfill_floor(con, table, when):
1214
+ _record_state(con, entity_key(BACKFILL_FLOOR_PREFIX + table), "done", detail=_iso(_utc(when)))
1215
+
1216
+
1217
+ def needs_first_load(tenant_key, rt=None):
1218
+ """A fresh account or an unfinished durable load uses bounded passes, even after one table finishes."""
1219
+ con, _path = _connect_tenant_mirror(tenant_key, rt)
1220
+ try:
1221
+ load = _read_state(con, entity_key(FIRST_LOAD_END_KEY))
1222
+ return bool(load and load["state"] != "done") or not any(
1223
+ read_watermark(con, table) for table in TABLES)
1224
+ finally:
1225
+ con.close()
1226
+
1227
+
1228
+ def first_load_status(tenant_key, rt=None):
1229
+ """Durable numeric progress for the API's safe-status projection; no vendor payloads."""
1230
+ con, _path = _connect_tenant_mirror(tenant_key, rt)
1231
+ try:
1232
+ state = _read_state(con, entity_key(FIRST_LOAD_STATUS_KEY))
1233
+ if not state:
1234
+ return None
1235
+ values = json.loads(state["detail"])
1236
+ return {"state": state["state"], "remaining": int(values["remaining"]),
1237
+ "etaSeconds": int(values["etaSeconds"]), "backfillDays": DEFAULT_BACKFILL_DAYS,
1238
+ "cause": first_load_eta_cause(values["remaining"])}
1239
+ finally:
1240
+ con.close()
1241
+
1242
+
1243
+ def sync_window(table, mark, *, now, since=None, backfill_days=None):
1244
+ """Pure request decision for one table: start, end, report, incremental and cause."""
1245
+ end = _utc(now)
1246
+ days = DEFAULT_BACKFILL_DAYS if backfill_days is None else backfill_days
1247
+ try:
1248
+ mark = _utc(mark) if mark else None
1249
+ except (ValueError, TypeError, AttributeError, OverflowError):
1250
+ mark = None
1251
+ if since is not None:
1252
+ start, incremental = _utc(since), False
1253
+ cause = f"caller named the window starting {_iso(start)}"
1254
+ elif mark is not None:
1255
+ start, incremental = mark, True
1256
+ cause = f"incremental window since watermark {_iso(mark)}"
1257
+ else:
1258
+ start, incremental = end - timedelta(days=days), False
1259
+ cause = f"first full load: {days} days of backfill"
1260
+ spec = _client().REPORT_TYPES.get(table) or {}
1261
+ rtype = spec.get("report_incremental") if incremental and spec.get("report_incremental") else spec.get("report")
1262
+ if table == "settlements":
1263
+ # The watermark filters issued reports by creation time; it never requests one.
1264
+ rtype = spec.get("report_discover")
1265
+ return {"start": start, "end": end, "report": rtype,
1266
+ "incremental": incremental, "cause": cause}
1267
+
1268
+
1269
  def _sync_event(kind, tenant, table, detail, rt, report, log):
1270
  """Sync is nondestructive: preserve C2's loud failure, but continue the data pull."""
1271
  try:
 
1303
 
1304
 
1305
  def sync(tenant_key="royal-imports", tables=None, since=None, log=print, rt=None,
1306
+ max_wait=900, *, scheduled=False, first_load=False, report_budget=None):
1307
  """Pull every declared table into the tenant's mirror. -> a report dict.
1308
 
1309
  Idempotent: a re-sync replaces each window rather than appending. Refuses, loudly and with the
 
1337
 
1338
  want = [t for t in (TABLES if tables is None else tables) if t in TABLES]
1339
  end = datetime.now(timezone.utc)
 
1340
  report = {"tenant": tenant_key, "tables": {}, "started": _iso(end),
1341
  "problems": [], "audit_problems": []}
1342
  enablement = read_database_enablement(rt)
 
1365
  if obligation["earliestTriggerAt"]:
1366
  con.close()
1367
  raise RuntimeError("Amazon deletion is outstanding; sync cannot repopulate purged data")
1368
+ load = _read_state(con, entity_key(FIRST_LOAD_END_KEY))
1369
+ if scheduled and load and load["state"] != "done":
1370
+ # The pinned inline producer still calls sync(scheduled=True). Do not let it start an
1371
+ # unbounded history pull between the resident writer's bounded first-load passes.
1372
+ for table in want:
1373
+ report["tables"][table] = {"state": "deferred", "reason":
1374
+ "The first load is continuing in bounded passes on the resident sync thread."}
1375
+ report["finished"] = _iso(datetime.now(timezone.utc))
1376
+ con.close()
1377
+ return report
1378
  if scheduled:
1379
  plan = sync_plan({table: _read_state(con, entity_key("sync_tick_" + table))
1380
  for table in TABLES}, want, now=end)
 
1383
  report["tables"][table] = {"state": "deferred", "reason": reason}
1384
  log(f"{table}: DEFERRED: {reason}")
1385
 
1386
+ # Heads go first even when the caller listed lines first. Scheduled plans still select one.
1387
+ pending = list(dict.fromkeys([head for head, sibling in SHARED_REPORT_SIBLINGS.items()
1388
+ if head in want and sibling in want] + want))
1389
+ handled, noted = set(), set()
1390
+
1391
+ def write_window(table, cols, types, rows):
1392
+ count, note = replace_window(con, table_name(table), cols, types, rows,
1393
+ replace_keys_for(table))
1394
+ if note and table not in noted:
1395
+ noted.add(table)
1396
+ report["problems"].append(note)
1397
+ log(note)
1398
+ return count
1399
+
1400
+ work, plans, paired, contexts = [], {}, {}, {}
1401
+ documents, settlement_remaining = 0, 0
1402
+ failed, landed = set(), set()
1403
+ if first_load:
1404
+ # Freeze the cohort end across passes. Recomputing it would move every window boundary.
1405
+ end = _utc(load["detail"]) if load else _utc(_iso(end))
1406
+ _record_state(con, entity_key(FIRST_LOAD_END_KEY), "running", detail=_iso(end))
1407
+ plans = {table: sync_window(table, None, now=end) for table in pending}
1408
+ floors = {table: read_backfill_floor(con, table) for table in pending}
1409
+ wanted = [table for table in pending if not read_watermark(con, table)
1410
+ or read_watermark(con, table) < end]
1411
+ for task in first_load_plan(wanted, end - timedelta(days=DEFAULT_BACKFILL_DAYS), end):
1412
+ table, w0, w1 = task
1413
+ if w1 is None or floors[table] is None or w1 <= floors[table]:
1414
+ work.append(task)
1415
+ # A failed enrichment after the final report retries completion without fetching again.
1416
+ for table in wanted:
1417
+ if not any(task[0] == table for task in work):
1418
+ work.append((table, None, None))
1419
+ # Reuse a document for head and lines only when BOTH durable cursors agree.
1420
+ for head, sibling in SHARED_REPORT_SIBLINGS.items():
1421
+ head_tasks = [(w0, w1) for table, w0, w1 in work if table == head]
1422
+ line_tasks = [(w0, w1) for table, w0, w1 in work if table == sibling]
1423
+ if head_tasks and head_tasks == line_tasks:
1424
+ paired[head] = sibling
1425
+ work = [task for task in work if task[0] != sibling]
1426
+ else:
1427
+ work = [(table, None, None) for table in pending]
1428
+
1429
+ for index, (table, task_start, task_end) in enumerate(work):
1430
+ if first_load and table in failed:
1431
+ continue
1432
+ if (first_load and isinstance(report_budget, int) and documents >= max(0, report_budget)
1433
+ and (task_end is not None or table == "settlements")):
1434
+ break
1435
+ if table in handled:
1436
+ continue
1437
+ plan = plans[table] if first_load else sync_window(
1438
+ table, read_watermark(con, table), now=end, since=since)
1439
+ rtype = plan["report"]
1440
  key, mirror = entity_key(table), table_name(table)
1441
  if not rtype:
1442
  report["tables"][table] = {"rows": 0, "state": "no confirmed bulk report"}
 
1444
  continue
1445
  prov = amazon_schema.PROVENANCE.get(table, "measured_at_sync")
1446
  root = (amazon_schema.ARTIFACTS.get(table) or (None, None))[1]
1447
+ log(f"{table}: {plan['cause']}")
1448
+ sibling = SHARED_REPORT_SIBLINGS.get(table)
1449
+ sibling_plan = None
1450
+ if first_load:
1451
+ sibling = paired.get(table)
1452
+ sibling_plan = plans.get(sibling)
1453
+ elif sibling in want and sibling not in handled:
1454
+ sibling_plan = sync_window(sibling, read_watermark(con, sibling), now=end, since=since)
1455
+ differs = [part for part in ("start", "end", "report")
1456
+ if plan[part] != sibling_plan[part]]
1457
+ if differs:
1458
+ log(f"{table} and {sibling}: cannot share this pass; differing {', '.join(differs)}")
1459
+ sibling = None
1460
+ else:
1461
+ handled.add(sibling)
1462
+ log(f"{sibling}: {sibling_plan['cause']}")
1463
+ else:
1464
+ sibling = None
1465
+ operation = "report_document"
1466
  try:
1467
  # Attempt first, so a refusal or process interruption cannot starve the other tables.
1468
+ if not first_load:
1469
+ _record_state(con, entity_key("sync_tick_" + table), "attempted")
1470
+ if sibling and not first_load:
1471
+ _record_state(con, entity_key("sync_tick_" + sibling), "attempted")
1472
+ sibling_written, sibling_cleared = 0, False
1473
  written, cols, cleared = 0, [], False
1474
  pricing_asins = []
1475
+ operations = {}
1476
+ if first_load:
1477
+ cleared = floors[table] is not None
1478
+ sibling_cleared = bool(sibling and floors[sibling] is not None)
1479
+ pricing_asins = contexts.setdefault(table, [])
1480
+ if table == "settlements":
1481
+ operation = "getReports"
1482
+ _sync_event("amazon.sync", tenant_key, table,
1483
+ {"state": "access", "operation": operation}, rt, report, log)
1484
+ discovered = cx.settlement_reports(
1485
+ fields, since=_iso(plan["start"]) if plan["incremental"] else None,
1486
+ log=lambda message: log(f"settlements: {message}"))
1487
+ downloaded, skipped = 0, 0
1488
+ for issued_index, issued in enumerate(discovered):
1489
+ document_id = str(issued.get("reportDocumentId") or "")
1490
+ if not document_id:
1491
+ raise cx.AmazonError("Settlement discovery returned no document ID.")
1492
+ if not settlement_is_new(con, document_id):
1493
+ skipped += 1
1494
+ continue
1495
+ if first_load and isinstance(report_budget, int) and documents >= max(0, report_budget):
1496
+ settlement_remaining = sum(settlement_is_new(con, str(item.get("reportDocumentId") or ""))
1497
+ for item in discovered[issued_index:])
1498
+ break
1499
+ operation = "getReportDocument"
1500
+ documents += 1
1501
+ _sync_event("amazon.sync", tenant_key, table,
1502
+ {"state": "access", "operation": operation}, rt, report, log)
1503
+ text = cx.settlement_document(fields, document_id)
1504
+ cols, rows = parse_flat_file(text, document_id=document_id, report_type=rtype)
1505
+ count = 0
1506
+ if cols and rows:
1507
+ types = _ensure(con, mirror, cols, rows, preserve_rows=True)
1508
+ # Never clear settlement history: durable markers prevent its replay.
1509
+ # A newly issued document still replaces its own settlement's lines.
1510
+ count = write_window(table, cols, types, rows)
1511
+ written += count
1512
+ # Record completion after the write, including header-only and blank files.
1513
+ _record_state(con, entity_key("settlement_doc_" + document_id), "done",
1514
+ count, issued.get("dataEndTime"))
1515
+ downloaded += 1
1516
+ operations["getReports"] = {
1517
+ "state": "done", "reports": len(discovered),
1518
+ "downloaded": downloaded, "skipped": skipped}
1519
+ if first_load and settlement_remaining:
1520
+ break
1521
+ windows = ()
1522
+ if table != "settlements":
1523
+ windows = (([(task_start, task_end)] if task_end is not None else []) if first_load
1524
+ else _windows(plan["start"], plan["end"]))
1525
+ for w0, w1 in (() if table == "settlements" else windows):
1526
  # W59-T43 / C2 has no amazon.access kind. Record each credential-backed
1527
  # vendor attempt as a sync ACCESS phase, distinct from the completion event.
1528
  # Lane F/combine own the vocabulary decision; no credential or row enters C2.
1529
+ documents += 1
1530
  _sync_event("amazon.sync", tenant_key, table,
1531
  {"state": "access", "operation": "report_document"}, rt, report, log)
1532
  text, meta = cx.report_document(
 
1566
  text, document_id=meta.get("reportDocumentId"), report_type=rtype)
1567
  declared = {}
1568
  if not cols:
1569
+ if first_load:
1570
+ # Even an empty recent window clears old rows exactly once.
1571
+ for empty_table in (table, sibling):
1572
+ if empty_table and floors[empty_table] is None:
1573
+ if con.execute("SELECT 1 FROM information_schema.tables WHERE table_name=?",
1574
+ [table_name(empty_table)]).fetchone():
1575
+ con.execute(f"DELETE FROM {table_name(empty_table)} WHERE 1=1")
1576
+ cleared = True
1577
+ else:
1578
+ continue
1579
+ if sibling:
1580
+ sibling_mirror = table_name(sibling)
1581
+ sibling_types = _ensure(con, sibling_mirror, cols, rows, declared=declared,
1582
+ preserve_rows=first_load) if cols else {}
1583
+ if cols and not sibling_cleared and not sibling_plan["incremental"]:
1584
+ con.execute(f"DELETE FROM {sibling_mirror} WHERE 1=1")
1585
+ sibling_cleared = True
1586
+ if cols:
1587
+ sibling_written += write_window(sibling, cols, sibling_types, rows)
1588
  if table == "orders":
1589
  rows = _dedupe_orders(rows)
1590
  if table == "inventory":
 
1593
  cols.append("inventory_source")
1594
  if table == "pricing":
1595
  pricing_asins.extend(_pricing_asins(rows))
1596
+ types = _ensure(con, mirror, cols, rows, declared=declared,
1597
+ preserve_rows=first_load) if cols else {}
1598
+ # Explicit windows and first loads replace the mirror; incremental passes keep history.
1599
+ if not cleared and not plan["incremental"]:
1600
  con.execute(f"DELETE FROM {mirror} WHERE 1=1")
1601
  cleared = True
1602
+ if cols:
1603
+ written += write_window(table, cols, types, rows)
1604
+ if first_load:
1605
+ for reached in (table, sibling):
1606
+ if reached:
1607
+ record_backfill_floor(con, reached, w0)
1608
+ floors[reached] = w0
1609
+
1610
+ if first_load:
1611
+ landed.add((table, task_start, task_end))
1612
+ if any(next_table == table for next_table, _w0, _w1 in work[index + 1:]):
1613
+ continue
1614
 
 
1615
  if table == "pricing":
1616
+ if first_load:
1617
+ # Completion can be in another process/pass than the recent ASIN report.
1618
+ available = {row[1] for row in con.execute(f"PRAGMA table_info('{mirror}')").fetchall()}
1619
+ pricing_asins = list(dict.fromkeys(pricing_asins + [str(row[0])
1620
+ for col in ("child_asin", "parent_asin") if col in available
1621
+ for row in con.execute(f'SELECT DISTINCT "{col}" FROM {mirror} WHERE "{col}" IS NOT NULL').fetchall()]))
1622
  state_key = ENTITY_STATE_KEYS[table]
1623
  if not pricing_asins:
1624
  _record_state(con, state_key, "unproven", 0,
 
1654
  awd_rows = _awd_rows(inventory)
1655
  if awd_rows:
1656
  awd_cols = sorted({c for row in awd_rows for c in row})
1657
+ awd_types = _ensure(con, mirror, awd_cols, awd_rows, preserve_rows=first_load)
1658
+ if not cleared and not plan["incremental"]:
1659
  con.execute(f"DELETE FROM {mirror} WHERE 1=1")
1660
  cleared = True
1661
+ written += write_window(table, awd_cols, awd_types, awd_rows)
1662
  _record_state(con, state_key, "done", len(awd_rows))
1663
  operations["listInventory"] = {
1664
  "state": "done", "rows": len(awd_rows)}
 
1675
  column_count, total_rows = len(cols), written
1676
  report["tables"][table] = {
1677
  "rows": total_rows, "columns": column_count, "provenance": prov,
1678
+ "report": rtype, "window": plan["cause"], "incremental": plan["incremental"],
1679
+ "operations": operations}
1680
  _record_state(con, key, "done", total_rows)
1681
  _record_state(con, entity_key("sync_tick_" + table), "done", total_rows)
1682
+ if not any(v["state"] in ("refused", "throttled") for v in operations.values()):
1683
+ record_watermark(con, table, plan["end"], total_rows)
1684
  _sync_event("amazon.sync", tenant_key, table,
1685
  {"state": "done", "rows": total_rows,
1686
  "operations_ok": all(v["state"] == "done" for v in operations.values())},
1687
  rt, report, log)
1688
  log(f" {table:<16} {total_rows:>7,} rows {column_count:>3} cols {prov}")
1689
+ if sibling:
1690
+ sibling_mirror = table_name(sibling)
1691
+ try:
1692
+ sibling_columns = len(con.execute(f"PRAGMA table_info('{sibling_mirror}')").fetchall())
1693
+ sibling_total = int(con.execute(f"SELECT COUNT(*) FROM {sibling_mirror}").fetchone()[0])
1694
+ except Exception:
1695
+ sibling_columns, sibling_total = len(cols), sibling_written
1696
+ sibling_prov = amazon_schema.PROVENANCE.get(sibling, "measured_at_sync")
1697
+ report["tables"][sibling] = {
1698
+ "rows": sibling_total, "columns": sibling_columns, "provenance": sibling_prov,
1699
+ "report": rtype, "window": sibling_plan["cause"],
1700
+ "incremental": sibling_plan["incremental"], "operations": {}}
1701
+ _record_state(con, entity_key(sibling), "done", sibling_total)
1702
+ _record_state(con, entity_key("sync_tick_" + sibling), "done", sibling_total)
1703
+ record_watermark(con, sibling, sibling_plan["end"], sibling_total)
1704
+ _sync_event("amazon.sync", tenant_key, sibling,
1705
+ {"state": "done", "rows": sibling_total, "operations_ok": True},
1706
+ rt, report, log)
1707
+ log(f" {sibling:<16} {sibling_total:>7,} rows {sibling_columns:>3} cols {sibling_prov}")
1708
  except Exception as e: # noqa: BLE001
1709
+ # A failed pass retains its previous watermark so its window will be retried.
1710
  # ⚠ ONE TABLE'S REFUSAL IS NOT THE SYNC'S. A 403 on brand_analytics is R5's UNPROVEN
1711
  # ticket answering itself, and it must not cost the six tables that did work.
1712
+ problem = _sync_problem(e, table, operation, tenant_key, rt, report, log)
1713
+ failed.add(table)
1714
  report["tables"][table] = {"rows": 0, **problem}
1715
  _sync_event("amazon.sync", tenant_key, table,
1716
  {"state": problem["state"], "rows": 0}, rt, report, log)
1717
+ if sibling:
1718
+ problem = _sync_problem(e, sibling, "report_document", tenant_key, rt, report, log)
1719
+ report["tables"][sibling] = {"rows": 0, **problem}
1720
+ _sync_event("amazon.sync", tenant_key, sibling,
1721
+ {"state": problem["state"], "rows": 0}, rt, report, log)
1722
+ if first_load:
1723
+ remaining = sum(task not in landed for task in work)
1724
+ if settlement_remaining:
1725
+ remaining += settlement_remaining - 1 # Replace the discovery slot with known documents.
1726
+ # Keep healthy tables moving after an isolated refusal. Once only failed work remains,
1727
+ # stop automatic continuation instead of endlessly retrying the same vendor refusal.
1728
+ continuable = any(task not in landed and task[0] not in failed for task in work)
1729
+ state = ("partial" if remaining and continuable else
1730
+ "failed" if remaining or report["problems"] or report["audit_problems"] else "done")
1731
+ progress = {"state": state, "remaining": remaining,
1732
+ "etaSeconds": first_load_eta(remaining, burst=0),
1733
+ "cause": first_load_eta_cause(remaining), "backfillDays": DEFAULT_BACKFILL_DAYS}
1734
+ report.update(progress)
1735
+ report["firstLoad"] = dict(progress)
1736
+ _record_state(con, entity_key(FIRST_LOAD_END_KEY), state, detail=_iso(end))
1737
+ _record_state(con, entity_key(FIRST_LOAD_STATUS_KEY), state,
1738
+ detail=json.dumps({key: progress[key] for key in ("remaining", "etaSeconds")}))
1739
  report["finished"] = _iso(datetime.now(timezone.utc))
1740
  con.close()
1741
  return report
platform/harness/measure_filter.py CHANGED
@@ -302,7 +302,7 @@ def _base_topic(key):
302
 
303
  def _related_scope(related, keys):
304
  """Prepare all bases before querying: a refused composite must be wholly unscoped."""
305
- if related is None or isinstance(related, _relations.MeasurePin):
306
  return related
307
  try:
308
  pin = _relations.measure_pin(related)
@@ -673,6 +673,10 @@ def resolve_values(key, window, today, team_id=None, allowed_pids=None, related=
673
  con = _sem._store_con()
674
  try:
675
  totals = _totals(con, key, window, today, team_id, related)
 
 
 
 
676
  finally:
677
  con.close()
678
  scale = ADMITTED.get(key, {}).get('input_scale') or 1.0
@@ -682,7 +686,8 @@ def resolve_values(key, window, today, team_id=None, allowed_pids=None, related=
682
  # round(x, None) returns an int — exactly the pool's `_round` convention for money/counts.
683
  # The pin narrows SQL input only. Keep the caller's entire pool for zero reconstruction.
684
  return _related_answer(
685
- {int(p): round(float(totals.get(p, 0.0)) * scale, nd) for p in universe}, related)
 
686
 
687
 
688
  #: The series channel's calendar buckets. `date_trunc` names in DuckDB; 'week' truncates to
 
302
 
303
  def _related_scope(related, keys):
304
  """Prepare all bases before querying: a refused composite must be wholly unscoped."""
305
+ if related is None or (isinstance(related, _relations.MeasurePin) and related.refusal):
306
  return related
307
  try:
308
  pin = _relations.measure_pin(related)
 
673
  con = _sem._store_con()
674
  try:
675
  totals = _totals(con, key, window, today, team_id, related)
676
+ guard = None
677
+ metric = _sem.metrics().get(key, {})
678
+ if related is not None and not related.refusal and metric.get('agg') == 'ratio':
679
+ guard = _totals(con, metric['denominator'], window, today, team_id, related)
680
  finally:
681
  con.close()
682
  scale = ADMITTED.get(key, {}).get('input_scale') or 1.0
 
686
  # round(x, None) returns an int — exactly the pool's `_round` convention for money/counts.
687
  # The pin narrows SQL input only. Keep the caller's entire pool for zero reconstruction.
688
  return _related_answer(
689
+ {int(p): round(float(totals.get(p, 0.0)) * scale, nd) for p in universe
690
+ if guard is None or guard.get(p)}, related)
691
 
692
 
693
  #: The series channel's calendar buckets. `date_trunc` names in DuckDB; 'week' truncates to
platform/harness/runtime.py CHANGED
@@ -126,6 +126,8 @@ class TenantRuntime:
126
  tenant: object # harness.base.Tenant
127
  store_namespace: str = ""
128
  pool_cache: dict = field(default_factory=dict)
 
 
129
  #: EXIT-5 — the API adapter's memos for `core.measure_resolve` (the Streamlit adapter keeps
130
  #: its own in st.session_state). Bounded by the module's own clear-past-cap rule; keyed on
131
  #: (stamp, scope, pool identity, question) so nothing user-shaped lives in them.
@@ -179,6 +181,29 @@ class TenantRuntime:
179
  """
180
  return self._store().get_projection(self.store_key(name), drop=drop)
181
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
182
  def revision(self, name):
183
  """Return the tiny durable change token for this tenant's addressed bucket."""
184
  return self._store().revision(self.store_key(name))
 
126
  tenant: object # harness.base.Tenant
127
  store_namespace: str = ""
128
  pool_cache: dict = field(default_factory=dict)
129
+ # Names/edges only, bounded and revision-keyed by routes_grid; no row pools live here.
130
+ related_cache: dict = field(default_factory=dict)
131
  #: EXIT-5 — the API adapter's memos for `core.measure_resolve` (the Streamlit adapter keeps
132
  #: its own in st.session_state). Bounded by the module's own clear-past-cap rule; keyed on
133
  #: (stamp, scope, pool identity, question) so nothing user-shaped lives in them.
 
181
  """
182
  return self._store().get_projection(self.store_key(name), drop=drop)
183
 
184
+ def get_projections(self, names, drop=(), timeout_s=None):
185
+ """Batch projected reads when the active bound store has a native batch operation.
186
+
187
+ ``None`` means the backend has no batch primitive; callers retain their ordinary per-key
188
+ path. That keeps this an optional optimization rather than widening the store contract.
189
+ """
190
+ reader = getattr(self._store(), "get_projections", None)
191
+ if not callable(reader):
192
+ return None
193
+ keyed = [(str(name), self.store_key(name)) for name in (names or ())]
194
+ answer = reader([store_key for _name, store_key in keyed], drop=drop,
195
+ timeout_s=timeout_s)
196
+ return {name: answer.get(store_key, {}) for name, store_key in keyed}
197
+
198
+ def get_many(self, names):
199
+ """Batch ordinary reads when the active bound store offers that optimization."""
200
+ reader = getattr(self._store(), "get_many", None)
201
+ if not callable(reader):
202
+ return None
203
+ keyed = [(str(name), self.store_key(name)) for name in (names or ())]
204
+ answer = reader([store_key for _name, store_key in keyed])
205
+ return {name: answer.get(store_key, {}) for name, store_key in keyed}
206
+
207
  def revision(self, name):
208
  """Return the tiny durable change token for this tenant's addressed bucket."""
209
  return self._store().revision(self.store_key(name))