fsanyoto commited on
Commit
2eae9f0
Β·
verified Β·
1 Parent(s): f131cb5

Deploy AIOS web (React glide grid + FastAPI slice)

Browse files
This view is limited to 50 files because it contains too many changes. Β  See raw diff
Files changed (50) hide show
  1. Dockerfile +15 -3
  2. RELEASES.json +1 -1
  3. VERSION +1 -1
  4. api/ai_enrich.py +6 -3
  5. api/amazon_relational.py +493 -35
  6. api/application_registry.py +72 -0
  7. api/assistant_mirror.py +240 -40
  8. api/connectors_amazon.py +201 -14
  9. api/content_studio.py +638 -0
  10. api/deps.py +232 -19
  11. api/formula_runtime.py +72 -3
  12. api/main.py +226 -35
  13. api/odoo_relational.py +491 -34
  14. api/routes_admin.py +6 -4
  15. api/routes_auth.py +14 -3
  16. api/routes_connected_tables.py +7 -3
  17. api/routes_connectors.py +337 -50
  18. api/routes_external.py +136 -25
  19. api/routes_keychain.py +62 -11
  20. api/routes_module_index.py +64 -5
  21. api/routes_nav.py +3 -3
  22. api/routes_odoo_tables.py +454 -47
  23. api/routes_platform_admin.py +57 -1
  24. api/routes_query.py +5 -0
  25. api/routes_sheet.py +881 -260
  26. api/routes_tables.py +1 -1
  27. api/routes_templates.py +132 -6
  28. platform/core/connector_spec.py +304 -1
  29. platform/core/daily_sales_repair.py +1146 -15
  30. platform/core/data_binding.py +64 -1
  31. platform/core/do_usage.py +583 -0
  32. platform/core/external_tools.py +55 -20
  33. platform/core/grid_events.py +60 -1
  34. platform/core/neon_usage.py +590 -0
  35. platform/core/perm_scope.py +74 -10
  36. platform/core/perms.py +11 -2
  37. platform/core/registry.py +10 -0
  38. platform/core/sheet_binding.py +60 -1
  39. platform/core/sheet_import.py +177 -4
  40. platform/core/sheet_structure.py +116 -16
  41. platform/core/sheets.py +104 -11
  42. platform/core/store_pg.py +133 -8
  43. platform/core/user_tables.py +67 -3
  44. platform/core/users.py +12 -2
  45. platform/harness/amazon_client.py +1057 -11
  46. platform/harness/amazon_schema.py +149 -8
  47. platform/harness/amazon_store.py +0 -0
  48. platform/harness/datastore.py +53 -4
  49. platform/harness/runtime.py +8 -0
  50. platform/harness/telemetry.py +7 -2
Dockerfile CHANGED
@@ -98,6 +98,18 @@ HEALTHCHECK --interval=30s --timeout=5s --start-period=60s --retries=3 \
98
  # '*' assumes the container is unreachable except through that trusted proxy, which must replace
99
  # client-supplied forwarded headers. For any direct ingress, set FORWARDED_ALLOW_IPS to the actual
100
  # proxy IPs/CIDRs before exposing the port; do not carry wildcard trust onto a public listener.
101
- # Suppress the server banner and release idle keep-alive sockets after five seconds. W59-T36
102
- # owns the API posture read-back and application host/header policy; these flags are its launcher seam.
103
- CMD ["sh", "-c", "uvicorn api.main:app --host 0.0.0.0 --port ${PORT} --proxy-headers --forwarded-allow-ips \"${FORWARDED_ALLOW_IPS}\" --no-server-header --timeout-keep-alive 5"]
 
 
 
 
 
 
 
 
 
 
 
 
 
98
  # '*' assumes the container is unreachable except through that trusted proxy, which must replace
99
  # client-supplied forwarded headers. For any direct ingress, set FORWARDED_ALLOW_IPS to the actual
100
  # proxy IPs/CIDRs before exposing the port; do not carry wildcard trust onto a public listener.
101
+ # Suppress the server banner. W59-T36 owns the API posture read-back and application host/header
102
+ # policy; these flags are its launcher seam.
103
+ #
104
+ # SDLC 67 close (2026-09-29): `--timeout-keep-alive 5` was the suspected cause of a measured
105
+ # 4-12% 502 rate at the HF proxy -- uvicorn closes an idle keep-alive socket after 5s, and if the
106
+ # proxy in front of it still believes that connection is reusable and forwards a request onto it,
107
+ # the client sees a 502 rather than a fresh connection (the classic reverse-proxy keep-alive race:
108
+ # https://adamcrowder.net/posts/node-express-api-and-aws-alb-502/ describes the same shape against
109
+ # an ALB). The fix is always the same direction -- make the BACKEND's idle timeout longer than
110
+ # whatever the front proxy holds a connection for, never shorter -- because only the side with the
111
+ # longer timeout can be the one to close first. HF's Spaces proxy timeout is not published; 75s is
112
+ # nginx's own keepalive_timeout default and comfortably exceeds the AWS ALB default (60s) other
113
+ # proxies in this class use, so it is the safest available floor without over-fitting to a number
114
+ # we cannot read. Re-aim this value again, never delete the flag, if a measurement finds otherwise.
115
+ CMD ["sh", "-c", "uvicorn api.main:app --host 0.0.0.0 --port ${PORT} --proxy-headers --forwarded-allow-ips \"${FORWARDED_ALLOW_IPS}\" --no-server-header --timeout-keep-alive 75"]
RELEASES.json CHANGED
@@ -1,4 +1,4 @@
1
  {
2
- "current": "37854d8928902b38cc4f5ddab2e2ddb0c9605671 (37854d8)",
3
  "releases": []
4
  }
 
1
  {
2
+ "current": "be8fb2f (be8fb2f)",
3
  "releases": []
4
  }
VERSION CHANGED
@@ -1 +1 @@
1
- 37854d8928902b38cc4f5ddab2e2ddb0c9605671 (37854d8)
 
1
+ be8fb2f (be8fb2f)
api/ai_enrich.py CHANGED
@@ -269,12 +269,15 @@ _SYSTEM = ('You fill in ONE cell of a spreadsheet. Answer with the value only: n
269
  'exactly: UNKNOWN')
270
 
271
 
272
- def _ask(provider, model, prompt, max_tokens, timeout):
273
  """One cell's answer. `(text, tokens_or_None, problem)`; a truthy `problem` means no text.
274
 
275
  ⚠ Both request shapes are `ai_review`'s, changed in exactly two ways: they carry the COLUMN's
276
  ceiling instead of a hardcoded 300, and they read `usage` back.
 
 
277
  """
 
278
  key = (os.environ.get(provider['env']) or '').strip()
279
  if not key:
280
  return '', None, f"{provider['name']} has no key on this deployment"
@@ -284,7 +287,7 @@ def _ask(provider, model, prompt, max_tokens, timeout):
284
  provider['url'], timeout=timeout,
285
  headers={'x-api-key': key, 'anthropic-version': '2023-06-01',
286
  'content-type': 'application/json'},
287
- json={'model': model, 'max_tokens': max_tokens, 'system': _SYSTEM,
288
  'messages': [{'role': 'user', 'content': prompt}]})
289
  if r.status_code >= 400:
290
  return '', None, f"{provider['name']} answered {r.status_code}"
@@ -300,7 +303,7 @@ def _ask(provider, model, prompt, max_tokens, timeout):
300
  provider['url'], timeout=timeout,
301
  headers={'Authorization': f"Bearer {key}", 'Content-Type': 'application/json'},
302
  json={'model': model, 'max_tokens': max_tokens, 'temperature': 0,
303
- 'messages': [{'role': 'system', 'content': _SYSTEM},
304
  {'role': 'user', 'content': prompt}]})
305
  if r.status_code >= 400:
306
  return '', None, f"{provider['name']} answered {r.status_code}"
 
269
  'exactly: UNKNOWN')
270
 
271
 
272
+ def _ask(provider, model, prompt, max_tokens, timeout, system=None):
273
  """One cell's answer. `(text, tokens_or_None, problem)`; a truthy `problem` means no text.
274
 
275
  ⚠ Both request shapes are `ai_review`'s, changed in exactly two ways: they carry the COLUMN's
276
  ceiling instead of a hardcoded 300, and they read `usage` back.
277
+ S67-T41: `system` replaces the one-cell instruction for a caller that is not filling a cell
278
+ (Content Studio's drafts); every existing caller passes nothing and keeps `_SYSTEM`.
279
  """
280
+ system = system or _SYSTEM
281
  key = (os.environ.get(provider['env']) or '').strip()
282
  if not key:
283
  return '', None, f"{provider['name']} has no key on this deployment"
 
287
  provider['url'], timeout=timeout,
288
  headers={'x-api-key': key, 'anthropic-version': '2023-06-01',
289
  'content-type': 'application/json'},
290
+ json={'model': model, 'max_tokens': max_tokens, 'system': system,
291
  'messages': [{'role': 'user', 'content': prompt}]})
292
  if r.status_code >= 400:
293
  return '', None, f"{provider['name']} answered {r.status_code}"
 
303
  provider['url'], timeout=timeout,
304
  headers={'Authorization': f"Bearer {key}", 'Content-Type': 'application/json'},
305
  json={'model': model, 'max_tokens': max_tokens, 'temperature': 0,
306
+ 'messages': [{'role': 'system', 'content': system},
307
  {'role': 'user', 'content': prompt}]})
308
  if r.status_code >= 400:
309
  return '', None, f"{provider['name']} answered {r.status_code}"
api/amazon_relational.py CHANGED
@@ -28,10 +28,8 @@ AMAZON TABLES: order lines to their order, pricing to its listing. **Nothing her
28
  removes the [[amazon-conversion-k-is-inverted]] defect class outright: no Odoo unit is ever
29
  converted to an Amazon unit, because the two never meet.
30
 
31
- ⚠ MATERIALISED, WITH THE CEILING REPORTED RATHER THAN ENFORCED. Same posture as Meta's: a table
32
- over `MAX_ROWS` is REFUSED with its cause and its fix named, never truncated. For Amazon that is
33
- standing rule 1's second half, and it is the likeliest table to hit it β€” an order-item history is
34
- one row per line, not per order.
35
  """
36
  import hashlib
37
  import json
@@ -45,7 +43,7 @@ if str(_API.parent.parent / "platform") not in sys.path:
45
 
46
  import odoo_relational as _odoo # noqa: E402
47
  from core import connector_spec # noqa: E402
48
- from harness import amazon_schema, amazon_store, datastore # noqa: E402
49
 
50
  Refused = _odoo.Refused
51
 
@@ -119,7 +117,7 @@ def source_attribution(key):
119
  #: Columns whose NAME makes them a date whatever they are stored as. Amazon reports ISO-8601.
120
  _DATEISH = ("purchase_date", "last_updated_date", "payments_date", "settlement_start_date",
121
  "settlement_end_date", "deposit_date", "posted_date", "open_date", "date",
122
- "report_date", "start_date", "end_date", "_time")
123
 
124
  #: β›” MONEY IS `currency`, NEVER `int`. An Amazon `item-price` is fractional in every marketplace,
125
  #: and an int column would round every row of a settlement. The grid shows what the report said and
@@ -130,6 +128,10 @@ _MONEY = {"item_price", "item_tax", "shipping_price", "shipping_tax", "gift_wrap
130
  "your_price", "landed_price", "listing_price", "open_date_price",
131
  "sales_by_asin_ordered_product_sales_amount",
132
  "sales_by_asin_ordered_product_sales_b2_b_amount"}
 
 
 
 
133
 
134
  #: Percentages and rates are fractional too, but they are not money. `currency` is the only
135
  #: fractional field type the grid offers, so they take it and the LABEL carries the meaning.
@@ -150,7 +152,7 @@ def _ftype(col, duck_type):
150
  """
151
  if any(col == s or col.endswith(s) for s in _DATEISH):
152
  return "date"
153
- if col in _MONEY or any(col.endswith(s) for s in _RATE_SUFFIX):
154
  return "currency"
155
  base = str(duck_type or "").upper().split("(")[0]
156
  if base in ("BIGINT", "INTEGER", "HUGEINT") or base.startswith("DECIMAL"):
@@ -168,6 +170,116 @@ def mirror_columns(cur, table):
168
  return []
169
 
170
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
171
  def _fields_for(cur, key):
172
  """One preset field per mirror column β€” the whole schema, derived. See the header."""
173
  cols = mirror_columns(cur, SOURCE[key])
@@ -175,14 +287,31 @@ def _fields_for(cur, key):
175
  return []
176
  names = {c[0] for c in cols}
177
  pinned = PINNED.get(key) if PINNED.get(key) in names else None
 
 
 
 
 
 
178
  out = []
179
- for name, dtype in cols:
 
 
 
 
180
  out.append(_odoo._preset({
181
  "key": name, "label": _label(name), "type": _ftype(name, dtype),
182
  "source": "overlay",
183
- "default": name == pinned or name in _DEFAULT_COLUMNS,
184
  "pinned": name == pinned,
185
  }, flow="amazon_relational"))
 
 
 
 
 
 
 
186
  # ⚠ BOTH SIDES MUST CARRY THE COLUMN, EVERY TIME. A `measured_at_sync` table's header is not
187
  # known until it syncs, so a link to a table that has never run would name a column that does
188
  # not exist. **Absent is a normal state here, not a failure** β€” which is exactly what lets the
@@ -205,7 +334,8 @@ def _fields_for(cur, key):
205
  out.append(_odoo._preset({
206
  "key": edge["key"], "label": edge["label"],
207
  "type": "link", "source": "overlay", "default": True,
208
- "link": {"table": edge["target"], "on": edge["to_col"], "from": edge["from_col"]},
 
209
  }, flow="amazon_relational"))
210
  out.append(_odoo._preset(_odoo._refreshed_field(), flow="amazon_relational"))
211
  return out
@@ -222,8 +352,128 @@ _DEFAULT_COLUMNS = {
222
  }
223
 
224
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
225
  def _row_id(row, key):
226
- """A STABLE id for one mirror row.
227
 
228
  ⭐ STABILITY IS THE WHOLE REQUIREMENT, and it is why a row number will not do. `_ensure_table_inplace`
229
  merges by `_id`, so an id that changes between syncs turns every re-sync into a full delete and
@@ -232,6 +482,24 @@ def _row_id(row, key):
232
  Falls back to a content hash rather than a counter for exactly that reason: a settlement line
233
  with no id column still hashes to the same value next week if its content has not changed.
234
  """
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
235
  for col in IDENTITY.get(key, ()):
236
  val = row.get(col)
237
  if val not in (None, ""):
@@ -242,22 +510,76 @@ def _row_id(row, key):
242
 
243
 
244
  def _read(cur, key):
245
- """[(row dict)] straight off the mirror, each carrying a stable `_id`."""
 
 
 
 
246
  cols = [c[0] for c in mirror_columns(cur, SOURCE[key])]
247
  if not cols:
248
  return []
249
  quoted = ", ".join(f'"{c}"' for c in cols)
250
- rows = cur.execute(f"SELECT {quoted} FROM {SOURCE[key]}").fetchall()
 
 
 
 
 
 
251
  out = []
252
  for r in rows:
253
  row = {}
254
- for name, val in zip(cols, r):
255
- row[name] = "" if val is None else (val if isinstance(val, (int, float)) else str(val))
256
- row["_id"] = _row_id(row, key)
 
 
257
  out.append(row)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
258
  return out
259
 
260
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
261
  def available(tenant_key="royal-imports"):
262
  """Has this tenant's mirror ever been fed by `amazon_store`? Cheap; never fetches."""
263
  try:
@@ -268,10 +590,8 @@ def available(tenant_key="royal-imports"):
268
 
269
 
270
  def plan(cur, rt=None):
271
- """{bucket_key: rows} plus `problems`. No store write; every cap checked BEFORE any commit."""
272
  built = {"problems": []}
273
- ut = _odoo._ut()
274
- cap = getattr(ut, "MAX_ROWS", 60_000)
275
  for key in SOURCE:
276
  # β›” COUNT BEFORE READ. `_read` materialises the whole table as python dicts, so checking
277
  # the cap afterwards means the one table too big to keep is the one built in memory first.
@@ -279,17 +599,8 @@ def plan(cur, rt=None):
279
  n = cur.execute(f"SELECT count(*) FROM {SOURCE[key]}").fetchone()[0]
280
  except Exception: # noqa: BLE001
281
  continue # absent = never synced
282
- if n > cap:
283
- # β›” REPORTED WITH ITS CAUSE AND ITS FIX, NEVER TRUNCATED. Standing rule 1's second
284
- # half, and `ut_amzn_order_items` is the table most likely to reach it: an order
285
- # history is one row per LINE, so a mid-size seller passes 60,000 in a year.
286
- built["problems"].append(
287
- f"{key}: the mirror holds {n:,} rows against MAX_ROWS={cap:,}. Nothing was "
288
- f"truncated and nothing was written for this table. Cause: this seller's history "
289
- f"is larger than a materialised ut_* table can hold. Fix: bind it read-through "
290
- f"off the mirror as `ut_odoo_gl_lines` is "
291
- f"(odoo_relational.READ_THROUGH_KEYS), which `routes_connected_tables` "
292
- f"already serves.")
293
  continue
294
  rows = _read(cur, key)
295
  if rows:
@@ -306,6 +617,9 @@ def plan_fingerprint(cur, built):
306
  """
307
  tables = []
308
  for key in SOURCE:
 
 
 
309
  rows = built.get(key) or []
310
  canonical = sorted(rows, key=lambda row: json.dumps(
311
  row, sort_keys=True, ensure_ascii=True, separators=(",", ":"), default=str))
@@ -315,6 +629,62 @@ def plan_fingerprint(cur, built):
315
  return hashlib.sha256(payload.encode("utf-8")).hexdigest()
316
 
317
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
318
  def apply_plan(rt, built, username="amazon", today=None):
319
  """Create-or-merge every table in `built`, in ONE `flush="sync"` write.
320
 
@@ -327,24 +697,103 @@ def apply_plan(rt, built, username="amazon", today=None):
327
  raise Refused("; ".join(built["problems"]))
328
  stamp = today or _odoo._iso_today()
329
  written = {}
330
- cur_mirror = datastore.ro_con()
331
- plans = [(key, LABELS[key], _fields_for(cur_mirror, key), built[key])
332
- for key in SOURCE if key in built]
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
333
 
334
  def _apply_all(doc):
335
  doc = doc if isinstance(doc, dict) else {}
 
336
  for key, label, fields, rows in plans:
337
- written[key] = _odoo._ensure_table_inplace(doc, key, label, fields, rows,
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
338
  username, stamp)
 
 
 
 
 
 
 
339
  # DPP 1.8 attribution is explicit in the stored definition, not inferred from a
340
  # mutable display label or from the process having imported this module.
341
  doc[key]["connector"] = source_attribution(key)
342
  return doc
343
 
344
  rt.update(_odoo._ut().STORE_KEY, _apply_all, flush="sync")
 
345
  return written
346
 
347
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
348
  def refresh_state(rt, why="boot", tenant_key=None, previous_fingerprint=None, log=print):
349
  """Open, plan and conditionally apply the Amazon mirror.
350
 
@@ -357,7 +806,14 @@ def refresh_state(rt, why="boot", tenant_key=None, previous_fingerprint=None, lo
357
  path = datastore.path_for(tenant)
358
  if Path(datastore.DB_PATH) != Path(path):
359
  datastore.use_path(path)
360
- cur = datastore.ro_con()
 
 
 
 
 
 
 
361
  if not any(mirror_columns(cur, t) for t in SOURCE.values()):
362
  log(f"[amazon_relational] {why}: no Amazon tables in {tenant}'s mirror, nothing to spawn "
363
  f"(run `python platform/harness/amazon_store.py --sync --tenant {tenant}` first)")
@@ -381,6 +837,8 @@ def refresh_state(rt, why="boot", tenant_key=None, previous_fingerprint=None, lo
381
  state["changed"] = True
382
  state["written"] = apply_plan(rt, applicable)
383
  state["applied"] = True
 
 
384
  for key, counts in state["written"].items():
385
  log(f"[amazon_relational] {why}: {key:<28} {counts}")
386
  return state
 
28
  removes the [[amazon-conversion-k-is-inverted]] defect class outright: no Odoo unit is ever
29
  converted to an Amazon unit, because the two never meet.
30
 
31
+ History stays in the mirror. Small snapshots may be materialised; a row or byte budget switches
32
+ them to read-through too. The shared document holds their schemas, never a truncated population.
 
 
33
  """
34
  import hashlib
35
  import json
 
43
 
44
  import odoo_relational as _odoo # noqa: E402
45
  from core import connector_spec # noqa: E402
46
+ from harness import amazon_client, amazon_schema, amazon_store, datastore # noqa: E402
47
 
48
  Refused = _odoo.Refused
49
 
 
117
  #: Columns whose NAME makes them a date whatever they are stored as. Amazon reports ISO-8601.
118
  _DATEISH = ("purchase_date", "last_updated_date", "payments_date", "settlement_start_date",
119
  "settlement_end_date", "deposit_date", "posted_date", "open_date", "date",
120
+ "report_date", "start_date", "end_date", "_snapshot_date", "month", "_time")
121
 
122
  #: β›” MONEY IS `currency`, NEVER `int`. An Amazon `item-price` is fractional in every marketplace,
123
  #: and an int column would round every row of a settlement. The grid shows what the report said and
 
128
  "your_price", "landed_price", "listing_price", "open_date_price",
129
  "sales_by_asin_ordered_product_sales_amount",
130
  "sales_by_asin_ordered_product_sales_b2_b_amount"}
131
+ _MONEY.update({"estimated_fee_total", "estimated_referral_fee_per_unit",
132
+ "estimated_variable_closing_fee", "estimated_order_handling_fee_per_order",
133
+ "estimated_pick_pack_fee_per_unit", "estimated_weight_handling_fee_per_unit",
134
+ "expected_fulfillment_fee_per_unit", "estimated_monthly_storage_fee"})
135
 
136
  #: Percentages and rates are fractional too, but they are not money. `currency` is the only
137
  #: fractional field type the grid offers, so they take it and the LABEL carries the meaning.
 
152
  """
153
  if any(col == s or col.endswith(s) for s in _DATEISH):
154
  return "date"
155
+ if col in _MONEY or col.endswith(("_amount", "_fee")) or any(col.endswith(s) for s in _RATE_SUFFIX):
156
  return "currency"
157
  base = str(duck_type or "").upper().split("(")[0]
158
  if base in ("BIGINT", "INTEGER", "HUGEINT") or base.startswith("DECIMAL"):
 
170
  return []
171
 
172
 
173
+ #: ⭐⭐ S67-T30 (instr 13, R29 / O23) β€” THE PRODUCT LOOKUPS. Every Amazon database with an ASIN
174
+ #: shows its "Parent ASIN" and "SKU" beside it, and every one with a SKU its "ASIN" and "Parent
175
+ #: ASIN", with no configuration. They are PLAIN TEXT fields whose value the SERVER derives, not
176
+ #: link fields, on purpose: a link renders as a count bubble, is not groupable
177
+ #: (`types.ts:isGroupableField`) and has no SQL binding for a filter (`routes_odoo_tables.
178
+ #: _filter_columns` and `_group_pin` read `spec["cols"]` only). A text field with a `cols`
179
+ #: expression groups, filters and sorts through the machinery every other column already uses.
180
+ #: ⚠ The ASIN / SKU column of a table is the `from_col` of its declared listings link (on `asin1`
181
+ #: / `seller_sku`), so the spec's links are the one statement of which column is which. A table
182
+ #: that carries its own `parent_asin` (pricing, economics) gains no derived Parent ASIN; one that
183
+ #: carries its own SKU or ASIN gains no duplicate of it; listings gains Parent ASIN only.
184
+ #: β›” Parent comes from `amazon_store.asin_parents_sql` ALONE (latest catalog, else economics, else
185
+ #: the ASIN itself), never from `amzn_catalog_items` directly: that table keeps older snapshot days
186
+ #: (D-3031) and five own ASINs have no catalog row. SKU and ASIN come from the LATEST listings day
187
+ #: only, because listings keeps one copy per snapshot date.
188
+ LISTINGS = PREFIX + "listings"
189
+ LOOKUP_PARENT, LOOKUP_SKU, LOOKUP_ASIN = "lookup_parent_asin", "lookup_sku", "lookup_asin"
190
+ LOOKUP_KEYS = frozenset((LOOKUP_PARENT, LOOKUP_SKU, LOOKUP_ASIN))
191
+ _LOOKUP_LABELS = {LOOKUP_PARENT: "Parent ASIN", LOOKUP_SKU: "SKU", LOOKUP_ASIN: "ASIN"}
192
+ _LOOKUP_DESCRIPTIONS = {
193
+ LOOKUP_PARENT: ("The parent ASIN of this row's product, from Amazon's catalog, else the "
194
+ "economics report, else the ASIN itself. Click it to open the listing."),
195
+ LOOKUP_SKU: ("Your seller SKUs for this row's ASIN in the latest listings report. "
196
+ "Click it to open the listing."),
197
+ LOOKUP_ASIN: ("The ASIN of this row's SKU in the latest listings report. "
198
+ "Click it to open the listing."),
199
+ }
200
+
201
+
202
+ def _product_columns(key):
203
+ """`(asin_col, sku_col, asin_link_key, sku_link_key)` as the spec declares them for `key`."""
204
+ if key == LISTINGS:
205
+ return "asin1", "seller_sku", None, None # the row IS the listing: Parent ASIN only
206
+ asin = sku = asin_via = sku_via = None
207
+ for edge in connector_spec.edges_for(key):
208
+ if edge["target"] != LISTINGS:
209
+ continue
210
+ if edge["to_col"] == "asin1" and asin is None:
211
+ asin, asin_via = edge["from_col"], edge["key"]
212
+ elif edge["to_col"] == "seller_sku" and sku is None:
213
+ sku, sku_via = edge["from_col"], edge["key"]
214
+ return asin, sku, asin_via, sku_via
215
+
216
+
217
+ def _latest_listings_sql(listing_names):
218
+ """`SELECT sku, asin` over the LATEST listings snapshot day, or None without both columns."""
219
+ if not {"seller_sku", "asin1"} <= set(listing_names):
220
+ return None
221
+ table = SOURCE[LISTINGS]
222
+ latest = (f' AND "_snapshot_date" = (SELECT MAX("_snapshot_date") FROM {table})'
223
+ if "_snapshot_date" in listing_names else "")
224
+ return ("SELECT DISTINCT sku, asin FROM (SELECT TRIM(CAST(seller_sku AS VARCHAR)) AS sku, "
225
+ f"TRIM(CAST(asin1 AS VARCHAR)) AS asin FROM {table} "
226
+ f"WHERE seller_sku IS NOT NULL AND asin1 IS NOT NULL{latest}) "
227
+ "WHERE sku <> '' AND asin <> ''")
228
+
229
+
230
+ def lookups(cur, key, names=None):
231
+ """`[{key, label, after, expr, opens, description}]`: the product lookups `key` gains.
232
+
233
+ `expr` is ONE correlated scalar subquery per lookup, qualified by the mirror table's own name
234
+ (every reader selects `FROM <table>` unaliased), so it binds to the outer row and never to the
235
+ resolver's own `asin`. DuckDB decorrelates it into a single hash join per query, which is what
236
+ keeps a grouped page of a read-through grid set-based rather than per-row. No match is NULL,
237
+ which every reader serialises as a blank cell. Nothing is capped (standing rule 1).
238
+ """
239
+ if names is None:
240
+ names = {name for name, _dtype in mirror_columns(cur, SOURCE[key])}
241
+ asin, sku, asin_via, sku_via = _product_columns(key)
242
+ asin = asin if asin in names else None
243
+ sku = sku if sku in names else None
244
+ if not asin and not sku:
245
+ return []
246
+ listing_names = {name for name, _dtype in mirror_columns(cur, SOURCE[LISTINGS])}
247
+ parents = None if "parent_asin" in names else amazon_store.asin_parents_sql(cur)
248
+ latest = _latest_listings_sql(listing_names)
249
+ table = SOURCE[key]
250
+ out = []
251
+
252
+ def add(lookup_key, after, expr, via):
253
+ if lookup_key in names:
254
+ return
255
+ out.append({"key": lookup_key, "label": _LOOKUP_LABELS[lookup_key], "after": after,
256
+ "expr": expr, "opens": {"via": via} if via else None,
257
+ "description": _LOOKUP_DESCRIPTIONS[lookup_key]})
258
+
259
+ if asin:
260
+ ref = f"TRIM(CAST({table}.{_quoted(asin)} AS VARCHAR))"
261
+ via = asin_via if "asin1" in listing_names else None
262
+ if parents:
263
+ add(LOOKUP_PARENT, asin,
264
+ f"(SELECT _p.parent_asin FROM ({parents}) AS _p WHERE _p.asin = {ref})", via)
265
+ if latest and not sku:
266
+ add(LOOKUP_SKU, asin,
267
+ f"(SELECT string_agg(DISTINCT _l.sku, ', ' ORDER BY _l.sku) "
268
+ f"FROM ({latest}) AS _l WHERE _l.asin = {ref})", via)
269
+ elif sku and latest:
270
+ ref = f"TRIM(CAST({table}.{_quoted(sku)} AS VARCHAR))"
271
+ via = sku_via if "seller_sku" in listing_names else None
272
+ add(LOOKUP_ASIN, sku,
273
+ f"(SELECT string_agg(DISTINCT _l.asin, ', ' ORDER BY _l.asin) "
274
+ f"FROM ({latest}) AS _l WHERE _l.sku = {ref})", via)
275
+ if parents:
276
+ add(LOOKUP_PARENT, sku,
277
+ f"(SELECT string_agg(DISTINCT _p.parent_asin, ', ' ORDER BY _p.parent_asin) "
278
+ f"FROM ({latest}) AS _l JOIN ({parents}) AS _p ON _p.asin = _l.asin "
279
+ f"WHERE _l.sku = {ref})", via)
280
+ return out
281
+
282
+
283
  def _fields_for(cur, key):
284
  """One preset field per mirror column β€” the whole schema, derived. See the header."""
285
  cols = mirror_columns(cur, SOURCE[key])
 
287
  return []
288
  names = {c[0] for c in cols}
289
  pinned = PINNED.get(key) if PINNED.get(key) in names else None
290
+ # S67-T30: each lookup lands IMMEDIATELY after the column it derives from, shown by default,
291
+ # and the ASIN / SKU column it sits beside is shown by default too (inventory's `msku` was not).
292
+ derived = {}
293
+ for lookup in lookups(cur, key, names):
294
+ derived.setdefault(lookup["after"], []).append(lookup)
295
+ product = {c for c in _product_columns(key)[:2] if c in names}
296
  out = []
297
+ # The identity column goes FIRST, the rest keep the report's own order. A record reached
298
+ # through a link is titled by its first field (the record drawer's frame label), and a
299
+ # report's first column is whatever Amazon put there (`item_name`, `_data_origin`), so a
300
+ # listing opened from an order must lead with its seller SKU, as Odoo's pinned fields do.
301
+ for name, dtype in sorted(cols, key=lambda col: col[0] != pinned):
302
  out.append(_odoo._preset({
303
  "key": name, "label": _label(name), "type": _ftype(name, dtype),
304
  "source": "overlay",
305
+ "default": name == pinned or name in _DEFAULT_COLUMNS or name in product,
306
  "pinned": name == pinned,
307
  }, flow="amazon_relational"))
308
+ for lookup in derived.get(name, ()):
309
+ field = {"key": lookup["key"], "label": lookup["label"], "type": "text",
310
+ "source": "overlay", "default": True,
311
+ "description": lookup["description"]}
312
+ if lookup["opens"]:
313
+ field["opens"] = dict(lookup["opens"])
314
+ out.append(_odoo._preset(field, flow="amazon_relational"))
315
  # ⚠ BOTH SIDES MUST CARRY THE COLUMN, EVERY TIME. A `measured_at_sync` table's header is not
316
  # known until it syncs, so a link to a table that has never run would name a column that does
317
  # not exist. **Absent is a normal state here, not a failure** β€” which is exactly what lets the
 
334
  out.append(_odoo._preset({
335
  "key": edge["key"], "label": edge["label"],
336
  "type": "link", "source": "overlay", "default": True,
337
+ "link": {"table": edge["target"], "on": edge["to_col"], "from": edge["from_col"],
338
+ "single": edge["cardinality"] == "many_to_one"},
339
  }, flow="amazon_relational"))
340
  out.append(_odoo._preset(_odoo._refreshed_field(), flow="amazon_relational"))
341
  return out
 
352
  }
353
 
354
 
355
+ #: The original seven used first-field identities. Retained for legacy-id migration
356
+ #: and for distinguishing history from the original small snapshots.
357
+ _LEGACY_IDENTITY_TABLES = frozenset(("orders", "order_items", "listings", "settlements",
358
+ "inventory", "pricing", "brand_analytics"))
359
+
360
+ # The original seven do not declare snapshot/window metadata. Keep their four small
361
+ # tables materialised; every newer non-snapshot report/API is history, including monthly ones.
362
+ READ_THROUGH = frozenset(
363
+ PREFIX + table for table, spec in amazon_client.REPORT_TYPES.items()
364
+ if table in {"settlements", "orders", "order_items", "transactions", "economics", "storage_fees"}
365
+ or (table not in _LEGACY_IDENTITY_TABLES and not spec.get("snapshot")
366
+ and (spec.get("report") or spec.get("api") or spec.get("monthly"))))
367
+
368
+
369
+ def _quoted(name):
370
+ return '"' + name.replace('"', '""') + '"'
371
+
372
+
373
+ #: β›”β›” HOTFIX 2026-09-27 (D-3284's cost): an Amazon snapshot table larger than this is served
374
+ #: READ-THROUGH, never stored in the tenant document. The per-table `MAX_TABLE_BYTES` (48 MB) let
375
+ #: catalog items (32.9 MB for 2,317 rows), pricing (11.7 MB) and FBA inventory (7.5 MB) in once
376
+ #: connector databases stopped counting toward MAX_TABLES, and `user_tables` doubled to 118 MB.
377
+ #: Every whole-document read and write doubled with it: the 2 GB DigitalOcean staging node ran
378
+ #: out of memory and PostgreSQL restarted (2026-09-27 16:36Z), and on Neon each read is billed at
379
+ #: the document's text size. Small tables stay stored; nothing is capped (standing rule 1).
380
+ AMAZON_STORED_BYTES = 4 * 1024 * 1024
381
+
382
+
383
+ def _read_through(cur, key, count):
384
+ """Choose storage before reading rows; estimate wide snapshots with a bounded sample."""
385
+ ut = _odoo._ut()
386
+ if key in READ_THROUGH or count > ut.MAX_ROWS:
387
+ return True
388
+ if not count:
389
+ return False
390
+ names = [name for name, _dtype in mirror_columns(cur, SOURCE[key])]
391
+ sample = cur.execute(f"SELECT * FROM {SOURCE[key]} LIMIT 32").fetchall()
392
+ # Include the stored id, refreshed stamp and row-object punctuation, with the same
393
+ # ASCII escaping as the document writer. The largest sampled row is conservative.
394
+ size = max((len(json.dumps(dict(zip(names, row)), ensure_ascii=True,
395
+ default=str).encode("utf-8")) + 96 for row in sample), default=0)
396
+ return count * size > min(ut.MAX_TABLE_BYTES, AMAZON_STORED_BYTES)
397
+
398
+
399
+ def missing_source(key):
400
+ return {"missing": (
401
+ f"The Amazon mirror in this container has not loaded Amazon "
402
+ f"{key.removeprefix(PREFIX).replace('_', ' ')} yet. "
403
+ "The next Amazon sync loads it. Nothing is lost.")}
404
+
405
+
406
+ def _text(value):
407
+ return "" if value is None else str(value)
408
+
409
+
410
+ def _number(value):
411
+ return None if value in (None, "") else float(value)
412
+
413
+
414
+ def _integer(value):
415
+ return None if value in (None, "") else int(value)
416
+
417
+
418
+ def grid_source(cur, key):
419
+ """One binding from the current mirror schema, or an explicit unloaded-mirror reason."""
420
+ columns = mirror_columns(cur, SOURCE[key])
421
+ if not columns:
422
+ return missing_source(key)
423
+ names = {name for name, _dtype in columns}
424
+ identity = [name for name in IDENTITY[key] if name in names]
425
+ coordinates = ["_report_document_id", "_report_row_number"]
426
+ if all(name in names for name in coordinates):
427
+ identity = list(dict.fromkeys(identity + coordinates))
428
+ if (not identity or identity == ["_report_row_number"]
429
+ or (key in {PREFIX + "settlements", PREFIX + "order_items"}
430
+ and not all(name in names for name in coordinates))):
431
+ identity = sorted(names)
432
+ # Report coordinates qualify line numbers by their document. If the declared identity
433
+ # is incomplete/null, hash the whole row instead of collapsing several unknown lines.
434
+ def digest(keys):
435
+ values = ", ".join(f"CAST({_quoted(name)} AS VARCHAR)" for name in keys)
436
+ return f"sha256(to_json(list_value({values})))"
437
+ complete = " AND ".join(f"NULLIF(CAST({_quoted(name)} AS VARCHAR), '') IS NOT NULL"
438
+ for name in identity)
439
+ digest_sql = f"CASE WHEN {complete} THEN {digest(identity)} ELSE {digest(sorted(names))} END"
440
+ # The shared grid, relation and selection contracts all use numeric pids. Thirteen
441
+ # hex digits retain a stable 52-bit identity that JavaScript represents exactly.
442
+ ident = f"greatest(1, CAST(('0x' || substr(({digest_sql}), 1, 13)) AS BIGINT))"
443
+ cols = {}
444
+ for name, dtype in columns:
445
+ kind = _ftype(name, dtype)
446
+ expr = _quoted(name)
447
+ if kind in ("currency", "int"):
448
+ expr = f"TRY_CAST({expr} AS {'DOUBLE' if kind == 'currency' else 'BIGINT'})"
449
+ cols[name] = (expr, {"currency": _number, "int": _integer}.get(kind, _text))
450
+ # S67-T30: the product lookups bind like any mirror column, so a filter (`=` a parent), a
451
+ # group (count by parent) and a sort compile against them in SQL with no per-row Python.
452
+ for lookup in lookups(cur, key, names):
453
+ cols[lookup["key"]] = (lookup["expr"], _text)
454
+ return {"table": SOURCE[key], "where": "", "id": ident,
455
+ "needs_excluded": False, "cols": cols}
456
+
457
+
458
+ def grid_sources():
459
+ """Only Amazon read-through grids; no keys at all on an Odoo-only mirror."""
460
+ cur = datastore.ro_cursor()
461
+ try:
462
+ present = {key for key, table in SOURCE.items() if mirror_columns(cur, table)}
463
+ if not present:
464
+ return {}
465
+ keys = set(READ_THROUGH)
466
+ for key in present - keys:
467
+ count = cur.execute(f"SELECT count(*) FROM {SOURCE[key]}").fetchone()[0]
468
+ if _read_through(cur, key, count):
469
+ keys.add(key)
470
+ return {key: grid_source(cur, key) for key in SOURCE if key in keys}
471
+ finally:
472
+ cur.close()
473
+
474
+
475
  def _row_id(row, key):
476
+ """The legacy stored identity, retained to migrate existing snapshot rows.
477
 
478
  ⭐ STABILITY IS THE WHOLE REQUIREMENT, and it is why a row number will not do. `_ensure_table_inplace`
479
  merges by `_id`, so an id that changes between syncs turns every re-sync into a full delete and
 
482
  Falls back to a content hash rather than a counter for exactly that reason: a settlement line
483
  with no id column still hashes to the same value next week if its content has not changed.
484
  """
485
+ if key.removeprefix(PREFIX) not in _LEGACY_IDENTITY_TABLES:
486
+ # These tables retain multiple days/items per SKU/transaction. First-field identity
487
+ # would collapse them in apply_plan even though all rows survived the raw mirror.
488
+ # S66-T40 (lead, observed live): a report whose header spells an identity column
489
+ # differently (the referral fee preview says `seller_sku`, not `sku`) made
490
+ # `row_identity_for` raise, and because `apply_plan` is ONE write for every Amazon table,
491
+ # one such row cost all of them. A row missing a component falls through to the full
492
+ # content hash below, which is still stable and never merges two different rows.
493
+ try:
494
+ identity = amazon_schema.row_identity_for(key.removeprefix(PREFIX), row)
495
+ except ValueError:
496
+ identity = None
497
+ if identity is not None:
498
+ return hashlib.sha256(json.dumps(identity, ensure_ascii=True,
499
+ default=str).encode()).hexdigest()[:32]
500
+ payload = json.dumps({k: row.get(k) for k in sorted(row)}, sort_keys=True,
501
+ ensure_ascii=True, separators=(",", ":"), default=str)
502
+ return hashlib.sha256(payload.encode("utf-8")).hexdigest()[:32]
503
  for col in IDENTITY.get(key, ()):
504
  val = row.get(col)
505
  if val not in (None, ""):
 
510
 
511
 
512
  def _read(cur, key):
513
+ """Small snapshots use the same numeric identity as their read-through grid.
514
+
515
+ A SKU or hex digest used as the stored row key is silently skipped by
516
+ perm_scope._read_user_table, and cannot be opened by the record drawer.
517
+ """
518
  cols = [c[0] for c in mirror_columns(cur, SOURCE[key])]
519
  if not cols:
520
  return []
521
  quoted = ", ".join(f'"{c}"' for c in cols)
522
+ spec = grid_source(cur, key)
523
+ # S67-T30: a stored snapshot carries the same product lookups, computed by the SAME SQL
524
+ # expressions the read-through binding uses, in this one SELECT (never per row).
525
+ derived = [(lookup["key"], lookup["expr"]) for lookup in lookups(cur, key)]
526
+ extra = "".join(f", {expr}" for _k, expr in derived)
527
+ rows = cur.execute(f'SELECT {spec["id"]}, {quoted}{extra} FROM {SOURCE[key]}').fetchall()
528
+ names = cols + [k for k, _expr in derived]
529
  out = []
530
  for r in rows:
531
  row = {}
532
+ for name, val in zip(names, r[1:]):
533
+ # Postgres JSON cannot hold U+0000, and one such byte refused the whole Amazon write.
534
+ row[name] = "" if val is None else (val if isinstance(val, (int, float))
535
+ else str(val).replace("\x00", ""))
536
+ row["_id"] = str(r[0])
537
  out.append(row)
538
+ def read_targets(target, column, values):
539
+ target_spec = grid_source(cur, target)
540
+ expr = _quoted(column)
541
+ found = cur.execute(
542
+ f'SELECT {target_spec["id"]}, {expr} FROM {SOURCE[target]} '
543
+ f'WHERE {expr} IN ({",".join("?" for _ in values)})', list(values)).fetchall()
544
+ return [{"pid": pid, column: value} for pid, value in found]
545
+
546
+ fill_link_cells(out, _fields_for(cur, key), read_targets)
547
+ return out
548
+
549
+
550
+ def link_edges(fields):
551
+ """`[(field, target, from, on)]` for every derived Amazon link `fill_link_cells` fills.
552
+
553
+ HOTFIX 2026-09-28: the ONE predicate, so the HTTP reader can see up front every column it
554
+ will be asked for on a target and read a STORED target once rather than once per edge."""
555
+ out = []
556
+ for field in fields:
557
+ bag = field.get("link") or {}
558
+ target, frm, on = bag.get("table"), bag.get("from"), bag.get("on")
559
+ if (field.get("type") != "link" or target not in SOURCE or not frm or not on
560
+ or bag.get("inverse") or bag.get("through") or bag.get("selectionMode") == "manual"):
561
+ continue
562
+ out.append((field, target, frm, on))
563
  return out
564
 
565
 
566
+ def fill_link_cells(rows, fields, read_targets):
567
+ """Populate derived Amazon links with target pids, with one read per edge.
568
+
569
+ The snapshot planner reads its tenant mirror; the HTTP reader supplies a scoped
570
+ target reader. Neither path stores history rows or caps the matching target set.
571
+ """
572
+ for field, target, frm, on in link_edges(fields):
573
+ values = {str(row[frm]) for row in rows if row.get(frm) not in (None, "")}
574
+ matches = {}
575
+ for row in read_targets(target, on, values) if values else ():
576
+ if row.get(on) not in (None, ""):
577
+ matches.setdefault(str(row[on]), set()).add(int(row["pid"]))
578
+ for row in rows:
579
+ row[field["key"]] = ",".join(str(pid) for pid in sorted(
580
+ matches.get(str(row.get(frm)), ())))
581
+
582
+
583
  def available(tenant_key="royal-imports"):
584
  """Has this tenant's mirror ever been fed by `amazon_store`? Cheap; never fetches."""
585
  try:
 
590
 
591
 
592
  def plan(cur, rt=None):
593
+ """Rows for small snapshots, schema/count stubs for history or oversized tables."""
594
  built = {"problems": []}
 
 
595
  for key in SOURCE:
596
  # β›” COUNT BEFORE READ. `_read` materialises the whole table as python dicts, so checking
597
  # the cap afterwards means the one table too big to keep is the one built in memory first.
 
599
  n = cur.execute(f"SELECT count(*) FROM {SOURCE[key]}").fetchone()[0]
600
  except Exception: # noqa: BLE001
601
  continue # absent = never synced
602
+ if _read_through(cur, key, n):
603
+ built[key] = {"readThrough": True, "fields": _fields_for(cur, key), "count": n}
 
 
 
 
 
 
 
 
 
604
  continue
605
  rows = _read(cur, key)
606
  if rows:
 
617
  """
618
  tables = []
619
  for key in SOURCE:
620
+ if isinstance(built.get(key), dict) and built[key].get("readThrough"):
621
+ tables.append({"key": key, **built[key]})
622
+ continue
623
  rows = built.get(key) or []
624
  canonical = sorted(rows, key=lambda row: json.dumps(
625
  row, sort_keys=True, ensure_ascii=True, separators=(",", ":"), default=str))
 
629
  return hashlib.sha256(payload.encode("utf-8")).hexdigest()
630
 
631
 
632
+ def _place_lookups(stored, fields):
633
+ """An EXISTING table's fields with each NEW product lookup placed right after its source.
634
+
635
+ `_ensure_table_inplace` only appends, so a lookup first planned for a table stored before
636
+ S67-T30 would land at the far end of the grid instead of beside the ASIN or SKU it derives
637
+ from. A lookup already stored keeps its place (a person may have moved it) and only has its
638
+ `opens` bag brought up to date, which `_ensure_table_inplace` does not forward-migrate.
639
+ """
640
+ out = list(stored)
641
+ have = {f.get("key"): i for i, f in enumerate(out)}
642
+ anchor = None
643
+ for field in fields:
644
+ fkey = field.get("key")
645
+ if fkey not in LOOKUP_KEYS:
646
+ anchor = fkey
647
+ continue
648
+ if fkey in have:
649
+ old = out[have[fkey]]
650
+ if (old.get("automation") or {}).get("flowId") == "amazon_relational" \
651
+ and not _odoo._ut().user_edited(old):
652
+ if field.get("opens"):
653
+ old["opens"] = dict(field["opens"])
654
+ else:
655
+ old.pop("opens", None)
656
+ continue
657
+ keys = [f.get("key") for f in out]
658
+ if anchor not in keys:
659
+ continue # no source column stored yet: the append places it
660
+ at = keys.index(anchor) + 1
661
+ while at < len(keys) and keys[at] in LOOKUP_KEYS:
662
+ at += 1 # after any lookup already beside the same source
663
+ out.insert(at, dict(field))
664
+ have = {f.get("key"): i for i, f in enumerate(out)}
665
+ anchor = fkey
666
+ return out
667
+
668
+
669
+ def _lead_with_identity(stored, report, pinned):
670
+ """An EXISTING table's fields with its identity column moved first, while the order is still
671
+ the report's own.
672
+
673
+ `_ensure_table_inplace` keeps a stored table's field order and only appends, so the
674
+ identity-first order `_fields_for` gives a NEW table never reaches one stored before it. The
675
+ move happens only when the stored mirror columns sit exactly in the report's order: any other
676
+ order is a person's arrangement, and theirs wins.
677
+ """
678
+ keys = [f.get("key") for f in stored]
679
+ if not pinned or pinned not in keys or keys[0] == pinned:
680
+ return stored
681
+ in_report = set(report)
682
+ if [k for k in keys if k in in_report] != [k for k in report if k in set(keys)]:
683
+ return stored
684
+ return ([f for f in stored if f.get("key") == pinned]
685
+ + [f for f in stored if f.get("key") != pinned])
686
+
687
+
688
  def apply_plan(rt, built, username="amazon", today=None):
689
  """Create-or-merge every table in `built`, in ONE `flush="sync"` write.
690
 
 
697
  raise Refused("; ".join(built["problems"]))
698
  stamp = today or _odoo._iso_today()
699
  written = {}
700
+ cur_mirror = datastore.ro_cursor()
701
+ try:
702
+ plans = [(key, LABELS[key], (built[key]["fields"] if isinstance(built[key], dict)
703
+ else _fields_for(cur_mirror, key)), built[key])
704
+ for key in SOURCE if key in built]
705
+ legacy_ids = {}
706
+ report_order = {key: [name for name, _dtype in mirror_columns(cur_mirror, SOURCE[key])]
707
+ for key, _label, _fields, _rows in plans}
708
+ for key, _label, _fields, rows in plans:
709
+ if isinstance(rows, dict):
710
+ continue
711
+ names = report_order[key]
712
+ for row in rows:
713
+ raw = {name: row.get(name) for name in names}
714
+ payload = json.dumps(raw, sort_keys=True, ensure_ascii=True,
715
+ separators=(",", ":"), default=str)
716
+ legacy_ids[(key, row["_id"])] = (
717
+ _row_id(raw, key), hashlib.sha256(payload.encode("utf-8")).hexdigest()[:32])
718
+ finally:
719
+ cur_mirror.close()
720
+
721
+ not_stored = {}
722
 
723
  def _apply_all(doc):
724
  doc = doc if isinstance(doc, dict) else {}
725
+ not_stored.clear()
726
  for key, label, fields, rows in plans:
727
+ # ⭐⭐ D-3284 (owner ruling 2026-09-27): S66-T42/T43 created a NEW Amazon table only while
728
+ # the workspace had room under MAX_TABLES and reported the rest `not_stored`; ten Amazon
729
+ # databases stayed missing that way. Connector databases now take no MAX_TABLES slot
730
+ # (`user_tables.counts_toward_table_cap`), so every planned Amazon table is created.
731
+ # `not_stored` stays as bookkeeping: empty, it clears any earlier record.
732
+ # Corrected/removed measured joins retire their machine-owned old fields.
733
+ # Preserve fields a person created or explicitly took over.
734
+ current = doc.get(key) or {}
735
+ wanted = {f["key"] for f in fields}
736
+ # S67-T30: a product lookup the mirror no longer supports retires the same way.
737
+ current["fields"] = _lead_with_identity(
738
+ [f for f in current.get("fields", [])
739
+ if not ((f.get("type") == "link" or f.get("key") in LOOKUP_KEYS)
740
+ and (f.get("automation") or {}).get("flowId") == "amazon_relational"
741
+ and not _odoo._ut().user_edited(f)
742
+ and f.get("key") not in wanted)],
743
+ report_order.get(key) or [], PINNED.get(key))
744
+ if current["fields"]:
745
+ current["fields"] = _place_lookups(current["fields"], fields)
746
+ stub = isinstance(rows, dict) and rows.get("readThrough") is True
747
+ if not stub:
748
+ stored = current.get("rows") or {}
749
+ for row in rows:
750
+ if row["_id"] in stored:
751
+ continue
752
+ for old_id in legacy_ids.get((key, row["_id"]), ()):
753
+ if old_id in stored:
754
+ # Carry authored cells along when replacing an unusable string id.
755
+ stored[row["_id"]] = stored.pop(old_id)
756
+ break
757
+ removed = len((doc.get(key) or {}).get("rows") or {}) if stub else 0
758
+ written[key] = _odoo._ensure_table_inplace(doc, key, label, fields, None if stub else rows,
759
  username, stamp)
760
+ if stub:
761
+ doc[key]["readThrough"] = True
762
+ doc[key]["rows"] = {}
763
+ doc[key].pop(_odoo._ut().OVERSIZE_FIELD, None)
764
+ written[key].update(rows=rows["count"], removed=removed, readThrough=True)
765
+ else:
766
+ doc[key].pop("readThrough", None)
767
  # DPP 1.8 attribution is explicit in the stored definition, not inferred from a
768
  # mutable display label or from the process having imported this module.
769
  doc[key]["connector"] = source_attribution(key)
770
  return doc
771
 
772
  rt.update(_odoo._ut().STORE_KEY, _apply_all, flush="sync")
773
+ _record_not_stored({key for key, *_rest in plans}, dict(not_stored))
774
  return written
775
 
776
 
777
+ def _record_not_stored(planned, not_stored):
778
+ """Durable per-table marker in the Amazon mirror, read by `connectors_amazon.sync_state`.
779
+
780
+ A table refused at the store's table cap gets `not_stored` with its reason; a table that was
781
+ stored clears any earlier marker. Best effort: the store write above already happened, and a
782
+ marker failure must not turn it into an error.
783
+ """
784
+ try:
785
+ con = datastore.connect()
786
+ for key in planned:
787
+ table = key[len(PREFIX):]
788
+ entity = amazon_store.entity_key("not_stored_" + table)
789
+ if key in not_stored:
790
+ amazon_store._record_state(con, entity, "not_stored", 0, not_stored[key])
791
+ else:
792
+ con.execute("DELETE FROM _sync_state WHERE entity = ?", [entity])
793
+ except Exception: # noqa: BLE001
794
+ pass
795
+
796
+
797
  def refresh_state(rt, why="boot", tenant_key=None, previous_fingerprint=None, log=print):
798
  """Open, plan and conditionally apply the Amazon mirror.
799
 
 
806
  path = datastore.path_for(tenant)
807
  if Path(datastore.DB_PATH) != Path(path):
808
  datastore.use_path(path)
809
+ cur = datastore.ro_cursor()
810
+ try:
811
+ return _refresh_from_cursor(cur, rt, why, tenant, previous_fingerprint, log, state)
812
+ finally:
813
+ cur.close()
814
+
815
+
816
+ def _refresh_from_cursor(cur, rt, why, tenant, previous_fingerprint, log, state):
817
  if not any(mirror_columns(cur, t) for t in SOURCE.values()):
818
  log(f"[amazon_relational] {why}: no Amazon tables in {tenant}'s mirror, nothing to spawn "
819
  f"(run `python platform/harness/amazon_store.py --sync --tenant {tenant}` first)")
 
837
  state["changed"] = True
838
  state["written"] = apply_plan(rt, applicable)
839
  state["applied"] = True
840
+ import routes_odoo_tables
841
+ routes_odoo_tables.invalidate_sources()
842
  for key, counts in state["written"].items():
843
  log(f"[amazon_relational] {why}: {key:<28} {counts}")
844
  return state
api/application_registry.py CHANGED
@@ -16,6 +16,11 @@ APPLICATION_TYPES = ("chart", "route", "map", "timeseries")
16
  APPLICATION_WIZARD_TEMPLATE_KINDS = ("route", "catalog")
17
  FINANCE_TEMPLATE_KIND = "finance"
18
  APPLICATION_TEMPLATE_KINDS = APPLICATION_WIZARD_TEMPLATE_KINDS + (FINANCE_TEMPLATE_KIND,)
 
 
 
 
 
19
  _KEY = "applications_v1"
20
  _MAX = 500
21
 
@@ -65,6 +70,17 @@ _TEMPLATE_CONTRACTS = {
65
  "fieldRoles": [],
66
  "defaultPlacement": {"section": "app", "order": 0},
67
  },
 
 
 
 
 
 
 
 
 
 
 
68
  }
69
 
70
 
@@ -363,6 +379,18 @@ def _template_definition(kind, payload, *, semantic_records=None, field_contract
363
  raise ApplicationError("navPlacement order must be an integer from 0 to 999")
364
  return {"templateKind": kind, "financeKey": finance_key, "deepLink": app["deepLink"],
365
  "navPlacement": {"section": "app", "order": order}}
 
 
 
 
 
 
 
 
 
 
 
 
366
  source = str(payload.get("sourceDatabase") or "").strip()[:120]
367
  if not source:
368
  raise ApplicationError("sourceDatabase is required")
@@ -470,12 +498,49 @@ def _public(app, user, is_admin=False, *, allowed_databases=None, target_databas
470
  "edit", "share", "delete", "duplicate", "star", "default")})
471
  row["starred"] = False
472
  row["isDefault"] = False
 
 
 
 
 
 
473
  target_database = target_database if target_database in compatible else (compatible[0] if compatible else None)
474
  row["openTarget"]={"kind":"application","id":row["id"],"type":row["type"],
475
  "database":target_database}
476
  return row
477
 
478
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
479
  def list_apps(runtime, user, is_admin=False, *, database=None, kind=None, starred=None,
480
  allowed_databases=None):
481
  doc = load(runtime)
@@ -491,6 +556,13 @@ def list_apps(runtime, user, is_admin=False, *, database=None, kind=None, starre
491
  rows.append(_public(app,user,is_admin,allowed_databases=allowed_databases,
492
  target_database=database,
493
  is_default=bool(default_key and doc["defaults"].get(default_key)==app.get("id"))))
 
 
 
 
 
 
 
494
  return sorted(rows,key=lambda x:(str(x.get("updatedAt") or ""),str(x.get("id"))),reverse=True)
495
 
496
 
 
16
  APPLICATION_WIZARD_TEMPLATE_KINDS = ("route", "catalog")
17
  FINANCE_TEMPLATE_KIND = "finance"
18
  APPLICATION_TEMPLATE_KINDS = APPLICATION_WIZARD_TEMPLATE_KINDS + (FINANCE_TEMPLATE_KIND,)
19
+ # S67-T40: Content Studio creates its OWN databases (content_studio.py), so it is never offered to
20
+ # the five-step wizard either; its catalogue POST provisions References + Posts and binds Posts.
21
+ # Appended on its own line so the wizard / Finance separation above reads exactly as S65-T10 pins it.
22
+ CONTENT_STUDIO_TEMPLATE_KIND = "content_studio"
23
+ APPLICATION_TEMPLATE_KINDS = APPLICATION_TEMPLATE_KINDS + (CONTENT_STUDIO_TEMPLATE_KIND,)
24
  _KEY = "applications_v1"
25
  _MAX = 500
26
 
 
70
  "fieldRoles": [],
71
  "defaultPlacement": {"section": "app", "order": 0},
72
  },
73
+ # S67-T40 (R17, R21, O3: one declarative row, no new type). S67-T41 owner correction: the App
74
+ # opens on its OWN surface in the App module (`content_studio.app_link`), like Catalog and
75
+ # Route, so the governed type only has to be a real one; its calendar is time.
76
+ "content_studio": {
77
+ "kind": "content_studio",
78
+ "label": "Content Studio",
79
+ "applicationType": "timeseries",
80
+ "displayMode": "timeseries",
81
+ "fieldRoles": [],
82
+ "defaultPlacement": {"section": "app", "order": 0},
83
+ },
84
  }
85
 
86
 
 
379
  raise ApplicationError("navPlacement order must be an integer from 0 to 999")
380
  return {"templateKind": kind, "financeKey": finance_key, "deepLink": app["deepLink"],
381
  "navPlacement": {"section": "app", "order": order}}
382
+ if kind == CONTENT_STUDIO_TEMPLATE_KIND:
383
+ # Created server-side by content_studio.create right after it made the Posts database, so
384
+ # the binding is the one database it just created; the link can only open that database.
385
+ source = str(payload.get("sourceDatabase") or "").strip()[:120]
386
+ if not source.startswith("ut_"):
387
+ raise ApplicationError("Content Studio binds the Posts database it created.")
388
+ if allowed_databases is not None and source not in set(allowed_databases):
389
+ raise PermissionError("an Application cannot bind a database you cannot open")
390
+ # S67-T41 owner correction: no stored link. `_public` derives the App's own address from
391
+ # its id, so every Studio, including one created before the correction, opens in App.
392
+ return {"templateKind": kind, "sourceDatabase": source,
393
+ "navPlacement": {"section": "app", "order": 0}}
394
  source = str(payload.get("sourceDatabase") or "").strip()[:120]
395
  if not source:
396
  raise ApplicationError("sourceDatabase is required")
 
498
  "edit", "share", "delete", "duplicate", "star", "default")})
499
  row["starred"] = False
500
  row["isDefault"] = False
501
+ if row.get("templateKind") == CONTENT_STUDIO_TEMPLATE_KIND:
502
+ # S67-T41 owner correction: a Studio opens on its own App surface, never in Data. The
503
+ # database-bound edit, share, duplicate and default doors cannot reach an App bound to a
504
+ # user table, so they are off; star and delete stay, routed to `/content-studio/apps`.
505
+ row["deepLink"] = f"/app?studio={row['id']}"
506
+ row["capabilities"].update({key: False for key in ("edit", "share", "duplicate", "default")})
507
  target_database = target_database if target_database in compatible else (compatible[0] if compatible else None)
508
  row["openTarget"]={"kind":"application","id":row["id"],"type":row["type"],
509
  "database":target_database}
510
  return row
511
 
512
 
513
+ #: `core.table_store.RESERVED_MEMBERS`: workspace members that are registries, not users.
514
+ _RESERVED_WORKSPACE_MEMBERS = ("__shared__", "__assigned__", "__audit__")
515
+
516
+
517
+ def dependent_view_counts(runtime, database):
518
+ """S67-T39 / R36: {app_id: number of saved Views on `database` bound to it}, over EVERY
519
+ user's Views in that database's workspace. The Delete App dialog names this count before
520
+ confirm. None when the workspace cannot be read, so the dialog never states a guessed 0.
521
+ """
522
+ from core.view_templates import workspace_key
523
+ key = workspace_key(database)
524
+ if not key:
525
+ return {}
526
+ try:
527
+ doc = runtime.get(key)
528
+ except Exception:
529
+ return None
530
+ counts = {}
531
+ for member, bucket in (doc.items() if isinstance(doc, dict) else ()):
532
+ if member in _RESERVED_WORKSPACE_MEMBERS or not isinstance(bucket, dict):
533
+ continue
534
+ views = bucket.get("views")
535
+ for view in (views.values() if isinstance(views, dict) else ()):
536
+ config = view.get("config") if isinstance(view, dict) else None
537
+ ref = config.get("application") if isinstance(config, dict) else None
538
+ app_id = str(ref.get("id") or "") if isinstance(ref, dict) else ""
539
+ if app_id:
540
+ counts[app_id] = counts.get(app_id, 0) + 1
541
+ return counts
542
+
543
+
544
  def list_apps(runtime, user, is_admin=False, *, database=None, kind=None, starred=None,
545
  allowed_databases=None):
546
  doc = load(runtime)
 
556
  rows.append(_public(app,user,is_admin,allowed_databases=allowed_databases,
557
  target_database=database,
558
  is_default=bool(default_key and doc["defaults"].get(default_key)==app.get("id"))))
559
+ # S67-T39: the dependent-View count rides a DATABASE-filtered read only, and only for rows
560
+ # the caller may delete, so the unfiltered nav read (App home tiles) pays nothing for it.
561
+ deletable = [row for row in rows if row["capabilities"].get("delete")]
562
+ if database and deletable:
563
+ counts = dependent_view_counts(runtime, database)
564
+ for row in deletable:
565
+ row["dependentViews"] = None if counts is None else counts.get(row["id"], 0)
566
  return sorted(rows,key=lambda x:(str(x.get("updatedAt") or ""),str(x.get("id"))),reverse=True)
567
 
568
 
api/assistant_mirror.py CHANGED
@@ -28,6 +28,15 @@ that need them. A module-level import either way is a cycle at startup.
28
  """
29
 
30
 
 
 
 
 
 
 
 
 
 
31
  class MirrorRefusal(Exception):
32
  """A named, fail-closed refusal with the status and code `deps` should raise it as.
33
 
@@ -49,9 +58,14 @@ def _rot():
49
  return routes_odoo_tables
50
 
51
 
52
- def open_cursor(session):
53
  """`(cursor, MirrorRefusal | None)` for THIS session's tenant. Exactly one is None.
54
 
 
 
 
 
 
55
  β›” THE TWO FAILURES ARE NOT THE SAME FAILURE and the statuses differ, exactly as they do on
56
  the grid door. `_mirror_cur` has already consumed `DatastoreMismatch` and hands it back as a
57
  sentence, so a bare `RuntimeError` escaping it means the mirror is completing its first sync
@@ -61,7 +75,7 @@ def open_cursor(session):
61
  """
62
  rot = _rot()
63
  try:
64
- cur, mismatch = rot._mirror_cur(session)
65
  except RuntimeError as exc:
66
  return None, MirrorRefusal(503, "assistant_store_not_ready", str(exc))
67
  except Exception as exc: # noqa: BLE001
@@ -118,8 +132,26 @@ def is_bound(cur, table_key):
118
  """
119
  from harness import datastore
120
  try:
121
- spec = _rot()._sources().get(str(table_key or ""))
122
- if not spec:
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
123
  return False
124
  aliases = dict(spec.get("tables") or {})
125
  if spec.get("table"):
@@ -129,8 +161,13 @@ def is_bound(cur, table_key):
129
  return False
130
 
131
 
132
- def declared_fields(table_key):
133
- """`mk_fields()` for one connected grid, or None when this key is not a connected Odoo grid.
 
 
 
 
 
134
 
135
  ⭐ THE STABLE MIRROR CONTRACT, NOT THE STORED DEFINITION. `odoo_relational.apply_plan` built
136
  each table's stored `fields` from this same function, so reading it directly is reading the
@@ -142,10 +179,20 @@ def declared_fields(table_key):
142
  if key == str(table_key or ""):
143
  return [dict(field) for field in (mk_fields() or [])
144
  if isinstance(field, dict) and field.get("key")]
145
- return None
 
 
 
 
 
 
 
 
 
 
146
 
147
 
148
- def answerable_fields(table_key, spec):
149
  """The declared field contract INTERSECTED with the columns the mirror can actually express.
150
 
151
  β›” THE INTERSECTION IS THE POINT, and dropping it would be the quiet defect. A declared field
@@ -155,7 +202,8 @@ def answerable_fields(table_key, spec):
155
  refused - a wrong answer with nothing on screen saying so. Left out, `deps` answers a request
156
  for it with a loud `assistant_unknown_field`.
157
  """
158
- fields = declared_fields(table_key)
 
159
  if fields is None:
160
  return None
161
  cols = set((spec or {}).get("cols") or ())
@@ -197,30 +245,14 @@ def _compiled(nodes, conj, columns, *, code, subject, rot):
197
  return pred
198
 
199
 
200
- def read_rows(session, table_key, cur, spec, fields, *, wall_tree=None, filter_tree=None,
201
- filter_keys=None):
202
- """Every permitted row of a mirror-served grid, or a refusal that names its cause.
203
-
204
- Returns `(rows, stamp, total)`. `rows` are FULL canonical rows in `scoped_pool`'s shape
205
- (`{pid, **cells}`), sorted by pid.
206
 
207
- β›” NO TRUNCATION, EVER, AND THAT IS WHY THIS REFUSES RATHER THAN PAGES. Standing rule 1: a
208
- connector's records get no cap, and a limit that genuinely cannot be removed must be REPORTED
209
- with its cause and a recommended fix, never silently enforced. `routes_query._system_prompt`
210
- states `PERMISSION-FILTERED RECORD COUNT: {len(records)}` to the model and the server computes
211
- every aggregate from those same records, so a short set does not produce a smaller answer, it
212
- produces a confidently WRONG one with nothing on screen saying so. `whole_pool` one file over
213
- already settled this policy in the same words: REFUSE, NEVER TRIM.
214
-
215
- β›” FULL CANONICAL ROWS, NOT THE PROJECTION. `deps.assistant_read_scope` runs the row wall and
216
- the field projection in Python AFTER this returns. Hand it a row missing a column the wall
217
- names and `filter_eval` reads `None` there and its verdict changes, so the belt-and-braces
218
- pass would quietly disagree with the pushdown it is meant to double-check.
219
  """
220
- from harness import datastore
221
- rot = _rot()
222
  fe = rot._fe()
223
-
224
  cols = rot._filter_columns(spec, fields)
225
 
226
  # ── the administrator's wall, compiled FIRST and pushed down ──────────────────────────────
@@ -267,23 +299,191 @@ def read_rows(session, table_key, cur, spec, fields, *, wall_tree=None, filter_t
267
  where = f"({where}) AND {pred.sql}" if where else pred.sql
268
  params.extend(pred.params)
269
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
270
  # ⚠ EXACTLY ONE of `table`/`from_sql` - `window` raises if both or neither arrive, so the
271
  # spec's own shape decides and a malformed spec fails loudly instead of serving a wrong FROM.
272
  select = ", ".join(f'{expr} AS "{key}"' for key, (expr, _c) in spec["cols"].items())
273
  frm = {"from_sql": spec["from_sql"]} if spec.get("from_sql") else {"table": spec["table"]}
274
- win = datastore.window(select=f'{spec["id"]} AS "_pid", {select}',
275
- where=where, params=tuple(params), order_by=spec["id"],
276
- limit=datastore.WINDOW_MAX, cur=cur, **frm)
277
- if win["total"] > len(win["rows"]):
 
 
 
 
 
 
 
 
 
278
  raise MirrorRefusal(
279
  409, "assistant_source_too_large",
280
- f"this database matches {win['total']:,} rows for that question, and the assistant "
281
- f"reads a connected database in one pass of at most "
282
- f"{datastore.WINDOW_MAX:,} rows, so answering now would mean counting only part of "
283
- f"them and reporting the total as if it were whole. Narrow it with a filter and ask "
284
- f"again")
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
285
 
286
- stamp = rot.mirror_stamp(spec, cur, table_key, getattr(session, "runtime", None))
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
287
  rows = rot._pool_from_window(spec, win, stamp=stamp)
288
  rows.sort(key=lambda row: row["pid"])
289
  return rows, stamp, int(win["total"])
 
28
  """
29
 
30
 
31
+ #: ⭐ S67-T33: the most rows ONE assistant answer materialises in this process. Not a cap on what
32
+ #: the connector may hold (rule 1: the read walks every row up to here, never a 5,000-row window):
33
+ #: it is the memory bound of computing an answer in Python. MEASURED on staging, 2026-09-28: an
34
+ #: unbounded read of one Amazon grid grew this process past 15 GB before it was stopped, which is
35
+ #: the Space's whole memory. Past it, the read REFUSES BY NAME with the cause and the fix (rule 1's
36
+ #: second sentence); server-side aggregates over larger tables are S67-T34 (D-2631).
37
+ ASSISTANT_READ_MAX = 200_000
38
+
39
+
40
  class MirrorRefusal(Exception):
41
  """A named, fail-closed refusal with the status and code `deps` should raise it as.
42
 
 
58
  return routes_odoo_tables
59
 
60
 
61
+ def open_cursor(session, table_key=None):
62
  """`(cursor, MirrorRefusal | None)` for THIS session's tenant. Exactly one is None.
63
 
64
+ ⭐ S67-T33: `table_key` CHOOSES THE CURSOR BY THE TABLE'S OWN SOURCE, through the grid's
65
+ `_table_cursor` (reached via `_mirror_cur`). An Amazon grid asserts the tenant and opens its
66
+ own mirror without the Odoo first-sync gate, exactly as the grid door has since be3edb6e, so
67
+ the assistant no longer answers 503 for Amazon while Odoo is still backfilling.
68
+
69
  β›” THE TWO FAILURES ARE NOT THE SAME FAILURE and the statuses differ, exactly as they do on
70
  the grid door. `_mirror_cur` has already consumed `DatastoreMismatch` and hands it back as a
71
  sentence, so a bare `RuntimeError` escaping it means the mirror is completing its first sync
 
75
  """
76
  rot = _rot()
77
  try:
78
+ cur, mismatch = rot._mirror_cur(session, table_key)
79
  except RuntimeError as exc:
80
  return None, MirrorRefusal(503, "assistant_store_not_ready", str(exc))
81
  except Exception as exc: # noqa: BLE001
 
132
  """
133
  from harness import datastore
134
  try:
135
+ rot = _rot()
136
+ key = str(table_key or "")
137
+ spec = rot._sources().get(key)
138
+ if not spec and key.startswith("ut_amzn_"):
139
+ # S67-T33: a registry built before the Amazon mirror loaded holds no Amazon key.
140
+ # `_source_for` and `_materialised_table` both learn them on demand; so must this.
141
+ rot._learn_amazon_sources()
142
+ spec = rot.GRID_SOURCES.get(key)
143
+ if not spec:
144
+ # A dynamically sized Amazon snapshot is admitted per key by the grid door, not by
145
+ # the provider build, so the registry can lack a key the grid serves (measured on
146
+ # staging: `ut_amzn_fba_inventory` listed unanswerable, then answered). Ask the
147
+ # provider's own per-key resolver, exactly as `require_amazon_mirror` does, without
148
+ # admitting it (the read path admits it through that same door).
149
+ import amazon_relational as amazon
150
+ if key not in amazon.SOURCE:
151
+ return False
152
+ current = amazon.grid_source(cur, key) or {}
153
+ return bool(current) and not current.get("missing")
154
+ if not spec or spec.get("missing"):
155
  return False
156
  aliases = dict(spec.get("tables") or {})
157
  if spec.get("table"):
 
161
  return False
162
 
163
 
164
+ def declared_fields(table_key, definition=None):
165
+ """`mk_fields()` for one connected grid, or None when this key is not a connected grid.
166
+
167
+ ⭐ S67-T33: THE SAME FALLBACK THE GRID TAKES (`routes_odoo_tables._compiled_read`). A key that
168
+ is not one of `odoo_relational.TABLES` but IS a registered read-through source (an Amazon grid,
169
+ a generated `ut_odoo_model_*` table) answers from its stored definition's `fields`. Before
170
+ this, those databases were listed as answerable and then refused 409 on every read.
171
 
172
  ⭐ THE STABLE MIRROR CONTRACT, NOT THE STORED DEFINITION. `odoo_relational.apply_plan` built
173
  each table's stored `fields` from this same function, so reading it directly is reading the
 
179
  if key == str(table_key or ""):
180
  return [dict(field) for field in (mk_fields() or [])
181
  if isinstance(field, dict) and field.get("key")]
182
+ key = str(table_key or "")
183
+ if not isinstance(definition, dict) or not key.startswith("ut_"):
184
+ return None
185
+ try:
186
+ if _rot()._materialised_table(key):
187
+ return None
188
+ except Exception: # noqa: BLE001
189
+ return None
190
+ fields = [dict(field) for field in (definition.get("fields") or [])
191
+ if isinstance(field, dict) and field.get("key")]
192
+ return fields or None
193
 
194
 
195
+ def answerable_fields(table_key, spec, fields=None):
196
  """The declared field contract INTERSECTED with the columns the mirror can actually express.
197
 
198
  β›” THE INTERSECTION IS THE POINT, and dropping it would be the quiet defect. A declared field
 
202
  refused - a wrong answer with nothing on screen saying so. Left out, `deps` answers a request
203
  for it with a loud `assistant_unknown_field`.
204
  """
205
+ if fields is None:
206
+ fields = declared_fields(table_key)
207
  if fields is None:
208
  return None
209
  cols = set((spec or {}).get("cols") or ())
 
245
  return pred
246
 
247
 
248
+ def _predicate(rot, spec, fields, wall_tree, filter_tree, filter_keys):
249
+ """`(where, params, cols)`: the ONE compiled predicate every mirror read runs. S67-T34.
 
 
 
 
250
 
251
+ Extracted verbatim from `read_rows` so the row walk, the page read and the server-side
252
+ aggregate run the SAME wall and the SAME caller filter, compiled once, by the grid's compiler.
253
+ Three readers with one predicate cannot disagree about which rows the caller may see.
 
 
 
 
 
 
 
 
 
254
  """
 
 
255
  fe = rot._fe()
 
256
  cols = rot._filter_columns(spec, fields)
257
 
258
  # ── the administrator's wall, compiled FIRST and pushed down ──────────────────────────────
 
299
  where = f"({where}) AND {pred.sql}" if where else pred.sql
300
  params.extend(pred.params)
301
 
302
+ return where, params, cols
303
+
304
+
305
+ def read_rows(session, table_key, cur, spec, fields, *, wall_tree=None, filter_tree=None,
306
+ filter_keys=None, defn=None):
307
+ """Every permitted row of a mirror-served grid, or a refusal that names its cause.
308
+
309
+ Returns `(rows, stamp, total)`. `rows` are FULL canonical rows in `scoped_pool`'s shape
310
+ (`{pid, **cells}`), sorted by pid.
311
+
312
+ β›” NO TRUNCATION, EVER, AND THAT IS WHY THIS REFUSES RATHER THAN PAGES. Standing rule 1: a
313
+ connector's records get no cap, and a limit that genuinely cannot be removed must be REPORTED
314
+ with its cause and a recommended fix, never silently enforced. `routes_query._system_prompt`
315
+ states `PERMISSION-FILTERED RECORD COUNT: {len(records)}` to the model and the server computes
316
+ every aggregate from those same records, so a short set does not produce a smaller answer, it
317
+ produces a confidently WRONG one with nothing on screen saying so. `whole_pool` one file over
318
+ already settled this policy in the same words: REFUSE, NEVER TRIM.
319
+
320
+ β›” FULL CANONICAL ROWS, NOT THE PROJECTION. `deps.assistant_read_scope` runs the row wall and
321
+ the field projection in Python AFTER this returns. Hand it a row missing a column the wall
322
+ names and `filter_eval` reads `None` there and its verdict changes, so the belt-and-braces
323
+ pass would quietly disagree with the pushdown it is meant to double-check.
324
+ """
325
+ from harness import datastore
326
+ rot = _rot()
327
+ where, params, _cols = _predicate(rot, spec, fields, wall_tree, filter_tree, filter_keys)
328
+
329
  # ⚠ EXACTLY ONE of `table`/`from_sql` - `window` raises if both or neither arrive, so the
330
  # spec's own shape decides and a malformed spec fails loudly instead of serving a wrong FROM.
331
  select = ", ".join(f'{expr} AS "{key}"' for key, (expr, _c) in spec["cols"].items())
332
  frm = {"from_sql": spec["from_sql"]} if spec.get("from_sql") else {"table": spec["table"]}
333
+ # ⭐⭐ S67-T33 / STANDING RULE 1: EVERY ROW, NOT THE FIRST 5,000. The single `WINDOW_MAX`
334
+ # pass this replaced refused any connected database past 5,000 matching rows, which is a cap
335
+ # on a connector's records. The read now walks the whole predicate in `WINDOW_MAX` pages,
336
+ # KEYSET-paged on the spec's own id (the order the pass already used), so a deep page costs
337
+ # one index seek rather than an OFFSET walk. The keyset bound is a pagination cursor, not a
338
+ # user condition: every condition above still comes from the grid's compiler.
339
+ # β›” STILL NEVER A PARTIAL ANSWER: the first page's `total` is the population, and a walk that
340
+ # ends anywhere else (the mirror resynced mid-read) refuses by name rather than answering short.
341
+ first = datastore.window(select=f'{spec["id"]} AS "_pid", {select}',
342
+ where=where, params=tuple(params), order_by=spec["id"],
343
+ limit=datastore.WINDOW_MAX, cur=cur, **frm)
344
+ total = int(first["total"])
345
+ if total > ASSISTANT_READ_MAX:
346
  raise MirrorRefusal(
347
  409, "assistant_source_too_large",
348
+ f"this database matches {total:,} rows for that question, and one assistant answer "
349
+ f"is computed over at most {ASSISTANT_READ_MAX:,} rows so that a single question "
350
+ f"cannot exhaust the server's memory. Answering over part of them would report a "
351
+ f"wrong total, so it is refused instead. Narrow it with a filter (for example a date "
352
+ f"range) and ask again, or ask for totals as aggregates, which the server computes "
353
+ f"over every row")
354
+ got = list(first["rows"])
355
+ page = list(first["rows"])
356
+ while len(got) < total and len(page) >= datastore.WINDOW_MAX:
357
+ last = page[-1][0]
358
+ more_where = f"({where}) AND {spec['id']} > ?" if where else f"{spec['id']} > ?"
359
+ nxt = datastore.window(select=f'{spec["id"]} AS "_pid", {select}',
360
+ where=more_where, params=tuple(params) + (last,),
361
+ order_by=spec["id"], limit=datastore.WINDOW_MAX, cur=cur, **frm)
362
+ page = list(nxt["rows"])
363
+ if not page:
364
+ break
365
+ got.extend(page)
366
+ if len(got) != total:
367
+ raise MirrorRefusal(
368
+ 409, "assistant_source_changed",
369
+ f"this database changed while it was being read ({len(got):,} rows read against "
370
+ f"{total:,} counted), so the answer would not describe one consistent set of rows. "
371
+ f"Ask again once its connector sync has finished")
372
+ win = {**first, "rows": got, "total": total}
373
+
374
+ # S67-T33: the caller's rows-free definition is LENT, as the grid lends it, so the stamp does
375
+ # not re-read the whole tenant document (measured about 3 s a question on staging).
376
+ stamp = rot.mirror_stamp(spec, cur, table_key, getattr(session, "runtime", None),
377
+ defn=defn if isinstance(defn, dict) else None)
378
+ rows = rot._pool_from_window(spec, win, stamp=stamp)
379
+ rows.sort(key=lambda row: row["pid"])
380
+ return rows, stamp, int(win["total"])
381
+
382
 
383
+ #: The connector's function names mapped onto the grid's aggregate compiler (`filter_sql`).
384
+ _GRID_AGG = {"sum": "sum", "avg": "average", "count_distinct": "count_distinct"}
385
+
386
+
387
+ def _json_value(rot, value):
388
+ """A DuckDB aggregate result as plain JSON: a number through the grid's `_agg_number`, a
389
+ date or timestamp as its ISO text, anything else as is. Never a substituted 0."""
390
+ if value is None:
391
+ return None
392
+ if isinstance(value, (int, float)) and not isinstance(value, bool):
393
+ return rot._agg_number(value)
394
+ iso = getattr(value, "isoformat", None)
395
+ if callable(iso):
396
+ return iso()
397
+ try:
398
+ from decimal import Decimal
399
+ if isinstance(value, Decimal):
400
+ return rot._agg_number(value)
401
+ except Exception: # noqa: BLE001
402
+ pass
403
+ return value
404
+
405
+
406
+ def aggregate(session, table_key, cur, spec, fields, specs, *, wall_tree=None,
407
+ filter_tree=None, filter_keys=None):
408
+ """Server-side aggregates over EVERY permitted row, in SQL. S67-T34 (D-616, D-2631).
409
+
410
+ Returns `(values, total)`: `values` is `[{fn, field?, value}]` in `specs` order, `total` the
411
+ count of rows the permitted predicate matches, from `datastore.window`'s own count statement
412
+ over the SAME `where` as the aggregates beside it (the grid's `_aggregate_over` contract).
413
+
414
+ β›”β›” THE PERMISSION FILTER IS INSIDE THE QUERY. `_predicate` compiles the administrator's wall
415
+ and the caller's filter with the grid's compiler, so a sum over a million-row table runs over
416
+ exactly the rows this caller may read and no row is pulled into Python to compute it. There
417
+ is no row ceiling here (rule 1): the answer is a handful of numbers whatever the table size.
418
+
419
+ β›” AN AGGREGATE MAY NAME ONLY A VISIBLE FIELD. `filter_keys` is the caller's visible list, and
420
+ a field outside it has no expression here at all, so a sum over a hidden column cannot be
421
+ built (the caller's `_normalise_aggregates` has already refused it by name).
422
+ """
423
+ from harness import datastore
424
+ rot = _rot()
425
+ where, params, cols = _predicate(rot, spec, fields, wall_tree, filter_tree, filter_keys)
426
+ visible = set(filter_keys) if filter_keys is not None else set(cols)
427
+ fs = rot._fs()
428
+ proj, slots = [], []
429
+ for item in specs or ():
430
+ fn, field = item.get("fn"), item.get("field")
431
+ if field is None:
432
+ slots.append(None) # count(*): the window's own total
433
+ continue
434
+ col = cols.get(field) if field in visible else None
435
+ if col is None:
436
+ raise MirrorRefusal(400, "assistant_unknown_field",
437
+ "every requested field must be readable in this database")
438
+ if fn in {"sum", "avg"} and col["type"] not in fs.NUMERIC_TYPES:
439
+ raise MirrorRefusal(400, "query_aggregate_type",
440
+ "sum and avg require a readable numeric field.")
441
+ if fn in _GRID_AGG:
442
+ expr = fs.aggregate_sql(col, _GRID_AGG[fn])
443
+ elif fn in {"min", "max"}:
444
+ expr = (fs.aggregate_sql(col, fn) if col["type"] in fs.NUMERIC_TYPES
445
+ else f"{fn}({col['sql']})")
446
+ elif fn == "count":
447
+ expr = f"count({col['sql']})"
448
+ else:
449
+ raise MirrorRefusal(400, "query_aggregates_invalid",
450
+ "That aggregate is not available.")
451
+ slots.append(len(proj))
452
+ proj.append(f'{expr} AS "a{len(proj)}"')
453
+ frm = {"from_sql": spec["from_sql"]} if spec.get("from_sql") else {"table": spec["table"]}
454
+ win = datastore.window(select=(", ".join(proj) or "1"), where=where, params=tuple(params),
455
+ order_by="1", limit=1, cur=cur, **frm)
456
+ row = (win["rows"] or [()])[0]
457
+ total = int(win["total"])
458
+ out = []
459
+ for item, slot in zip(specs or (), slots):
460
+ value = total if slot is None else (_json_value(rot, row[slot]) if slot < len(row)
461
+ else None)
462
+ out.append({**item, "value": value})
463
+ return out, total
464
+
465
+
466
+ def read_page(session, table_key, cur, spec, fields, *, offset, limit, wall_tree=None,
467
+ filter_tree=None, filter_keys=None, defn=None):
468
+ """ONE page of the permitted rows, paged IN SQL, in pid order. S67-T34 (rule 1).
469
+
470
+ Returns `(rows, stamp, total)` in `read_rows`'s shape. This is how a table past
471
+ `ASSISTANT_READ_MAX` still answers its rows: the page is `LIMIT/OFFSET` over the same
472
+ predicate `read_rows` runs, so only `limit` rows ever reach this process, and `total` is the
473
+ population that predicate matches. It carries no sort other than the pid; a caller asking for
474
+ one past the ceiling is refused by name upstream, because sorting a million rows in Python
475
+ is the memory failure the ceiling exists to prevent.
476
+ """
477
+ from harness import datastore
478
+ rot = _rot()
479
+ where, params, _cols = _predicate(rot, spec, fields, wall_tree, filter_tree, filter_keys)
480
+ select = ", ".join(f'{expr} AS "{key}"' for key, (expr, _c) in spec["cols"].items())
481
+ frm = {"from_sql": spec["from_sql"]} if spec.get("from_sql") else {"table": spec["table"]}
482
+ win = datastore.window(select=f'{spec["id"]} AS "_pid", {select}', where=where,
483
+ params=tuple(params), order_by=spec["id"], offset=int(offset),
484
+ limit=int(limit), cur=cur, **frm)
485
+ stamp = rot.mirror_stamp(spec, cur, table_key, getattr(session, "runtime", None),
486
+ defn=defn if isinstance(defn, dict) else None)
487
  rows = rot._pool_from_window(spec, win, stamp=stamp)
488
  rows.sort(key=lambda row: row["pid"])
489
  return rows, stamp, int(win["total"])
api/connectors_amazon.py CHANGED
@@ -49,11 +49,80 @@ class AmazonStatusSafetyError(ValueError):
49
  """A status response contains a forbidden credential shape."""
50
 
51
 
52
- def status_payload_has_secret(serialized_payload):
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
53
  """Scan serialized JSON for C2's credential keys, token shapes and oversized values.
54
 
55
  Pure and usable without a server. Invalid JSON is unsafe too; never echo rejected content.
56
  The credential key roster is the same one the connector accepts at connect time.
 
57
  """
58
  import json
59
  import re
@@ -73,10 +142,11 @@ def status_payload_has_secret(serialized_payload):
73
  elif isinstance(value, list):
74
  pending.extend(value)
75
  elif isinstance(value, str):
76
- if (len(value) > 512 or 'Atzr|' in value
77
  or re.search(r'\bbearer\s+\S+', value, flags=re.IGNORECASE)):
78
  return True
79
- elif value is not None and len(str(value)) > 512:
 
80
  return True
81
  return False
82
 
@@ -132,8 +202,51 @@ def sync_state(rt, tenant_key, *, cadence_seconds=None, now=None):
132
  from datetime import datetime, timezone
133
  import duckdb
134
  from harness import amazon_store as az
 
135
  from routes_odoo_tables import _sync_entities, stuck_verdict
136
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
137
  def stamp(value):
138
  if not value:
139
  return None
@@ -150,7 +263,7 @@ def sync_state(rt, tenant_key, *, cadence_seconds=None, now=None):
150
  try:
151
  path = az._tenant_mirror_path(tenant_key, rt=rt)
152
  if not path.is_file():
153
- return assert_safe_status(None)
154
  # β›”β›” A SECOND `duckdb.connect(read_only=True)` ON A FILE THIS PROCESS ALREADY HOLDS RAISES
155
  # `ConnectionException: Can't open a connection to same database file with a different
156
  # configuration`. On the DEFAULT tenant layout the Amazon mirror IS the process-global
@@ -174,9 +287,9 @@ def sync_state(rt, tenant_key, *, cadence_seconds=None, now=None):
174
  mirrored = _mirror_counts(cur, az)
175
  except Exception: # noqa: BLE001
176
  # No exception text: paths, provider payloads and credentials never become status copy.
177
- return assert_safe_status(None)
178
  if entities is None:
179
- return assert_safe_status(None)
180
 
181
  clock = stamp(now) if now is not None else datetime.now(timezone.utc)
182
  states, bad_ticks = {}, set()
@@ -209,12 +322,12 @@ def sync_state(rt, tenant_key, *, cadence_seconds=None, now=None):
209
  elif (entities.get(az.entity_key(az.FIRST_LOAD_END_KEY)) or {}).get("phase") not in (None, "done"):
210
  load_unknown = "First-load progress is not available yet. Keep the connection enabled and check again after this pass."
211
 
212
- answer = {}
213
  for table in az.TABLES:
214
  landed = entities.get(az.entity_key(table)) or {}
215
  tick = entities.get(az.entity_key("sync_tick_" + table)) or {}
216
  watermark = entities.get(az.entity_key(az.WATERMARK_PREFIX + table)) or {}
217
- rows = landed.get("rows")
218
  if type(rows) is not int or rows < 0:
219
  rows = None
220
  # Hotfix 2026-09-24: during the first load no per-table marker exists yet, so the screen
@@ -222,7 +335,9 @@ def sync_state(rt, tenant_key, *, cadence_seconds=None, now=None):
222
  # own count is the measured answer.
223
  if rows is None and table in mirrored:
224
  rows = mirrored[table]
225
- successes = [when for marker in (landed, tick, watermark)
 
 
226
  if marker.get("phase") == "done"
227
  and (when := stamp(marker.get("updatedAt"))) is not None]
228
  stuck, stuck_cause = tick.get("stuck"), tick.get("stuckCause") or ""
@@ -237,8 +352,20 @@ def sync_state(rt, tenant_key, *, cadence_seconds=None, now=None):
237
  # a promise about when a later resident tick will select this table.
238
  next_due = stamp(reason.partition("next due ")[2]) if reason else None
239
  unknown = [load_unknown] if load_unknown else []
 
 
 
 
 
 
 
240
  if rows is None:
241
- unknown.append("This container has no readable row count for this table.")
 
 
 
 
 
242
  # Standing rule 1's second half: a limit that cannot be removed is REPORTED, with cause
243
  # and fix. `amazon_relational.plan` refuses a table over MAX_ROWS and said so only in the
244
  # server log, so the owner saw an empty database and no reason.
@@ -246,10 +373,68 @@ def sync_state(rt, tenant_key, *, cadence_seconds=None, now=None):
246
  if over is not None and over > _stored_table_cap():
247
  unknown.append(
248
  f"The mirror holds {over:,} rows, more than a stored table can hold "
249
- f"({_stored_table_cap():,}), so this database shows none yet. Nothing was "
250
- "truncated. Serving it straight from the mirror is scheduled for SDLC 66.")
251
- if plan is None:
 
 
 
 
 
 
 
252
  unknown.append("The sync schedule is unknown because a saved timestamp could not be read. Retry sync to refresh it.")
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
253
  answer[table] = {
254
  "rowsLoaded": rows,
255
  "lastSuccess": iso(max(successes)) if successes else None,
@@ -260,7 +445,9 @@ def sync_state(rt, tenant_key, *, cadence_seconds=None, now=None):
260
  "stuck": stuck, "stuckCause": stuck_cause,
261
  "unknownCause": " ".join(unknown),
262
  }
263
- return assert_safe_status(answer)
 
 
264
 
265
 
266
  #: W55-T19 / R7 / C10 β€” AMAZON'S OWN ENABLEMENT DOCUMENT. This is deliberately not Odoo's
 
49
  """A status response contains a forbidden credential shape."""
50
 
51
 
52
+ class AmazonStatusRowsRefused(AmazonStatusSafetyError):
53
+ """S67-T31: some status ROWS were refused; the rest are carried, already guarded.
54
+
55
+ Still an `AmazonStatusSafetyError`, so every caller that refuses on the base class keeps
56
+ refusing. `safe_status` holds only rows that passed the guard and `refused` only table NAMES:
57
+ neither carries the refused content, so a caller may degrade those rows and keep the others.
58
+ """
59
+
60
+ def __init__(self, safe_status, refused):
61
+ super().__init__('Amazon status contains a forbidden credential shape.')
62
+ self.safe_status = safe_status
63
+ self.refused = tuple(refused)
64
+
65
+
66
+ #: The status guard's scalar ceiling, and the sentence a row wears when its status is refused.
67
+ STATUS_SCALAR_LIMIT = 512
68
+ STATUS_UNAVAILABLE = "Sync status is unavailable for this database."
69
+
70
+
71
+ def trim_status(value, limit=STATUS_SCALAR_LIMIT):
72
+ """Cut every string over `limit` to a word-bounded excerpt that fits, recursively.
73
+
74
+ S67-T31: a long, honest cause (staging's Search Query Performance cause measured 935
75
+ characters) used to trip the length branch of the guard and take the whole connector modal
76
+ down with a 500. β›” Call this only AFTER the credential-shape scan of the UNTRIMMED value
77
+ (`screen_status_rows`): cutting first could slice a token shape out of a string and accept
78
+ the rest, which would trim a refusal into acceptance.
79
+ """
80
+ if isinstance(value, dict):
81
+ return {key: trim_status(item, limit) for key, item in value.items()}
82
+ if isinstance(value, list):
83
+ return [trim_status(item, limit) for item in value]
84
+ if isinstance(value, tuple):
85
+ return tuple(trim_status(item, limit) for item in value)
86
+ if isinstance(value, str) and len(value) > limit:
87
+ head = value[:limit - 3]
88
+ cut = head.rsplit(" ", 1)[0] if " " in head else head
89
+ return cut.rstrip(" ,;:") + "..."
90
+ return value
91
+
92
+
93
+ def screen_status_rows(status):
94
+ """`(safe, refused)` for a `{table: row}` status: each row judged on its own.
95
+
96
+ A row carrying a credential key, an `Atzr|` token or a bearer shape ANYWHERE in its untrimmed
97
+ form is refused whole (NC: never trimmed into acceptance, its value never returned). A row that
98
+ is only too long is trimmed, then held to the full guard again.
99
+ """
100
+ import json
101
+
102
+ safe, refused = {}, []
103
+ for name, row in (status or {}).items():
104
+ try:
105
+ whole = json.dumps(row, ensure_ascii=False, allow_nan=False)
106
+ if status_payload_has_secret(whole, max_length=None):
107
+ refused.append(name)
108
+ continue
109
+ trimmed = trim_status(row)
110
+ if status_payload_has_secret(json.dumps(trimmed, ensure_ascii=False, allow_nan=False)):
111
+ refused.append(name)
112
+ continue
113
+ except (TypeError, ValueError, RecursionError):
114
+ refused.append(name)
115
+ continue
116
+ safe[name] = trimmed
117
+ return safe, refused
118
+
119
+
120
+ def status_payload_has_secret(serialized_payload, *, max_length=STATUS_SCALAR_LIMIT):
121
  """Scan serialized JSON for C2's credential keys, token shapes and oversized values.
122
 
123
  Pure and usable without a server. Invalid JSON is unsafe too; never echo rejected content.
124
  The credential key roster is the same one the connector accepts at connect time.
125
+ `max_length=None` scans for credential shapes only (the pre-trim pass of `screen_status_rows`).
126
  """
127
  import json
128
  import re
 
142
  elif isinstance(value, list):
143
  pending.extend(value)
144
  elif isinstance(value, str):
145
+ if ((max_length is not None and len(value) > max_length) or 'Atzr|' in value
146
  or re.search(r'\bbearer\s+\S+', value, flags=re.IGNORECASE)):
147
  return True
148
+ elif (value is not None and max_length is not None
149
+ and len(str(value)) > max_length):
150
  return True
151
  return False
152
 
 
202
  from datetime import datetime, timezone
203
  import duckdb
204
  from harness import amazon_store as az
205
+ from harness.amazon_client import DECLARED_DATASETS
206
  from routes_odoo_tables import _sync_entities, stuck_verdict
207
 
208
+ # S66-T46 (lead): an unreadable deletion ledger is flagged once per row as
209
+ # `purgeScheduleUnknown`, never written into every table's cause, which would push out the
210
+ # reason each table already names (Amazon's refusal, the empty period, the declared reason).
211
+ purge_unknown = False
212
+ try:
213
+ purge_plan = az.scheduled_purge(tenant_key, rt, now=now)
214
+ purge_message = az.deletion_schedule_message(purge_plan) if purge_plan else ""
215
+ except Exception:
216
+ purge_plan, purge_message, purge_unknown = None, "", True
217
+
218
+ def finish(status):
219
+ for row in status.values():
220
+ # A key only when there is something to say: a declared dataset's row stays exactly
221
+ # its declaration while no purge is scheduled (the group (a) status contract).
222
+ if purge_plan:
223
+ row["purgeScheduledAt"] = purge_plan["purgeAt"]
224
+ if purge_unknown and not row.get("declared"):
225
+ row["purgeScheduleUnknown"] = True
226
+ if purge_message:
227
+ cause = row.get("unknownCause") or ""
228
+ # Status scalars have a 512-character safety ceiling. Keep the full existing
229
+ # cause separately if the deletion sentence needs a shorter screen excerpt.
230
+ budget = 512 - len(purge_message) - 1
231
+ if len(cause) > budget:
232
+ row["syncCause"] = cause
233
+ cause = cause[:budget - 3].rsplit(" ", 1)[0] + "..."
234
+ row["unknownCause"] = " ".join(part for part in (cause, purge_message) if part)
235
+ # S67-T31: judged PER ROW. An over-long cause is trimmed (staging's 500 was
236
+ # `search_query_performance.unknownCause` at 935 characters); a credential shape still
237
+ # refuses, now naming only its rows so the caller degrades those and keeps the rest.
238
+ safe, refused = screen_status_rows(status)
239
+ if refused:
240
+ raise AmazonStatusRowsRefused(assert_safe_status(safe), refused)
241
+ return assert_safe_status(safe)
242
+
243
+ declared = {name: {"rowsLoaded": None, "declared": True, "unknownCause": spec["reason"]}
244
+ for name, spec in DECLARED_DATASETS.items()}
245
+ unavailable = {table: {"rowsLoaded": None, "lastSuccess": None, "inProgress": False,
246
+ "unknownCause": "This container has no readable sync status. Sync this database to load its rows."}
247
+ for table in az.TABLES}
248
+ unavailable.update(declared)
249
+
250
  def stamp(value):
251
  if not value:
252
  return None
 
263
  try:
264
  path = az._tenant_mirror_path(tenant_key, rt=rt)
265
  if not path.is_file():
266
+ return finish(unavailable)
267
  # β›”β›” A SECOND `duckdb.connect(read_only=True)` ON A FILE THIS PROCESS ALREADY HOLDS RAISES
268
  # `ConnectionException: Can't open a connection to same database file with a different
269
  # configuration`. On the DEFAULT tenant layout the Amazon mirror IS the process-global
 
287
  mirrored = _mirror_counts(cur, az)
288
  except Exception: # noqa: BLE001
289
  # No exception text: paths, provider payloads and credentials never become status copy.
290
+ return finish(unavailable)
291
  if entities is None:
292
+ return finish(unavailable)
293
 
294
  clock = stamp(now) if now is not None else datetime.now(timezone.utc)
295
  states, bad_ticks = {}, set()
 
322
  elif (entities.get(az.entity_key(az.FIRST_LOAD_END_KEY)) or {}).get("phase") not in (None, "done"):
323
  load_unknown = "First-load progress is not available yet. Keep the connection enabled and check again after this pass."
324
 
325
+ answer = dict(declared)
326
  for table in az.TABLES:
327
  landed = entities.get(az.entity_key(table)) or {}
328
  tick = entities.get(az.entity_key("sync_tick_" + table)) or {}
329
  watermark = entities.get(az.entity_key(az.WATERMARK_PREFIX + table)) or {}
330
+ rows = mirrored.get(table, landed.get("rows"))
331
  if type(rows) is not int or rows < 0:
332
  rows = None
333
  # Hotfix 2026-09-24: during the first load no per-table marker exists yet, so the screen
 
335
  # own count is the measured answer.
336
  if rows is None and table in mirrored:
337
  rows = mirrored[table]
338
+ # A refused pass closes its tick (S66-T42); only the watermark can then prove a success.
339
+ refused_now = landed.get("phase") in ("refused", "failed", "throttled")
340
+ successes = [when for marker in ((watermark,) if refused_now else (landed, tick, watermark))
341
  if marker.get("phase") == "done"
342
  and (when := stamp(marker.get("updatedAt"))) is not None]
343
  stuck, stuck_cause = tick.get("stuck"), tick.get("stuckCause") or ""
 
352
  # a promise about when a later resident tick will select this table.
353
  next_due = stamp(reason.partition("next due ")[2]) if reason else None
354
  unknown = [load_unknown] if load_unknown else []
355
+ proof = entities.get(az.entity_key("reconcile_" + table)) or {}
356
+ try:
357
+ period = json.loads(proof.get("cursor") or "{}")
358
+ if not isinstance(period, dict):
359
+ period = {}
360
+ except (ValueError, TypeError):
361
+ period = {}
362
  if rows is None:
363
+ unknown.append("This table has not loaded yet. Sync it to request rows from Amazon.")
364
+ if landed.get("phase") in ("refused", "failed", "throttled"):
365
+ unknown = [str(landed.get("cursor") or "Amazon refused this dataset. Retry sync to refresh its reason.")]
366
+ elif rows == 0 or period.get("count") == 0:
367
+ span = " to ".join(dict.fromkeys(str(period[k]) for k in ("start", "end") if period.get(k)))
368
+ unknown.append("Amazon returned no rows for " + (span or "the last requested period") + ".")
369
  # Standing rule 1's second half: a limit that cannot be removed is REPORTED, with cause
370
  # and fix. `amazon_relational.plan` refuses a table over MAX_ROWS and said so only in the
371
  # server log, so the owner saw an empty database and no reason.
 
373
  if over is not None and over > _stored_table_cap():
374
  unknown.append(
375
  f"The mirror holds {over:,} rows, more than a stored table can hold "
376
+ f"({_stored_table_cap():,}). This table is served read-through from the mirror "
377
+ "with no row cap.")
378
+ # S66-T42/T43 (lead): a table `amazon_relational.apply_plan` could not create because the
379
+ # workspace is at the store's table cap says so here, with the cause and the fix.
380
+ not_stored = entities.get(az.entity_key("not_stored_" + table)) or {}
381
+ if not_stored.get("phase") == "not_stored":
382
+ unknown.append(str(not_stored.get("cursor") or
383
+ "This table is not in this workspace yet because the workspace is at "
384
+ "its database limit. Its rows are kept in the Amazon mirror."))
385
+ if plan is None and landed.get("phase") not in ("refused", "failed", "throttled"):
386
  unknown.append("The sync schedule is unknown because a saved timestamp could not be read. Retry sync to refresh it.")
387
+ limit = None
388
+ if table in az.GROUP_A_TABLES and table != "catalog_items":
389
+ marker = entities.get(az.entity_key("limit_" + table)) or {}
390
+ try:
391
+ limit = json.loads(marker.get("cursor") or "{}").get("reason")
392
+ except (ValueError, TypeError, AttributeError):
393
+ limit = "History progress is unreadable. Retry sync to refresh its saved date."
394
+ if limit:
395
+ unknown.append(limit)
396
+ if table == "catalog_items":
397
+ marker = entities.get(az.entity_key("limit_" + table)) or {}
398
+ try:
399
+ limit = json.loads(marker.get("cursor") or "{}").get("reason")
400
+ except (ValueError, TypeError, AttributeError):
401
+ limit = "Catalog request progress is unreadable. Sync Amazon listings and retry."
402
+ limit = limit or ("Amazon Catalog allows 20 ASINs per search and 2 requests per second. "
403
+ "Every own ASIN and page is followed; sync Amazon listings first.")
404
+ unknown.append(limit)
405
+ if table == "listing_items":
406
+ limit = ("Amazon Listings allows 1,000 SKUs per query and 5 requests per second. "
407
+ "Date ranges are split and every page followed. Keep the connection enabled.")
408
+ unknown.append(limit)
409
+ if table == az.SQP_TABLE:
410
+ marker = entities.get(az.SQP_LIMIT_KEY) or {}
411
+ if marker.get("cursor"):
412
+ try:
413
+ detail = json.loads(marker["cursor"])
414
+ limit = az.sqp_limit_sentence(detail)
415
+ unknown.append(limit)
416
+ # S66-T41 (lead): one sentence PER REASON, so a whole refused batch (18
417
+ # ASINs) stays inside `assert_safe_status`'s 512-character scalar ceiling and
418
+ # every refused ASIN is still named on screen.
419
+ by_reason = {}
420
+ for asin, refusal in detail.get("refusedAsins", {}).items():
421
+ by_reason.setdefault(str(refusal), []).append(str(asin))
422
+ for refusal, asins in by_reason.items():
423
+ unknown.append(az.sqp_refusal_message(asins[0], refusal) if len(asins) == 1
424
+ else f"Amazon refused ASINs {', '.join(sorted(asins))} for "
425
+ f"Search Query Performance: {refusal}")
426
+ unknown.extend(detail.get("errors", []))
427
+ if detail.get("deferredReason"):
428
+ reason = detail["deferredReason"]
429
+ if detail.get("partialMonthReason"):
430
+ unknown.append(detail["partialMonthReason"])
431
+ except (ValueError, TypeError, KeyError, AttributeError):
432
+ unknown.append("Search Query Performance limits are unknown because the saved marker could not be read. Retry sync to refresh it.")
433
+ if table in az.GROUP_A_TABLES and len(" ".join(unknown)) > 512:
434
+ # Preserve the dataset's refusal/empty-period and vendor limit. Generic schedule
435
+ # diagnostics can otherwise overflow the status guard and hide every database.
436
+ unknown = [part for part in unknown if part != load_unknown
437
+ and not part.startswith(("The sync schedule is unknown", "This table has not loaded yet."))]
438
  answer[table] = {
439
  "rowsLoaded": rows,
440
  "lastSuccess": iso(max(successes)) if successes else None,
 
445
  "stuck": stuck, "stuckCause": stuck_cause,
446
  "unknownCause": " ".join(unknown),
447
  }
448
+ if table == az.SQP_TABLE or table in az.GROUP_A_TABLES:
449
+ answer[table]["limit"] = limit
450
+ return finish(answer)
451
 
452
 
453
  #: W55-T19 / R7 / C10 β€” AMAZON'S OWN ENABLEMENT DOCUMENT. This is deliberately not Odoo's
api/content_studio.py ADDED
@@ -0,0 +1,638 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Content Studio: an App template that creates its own References and Posts databases.
2
+
3
+ S67-T40 (owner instruction 1; rulings R17, R18, R21, R11). Plan-only (round 1 R5): nothing here
4
+ publishes to any network. The template creates two TENANT-OWNED databases and binds the App to
5
+ Posts, opening on a calendar of posts. The App is delete-safe (rule 13, ARCHITECTURE section 24):
6
+ deleting it removes only the App record, so References and Posts and every row in them outlive it.
7
+
8
+ This module is the App's own code. Shared files reach it only through a guarded import
9
+ (`routes_templates.py`), so deleting this file removes the template and nothing else breaks.
10
+ """
11
+ from __future__ import annotations
12
+
13
+ TEMPLATE_KEY = "content_studio"
14
+ LABEL = "Content Studio"
15
+ DESCRIPTION = "Plan posts from your references. Creates References and Posts databases."
16
+
17
+ #: R4: the channels a post is planned for. Channel length limits are ADVISED, never enforced (O4).
18
+ CHANNELS = ["LinkedIn", "X", "Instagram", "TikTok", "YouTube", "Reddit"]
19
+ NETWORKS = CHANNELS + ["Other"]
20
+ STATUSES = ["Idea", "Draft", "Ready", "Posted", "Skipped"]
21
+ CALENDAR_VIEW_ID = "cs_calendar"
22
+
23
+ #: R15 (structure, not words): the three fields a Reference is drafted FROM are extracted from its
24
+ #: pasted text by the per-row AI field machinery (`ai_enrich`), re-run when that text changes.
25
+ _EXTRACT = {
26
+ "hook": ("Hook", "In one sentence, describe the opening move this post uses to earn attention "
27
+ "(for example a surprising number, a contrarian claim, a question). Describe "
28
+ "the technique; do not quote the post. Post: {pasted_text}"),
29
+ "structure": ("Structure", "List the post's structure as 3 to 6 short numbered beats, each "
30
+ "naming what that part does (setup, proof, turn, ask). Describe, "
31
+ "never quote. Post: {pasted_text}"),
32
+ "angle": ("Angle", "In one sentence, name the point of view or argument the post takes and "
33
+ "who it is for. Do not quote the post. Post: {pasted_text}"),
34
+ }
35
+
36
+
37
+ #: The platform ladder leads with a REASONING model, which spends a few hundred tokens thinking
38
+ #: before it answers; at 300 the column's ceiling was reached with no answer ("the model had
39
+ #: nothing to answer from"), so each extraction gets room for thought plus a short answer.
40
+ EXTRACT_MAX_TOKENS = 1500
41
+
42
+
43
+ def reference_fields():
44
+ """References: URL, network, author, pasted text, tags, hook, structure, angle (T40 done-when)."""
45
+ fields = [
46
+ {"key": "title", "label": "Title", "type": "text"},
47
+ {"key": "url", "label": "URL", "type": "url"},
48
+ {"key": "network", "label": "Network", "type": "select", "options": list(NETWORKS)},
49
+ {"key": "author", "label": "Author", "type": "text"},
50
+ {"key": "pasted_text", "label": "Pasted text", "type": "text"},
51
+ {"key": "tags", "label": "Tags", "type": "multiselect", "options": []},
52
+ ]
53
+ for key, (label, prompt) in _EXTRACT.items():
54
+ fields.append({"key": key, "label": label, "type": "ai_enrich",
55
+ "aiEnrich": {"prompt": prompt, "maxTokens": EXTRACT_MAX_TOKENS,
56
+ "trigger": {"mode": "on_change"}, "overwrite": "always"}})
57
+ return fields
58
+
59
+
60
+ def post_fields(references_key):
61
+ """Posts: caption, channel, scheduled, status, media, campaign, post URL, References link,
62
+ attribution (T40 done-when). `media` is an `image` field: the store has no attachment type."""
63
+ return [
64
+ {"key": "caption", "label": "Caption", "type": "text"},
65
+ {"key": "channel", "label": "Channel", "type": "select", "options": list(CHANNELS)},
66
+ {"key": "scheduled", "label": "Scheduled", "type": "date"},
67
+ {"key": "status", "label": "Status", "type": "select", "options": list(STATUSES)},
68
+ {"key": "media", "label": "Media", "type": "image"},
69
+ {"key": "campaign", "label": "Campaign", "type": "text"},
70
+ {"key": "post_url", "label": "Post URL", "type": "url"},
71
+ {"key": "references", "label": "References", "type": "link",
72
+ "link": {"table": references_key, "selectionMode": "manual"}},
73
+ {"key": "attribution", "label": "Attribution", "type": "text"},
74
+ ]
75
+
76
+
77
+ class ContentStudioError(RuntimeError):
78
+ """A refusal the route turns into a plain sentence; nothing half-created is left behind."""
79
+
80
+
81
+ def _create_table(label, username, fields, st):
82
+ """`user_tables.create` refuses a duplicate label, so a second Studio gets "Posts 2" and so on."""
83
+ from core import user_tables
84
+ for attempt in range(1, 10):
85
+ name = label if attempt == 1 else f"{label} {attempt}"
86
+ key = user_tables.create(name, username, fields=fields, st=st)
87
+ if key:
88
+ return key
89
+ raise ContentStudioError(f"The {label} database could not be created. "
90
+ "Rename or delete an older one, then try again.")
91
+
92
+
93
+ def create(session, name=""):
94
+ """Create References + Posts (tenant-owned), a calendar View on Posts, and the App bound to
95
+ Posts. Returns `{address, application, databases}`; on a failure part-way, the databases this
96
+ call created are deleted again, so a refusal leaves the workspace as it was."""
97
+ import application_registry
98
+ from core import user_tables, view_templates
99
+ from core import table_store
100
+
101
+ st = session.runtime
102
+ user = session.uname
103
+ app_name = " ".join(str(name or "").split())[:120] or LABEL
104
+ created = []
105
+ try:
106
+ references = _create_table("References", user, reference_fields(), st)
107
+ created.append(references)
108
+ posts = _create_table("Posts", user, post_fields(references), st)
109
+ created.append(posts)
110
+ ops = table_store.make(view_templates.workspace_key(posts), st=st)
111
+ ops.save_view(user, {"id": CALENDAR_VIEW_ID, "name": "Calendar", "createdBy": user,
112
+ "config": {"filters": [], "sorts": [],
113
+ "display": {"mode": "calendar", "dateField": "scheduled",
114
+ "titleField": "caption"}}})
115
+ application = application_registry.create(
116
+ st, session.tenant, user,
117
+ {"templateKind": TEMPLATE_KEY, "name": app_name, "sourceDatabase": posts,
118
+ "navPlacement": {"section": "app", "order": 0}},
119
+ allowed_databases=None)
120
+ except Exception:
121
+ for key in reversed(created):
122
+ try:
123
+ user_tables.delete(key, st=st)
124
+ except Exception:
125
+ pass
126
+ raise
127
+ # S67-T41 owner correction: the App opens on its own surface in the App module, not in Data.
128
+ return {"address": "#" + app_link(application["id"]), "application": application,
129
+ "databases": {"references": references, "posts": posts}}
130
+
131
+
132
+ # ---------------------------------------------------------------------------------------------
133
+ # S67-T41 (R13, R14, R15, R19, R20, R5, O4): DRAFTING. One Draft row per chosen channel, from
134
+ # several selected References plus a one-line topic, written from their extracted STRUCTURE
135
+ # (hook, structure, angle), never their words, with every author credited. Plan-only: a draft
136
+ # is a Posts row with status Draft; nothing is published anywhere.
137
+ # ---------------------------------------------------------------------------------------------
138
+ import re
139
+
140
+ from fastapi import APIRouter, Body, Depends
141
+
142
+ from deps import Session, err, require_session
143
+
144
+ router = APIRouter(prefix="/content-studio")
145
+
146
+ #: O4: channel limits are ADVICE in the prompt and on screen, never a truncation.
147
+ CHANNEL_ADVICE = {
148
+ "LinkedIn": "LinkedIn post: up to about 3,000 characters, short paragraphs, no hashtag wall.",
149
+ "X": "X post: aim for 280 characters or fewer.",
150
+ "Instagram": "Instagram caption: up to about 2,200 characters; a strong first line.",
151
+ "TikTok": "TikTok caption: short, about 150 characters.",
152
+ "YouTube": "YouTube: a title line, then a two or three sentence description.",
153
+ "Reddit": "Reddit: a title line, then the body; plain, specific, no marketing tone.",
154
+ }
155
+ DRAFT_SYSTEM = (
156
+ "You draft ONE original social media post for the channel named. You are given a one-line "
157
+ "topic and the STRUCTURE of several reference posts: the technique of each hook, its beats "
158
+ "and its angle. Borrow those structural moves; write entirely new sentences. Never copy or "
159
+ "closely paraphrase any sentence from anywhere. Answer with the post text only: no preamble, "
160
+ "no quotation marks around it, no markdown heading. Never use an em dash or an en dash; use "
161
+ "a comma, a colon or a full stop.")
162
+ LEDGER_SURFACE = "field_agent"
163
+ DRAFT_MAX_TOKENS = 3000 # a reasoning model thinks first; see EXTRACT_MAX_TOKENS
164
+ DRAFT_TIMEOUT_S = 60
165
+ #: A shared run of this many words is a copied sentence, whatever surrounds it (R15).
166
+ VERBATIM_WORDS = 8
167
+
168
+
169
+ def _studio(session, app_id):
170
+ """(app, posts_key, references_key) for a Content Studio App this session may open."""
171
+ import application_registry
172
+ from core import user_tables
173
+ doc = application_registry.load(session.runtime)
174
+ app = doc["applications"].get(str(app_id or ""))
175
+ if (not isinstance(app, dict) or app.get("templateKind") != TEMPLATE_KEY
176
+ or not application_registry.may_read(app, session.uname, session.admin)):
177
+ raise err(404, "content_studio_not_found", "That Content Studio does not exist.")
178
+ posts = str(app.get("sourceDatabase") or "")
179
+ defn = user_tables.get(posts, session.runtime) or {}
180
+ link = next((f.get("link") for f in defn.get("fields") or ()
181
+ if isinstance(f, dict) and f.get("key") == "references"), None)
182
+ references = str((link or {}).get("table") or "")
183
+ if not posts or not references:
184
+ raise err(409, "content_studio_incomplete",
185
+ "This Content Studio has lost its Posts or References database.")
186
+ return app, posts, references
187
+
188
+
189
+ def _reference_rows(session, references):
190
+ from core import user_tables
191
+ defn = user_tables.get(references, session.runtime) or {}
192
+ rows = defn.get("rows") or {}
193
+ return {str(rid): (row if isinstance(row, dict) else {}) for rid, row in rows.items()}
194
+
195
+
196
+ # ---------------------------------------------------------------------------------------------
197
+ # S67-T42 (R35, standing rule 1): THE MONTHLY DRAFTING ALLOWANCE. Drafting runs on Loopable's
198
+ # platform keys, so each tenant gets a monthly count of drafts (one Draft row = one draft). Past
199
+ # it, Draft pauses with a sentence naming the reset date; the platform admin raises it per tenant
200
+ # (`PUT /content-studio/allowance`). A tenant admin cannot: that would defeat R35.
201
+ # One per-tenant bucket, owned by this module: {"allowance": int (absent = default),
202
+ # "months": {"YYYY-MM": drafts made}}. Month and reset are UTC.
203
+ # ---------------------------------------------------------------------------------------------
204
+ ALLOWANCE_KEY = "content_studio_usage"
205
+ DEFAULT_MONTHLY_DRAFTS = 100
206
+ ALLOWANCE_KEEP_MONTHS = 12
207
+ _MONTH_NAMES = ("January", "February", "March", "April", "May", "June", "July", "August",
208
+ "September", "October", "November", "December")
209
+
210
+
211
+ def _now():
212
+ from datetime import datetime, timezone
213
+ return datetime.now(timezone.utc)
214
+
215
+
216
+ def _month_key(now):
217
+ return f"{now.year:04d}-{now.month:02d}"
218
+
219
+
220
+ def _reset_on(now):
221
+ """The first day of next month (UTC): the day this month's count stops counting."""
222
+ from datetime import date
223
+ return date(now.year + 1, 1, 1) if now.month == 12 else date(now.year, now.month + 1, 1)
224
+
225
+
226
+ def _spoken_date(day):
227
+ return f"{_MONTH_NAMES[day.month - 1]} {day.day}, {day.year}"
228
+
229
+
230
+ def _limit(doc):
231
+ raw = (doc or {}).get("allowance")
232
+ if isinstance(raw, bool) or not isinstance(raw, int) or raw < 0:
233
+ return DEFAULT_MONTHLY_DRAFTS
234
+ return raw
235
+
236
+
237
+ def allowance_state(doc, now=None):
238
+ """{used, limit, remaining, resetsOn, paused, message}: the ONE reading of month and reset
239
+ that both the panel and the enforcement use."""
240
+ now = now or _now()
241
+ doc = doc if isinstance(doc, dict) else {}
242
+ months = doc.get("months") if isinstance(doc.get("months"), dict) else {}
243
+ try:
244
+ used = max(0, int(months.get(_month_key(now)) or 0))
245
+ except (TypeError, ValueError):
246
+ used = 0
247
+ limit = _limit(doc)
248
+ remaining = max(0, limit - used)
249
+ reset = _reset_on(now)
250
+ paused = remaining <= 0
251
+ message = (f"This workspace has used this month's drafting allowance. "
252
+ f"It resets on {_spoken_date(reset)}.") if paused else ""
253
+ return {"used": used, "limit": limit, "remaining": remaining,
254
+ "resetsOn": reset.isoformat(), "paused": paused, "message": message}
255
+
256
+
257
+ def _short_message(remaining):
258
+ drafts = "1 draft" if remaining == 1 else f"{remaining} drafts"
259
+ channels = "1 channel" if remaining == 1 else f"up to {remaining} channels"
260
+ return f"This workspace has {drafts} left this month. Choose {channels}."
261
+
262
+
263
+ def _refusal(state, wanted):
264
+ """The 429 a draft of `wanted` drafts earns against `state`, or None when it fits."""
265
+ if state["remaining"] <= 0:
266
+ return err(429, "drafting_allowance_used", state["message"])
267
+ if state["remaining"] < wanted:
268
+ return err(429, "drafting_allowance_short", _short_message(state["remaining"]))
269
+ return None
270
+
271
+
272
+ def _read_allowance(st):
273
+ doc = st.get(ALLOWANCE_KEY)
274
+ return doc if isinstance(doc, dict) else {}
275
+
276
+
277
+ def _count(st, month, delta):
278
+ """Add `delta` drafts to `month` in one read-modify-write; keeps the last 12 months."""
279
+ def _up(doc):
280
+ doc = dict(doc) if isinstance(doc, dict) else {}
281
+ months = dict(doc.get("months")) if isinstance(doc.get("months"), dict) else {}
282
+ months[month] = max(0, int(months.get(month) or 0) + int(delta))
283
+ for stale in sorted(months)[:-ALLOWANCE_KEEP_MONTHS]:
284
+ months.pop(stale, None)
285
+ doc["months"] = months
286
+ return doc
287
+ st.update(ALLOWANCE_KEY, _up, flush="sync")
288
+
289
+
290
+ def reserve_drafts(st, wanted, now=None):
291
+ """Check AND reserve `wanted` drafts in ONE store update, so two concurrent drafts cannot both
292
+ take the last slot. Raises the 429 (no write) when it does not fit; returns the month key the
293
+ reservation was booked to, for `release_drafts` on a failure."""
294
+ now = now or _now()
295
+ try:
296
+ current = _read_allowance(st)
297
+ except Exception: # noqa: BLE001
298
+ raise err(503, "allowance_unavailable",
299
+ "The drafting allowance could not be checked, so nothing was drafted. Try again.")
300
+ refusal = _refusal(allowance_state(current, now), wanted)
301
+ if refusal is not None: # fast path: a paused workspace writes nothing
302
+ raise refusal
303
+ month = _month_key(now)
304
+ verdict = {}
305
+
306
+ def _up(doc):
307
+ doc = dict(doc) if isinstance(doc, dict) else {}
308
+ refused = _refusal(allowance_state(doc, now), wanted)
309
+ if refused is not None:
310
+ verdict["refusal"] = refused
311
+ return doc
312
+ months = dict(doc.get("months")) if isinstance(doc.get("months"), dict) else {}
313
+ months[month] = max(0, int(months.get(month) or 0)) + int(wanted)
314
+ for stale in sorted(months)[:-ALLOWANCE_KEEP_MONTHS]:
315
+ months.pop(stale, None)
316
+ doc["months"] = months
317
+ return doc
318
+
319
+ try:
320
+ st.update(ALLOWANCE_KEY, _up, flush="sync")
321
+ except Exception: # noqa: BLE001
322
+ raise err(503, "allowance_unavailable",
323
+ "The drafting allowance could not be checked, so nothing was drafted. Try again.")
324
+ if "refusal" in verdict:
325
+ raise verdict["refusal"]
326
+ return month
327
+
328
+
329
+ def release_drafts(st, month, n):
330
+ """Hand back a reservation whose drafts were never written (AI failure, copy refusal, store)."""
331
+ try:
332
+ _count(st, month, -int(n))
333
+ except Exception: # noqa: BLE001
334
+ pass
335
+
336
+
337
+ @router.put("/allowance")
338
+ def set_allowance(body: dict = Body(default=None), session: Session = Depends(require_session)):
339
+ """The owner raises (or resets) one tenant's monthly drafting allowance. Platform admin only."""
340
+ import core.platform_admin as platform_admin
341
+ from harness import runtime
342
+ if not platform_admin.is_platform_admin(session.user):
343
+ raise err(403, "forbidden", "your account does not have access to this surface")
344
+ body = body if isinstance(body, dict) else {}
345
+ slug = str(body.get("tenant") or "").strip().lower()
346
+ monthly = body.get("monthly")
347
+ if monthly is not None and (isinstance(monthly, bool) or not isinstance(monthly, int)
348
+ or monthly < 0):
349
+ raise err(400, "bad_allowance",
350
+ "The monthly allowance must be a whole number of drafts, 0 or more.")
351
+ if not slug or slug not in runtime.known_tenants():
352
+ raise err(404, "tenant_not_found", "That workspace does not exist.")
353
+ try:
354
+ st = runtime.get_runtime(slug)
355
+ except KeyError:
356
+ raise err(404, "tenant_not_found", "That workspace does not exist.")
357
+
358
+ def _up(doc):
359
+ doc = dict(doc) if isinstance(doc, dict) else {}
360
+ if monthly is None:
361
+ doc.pop("allowance", None)
362
+ else:
363
+ doc["allowance"] = int(monthly)
364
+ return doc
365
+
366
+ st.update(ALLOWANCE_KEY, _up, flush="sync")
367
+ return {"tenant": slug, "allowance": allowance_state(_read_allowance(st))}
368
+
369
+
370
+ @router.get("/references")
371
+ def list_references(app: str = "", session: Session = Depends(require_session)):
372
+ """The References a Draft can use: id, title, author, network, and whether its structure is in.
373
+ Also this workspace's drafting allowance, so the panel reports the limit before it bites."""
374
+ _app, _posts, references = _studio(session, app)
375
+ rows = _reference_rows(session, references)
376
+ try:
377
+ allowance = allowance_state(_read_allowance(session.runtime))
378
+ except Exception: # noqa: BLE001
379
+ allowance = None # unreadable: the draft route still enforces it
380
+ return {"references": [
381
+ {"id": rid, "title": str(row.get("title") or row.get("url") or "Untitled reference")[:160],
382
+ "author": str(row.get("author") or "")[:120], "network": str(row.get("network") or ""),
383
+ "extracted": bool(str(row.get("structure") or "").strip())}
384
+ for rid, row in rows.items()],
385
+ "channels": list(CHANNELS), "advice": dict(CHANNEL_ADVICE), "allowance": allowance}
386
+
387
+
388
+ # ---------------------------------------------------------------------------------------------
389
+ # S67-T41 OWNER CORRECTION (Telegram update 437342853, 2026-09-28): "Content studio should be an
390
+ # App ... it should act like Catalog / Route where it just uses the data module as what its
391
+ # building on top." So the App opens in the App module on its OWN surface (`#/app?studio=<id>`,
392
+ # `apps/contentStudio/ContentStudioApp.tsx`): a calendar and a board over Posts, the Draft door,
393
+ # and the two databases underneath. Data still holds References and Posts; the App never sends
394
+ # anyone there to use it.
395
+ # ⚠ The shared `/applications` doors (`routes_assets.py`) admit only topic-backed databases, so
396
+ # they refuse an App bound to a user table; the read, star and delete this App needs live here,
397
+ # on the registry's own writers, behind the same `_studio` + `may_read` / `may_edit` walls.
398
+ # ---------------------------------------------------------------------------------------------
399
+ def app_link(app_id):
400
+ """The one address a Content Studio App opens at (App module, its own surface)."""
401
+ return f"/app?studio={app_id}"
402
+
403
+
404
+ #: The Posts fields the surface draws, in the grid's own Field shape so `viewModes.CalendarView`
405
+ #: (by Scheduled) and `KanbanView` (by Status) render them unchanged.
406
+ SURFACE_FIELDS = ("caption", "channel", "scheduled", "status", "attribution", "campaign")
407
+
408
+
409
+ def _surface_field(defn):
410
+ kind = {"date": "date", "select": "select"}.get(defn.get("type"), "text")
411
+ field = {"key": defn["key"], "label": str(defn.get("label") or defn["key"]), "type": kind,
412
+ "source": "overlay"}
413
+ if kind == "select":
414
+ field["options"] = [str(o) for o in (defn.get("options") or ())]
415
+ return field
416
+
417
+
418
+ def _studio_or_missing(session, app_id):
419
+ """`_studio`, answering an absent App with the code the App home already reads as 'gone'."""
420
+ try:
421
+ return _studio(session, app_id)
422
+ except Exception as exc: # noqa: BLE001
423
+ if getattr(exc, "status_code", None) == 404:
424
+ raise err(404, "application_not_found", "That Content Studio no longer exists.")
425
+ raise
426
+
427
+
428
+ @router.get("/studio")
429
+ def studio(app: str = "", session: Session = Depends(require_session)):
430
+ """Everything the App surface draws: the App, Posts (surface fields + rows), References'
431
+ count, the statuses in board order, and the drafting allowance."""
432
+ from core import user_tables
433
+ application, posts, references = _studio(session, app)
434
+ if not user_tables.may_open(posts, session.uname, session.admin, st=session.runtime):
435
+ raise err(404, "content_studio_not_found", "That Content Studio does not exist.")
436
+ posts_def = user_tables.get(posts, session.runtime) or {}
437
+ refs_def = user_tables.get(references, session.runtime) or {}
438
+ by_key = {f.get("key"): f for f in posts_def.get("fields") or () if isinstance(f, dict)}
439
+ fields = [_surface_field(by_key[key]) for key in SURFACE_FIELDS if key in by_key]
440
+ rows = []
441
+ for rid, row in (posts_def.get("rows") or {}).items():
442
+ try:
443
+ pid = int(rid)
444
+ except (TypeError, ValueError):
445
+ continue
446
+ row = row if isinstance(row, dict) else {}
447
+ rows.append({"pid": pid, **{key: row.get(key) for key in SURFACE_FIELDS}})
448
+ rows.sort(key=lambda r: r["pid"])
449
+ try:
450
+ allowance = allowance_state(_read_allowance(session.runtime))
451
+ except Exception: # noqa: BLE001
452
+ allowance = None
453
+ return {"app": {"id": application["id"], "name": application.get("name") or LABEL},
454
+ "posts": {"database": posts, "label": str(posts_def.get("label") or "Posts"),
455
+ "fields": fields, "rows": rows},
456
+ "references": {"database": references,
457
+ "label": str(refs_def.get("label") or "References"),
458
+ "count": len(refs_def.get("rows") or {})},
459
+ "statuses": list(STATUSES), "allowance": allowance}
460
+
461
+
462
+ @router.patch("/apps/{app_id}")
463
+ def star_app(app_id: str, body: dict = Body(default=None),
464
+ session: Session = Depends(require_session)):
465
+ """Star or unstar a Content Studio on the App home (the registry's own star writer)."""
466
+ import application_registry
467
+ _app, posts, _references = _studio_or_missing(session, app_id)
468
+ body = body if isinstance(body, dict) else {}
469
+ if set(body) != {"starred"} or not isinstance(body.get("starred"), bool):
470
+ raise err(400, "bad_patch", "Only the star of a Content Studio changes here.")
471
+ try:
472
+ landed = application_registry.mutate(session.runtime, app_id, session.uname, session.admin,
473
+ {"starred": body["starred"]}, allowed_databases=[posts])
474
+ except PermissionError:
475
+ raise err(403, "forbidden", "You cannot change this App.")
476
+ except application_registry.ApplicationError:
477
+ raise err(404, "application_not_found", "That Content Studio no longer exists.")
478
+ return {"application": landed}
479
+
480
+
481
+ @router.delete("/apps/{app_id}")
482
+ def delete_app(app_id: str, body: dict = Body(default=None),
483
+ session: Session = Depends(require_session)):
484
+ """Delete the App and nothing else (rule 13): References and Posts, every row in them, and
485
+ every View on them outlive it. The name is confirmed inside the registry's own update."""
486
+ import application_registry
487
+ application, posts, _references = _studio_or_missing(session, app_id)
488
+ confirm = (body or {}).get("confirmName") if isinstance(body, dict) else None
489
+ if not isinstance(confirm, str) or confirm != application.get("name"):
490
+ raise err(400, "confirm_name", "Enter the exact App name to confirm.")
491
+
492
+ class _NameConfirmed:
493
+ def update(self, key, delete):
494
+ def confirmed(raw):
495
+ deleted = delete(raw)
496
+ if (raw.get("applications") or {}).get(app_id, {}).get("name") != confirm:
497
+ raise application_registry.ApplicationError("the App name changed")
498
+ return deleted
499
+ return session.runtime.update(key, confirmed)
500
+
501
+ try:
502
+ return application_registry.delete(_NameConfirmed(), app_id, session.uname, session.admin,
503
+ allowed_databases=[posts])
504
+ except PermissionError:
505
+ raise err(403, "forbidden", "You no longer have permission to delete this App.")
506
+ except application_registry.ApplicationError:
507
+ raise err(400, "confirm_name", "The App name may have changed. Close this dialog and reload to retry.")
508
+
509
+
510
+ def _words(text):
511
+ return re.findall(r"[a-z0-9']+", str(text or "").lower())
512
+
513
+
514
+ def copied_sentence(draft, sources):
515
+ """The first run of VERBATIM_WORDS consecutive words a draft shares with any source, or ''."""
516
+ grams = set()
517
+ for source in sources:
518
+ words = _words(source)
519
+ grams.update(" ".join(words[i:i + VERBATIM_WORDS])
520
+ for i in range(0, max(0, len(words) - VERBATIM_WORDS + 1)))
521
+ words = _words(draft)
522
+ for i in range(0, max(0, len(words) - VERBATIM_WORDS + 1)):
523
+ gram = " ".join(words[i:i + VERBATIM_WORDS])
524
+ if gram in grams:
525
+ return gram
526
+ return ""
527
+
528
+
529
+ def attribution(rows):
530
+ """Every selected reference's author, credited in plain words (R14 traceability)."""
531
+ names = list(dict.fromkeys(
532
+ str(row.get("author") or row.get("title") or "an unnamed source").strip() for row in rows))
533
+ joined = names[0] if len(names) == 1 else ", ".join(names[:-1]) + " and " + names[-1]
534
+ return f"Structure drawn from posts by {joined}."
535
+
536
+
537
+ def draft_prompt(topic, channel, rows):
538
+ parts = [f"Channel: {channel}. {CHANNEL_ADVICE.get(channel, '')}", f"Topic: {topic}", ""]
539
+ for n, row in enumerate(rows, 1):
540
+ parts.append(f"Reference {n} ({row.get('network') or 'post'}):")
541
+ for key, label in (("hook", "Hook technique"), ("structure", "Structure"), ("angle", "Angle")):
542
+ value = " ".join(str(row.get(key) or "").split())[:900]
543
+ if value:
544
+ parts.append(f" {label}: {value}")
545
+ parts.append("")
546
+ parts.append("Write the post now.")
547
+ return "\n".join(parts)
548
+
549
+
550
+ def draft_posts(session, app_id, reference_ids, topic, channels, *, ask=None):
551
+ """Generate EVERY channel first; write the Posts rows only if all succeeded (no half row)."""
552
+ topic = " ".join(str(topic or "").split())[:200]
553
+ ids = [str(x) for x in dict.fromkeys(reference_ids or ())][:50]
554
+ wanted = [c for c in dict.fromkeys(channels or ()) if c in CHANNELS]
555
+ if not ids:
556
+ raise err(400, "no_references", "Select at least one reference to draft from.")
557
+ if not topic:
558
+ raise err(400, "no_topic", "Type a one-line topic for the draft.")
559
+ if not wanted:
560
+ raise err(400, "no_channels", "Choose at least one channel.")
561
+ _app, posts, references = _studio(session, app_id)
562
+ rows_by_id = _reference_rows(session, references)
563
+ missing = [rid for rid in ids if rid not in rows_by_id]
564
+ if missing:
565
+ raise err(409, "references_missing", "A selected reference no longer exists. Reload and try again.")
566
+ chosen = [rows_by_id[rid] for rid in ids]
567
+ if not any(str(row.get("structure") or "").strip() for row in chosen):
568
+ raise err(409, "references_not_extracted",
569
+ "The selected references have no extracted structure yet. Paste their text first.")
570
+ # R35: the allowance is checked and reserved BEFORE any provider is resolved or called. One
571
+ # draft per channel; the reservation is handed back unless every row is written.
572
+ month = reserve_drafts(session.runtime, len(wanted))
573
+ try:
574
+ return _generate_and_save(session, posts, ids, topic, wanted, chosen, ask=ask)
575
+ except BaseException:
576
+ release_drafts(session.runtime, month, len(wanted))
577
+ raise
578
+
579
+
580
+ def _generate_and_save(session, posts, ids, topic, wanted, chosen, *, ask=None):
581
+ import ai_enrich
582
+ import ai_review
583
+ import usage_ledger
584
+ from core import user_tables
585
+
586
+ live = ai_review.ladder()
587
+ if not live and ask is None:
588
+ raise err(503, "ai_unavailable", "No AI provider is configured here, so nothing was drafted.")
589
+ provider = (live or [{"name": "injected", "model": "injected"}])[0]
590
+ model = provider["model"]
591
+ caller = ask or ai_enrich._ask
592
+ sources = [row.get("pasted_text") or "" for row in chosen]
593
+ drafts, tokens, calls = [], 0, 0
594
+ try:
595
+ for channel in wanted:
596
+ prompt = draft_prompt(topic, channel, chosen)
597
+ text = ""
598
+ for attempt in range(2):
599
+ answer, used, problem = caller(provider, model, prompt, DRAFT_MAX_TOKENS,
600
+ DRAFT_TIMEOUT_S, system=DRAFT_SYSTEM)
601
+ calls += 1
602
+ tokens += int(used or 0)
603
+ if problem or not str(answer or "").strip():
604
+ raise err(502, "draft_failed", f"The {channel} draft could not be written "
605
+ f"({problem or 'the model returned nothing'}). Nothing was saved.")
606
+ text = ai_enrich.no_dashes(str(answer).strip())
607
+ copied = copied_sentence(text, sources)
608
+ if not copied:
609
+ break
610
+ prompt += f"\n\nYour last draft reused this wording from a reference: \"{copied}\". Rewrite it in new words."
611
+ text = ""
612
+ if not text:
613
+ raise err(502, "draft_copied", f"The {channel} draft kept repeating a reference's "
614
+ "wording, so it was not saved. Nothing was saved.")
615
+ drafts.append({"channel": channel, "caption": text})
616
+ finally:
617
+ if calls:
618
+ # `usage_ledger.SURFACES` names four surfaces and refuses any other, so drafts are
619
+ # booked as `field_agent` (AI-written cells of a user table) until a `content_studio`
620
+ # surface exists there (a finding for the ledger's owner; this module cannot add it).
621
+ usage_ledger.record(LEDGER_SURFACE, provider["name"], model, total=tokens,
622
+ calls=calls, st=session.runtime, user=session.uname)
623
+ credit = attribution(chosen)
624
+ new_rows = [{"caption": d["caption"], "channel": d["channel"], "status": "Draft",
625
+ "references": ",".join(ids), "attribution": credit} for d in drafts]
626
+ created = user_tables.add_rows(posts, new_rows, username=session.uname, st=session.runtime)
627
+ if not created:
628
+ raise err(503, "store_unavailable", "The drafts could not be saved. Nothing was saved.")
629
+ return {"created": list(created) if isinstance(created, (list, tuple)) else created,
630
+ "drafts": drafts, "attribution": credit, "tokens": tokens, "calls": calls,
631
+ "provider": provider["name"], "model": model}
632
+
633
+
634
+ @router.post("/draft")
635
+ def draft(body: dict = Body(default=None), session: Session = Depends(require_session)):
636
+ body = body if isinstance(body, dict) else {}
637
+ return draft_posts(session, body.get("app"), body.get("references"), body.get("topic"),
638
+ body.get("channels"))
api/deps.py CHANGED
@@ -232,6 +232,28 @@ def _assistant_cached_builtin(session: Session, database: str, requested_fields=
232
  else ("product_pool", team_id))
233
  cache = getattr(session.runtime, "pool_cache", None)
234
  entry = cache.get(cache_key) if isinstance(cache, dict) else None
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
235
  if (not isinstance(entry, tuple) or len(entry) < 2
236
  or not isinstance(entry[1], (list, tuple))
237
  or any(not isinstance(row, dict) for row in entry[1])):
@@ -247,6 +269,44 @@ def _assistant_cached_builtin(session: Session, database: str, requested_fields=
247
  _ASSISTANT_USER_TABLE_KINDS = frozenset({"materialised_user_table", "mirror_user_table"})
248
 
249
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
250
  def _assistant_table_defn(session: Session, database: str):
251
  """The rows-free definition for one `ut_*` database plus its lent runtime, after `may_open`.
252
 
@@ -276,7 +336,9 @@ def _assistant_table_defn(session: Session, database: str):
276
  if not isinstance(definition, dict):
277
  _assistant_refuse(404, "assistant_unknown_database", "that database is not available")
278
  definition_runtime = user_tables.lend(session.runtime, user_tables=definitions)
279
- if not user_tables.may_open(database, session.uname, session.admin, st=definition_runtime):
 
 
280
  # Deliberately no label/owner in this response: knowing a key must not become an oracle
281
  # for somebody else's database.
282
  _assistant_refuse(403, "assistant_forbidden",
@@ -346,7 +408,7 @@ def _assistant_user_table(session: Session, database: str, requested_fields=None
346
 
347
 
348
  def _assistant_mirror_table(session: Session, database: str, requested_fields=None, filters=None,
349
- *, defn=None, lent=None):
350
  """Read a MIRROR-SERVED user table through the same filter compiler the grid uses (R11 / C7).
351
 
352
  Returns `_assistant_user_table`'s 4-tuple - `(rows, fields, source_version, label)` - so the
@@ -382,11 +444,24 @@ def _assistant_mirror_table(session: Session, database: str, requested_fields=No
382
  # THE CHEAP QUESTION FIRST, and it needs no cursor: is this key a connected Odoo grid at all?
383
  # A `readThrough` table that is not one has no mirror rows anywhere, and asking the mirror
384
  # about it would open a cursor to learn nothing.
385
- if assistant_mirror.declared_fields(database) is None:
 
 
 
 
 
 
 
 
 
 
386
  _assistant_refuse(409, "assistant_snapshot_unavailable",
387
  "this database does not store its rows and it is not one of the "
388
  "connected databases the assistant can read from the mirror")
389
- cur, refusal = assistant_mirror.open_cursor(session)
 
 
 
390
  if refusal is not None:
391
  _assistant_refuse(refusal.status, refusal.code, refusal.message)
392
  spec = assistant_mirror.binding(cur, database)
@@ -394,7 +469,7 @@ def _assistant_mirror_table(session: Session, database: str, requested_fields=No
394
  _assistant_refuse(409, "assistant_snapshot_unavailable",
395
  "this database is served live from its connector and this deployment "
396
  "has no read-through binding for it yet, so it holds no rows to read")
397
- fields = assistant_mirror.answerable_fields(database, spec)
398
  if not fields:
399
  _assistant_refuse(409, "assistant_snapshot_unavailable",
400
  "this database's connector mirror carries none of its declared columns "
@@ -422,12 +497,36 @@ def _assistant_mirror_table(session: Session, database: str, requested_fields=No
422
  if grant is None:
423
  wall_tree = None
424
 
 
 
 
 
 
 
 
425
  try:
426
- rows, stamp, total = assistant_mirror.read_rows(
427
- session, database, cur, spec, fields,
428
- wall_tree=wall_tree, filter_tree=filter_tree,
429
- filter_keys=[field["key"] for field in visible
430
- if isinstance(field, dict) and field.get("key")])
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
431
  except assistant_mirror.MirrorRefusal as exc:
432
  _assistant_refuse(exc.status, exc.code, exc.message)
433
  raise # unreachable; see above
@@ -459,7 +558,33 @@ def _assistant_requested_fields(fields, visible_fields):
459
  return out
460
 
461
 
462
- def assistant_read_scope(session: Session, database, fields=None, filters=None):
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
463
  """Return a detached, permissioned snapshot for one Assistant tool call.
464
 
465
  Signature (the C8 contract for E):
@@ -468,6 +593,12 @@ def assistant_read_scope(session: Session, database, fields=None, filters=None):
468
  permission_scope_applied, fields, filters, records
469
  }
470
 
 
 
 
 
 
 
471
  `database` is exactly one source. `fields` is an explicit list of visible field keys (or
472
  ``None`` for every visible field). `filters` is ``None`` or the strict tree accepted by
473
  `core.perm_scope.validate_assistant_filter`; it rejects unknown, hidden, inactive and
@@ -478,6 +609,7 @@ def assistant_read_scope(session: Session, database, fields=None, filters=None):
478
  from harness import filter_eval
479
 
480
  key = str(database or "").strip()
 
481
  if key in _ASSISTANT_BUILT_INS:
482
  rows, canonical_fields, source_version = _assistant_cached_builtin(session, key, fields, filters)
483
  label = _ASSISTANT_BUILT_INS[key]["label"]
@@ -494,12 +626,15 @@ def assistant_read_scope(session: Session, database, fields=None, filters=None):
494
  from core import user_tables
495
  _defn, _lent = _assistant_table_defn(session, key)
496
  if user_tables.materialises(key, st=_lent, defn=_defn):
 
 
 
497
  rows, canonical_fields, source_version, label = _assistant_user_table(
498
  session, key, fields, filters, defn=_defn, lent=_lent)
499
  source_kind = "materialised_user_table"
500
  else:
501
  rows, canonical_fields, source_version, label = _assistant_mirror_table(
502
- session, key, fields, filters, defn=_defn, lent=_lent)
503
  source_kind = "mirror_user_table"
504
  else:
505
  _assistant_refuse(404, "assistant_unknown_database", "that database is not available")
@@ -556,7 +691,7 @@ def assistant_read_scope(session: Session, database, fields=None, filters=None):
556
  records.append({"pid": copy.deepcopy(row["pid"]),
557
  **{field: copy.deepcopy(row.get(field)) for field in selected_keys}})
558
 
559
- return {
560
  "database": key,
561
  "label": str(label),
562
  "source_kind": source_kind,
@@ -568,6 +703,11 @@ def assistant_read_scope(session: Session, database, fields=None, filters=None):
568
  "filters": copy.deepcopy(requested_filter),
569
  "records": records,
570
  }
 
 
 
 
 
571
 
572
 
573
  def assistant_permitted_sources(session: Session, databases):
@@ -672,6 +812,19 @@ def assistant_source_status(session: Session, databases=None):
672
  mirror_cur = None
673
  return mirror_cur
674
 
 
 
 
 
 
 
 
 
 
 
 
 
 
675
  def _definitions():
676
  nonlocal definitions, read_definitions
677
  if not read_definitions:
@@ -752,6 +905,10 @@ def assistant_source_status(session: Session, databases=None):
752
  entry = cache.get(cache_key) if isinstance(cache, dict) else None
753
  if isinstance(entry, tuple) and len(entry) >= 2 and isinstance(entry[1], (list, tuple)):
754
  row["answerable"] = True
 
 
 
 
755
  else:
756
  row["reason"] = "no snapshot loaded yet. Open this database once, then ask"
757
  continue
@@ -762,10 +919,16 @@ def assistant_source_status(session: Session, databases=None):
762
  if not isinstance(definition, dict):
763
  continue
764
  lent = user_tables.lend(session.runtime, user_tables=defs)
765
- if not user_tables.may_open(key, session.uname, session.admin, st=lent):
 
766
  continue
767
  row["permitted"] = True
768
  if user_tables.materialises(key, st=lent, defn=definition):
 
 
 
 
 
769
  row["answerable"] = True
770
  continue
771
  # ⭐⭐ W47-T23 / R11 - THE SECOND WAY TO BE ANSWERABLE. This used to end here with a flat
@@ -778,7 +941,9 @@ def assistant_source_status(session: Session, databases=None):
778
  # is transient and retrying works, while a grid this deployment has no read-through
779
  # binding for will not fix itself by waiting. One sentence for both would make the first
780
  # look permanent, which is the failure mode the ruling exists to prevent.
781
- cur = _mirror()
 
 
782
  if cur is None:
783
  row["reason"] = ("this database is read live from its connector mirror, and that "
784
  "mirror is not open on this deployment right now; try again shortly")
@@ -801,6 +966,49 @@ def assistant_source_status(session: Session, databases=None):
801
  return out
802
 
803
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
804
  def _user_for(claims):
805
  """The account named by a verified cookie, or None β€” with the epoch check that makes a
806
  stateless cookie revocable.
@@ -829,11 +1037,16 @@ def _user_for(claims):
829
  if (qa and claim_tenant == qa.get("tenant")
830
  and int(claims.get("e") or 0) == int(qa.get("epoch") or 0)):
831
  return qa
 
 
 
832
  try:
833
- reg = users.registry() or {}
834
- except Exception:
835
- reg = {}
836
- rec = reg.get(uname)
 
 
837
  if (rec and rec.get("active", True)
838
  and int(rec.get("epoch") or 0) == int(claims.get("e") or 0)):
839
  pub = users._public(uname, rec)
 
232
  else ("product_pool", team_id))
233
  cache = getattr(session.runtime, "pool_cache", None)
234
  entry = cache.get(cache_key) if isinstance(cache, dict) else None
235
+ if not isinstance(entry, tuple) and isinstance(cache, dict):
236
+ # ⭐⭐ S67-T33 / D-778: A DATABASE NEVER OPENED ANSWERS. The snapshot used to exist only
237
+ # once somebody opened the grid in this process, so after every restart the assistant
238
+ # refused a database its caller demonstrably holds. The miss is now filled by the GRID'S
239
+ # OWN pool builder, for the SAME scope key the grid would fill (`derive_pool_scope`
240
+ # above), so this is the grid's first open, not a second reader. It is read-only on Odoo
241
+ # (rule 4) and honours the connector pause exactly as the grid does (503 when paused
242
+ # with nothing cached). Permission and field checks above have already run.
243
+ try:
244
+ if database == "customer_data":
245
+ import routes_customers
246
+ routes_customers._pool_for(session.runtime, team_id, agent)
247
+ else:
248
+ import routes_products
249
+ routes_products._pool_for(session.runtime, team_id)
250
+ except HTTPException:
251
+ raise
252
+ except Exception as exc: # noqa: BLE001
253
+ _assistant_refuse(409, "assistant_snapshot_unavailable",
254
+ f"this database's snapshot could not be built for your current "
255
+ f"scope ({type(exc).__name__}); try again shortly")
256
+ entry = cache.get(cache_key)
257
  if (not isinstance(entry, tuple) or len(entry) < 2
258
  or not isinstance(entry[1], (list, tuple))
259
  or any(not isinstance(row, dict) for row in entry[1])):
 
269
  _ASSISTANT_USER_TABLE_KINDS = frozenset({"materialised_user_table", "mirror_user_table"})
270
 
271
 
272
+ def _assistant_nav_opens(session: Session, database: str, lent, definition=None) -> bool:
273
+ """Does the NAVIGATION offer this `ut_*` database to this caller? S67-T33 / R33.
274
+
275
+ ⭐⭐ THE AI CONNECTOR'S LIST IS THE NAV'S LIST, BY CONSTRUCTION RATHER THAN BY AGREEMENT.
276
+ `routes_nav.nav` admits a user table when `user_tables.nav_entries` lists it (which is
277
+ `may_open` plus the route-system exclusion), it is not a retired key, and
278
+ `perm_scope.nav_may_open` agrees (an administrator's explicit untick denies). The assistant
279
+ used to ask `may_open` alone, so a database the member's navigation did not show could still
280
+ be listed and read through the connector. This asks the nav's own predicates, in the nav's
281
+ order; it is not a second permission rule.
282
+ ⚠ `session.admin` short-circuits leg 1 of `nav_may_open` exactly as the nav's own admin
283
+ session does; `nav_may_open` itself returns True for an administrator.
284
+ """
285
+ import core.perm_scope as perm_scope
286
+ from core import route_system, user_tables
287
+
288
+ key = str(database or "")
289
+ try:
290
+ from odoo_relational import RETIRED_KEYS as retired
291
+ except Exception: # noqa: BLE001
292
+ retired = ()
293
+ if key in retired:
294
+ return False
295
+ if not user_tables.may_open(key, session.uname, session.admin, st=lent):
296
+ return False
297
+ if isinstance(definition, dict) and (route_system.is_managed(definition)
298
+ or route_system.is_managed(key)):
299
+ return False
300
+ if session.admin:
301
+ return True
302
+ user = getattr(session, "user", None)
303
+ if not isinstance(user, dict):
304
+ # No account record means no explicit per-key deny can exist, and `nav_may_open` leg 3
305
+ # then defers to `may_open`, which has already answered above.
306
+ return True
307
+ return bool(perm_scope.nav_may_open(user, key, st=lent))
308
+
309
+
310
  def _assistant_table_defn(session: Session, database: str):
311
  """The rows-free definition for one `ut_*` database plus its lent runtime, after `may_open`.
312
 
 
336
  if not isinstance(definition, dict):
337
  _assistant_refuse(404, "assistant_unknown_database", "that database is not available")
338
  definition_runtime = user_tables.lend(session.runtime, user_tables=definitions)
339
+ # S67-T33: the NAV's predicate, not `may_open` alone, so a database the caller's navigation
340
+ # does not show is refused here too (403), not merely left off the list.
341
+ if not _assistant_nav_opens(session, database, definition_runtime, definition):
342
  # Deliberately no label/owner in this response: knowing a key must not become an oracle
343
  # for somebody else's database.
344
  _assistant_refuse(403, "assistant_forbidden",
 
408
 
409
 
410
  def _assistant_mirror_table(session: Session, database: str, requested_fields=None, filters=None,
411
+ *, defn=None, lent=None, mode=None, extra=None):
412
  """Read a MIRROR-SERVED user table through the same filter compiler the grid uses (R11 / C7).
413
 
414
  Returns `_assistant_user_table`'s 4-tuple - `(rows, fields, source_version, label)` - so the
 
444
  # THE CHEAP QUESTION FIRST, and it needs no cursor: is this key a connected Odoo grid at all?
445
  # A `readThrough` table that is not one has no mirror rows anywhere, and asking the mirror
446
  # about it would open a cursor to learn nothing.
447
+ if database.startswith("ut_amzn_"):
448
+ # ⭐ S67-T33: THE GRID'S OWN ADMISSION FIRST. A dynamically sized Amazon snapshot enters
449
+ # the source registry only through `require_amazon_mirror`, so without it a database
450
+ # never opened in this process had no binding here (measured on staging for
451
+ # `ut_amzn_fba_inventory`). It refuses by name (503) while the Amazon mirror is unloaded.
452
+ import routes_odoo_tables
453
+ routes_odoo_tables.require_amazon_mirror(database, session, definition=definition)
454
+ # ⭐ S67-T33: the stored definition rides along, so an Amazon grid or a generated
455
+ # `ut_odoo_model_*` table answers from its own field contract, as it does on the grid.
456
+ declared = assistant_mirror.declared_fields(database, definition)
457
+ if declared is None:
458
  _assistant_refuse(409, "assistant_snapshot_unavailable",
459
  "this database does not store its rows and it is not one of the "
460
  "connected databases the assistant can read from the mirror")
461
+ # ⭐ S67-T33: THE TABLE'S OWN CURSOR (`_table_cursor`), so Amazon never waits on Odoo's
462
+ # first sync. Passed only for an Amazon key, which is the one family it changes.
463
+ cur, refusal = (assistant_mirror.open_cursor(session, database)
464
+ if database.startswith("ut_amzn_") else assistant_mirror.open_cursor(session))
465
  if refusal is not None:
466
  _assistant_refuse(refusal.status, refusal.code, refusal.message)
467
  spec = assistant_mirror.binding(cur, database)
 
469
  _assistant_refuse(409, "assistant_snapshot_unavailable",
470
  "this database is served live from its connector and this deployment "
471
  "has no read-through binding for it yet, so it holds no rows to read")
472
+ fields = assistant_mirror.answerable_fields(database, spec, declared)
473
  if not fields:
474
  _assistant_refuse(409, "assistant_snapshot_unavailable",
475
  "this database's connector mirror carries none of its declared columns "
 
497
  if grant is None:
498
  wall_tree = None
499
 
500
+ filter_keys = [field["key"] for field in visible
501
+ if isinstance(field, dict) and field.get("key")]
502
+ # ⭐⭐ S67-T34: THREE READS BEHIND THE ONE PREFLIGHT ABOVE, chosen by `mode`, all compiling the
503
+ # SAME wall and filter (`assistant_mirror._predicate`). "schema" reads and counts no row
504
+ # (D-2630: describe answers the field list of a million-row table without counting it);
505
+ # ("aggregate", specs) runs the aggregates IN SQL over every permitted row (D-616, D-2631);
506
+ # ("page", offset, limit) pages IN SQL for a table past the one-pass ceiling (rule 1).
507
  try:
508
+ if mode == "schema":
509
+ return [], fields, {"read": "connector mirror"}, definition.get("label") or database
510
+ if isinstance(mode, tuple) and mode and mode[0] == "aggregate":
511
+ values, total = assistant_mirror.aggregate(
512
+ session, database, cur, spec, fields, mode[1],
513
+ wall_tree=wall_tree, filter_tree=filter_tree, filter_keys=filter_keys)
514
+ if isinstance(extra, dict):
515
+ extra["aggregates"], extra["total"] = values, total
516
+ return ([], fields, {"read": "connector mirror", "rows": total},
517
+ definition.get("label") or database)
518
+ if isinstance(mode, tuple) and mode and mode[0] == "page":
519
+ rows, stamp, total = assistant_mirror.read_page(
520
+ session, database, cur, spec, fields, offset=mode[1], limit=mode[2],
521
+ wall_tree=wall_tree, filter_tree=filter_tree, filter_keys=filter_keys,
522
+ defn=definition)
523
+ if isinstance(extra, dict):
524
+ extra["total"] = total
525
+ else:
526
+ rows, stamp, total = assistant_mirror.read_rows(
527
+ session, database, cur, spec, fields,
528
+ wall_tree=wall_tree, filter_tree=filter_tree, filter_keys=filter_keys,
529
+ defn=definition)
530
  except assistant_mirror.MirrorRefusal as exc:
531
  _assistant_refuse(exc.status, exc.code, exc.message)
532
  raise # unreachable; see above
 
558
  return out
559
 
560
 
561
+ def _assistant_oversize_reason(database: str, definition, lent) -> str:
562
+ """The named cause an oversize stored table carries, or "". S67-T33 / rule 1.
563
+
564
+ β›” AN EVICTED TABLE KEEPS ITS DEFINITION AND LOSES ITS ROWS, so the materialised reader used
565
+ to answer it as a complete table of 0 rows. `user_tables.limit_report` is the one evaluator
566
+ of that stamp (the grid serves the same sentence), so its cause and recommendation are read
567
+ from there rather than re-worded here.
568
+ """
569
+ from core import user_tables
570
+
571
+ if not user_tables.oversize_of(database, defn=definition if isinstance(definition, dict)
572
+ else {}):
573
+ return ""
574
+ report = user_tables.limit_report(database, st=lent) or {}
575
+ label = str((definition or {}).get("label") or database)
576
+ return (f"{label} is not stored: {report.get('cause') or 'its rows were not stored'}. "
577
+ f"{str(report.get('recommendation') or '').rstrip('.')}. The assistant refuses "
578
+ f"rather than answering 0 rows").replace("\u2014", ",").replace("\u2013", "-")
579
+
580
+
581
+ def _assistant_refuse_oversize(database: str, definition, lent) -> None:
582
+ reason = _assistant_oversize_reason(database, definition, lent)
583
+ if reason:
584
+ _assistant_refuse(409, "assistant_source_too_large", reason)
585
+
586
+
587
+ def assistant_read_scope(session: Session, database, fields=None, filters=None, *, mode=None):
588
  """Return a detached, permissioned snapshot for one Assistant tool call.
589
 
590
  Signature (the C8 contract for E):
 
593
  permission_scope_applied, fields, filters, records
594
  }
595
 
596
+ ⭐ S67-T34: `mode` changes only HOW a mirror-served database is read, never who may read it
597
+ (every wall below runs unchanged): "schema" returns the field contract with no row read,
598
+ ("aggregate", specs) adds `server_aggregates` and `total_rows` computed in SQL over every
599
+ permitted row, ("page", offset, limit) returns one SQL page plus `total_rows`. Every other
600
+ source kind ignores it and answers as before.
601
+
602
  `database` is exactly one source. `fields` is an explicit list of visible field keys (or
603
  ``None`` for every visible field). `filters` is ``None`` or the strict tree accepted by
604
  `core.perm_scope.validate_assistant_filter`; it rejects unknown, hidden, inactive and
 
609
  from harness import filter_eval
610
 
611
  key = str(database or "").strip()
612
+ extra = {}
613
  if key in _ASSISTANT_BUILT_INS:
614
  rows, canonical_fields, source_version = _assistant_cached_builtin(session, key, fields, filters)
615
  label = _ASSISTANT_BUILT_INS[key]["label"]
 
626
  from core import user_tables
627
  _defn, _lent = _assistant_table_defn(session, key)
628
  if user_tables.materialises(key, st=_lent, defn=_defn):
629
+ # S67-T33: a stored table whose rows were evicted refuses by name, never answers 0.
630
+ # Only on the STORED reader: a read-through table's rows come from the mirror.
631
+ _assistant_refuse_oversize(key, _defn, _lent)
632
  rows, canonical_fields, source_version, label = _assistant_user_table(
633
  session, key, fields, filters, defn=_defn, lent=_lent)
634
  source_kind = "materialised_user_table"
635
  else:
636
  rows, canonical_fields, source_version, label = _assistant_mirror_table(
637
+ session, key, fields, filters, defn=_defn, lent=_lent, mode=mode, extra=extra)
638
  source_kind = "mirror_user_table"
639
  else:
640
  _assistant_refuse(404, "assistant_unknown_database", "that database is not available")
 
691
  records.append({"pid": copy.deepcopy(row["pid"]),
692
  **{field: copy.deepcopy(row.get(field)) for field in selected_keys}})
693
 
694
+ out = {
695
  "database": key,
696
  "label": str(label),
697
  "source_kind": source_kind,
 
703
  "filters": copy.deepcopy(requested_filter),
704
  "records": records,
705
  }
706
+ if "aggregates" in extra:
707
+ out["server_aggregates"] = copy.deepcopy(extra["aggregates"])
708
+ if "total" in extra:
709
+ out["total_rows"] = int(extra["total"])
710
+ return out
711
 
712
 
713
  def assistant_permitted_sources(session: Session, databases):
 
812
  mirror_cur = None
813
  return mirror_cur
814
 
815
+ amazon_cur = {}
816
+
817
+ def _amazon_mirror(key):
818
+ """ONE Amazon cursor for the batch, chosen by `_table_cursor` for an Amazon key."""
819
+ if "cur" not in amazon_cur:
820
+ try:
821
+ import assistant_mirror
822
+ cur, refusal = assistant_mirror.open_cursor(session, key)
823
+ amazon_cur["cur"] = None if refusal is not None else cur
824
+ except Exception: # noqa: BLE001
825
+ amazon_cur["cur"] = None
826
+ return amazon_cur["cur"]
827
+
828
  def _definitions():
829
  nonlocal definitions, read_definitions
830
  if not read_definitions:
 
905
  entry = cache.get(cache_key) if isinstance(cache, dict) else None
906
  if isinstance(entry, tuple) and len(entry) >= 2 and isinstance(entry[1], (list, tuple)):
907
  row["answerable"] = True
908
+ elif isinstance(cache, dict):
909
+ # ⭐ S67-T33 / D-778: a miss is no longer a refusal; the read builds the snapshot
910
+ # through the grid's own pool builder, so this source is answerable now.
911
+ row["answerable"] = True
912
  else:
913
  row["reason"] = "no snapshot loaded yet. Open this database once, then ask"
914
  continue
 
919
  if not isinstance(definition, dict):
920
  continue
921
  lent = user_tables.lend(session.runtime, user_tables=defs)
922
+ # ⭐ S67-T33 / R33: the NAV's predicate, so this list and the navigation are one set.
923
+ if not _assistant_nav_opens(session, key, lent, definition):
924
  continue
925
  row["permitted"] = True
926
  if user_tables.materialises(key, st=lent, defn=definition):
927
+ oversize = _assistant_oversize_reason(key, definition, lent)
928
+ if oversize:
929
+ # Rule 1's second sentence: listed, with its cause, never answered as 0 rows.
930
+ row["reason"] = oversize
931
+ continue
932
  row["answerable"] = True
933
  continue
934
  # ⭐⭐ W47-T23 / R11 - THE SECOND WAY TO BE ANSWERABLE. This used to end here with a flat
 
941
  # is transient and retrying works, while a grid this deployment has no read-through
942
  # binding for will not fix itself by waiting. One sentence for both would make the first
943
  # look permanent, which is the failure mode the ruling exists to prevent.
944
+ # ⭐ S67-T33: an Amazon grid asks ITS OWN mirror (`_table_cursor`), never the Odoo
945
+ # first-sync gate, so "not open right now" is never Odoo's backfill answering for Amazon.
946
+ cur = _amazon_mirror(key) if key.startswith("ut_amzn_") else _mirror()
947
  if cur is None:
948
  row["reason"] = ("this database is read live from its connector mirror, and that "
949
  "mirror is not open on this deployment right now; try again shortly")
 
966
  return out
967
 
968
 
969
+ #: S67 lane D (S67-T28 finding): the session resolver re-read the WHOLE `users` bucket on every
970
+ #: authenticated request, uncached; on a saturated cluster that read queued ahead of every page.
971
+ #: It is now held against the bucket's durable revision token (`store.revision`), the S67-T28
972
+ #: `_REV_MEMO` pattern: the token is read BEFORE the value, so a write racing the read can only
973
+ #: cache newer bytes under the older token (a miss next time, never a stale hit). Every Postgres
974
+ #: `put` / `update` bumps `rev`, so a deactivation or an epoch rotation misses on the very next
975
+ #: request. β›” FAIL-CLOSED: no token, or a token read that fails, means a direct read (which still
976
+ #: answers 503 when the store is unreadable); a cached registry is never served in its place.
977
+ #: The key carries the identity of the reader and the store it reads, so a swapped store or a
978
+ #: patched `users.registry` / `store.get` misses instead of replaying another registry.
979
+ _SESSION_REGISTRY = {"key": None, "reg": None}
980
+ _SESSION_REGISTRY_LOCK = __import__("threading").Lock()
981
+
982
+
983
+ def _session_registry():
984
+ """`users.registry()` for `_user_for`, re-read only when the `users` bucket changed.
985
+
986
+ β›” POSTGRES ONLY. There `rev` is a durable column every `put` / `update` bumps (`store_pg`),
987
+ and nothing else writes `store_kv`. The HF backend's token is a per-process counter that a
988
+ fixture or a cache-level write can leave unmoved (measured: `verify_scopes` 346/687 with the
989
+ memo on HF), and its `get()` already holds the bucket for the life of the process, so the
990
+ re-read this memo removes does not exist there: it reads straight through.
991
+ """
992
+ key = None
993
+ try:
994
+ backend = store.backend()
995
+ token = (store.revision("users") or {}).get("token") if backend == "pg" else None
996
+ if token not in (None, ""):
997
+ where = os.environ.get("AIOS_TENANT") or "royal-imports"
998
+ key = (backend, where, id(users.registry), id(store.get), str(token))
999
+ except Exception: # noqa: BLE001 -- fail-closed below
1000
+ key = None
1001
+ if key is not None:
1002
+ with _SESSION_REGISTRY_LOCK:
1003
+ if _SESSION_REGISTRY["key"] == key:
1004
+ return _SESSION_REGISTRY["reg"]
1005
+ reg = users.registry() or {}
1006
+ if key is not None:
1007
+ with _SESSION_REGISTRY_LOCK:
1008
+ _SESSION_REGISTRY["key"], _SESSION_REGISTRY["reg"] = key, copy.deepcopy(reg)
1009
+ return reg
1010
+
1011
+
1012
  def _user_for(claims):
1013
  """The account named by a verified cookie, or None β€” with the epoch check that makes a
1014
  stateless cookie revocable.
 
1037
  if (qa and claim_tenant == qa.get("tenant")
1038
  and int(claims.get("e") or 0) == int(qa.get("epoch") or 0)):
1039
  return qa
1040
+ # β›” HOTFIX 2026-09-28: AN UNREADABLE REGISTRY IS A 503, NEVER "NO SUCH ACCOUNT". This caught
1041
+ # `Exception` into `{}`, so a store `PoolTimeout` on staging answered a valid cookie with 401
1042
+ # `invalid_session` and the client signed the owner out. Still fail-closed: nobody is admitted.
1043
  try:
1044
+ reg = _session_registry()
1045
+ except Exception as exc:
1046
+ print(f"[aios-api] session account store unavailable: {type(exc).__name__}: {exc}")
1047
+ raise err(503, "session_store_unavailable",
1048
+ "your account could not be checked just now. Try again in a moment")
1049
+ rec = copy.deepcopy(reg.get(uname)) # the registry may be the held copy: never mutate it
1050
  if (rec and rec.get("active", True)
1051
  and int(rec.get("epoch") or 0) == int(claims.get("e") or 0)):
1052
  pub = users._public(uname, rec)
api/formula_runtime.py CHANGED
@@ -83,6 +83,8 @@ FN_ARITY = {
83
  "ISNUMBER": (1, 1), "ISTEXT": (1, 1), "ISERROR": (1, 1),
84
  "OFFSET": (3, 5), "INDIRECT": (1, 2), "RAND": (0, 0),
85
  "FILTER": (2, 3), "SORT": (1, 4),
 
 
86
  }
87
  # Excel workbooks in the Daily Sales lineage use both spellings. Keep the authored formula text
88
  # untouched in the workbook; normalization maps AVG to the same AST/function the catalogue's
@@ -457,6 +459,15 @@ FN_CATALOGUE = {
457
  "description": "Returns TRUE when a value is a number.", "contexts": _BOTH},
458
  "ISTEXT": {"category": "Information", "signature": "ISTEXT(value)",
459
  "description": "Returns TRUE when a value is text.", "contexts": _BOTH},
 
 
 
 
 
 
 
 
 
460
  "ISERROR": {"category": "Information", "signature": "ISERROR(value)",
461
  "description": "Returns TRUE when a formula would refuse to produce a value.",
462
  "contexts": _BOTH},
@@ -625,6 +636,9 @@ PARITY_FIXTURES = {
625
  "EXACT": ("EXACT(\"A\", \"a\")", "FALSE"),
626
  "ISNUMBER": ("ISNUMBER(1)", "TRUE"),
627
  "ISTEXT": ("ISTEXT(\"a\")", "TRUE"),
 
 
 
628
  "ISERROR": ("ISERROR(1 / 0)", "TRUE"),
629
  }
630
 
@@ -917,7 +931,7 @@ RANGE_ARGS = {
917
 
918
  #: ⭐ THE ARGUMENT IS AN ADDRESS, NOT A VALUE. `ROW(B4)` asks what row `B4` is on and must not read
919
  #: what is written there; the cell may well be empty. Handled beside `RANGE_ARGS` in `_eval`.
920
- ADDR_ARGS = {"ROW": (0,), "COLUMN": (0,)}
921
 
922
 
923
  def _is_range_position(fn, index):
@@ -1792,6 +1806,11 @@ def _eval_value(node, row, today, diagnostics=None):
1792
  return _array_binary(a, b, node["op"], diagnostics)
1793
  if isinstance(row, _SheetInputs):
1794
  def comparison(value):
 
 
 
 
 
1795
  if isinstance(value, bool): return (2, value)
1796
  if isinstance(value, str): return (1, value.casefold())
1797
  return (0, _excel_number(value or 0))
@@ -1858,6 +1877,17 @@ def _eval_value(node, row, today, diagnostics=None):
1858
  for code in local:
1859
  _diagnose(diagnostics, code)
1860
  return "" if first is None and isinstance(row, _SheetInputs) and not local else first
 
 
 
 
 
 
 
 
 
 
 
1861
  if fn == "ISERROR":
1862
  # The whole job is to ask whether a refusal happened, so the refusal must stay LOCAL.
1863
  local = []
@@ -1888,6 +1918,11 @@ def _eval_value(node, row, today, diagnostics=None):
1888
  _diagnose(diagnostics, "bad_value")
1889
  return None
1890
  args.append(here)
 
 
 
 
 
1891
  return _call(fn, args, today, diagnostics)
1892
  # ⭐ RANGES FLATTEN HERE, INTO THE SAME FLAT LIST `_call` ALREADY TAKES. `SUM`, `AVERAGE`,
1893
  # `COUNT`, `MIN` and `MAX` are declared `(1, MANY)` and simply iterate their argument list,
@@ -1923,6 +1958,27 @@ def _eval_value(node, row, today, diagnostics=None):
1923
  raise FormulaError("formula AST node is not supported")
1924
 
1925
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1926
  def _addr_arg(node, diagnostics):
1927
  """The ADDRESS a1 node names, as a key. Anything else is not an address and refuses."""
1928
  if isinstance(node, dict) and node.get("t") == "a1":
@@ -2319,6 +2375,19 @@ def _call(fn, a, today, diagnostics=None):
2319
  if fn == "FIND":
2320
  start = int(nums[2] or 1)-1 if len(a)>2 else 0; pos = _text(a[1]).find(_text(a[0]), start)
2321
  return pos+1 if pos >= 0 else None
 
 
 
 
 
 
 
 
 
 
 
 
 
2322
  if fn in ("REGEX_EXTRACT", "REGEX_REPLACE"):
2323
  pattern, subject = _safe_regex(a[1], a[0]); m = re.search(pattern, subject)
2324
  if fn == "REGEX_EXTRACT": return m.group(0) if m else None
@@ -3722,7 +3791,7 @@ def recompute_document(book, *, cache_key=None, changes=None, today=None):
3722
  excel = {**REFERENCE_ERRORS, "no_match": "#N/A", "bad_value": "#VALUE!", "div_zero": "#DIV/0!", "numeric_error": "#NUM!"}.get(code)
3723
  if excel: cell["error"]["excelCode"] = excel
3724
  else:
3725
- cell.update(state="fresh", display="" if cell["value"] is None else str(cell["value"]))
3726
  cell.pop("refusal", None)
3727
  cell.pop("error", None)
3728
  cell.pop("spill", None)
@@ -3765,7 +3834,7 @@ def recompute_document(book, *, cache_key=None, changes=None, today=None):
3765
  base = engine.specs.get(child, {})
3766
  style = {**(owner.get("style") or {}), **(base.get("style") or {})}
3767
  record = {"kind": "literal", "value": engine.values.get(child), "state": "fresh",
3768
- "display": "" if engine.values.get(child) is None else str(engine.values[child]),
3769
  "spillParent": origin, "style": style,
3770
  "formulaMeta": {"spillChild": {"version": 1, "origin": origin, "address": child,
3771
  **({"base": base} if base else {})}}}
 
83
  "ISNUMBER": (1, 1), "ISTEXT": (1, 1), "ISERROR": (1, 1),
84
  "OFFSET": (3, 5), "INDIRECT": (1, 2), "RAND": (0, 0),
85
  "FILTER": (2, 3), "SORT": (1, 4),
86
+ # S67-T24: the reference workbooks' `IF(ISNUMBER(SEARCH(...)))` and `ISFORMULA(...)`.
87
+ "SEARCH": (2, 3), "SEARCHB": (2, 3), "ISFORMULA": (1, 1),
88
  }
89
  # Excel workbooks in the Daily Sales lineage use both spellings. Keep the authored formula text
90
  # untouched in the workbook; normalization maps AVG to the same AST/function the catalogue's
 
459
  "description": "Returns TRUE when a value is a number.", "contexts": _BOTH},
460
  "ISTEXT": {"category": "Information", "signature": "ISTEXT(value)",
461
  "description": "Returns TRUE when a value is text.", "contexts": _BOTH},
462
+ "ISFORMULA": {"category": "Information", "signature": "ISFORMULA(reference)",
463
+ "description": "Returns TRUE when the cell it names holds a formula.",
464
+ "contexts": _SHEET},
465
+ "SEARCH": {"category": "Text", "signature": "SEARCH(find_text, within_text, [start_num])",
466
+ "description": "Returns the position of one text value inside another, ignoring "
467
+ "case; ? matches one character and * any run.", "contexts": _BOTH},
468
+ "SEARCHB": {"category": "Text", "signature": "SEARCHB(find_text, within_text, [start_num])",
469
+ "description": "Counts like SEARCH; each character is one byte in this workbook.",
470
+ "contexts": _BOTH},
471
  "ISERROR": {"category": "Information", "signature": "ISERROR(value)",
472
  "description": "Returns TRUE when a formula would refuse to produce a value.",
473
  "contexts": _BOTH},
 
636
  "EXACT": ("EXACT(\"A\", \"a\")", "FALSE"),
637
  "ISNUMBER": ("ISNUMBER(1)", "TRUE"),
638
  "ISTEXT": ("ISTEXT(\"a\")", "TRUE"),
639
+ "SEARCH": ("SEARCH(\"B\", \"abc\")", 2),
640
+ "SEARCHB": ("SEARCHB(\"c?\", \"abcd\")", 3),
641
+ "ISFORMULA": ("ISFORMULA(B2)", "FALSE"),
642
  "ISERROR": ("ISERROR(1 / 0)", "TRUE"),
643
  }
644
 
 
931
 
932
  #: ⭐ THE ARGUMENT IS AN ADDRESS, NOT A VALUE. `ROW(B4)` asks what row `B4` is on and must not read
933
  #: what is written there; the cell may well be empty. Handled beside `RANGE_ARGS` in `_eval`.
934
+ ADDR_ARGS = {"ROW": (0,), "COLUMN": (0,), "ISFORMULA": (0,)}
935
 
936
 
937
  def _is_range_position(fn, index):
 
1806
  return _array_binary(a, b, node["op"], diagnostics)
1807
  if isinstance(row, _SheetInputs):
1808
  def comparison(value):
1809
+ # HOTFIX 2026-09-28: AN EMPTY CELL COMPARES AS EMPTY TEXT, exactly as the browser's
1810
+ # `formulaEval.ts` does ("value" in cell ? cell.value : ""). As a number it made
1811
+ # `=IF(E13="",E12,E13)` FALSE here and TRUE in the browser, so Daily Sales showed a
1812
+ # date the server refused and every binding filtered on row 7 read #ERROR!.
1813
+ if value is None: return (1, "")
1814
  if isinstance(value, bool): return (2, value)
1815
  if isinstance(value, str): return (1, value.casefold())
1816
  return (0, _excel_number(value or 0))
 
1877
  for code in local:
1878
  _diagnose(diagnostics, code)
1879
  return "" if first is None and isinstance(row, _SheetInputs) and not local else first
1880
+ if fn in ("ISNUMBER", "ISTEXT") and len(node.get("args") or ()) == 1:
1881
+ # ⭐ S67-T24: AN IS-FUNCTION NEVER PASSES AN ERROR ON. Excel answers FALSE for
1882
+ # `ISNUMBER(#VALUE!)`, which is what makes `IF(ISNUMBER(SEARCH(x, y)), ...)` (2,512 cells
1883
+ # in the Inventory System's Catalog) take its FALSE branch instead of failing whole.
1884
+ local = []
1885
+ try:
1886
+ value = _eval(node["args"][0], row, today, local)
1887
+ except (ArithmeticError, TypeError, ValueError, OverflowError, KeyError, re.error,
1888
+ FormulaError, _ReferenceFault):
1889
+ return False
1890
+ return False if local else _call(fn, [value], today, diagnostics)
1891
  if fn == "ISERROR":
1892
  # The whole job is to ask whether a refusal happened, so the refusal must stay LOCAL.
1893
  local = []
 
1918
  _diagnose(diagnostics, "bad_value")
1919
  return None
1920
  args.append(here)
1921
+ if fn == "ISFORMULA":
1922
+ # ⭐ S67-T24: ASKED OF THE CELL, NOT OF ITS VALUE. Only a Sheet document knows which
1923
+ # cells hold formulas (`SheetCalculation.specs`); a grid row has no formula cells.
1924
+ spec = row.engine.specs.get(args[0]) if isinstance(row, _SheetInputs) and args[0] else None
1925
+ return bool(isinstance(spec, dict) and spec.get("formula"))
1926
  return _call(fn, args, today, diagnostics)
1927
  # ⭐ RANGES FLATTEN HERE, INTO THE SAME FLAT LIST `_call` ALREADY TAKES. `SUM`, `AVERAGE`,
1928
  # `COUNT`, `MIN` and `MAX` are declared `(1, MANY)` and simply iterate their argument list,
 
1958
  raise FormulaError("formula AST node is not supported")
1959
 
1960
 
1961
+ def _display_text(value):
1962
+ """A calculated value as the grid shows it: Excel's TRUE / FALSE for a boolean (S67-T24; a
1963
+ Python `str(True)` painted "True"), blank for nothing, else the value's own text."""
1964
+ if value is None:
1965
+ return ""
1966
+ if isinstance(value, bool):
1967
+ return "TRUE" if value else "FALSE"
1968
+ return str(value)
1969
+
1970
+
1971
+ def _search_pattern(text):
1972
+ """SEARCH's wildcard text as a regex: `?` one character, `*` any run, `~x` a literal x."""
1973
+ out, index = [], 0
1974
+ while index < len(text):
1975
+ ch = text[index]
1976
+ if ch == "~" and index + 1 < len(text) and text[index + 1] in "?*~":
1977
+ out.append(re.escape(text[index + 1])); index += 2; continue
1978
+ out.append(".*?" if ch == "*" else "." if ch == "?" else re.escape(ch)); index += 1
1979
+ return "".join(out)
1980
+
1981
+
1982
  def _addr_arg(node, diagnostics):
1983
  """The ADDRESS a1 node names, as a key. Anything else is not an address and refuses."""
1984
  if isinstance(node, dict) and node.get("t") == "a1":
 
2375
  if fn == "FIND":
2376
  start = int(nums[2] or 1)-1 if len(a)>2 else 0; pos = _text(a[1]).find(_text(a[0]), start)
2377
  return pos+1 if pos >= 0 else None
2378
+ if fn in ("SEARCH", "SEARCHB"):
2379
+ # Excel's SEARCH: case-insensitive, `?` one character, `*` any run, `~` escapes the next;
2380
+ # absent text or a start outside the text is #VALUE!. SEARCHB counts bytes only in a
2381
+ # double-byte locale, so in this one it is SEARCH.
2382
+ within = _text(a[1])
2383
+ start = _strict_num(a[2], diagnostics) if len(a) > 2 else 1
2384
+ if start is None or not 1 <= int(start) <= len(within) + 1:
2385
+ _diagnose(diagnostics, "bad_value"); return None
2386
+ found = re.compile(_search_pattern(_text(a[0])), re.IGNORECASE | re.DOTALL).search(
2387
+ within, int(start) - 1)
2388
+ if not found:
2389
+ _diagnose(diagnostics, "bad_value"); return None
2390
+ return found.start() + 1
2391
  if fn in ("REGEX_EXTRACT", "REGEX_REPLACE"):
2392
  pattern, subject = _safe_regex(a[1], a[0]); m = re.search(pattern, subject)
2393
  if fn == "REGEX_EXTRACT": return m.group(0) if m else None
 
3791
  excel = {**REFERENCE_ERRORS, "no_match": "#N/A", "bad_value": "#VALUE!", "div_zero": "#DIV/0!", "numeric_error": "#NUM!"}.get(code)
3792
  if excel: cell["error"]["excelCode"] = excel
3793
  else:
3794
+ cell.update(state="fresh", display=_display_text(cell["value"]))
3795
  cell.pop("refusal", None)
3796
  cell.pop("error", None)
3797
  cell.pop("spill", None)
 
3834
  base = engine.specs.get(child, {})
3835
  style = {**(owner.get("style") or {}), **(base.get("style") or {})}
3836
  record = {"kind": "literal", "value": engine.values.get(child), "state": "fresh",
3837
+ "display": _display_text(engine.values.get(child)),
3838
  "spillParent": origin, "style": style,
3839
  "formulaMeta": {"spillChild": {"version": 1, "origin": origin, "address": child,
3840
  **({"base": base} if base else {})}}}
api/main.py CHANGED
@@ -92,6 +92,7 @@ from core import store # noqa: E402
92
  # 409: a view write whose caller was looking at an older revision. See its handler below.
93
  from core import view_audit # noqa: E402
94
  from deps import Session, module_gate # noqa: E402
 
95
 
96
  _WEB_DIST = _HERE.parents[1] / "web" / "dist"
97
 
@@ -351,6 +352,21 @@ app.include_router(routes_uploads.router)
351
  # control is that the enumeration must NAME every router in this file.**
352
  app.include_router(routes_templates.router) # item 7 / C12 β€” template registry, session-gated
353
  app.include_router(routes_connectors.router) # item 10 / C11 β€” the connectors directory
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
354
  # ⚠ routes_forms is the FIRST NEW PUBLIC DOOR since the `_DEV_FIXTURES` scar (see :257 below). Its
355
  # two paths are DELIBERATELY unauthenticated β€” a form is filled in by someone with no account β€” so
356
  # it joins `/api/health`, the automation hook and the tick on the exempt list. It resolves a token
@@ -527,6 +543,45 @@ def _assert_contract_floor():
527
  "Run aios-web/verify_fields_contract.py.")
528
 
529
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
530
  def _startup_notes():
531
  """Say the two things an operator must know, once, at import β€” never in a response body."""
532
  if aios_session.EPHEMERAL_SECRET:
@@ -535,10 +590,6 @@ def _startup_notes():
535
  "Set it in production.")
536
 
537
 
538
- _assert_contract_floor()
539
- _startup_notes()
540
-
541
-
542
  # β›” THE STATIC MOUNT IS NOW UNAUTHENTICATED, and that is a change this wave made on purpose.
543
  # Before EXIT-3a, `BasicAuth` middleware gated the WHOLE app including this mount. A branded login
544
  # page cannot live behind a password prompt, so the shell's own assets must be public β€” which they
@@ -746,6 +797,7 @@ def _pull_amazon(why):
746
  try:
747
  from harness import amazon_store as _amazon
748
  from harness import runtime as _runtime
 
749
  # The existing boot/resync projection targets tenant #0. Keep credential, mirror and
750
  # projection on that same tenant; never fall back to an operator's environment credential.
751
  tenant_key = "royal-imports"
@@ -758,7 +810,19 @@ def _pull_amazon(why):
758
  # bounded passes requeue themselves until every table is loaded. `scheduled=True` loaded
759
  # ONE table per resync (a 4 hour cadence), so five of seven tables were never pulled, and
760
  # the markers live in the container's mirror, so every restart began again from orders.
761
- if _amazon.needs_first_load(tenant_key, rt=rt):
 
 
 
 
 
 
 
 
 
 
 
 
762
  from core import connector_sync as _connector_sync
763
  receipt = _connector_sync.request_sync("amazon_sp", tenant_key, requested_by="first load")
764
  print(f"[aios-api] amazon sync ({why}): first load "
@@ -766,6 +830,8 @@ def _pull_amazon(why):
766
  return True
767
  rep = _amazon.sync(tenant_key, rt=rt, scheduled=True,
768
  log=lambda message: print(f"[aios-api] amazon sync ({why}): {message}"))
 
 
769
  for problem in rep.get("problems") or []:
770
  print(f"[aios-api] amazon sync PROBLEM ({why}): {problem}")
771
  print(f"[aios-api] amazon sync done ({why}): " + ", ".join(
@@ -1184,7 +1250,7 @@ def _sweep_automation_schemas(why):
1184
 
1185
 
1186
  def _rebuild_odoo_relational(why, previous_meta_fingerprint=None, meta_state_out=None, *,
1187
- tenant=None, cadence_seconds=None):
1188
  """Spawn/refresh the four locked Odoo databases, then compute their cells. `why` is 'boot' or
1189
  'resync' and rides every log line, because "it failed" and "it failed at boot, before anyone
1190
  could have asked" are different diagnoses.
@@ -1207,13 +1273,14 @@ def _rebuild_odoo_relational(why, previous_meta_fingerprint=None, meta_state_out
1207
  from core import user_tables as _ut
1208
  default_cid = routes_connectors._odoo_connection_slot(_rt, None)
1209
  connection_ids = [default_cid]
1210
- try:
1211
- import core.keychain as _keychain
1212
- connection_ids.extend(str(row.get("connection_id") or row.get("id"))
1213
- for row in _keychain.list_entries(_rt)
1214
- if row.get("type") == "odoo")
1215
- except Exception: # noqa: BLE001
1216
- pass
 
1217
  connection_ids = list(dict.fromkeys(cid for cid in connection_ids if cid))
1218
 
1219
  def _health(cid, stage, state, **kwargs):
@@ -1240,10 +1307,14 @@ def _rebuild_odoo_relational(why, previous_meta_fingerprint=None, meta_state_out
1240
  # because a gate that counts RAW occurrences of a call would count the one in this
1241
  # comment too. That gate happens to strip comments first; the next one may not.
1242
  # See `odoo_relational.refresh` and its `all_connections` arm.
1243
- counts = _rel.refresh(_rt, tenant_key, all_connections=True)
1244
- raw_models, schema_errors = _rel.authoritative_schema_snapshot()
1245
- _ut.record_odoo_schema_snapshot(raw_models, complete=not schema_errors,
1246
- errors=schema_errors, st=_rt)
 
 
 
 
1247
  rebuild_errors = []
1248
  # ⭐⭐ AND THEN COMPUTE THE CELLS, which `refresh` does NOT do.
1249
  #
@@ -1319,14 +1390,25 @@ def _rebuild_odoo_relational(why, previous_meta_fingerprint=None, meta_state_out
1319
  cid = str(cid or default_cid)
1320
  report = report if isinstance(report, dict) else {}
1321
  failed = report.get("failed")
 
 
1322
  database_failures, unmatched_failures = (
1323
- routes_connectors._odoo_result_database_failures(_rt, cid, report))
 
1324
  failure = (routes_connectors._odoo_failure(failed, "materialization")
1325
  if failed else None)
1326
- if not failure and (database_failures or unmatched_failures):
 
 
 
1327
  failure = routes_connectors._odoo_failure(
1328
  "one or more databases failed", "materialization")
1329
  tables = report.get("tables") if isinstance(report.get("tables"), dict) else {}
 
 
 
 
 
1330
  database_receipts = report.get("databases")
1331
  database_receipts = database_receipts if isinstance(database_receipts, dict) else {}
1332
  canonical_receipts = report.get("canonicalDatabases")
@@ -1395,6 +1477,13 @@ def _rebuild_odoo_relational(why, previous_meta_fingerprint=None, meta_state_out
1395
  _ut.record_odoo_sync_status(
1396
  phase="error" if safe_errors else "ok", errors=safe_errors,
1397
  st=_rt, connection_id=cid)
 
 
 
 
 
 
 
1398
  meta_state = _rebuild_meta_relational(
1399
  why, _rt, previous_fingerprint=previous_meta_fingerprint, force_relations=True)
1400
  if isinstance(meta_state_out, dict):
@@ -1424,26 +1513,96 @@ def _rebuild_odoo_relational(why, previous_meta_fingerprint=None, meta_state_out
1424
  _ut.record_odoo_sync_status(
1425
  phase="error", errors=[f"{failure['code']}: {failure['message']}"],
1426
  st=_rt, connection_id=cid)
 
 
 
1427
  except Exception:
1428
  pass
1429
  return False
1430
 
1431
 
1432
- # W39-T31 / A63. `sync_all` already knows whether Odoo supplied any change, but this loop used to
1433
- # rebuild every relational table (and its large `user_tables` document) after every poll anyway.
1434
- # A nonempty all-live zero report is the only evidence strong enough to skip that Odoo half. An
1435
- # absent report is not zero: it means manual mode, paused source, or a failed/unknown sync, all of
1436
- # which force the pre-existing rebuild path. `delete_report=None` is one of the ordinary passes
1437
- # between delete reconciliations; it is unknown, not falsely claimed clean.
1438
  _RESYNC_REBUILD_MAX_AGE_SECONDS = 24 * 60 * 60
1439
 
1440
 
1441
- def _resync_rebuild_reason(sync_report, delete_report, last_success_at, now):
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1442
  """Return a fail-safe Odoo-rebuild reason, or ``None`` for a proven no-op.
1443
 
1444
  The policy stays pure so it can be tested without Odoo, DuckDB, or PostgreSQL. Hard deletes
1445
  are noticed by the existing every-fourth-pass reconciliation; an intervening pass does not
1446
- pretend that it checked them, preserving the old visibility bound.
 
 
1447
  """
1448
  if not isinstance(sync_report, dict) or not sync_report:
1449
  return "sync-unconfirmed"
@@ -1451,7 +1610,7 @@ def _resync_rebuild_reason(sync_report, delete_report, last_success_at, now):
1451
  if not isinstance(report, dict) or report.get("phase") != "live":
1452
  return "sync-unconfirmed"
1453
  try:
1454
- if int(report.get("pulled") or 0) > 0:
1455
  return "source-change"
1456
  except (TypeError, ValueError):
1457
  return "sync-unconfirmed"
@@ -1463,8 +1622,20 @@ def _resync_rebuild_reason(sync_report, delete_report, last_success_at, now):
1463
  return "delete-change"
1464
  except (TypeError, ValueError):
1465
  return "sync-unconfirmed"
 
 
 
 
 
1466
  if last_success_at is None or now - last_success_at >= _RESYNC_REBUILD_MAX_AGE_SECONDS:
1467
  return "daily-recovery"
 
 
 
 
 
 
 
1468
  return None
1469
 
1470
 
@@ -1619,6 +1790,14 @@ def _resolve_resync_cadence():
1619
  "manual": manual, "disagreement": disagreement}
1620
 
1621
 
 
 
 
 
 
 
 
 
1622
  def _resync_mirror_pass(manual, passes):
1623
  """ONE bounded mirror pass: DuckDB and the Odoo XML-RPC client, and nothing else.
1624
 
@@ -1729,6 +1908,7 @@ def _store_resync_loop():
1729
  from core import connector_sync
1730
  passes = 0
1731
  last_relational_success_at = None
 
1732
  last_meta_fingerprint = None
1733
  last_meta_applied_at = None
1734
  # ⚠ AMAZON KEEPS ITS OWN FINGERPRINT, never sharing Meta's. Two connectors whose plans change
@@ -1844,6 +2024,7 @@ def _store_resync_loop():
1844
  if _odoo_requested is None:
1845
  _odoo_requested = _run_sync_requests()
1846
  sync_report, delete_report, _err = _resync_mirror_pass(_manual, passes)
 
1847
  RESYNC_STATE["lastPassAt"] = _t.strftime("%Y-%m-%dT%H:%M:%SZ", _t.gmtime())
1848
  RESYNC_STATE["lastPassOk"] = _err is None
1849
  RESYNC_STATE["lastError"] = _err
@@ -1896,14 +2077,19 @@ def _store_resync_loop():
1896
  _meta_previous = last_meta_fingerprint if _meta_reason is None else None
1897
  _amzn_previous = last_amzn_fingerprint if _meta_reason is None else None
1898
  _reason = _resync_rebuild_reason(sync_report, delete_report,
1899
- last_relational_success_at, _meta_now)
 
1900
  # The scheduled pass may now be quiet because Sync now just advanced this mirror.
1901
  # Its zero-change report cannot suppress projection of the explicit Odoo pull.
1902
  if _odoo_requested:
1903
  _reason = "sync now"
1904
  _meta_state = {}
1905
  if _reason is not None:
1906
- print(f"[aios-api] odoo relational rebuild required ({_reason})")
 
 
 
 
1907
  # Test/recovery harnesses replace the rebuild seam with its historical three-argument
1908
  # callable. Bind tenant/cadence whenever the active implementation accepts them,
1909
  # without turning an internal TypeError into a second materialization attempt.
@@ -1913,14 +2099,20 @@ def _store_resync_loop():
1913
  p.kind == _inspect.Parameter.VAR_KEYWORD for p in _rebuild_params.values())
1914
  _rebuild_scope = ({"tenant": _cad["tenant"], "cadence_seconds": _every}
1915
  if _rebuild_variadic or "tenant" in _rebuild_params else {})
 
 
1916
  if _rebuild_odoo_relational(
1917
  "resync", _meta_previous, _meta_state, **_rebuild_scope):
1918
- last_relational_success_at = _t.time()
 
 
 
1919
  else:
1920
  # A prior successful projection no longer proves the changed mirror was applied.
1921
  # Keep recovery due on the next cadence even when that sync has no new rows.
1922
  last_relational_success_at = None
1923
  else:
 
1924
  # A zero Odoo report says nothing about Meta. Its exact-plan fingerprint decides
1925
  # whether this cadence may skip both the full-document write and relation pass.
1926
  try:
@@ -2113,10 +2305,9 @@ if os.environ.get("AIOS_PREWARM") == "1":
2113
  #
2114
  # ⭐ THE HOTFIX MOVED THE CADENCE ONLY, 3600 -> 14400 (R11's existing four-hourly preset), which
2115
  # is safe at any value because the wait below is `sync_wake_event().wait(_every)` and a Sync now
2116
- # SETS it. TWO STRUCTURAL CAUSES REMAIN OPEN AS REGISTER ROWS, and neither is a hotfix:
2117
- # * `_resync_rebuild_reason` treats `pulled > 0` as a change, but `pulled` counts rows FETCHED,
2118
- # not rows CHANGED β€” live rebuilt on passes its own report scored `added/updated/removed: 0`.
2119
- # Narrowing it is what would make an hourly cadence nearly free.
2120
  # * the whole-document store write itself. A partial-JSONB write is a wave.
2121
  # ⚠ SO DO NOT PUT THIS BACK TO 3600 to "fix staleness" without closing one of those two first.
2122
  if os.environ.get("AIOS_STORE_RESYNC") == "1":
 
92
  # 409: a view write whose caller was looking at an older revision. See its handler below.
93
  from core import view_audit # noqa: E402
94
  from deps import Session, module_gate # noqa: E402
95
+ from deps import require_session as deps_require_session # noqa: E402 (S67-T20 freshness door)
96
 
97
  _WEB_DIST = _HERE.parents[1] / "web" / "dist"
98
 
 
352
  # control is that the enumeration must NAME every router in this file.**
353
  app.include_router(routes_templates.router) # item 7 / C12 β€” template registry, session-gated
354
  app.include_router(routes_connectors.router) # item 10 / C11 β€” the connectors directory
355
+
356
+
357
+ @app.get("/api/v1/connectors/odoo/freshness")
358
+ def odoo_sync_freshness(session: Session = Depends(deps_require_session)):
359
+ """⭐⭐ S67-T20 / C4 β€” IS THIS WORKSPACE'S ODOO DATA STALE, AND DID THE LAST SYNC FAIL.
360
+
361
+ `{stale, lastGoodSyncAt, schedule, staleAfterSeconds, lastRunAt, phase, frozen, failure}`
362
+ from `odoo_relational.sync_freshness`. The Connectors card and Connectors > Odoo word it.
363
+ ⚠ A SIBLING OF `GET /connectors/odoo/status`, not a field on it, because that route is another
364
+ lane's file this wave; the integrator may fold these fields into it (COMBINE-AMEND, S67-T20).
365
+ Same administrator door as that route.
366
+ """
367
+ routes_connectors._odoo_admin(session)
368
+ import odoo_relational as _rel_fresh
369
+ return _rel_fresh.sync_freshness(session.runtime)
370
  # ⚠ routes_forms is the FIRST NEW PUBLIC DOOR since the `_DEV_FIXTURES` scar (see :257 below). Its
371
  # two paths are DELIBERATELY unauthenticated β€” a form is filled in by someone with no account β€” so
372
  # it joins `/api/health`, the automation hook and the tick on the exempt list. It resolves a token
 
543
  "Run aios-web/verify_fields_contract.py.")
544
 
545
 
546
+ def _assert_store_guards(cadence=None):
547
+ """Refuse an unbounded store or a cadence below the deployment's own floor."""
548
+ from core import store_pg
549
+
550
+ budget = store_pg.byte_budget_status()
551
+ if not budget["armed"]:
552
+ raise RuntimeError("byte budget disarmed: " + "; ".join(budget["reasons"]))
553
+
554
+ # Read the deployment independently of the resolver: a broken resolver must
555
+ # not be able to supply both sides of its own comparison.
556
+ raw = os.environ.get("AIOS_STORE_RESYNC_SECONDS")
557
+ deployed = None
558
+ if raw is not None:
559
+ try:
560
+ deployed = int(raw.strip())
561
+ except (TypeError, ValueError):
562
+ raise RuntimeError("resync guard disarmed: AIOS_STORE_RESYNC_SECONDS must be a "
563
+ "positive integer") from None
564
+ if deployed <= 0:
565
+ raise RuntimeError("resync guard disarmed: AIOS_STORE_RESYNC_SECONDS must be a "
566
+ "positive integer")
567
+ if cadence is None:
568
+ cadence = _resolve_resync_cadence()
569
+ resolved = cadence.get("resolvedSeconds")
570
+ if type(resolved) is not int or resolved <= 0:
571
+ raise RuntimeError("resync guard disarmed: resolved cadence must be a positive integer")
572
+ if deployed is not None and resolved < deployed:
573
+ raise RuntimeError(f"resync guard disarmed: resolved cadence {resolved}s is below "
574
+ f"deployed floor {deployed}s")
575
+ deployment = f"{deployed}s deployed" if deployed is not None else "not deployed"
576
+ return {"ok": True, "byte_budget": budget,
577
+ "resync": {"ok": True, "resolved_seconds": resolved, "deployed_seconds": deployed},
578
+ "message": ("Byte budget armed: yes. "
579
+ f"Table {budget['max_table_bytes'] / 1048576:g} MB; "
580
+ f"document {budget['max_document_bytes'] / 1048576:g} MB; "
581
+ f"store ceiling {budget['document_ceiling_bytes'] / 1048576:g} MB. "
582
+ f"Resync: {resolved}s resolved vs {deployment}.")}
583
+
584
+
585
  def _startup_notes():
586
  """Say the two things an operator must know, once, at import β€” never in a response body."""
587
  if aios_session.EPHEMERAL_SECRET:
 
590
  "Set it in production.")
591
 
592
 
 
 
 
 
593
  # β›” THE STATIC MOUNT IS NOW UNAUTHENTICATED, and that is a change this wave made on purpose.
594
  # Before EXIT-3a, `BasicAuth` middleware gated the WHOLE app including this mount. A branded login
595
  # page cannot live behind a password prompt, so the shell's own assets must be public β€” which they
 
797
  try:
798
  from harness import amazon_store as _amazon
799
  from harness import runtime as _runtime
800
+ import routes_odoo_tables as _rot
801
  # The existing boot/resync projection targets tenant #0. Keep credential, mirror and
802
  # projection on that same tenant; never fall back to an operator's environment credential.
803
  tenant_key = "royal-imports"
 
810
  # bounded passes requeue themselves until every table is loaded. `scheduled=True` loaded
811
  # ONE table per resync (a 4 hour cadence), so five of seven tables were never pulled, and
812
  # the markers live in the container's mirror, so every restart began again from orders.
813
+ first_load_needed = _amazon.needs_first_load(tenant_key, rt=rt)
814
+ # D-3495: `needs_first_load` (and `sync`, below) call `restore_mirror` as their very
815
+ # first step, which writes the Amazon mirror tables directly on the shared datastore
816
+ # instance. Nothing else republished the read-through registry until the Odoo relational
817
+ # rebuild's `refresh_state` ran, minutes later (`_rebuild_odoo_relational`, further down
818
+ # this boot) β€” so a grid read landing in between (staging, 2026-09-29: `ut_amzn_orders`
819
+ # 409 for ~30 min after "Amazon mirror restored") answered stale or absent. Invalidate
820
+ # right here, cheaply: this only clears the registry and its built flag, so the next read
821
+ # (or `_sources()` below) does a fresh per-table `count(*)` probe against the mirror that
822
+ # was just restored β€” never the slow `plan`/`apply_plan` write `refresh_state` does.
823
+ _rot.invalidate_sources()
824
+ _rot._sources()
825
+ if first_load_needed:
826
  from core import connector_sync as _connector_sync
827
  receipt = _connector_sync.request_sync("amazon_sp", tenant_key, requested_by="first load")
828
  print(f"[aios-api] amazon sync ({why}): first load "
 
830
  return True
831
  rep = _amazon.sync(tenant_key, rt=rt, scheduled=True,
832
  log=lambda message: print(f"[aios-api] amazon sync ({why}): {message}"))
833
+ _rot.invalidate_sources()
834
+ _rot._sources()
835
  for problem in rep.get("problems") or []:
836
  print(f"[aios-api] amazon sync PROBLEM ({why}): {problem}")
837
  print(f"[aios-api] amazon sync done ({why}): " + ", ".join(
 
1250
 
1251
 
1252
  def _rebuild_odoo_relational(why, previous_meta_fingerprint=None, meta_state_out=None, *,
1253
+ tenant=None, cadence_seconds=None, table_keys=None):
1254
  """Spawn/refresh the four locked Odoo databases, then compute their cells. `why` is 'boot' or
1255
  'resync' and rides every log line, because "it failed" and "it failed at boot, before anyone
1256
  could have asked" are different diagnoses.
 
1273
  from core import user_tables as _ut
1274
  default_cid = routes_connectors._odoo_connection_slot(_rt, None)
1275
  connection_ids = [default_cid]
1276
+ if table_keys is None:
1277
+ try:
1278
+ import core.keychain as _keychain
1279
+ connection_ids.extend(str(row.get("connection_id") or row.get("id"))
1280
+ for row in _keychain.list_entries(_rt)
1281
+ if row.get("type") == "odoo")
1282
+ except Exception: # noqa: BLE001
1283
+ pass
1284
  connection_ids = list(dict.fromkeys(cid for cid in connection_ids if cid))
1285
 
1286
  def _health(cid, stage, state, **kwargs):
 
1307
  # because a gate that counts RAW occurrences of a call would count the one in this
1308
  # comment too. That gate happens to strip comments first; the next one may not.
1309
  # See `odoo_relational.refresh` and its `all_connections` arm.
1310
+ # Only the designated default owns this mirror. A selection must never be lent to
1311
+ # another connection; the full recovery path still refreshes every connection.
1312
+ counts = _rel.refresh(_rt, tenant_key, all_connections=table_keys is None,
1313
+ **({"table_keys": table_keys} if table_keys is not None else {}))
1314
+ if table_keys is None:
1315
+ raw_models, schema_errors = _rel.authoritative_schema_snapshot()
1316
+ _ut.record_odoo_schema_snapshot(raw_models, complete=not schema_errors,
1317
+ errors=schema_errors, st=_rt)
1318
  rebuild_errors = []
1319
  # ⭐⭐ AND THEN COMPUTE THE CELLS, which `refresh` does NOT do.
1320
  #
 
1390
  cid = str(cid or default_cid)
1391
  report = report if isinstance(report, dict) else {}
1392
  failed = report.get("failed")
1393
+ # S67-T19: a read-through `limits` notice is the design working, not a failed
1394
+ # database; health reads only the entries that withheld rows (`_rel.health_view`).
1395
  database_failures, unmatched_failures = (
1396
+ routes_connectors._odoo_result_database_failures(
1397
+ _rt, cid, _rel.health_view(report)))
1398
  failure = (routes_connectors._odoo_failure(failed, "materialization")
1399
  if failed else None)
1400
+ # S67-T19: count drift within tolerance is recorded per database (below) and does not
1401
+ # fail the connection; a real failure or a stale mirror's gap still does.
1402
+ if not failure and (_rel.blocking_failures(database_failures, report)
1403
+ or unmatched_failures):
1404
  failure = routes_connectors._odoo_failure(
1405
  "one or more databases failed", "materialization")
1406
  tables = report.get("tables") if isinstance(report.get("tables"), dict) else {}
1407
+ for key, delta in sorted(tables.items()):
1408
+ if isinstance(delta, dict):
1409
+ print(f"[aios-api] odoo relational table rebuilt ({why}): {key} "
1410
+ + str({name: delta.get(name, 0)
1411
+ for name in ("added", "updated", "removed")}))
1412
  database_receipts = report.get("databases")
1413
  database_receipts = database_receipts if isinstance(database_receipts, dict) else {}
1414
  canonical_receipts = report.get("canonicalDatabases")
 
1477
  _ut.record_odoo_sync_status(
1478
  phase="error" if safe_errors else "ok", errors=safe_errors,
1479
  st=_rt, connection_id=cid)
1480
+ # S67-T20 / C4: the tenant-wide freshness stamp follows the DEFAULT connection, the
1481
+ # one this mirror serves. Best effort: a stamp must never fail the rebuild it records.
1482
+ if cid == str(default_cid):
1483
+ try:
1484
+ _rel.record_sync_outcome(_rt, ok=not safe_errors, at=refreshed_at)
1485
+ except Exception: # noqa: BLE001
1486
+ pass
1487
  meta_state = _rebuild_meta_relational(
1488
  why, _rt, previous_fingerprint=previous_meta_fingerprint, force_relations=True)
1489
  if isinstance(meta_state_out, dict):
 
1513
  _ut.record_odoo_sync_status(
1514
  phase="error", errors=[f"{failure['code']}: {failure['message']}"],
1515
  st=_rt, connection_id=cid)
1516
+ # S67-T20 / C4: a failed rebuild is a sync that did NOT end well.
1517
+ import odoo_relational as _rel_failed
1518
+ _rel_failed.record_sync_outcome(_rt, ok=False)
1519
  except Exception:
1520
  pass
1521
  return False
1522
 
1523
 
1524
+ # D-780: fetched rows include an idempotent cursor overlap. Only mirror content compared with
1525
+ # the last successful apply proves equality; scalar backfills need not move write_date.
 
 
 
 
1526
  _RESYNC_REBUILD_MAX_AGE_SECONDS = 24 * 60 * 60
1527
 
1528
 
1529
+ # Dependencies of odoo_relational.plan/read_* and the two read-through table specs. Customers
1530
+ # and products are compiled databases, no longer refreshable ut_odoo_* tables. Unknown entities
1531
+ # (including their sources) deliberately select full recovery, never an empty selection.
1532
+ _RESYNC_ENTITY_TABLE_KEYS = {
1533
+ "account_move": ("ut_odoo_invoices", "ut_odoo_bills", "ut_odoo_vendors"),
1534
+ "sale_order": ("ut_odoo_orders",),
1535
+ "res_partner": ("ut_odoo_invoices", "ut_odoo_orders", "ut_odoo_agents", "ut_odoo_vendors"),
1536
+ "account_invoice_line_agent": ("ut_odoo_invoices", "ut_odoo_agents"),
1537
+ "account_account": ("ut_odoo_accounts",),
1538
+ "sale_order_line": ("ut_odoo_order_lines",),
1539
+ "account_move_line": ("ut_odoo_gl_lines",),
1540
+ }
1541
+
1542
+
1543
+ def _resync_source_fingerprints(sync_report, cur=None):
1544
+ """Count and hash every column, without fetching rows out of the local mirror.
1545
+
1546
+ Use the sync writer's cursor factory (also used for status/validation), never a second
1547
+ DuckDB instance or a tenant-store read. A missing optional table has an explicit fingerprint;
1548
+ a failed read returns unknown and forces recovery. A lent fixture cursor is never closed.
1549
+ """
1550
+ owned = cur is None
1551
+ try:
1552
+ if owned:
1553
+ from harness import datastore as _ds
1554
+ cur = _ds.connect()
1555
+ cur.execute("BEGIN TRANSACTION")
1556
+ available = {row[0] for row in cur.execute(
1557
+ "SELECT table_name FROM information_schema.tables WHERE table_schema = 'main'"
1558
+ ).fetchall()}
1559
+ entities = set(_RESYNC_ENTITY_TABLE_KEYS) | set(sync_report or {})
1560
+ fingerprints = {}
1561
+ for entity in sorted(entities):
1562
+ if entity not in available:
1563
+ fingerprints[entity] = None
1564
+ continue
1565
+ quoted = '"' + entity.replace('"', '""') + '"'
1566
+ fingerprints[entity] = tuple(cur.execute(
1567
+ f"SELECT count(*), coalesce(bit_xor(hash(t)), 0) FROM {quoted} AS t"
1568
+ ).fetchone())
1569
+ if owned:
1570
+ cur.execute("COMMIT")
1571
+ return fingerprints
1572
+ except Exception as exc: # noqa: BLE001
1573
+ print(f"[aios-api] odoo source fingerprint unconfirmed ({type(exc).__name__})")
1574
+ return {}
1575
+ finally:
1576
+ if owned and cur is not None:
1577
+ cur.close()
1578
+
1579
+
1580
+ def _resync_changed_entities(previous_fingerprints, current_fingerprints):
1581
+ """The union catches disappearing entities as well as additions and scalar changes."""
1582
+ previous = previous_fingerprints or {}
1583
+ current = current_fingerprints or {}
1584
+ return sorted(entity for entity in set(previous) | set(current)
1585
+ if entity not in previous or entity not in current
1586
+ or previous[entity] != current[entity])
1587
+
1588
+
1589
+ def _resync_table_keys(changed_entities):
1590
+ """None means full recovery; an unmapped entity can never suppress an apply."""
1591
+ if not changed_entities or any(entity not in _RESYNC_ENTITY_TABLE_KEYS
1592
+ for entity in changed_entities):
1593
+ return None
1594
+ return sorted({key for entity in changed_entities for key in _RESYNC_ENTITY_TABLE_KEYS[entity]})
1595
+
1596
+
1597
+ def _resync_rebuild_reason(sync_report, delete_report, last_success_at, now,
1598
+ previous_fingerprints=None, current_fingerprints=None):
1599
  """Return a fail-safe Odoo-rebuild reason, or ``None`` for a proven no-op.
1600
 
1601
  The policy stays pure so it can be tested without Odoo, DuckDB, or PostgreSQL. Hard deletes
1602
  are noticed by the existing every-fourth-pass reconciliation; an intervening pass does not
1603
+ pretend that it checked them, preserving the old visibility bound. The four-argument legacy
1604
+ caller retains its conservative fetched-row signal; the resident loop always supplies a
1605
+ fingerprint map (an empty map means a failed read, never equality).
1606
  """
1607
  if not isinstance(sync_report, dict) or not sync_report:
1608
  return "sync-unconfirmed"
 
1610
  if not isinstance(report, dict) or report.get("phase") != "live":
1611
  return "sync-unconfirmed"
1612
  try:
1613
+ if int(report.get("pulled") or 0) > 0 and current_fingerprints is None:
1614
  return "source-change"
1615
  except (TypeError, ValueError):
1616
  return "sync-unconfirmed"
 
1622
  return "delete-change"
1623
  except (TypeError, ValueError):
1624
  return "sync-unconfirmed"
1625
+ if current_fingerprints is not None:
1626
+ if not isinstance(current_fingerprints, dict) or not current_fingerprints:
1627
+ return "fingerprint-unconfirmed"
1628
+ if not isinstance(previous_fingerprints, dict) or not previous_fingerprints:
1629
+ return "no-fingerprint"
1630
  if last_success_at is None or now - last_success_at >= _RESYNC_REBUILD_MAX_AGE_SECONDS:
1631
  return "daily-recovery"
1632
+ if current_fingerprints is not None:
1633
+ # Kept inline so the pure decision can still be AST-extracted on its own by old gates.
1634
+ changed = sorted(entity for entity in set(previous_fingerprints) | set(current_fingerprints)
1635
+ if entity not in previous_fingerprints or entity not in current_fingerprints
1636
+ or previous_fingerprints[entity] != current_fingerprints[entity])
1637
+ if changed:
1638
+ return "source-change: " + ",".join(changed)
1639
  return None
1640
 
1641
 
 
1790
  "manual": manual, "disagreement": disagreement}
1791
 
1792
 
1793
+ # All import-time refusals run before any boot or resync thread can start.
1794
+ _assert_contract_floor()
1795
+ STORE_GUARDS = _assert_store_guards()
1796
+ from core import neon_usage as _neon_usage # noqa: E402
1797
+ _neon_usage.set_guards(STORE_GUARDS)
1798
+ _startup_notes()
1799
+
1800
+
1801
  def _resync_mirror_pass(manual, passes):
1802
  """ONE bounded mirror pass: DuckDB and the Odoo XML-RPC client, and nothing else.
1803
 
 
1908
  from core import connector_sync
1909
  passes = 0
1910
  last_relational_success_at = None
1911
+ last_source_fingerprints = None
1912
  last_meta_fingerprint = None
1913
  last_meta_applied_at = None
1914
  # ⚠ AMAZON KEEPS ITS OWN FINGERPRINT, never sharing Meta's. Two connectors whose plans change
 
2024
  if _odoo_requested is None:
2025
  _odoo_requested = _run_sync_requests()
2026
  sync_report, delete_report, _err = _resync_mirror_pass(_manual, passes)
2027
+ source_fingerprints = _resync_source_fingerprints(sync_report)
2028
  RESYNC_STATE["lastPassAt"] = _t.strftime("%Y-%m-%dT%H:%M:%SZ", _t.gmtime())
2029
  RESYNC_STATE["lastPassOk"] = _err is None
2030
  RESYNC_STATE["lastError"] = _err
 
2077
  _meta_previous = last_meta_fingerprint if _meta_reason is None else None
2078
  _amzn_previous = last_amzn_fingerprint if _meta_reason is None else None
2079
  _reason = _resync_rebuild_reason(sync_report, delete_report,
2080
+ last_relational_success_at, _meta_now,
2081
+ last_source_fingerprints, source_fingerprints)
2082
  # The scheduled pass may now be quiet because Sync now just advanced this mirror.
2083
  # Its zero-change report cannot suppress projection of the explicit Odoo pull.
2084
  if _odoo_requested:
2085
  _reason = "sync now"
2086
  _meta_state = {}
2087
  if _reason is not None:
2088
+ _table_keys = (_resync_table_keys(_resync_changed_entities(
2089
+ last_source_fingerprints, source_fingerprints))
2090
+ if _reason.startswith("source-change:") else None)
2091
+ print(f"[aios-api] odoo relational rebuild required ({_reason}); "
2092
+ f"table_keys={_table_keys if _table_keys is not None else 'all'}")
2093
  # Test/recovery harnesses replace the rebuild seam with its historical three-argument
2094
  # callable. Bind tenant/cadence whenever the active implementation accepts them,
2095
  # without turning an internal TypeError into a second materialization attempt.
 
2099
  p.kind == _inspect.Parameter.VAR_KEYWORD for p in _rebuild_params.values())
2100
  _rebuild_scope = ({"tenant": _cad["tenant"], "cadence_seconds": _every}
2101
  if _rebuild_variadic or "tenant" in _rebuild_params else {})
2102
+ if _rebuild_variadic or "table_keys" in _rebuild_params:
2103
+ _rebuild_scope["table_keys"] = _table_keys
2104
  if _rebuild_odoo_relational(
2105
  "resync", _meta_previous, _meta_state, **_rebuild_scope):
2106
+ last_source_fingerprints = source_fingerprints or None
2107
+ # Partial successes cannot postpone the mandatory full daily recovery forever.
2108
+ if _table_keys is None:
2109
+ last_relational_success_at = _t.time()
2110
  else:
2111
  # A prior successful projection no longer proves the changed mirror was applied.
2112
  # Keep recovery due on the next cadence even when that sync has no new rows.
2113
  last_relational_success_at = None
2114
  else:
2115
+ print("[aios-api] odoo relational rebuild skipped (unchanged)")
2116
  # A zero Odoo report says nothing about Meta. Its exact-plan fingerprint decides
2117
  # whether this cadence may skip both the full-document write and relation pass.
2118
  try:
 
2305
  #
2306
  # ⭐ THE HOTFIX MOVED THE CADENCE ONLY, 3600 -> 14400 (R11's existing four-hourly preset), which
2307
  # is safe at any value because the wait below is `sync_wake_event().wait(_every)` and a Sync now
2308
+ # SETS it. D-780 now compares local content fingerprints with the last successful apply and
2309
+ # logs unchanged skips; changed mapped entities select only their dependent table keys.
2310
+ # The other structural cost remains outside that skip policy:
 
2311
  # * the whole-document store write itself. A partial-JSONB write is a wave.
2312
  # ⚠ SO DO NOT PUT THIS BACK TO 3600 to "fix staleness" without closing one of those two first.
2313
  if os.environ.get("AIOS_STORE_RESYNC") == "1":
api/odoo_relational.py CHANGED
@@ -60,6 +60,7 @@ RI_SLUGS = ("", "royal-imports")
60
  INVOICES_KEY = "ut_odoo_invoices"
61
  CUSTOMERS_KEY = "ut_odoo_customers"
62
  ORDERS_KEY = "ut_odoo_orders"
 
63
  PRODUCTS_KEY = "ut_odoo_products"
64
  #: ⭐ WAVE 28 (owner R1): *"ALL of Unique ID in Odoo is a database e.g. Customers/Products/Agents,
65
  #: etc. Including expenses and GL codes."* Four more DOCUMENT/REGISTRY grains, each measured to
@@ -169,6 +170,34 @@ def _refreshed_field():
169
  "default": False, "description": "When this row was last reconciled against Odoo."}
170
 
171
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
172
  def agent_fields():
173
  """One row per SALES AGENT, keyed on the `res.partner` id.
174
 
@@ -1854,6 +1883,9 @@ ODOO_TABLE_MODELS = {
1854
  ACCOUNTS_KEY: "account.account", VENDORS_KEY: "res.partner", BILLS_KEY: "account.move",
1855
  ORDER_LINES_KEY: "sale.order.line", GL_LINES_KEY: "account.move.line",
1856
  }
 
 
 
1857
 
1858
 
1859
  def authoritative_schema_snapshot():
@@ -2589,6 +2621,7 @@ def _aios_grid():
2589
  #: crashes" posture `routes_odoo_tables._degrade` already takes for the eight Odoo grids.
2590
  _REGISTRY_COL_TYPE = {
2591
  "int": "BIGINT", "currency": "DOUBLE", "number": "DOUBLE", "pct": "DOUBLE",
 
2592
  "measure": "DOUBLE", "checkbox": "BOOLEAN", "date": "DATE", "datetime": "TIMESTAMP",
2593
  }
2594
 
@@ -2610,7 +2643,7 @@ def _registry_col_value(value, kind):
2610
  try:
2611
  if kind == "int":
2612
  return int(value)
2613
- if kind in ("currency", "number", "pct", "measure"):
2614
  return float(value)
2615
  if kind == "checkbox":
2616
  return bool(value)
@@ -2623,7 +2656,7 @@ def _registry_col_value(value, kind):
2623
  return str(value)
2624
 
2625
 
2626
- def registry_grid_spec(cur, table_key, fields, rows):
2627
  """A `_source_for`-shaped spec for one compiled registry grid, materialised into a throwaway
2628
  DuckDB TEMP TABLE on THIS cursor (W52-T11).
2629
 
@@ -2637,7 +2670,8 @@ def registry_grid_spec(cur, table_key, fields, rows):
2637
  request simply gets `CREATE OR REPLACE`d over with that request's own fresh, freshly-scoped
2638
  read. Nothing here survives the cursor closing, and nothing here is written to `royal.duckdb`.
2639
 
2640
- ⭐ THE JOIN KEY IS ALWAYS `pid`, NEVER A DECLARED FIELD. Both grains carry it as a raw row
 
2641
  datum rather than a column a person can hide (`customer_data.pid` IS the `res.partner` id;
2642
  `product_data.pid` is `crc32(default_code)` β€” see `_related_source_value`'s own note), so it
2643
  is forced into the materialised columns here even when the caller's field list never
@@ -2645,10 +2679,10 @@ def registry_grid_spec(cur, table_key, fields, rows):
2645
  """
2646
  keys = list(dict.fromkeys(
2647
  str(f.get("key") or "") for f in (fields or []) if isinstance(f, dict) and f.get("key")))
2648
- if "pid" not in keys:
2649
- keys.insert(0, "pid")
2650
  by_key = {str(f.get("key")): f for f in (fields or []) if isinstance(f, dict)}
2651
- by_key.setdefault("pid", {"type": "int"})
2652
 
2653
  table = f"_registry_grid_{table_key}"
2654
  cols_sql = ", ".join(
@@ -2658,9 +2692,15 @@ def registry_grid_spec(cur, table_key, fields, rows):
2658
  values = [tuple(_registry_col_value(r.get(k), (by_key.get(k) or {}).get("type")) for k in keys)
2659
  for r in (rows or []) if isinstance(r, dict)]
2660
  if values:
2661
- placeholders = ", ".join(["?"] * len(keys))
2662
- cur.executemany(f'INSERT INTO "{table}" VALUES ({placeholders})', values)
2663
- return {"table": table, "id": '"pid"', "cols": {k: (f'"{k}"', str) for k in keys}}
 
 
 
 
 
 
2664
 
2665
 
2666
  def _declared_from_spec(fields):
@@ -2766,6 +2806,7 @@ TABLE_SUBJECTS = {
2766
  # spec row tried to bring it back.
2767
  INVOICES_KEY: "odoo:account.move.customer",
2768
  ORDERS_KEY: "odoo:sale.order",
 
2769
  AGENTS_KEY: "odoo:res.partner.agent",
2770
  ACCOUNTS_KEY: "odoo:account.account",
2771
  VENDORS_KEY: "odoo:res.partner.vendor",
@@ -3150,7 +3191,8 @@ DEFAULT_CONNECTION = "default"
3150
  #: Every table key this connector owns, un-namespaced. The two RETIRED keys are members so a
3151
  #: document that still holds one resolves to its connection rather than reading as a stranger's
3152
  #: table β€” `apply_plan` has to know whose copy it is popping.
3153
- BASE_TABLE_KEYS = frozenset([key for _b, key, _l, _f in TABLES] + list(RETIRED_KEYS))
 
3154
 
3155
  #: `core.user_tables._field_key` truncates a table key at 60 characters SILENTLY, and a truncated
3156
  #: key is a key that can collide with another connection's. The longest base key is
@@ -3577,6 +3619,112 @@ def sync_seconds(rt):
3577
  return max(int(secs), SYNC_FLOOR_SECONDS)
3578
 
3579
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
3580
  def frozen(rt):
3581
  """Has this tenant DISCONNECTED Odoo (R10)? Frozen grids keep every row and every field and
3582
  stop being refreshed β€” distinct from PAUSED, which is temporary and keeps the credential."""
@@ -3697,15 +3845,9 @@ def plan(cur, rt=None, connection_id=None, buckets=None):
3697
  continue
3698
  built[bucket] = reader(cur, excluded)
3699
  problems = []
3700
-
3701
- if rt is not None:
3702
- needed = [k for k in keys.values() if k not in existing]
3703
- if needed and len(existing) + len(needed) > ut.MAX_TABLES:
3704
- problems.append(
3705
- f"tenant holds {len(existing)} of MAX_TABLES={ut.MAX_TABLES} user tables and "
3706
- f"needs {len(needed)} more ({', '.join(needed)}) for connection "
3707
- f"{connection_id or 'default'}; refusing rather than creating "
3708
- f"part of a linked set")
3709
 
3710
  for bucket, _base, _label, _fields in tables:
3711
  key = keys[bucket]
@@ -3751,6 +3893,221 @@ def plan(cur, rt=None, connection_id=None, buckets=None):
3751
  return built
3752
 
3753
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
3754
  def apply_plan(rt, built, username="automation", today=None, report=None,
3755
  connection_id=None, retire=True):
3756
  """Create-or-merge every table in `built` and its rows. Idempotent by construction.
@@ -3864,10 +4221,8 @@ def _ensure_table_inplace(cur, key, label, fields, rows, username, stamp):
3864
  subject = table_subject_for(key)
3865
  table = cur.get(key)
3866
  if table is None:
3867
- if len(cur) >= ut.MAX_TABLES:
3868
- # ⚠ `ut_ensure` returns silently at this cap; a silent no-op here would report a
3869
- # successful refresh over a table that does not exist.
3870
- raise Refused(f"{key}: tenant is at MAX_TABLES={ut.MAX_TABLES}; nothing created")
3871
  # ⭐⭐ W33-T41 / item 12a β€” THE UNIQUENESS CHECK, ON THE LINE THAT MINTED THE DUPLICATE.
3872
  #
3873
  # β›” IT GUARDS **CREATION ONLY**, AND THAT IS THE WHOLE DESIGN, NOT A WEAKENING. This
@@ -4195,11 +4550,8 @@ def sync_discovered_models(rt, source_connection_id=None, namespace=None,
4195
  f"{choice['model']}: too large to store, served nowhere until read-through"
4196
  for choice in oversized]}
4197
  keys = [str(choice["key"]) for choice in selected]
4198
- missing = [key for key in keys if key not in existing]
4199
- if missing and len(existing) + len(missing) > ut.MAX_TABLES:
4200
- raise Refused(f"tenant holds {len(existing)} of MAX_TABLES={ut.MAX_TABLES} databases and "
4201
- f"needs {len(missing)} more enabled Odoo models; nothing was created")
4202
- source = (rt.odoo_source() if source_connection_id is None
4203
  else rt.odoo_source(source_connection_id))
4204
  if source is None:
4205
  raise Refused("the selected Odoo connection is unavailable, so no discovered model was read")
@@ -4312,6 +4664,93 @@ def available(rt=None, tenant_key=None, connection_id=None):
4312
  return False
4313
 
4314
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
4315
  def refresh_curated(rt, cur, buckets, *, connection_id=None, username="automation",
4316
  today=None, retire=True, on_apply=None, source_client=None):
4317
  """Refresh each enabled database independently; retain failed databases unchanged.
@@ -4598,6 +5037,16 @@ def refresh(rt, tenant, username="automation", cur=None, today=None, connection_
4598
  return out
4599
 
4600
 
 
 
 
 
 
 
 
 
 
 
4601
  def refresh_all(rt, tenant, username="automation", cur=None, today=None):
4602
  """Refresh EVERY Odoo connection this tenant holds, one at a time (R4 / R7 / C4).
4603
 
@@ -4634,12 +5083,20 @@ def refresh_all(rt, tenant, username="automation", cur=None, today=None):
4634
  primary = None
4635
  for entry_cid in order:
4636
  named = None if (entry_cid is None or entry_cid == default) else entry_cid
4637
- try:
4638
- one = refresh(rt, tenant, username=username,
4639
- cur=(cur if named is None else None),
4640
- today=today, connection_id=named)
4641
- except Exception as exc: # noqa: BLE001
4642
- one = {"tables": {}, "failed": f"{type(exc).__name__}: Odoo connection refresh failed"}
 
 
 
 
 
 
 
 
4643
  reports[str(entry_cid or default or DEFAULT_CONNECTION)] = one
4644
  if named is None and primary is None:
4645
  primary = one
 
60
  INVOICES_KEY = "ut_odoo_invoices"
61
  CUSTOMERS_KEY = "ut_odoo_customers"
62
  ORDERS_KEY = "ut_odoo_orders"
63
+ SHIPMENTS_KEY = "ut_odoo_shipments"
64
  PRODUCTS_KEY = "ut_odoo_products"
65
  #: ⭐ WAVE 28 (owner R1): *"ALL of Unique ID in Odoo is a database e.g. Customers/Products/Agents,
66
  #: etc. Including expenses and GL codes."* Four more DOCUMENT/REGISTRY grains, each measured to
 
170
  "default": False, "description": "When this row was last reconciled against Odoo."}
171
 
172
 
173
+ def shipment_fields():
174
+ """Outgoing pickings, with elapsed delivery time measured at actual completion."""
175
+ return [_preset({"source": "overlay", **field}) for field in (
176
+ {"key": "picking_no", "type": "text", "label": "Shipment", "pinned": True,
177
+ "default": True},
178
+ {"key": "odoo_id", "type": "int", "label": "Odoo ID", "default": False,
179
+ "description": "The `stock.picking` id. Also this row's id."},
180
+ {"key": "origin", "type": "text", "label": "Source document", "default": True},
181
+ {"key": "order_no", "type": "text", "label": "Order", "default": True},
182
+ {"key": "order_id", "type": "int", "label": "Odoo order id", "default": False},
183
+ {"key": "order_link", "type": "link", "label": "Order record", "default": True,
184
+ "link": {"on": "odoo_id", "from": "order_id", "table": ORDERS_KEY}},
185
+ {"key": "customer", "type": "text", "label": "Customer", "default": True},
186
+ {"key": "partner_id", "type": "int", "label": "Odoo partner id", "default": False},
187
+ {"key": "customer_link", "type": "link", "label": "Customer record", "default": False,
188
+ "link": {"on": "pid", "from": "partner_id", "table": "customer_data"}},
189
+ {"key": "state", "type": "select", "label": "State", "default": True,
190
+ "options": ["draft", "waiting", "confirmed", "assigned", "done", "cancel"]},
191
+ {"key": "scheduled_date", "type": "date", "label": "Scheduled date", "default": True},
192
+ {"key": "date_done", "type": "date", "label": "Ship date", "default": True},
193
+ {"key": "order_date", "type": "date", "label": "Order date", "default": True},
194
+ {"key": "delivery_days", "type": "float", "agg": "avg", "label": "Delivery days",
195
+ "default": True, "description": "Elapsed days from the sale order date to the actual "
196
+ "shipment completion. Blank when either date is missing."},
197
+ _refreshed_field(),
198
+ )]
199
+
200
+
201
  def agent_fields():
202
  """One row per SALES AGENT, keyed on the `res.partner` id.
203
 
 
1883
  ACCOUNTS_KEY: "account.account", VENDORS_KEY: "res.partner", BILLS_KEY: "account.move",
1884
  ORDER_LINES_KEY: "sale.order.line", GL_LINES_KEY: "account.move.line",
1885
  }
1886
+ # Shipments deliberately stay outside this map and TABLES: its consumers route mapped models
1887
+ # through the DuckDB refresh and suppress their discovered-model reads. There is no picking
1888
+ # mirror. The live shipment writer below stamps its model directly on the stored connector.
1889
 
1890
 
1891
  def authoritative_schema_snapshot():
 
2621
  #: crashes" posture `routes_odoo_tables._degrade` already takes for the eight Odoo grids.
2622
  _REGISTRY_COL_TYPE = {
2623
  "int": "BIGINT", "currency": "DOUBLE", "number": "DOUBLE", "pct": "DOUBLE",
2624
+ "float": "DOUBLE",
2625
  "measure": "DOUBLE", "checkbox": "BOOLEAN", "date": "DATE", "datetime": "TIMESTAMP",
2626
  }
2627
 
 
2643
  try:
2644
  if kind == "int":
2645
  return int(value)
2646
+ if kind in ("currency", "number", "pct", "measure", "float"):
2647
  return float(value)
2648
  if kind == "checkbox":
2649
  return bool(value)
 
2656
  return str(value)
2657
 
2658
 
2659
+ def registry_grid_spec(cur, table_key, fields, rows, *, id_key="pid", bulk=False):
2660
  """A `_source_for`-shaped spec for one compiled registry grid, materialised into a throwaway
2661
  DuckDB TEMP TABLE on THIS cursor (W52-T11).
2662
 
 
2670
  request simply gets `CREATE OR REPLACE`d over with that request's own fresh, freshly-scoped
2671
  read. Nothing here survives the cursor closing, and nothing here is written to `royal.duckdb`.
2672
 
2673
+ The default identity is `pid`; stored shipments explicitly use `odoo_id` instead.
2674
+ Both registry grains carry `pid` as a raw row
2675
  datum rather than a column a person can hide (`customer_data.pid` IS the `res.partner` id;
2676
  `product_data.pid` is `crc32(default_code)` β€” see `_related_source_value`'s own note), so it
2677
  is forced into the materialised columns here even when the caller's field list never
 
2679
  """
2680
  keys = list(dict.fromkeys(
2681
  str(f.get("key") or "") for f in (fields or []) if isinstance(f, dict) and f.get("key")))
2682
+ if id_key not in keys:
2683
+ keys.insert(0, id_key)
2684
  by_key = {str(f.get("key")): f for f in (fields or []) if isinstance(f, dict)}
2685
+ by_key.setdefault(id_key, {"type": "int"})
2686
 
2687
  table = f"_registry_grid_{table_key}"
2688
  cols_sql = ", ".join(
 
2692
  values = [tuple(_registry_col_value(r.get(k), (by_key.get(k) or {}).get("type")) for k in keys)
2693
  for r in (rows or []) if isinstance(r, dict)]
2694
  if values:
2695
+ if bulk:
2696
+ # Column arrays avoid 40,000 Python/DuckDB execute round trips for stored shipments.
2697
+ selects = ", ".join(
2698
+ f'unnest(?)::{_REGISTRY_COL_TYPE.get(by_key[k].get("type"), "VARCHAR")}' for k in keys)
2699
+ cur.execute(f'INSERT INTO "{table}" SELECT {selects}', [list(col) for col in zip(*values)])
2700
+ else:
2701
+ placeholders = ", ".join(["?"] * len(keys))
2702
+ cur.executemany(f'INSERT INTO "{table}" VALUES ({placeholders})', values)
2703
+ return {"table": table, "id": f'"{id_key}"', "cols": {k: (f'"{k}"', str) for k in keys}}
2704
 
2705
 
2706
  def _declared_from_spec(fields):
 
2806
  # spec row tried to bring it back.
2807
  INVOICES_KEY: "odoo:account.move.customer",
2808
  ORDERS_KEY: "odoo:sale.order",
2809
+ SHIPMENTS_KEY: "odoo:stock.picking.outgoing",
2810
  AGENTS_KEY: "odoo:res.partner.agent",
2811
  ACCOUNTS_KEY: "odoo:account.account",
2812
  VENDORS_KEY: "odoo:res.partner.vendor",
 
3191
  #: Every table key this connector owns, un-namespaced. The two RETIRED keys are members so a
3192
  #: document that still holds one resolves to its connection rather than reading as a stranger's
3193
  #: table β€” `apply_plan` has to know whose copy it is popping.
3194
+ BASE_TABLE_KEYS = frozenset([key for _b, key, _l, _f in TABLES]
3195
+ + list(RETIRED_KEYS) + [SHIPMENTS_KEY])
3196
 
3197
  #: `core.user_tables._field_key` truncates a table key at 60 characters SILENTLY, and a truncated
3198
  #: key is a key that can collide with another connection's. The longest base key is
 
3619
  return max(int(secs), SYNC_FLOOR_SECONDS)
3620
 
3621
 
3622
+ #: ⭐⭐ S67-T20 / C4 β€” how far past its cadence a sync may run before its data is called stale.
3623
+ #: Two hours of grace on top of the interval: a daily sync is stale after 26 hours (C4's number),
3624
+ #: long enough that one slow pass does not raise a warning and short enough that a dead sync is
3625
+ #: named on the next day, not three weeks later as staging's was.
3626
+ STALE_GRACE_SECONDS = 2 * 3600
3627
+
3628
+
3629
+ def _utc_now():
3630
+ return _dt.datetime.now(_dt.timezone.utc)
3631
+
3632
+
3633
+ def _parse_iso(value):
3634
+ """An ISO timestamp as an aware UTC datetime, or None when it is not one."""
3635
+ if not value:
3636
+ return None
3637
+ try:
3638
+ at = _dt.datetime.fromisoformat(str(value).replace("Z", "+00:00"))
3639
+ except ValueError:
3640
+ return None
3641
+ return at if at.tzinfo else at.replace(tzinfo=_dt.timezone.utc)
3642
+
3643
+
3644
+ def record_sync_outcome(rt, ok, at=None):
3645
+ """Stamp this tenant's last sync, and its last GOOD sync when `ok`, on the connector's config.
3646
+
3647
+ β›” ON THE CONFIG DOCUMENT, NOT ON `odoo_sync_status`, because that document is REPLACED whole
3648
+ on every phase write (`core.user_tables.record_odoo_sync_status`), so a stamp kept there would
3649
+ be erased by the next `syncing`. Every writer of this config document updates it in place.
3650
+ """
3651
+ stamp = at or _utc_now().isoformat()
3652
+
3653
+ def _up(cur):
3654
+ cur = dict(cur) if isinstance(cur, dict) else {}
3655
+ cur["lastSyncAt"] = stamp
3656
+ if ok:
3657
+ cur["lastGoodSyncAt"] = stamp
3658
+ return cur
3659
+
3660
+ rt.update(CONFIG_KEY, _up, flush="sync")
3661
+ return stamp
3662
+
3663
+
3664
+ def _effective_sync_seconds(preset):
3665
+ """The interval a sync can actually keep: the tenant's preset, floored, and never faster than
3666
+ the deployment's own resync floor (`AIOS_STORE_RESYNC_SECONDS`), which `main.py` applies with
3667
+ `max()`. None for `manual`: a person who asked not to be synced has no schedule to fall
3668
+ behind."""
3669
+ secs = SYNC_PRESETS.get(preset)
3670
+ if secs is None:
3671
+ return None
3672
+ secs = max(int(secs), SYNC_FLOOR_SECONDS)
3673
+ import os
3674
+ try:
3675
+ env = int(str(os.environ.get("AIOS_STORE_RESYNC_SECONDS") or "").strip() or 0)
3676
+ except ValueError:
3677
+ env = 0
3678
+ return max(secs, env)
3679
+
3680
+
3681
+ def sync_freshness(rt, now=None):
3682
+ """C4: is this tenant's Odoo data stale, when was it last synced well, and did the last run
3683
+ fail. PURE over the store reads.
3684
+
3685
+ `stale` is true when the last good sync is older than the effective cadence plus
3686
+ `STALE_GRACE_SECONDS`, or when none has ever been recorded. A `manual` or disconnected
3687
+ (frozen) connector is never stale: it is not behind a schedule it does not have. `failure` is
3688
+ the last run's own reason, its code prefix removed, when that run ended in error.
3689
+ """
3690
+ now = now or _utc_now()
3691
+ try:
3692
+ raw = rt.get(CONFIG_KEY) if rt is not None else None
3693
+ except Exception: # noqa: BLE001
3694
+ raw = None
3695
+ raw = raw if isinstance(raw, dict) else {}
3696
+ cfg = read_config(rt)
3697
+ try:
3698
+ status = rt.get(_ut().ODOO_SYNC_STATUS_KEY) if rt is not None else None
3699
+ except Exception: # noqa: BLE001
3700
+ status = None
3701
+ status = status if isinstance(status, dict) else {}
3702
+ phase = str(status.get("phase") or "unknown")
3703
+ last_good = raw.get("lastGoodSyncAt") or None
3704
+ # Before the first stamp this deployment writes, a status that last read `ok` IS a good sync.
3705
+ if not last_good and phase == "ok":
3706
+ last_good = status.get("lastRun") or None
3707
+ every = _effective_sync_seconds(cfg["syncEvery"])
3708
+ allowed = None if every is None else every + STALE_GRACE_SECONDS
3709
+ stale = False
3710
+ if allowed is not None and not cfg["frozen"]:
3711
+ good_at = _parse_iso(last_good)
3712
+ stale = good_at is None or (now - good_at).total_seconds() > allowed
3713
+ failure = None
3714
+ errors = status.get("errors") if isinstance(status.get("errors"), list) else []
3715
+ if phase == "error" and errors:
3716
+ text = str(errors[0])
3717
+ code, _sep, message = text.partition(": ")
3718
+ if not _sep:
3719
+ code, message = "sync_failed", text
3720
+ failure = {"code": code.strip()[:80], "message": message.strip()[:300],
3721
+ "at": status.get("lastRun") or None}
3722
+ return {"stale": bool(stale), "lastGoodSyncAt": last_good,
3723
+ "schedule": cfg["syncEvery"], "staleAfterSeconds": allowed,
3724
+ "lastRunAt": status.get("lastRun") or None, "phase": phase,
3725
+ "frozen": cfg["frozen"], "failure": failure}
3726
+
3727
+
3728
  def frozen(rt):
3729
  """Has this tenant DISCONNECTED Odoo (R10)? Frozen grids keep every row and every field and
3730
  stop being refreshed β€” distinct from PAUSED, which is temporary and keeps the credential."""
 
3845
  continue
3846
  built[bucket] = reader(cur, excluded)
3847
  problems = []
3848
+ # ⭐⭐ D-3284 (owner ruling 2026-09-27): connector databases take NO `MAX_TABLES` slot
3849
+ # (`user_tables.counts_toward_table_cap`), so the cap can no longer refuse, or half-spawn, a
3850
+ # linked set. The refusal that stood here counted every database, people's and connectors'.
 
 
 
 
 
 
3851
 
3852
  for bucket, _base, _label, _fields in tables:
3853
  key = keys[bucket]
 
3893
  return built
3894
 
3895
 
3896
+ def _shipment_datetime(value):
3897
+ """Odoo's naive datetimes are UTC; preserve seconds before deriving date cells."""
3898
+ if not value:
3899
+ return None
3900
+ parsed = _dt.datetime.fromisoformat(str(value).replace("Z", "+00:00"))
3901
+ return (parsed.replace(tzinfo=_dt.timezone.utc) if parsed.tzinfo is None
3902
+ else parsed.astimezone(_dt.timezone.utc))
3903
+
3904
+
3905
+ def _shipment_relation_id(value):
3906
+ if isinstance(value, (list, tuple)):
3907
+ value = value[0] if value else None
3908
+ return int(value) if value else None
3909
+
3910
+
3911
+ def read_shipments_live(client):
3912
+ """All outgoing pickings in id pages, then their sale orders in bounded read batches.
3913
+
3914
+ Only guarded read methods are used. The guarded OdooClient has no ``read`` convenience
3915
+ method, so sale-order batches use search_read with an explicit id domain as well.
3916
+ """
3917
+ pickings, last_id = [], 0
3918
+ fields = ["name", "origin", "sale_id", "partner_id", "state", "scheduled_date", "date_done"]
3919
+ while True:
3920
+ domain = [("picking_type_code", "=", "outgoing")]
3921
+ if last_id:
3922
+ domain.append(("id", ">", last_id))
3923
+ page = client.search_read("stock.picking", domain, fields,
3924
+ limit=MODEL_READ_PAGE, order="id")
3925
+ if not page:
3926
+ break
3927
+ for picking in page:
3928
+ rid = int(picking["id"])
3929
+ if rid <= last_id:
3930
+ raise Refused("stock.picking: source ids did not advance; no shipments written")
3931
+ last_id = rid
3932
+ pickings.append(picking)
3933
+ # Continue until an empty page, even if the server returned less than requested.
3934
+
3935
+ ids = sorted({oid for picking in pickings
3936
+ if (oid := _shipment_relation_id(picking.get("sale_id"))) is not None})
3937
+ orders = {}
3938
+ for start in range(0, len(ids), MODEL_READ_PAGE):
3939
+ chunk = ids[start:start + MODEL_READ_PAGE]
3940
+ batch = client.search_read("sale.order", [("id", "in", chunk)],
3941
+ ["name", "date_order"], limit=len(chunk), order="id")
3942
+ if len(batch) != len(chunk) or {int(row["id"]) for row in batch} != set(chunk):
3943
+ raise Refused("sale.order: referenced shipment orders were not all readable; "
3944
+ "the prior database was preserved. Check Odoo read access and retry.")
3945
+ orders.update((int(row["id"]), row) for row in batch)
3946
+
3947
+ rows, stamp = {}, _iso_today()
3948
+ for picking in pickings:
3949
+ rid = int(picking["id"])
3950
+ oid = _shipment_relation_id(picking.get("sale_id"))
3951
+ partner = picking.get("partner_id")
3952
+ pid = _shipment_relation_id(partner)
3953
+ order = orders.get(oid) or {}
3954
+ ordered = _shipment_datetime(order.get("date_order"))
3955
+ done = _shipment_datetime(picking.get("date_done"))
3956
+ scheduled = _shipment_datetime(picking.get("scheduled_date"))
3957
+ rows[str(rid)] = {
3958
+ "picking_no": picking.get("name") or "", "odoo_id": rid,
3959
+ "origin": picking.get("origin") or "", "order_no": order.get("name") or "",
3960
+ "order_id": oid, **({"order_link": str(oid)} if oid is not None else {}),
3961
+ "customer": (partner[1] or "")
3962
+ if isinstance(partner, (list, tuple)) and len(partner) > 1 else "",
3963
+ "partner_id": pid, **({"customer_link": str(pid)} if pid is not None else {}),
3964
+ "state": picking.get("state") or "",
3965
+ "scheduled_date": scheduled.date().isoformat() if scheduled else "",
3966
+ "date_done": done.date().isoformat() if done else "",
3967
+ "order_date": ordered.date().isoformat() if ordered else "",
3968
+ "delivery_days": (done - ordered).total_seconds() / 86400
3969
+ if done and ordered else None,
3970
+ "refreshed": stamp,
3971
+ }
3972
+ return rows
3973
+
3974
+
3975
+ def reconcile_shipments(rows, client, year):
3976
+ """Independent source count for completed outgoing shipments in one UTC calendar year."""
3977
+ year = int(year)
3978
+ start, end = f"{year:04d}-01-01", f"{year + 1:04d}-01-01"
3979
+ values = rows.values() if isinstance(rows, dict) else rows
3980
+ count = sum(1 for row in values if row.get("state") == "done"
3981
+ and start <= (row.get("date_done") or "") < end)
3982
+ source_count = client.search_count("stock.picking", [
3983
+ ("picking_type_code", "=", "outgoing"), ("state", "=", "done"),
3984
+ ("date_done", ">=", start + " 00:00:00"), ("date_done", "<", end + " 00:00:00")])
3985
+ return {"year": year, "rowCount": count, "sourceCount": source_count,
3986
+ "ok": type(source_count) is int and source_count == count}
3987
+
3988
+
3989
+ def _shipment_problems(rows, existing, key):
3990
+ ut = _ut()
3991
+ problems = []
3992
+ stored = len((existing.get(key) or {}).get("rows") or {})
3993
+ if stored and len(rows) < stored * MAX_SHRINK:
3994
+ problems.append(f"{key}: read returned {len(rows)} rows against {stored} stored; "
3995
+ "preserved the prior database. Check Odoo read access and retry sync.")
3996
+ # D-3284: no MAX_TABLES check here; a connector database takes no slot.
3997
+ size = ut.rows_bytes(rows)
3998
+ if size > ut.MAX_TABLE_BYTES:
3999
+ problems.append(f"{key}: {size} bytes exceeds MAX_TABLE_BYTES={ut.MAX_TABLE_BYTES}; "
4000
+ "the prior database was preserved. Use read-through storage before retrying.")
4001
+ return problems
4002
+
4003
+
4004
+ def plan_shipments(rt, connection_id=None, client=None, today=None):
4005
+ """Prepare shipments without writing a store; injected clients keep gates hermetic."""
4006
+ if client is None:
4007
+ source = rt.odoo_source() if connection_id is None else rt.odoo_source(connection_id)
4008
+ if source is None:
4009
+ raise Refused("the selected Odoo connection is unavailable; no shipments read")
4010
+ from core.odoo import get_odoo
4011
+ with source._bound():
4012
+ return plan_shipments(rt, connection_id, get_odoo(), today)
4013
+ _cid, namespace = connection_namespace(rt, connection_id)
4014
+ rows = read_shipments_live(client)
4015
+ for row in rows.values():
4016
+ row["refreshed"] = today or _iso_today()
4017
+ if namespace:
4018
+ row.pop("customer_link", None) # customer_data belongs to the default connection
4019
+ key = table_key_for(SHIPMENTS_KEY, namespace)
4020
+ result = {"rows": rows, "problems": _shipment_problems(
4021
+ rows, rt.get(_ut().STORE_KEY) or {}, key)}
4022
+ if namespace:
4023
+ result["connection_id"] = namespace
4024
+ return result
4025
+
4026
+
4027
+ def apply_shipments(rt, planned, username="automation", today=None):
4028
+ """Publish via the relational writer, refusing partial reads before touching stored rows."""
4029
+ from copy import deepcopy
4030
+ from core import connector_sync
4031
+ if planned.get("problems"):
4032
+ raise Refused("; ".join(planned["problems"]))
4033
+ ut, stamp = _ut(), today or _iso_today()
4034
+ namespace = planned.get("connection_id")
4035
+ key = table_key_for(SHIPMENTS_KEY, namespace)
4036
+ rows = {rid: {**row, "refreshed": stamp} for rid, row in planned["rows"].items()}
4037
+ problems = _shipment_problems(rows, rt.get(ut.STORE_KEY) or {}, key)
4038
+ if problems:
4039
+ raise Refused("; ".join(problems))
4040
+ written = {}
4041
+
4042
+ def apply(current):
4043
+ current = current if isinstance(current, dict) else {}
4044
+ # Recheck the current document under the store's update lock, including concurrent growth.
4045
+ problems = _shipment_problems(rows, current, key)
4046
+ if problems:
4047
+ raise Refused("; ".join(problems))
4048
+ current = dict(current)
4049
+ if key in current:
4050
+ current[key] = deepcopy(current[key])
4051
+ previous = (current.get(key) or {}).get("rows") or {}
4052
+ link_only_changes = sum(
4053
+ 1 for rid, row in rows.items() if rid in previous
4054
+ and any(link in previous[rid] and link not in row
4055
+ for link in ("order_link", "customer_link"))
4056
+ and all(str(previous[rid].get(k, "")) == str(v)
4057
+ for k, v in row.items() if k != "refreshed"))
4058
+ new = key not in current
4059
+ written[key] = _ensure_table_inplace(
4060
+ current, key, table_label_for("Odoo shipments", connection_label(rt, namespace)),
4061
+ namespaced_fields(shipment_fields(), namespace),
4062
+ [{**row, "_id": rid} for rid, row in rows.items()], username, stamp)
4063
+ if new:
4064
+ current[key].pop("created", None) # match the existing staging definition
4065
+ for rid, row in rows.items():
4066
+ for link in ("order_link", "customer_link"):
4067
+ if link not in row:
4068
+ current[key]["rows"][rid].pop(link, None)
4069
+ written[key]["updated"] += link_only_changes
4070
+ current[key]["connector"] = {
4071
+ **connector_sync.database_identity(
4072
+ "odoo", connection_for_table_key(key, rt), key, "stock.picking"),
4073
+ "key": "odoo", "readOnly": True}
4074
+ # Existing user cells also count toward the unchanged byte budget.
4075
+ problems = _shipment_problems(current[key]["rows"], current, key)
4076
+ if problems:
4077
+ raise Refused("; ".join(problems))
4078
+ return current
4079
+
4080
+ rt.update(ut.STORE_KEY, apply, flush="sync")
4081
+ ut.register_connected(key)
4082
+ return written
4083
+
4084
+
4085
+ def refresh_shipments(rt, *, connection_id=None, year=None, today=None,
4086
+ username="automation", client=None):
4087
+ """Reconcile before publication, then independently count the rows read back from storage."""
4088
+ if client is None:
4089
+ source = rt.odoo_source() if connection_id is None else rt.odoo_source(connection_id)
4090
+ if source is None:
4091
+ raise Refused("the selected Odoo connection is unavailable; no shipments read")
4092
+ from core.odoo import get_odoo
4093
+ with source._bound():
4094
+ return refresh_shipments(rt, connection_id=connection_id, year=year, today=today,
4095
+ username=username, client=get_odoo())
4096
+ stamp = today or _iso_today()
4097
+ year = int(year) if year is not None else int(stamp[:4])
4098
+ planned = plan_shipments(rt, connection_id, client, stamp)
4099
+ reconciled = reconcile_shipments(planned["rows"], client, year)
4100
+ if not reconciled["ok"]:
4101
+ raise Refused(f"stock.picking: {year} done outgoing shipments do not reconcile "
4102
+ f"({reconciled['rowCount']} planned, {reconciled['sourceCount']} source); "
4103
+ "nothing written")
4104
+ applied = apply_shipments(rt, planned, username, stamp)
4105
+ key = table_key_for(SHIPMENTS_KEY, planned.get("connection_id"))
4106
+ stored = ((rt.get(_ut().STORE_KEY, fresh=True) or {}).get(key) or {}).get("rows") or {}
4107
+ return {"planned": planned, "reconcile_planned": reconciled, "apply": applied,
4108
+ "stored_rows": stored, "reconcile_stored": reconcile_shipments(stored, client, year)}
4109
+
4110
+
4111
  def apply_plan(rt, built, username="automation", today=None, report=None,
4112
  connection_id=None, retire=True):
4113
  """Create-or-merge every table in `built` and its rows. Idempotent by construction.
 
4221
  subject = table_subject_for(key)
4222
  table = cur.get(key)
4223
  if table is None:
4224
+ # D-3284 (owner ruling 2026-09-27): every table this writer creates is a connector
4225
+ # database (Odoo, Meta, Amazon), and those take no MAX_TABLES slot, so no cap refusal here.
 
 
4226
  # ⭐⭐ W33-T41 / item 12a β€” THE UNIQUENESS CHECK, ON THE LINE THAT MINTED THE DUPLICATE.
4227
  #
4228
  # β›” IT GUARDS **CREATION ONLY**, AND THAT IS THE WHOLE DESIGN, NOT A WEAKENING. This
 
4550
  f"{choice['model']}: too large to store, served nowhere until read-through"
4551
  for choice in oversized]}
4552
  keys = [str(choice["key"]) for choice in selected]
4553
+ # D-3284: enabled Odoo models are connector databases and take no MAX_TABLES slot.
4554
+ source =(rt.odoo_source() if source_connection_id is None
 
 
 
4555
  else rt.odoo_source(source_connection_id))
4556
  if source is None:
4557
  raise Refused("the selected Odoo connection is unavailable, so no discovered model was read")
 
4664
  return False
4665
 
4666
 
4667
+ #: ⭐ S67-T19 β€” the `limits` effects that REPORT a serving choice rather than withheld rows.
4668
+ #: `read_through` says a database is served THROUGH the mirror with no ceiling at all
4669
+ #: (`core.user_tables.limit_report`), which is the design working, not a failure.
4670
+ INFORMATIONAL_LIMIT_EFFECTS = frozenset({"read_through"})
4671
+
4672
+
4673
+ def failing_limits(limits):
4674
+ """Only the `limits` entries that describe rows a pass did NOT serve, PURE.
4675
+
4676
+ β›”β›” STAGING'S `materialization_failed` WAS THIS, NAMED 2026-09-28 FROM ITS OWN RECEIPTS. The
4677
+ rebuild handed its whole `limits` map to the health classifier, which reads EVERY entry as a
4678
+ materialization failure, so the two read-through grains (`ut_odoo_order_lines`,
4679
+ `ut_odoo_gl_lines`, "no ceiling applies to them") failed the connection on every single sync
4680
+ and `odoo_sync_status` could never read `ok`. The report keeps them; only health drops them.
4681
+ """
4682
+ if not isinstance(limits, dict):
4683
+ return limits
4684
+ return {key: entry for key, entry in limits.items()
4685
+ if not (isinstance(entry, dict)
4686
+ and entry.get("effect") in INFORMATIONAL_LIMIT_EFFECTS)}
4687
+
4688
+
4689
+ #: ⭐ S67-T19 β€” the reconciliation DRIFT a good sync may carry: Odoo keeps taking invoices while a
4690
+ #: rebuild runs, and a hard delete leaves a mirror row until `reconcile_deletes` (every fourth
4691
+ #: pass). Measured on a mirror pulled 25 minutes before its rebuild: invoices 32,980 vs 32,978,
4692
+ #: orders 34,059 vs 34,058, agents 17 vs 18, GL lines 1,024,102 vs 1,024,178, customers 3,715 vs
4693
+ #: 3,716. A stale mirror is nothing like that: staging's unpulled seed was 626 orders and 732
4694
+ #: invoices short. The receipt keeps its exact counts either way; this only decides whether the
4695
+ #: CONNECTION's sync counts as good.
4696
+ RECONCILE_DRIFT_ROWS = 10
4697
+ RECONCILE_DRIFT_RATIO = 0.002
4698
+
4699
+
4700
+ def reconciliation_drift(receipt):
4701
+ """`(difference, tolerance)` for a count-reconciliation receipt, or None when it has no counts."""
4702
+ if not isinstance(receipt, dict):
4703
+ return None
4704
+ source = receipt.get("sourceCount")
4705
+ if isinstance(receipt.get("unexplainedCount"), int):
4706
+ diff = int(receipt["unexplainedCount"])
4707
+ else:
4708
+ mirror = receipt.get("mirrorCount")
4709
+ if not isinstance(source, int) or not isinstance(mirror, int):
4710
+ return None
4711
+ diff = abs(source - mirror)
4712
+ base = source if isinstance(source, int) and source > 0 else 0
4713
+ return diff, max(RECONCILE_DRIFT_ROWS, int(base * RECONCILE_DRIFT_RATIO))
4714
+
4715
+
4716
+ def blocking_failures(database_failures, report):
4717
+ """The database failures that make a sync NOT good, PURE.
4718
+
4719
+ A reconciliation failure whose count drift is within `reconciliation_drift`'s tolerance is
4720
+ recorded against its database and does not fail the connection; every other failure does.
4721
+ """
4722
+ receipts = {}
4723
+ for name in ("databases", "canonicalDatabases"):
4724
+ group = report.get(name) if isinstance(report, dict) else None
4725
+ if isinstance(group, dict):
4726
+ receipts.update(group)
4727
+ out = {}
4728
+ for key, failure in (database_failures or {}).items():
4729
+ if isinstance(failure, dict) and failure.get("code") == "reconciliation_incomplete":
4730
+ drift = reconciliation_drift(receipts.get(key))
4731
+ if drift is not None and drift[0] <= drift[1]:
4732
+ continue
4733
+ out[key] = failure
4734
+ return out
4735
+
4736
+
4737
+ def health_view(report):
4738
+ """The refresh report as the sync-health classifier must read it, PURE.
4739
+
4740
+ Identical to `report` except that `limits` carries only `failing_limits`; the returned report
4741
+ itself (the log line, the Sync report) still names every read-through database.
4742
+ """
4743
+ if not isinstance(report, dict) or not isinstance(report.get("limits"), dict):
4744
+ return report
4745
+ view = dict(report)
4746
+ remaining = failing_limits(report["limits"])
4747
+ if remaining:
4748
+ view["limits"] = remaining
4749
+ else:
4750
+ view.pop("limits", None)
4751
+ return view
4752
+
4753
+
4754
  def refresh_curated(rt, cur, buckets, *, connection_id=None, username="automation",
4755
  today=None, retire=True, on_apply=None, source_client=None):
4756
  """Refresh each enabled database independently; retain failed databases unchanged.
 
5037
  return out
5038
 
5039
 
5040
+ #: ⭐ S67-T19 β€” the store errors a connection refresh retries ONCE. Staging's rebuild at
5041
+ #: 2026-09-28 16:03Z failed whole with `ProtocolViolation` (SQLSTATE 08P01), which is how the
5042
+ #: DigitalOcean pool (PgBouncer) reports its own timeouts while every lane container shares it;
5043
+ #: nothing was written and the next scheduled pass was a day away. `refresh` is idempotent, so a
5044
+ #: second attempt after a pause is safe; a refusal or any other exception is never retried.
5045
+ TRANSIENT_STORE_ERRORS = frozenset({"ProtocolViolation", "OperationalError", "PoolTimeout",
5046
+ "ConnectionTimeout", "AdminShutdown"})
5047
+ REFRESH_RETRY_SECONDS = 20
5048
+
5049
+
5050
  def refresh_all(rt, tenant, username="automation", cur=None, today=None):
5051
  """Refresh EVERY Odoo connection this tenant holds, one at a time (R4 / R7 / C4).
5052
 
 
5083
  primary = None
5084
  for entry_cid in order:
5085
  named = None if (entry_cid is None or entry_cid == default) else entry_cid
5086
+ for attempt in (1, 2):
5087
+ try:
5088
+ one = refresh(rt, tenant, username=username,
5089
+ cur=(cur if named is None else None),
5090
+ today=today, connection_id=named)
5091
+ break
5092
+ except Exception as exc: # noqa: BLE001
5093
+ # S67-T19: one retry for a TRANSIENT store error only; every other failure is
5094
+ # recorded exactly as before.
5095
+ if attempt == 1 and type(exc).__name__ in TRANSIENT_STORE_ERRORS:
5096
+ _time.sleep(REFRESH_RETRY_SECONDS)
5097
+ continue
5098
+ one = {"tables": {}, "failed": f"{type(exc).__name__}: Odoo connection refresh failed"}
5099
+ break
5100
  reports[str(entry_cid or default or DEFAULT_CONNECTION)] = one
5101
  if named is None and primary is None:
5102
  primary = one
api/routes_admin.py CHANGED
@@ -605,7 +605,8 @@ def _perm_modules(session, surfaces=False):
605
  skeys = set()
606
  if surfaces:
607
  # ⭐⭐ OWNER RULING 2026-08-18 β€” the Assistant and Agents surfaces are
608
- # governable ACCESS toggles in the ACCOUNT editor. `surfaces` is caller-explicit and
 
609
  # defaults CLOSED because the other consumer of this list is the CHANNEL-AGENT editor
610
  # (`routes_slack.get_channel_agent`), whose principal is enforced per DATABASE
611
  # (`agent_rows`/`agent_may_read`) and never opens an app surface: an Assistant toggle
@@ -869,11 +870,11 @@ def _module_fields(key, session=None):
869
  import aios_grid
870
  key = str(key or "")
871
  if key in {s["key"] for s in perms.governable_surface_modules()}:
872
- # ⭐ A SURFACE (Assistant, Agents) has NO field schema, and `[]` is the honest
873
  # answer rather than the wave-26 empty-option trap: the editor's schemaless rendering
874
  # is a deliberate state (access toggle only, R9), and `_clean_perms` refuses any
875
  # filter or hiddenFields validated against this empty vocabulary. An unknown key
876
- # still RAISES below β€” this arm admits exactly the derived surface list.
877
  return []
878
  if key.startswith("ut_"):
879
  if session is None:
@@ -2337,7 +2338,8 @@ def get_perms(username: str, session: Session = Depends(admin_gate)):
2337
  # save of ANY unrelated change would write the explicit deny for real. `nav_may_open`
2338
  # is the evaluator the nav and the route gate actually ask, legacy fallback included,
2339
  # so the box shows what the account can reach and an untouched save stays a no-op.
2340
- _default = (perm_scope.nav_may_open(users._public(uname, rec), k)
 
2341
  if any(m.get("surface") and m["key"] == k for m in modules) else
2342
  perm_scope.may_read(users._public(uname, rec), k, st=session.runtime))
2343
  # ⭐⭐ W38-T19 β€” `metrics` RIDES EVERY ENTRY, BACKFILLED TO GRANTED ON A RECORD WRITTEN
 
605
  skeys = set()
606
  if surfaces:
607
  # ⭐⭐ OWNER RULING 2026-08-18 β€” the Assistant and Agents surfaces are
608
+ # governable ACCESS toggles in the ACCOUNT editor, alongside R3/C6's AI apps (MCP)
609
+ # gateway capability. `surfaces` is caller-explicit and
610
  # defaults CLOSED because the other consumer of this list is the CHANNEL-AGENT editor
611
  # (`routes_slack.get_channel_agent`), whose principal is enforced per DATABASE
612
  # (`agent_rows`/`agent_may_read`) and never opens an app surface: an Assistant toggle
 
870
  import aios_grid
871
  key = str(key or "")
872
  if key in {s["key"] for s in perms.governable_surface_modules()}:
873
+ # ⭐ A SURFACE (Assistant, Agents, AI apps) has NO field schema, and `[]` is the honest
874
  # answer rather than the wave-26 empty-option trap: the editor's schemaless rendering
875
  # is a deliberate state (access toggle only, R9), and `_clean_perms` refuses any
876
  # filter or hiddenFields validated against this empty vocabulary. An unknown key
877
+ # still RAISES below: this arm admits the page surfaces and R3/C6's gateway capability.
878
  return []
879
  if key.startswith("ut_"):
880
  if session is None:
 
2338
  # save of ANY unrelated change would write the explicit deny for real. `nav_may_open`
2339
  # is the evaluator the nav and the route gate actually ask, legacy fallback included,
2340
  # so the box shows what the account can reach and an untouched save stays a no-op.
2341
+ # AI apps also uses this evaluator: absence grants even for a restricted legacy list.
2342
+ _default = (perm_scope.nav_may_open(users._public(uname, rec), k, st=session.runtime)
2343
  if any(m.get("surface") and m["key"] == k for m in modules) else
2344
  perm_scope.may_read(users._public(uname, rec), k, st=session.runtime))
2345
  # ⭐⭐ W38-T19 β€” `metrics` RIDES EVERY ENTRY, BACKFILLED TO GRANTED ON A RECORD WRITTEN
api/routes_auth.py CHANGED
@@ -69,6 +69,17 @@ def _throttled(key, now=None):
69
  raise _auth_store_down("_throttled", exc)
70
 
71
 
 
 
 
 
 
 
 
 
 
 
 
72
  def _note_failure(key, now=None):
73
  if key is None:
74
  return 0
@@ -196,7 +207,7 @@ def login(request: Request, response: Response, body: dict = Body(default=None))
196
  raise err(429, "too_many_attempts",
197
  f"too many failed attempts. Try again in {wait} seconds")
198
  if not user:
199
- user = users.verify(uname, pw) if (uname and pw) else None
200
  if not user:
201
  # CONSTANT-TIME-ISH 401. `users.verify` returns immediately for an unknown username (no
202
  # record, no PBKDF2) and burns ~200k iterations for a known one, so the response time
@@ -356,7 +367,7 @@ def _mfa_credentials(uname, password):
356
  print(f"[aios-api] {exc}")
357
  raise err(429, "too_many_attempts",
358
  f"too many failed attempts. Try again in {wait} seconds")
359
- user = users.verify(uname, password) if (uname and password) else None
360
  if not user:
361
  try:
362
  users._hash(password or "x", "00" * 16)
@@ -620,7 +631,7 @@ def change_password(body: dict = Body(default=None)):
620
  print(f"[aios-api] {exc}")
621
  raise err(429, "too_many_attempts",
622
  f"too many failed attempts. Try again in {wait} seconds")
623
- user = users.verify(uname, pw) if (uname and pw) else None
624
  if not user:
625
  try:
626
  users._hash(pw or "x", "00" * 16)
 
69
  raise _auth_store_down("_throttled", exc)
70
 
71
 
72
+ def _verify(uname, pw):
73
+ """`users.verify`, with an UNREADABLE registry answered 503 before `_note_failure` runs.
74
+
75
+ HOTFIX 2026-09-28: a store `PoolTimeout` used to read as "no such account", so a correct
76
+ password got 401 `invalid_credentials` and was charged to the lockout budget."""
77
+ try:
78
+ return users.verify(uname, pw) if (uname and pw) else None
79
+ except users.RegistryUnavailable as exc:
80
+ raise _auth_store_down("verify", exc)
81
+
82
+
83
  def _note_failure(key, now=None):
84
  if key is None:
85
  return 0
 
207
  raise err(429, "too_many_attempts",
208
  f"too many failed attempts. Try again in {wait} seconds")
209
  if not user:
210
+ user = _verify(uname, pw)
211
  if not user:
212
  # CONSTANT-TIME-ISH 401. `users.verify` returns immediately for an unknown username (no
213
  # record, no PBKDF2) and burns ~200k iterations for a known one, so the response time
 
367
  print(f"[aios-api] {exc}")
368
  raise err(429, "too_many_attempts",
369
  f"too many failed attempts. Try again in {wait} seconds")
370
+ user = _verify(uname, password)
371
  if not user:
372
  try:
373
  users._hash(password or "x", "00" * 16)
 
631
  print(f"[aios-api] {exc}")
632
  raise err(429, "too_many_attempts",
633
  f"too many failed attempts. Try again in {wait} seconds")
634
+ user = _verify(uname, pw)
635
  if not user:
636
  try:
637
  users._hash(pw or "x", "00" * 16)
api/routes_connected_tables.py CHANGED
@@ -30,7 +30,7 @@ behind green gates in wave 23, and wave 28 proved a MOUNTED router can still be
30
  (`session.is_admin`, an attribute that does not exist, raising above the route's own `try`). So
31
  `verify_meta` both enumerates `main.app.routes` for this path AND calls it.
32
  """
33
- from fastapi import APIRouter, Depends, Query
34
 
35
  from deps import Session, err, require_session
36
 
@@ -45,6 +45,8 @@ def connected_table_identity(table_key: str, session: Session = Depends(require_
45
 
46
  # Definitions only: this endpoint neither reads rows nor bypasses the table permission wall.
47
  definition = _defn_or_refuse(session, table_key, defs_only=True)
 
 
48
  identity = user_tables.connector_database_identity(
49
  table_key, rt=session.runtime, definition=definition)
50
  if identity is None:
@@ -54,6 +56,7 @@ def connected_table_identity(table_key: str, session: Session = Depends(require_
54
 
55
 
56
  @router.get("/connected-tables/{table_key}/rows")
 
57
  def connected_table_rows(table_key: str,
58
  offset: int = Query(default=0, ge=0),
59
  limit: int = Query(default=0),
@@ -64,7 +67,8 @@ def connected_table_rows(table_key: str,
64
  related: str = Query(default=None),
65
  groupBy: str = Query(default=None),
66
  groupKey: str = Query(default=None),
67
- session: Session = Depends(require_session)):
 
68
  """ONE WINDOW over a connected grid β€” `{fields, rows, total, totalUnfiltered, offset, limit,
69
  limits, identity, recordsMutable}`, whichever connector the rows come from.
70
 
@@ -92,4 +96,4 @@ def connected_table_rows(table_key: str,
92
  return routes_odoo_tables.odoo_table_rows(
93
  table_key, offset=offset, limit=limit, filters=filters, filterConj=filterConj,
94
  sorts=sorts, search=search, related=related, groupBy=groupBy, groupKey=groupKey,
95
- session=session)
 
30
  (`session.is_admin`, an attribute that does not exist, raising above the route's own `try`). So
31
  `verify_meta` both enumerates `main.app.routes` for this path AND calls it.
32
  """
33
+ from fastapi import APIRouter, Body, Depends, Query
34
 
35
  from deps import Session, err, require_session
36
 
 
45
 
46
  # Definitions only: this endpoint neither reads rows nor bypasses the table permission wall.
47
  definition = _defn_or_refuse(session, table_key, defs_only=True)
48
+ from routes_odoo_tables import require_amazon_mirror
49
+ require_amazon_mirror(table_key, session, definition)
50
  identity = user_tables.connector_database_identity(
51
  table_key, rt=session.runtime, definition=definition)
52
  if identity is None:
 
56
 
57
 
58
  @router.get("/connected-tables/{table_key}/rows")
59
+ @router.post("/connected-tables/{table_key}/rows")
60
  def connected_table_rows(table_key: str,
61
  offset: int = Query(default=0, ge=0),
62
  limit: int = Query(default=0),
 
67
  related: str = Query(default=None),
68
  groupBy: str = Query(default=None),
69
  groupKey: str = Query(default=None),
70
+ session: Session = Depends(require_session),
71
+ body: dict = Body(default=None)):
72
  """ONE WINDOW over a connected grid β€” `{fields, rows, total, totalUnfiltered, offset, limit,
73
  limits, identity, recordsMutable}`, whichever connector the rows come from.
74
 
 
96
  return routes_odoo_tables.odoo_table_rows(
97
  table_key, offset=offset, limit=limit, filters=filters, filterConj=filterConj,
98
  sorts=sorts, search=search, related=related, groupBy=groupBy, groupKey=groupKey,
99
+ session=session, body=body)
api/routes_connectors.py CHANGED
@@ -21,14 +21,17 @@ that reported another user's Gmail as "connected" would be both wrong and a smal
21
  from __future__ import annotations
22
 
23
  import os
 
24
  import ssl
25
  import time
26
  import threading
27
  import xmlrpc.client
28
  from concurrent.futures import ThreadPoolExecutor
29
  from datetime import datetime, timezone
 
 
30
 
31
- from fastapi import APIRouter, Body, Depends
32
 
33
  import oauth_connect
34
  from deps import Session, err, require_session, running_main
@@ -83,6 +86,75 @@ from routes_monid import router as monid_router
83
  router = APIRouter(prefix="/api/v1")
84
 
85
  CONNECTOR_DETAIL_DEADLINE_SECONDS = 30
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
86
  SCRAPER_RECIPE_KINDS = frozenset(("instagram", "tiktok", "google_maps"))
87
  SCRAPER_RECIPE_MAX_ROWS = 30
88
  SCRAPER_RUN_HISTORY_LIMIT = 20
@@ -254,9 +326,11 @@ TOKEN_CONNECTORS = [
254
  "hint": "Create a Production app client in the Solution Provider Portal, self authorize "
255
  "it, then paste client_id, client_secret, refresh_token, seller_id and "
256
  "marketplace_id.",
257
- "note": "The credential is verified live against your marketplace participations before "
258
- "it is stored. Bulk reads use the Reports API, so a backfill is one document "
259
- "rather than a page loop."},
 
 
260
  ]
261
 
262
  #: ⭐⭐ WAVE 31 Β· R7 β€” THE NANGO BREADTH REGISTRY. Owner: *"for Nango, i want to explore as many
@@ -374,6 +448,11 @@ _register_probers()
374
  # W48C-T08 / C6 β€” the graph is a reader-scoped DTO over tables that exist in THIS workspace,
375
  # never a provider-shaped example. `CONNECTED_PREFIXES` only classifies the connector family;
376
  # `user_tables.is_connected` below is the second proof that a user-named lookalike did not enter.
 
 
 
 
 
377
  def _graph_refusal(connection_key, reason, status="refused"):
378
  return {"connectionKey": str(connection_key or ""), "status": status,
379
  "reason": str(reason or ""), "tables": [], "relationships": []}
@@ -526,7 +605,12 @@ def relationship_graphs(rows, user, rt, uname, is_admin=False):
526
  graphs = {}
527
  connected = set()
528
  for connection_key, row in row_by_key.items():
529
- if row.get("state") == "connected":
 
 
 
 
 
530
  connected.add(connection_key)
531
  graphs[connection_key] = _graph_refusal(
532
  connection_key,
@@ -906,9 +990,21 @@ def directory(rt, uname, is_admin=False, incomplete=None):
906
  else "keychain_personal"),
907
  # Said once, on the row, rather than in a paragraph somewhere (R13): storing the
908
  # key is real and useful, and it is not the same as the data being on screen.
909
- "note": "Storing the key connects the account; reading its data lands in a later "
910
- "release.",
 
 
 
 
 
911
  })
 
 
 
 
 
 
 
912
 
913
  contribute("token", _tokens,
914
  "the token-connector registry could not be composed",
@@ -2150,19 +2246,21 @@ def _schema_catalog(session):
2150
  return _schema_catalog_rt(session.runtime)
2151
 
2152
 
2153
- def _odoo_raw_models(rt, snapshot=None, connection_id=None):
2154
  """Raw fields from stamped discovery, or the legacy snapshot before discovery exists."""
2155
- if snapshot is None:
2156
- import core.user_tables as user_tables
2157
- snapshot = user_tables.odoo_schema_snapshot(rt)
2158
- legacy = dict((snapshot or {}).get("models") or {}) if connection_id is None else {}
2159
- try:
2160
- import odoo_relational
2161
- discovered = odoo_relational.readable_model_catalog(rt, connection_id)
2162
- except Exception: # noqa: BLE001
2163
- discovered = {}
2164
  if not discovered.get("capturedAt"):
2165
- return legacy
 
 
 
 
 
2166
  # A stamp means this tenant-bound account has spoken. Even a partial stamped set is safer
2167
  # than adding snapshot-only models that the current account never proved it may read.
2168
  return {str(item.get("model") or ""): dict(item.get("fields") or {})
@@ -2170,7 +2268,7 @@ def _odoo_raw_models(rt, snapshot=None, connection_id=None):
2170
  if str(item.get("model") or "")}
2171
 
2172
 
2173
- def _schema_catalog_rt(rt, database_key=None, connection_id=None):
2174
  """The same catalogue, over a RUNTIME rather than a session.
2175
 
2176
  ⚠ SPLIT IN W49-T12 SO THE DETAIL ROUTE CAN REUSE IT RATHER THAN RE-DERIVE IT. The connector
@@ -2179,10 +2277,9 @@ def _schema_catalog_rt(rt, database_key=None, connection_id=None):
2179
  already paid for twice ([[one-question-two-normalizers]]). `_schema_catalog` keeps its
2180
  signature so every existing caller is untouched.
2181
  """
2182
- import core.user_tables as user_tables
2183
  enabled = rt.get(_ODOO_SCHEMA_ENABLEMENTS) or {}
2184
- snapshot = user_tables.odoo_schema_snapshot(rt)
2185
- raw_models = _odoo_raw_models(rt, snapshot, connection_id)
2186
  if raw_models:
2187
  rows = []
2188
  for model, raw_fields in raw_models.items():
@@ -3089,34 +3186,72 @@ def _detail_keychain(rt, uname, is_admin, row):
3089
  "connect": _detail_connect(entry_type, bool(is_admin)) if can_connect else None}
3090
 
3091
 
3092
- def _odoo_catalog_by_model(rt, connection_id=None):
3093
  """`{odoo model: (fields, excluded)}` from the schema catalogue this module already serves.
3094
 
3095
  `excluded` is a DIFF, not a guess: every raw field the snapshot captured that the catalogue did
3096
  not offer. Today that diff is empty, which is the honest answer, and it fills itself the moment
3097
  the catalogue starts filtering instead of needing somebody to remember.
3098
  """
3099
- raw_models = _odoo_raw_models(rt, connection_id=connection_id)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
3100
  offered = {}
3101
- for row in _schema_catalog_rt(rt, connection_id=connection_id):
 
3102
  offered.setdefault(str(row.get("model") or ""), []).append(row)
3103
- out = {}
3104
- for model, raw_fields in raw_models.items():
3105
  rows = offered.get(str(model), [])
3106
  offered_raw = {str(row.get("rawName") or "") for row in rows}
3107
- out[str(model)] = (
3108
  [{"key": str(row.get("key") or ""),
3109
  "label": str(row.get("label") or ""),
3110
  "type": str(row.get("type") or "text"),
3111
  "source_field": str(row.get("rawName") or ""),
3112
- "enabled": bool(row.get("enabled"))}
3113
  for row in rows],
3114
  [{"source_field": str(name),
3115
  "reason": "This source field was captured from Odoo but the field catalogue did not "
3116
  "offer it."}
3117
  for name in sorted(raw_fields or {}) if str(name) not in offered_raw],
3118
  )
3119
- return out
 
 
 
 
 
 
 
 
 
 
 
 
 
 
3120
 
3121
 
3122
  def _spec_field(item, name):
@@ -3495,7 +3630,8 @@ def _scraper_databases(rt, is_admin, uname):
3495
  return _named_databases(rt, is_admin, uname, rows)
3496
 
3497
 
3498
- def _detail_databases(rt, is_admin, connector_key, uname="", connection_id=None):
 
3499
  """C4's `databases` half: what this connector materialises, and the fields behind each one.
3500
 
3501
  β›” ODOO IS THE ONLY INTROSPECTABLE CONNECTOR TODAY and every other card answers `[]`. That is
@@ -3523,19 +3659,34 @@ def _detail_databases(rt, is_admin, connector_key, uname="", connection_id=None)
3523
  reason: what a database is CALLED is a fact about the reader's own workspace. Only the WRITE
3524
  is walled, per row, by `canRename`.
3525
  """
 
 
 
 
 
 
 
 
 
3526
  if str(connector_key) == "scraper":
3527
- return _scraper_databases(rt, is_admin, uname)
3528
  if not connector_directory.is_introspectable(connector_key):
3529
  return []
3530
  spec = _connector_spec_tables(connector_key)
3531
  if str(connector_key) == "amazon_sp":
3532
- return _named_databases(rt, is_admin, uname, _amazon_databases(rt, is_admin, spec))
 
3533
  try:
3534
  import odoo_relational
3535
  namespace = (odoo_relational.connection_namespace(rt, connection_id)[1]
3536
  if connection_id else None)
3537
  choices = odoo_relational.grid_choices(rt, connection_id)
3538
  models = dict(getattr(odoo_relational, "ODOO_TABLE_MODELS", {}) or {})
 
 
 
 
 
3539
  # This read never contacts Odoo. A 499-model exact pass measured 67.87 s after the
3540
  # catalogue was warm, so modal detail must never turn a missing cache into inline work.
3541
  census = odoo_relational.population_census(rt, namespace)
@@ -3547,9 +3698,11 @@ def _detail_databases(rt, is_admin, connector_key, uname="", connection_id=None)
3547
  census_at = str(census.get("capturedAt") or "")
3548
  counts = census.get("counts") if isinstance(census.get("counts"), dict) else {}
3549
  catalog = {}
3550
- if is_admin:
3551
  try:
3552
- catalog = _odoo_catalog_by_model(rt, namespace)
 
 
3553
  except Exception: # noqa: BLE001
3554
  catalog = {}
3555
  enabled_fields = rt.get(_ODOO_SCHEMA_ENABLEMENTS) or {} if is_admin else {}
@@ -3576,9 +3729,9 @@ def _detail_databases(rt, is_admin, connector_key, uname="", connection_id=None)
3576
  key = str(choice.get("key") or "")
3577
  resolved_connection_id = resolve_connection_id(key)
3578
  model = str(choice.get("model") or models.get(key) or "")
3579
- fields, excluded = catalog.get(model, ([], []))
3580
  scoped_fields = []
3581
- for field in fields:
3582
  field_key = _odoo_field_key(rt, key, model, field["source_field"],
3583
  connection_id=resolved_connection_id)
3584
  scoped_fields.append({**field, "key": field_key,
@@ -3612,7 +3765,7 @@ def _detail_databases(rt, is_admin, connector_key, uname="", connection_id=None)
3612
  "fields": scoped_fields,
3613
  "excluded": [dict(item) for item in excluded],
3614
  "sharesFieldsWith": []})
3615
- return _named_databases(rt, is_admin, uname, out)
3616
 
3617
 
3618
  def _amazon_report_name(table):
@@ -3662,7 +3815,17 @@ def _amazon_databases(rt, is_admin, spec=None):
3662
  enabled = enablement.get("databases") or {}
3663
  cat = amazon_schema.catalogue()
3664
  tables = cat.get("tables") or {}
3665
- states = cx.sync_state(rt, getattr(rt, "key", "")) if is_admin else None
 
 
 
 
 
 
 
 
 
 
3666
 
3667
  # Which OTHER declared tables are filled from the SAME report document. Derived from the
3668
  # report map so a table added to `REPORT_TYPES` gets this disclosure for free.
@@ -3833,12 +3996,14 @@ def _detail_introspection(rt, is_admin, row):
3833
  captured = ""
3834
  census = {}
3835
  try:
3836
- import core.user_tables as user_tables
3837
- snapshot = user_tables.odoo_schema_snapshot(rt)
3838
  import odoo_relational
3839
  discovered = odoo_relational.readable_model_catalog(rt)
3840
  census = odoo_relational.population_census(rt)
3841
  has_discovery = bool(discovered.get("capturedAt"))
 
 
 
 
3842
  complete = (discovered.get("complete") is True if has_discovery
3843
  else snapshot.get("complete") is True)
3844
  captured = str((discovered.get("capturedAt") if has_discovery
@@ -4130,13 +4295,16 @@ def _scraper_database_run(rt, uname, is_admin, database_key):
4130
  return {"key": key, "state": state, "counts": counts, "receipt": receipt}
4131
 
4132
 
4133
- def connector_detail(rt, user, uname, is_admin, connector_key):
 
4134
  """The whole C4 detail DTO for one connector, or `None` when this reader has no such card.
4135
 
4136
  Pure over the session for the same reason `directory` is: a gate can drive it without a
4137
  request, so the modal's payload is provable without a browser.
4138
  """
4139
  budget = _ConnectorDetailBudget()
 
 
4140
  rows = directory(rt, uname, bool(is_admin))
4141
  budget.check()
4142
  key = str(connector_key or "")
@@ -4144,15 +4312,23 @@ def connector_detail(rt, user, uname, is_admin, connector_key):
4144
  if str(candidate.get("key") or "") == key), None)
4145
  if row is None:
4146
  return None
4147
- graph = relationship_graphs(rows, user, rt, uname, bool(is_admin)).get(key) or _graph_refusal(
4148
- key, "No actual relationship graph was returned for this connection. A provider template "
4149
- "is not shown.")
 
 
 
4150
  budget.check()
4151
  # ⚠ `uname` REACHES THE DATABASES HALF NOW (W53-T27): `canRename` is `may_open`, which is a
4152
  # question about a PERSON and a table, not about an administrator flag alone.
4153
- databases = _detail_databases(rt, bool(is_admin), key, uname=str(uname or ""))
 
 
 
4154
  budget.check()
4155
  overview = _detail_overview(row, databases, graph)
 
 
4156
  budget.check()
4157
  keychain = _detail_keychain(rt, uname, bool(is_admin), row)
4158
  budget.check()
@@ -4163,6 +4339,9 @@ def connector_detail(rt, user, uname, is_admin, connector_key):
4163
  return _safe_connector_status(key, {"overview": overview,
4164
  "keychain": keychain,
4165
  "databases": databases,
 
 
 
4166
  "syncNow": ({"provider": key, "path": f"/connectors/{key}/sync/now"}
4167
  if is_admin and key in connector_sync.providers() else None),
4168
  "materialization": ({"provider": key, "statusPath": f"/connectors/{key}/jobs",
@@ -4182,6 +4361,36 @@ def connector_detail(rt, user, uname, is_admin, connector_key):
4182
  "relationships": graph})
4183
 
4184
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
4185
  def _set_amazon_database_enablement(database_key, body, session):
4186
  """The testable core of Amazon's tenant-scoped administrator enablement door."""
4187
  if not session.admin:
@@ -4253,22 +4462,100 @@ def scraper_database_run(database_key: str, session: Session = Depends(require_s
4253
  # the habit that keeps it impossible tomorrow. `/connectors/directory` is two segments and cannot
4254
  # collide with this three-segment path at all.
4255
  @router.get("/connectors/{connector_key}/detail")
4256
- def connectors_detail(connector_key: str, session: Session = Depends(require_session)):
 
 
 
4257
  """C4 / R26: everything the connector modal shows, for THIS reader, in one read.
4258
 
4259
  Session-gated exactly like the directory it is opened from: a member may look at what the
4260
  workspace is connected to. What a member does NOT get is the tenant-wide credential door
4261
  (`canConnect` carries the card's own answer) or the administrator-only Odoo field catalogue.
4262
  """
4263
- detail = connector_detail(
4264
- session.runtime, session.user, session.uname,
4265
- bool(session.admin), connector_key)
 
 
 
 
 
 
 
4266
  if detail is None:
4267
  raise err(404, "unknown_connector",
4268
  "this workspace has no connector with that key")
4269
  return detail
4270
 
4271
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
4272
  class _ReplacementConflict(Exception):
4273
  """The addressed keychain row changed between replacement preparation and its store update."""
4274
 
 
21
  from __future__ import annotations
22
 
23
  import os
24
+ import copy
25
  import ssl
26
  import time
27
  import threading
28
  import xmlrpc.client
29
  from concurrent.futures import ThreadPoolExecutor
30
  from datetime import datetime, timezone
31
+ from collections import OrderedDict
32
+ from typing import Annotated, Literal
33
 
34
+ from fastapi import APIRouter, Body, Depends, HTTPException, Query
35
 
36
  import oauth_connect
37
  from deps import Session, err, require_session, running_main
 
86
  router = APIRouter(prefix="/api/v1")
87
 
88
  CONNECTOR_DETAIL_DEADLINE_SECONDS = 30
89
+ _ODOO_DETAIL_CATALOG_CACHE = OrderedDict()
90
+ _ODOO_DETAIL_CATALOG_LOCK = threading.Lock()
91
+ _ODOO_DETAIL_CATALOG_CACHE_SIZE = 16
92
+
93
+
94
+ class _ConnectorDetailReads:
95
+ """Request-local copies only; credentials, grants and enablements never outlive a read.
96
+
97
+ Keep projections separate from whole documents and honour fresh reads. HF projections still
98
+ download the whole document on a cold store; removing that requires a store-layer change.
99
+ """
100
+
101
+ def __init__(self, runtime):
102
+ self.runtime = runtime
103
+ self.documents = {}
104
+ self.projections = {}
105
+
106
+ def get(self, name, fresh=False):
107
+ if fresh:
108
+ return self.runtime.get(name, fresh=True)
109
+ if name not in self.documents:
110
+ self.documents[name] = self.runtime.get(name)
111
+ return copy.deepcopy(self.documents[name])
112
+
113
+ def get_projection(self, name, drop=()):
114
+ key = (name, tuple(sorted(drop)))
115
+ if key not in self.projections:
116
+ self.projections[key] = self.runtime.get_projection(name, drop=drop)
117
+ return copy.deepcopy(self.projections[key])
118
+
119
+ def __getattr__(self, name):
120
+ return getattr(self.runtime, name)
121
+
122
+
123
+ class _DatabasePage:
124
+ """Select before hydrating Odoo fields; the total always describes the unsliced list."""
125
+
126
+ def __init__(self, offset=0, limit=50, database_key=None):
127
+ self.offset, self.limit, self.database_key = offset, limit, database_key
128
+ self.total = 0
129
+ self.creates = []
130
+
131
+ def select(self, rows, *, overview_rows=None):
132
+ self.total = len(rows)
133
+ self.creates = [str(row.get("label") or "")
134
+ for row in (rows if overview_rows is None else overview_rows)
135
+ if row.get("enabled") is True and row.get("label")]
136
+ if self.database_key is not None:
137
+ return [row for row in rows if row.get("key") == self.database_key]
138
+ return rows[self.offset:self.offset + self.limit]
139
+
140
+ def dto(self):
141
+ next_offset = self.offset + self.limit
142
+ return {"offset": self.offset, "limit": self.limit, "total": self.total,
143
+ "next": next_offset if next_offset < self.total else None}
144
+
145
+
146
+ def _detail_page(databases_offset, databases_limit, fields):
147
+ if fields not in ("summary", "full"):
148
+ raise err(422, "invalid_database_fields", "Fields must be summary or full.")
149
+ if databases_offset is None and databases_limit is None and fields == "full":
150
+ return None
151
+ offset = 0 if databases_offset is None else databases_offset
152
+ limit = 50 if databases_limit is None else databases_limit
153
+ if (type(offset) is not int or offset < 0 or type(limit) is not int or limit < 1):
154
+ raise err(422, "invalid_database_page", "Use a nonnegative offset and a positive limit.")
155
+ return _DatabasePage(offset, limit)
156
+
157
+
158
  SCRAPER_RECIPE_KINDS = frozenset(("instagram", "tiktok", "google_maps"))
159
  SCRAPER_RECIPE_MAX_ROWS = 30
160
  SCRAPER_RUN_HISTORY_LIMIT = 20
 
326
  "hint": "Create a Production app client in the Solution Provider Portal, self authorize "
327
  "it, then paste client_id, client_secret, refresh_token, seller_id and "
328
  "marketplace_id.",
329
+ # S66-T27: storing is offline (`keychain._test_amazon_entry` probes only on an explicit
330
+ # Test), so this no longer claims the key was verified before it was stored.
331
+ "note": "Choose Test in the Keychain tab to verify the credential live against your "
332
+ "marketplace participations. Bulk reads use the Reports API, so a backfill is one "
333
+ "document rather than a page loop."},
334
  ]
335
 
336
  #: ⭐⭐ WAVE 31 Β· R7 β€” THE NANGO BREADTH REGISTRY. Owner: *"for Nango, i want to explore as many
 
448
  # W48C-T08 / C6 β€” the graph is a reader-scoped DTO over tables that exist in THIS workspace,
449
  # never a provider-shaped example. `CONNECTED_PREFIXES` only classifies the connector family;
450
  # `user_tables.is_connected` below is the second proof that a user-named lookalike did not enter.
451
+ #: The directory states that mean "this workspace holds this connection": connected, stored but
452
+ #: asking to be reconnected, or stored past its rotation deadline (`ConnectorsPage`'s `held`).
453
+ _GRAPH_HELD_STATES = frozenset(("connected", "reconnect", "expired"))
454
+
455
+
456
  def _graph_refusal(connection_key, reason, status="refused"):
457
  return {"connectionKey": str(connection_key or ""), "status": status,
458
  "reason": str(reason or ""), "tables": [], "relationships": []}
 
605
  graphs = {}
606
  connected = set()
607
  for connection_key, row in row_by_key.items():
608
+ # ⭐ S67-T31 / D-718 β€” A HELD CONNECTION HAS A GRAPH. Staging's Amazon card reads
609
+ # `reconnect` (its stored key has no passing test on record) while 38 `ut_amzn_*` tables
610
+ # are materialised, and the old `== "connected"` test refused the graph as "Connect this
611
+ # provider" over a connection that is plainly there. The nodes are still only tables that
612
+ # exist and that this reader may open, so admitting the held states draws no template.
613
+ if row.get("state") in _GRAPH_HELD_STATES:
614
  connected.add(connection_key)
615
  graphs[connection_key] = _graph_refusal(
616
  connection_key,
 
990
  else "keychain_personal"),
991
  # Said once, on the row, rather than in a paragraph somewhere (R13): storing the
992
  # key is real and useful, and it is not the same as the data being on screen.
993
+ # ⭐ S66-T27 / S67-T31 β€” EXCEPT AMAZON, whose data DOES land (38 `ut_amzn_*`
994
+ # tables on staging), so the generic sentence told an administrator the opposite
995
+ # of the truth. Amazon carries its own authored note.
996
+ # S67 lane D: so does Meta Ads, whose authored note this line used to overwrite
997
+ # with the generic one; every card now shows its own (Stripe's and Shopify's own
998
+ # note IS the generic sentence, so theirs is unchanged).
999
+ "note": t["note"],
1000
  })
1001
+ # S66-T27 β€” WHAT `reconnect` MEANS, said on the card. On Amazon it is "a key is stored
1002
+ # but no passing Test is on record for it" (`keychain.entry_verified`), measured on
1003
+ # staging's admin view with the data still syncing. The bare word read as a broken
1004
+ # connection with no way to tell what to do.
1005
+ if rows[-1]["state"] == "reconnect" and t["entry"] == "amazon_sp":
1006
+ rows[-1]["needs"] = ("Amazon access for the stored key is not confirmed yet. "
1007
+ "Choose Test in the Keychain tab, or replace the key.")
1008
 
1009
  contribute("token", _tokens,
1010
  "the token-connector registry could not be composed",
 
2246
  return _schema_catalog_rt(session.runtime)
2247
 
2248
 
2249
+ def _odoo_raw_models(rt, snapshot=None, connection_id=None, *, discovered=None):
2250
  """Raw fields from stamped discovery, or the legacy snapshot before discovery exists."""
2251
+ if discovered is None:
2252
+ try:
2253
+ import odoo_relational
2254
+ discovered = odoo_relational.readable_model_catalog(rt, connection_id)
2255
+ except Exception: # noqa: BLE001
2256
+ discovered = {}
 
 
 
2257
  if not discovered.get("capturedAt"):
2258
+ if connection_id is not None:
2259
+ return {}
2260
+ if snapshot is None:
2261
+ import core.user_tables as user_tables
2262
+ snapshot = user_tables.odoo_schema_snapshot(rt)
2263
+ return dict((snapshot or {}).get("models") or {})
2264
  # A stamp means this tenant-bound account has spoken. Even a partial stamped set is safer
2265
  # than adding snapshot-only models that the current account never proved it may read.
2266
  return {str(item.get("model") or ""): dict(item.get("fields") or {})
 
2268
  if str(item.get("model") or "")}
2269
 
2270
 
2271
+ def _schema_catalog_rt(rt, database_key=None, connection_id=None, *, raw_models=None):
2272
  """The same catalogue, over a RUNTIME rather than a session.
2273
 
2274
  ⚠ SPLIT IN W49-T12 SO THE DETAIL ROUTE CAN REUSE IT RATHER THAN RE-DERIVE IT. The connector
 
2277
  already paid for twice ([[one-question-two-normalizers]]). `_schema_catalog` keeps its
2278
  signature so every existing caller is untouched.
2279
  """
 
2280
  enabled = rt.get(_ODOO_SCHEMA_ENABLEMENTS) or {}
2281
+ if raw_models is None:
2282
+ raw_models = _odoo_raw_models(rt, connection_id=connection_id)
2283
  if raw_models:
2284
  rows = []
2285
  for model, raw_fields in raw_models.items():
 
3186
  "connect": _detail_connect(entry_type, bool(is_admin)) if can_connect else None}
3187
 
3188
 
3189
+ def _odoo_catalog_by_model(rt, connection_id=None, *, models=None):
3190
  """`{odoo model: (fields, excluded)}` from the schema catalogue this module already serves.
3191
 
3192
  `excluded` is a DIFF, not a guess: every raw field the snapshot captured that the catalogue did
3193
  not offer. Today that diff is empty, which is the honest answer, and it fills itself the moment
3194
  the catalogue starts filtering instead of needing somebody to remember.
3195
  """
3196
+ import odoo_relational
3197
+ discovered = odoo_relational.readable_model_catalog(rt, connection_id)
3198
+ marker = ("discovered", discovered.get("version"), discovered.get("capturedAt"),
3199
+ discovered.get("complete"))
3200
+ snapshot = None
3201
+ if not discovered.get("capturedAt"):
3202
+ import core.user_tables as user_tables
3203
+ snapshot = user_tables.odoo_schema_snapshot(rt) if connection_id is None else {}
3204
+ marker = ("snapshot", snapshot.get("version"), snapshot.get("capturedAt"),
3205
+ snapshot.get("complete"))
3206
+ owner = rt.runtime if isinstance(rt, _ConnectorDetailReads) else rt
3207
+ cache_key = (id(owner), str(connection_id or ""), marker)
3208
+ wanted = None if models is None else set(models)
3209
+ # Cache only normalized metadata, never permissions or field switch values. Keep the owner
3210
+ # alive with the entry so Python cannot reuse a runtime id for another tenant's catalogue.
3211
+ with _ODOO_DETAIL_CATALOG_LOCK:
3212
+ entry = _ODOO_DETAIL_CATALOG_CACHE.get(cache_key) if marker[2] else None
3213
+ cached = dict(entry[1]) if entry else {}
3214
+ if entry:
3215
+ _ODOO_DETAIL_CATALOG_CACHE.move_to_end(cache_key)
3216
+ raw_models = _odoo_raw_models(rt, snapshot, connection_id, discovered=discovered)
3217
+ if wanted is not None:
3218
+ raw_models = {model: fields for model, fields in raw_models.items() if model in wanted}
3219
+ missing = {model: fields for model, fields in raw_models.items() if model not in cached}
3220
  offered = {}
3221
+ for row in (_schema_catalog_rt(rt, connection_id=connection_id, raw_models=missing)
3222
+ if missing else ()):
3223
  offered.setdefault(str(row.get("model") or ""), []).append(row)
3224
+ built = {}
3225
+ for model, raw_fields in missing.items():
3226
  rows = offered.get(str(model), [])
3227
  offered_raw = {str(row.get("rawName") or "") for row in rows}
3228
+ built[str(model)] = (
3229
  [{"key": str(row.get("key") or ""),
3230
  "label": str(row.get("label") or ""),
3231
  "type": str(row.get("type") or "text"),
3232
  "source_field": str(row.get("rawName") or ""),
3233
+ "enabled": False}
3234
  for row in rows],
3235
  [{"source_field": str(name),
3236
  "reason": "This source field was captured from Odoo but the field catalogue did not "
3237
  "offer it."}
3238
  for name in sorted(raw_fields or {}) if str(name) not in offered_raw],
3239
  )
3240
+ if marker[2]:
3241
+ with _ODOO_DETAIL_CATALOG_LOCK:
3242
+ entry = _ODOO_DETAIL_CATALOG_CACHE.get(cache_key)
3243
+ merged = {**(entry[1] if entry else {}), **built}
3244
+ _ODOO_DETAIL_CATALOG_CACHE[cache_key] = (owner, merged)
3245
+ _ODOO_DETAIL_CATALOG_CACHE.move_to_end(cache_key)
3246
+ while len(_ODOO_DETAIL_CATALOG_CACHE) > _ODOO_DETAIL_CATALOG_CACHE_SIZE:
3247
+ _ODOO_DETAIL_CATALOG_CACHE.popitem(last=False)
3248
+ cached.update(built)
3249
+ enabled = rt.get(_ODOO_SCHEMA_ENABLEMENTS) or {}
3250
+ return {model: ([{**field, "enabled": connection_id is None
3251
+ and enabled.get(field["key"]) is True}
3252
+ for field in cached[model][0]],
3253
+ [dict(item) for item in cached[model][1]])
3254
+ for model in raw_models}
3255
 
3256
 
3257
  def _spec_field(item, name):
 
3630
  return _named_databases(rt, is_admin, uname, rows)
3631
 
3632
 
3633
+ def _detail_databases(rt, is_admin, connector_key, uname="", connection_id=None, *,
3634
+ page=None, fields="full"):
3635
  """C4's `databases` half: what this connector materialises, and the fields behind each one.
3636
 
3637
  β›” ODOO IS THE ONLY INTROSPECTABLE CONNECTOR TODAY and every other card answers `[]`. That is
 
3659
  reason: what a database is CALLED is a fact about the reader's own workspace. Only the WRITE
3660
  is walled, per row, by `canRename`.
3661
  """
3662
+ def select(rows):
3663
+ return page.select(rows) if page is not None else rows
3664
+
3665
+ def disclose(rows):
3666
+ if fields == "summary":
3667
+ return [{**row, "fields": [], "excluded": [], "fieldsDeferred": True}
3668
+ for row in rows]
3669
+ return rows
3670
+
3671
  if str(connector_key) == "scraper":
3672
+ return disclose(select(_scraper_databases(rt, is_admin, uname)))
3673
  if not connector_directory.is_introspectable(connector_key):
3674
  return []
3675
  spec = _connector_spec_tables(connector_key)
3676
  if str(connector_key) == "amazon_sp":
3677
+ return disclose(select(_named_databases(
3678
+ rt, is_admin, uname, _amazon_databases(rt, is_admin, spec))))
3679
  try:
3680
  import odoo_relational
3681
  namespace = (odoo_relational.connection_namespace(rt, connection_id)[1]
3682
  if connection_id else None)
3683
  choices = odoo_relational.grid_choices(rt, connection_id)
3684
  models = dict(getattr(odoo_relational, "ODOO_TABLE_MODELS", {}) or {})
3685
+ if page is not None:
3686
+ choices = page.select(choices, overview_rows=schema_overlay.merge_database_labels([
3687
+ {**choice, "label": str((spec.get(choice.get("baseKey") or choice["key"])
3688
+ or {}).get("label") or choice.get("label") or choice["key"])}
3689
+ for choice in choices], _database_name_overrides(rt)))
3690
  # This read never contacts Odoo. A 499-model exact pass measured 67.87 s after the
3691
  # catalogue was warm, so modal detail must never turn a missing cache into inline work.
3692
  census = odoo_relational.population_census(rt, namespace)
 
3698
  census_at = str(census.get("capturedAt") or "")
3699
  counts = census.get("counts") if isinstance(census.get("counts"), dict) else {}
3700
  catalog = {}
3701
+ if is_admin and fields == "full" and choices:
3702
  try:
3703
+ catalog = _odoo_catalog_by_model(rt, namespace, models={
3704
+ str(choice.get("model") or models.get(choice.get("key")) or "")
3705
+ for choice in choices})
3706
  except Exception: # noqa: BLE001
3707
  catalog = {}
3708
  enabled_fields = rt.get(_ODOO_SCHEMA_ENABLEMENTS) or {} if is_admin else {}
 
3729
  key = str(choice.get("key") or "")
3730
  resolved_connection_id = resolve_connection_id(key)
3731
  model = str(choice.get("model") or models.get(key) or "")
3732
+ field_rows, excluded = catalog.get(model, ([], []))
3733
  scoped_fields = []
3734
+ for field in field_rows:
3735
  field_key = _odoo_field_key(rt, key, model, field["source_field"],
3736
  connection_id=resolved_connection_id)
3737
  scoped_fields.append({**field, "key": field_key,
 
3765
  "fields": scoped_fields,
3766
  "excluded": [dict(item) for item in excluded],
3767
  "sharesFieldsWith": []})
3768
+ return disclose(_named_databases(rt, is_admin, uname, out))
3769
 
3770
 
3771
  def _amazon_report_name(table):
 
3815
  enabled = enablement.get("databases") or {}
3816
  cat = amazon_schema.catalogue()
3817
  tables = cat.get("tables") or {}
3818
+ # ⭐ S67-T31 β€” A STATUS THE GUARD REFUSES COSTS ITS OWN ROW, NEVER THE MODAL. `sync_state`
3819
+ # trims an over-long cause itself; a credential shape still refuses, and the refusal carries
3820
+ # the guarded rows that passed, so only the refused tables read "Sync status is unavailable
3821
+ # for this database." (a missing status is exactly what `_amazon_database_sync_health`
3822
+ # degrades). Any other failure reading status degrades every row, never the whole page.
3823
+ try:
3824
+ states = cx.sync_state(rt, getattr(rt, "key", "")) if is_admin else None
3825
+ except cx.AmazonStatusSafetyError as refused:
3826
+ states = dict(getattr(refused, "safe_status", None) or {})
3827
+ except Exception: # noqa: BLE001
3828
+ states = {}
3829
 
3830
  # Which OTHER declared tables are filled from the SAME report document. Derived from the
3831
  # report map so a table added to `REPORT_TYPES` gets this disclosure for free.
 
3996
  captured = ""
3997
  census = {}
3998
  try:
 
 
3999
  import odoo_relational
4000
  discovered = odoo_relational.readable_model_catalog(rt)
4001
  census = odoo_relational.population_census(rt)
4002
  has_discovery = bool(discovered.get("capturedAt"))
4003
+ snapshot = {}
4004
+ if not has_discovery:
4005
+ import core.user_tables as user_tables
4006
+ snapshot = user_tables.odoo_schema_snapshot(rt)
4007
  complete = (discovered.get("complete") is True if has_discovery
4008
  else snapshot.get("complete") is True)
4009
  captured = str((discovered.get("capturedAt") if has_discovery
 
4295
  return {"key": key, "state": state, "counts": counts, "receipt": receipt}
4296
 
4297
 
4298
+ def connector_detail(rt, user, uname, is_admin, connector_key, *, databases_offset=None,
4299
+ databases_limit=None, fields="full"):
4300
  """The whole C4 detail DTO for one connector, or `None` when this reader has no such card.
4301
 
4302
  Pure over the session for the same reason `directory` is: a gate can drive it without a
4303
  request, so the modal's payload is provable without a browser.
4304
  """
4305
  budget = _ConnectorDetailBudget()
4306
+ page = _detail_page(databases_offset, databases_limit, fields)
4307
+ rt = _ConnectorDetailReads(rt)
4308
  rows = directory(rt, uname, bool(is_admin))
4309
  budget.check()
4310
  key = str(connector_key or "")
 
4312
  if str(candidate.get("key") or "") == key), None)
4313
  if row is None:
4314
  return None
4315
+ # Summary pages defer the independent graph as well, otherwise an all-table field graph
4316
+ # would recreate the unbounded payload beside the paged database list.
4317
+ graph = (None if fields == "summary" else
4318
+ relationship_graphs([row], user, rt, uname, bool(is_admin)).get(key)
4319
+ or _graph_refusal(key, "No actual relationship graph was returned for this connection. "
4320
+ "A provider template is not shown."))
4321
  budget.check()
4322
  # ⚠ `uname` REACHES THE DATABASES HALF NOW (W53-T27): `canRename` is `may_open`, which is a
4323
  # question about a PERSON and a table, not about an administrator flag alone.
4324
+ databases = _detail_databases(rt, bool(is_admin), key, uname=str(uname or ""),
4325
+ page=page, fields=fields)
4326
+ if key == "amazon_sp":
4327
+ databases = _screen_amazon_database_rows(databases)
4328
  budget.check()
4329
  overview = _detail_overview(row, databases, graph)
4330
+ if page is not None and page.total:
4331
+ overview["creates"] = page.creates
4332
  budget.check()
4333
  keychain = _detail_keychain(rt, uname, bool(is_admin), row)
4334
  budget.check()
 
4339
  return _safe_connector_status(key, {"overview": overview,
4340
  "keychain": keychain,
4341
  "databases": databases,
4342
+ **({"databasesPage": page.dto()} if page is not None else {}),
4343
+ **({"relationshipsPath": f"/connectors/{key}/relationships"}
4344
+ if fields == "summary" else {}),
4345
  "syncNow": ({"provider": key, "path": f"/connectors/{key}/sync/now"}
4346
  if is_admin and key in connector_sync.providers() else None),
4347
  "materialization": ({"provider": key, "statusPath": f"/connectors/{key}/jobs",
 
4361
  "relationships": graph})
4362
 
4363
 
4364
+ def _screen_amazon_database_rows(rows):
4365
+ """S67-T31: each Amazon database row answers the status guard ON ITS OWN.
4366
+
4367
+ `_safe_connector_status` still guards the whole detail below, and it used to be the first and
4368
+ only judge, so one oversized scalar in one row aborted every row, the Overview and the
4369
+ Keychain with a bare 500. Here an over-long string is trimmed, and a row that still fails (a
4370
+ credential shape anywhere in it) keeps only its key, its shipped name and its switch, with
4371
+ "Sync status is unavailable for this database." as its status. The refused value is dropped,
4372
+ never echoed.
4373
+ """
4374
+ import connectors_amazon as cx # noqa: PLC0415 - provider owns the guard
4375
+ listed = list(rows or [])
4376
+ safe, _refused = cx.screen_status_rows(dict(enumerate(listed)))
4377
+ out = []
4378
+ for index, row in enumerate(listed):
4379
+ if index in safe:
4380
+ out.append(safe[index])
4381
+ continue
4382
+ key = str((row or {}).get("key") or "")
4383
+ enabled = (row or {}).get("enabled") is True
4384
+ label = LABELS_AMAZON.get(key) or key
4385
+ out.append({"key": key, "label": label, "defaultLabel": label, "native": "",
4386
+ "enabled": enabled,
4387
+ **({"sync": None} if "sync" in (row or {}) else {}),
4388
+ "syncHealth": _amazon_database_sync_health(None, enabled),
4389
+ "fields": [], "excluded": [], "sharesFieldsWith": [],
4390
+ **({"fieldsDeferred": True} if (row or {}).get("fieldsDeferred") else {})})
4391
+ return out
4392
+
4393
+
4394
  def _set_amazon_database_enablement(database_key, body, session):
4395
  """The testable core of Amazon's tenant-scoped administrator enablement door."""
4396
  if not session.admin:
 
4462
  # the habit that keeps it impossible tomorrow. `/connectors/directory` is two segments and cannot
4463
  # collide with this three-segment path at all.
4464
  @router.get("/connectors/{connector_key}/detail")
4465
+ def connectors_detail(connector_key: str, session: Session = Depends(require_session),
4466
+ databases_offset: Annotated[int | None, Query(ge=0)] = None,
4467
+ databases_limit: Annotated[int | None, Query(ge=1)] = None,
4468
+ fields: Literal["summary", "full"] = "full"):
4469
  """C4 / R26: everything the connector modal shows, for THIS reader, in one read.
4470
 
4471
  Session-gated exactly like the directory it is opened from: a member may look at what the
4472
  workspace is connected to. What a member does NOT get is the tenant-wide credential door
4473
  (`canConnect` carries the card's own answer) or the administrator-only Odoo field catalogue.
4474
  """
4475
+ try:
4476
+ detail = connector_detail(
4477
+ session.runtime, session.user, session.uname,
4478
+ bool(session.admin), connector_key, databases_offset=databases_offset,
4479
+ databases_limit=databases_limit, fields=fields)
4480
+ except HTTPException:
4481
+ raise
4482
+ except Exception as exc: # noqa: BLE001
4483
+ raise _plain_read_failure(exc, "connector_detail_failed",
4484
+ "The connector details could not be read.")
4485
  if detail is None:
4486
  raise err(404, "unknown_connector",
4487
  "this workspace has no connector with that key")
4488
  return detail
4489
 
4490
 
4491
+ def _plain_read_failure(exc, code, sentence):
4492
+ """S67-T31: an unexpected failure on a connector read answers the app's JSON error shape.
4493
+
4494
+ The owner's modal read "The server answered 500." because a guard refusal escaped as a bare
4495
+ `Internal Server Error` with no body. Every read the modal makes now answers
4496
+ `{error: {code, message}}` with a plain cause instead; the traceback goes to the server log,
4497
+ and no exception text (which can echo a payload) reaches the wire.
4498
+ """
4499
+ import logging # noqa: PLC0415
4500
+ logging.getLogger(__name__).exception("connector read failed: %s", code)
4501
+ name = type(exc).__name__
4502
+ if name in ("StoreUnavailable", "PoolTimeout"):
4503
+ return err(503, "store_unavailable",
4504
+ "The workspace store did not answer in time, so nothing could be read.")
4505
+ try:
4506
+ import connectors_amazon as cx # noqa: PLC0415
4507
+ if isinstance(exc, cx.AmazonStatusSafetyError):
4508
+ return err(500, "connector_status_withheld",
4509
+ "This connector's status could not be shown safely, so it was withheld.")
4510
+ except Exception: # noqa: BLE001
4511
+ pass
4512
+ return err(500, code, sentence)
4513
+
4514
+
4515
+ def _detail_row(rt, uname, is_admin, connector_key):
4516
+ row = next((row for row in directory(rt, uname, is_admin)
4517
+ if row.get("key") == connector_key), None)
4518
+ if row is None:
4519
+ raise err(404, "unknown_connector", "this workspace has no connector with that key")
4520
+ return row
4521
+
4522
+
4523
+ @router.get("/connectors/{connector_key}/databases/{database_key}/fields")
4524
+ def connector_database_fields(connector_key: str, database_key: str,
4525
+ session: Session = Depends(require_session)):
4526
+ """The same field disclosure and database access as detail, for one database only."""
4527
+ rt = _ConnectorDetailReads(session.runtime)
4528
+ try:
4529
+ _detail_row(rt, session.uname, bool(session.admin), connector_key)
4530
+ rows = _detail_databases(rt, bool(session.admin), connector_key, uname=session.uname,
4531
+ page=_DatabasePage(database_key=database_key))
4532
+ if not rows:
4533
+ raise err(404, "unknown_database",
4534
+ "This connector has no visible database with that key.")
4535
+ return _safe_connector_status(connector_key, {
4536
+ "key": database_key, "fields": rows[0]["fields"], "excluded": rows[0]["excluded"]})
4537
+ except HTTPException:
4538
+ raise
4539
+ except Exception as exc: # noqa: BLE001
4540
+ raise _plain_read_failure(exc, "connector_fields_failed",
4541
+ "The fields of this database could not be read.")
4542
+
4543
+
4544
+ @router.get("/connectors/{connector_key}/relationships")
4545
+ def connector_relationships(connector_key: str, session: Session = Depends(require_session)):
4546
+ """Load the permitted graph when the summary client's Relationships tab opens."""
4547
+ rt = _ConnectorDetailReads(session.runtime)
4548
+ try:
4549
+ row = _detail_row(rt, session.uname, bool(session.admin), connector_key)
4550
+ return _safe_connector_status(connector_key, relationship_graphs(
4551
+ [row], session.user, rt, session.uname, bool(session.admin))[connector_key])
4552
+ except HTTPException:
4553
+ raise
4554
+ except Exception as exc: # noqa: BLE001
4555
+ raise _plain_read_failure(exc, "connector_relationships_failed",
4556
+ "The relationship graph could not be read.")
4557
+
4558
+
4559
  class _ReplacementConflict(Exception):
4560
  """The addressed keychain row changed between replacement preparation and its store update."""
4561
 
api/routes_external.py CHANGED
@@ -19,7 +19,7 @@ from starlette.concurrency import run_in_threadpool
19
  import deps
20
  import aios_session
21
  from deps import Session, err, require_session
22
- from core import audit, external_access, external_oauth, external_tools, sheets
23
  from harness import runtime
24
 
25
 
@@ -621,14 +621,16 @@ def require_external_session(
621
  def require_external_analyst(
622
  session: Session = Depends(require_external_session),
623
  ) -> Session:
624
- """Apply the same Analyst grant wall used by the in-app Assistant and Query."""
625
- session.require("analyst")
 
 
626
  return session
627
 
628
 
629
  def _can_use_analyst(session: Session) -> bool:
630
  try:
631
- session.require("analyst")
632
  except HTTPException as exc:
633
  if exc.status_code == 403:
634
  return False
@@ -726,8 +728,12 @@ def _resolve_external_scope(
726
  if include_schema:
727
  if row["answerable"] or strict_schema:
728
  try:
 
 
 
 
729
  scoped = deps.assistant_read_scope(
730
- session, database, fields=None, filters=None,
731
  )
732
  except HTTPException as exc:
733
  # HOTFIX 2026-09-24: ONE refusing source (measured: a 996,203-row mirror
@@ -999,7 +1005,7 @@ def _portable_skill_document(session: Session | None, base_url: str) -> str:
999
  f" '{surface_url}/mcp'",
1000
  "```",
1001
  "",
1002
- ("Sheet tools require analyst and sheets grants. Geometry patches carry the revision "
1003
  "read by the server; pass rev to also protect the revision you read. Cell writes "
1004
  "are previewed before confirmation. A partially refused patch changes no cells."),
1005
  "",
@@ -1043,7 +1049,7 @@ def _portable_skill_document(session: Session | None, base_url: str) -> str:
1043
  @router.post("/keys", status_code=201)
1044
  def create_key(body=Body(default=None), session: Session = Depends(require_session)):
1045
  """Mint a credential for the signed-in human. Bearer auth is not accepted here."""
1046
- session.require("analyst")
1047
  label = body.get("label") if isinstance(body, dict) else None
1048
  access = body.get("access") if isinstance(body, dict) else None
1049
  databases = body.get("databases") if isinstance(body, dict) else None
@@ -1182,7 +1188,7 @@ def oauth_metadata(request: Request):
1182
  @router.post("/oauth/clients", status_code=201)
1183
  def create_oauth_client(body=Body(default=None), session: Session = Depends(require_session)):
1184
  """Register one user-owned public client; PKCE replaces a shared client secret."""
1185
- session.require("analyst")
1186
  body = body if isinstance(body, dict) else {}
1187
  try:
1188
  client = external_oauth.register_client(
@@ -1470,16 +1476,18 @@ def oauth_authorize(
1470
  if redirect_uri not in meta["r"]:
1471
  return _refusal_page("The app sent a callback it did not register. Start the "
1472
  "connection again from the app.")
 
 
 
1473
  try:
1474
- session.require("analyst")
1475
  _require_oauth_resource(request, resource)
1476
  if response_type != "code":
1477
  raise external_oauth.ExternalOAuthError("oauth_response_type_invalid")
1478
  external_oauth._normalise_state(state)
1479
  external_oauth._normalise_challenge(code_challenge, code_challenge_method)
1480
  except HTTPException:
1481
- return _refusal_page("Your Loopable role cannot connect AI apps. Ask an "
1482
- "administrator.", 403)
1483
  except external_access.ExternalAccessError:
1484
  return _refusal_page("The app sent an incomplete sign-in request. Start the "
1485
  "connection again from the app.")
@@ -1490,7 +1498,9 @@ def oauth_authorize(
1490
  return _consent_page(request, session, meta, fields)
1491
  if session is None:
1492
  raise err(401, "no_session", "sign in to continue")
1493
- session.require("analyst")
 
 
1494
  try:
1495
  audience = _require_oauth_resource(request, resource)
1496
  result = external_oauth.authorize(
@@ -1532,11 +1542,9 @@ async def oauth_consent(request: Request, session: Session = Depends(require_ses
1532
  databases = chosen
1533
  else:
1534
  databases = external_access.ALL_DATABASES
1535
- try:
1536
- session.require("analyst")
1537
- except HTTPException:
1538
- return _refusal_page("Your Loopable role cannot connect AI apps. Ask an administrator.",
1539
- 403)
1540
  try:
1541
  audience = _require_oauth_resource(request, fields.get("resource"))
1542
  client = external_oauth.adopt_dynamic_client(
@@ -1719,7 +1727,7 @@ def external_setup(request: Request):
1719
  @router.get("/skill.md")
1720
  def portable_skill(request: Request, session: Session = Depends(require_session)):
1721
  """Download current governed external read and build instructions."""
1722
- session.require("analyst")
1723
  document = _portable_skill_document(session, str(request.base_url))
1724
  return Response(
1725
  content=document,
@@ -1750,7 +1758,7 @@ def whoami(session: Session = Depends(require_external_analyst)):
1750
  @router.get("/scope/databases")
1751
  def scope_databases(session: Session = Depends(require_session)):
1752
  """List the databases this signed-in maker could put inside a new credential's scope."""
1753
- session.require("analyst")
1754
  return {"databases": [
1755
  {key: row[key] for key in ("database", "label", "permitted", "answerable")}
1756
  for row in _resolve_external_scope(session)
@@ -1817,8 +1825,11 @@ def _query_database(session: Session, database: str, payload: dict):
1817
 
1818
  # This is the same permissioned record reader the in-app Assistant uses. It resolves the
1819
  # database, field and row walls on every call before this adapter sorts, caps, or aggregates.
 
 
1820
  scoped = deps.assistant_read_scope(
1821
- session, database, fields=None, filters=payload.get("filters"))
 
1822
  visible_fields = list(scoped.get("fields") or [])
1823
  visible_keys = {
1824
  field.get("key") for field in visible_fields
@@ -1847,6 +1858,104 @@ def _query_database(session: Session, database: str, payload: dict):
1847
  sorts.append({"field": rule["field"], "direction": rule["direction"]})
1848
  aggregates = _normalise_aggregates(payload.get("aggregates"), visible_keys)
1849
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1850
  rows = list(scoped.get("records") or [])
1851
  inspected_keys = (set(selected_keys)
1852
  | {rule["field"] for rule in sorts}
@@ -1897,7 +2006,8 @@ def _query_database(session: Session, database: str, payload: dict):
1897
  "filters": copy.deepcopy(scoped.get("filters")),
1898
  "sorts": copy.deepcopy(sorts),
1899
  "records": records,
1900
- "aggregates": _aggregate_records(rows, aggregates),
 
1901
  "page": {"limit": limit, "returned": len(records), "offset": offset,
1902
  "has_more": next_cursor is not None, "next_cursor": next_cursor},
1903
  "caps": {"max_page_size": _QUERY_MAX_LIMIT, "max_sorts": _QUERY_MAX_SORTS,
@@ -2730,7 +2840,7 @@ def _tool_context(session: Session, *, preview=False) -> external_tools.ToolCont
2730
  def _strong_confirmation_required(name, args):
2731
  if name == "database_object":
2732
  return args.get("action") in {"record_delete", "field_create", "field_update", "field_delete"}
2733
- if name in {"view_delete", "app_delete"}:
2734
  return True
2735
  if name == "sheet_write_cells":
2736
  return any(isinstance(cell, dict) and cell.get("kind") == "empty"
@@ -2913,7 +3023,7 @@ def _confirm_change(session, args):
2913
  "Enter the exact stronger confirmation phrase shown in the preview.")
2914
  _confirmation_already_used(session, payload)
2915
 
2916
- # Freshly resolve the account, Analyst grant, credential revocation/label, credential scope,
2917
  # database walls and writer semantics before claiming the nonce or touching customer state.
2918
  fresh_user = external_access.resolve_active_user(session.uname, session.tenant)
2919
  fresh_session = Session(session.tenant, fresh_user, copy.deepcopy(session.claims), session.runtime)
@@ -3100,7 +3210,7 @@ def _handle_mcp(req, session: Session, base_url: str = ""):
3100
  return protocol_error(rid, -32602, "MCP params must be an object.")
3101
  name = params.get("name")
3102
  try:
3103
- # Same Analyst wall as the read doors, inside the dispatcher so a grant refusal
3104
  # remains MCP content. Only the unchanged bearer dependency produces an HTTP 401.
3105
  require_external_analyst(session)
3106
  if method == "initialize":
@@ -3163,7 +3273,7 @@ def _handle_mcp(req, session: Session, base_url: str = ""):
3163
 
3164
  @router.post("/mcp")
3165
  async def external_mcp(request: Request, session: Session = Depends(require_external_session)):
3166
- """Remote MCP without a browser session, with the read doors' maker and Analyst wall."""
3167
  try:
3168
  req = await request.json()
3169
  except (ValueError, UnicodeError):
@@ -3177,6 +3287,7 @@ async def external_mcp(request: Request, session: Session = Depends(require_exte
3177
  # A new write in the registry without a matching literal raises at import, before serving.
3178
  _WRITE_TOOLS = frozenset({
3179
  "sheet_create_workbook", "sheet_add_tab", "sheet_write_cells", "sheet_set_binding",
 
3180
  "sheet_set_styles", "sheet_set_dimensions", "sheet_set_validation", "sheet_set_freeze",
3181
  "sheet_add_rule", "sheet_set_chart", "sheet_structure",
3182
  "sheet_save_template",
 
19
  import deps
20
  import aios_session
21
  from deps import Session, err, require_session
22
+ from core import audit, external_access, external_oauth, external_tools, perm_scope, sheets
23
  from harness import runtime
24
 
25
 
 
621
  def require_external_analyst(
622
  session: Session = Depends(require_external_session),
623
  ) -> Session:
624
+ """Check the separate AI apps grant on every request, including existing bearer tokens."""
625
+ reason = perm_scope.ai_apps_refusal(session.user, session.runtime)
626
+ if reason:
627
+ raise err(403, "forbidden", reason)
628
  return session
629
 
630
 
631
  def _can_use_analyst(session: Session) -> bool:
632
  try:
633
+ require_external_analyst(session)
634
  except HTTPException as exc:
635
  if exc.status_code == 403:
636
  return False
 
728
  if include_schema:
729
  if row["answerable"] or strict_schema:
730
  try:
731
+ # ⭐ S67-T34 (D-2630): THE FIELD CONTRACT, NOT THE ROWS. Describing a
732
+ # database used to read (and count) every row first, so the largest
733
+ # connected table refused its own schema. "schema" runs every wall and
734
+ # reads no row; sources that are not mirror-served ignore it.
735
  scoped = deps.assistant_read_scope(
736
+ session, database, fields=None, filters=None, mode="schema",
737
  )
738
  except HTTPException as exc:
739
  # HOTFIX 2026-09-24: ONE refusing source (measured: a 996,203-row mirror
 
1005
  f" '{surface_url}/mcp'",
1006
  "```",
1007
  "",
1008
+ ("Sheet tools require AI apps (MCP) and Sheet access. Geometry patches carry the revision "
1009
  "read by the server; pass rev to also protect the revision you read. Cell writes "
1010
  "are previewed before confirmation. A partially refused patch changes no cells."),
1011
  "",
 
1049
  @router.post("/keys", status_code=201)
1050
  def create_key(body=Body(default=None), session: Session = Depends(require_session)):
1051
  """Mint a credential for the signed-in human. Bearer auth is not accepted here."""
1052
+ require_external_analyst(session)
1053
  label = body.get("label") if isinstance(body, dict) else None
1054
  access = body.get("access") if isinstance(body, dict) else None
1055
  databases = body.get("databases") if isinstance(body, dict) else None
 
1188
  @router.post("/oauth/clients", status_code=201)
1189
  def create_oauth_client(body=Body(default=None), session: Session = Depends(require_session)):
1190
  """Register one user-owned public client; PKCE replaces a shared client secret."""
1191
+ require_external_analyst(session)
1192
  body = body if isinstance(body, dict) else {}
1193
  try:
1194
  client = external_oauth.register_client(
 
1476
  if redirect_uri not in meta["r"]:
1477
  return _refusal_page("The app sent a callback it did not register. Start the "
1478
  "connection again from the app.")
1479
+ reason = perm_scope.ai_apps_refusal(session.user, session.runtime)
1480
+ if reason:
1481
+ return _refusal_page(reason, 403)
1482
  try:
 
1483
  _require_oauth_resource(request, resource)
1484
  if response_type != "code":
1485
  raise external_oauth.ExternalOAuthError("oauth_response_type_invalid")
1486
  external_oauth._normalise_state(state)
1487
  external_oauth._normalise_challenge(code_challenge, code_challenge_method)
1488
  except HTTPException:
1489
+ return _refusal_page("The app requested an unavailable resource. Start the "
1490
+ "connection again from the app.", 403)
1491
  except external_access.ExternalAccessError:
1492
  return _refusal_page("The app sent an incomplete sign-in request. Start the "
1493
  "connection again from the app.")
 
1498
  return _consent_page(request, session, meta, fields)
1499
  if session is None:
1500
  raise err(401, "no_session", "sign in to continue")
1501
+ reason = perm_scope.ai_apps_refusal(session.user, session.runtime)
1502
+ if reason:
1503
+ return _refusal_page(reason, 403)
1504
  try:
1505
  audience = _require_oauth_resource(request, resource)
1506
  result = external_oauth.authorize(
 
1542
  databases = chosen
1543
  else:
1544
  databases = external_access.ALL_DATABASES
1545
+ reason = perm_scope.ai_apps_refusal(session.user, session.runtime)
1546
+ if reason:
1547
+ return _refusal_page(reason, 403)
 
 
1548
  try:
1549
  audience = _require_oauth_resource(request, fields.get("resource"))
1550
  client = external_oauth.adopt_dynamic_client(
 
1727
  @router.get("/skill.md")
1728
  def portable_skill(request: Request, session: Session = Depends(require_session)):
1729
  """Download current governed external read and build instructions."""
1730
+ require_external_analyst(session)
1731
  document = _portable_skill_document(session, str(request.base_url))
1732
  return Response(
1733
  content=document,
 
1758
  @router.get("/scope/databases")
1759
  def scope_databases(session: Session = Depends(require_session)):
1760
  """List the databases this signed-in maker could put inside a new credential's scope."""
1761
+ require_external_analyst(session)
1762
  return {"databases": [
1763
  {key: row[key] for key in ("database", "label", "permitted", "answerable")}
1764
  for row in _resolve_external_scope(session)
 
1825
 
1826
  # This is the same permissioned record reader the in-app Assistant uses. It resolves the
1827
  # database, field and row walls on every call before this adapter sorts, caps, or aggregates.
1828
+ # ⭐ S67-T34: a MIRROR-SERVED database is first resolved schema-only (every wall, no row), so
1829
+ # its aggregates run in SQL over every permitted row and its rows are read only as needed.
1830
  scoped = deps.assistant_read_scope(
1831
+ session, database, fields=None, filters=payload.get("filters"), mode="schema")
1832
+ mirror = scoped.get("source_kind") == "mirror_user_table"
1833
  visible_fields = list(scoped.get("fields") or [])
1834
  visible_keys = {
1835
  field.get("key") for field in visible_fields
 
1858
  sorts.append({"field": rule["field"], "direction": rule["direction"]})
1859
  aggregates = _normalise_aggregates(payload.get("aggregates"), visible_keys)
1860
 
1861
+ server_aggregates = None
1862
+ paged_total = None
1863
+ if mirror:
1864
+ # ⭐⭐ S67-T34 (D-616, D-2631): THE PERMISSION FILTER IS INSIDE THE QUERY. The aggregates
1865
+ # are computed by the mirror in SQL over exactly the rows this caller may read, never
1866
+ # over a fetched slice, and never by pulling a million rows into this process.
1867
+ if aggregates:
1868
+ summary = deps.assistant_read_scope(
1869
+ session, database, fields=None, filters=payload.get("filters"),
1870
+ mode=("aggregate", aggregates))
1871
+ server_aggregates = summary.get("server_aggregates")
1872
+ import assistant_mirror
1873
+ # The summary already counted the permitted population, so a table past the one-pass
1874
+ # ceiling goes straight to SQL paging instead of pulling a first 5,000-row page only to
1875
+ # refuse it.
1876
+ known = (summary.get("total_rows") if aggregates else None)
1877
+ try:
1878
+ if known is not None and int(known) > assistant_mirror.ASSISTANT_READ_MAX:
1879
+ raise err(409, "assistant_source_too_large",
1880
+ f"this database matches {int(known):,} rows for that question, "
1881
+ f"past the {assistant_mirror.ASSISTANT_READ_MAX:,} rows one answer "
1882
+ f"holds in memory")
1883
+ scoped = deps.assistant_read_scope(
1884
+ session, database, fields=None, filters=payload.get("filters"))
1885
+ except HTTPException as exc:
1886
+ info = exc.detail.get("error", {}) if isinstance(exc.detail, dict) else {}
1887
+ if info.get("code") != "assistant_source_too_large":
1888
+ raise
1889
+ # ⭐ RULE 1: PAST THE ONE-PASS CEILING THE ROWS ARE PAGED IN SQL, NOT REFUSED. Only
1890
+ # this page reaches the process. A sort cannot be honoured at this size without
1891
+ # holding every row, so it is refused by name with the fix, never silently dropped.
1892
+ if sorts:
1893
+ raise err(409, "query_sort_too_large",
1894
+ f"{info.get('message') or 'This database is too large to sort here.'} "
1895
+ f"Sorting is refused at this size because it would need every row in "
1896
+ f"memory. Ask without sorts to page it in record order, or narrow it "
1897
+ f"with a filter and sort again.") from exc
1898
+ paged_total = True
1899
+ if paged_total:
1900
+ offset = (0 if payload.get("cursor") is None else None)
1901
+ page_digest = _json_digest({
1902
+ "database": database, "fields": selected_keys, "filters": payload.get("filters"),
1903
+ "sorts": [], "aggregates": aggregates, "limit": limit, "paged": True,
1904
+ })
1905
+ first = deps.assistant_read_scope(
1906
+ session, database, fields=None, filters=payload.get("filters"), mode=("page", 0, 1))
1907
+ snap = _json_digest({
1908
+ "source_version": first.get("source_version"),
1909
+ "visible_fields": [field["key"] for field in visible_fields
1910
+ if isinstance(field, dict) and field.get("key") in visible_keys],
1911
+ "total_rows": first.get("total_rows"),
1912
+ })
1913
+ if offset is None:
1914
+ offset = _decode_query_cursor(session, payload.get("cursor"), page_digest, snap)
1915
+ total_rows = int(first.get("total_rows") or 0)
1916
+ if offset > total_rows:
1917
+ raise err(409, "query_cursor_stale",
1918
+ "The permitted result set changed. Start again without a cursor.")
1919
+ page = deps.assistant_read_scope(
1920
+ session, database, fields=None, filters=payload.get("filters"),
1921
+ mode=("page", offset, limit))
1922
+ page_rows = list(page.get("records") or [])
1923
+ for row in page_rows:
1924
+ _assert_finite_query_value({"pid": row.get("pid"),
1925
+ **{key: row.get(key) for key in selected_keys}})
1926
+ records = [{"pid": copy.deepcopy(row.get("pid")),
1927
+ **{key: copy.deepcopy(row.get(key)) for key in selected_keys}}
1928
+ for row in page_rows]
1929
+ next_offset = offset + limit
1930
+ next_cursor = (_encode_query_cursor(session, page_digest, snap, next_offset)
1931
+ if next_offset < total_rows else None)
1932
+ result = {
1933
+ "database": page.get("database"), "label": page.get("label"),
1934
+ "source_kind": page.get("source_kind"),
1935
+ "source_version": copy.deepcopy(page.get("source_version")),
1936
+ "retrieved_at": page.get("retrieved_at"), "permission_scope_applied": True,
1937
+ "fields": [copy.deepcopy(field) for field in visible_fields
1938
+ if field.get("key") in selected_keys],
1939
+ "filters": copy.deepcopy(page.get("filters")), "sorts": [],
1940
+ "records": records, "aggregates": copy.deepcopy(server_aggregates or []),
1941
+ "page": {"limit": limit, "returned": len(records), "offset": offset,
1942
+ "has_more": next_cursor is not None, "next_cursor": next_cursor},
1943
+ "caps": {"max_page_size": _QUERY_MAX_LIMIT, "max_sorts": _QUERY_MAX_SORTS,
1944
+ "max_aggregates": _QUERY_MAX_AGGREGATES,
1945
+ "max_filter_depth": _QUERY_MAX_FILTER_DEPTH,
1946
+ "max_filter_nodes": _QUERY_MAX_FILTER_NODES,
1947
+ "max_filter_siblings": _QUERY_MAX_FILTER_SIBLINGS},
1948
+ }
1949
+ _assert_finite_query_value(result)
1950
+ audit.record("external.query", session.tenant, session.uname, {
1951
+ "surface": "ai_api",
1952
+ "authentication": session.claims.get("external_auth_type", "api_key"),
1953
+ "field_count": len(selected_keys), "filter_applied": page.get("filters") is not None,
1954
+ "sort_count": 0, "aggregate_count": len(aggregates), "page_size": limit,
1955
+ "returned_count": len(records), "cursor_used": offset > 0,
1956
+ })
1957
+ return result
1958
+
1959
  rows = list(scoped.get("records") or [])
1960
  inspected_keys = (set(selected_keys)
1961
  | {rule["field"] for rule in sorts}
 
2006
  "filters": copy.deepcopy(scoped.get("filters")),
2007
  "sorts": copy.deepcopy(sorts),
2008
  "records": records,
2009
+ "aggregates": (copy.deepcopy(server_aggregates) if server_aggregates is not None
2010
+ else _aggregate_records(rows, aggregates)),
2011
  "page": {"limit": limit, "returned": len(records), "offset": offset,
2012
  "has_more": next_cursor is not None, "next_cursor": next_cursor},
2013
  "caps": {"max_page_size": _QUERY_MAX_LIMIT, "max_sorts": _QUERY_MAX_SORTS,
 
2840
  def _strong_confirmation_required(name, args):
2841
  if name == "database_object":
2842
  return args.get("action") in {"record_delete", "field_create", "field_update", "field_delete"}
2843
+ if name in {"view_delete", "app_delete", "sheet_delete_tab"}:
2844
  return True
2845
  if name == "sheet_write_cells":
2846
  return any(isinstance(cell, dict) and cell.get("kind") == "empty"
 
3023
  "Enter the exact stronger confirmation phrase shown in the preview.")
3024
  _confirmation_already_used(session, payload)
3025
 
3026
+ # Freshly resolve the account, AI apps grant, credential revocation/label, credential scope,
3027
  # database walls and writer semantics before claiming the nonce or touching customer state.
3028
  fresh_user = external_access.resolve_active_user(session.uname, session.tenant)
3029
  fresh_session = Session(session.tenant, fresh_user, copy.deepcopy(session.claims), session.runtime)
 
3210
  return protocol_error(rid, -32602, "MCP params must be an object.")
3211
  name = params.get("name")
3212
  try:
3213
+ # Same AI apps wall as the read doors, inside the dispatcher so a grant refusal
3214
  # remains MCP content. Only the unchanged bearer dependency produces an HTTP 401.
3215
  require_external_analyst(session)
3216
  if method == "initialize":
 
3273
 
3274
  @router.post("/mcp")
3275
  async def external_mcp(request: Request, session: Session = Depends(require_external_session)):
3276
+ """Remote MCP without a browser session, with the read doors' maker and AI apps wall."""
3277
  try:
3278
  req = await request.json()
3279
  except (ValueError, UnicodeError):
 
3287
  # A new write in the registry without a matching literal raises at import, before serving.
3288
  _WRITE_TOOLS = frozenset({
3289
  "sheet_create_workbook", "sheet_add_tab", "sheet_write_cells", "sheet_set_binding",
3290
+ "sheet_delete_tab",
3291
  "sheet_set_styles", "sheet_set_dimensions", "sheet_set_validation", "sheet_set_freeze",
3292
  "sheet_add_rule", "sheet_set_chart", "sheet_structure",
3293
  "sheet_save_template",
api/routes_keychain.py CHANGED
@@ -782,23 +782,63 @@ def _credential_revocation(row, rt):
782
 
783
 
784
  def _revocation_residue(entry_type):
785
- """Standing rule 1 requires the cause AND a fix for the mirror this route cannot clear.
786
-
787
- Clearing the app's materialised copy is not evidence that the raw reports disappeared. Mirror
788
- deletion stays operator-only because it is permanent, exact-tenant work outside the product
789
- store transaction.
790
- """
791
  if entry_type != "amazon_sp":
792
  return []
793
  return [{
794
  "where": "The tenant DuckDB mirror",
795
  "why": "Connection removal clears product databases but not the raw Amazon mirror.",
796
- "fix": "After every Amazon credential is revoked, an operator must run "
797
  "platform/harness/amazon_store.py --purge --tenant <tenant> "
798
  "--confirm-tenant <tenant> and retain the residue-free receipt.",
799
  }]
800
 
801
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
802
  def _write_revocation_warning(rt, entry_id, warning=None):
803
  """Prune expired consent on every write; failed persistence requires explicit confirmation."""
804
  now = int(time.time())
@@ -852,6 +892,7 @@ def list_keychain(session: Session = Depends(require_session)):
852
  @router.post("/admin/keychain", status_code=201)
853
  def add_key(body: dict = Body(default=None), session: Session = Depends(require_session)):
854
  kc = _kc()
 
855
  body = body or {}
856
  if not session.runtime.available():
857
  raise err(503, "store_unavailable", "the tenant store is unavailable")
@@ -921,6 +962,11 @@ def add_key(body: dict = Body(default=None), session: Session = Depends(require_
921
  except Exception: # noqa: BLE001
922
  pass
923
  elif etype == "amazon_sp":
 
 
 
 
 
924
  # D-29 / intake F6: the HTTP thread only records intent. The resident resync thread
925
  # remains the mirror's single writer. A queue failure must not undo a stored credential.
926
  try:
@@ -939,7 +985,7 @@ def add_key(body: dict = Body(default=None), session: Session = Depends(require_
939
  raise err(503, "store_unavailable",
940
  "the key was not saved; its sharing setting could not be stored, so "
941
  "nothing was kept. Try again.")
942
- return {"entry": {**_identified(row), "scope": scope, "owner": session.uname}}
943
 
944
 
945
  @router.put("/admin/keychain/{entry_id}")
@@ -1089,14 +1135,16 @@ def delete_key(entry_id: str, session: Session = Depends(require_session), confi
1089
  raise err(503, "store_unavailable", "The delete did not land. Try again.")
1090
  if not purges:
1091
  _write_revocation_warning(rt, entry_id)
 
 
1092
  if footprint["rowsKnown"]:
1093
- return {"ok": True}
1094
  # Fail soft costs the census, never the ability to revoke access. Standing rule 1 makes
1095
  # that incomplete cleanup explicit instead of returning an invented "nothing to purge".
1096
  return {"ok": True, "registry": registered, "footprint": footprint,
1097
  "purged": {"tables": [], "rows": 0, "rowsKnown": False,
1098
  "refused": [], "failed": []},
1099
- "residue": _revocation_residue(row["type"])}
1100
  from core import user_tables as ut
1101
  try:
1102
  result = ut.purge_credential_tables(row["type"], entry_id, rt=_Ownership(), st=rt)
@@ -1105,11 +1153,14 @@ def delete_key(entry_id: str, session: Session = Depends(require_session), confi
1105
  result = {"purged": [], "rows": 0, "rowsKnown": False, "refused": [],
1106
  "failed": [{"key": key, "error": type(exc).__name__} for key in tables]}
1107
  _write_revocation_warning(rt, entry_id)
 
 
 
1108
  return {"ok": True,
1109
  "purged": {"tables": result["purged"], "rows": result["rows"],
1110
  "rowsKnown": result["rowsKnown"], "refused": result["refused"],
1111
  "failed": result["failed"]},
1112
- "residue": _revocation_residue(row["type"])}
1113
 
1114
 
1115
  @router.post("/admin/keychain/{entry_id}/test")
 
782
 
783
 
784
  def _revocation_residue(entry_type):
785
+ """Fallback cleanup instructions when automatic retention could not be scheduled."""
 
 
 
 
 
786
  if entry_type != "amazon_sp":
787
  return []
788
  return [{
789
  "where": "The tenant DuckDB mirror",
790
  "why": "Connection removal clears product databases but not the raw Amazon mirror.",
791
+ "fix": "If automatic deletion could not be scheduled, after every Amazon credential is revoked an operator must run "
792
  "platform/harness/amazon_store.py --purge --tenant <tenant> "
793
  "--confirm-tenant <tenant> and retain the residue-free receipt.",
794
  }]
795
 
796
 
797
+ def _amazon_lifecycle_warning(session, action):
798
+ """Access already changed. Warn and audit a lifecycle failure without turning it into 5xx."""
799
+ rt = session.runtime
800
+ removing = action == "revocation"
801
+ warning = ("The Amazon credential was removed, but its automatic data deletion could not be scheduled. "
802
+ "Contact an administrator to arrange deletion within 30 days."
803
+ if removing else
804
+ "The Amazon credential was saved, but its pending data deletion could not be cancelled. "
805
+ "Contact an administrator before the scheduled deletion date.")
806
+ kind = "credential.revoked" if removing else "credential.added"
807
+ try:
808
+ from core import audit
809
+ detail = {"connector": "amazon_sp",
810
+ "deletion_trigger_recorded" if removing else "deletion_cancellation_recorded": False}
811
+ if audit.record(kind, rt.key, session.uname, detail) is None:
812
+ raise RuntimeError("lifecycle audit could not be saved")
813
+ except Exception:
814
+ try:
815
+ from harness import amazon_store as az
816
+ az._append_ledger(rt.key, rt, "audit_failures",
817
+ {"tenant": rt.key, "kind": kind, "checked_at": az._control_stamp()})
818
+ except Exception:
819
+ pass
820
+ warning += " The audit event could not be saved."
821
+ print(f"Amazon lifecycle warning: {warning}")
822
+ return {"warning": warning}
823
+
824
+
825
+ def _amazon_deletion_receipt(session, *, product_purge_succeeded=True):
826
+ try:
827
+ if not product_purge_succeeded:
828
+ raise RuntimeError("product cleanup is incomplete")
829
+ from harness import amazon_store as az
830
+ rt = session.runtime
831
+ az.record_deletion_trigger(rt.key, "revocation", rt=rt, principal=session.uname)
832
+ plan = az.scheduled_purge(rt.key, rt)
833
+ return {"purgeScheduledAt": plan["purgeAt"], "deadlineAt": plan["deadlineAt"],
834
+ "residue": [{"where": "Amazon data in this workspace",
835
+ "why": az.deletion_schedule_message(plan),
836
+ "fix": "Add the Amazon credential again before the scheduled date to cancel deletion."}]}
837
+ except Exception:
838
+ return {**_amazon_lifecycle_warning(session, "revocation"),
839
+ "residue": _revocation_residue("amazon_sp")}
840
+
841
+
842
  def _write_revocation_warning(rt, entry_id, warning=None):
843
  """Prune expired consent on every write; failed persistence requires explicit confirmation."""
844
  now = int(time.time())
 
892
  @router.post("/admin/keychain", status_code=201)
893
  def add_key(body: dict = Body(default=None), session: Session = Depends(require_session)):
894
  kc = _kc()
895
+ lifecycle = {}
896
  body = body or {}
897
  if not session.runtime.available():
898
  raise err(503, "store_unavailable", "the tenant store is unavailable")
 
962
  except Exception: # noqa: BLE001
963
  pass
964
  elif etype == "amazon_sp":
965
+ try:
966
+ from harness import amazon_store as az
967
+ az.cancel_deletion_trigger(session.runtime.key, rt=session.runtime, principal=session.uname)
968
+ except Exception:
969
+ lifecycle = _amazon_lifecycle_warning(session, "cancellation")
970
  # D-29 / intake F6: the HTTP thread only records intent. The resident resync thread
971
  # remains the mirror's single writer. A queue failure must not undo a stored credential.
972
  try:
 
985
  raise err(503, "store_unavailable",
986
  "the key was not saved; its sharing setting could not be stored, so "
987
  "nothing was kept. Try again.")
988
+ return {"entry": {**_identified(row), "scope": scope, "owner": session.uname}, **lifecycle}
989
 
990
 
991
  @router.put("/admin/keychain/{entry_id}")
 
1135
  raise err(503, "store_unavailable", "The delete did not land. Try again.")
1136
  if not purges:
1137
  _write_revocation_warning(rt, entry_id)
1138
+ lifecycle = (_amazon_deletion_receipt(session, product_purge_succeeded=footprint["rowsKnown"])
1139
+ if row["type"] == "amazon_sp" else {})
1140
  if footprint["rowsKnown"]:
1141
+ return {"ok": True, **lifecycle}
1142
  # Fail soft costs the census, never the ability to revoke access. Standing rule 1 makes
1143
  # that incomplete cleanup explicit instead of returning an invented "nothing to purge".
1144
  return {"ok": True, "registry": registered, "footprint": footprint,
1145
  "purged": {"tables": [], "rows": 0, "rowsKnown": False,
1146
  "refused": [], "failed": []},
1147
+ "residue": _revocation_residue(row["type"]), **lifecycle}
1148
  from core import user_tables as ut
1149
  try:
1150
  result = ut.purge_credential_tables(row["type"], entry_id, rt=_Ownership(), st=rt)
 
1153
  result = {"purged": [], "rows": 0, "rowsKnown": False, "refused": [],
1154
  "failed": [{"key": key, "error": type(exc).__name__} for key in tables]}
1155
  _write_revocation_warning(rt, entry_id)
1156
+ lifecycle = (_amazon_deletion_receipt(
1157
+ session, product_purge_succeeded=not result["failed"] and not result["refused"])
1158
+ if row["type"] == "amazon_sp" else {})
1159
  return {"ok": True,
1160
  "purged": {"tables": result["purged"], "rows": result["rows"],
1161
  "rowsKnown": result["rowsKnown"], "refused": result["refused"],
1162
  "failed": result["failed"]},
1163
+ "residue": _revocation_residue(row["type"]), **lifecycle}
1164
 
1165
 
1166
  @router.post("/admin/keychain/{entry_id}/test")
api/routes_module_index.py CHANGED
@@ -1,12 +1,15 @@
1
- """W61-T20 / C6: personal App and Agents folder trees in the tenant store."""
2
  import copy
3
 
4
  from fastapi import APIRouter, Body, Depends, HTTPException
5
 
6
  from core.sheets import SheetFolderRefusal, validate_folder_tree
7
  from deps import Session, err, module_gate, require_session
 
8
  import routes_assets
9
  import routes_automation
 
 
10
  import voice_control_plane as voice
11
 
12
  router = APIRouter(prefix="/api/v1/module-index")
@@ -17,16 +20,22 @@ def _sentence(value):
17
  return " ".join(str(value).replace("\u2014", "-").replace("\u2013", "-").split())
18
 
19
 
 
 
 
 
 
 
20
  def _refuse(exc):
21
  status = {"module_index_unknown": 404, "module_index_user_forbidden": 403,
22
  "module_index_item_forbidden": 403, "index_revision_stale": 409}.get(exc.code, 400)
23
- return err(status, exc.code, _sentence(exc.message))
24
 
25
 
26
  def _access(module: str, session: Session = Depends(require_session)):
27
- if module not in {"app", "agents"}:
28
  raise _refuse(SheetFolderRefusal(
29
- "module_index_unknown", f'Folder index "{module}" is not supported; use app or agents.'))
30
  # App has the same session-only door as GET /applications; its rows have their own wall.
31
  if module == "agents":
32
  try:
@@ -47,11 +56,28 @@ def _owner(session, user):
47
 
48
 
49
  def _readable_ids(module, session):
 
 
 
 
 
50
  # The projection is hygiene; the per-item list door is the wall. The client renders
51
  # only listed items, so an unverifiable placement renders no tile. Use local voice
52
  # records, never audience-option assembly, just to verify placement ids.
53
  if module == "app":
54
- return {row["id"] for row in routes_assets.application_index(session=session)["applications"]}
 
 
 
 
 
 
 
 
 
 
 
 
55
  # ⭐⭐ W62-T17 β€” THE ID-ONLY ACCESSOR THIS COMMENT USED TO ASK FOR NOW EXISTS. It read
56
  # *"all_definitions omits synthetic field/Odoo agents. Keep the complete automation list until
57
  # it has an id-only accessor"*, and the whole Agents rail payload was being built to answer a
@@ -94,11 +120,41 @@ def _available(session):
94
  raise err(503, "store_unavailable", "The tenant store is unavailable. Reload folders to retry.")
95
 
96
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
97
  @router.get("/{module}")
98
  def read_module_index(module: str, user: str = "", session: Session = Depends(_access)):
99
  try:
100
  owner = _owner(session, user)
101
  _available(session)
 
102
  doc = session.runtime.get("module_index_" + module) or {}
103
  bucket = (doc.get("users") or {}).get(owner) or {}
104
  ids = _readable_ids(module, session) if bucket.get("itemFolders") else set()
@@ -117,6 +173,7 @@ def write_module_index(module: str, body: dict = Body(default=None), user: str =
117
  raise SheetFolderRefusal("module_index_user_forbidden",
118
  f'The folder tree for "{owner}" can only be changed by its owner.')
119
  _available(session)
 
120
  raw = body or {}
121
  expect_rev = raw.get("rev")
122
  if type(expect_rev) is not int or expect_rev < 0:
@@ -141,6 +198,8 @@ def write_module_index(module: str, body: dict = Body(default=None), user: str =
141
  raise SheetFolderRefusal("index_revision_stale",
142
  "Folders changed somewhere else. Reload folders and try again.")
143
  saved.update(folders=folders, itemFolders=placed, rev=rev + 1)
 
 
144
  if collapsed is not None:
145
  saved["collapsedGroups"] = collapsed
146
  elif "collapsedGroups" in (users.get(owner) or {}):
 
1
+ """Personal App, Agents and Data folder trees in the tenant store."""
2
  import copy
3
 
4
  from fastapi import APIRouter, Body, Depends, HTTPException
5
 
6
  from core.sheets import SheetFolderRefusal, validate_folder_tree
7
  from deps import Session, err, module_gate, require_session
8
+ import application_registry
9
  import routes_assets
10
  import routes_automation
11
+ from routes_applications import _may_open
12
+ from routes_nav import _placeable_top_keys
13
  import voice_control_plane as voice
14
 
15
  router = APIRouter(prefix="/api/v1/module-index")
 
20
  return " ".join(str(value).replace("\u2014", "-").replace("\u2013", "-").split())
21
 
22
 
23
+ # D-3282 (standing rule 1): a limit the server cannot lift is reported with its cause and the fix.
24
+ _REFUSAL_SENTENCES = {
25
+ "too_many_folders": "Folders hold at most 200 per rail. Delete or merge a folder, then try again.",
26
+ }
27
+
28
+
29
  def _refuse(exc):
30
  status = {"module_index_unknown": 404, "module_index_user_forbidden": 403,
31
  "module_index_item_forbidden": 403, "index_revision_stale": 409}.get(exc.code, 400)
32
+ return err(status, exc.code, _sentence(_REFUSAL_SENTENCES.get(exc.code, exc.message)))
33
 
34
 
35
  def _access(module: str, session: Session = Depends(require_session)):
36
+ if module not in {"app", "agents", "data"}:
37
  raise _refuse(SheetFolderRefusal(
38
+ "module_index_unknown", f'Folder index "{module}" is not supported; use app, agents or data.'))
39
  # App has the same session-only door as GET /applications; its rows have their own wall.
40
  if module == "agents":
41
  try:
 
56
 
57
 
58
  def _readable_ids(module, session):
59
+ if module == "data":
60
+ keys, enumerated = _placeable_top_keys(session)
61
+ if not enumerated:
62
+ raise err(503, "store_unavailable", "Databases could not be loaded. Reload folders to retry.")
63
+ return keys
64
  # The projection is hygiene; the per-item list door is the wall. The client renders
65
  # only listed items, so an unverifiable placement renders no tile. Use local voice
66
  # records, never audience-option assembly, just to verify placement ids.
67
  if module == "app":
68
+ # S67-T38 (O24): the App home also renders the seeded Finance projections, so the wall
69
+ # admits their ids through the SAME grant filter /nav/applications uses. Never ungated:
70
+ # a Finance card the session cannot open is not placeable.
71
+ stored = {row["id"] for row in routes_assets.application_index(session=session)["applications"]}
72
+ finance = application_registry.finance_application_records(
73
+ may_open=lambda permission: _may_open(session, permission))
74
+ # S67-T41 owner correction: a Content Studio App is bound to a USER table, which the
75
+ # topic-only `/applications` index refuses, so its tile arrives from /nav/applications.
76
+ # Its id is admitted through the registry's own `may_read` (the same read that lists it).
77
+ studios = {row["id"] for row in application_registry.list_apps(
78
+ session.runtime, session.uname, session.admin)
79
+ if row.get("templateKind") == application_registry.CONTENT_STUDIO_TEMPLATE_KIND}
80
+ return stored | {row["id"] for row in finance} | studios
81
  # ⭐⭐ W62-T17 β€” THE ID-ONLY ACCESSOR THIS COMMENT USED TO ASK FOR NOW EXISTS. It read
82
  # *"all_definitions omits synthetic field/Odoo agents. Keep the complete automation list until
83
  # it has an id-only accessor"*, and the whole Agents rail payload was being built to answer a
 
120
  raise err(503, "store_unavailable", "The tenant store is unavailable. Reload folders to retry.")
121
 
122
 
123
+ def _carry_data_index(module, owner, session):
124
+ """Copy the legacy bucket once, inside the index transaction; never clean or write nav_prefs.
125
+
126
+ Presence, including an empty bucket, is authoritative. Revision one makes an unread rev-zero
127
+ PUT stale even when it is the first request after upgrade (W61-T19's wholesale-erase guard).
128
+ """
129
+ if module != "data":
130
+ return
131
+ doc = session.runtime.get("module_index_data") or {}
132
+ if owner in (doc.get("users") or {}):
133
+ return
134
+ legacy = (session.runtime.get("nav_prefs") or {}).get(owner) or {}
135
+
136
+ def carry(current):
137
+ state = copy.deepcopy(current) if isinstance(current, dict) else {}
138
+ users = state.setdefault("users", {})
139
+ if owner not in users:
140
+ users[owner] = {
141
+ "folders": [{**folder, "parent": None} for folder in legacy.get("folders", [])],
142
+ "itemFolders": copy.deepcopy(legacy.get("placement", {})),
143
+ "rev": 1,
144
+ "carriedFrom": "nav_prefs",
145
+ }
146
+ state["schema"] = 1
147
+ return state
148
+
149
+ session.runtime.update("module_index_data", carry, flush="sync")
150
+
151
+
152
  @router.get("/{module}")
153
  def read_module_index(module: str, user: str = "", session: Session = Depends(_access)):
154
  try:
155
  owner = _owner(session, user)
156
  _available(session)
157
+ _carry_data_index(module, owner, session)
158
  doc = session.runtime.get("module_index_" + module) or {}
159
  bucket = (doc.get("users") or {}).get(owner) or {}
160
  ids = _readable_ids(module, session) if bucket.get("itemFolders") else set()
 
173
  raise SheetFolderRefusal("module_index_user_forbidden",
174
  f'The folder tree for "{owner}" can only be changed by its owner.')
175
  _available(session)
176
+ _carry_data_index(module, owner, session)
177
  raw = body or {}
178
  expect_rev = raw.get("rev")
179
  if type(expect_rev) is not int or expect_rev < 0:
 
198
  raise SheetFolderRefusal("index_revision_stale",
199
  "Folders changed somewhere else. Reload folders and try again.")
200
  saved.update(folders=folders, itemFolders=placed, rev=rev + 1)
201
+ if "carriedFrom" in (users.get(owner) or {}):
202
+ saved["carriedFrom"] = users[owner]["carriedFrom"]
203
  if collapsed is not None:
204
  saved["collapsedGroups"] = collapsed
205
  elif "collapsedGroups" in (users.get(owner) or {}):
api/routes_nav.py CHANGED
@@ -528,7 +528,7 @@ def _granted_view_ids(session):
528
  #:
529
  #: These two reserved words are the rail's OWN vocabulary, not a connection's to take:
530
  #: `dataPath.ts::dataDatabaseOrigin` folds `loopable` / `blank` / `automation` / empty into the
531
- #: "Loopable" group, and the branch below puts user tables in "Custom". A connection someone named
532
  #: "Blank" would otherwise dissolve into one of those folders and show a client's invoices beside
533
  #: somebody's scratch table, so such a label falls back to the generic Odoo folder instead.
534
  _RESERVED_GROUP_LABELS = frozenset({"loopable", "blank", "automation", "custom", "odoo"})
@@ -704,7 +704,7 @@ def nav(session: Session = Depends(require_session)):
704
  continue
705
  pages.append({"key": e["key"], "label": e["label"],
706
  "source": e.get("source") or "Blank",
707
- "sourceGroup": e.get("sourceGroup") or "Custom", "chrome": "main"})
708
  except Exception:
709
  # ⭐⭐ W31-T11 (owner item 6b) β€” STILL SWALLOWED, NO LONGER SILENT.
710
  #
@@ -757,7 +757,7 @@ def nav(session: Session = Depends(require_session)):
757
  if _folder:
758
  page["sourceGroup"] = _folder
759
  elif page.get("user_table") or source in {"", "Blank", "Custom"}:
760
- page.setdefault("sourceGroup", "Custom")
761
  # β›”β›” W35-T43 (rulings R4/R7) β€” THE MARK-IMPORTANT COUNTS ARE NO LONGER COMPUTED HERE.
762
  #
763
  # W34-T10 put them on this payload with a 2.5 s budget and a `degraded` entry, spent in RECENTS
 
528
  #:
529
  #: These two reserved words are the rail's OWN vocabulary, not a connection's to take:
530
  #: `dataPath.ts::dataDatabaseOrigin` folds `loopable` / `blank` / `automation` / empty into the
531
+ #: "Loopable" group; the legacy "Custom" group is reserved too. A connection someone named
532
  #: "Blank" would otherwise dissolve into one of those folders and show a client's invoices beside
533
  #: somebody's scratch table, so such a label falls back to the generic Odoo folder instead.
534
  _RESERVED_GROUP_LABELS = frozenset({"loopable", "blank", "automation", "custom", "odoo"})
 
704
  continue
705
  pages.append({"key": e["key"], "label": e["label"],
706
  "source": e.get("source") or "Blank",
707
+ "sourceGroup": e.get("sourceGroup") or "Loopable", "chrome": "main"})
708
  except Exception:
709
  # ⭐⭐ W31-T11 (owner item 6b) β€” STILL SWALLOWED, NO LONGER SILENT.
710
  #
 
757
  if _folder:
758
  page["sourceGroup"] = _folder
759
  elif page.get("user_table") or source in {"", "Blank", "Custom"}:
760
+ page.setdefault("sourceGroup", "Loopable")
761
  # β›”β›” W35-T43 (rulings R4/R7) β€” THE MARK-IMPORTANT COUNTS ARE NO LONGER COMPUTED HERE.
762
  #
763
  # W34-T10 put them on this payload with a 2.5 s budget and a `degraded` entry, spent in RECENTS
api/routes_odoo_tables.py CHANGED
@@ -23,6 +23,7 @@ import hashlib
23
  import hmac
24
  import json
25
  import math
 
26
  import time
27
  from contextlib import contextmanager
28
 
@@ -66,7 +67,7 @@ def _rt():
66
  # takes a mirror cursor with NO tenant gate at all (`is_royal` appears only as an `applicable`
67
  # FIELD in its response), so any authenticated tenant reached tenant #0's file and was served row
68
  # counts from it. That is D-169 firing in production, and it is what `_mirror_cur` closes.
69
- def _mirror_cur(session):
70
  """A mirror cursor for THIS session's tenant, or a refusal that says which failure it is.
71
 
72
  Returns `(cursor, mismatch_sentence)` β€” exactly one of the two is None, so a caller cannot
@@ -88,11 +89,32 @@ def _mirror_cur(session):
88
  """
89
  rtm = _rt()
90
  try:
91
- return rtm.mirror_cursor(session.runtime), None
92
  except rtm.DatastoreMismatch as e:
93
  return None, str(e)
94
 
95
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
96
  @router.post("/odoo-tables/refresh")
97
  def refresh_odoo_tables(session: Session = Depends(require_session)):
98
  """Rebuild all four locked Odoo databases from the tenant's mirror β€” customers, products,
@@ -178,10 +200,15 @@ def odoo_tables_status(brief: bool = Query(False),
178
  rel = _rel()
179
  if brief:
180
  sources = _sources()
181
- return {"ok": True, "tables": {
182
- key: {"readThrough": key in sources}
183
- for _bucket, key, _label, _fields in rel.TABLES
184
- }}
 
 
 
 
 
185
 
186
  import core.user_tables as user_tables
187
  # The detailed operator view genuinely needs rows to report counts/freshness, but it needs
@@ -596,6 +623,28 @@ GRID_SOURCES = {}
596
  _SOURCE_PROVIDERS = []
597
  _SOURCE_ANALYSES = {}
598
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
599
 
600
  def register_source_provider(fn):
601
  """Add a `() -> {table_key: spec}` builder to the connected-grid registry.
@@ -641,20 +690,60 @@ def _sources():
641
  ⚠ STILL LAZY, for the original reason: `_odoo_sources` names `odoo_relational` constants, and
642
  importing that module at file-import time would drag the Odoo layer into every process that
643
  mounts a router. A provider is a CALLABLE precisely so it stays unimported until first read.
 
 
 
 
644
  """
645
- if GRID_SOURCES:
 
646
  return GRID_SOURCES
647
- for build in list(_SOURCE_PROVIDERS):
 
 
 
648
  try:
649
- extra = build() or {}
650
- except Exception: # noqa: BLE001
651
- # β›” ONE PROVIDER'S FAILURE COSTS ITS OWN GRIDS, NEVER THE PAGE. The Odoo grids must
652
- # not go dark because a newer connector's module is unhappy β€” and vice versa.
653
- continue
654
- if extra:
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
655
  GRID_SOURCES.update(extra)
656
  _ut().register_connected(*extra)
657
- return GRID_SOURCES
658
 
659
 
660
  def _odoo_sources():
@@ -1014,7 +1103,67 @@ def _odoo_spawned():
1014
  #: Odoo registers itself, at import, exactly as any other connector does. ⚠ The order providers
1015
  #: are registered in is the order their specs land; keys are namespaced by connector (`ut_odoo_`,
1016
  #: `ut_meta_`), so a later provider cannot shadow an earlier one's grid.
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1017
  register_source_provider(_odoo_sources)
 
1018
  register_source_analysis("odoo", folds=lambda: _fold_reasons(_rel()), spawned=_odoo_spawned)
1019
 
1020
 
@@ -1243,8 +1392,18 @@ def _source_for(cur, table_key):
1243
  if module_cache_source(key) is not None:
1244
  return None
1245
  spec = _sources().get(key)
 
 
 
 
 
 
 
 
1246
  if not spec:
1247
  return None
 
 
1248
  from harness import datastore
1249
  for opt in (spec.get("optional") or ()):
1250
  wants = tuple(opt.get("absent") or ())
@@ -1271,6 +1430,10 @@ def _source_for(cur, table_key):
1271
  for alias, tname in aliases.items():
1272
  cols = datastore.columns_of(tname, cur=cur)
1273
  if not cols:
 
 
 
 
1274
  return None # the mirror has no such table on this deployment
1275
  have_by_alias[alias] = cols
1276
  excluded = ""
@@ -1624,11 +1787,133 @@ def _related_source_value(session, entry, record):
1624
 
1625
 
1626
  def _materialised_table(table_key):
1627
- """Storage dispatch, independent of the browser's fallible status roster."""
1628
- return module_cache_source(table_key) is not None or table_key not in _sources()
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1629
 
 
 
 
1630
 
1631
- def _traversal_defn_or_refuse(session, table_key):
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1632
  """R50 / C4: admit an intermediate after the caller has established both end grants.
1633
 
1634
  The session may already open both ends of a declared two-hop link. Its intermediate
@@ -1636,7 +1921,7 @@ def _traversal_defn_or_refuse(session, table_key):
1636
  read still applies the row wall, hidden-field closure, BU scope and mirror guard.
1637
  """
1638
  ut = _ut()
1639
- st = ut.lend_defs(session.runtime)
1640
  defn = ut.get(table_key, st=st)
1641
  if not defn:
1642
  raise err(404, "unknown_table", "that database does not exist")
@@ -1644,21 +1929,26 @@ def _traversal_defn_or_refuse(session, table_key):
1644
 
1645
 
1646
  def _relation_rows(session, table, columns, *, ids=None, match=None, where=None,
1647
- traversal_of=None):
1648
  """Internal join values, after the ordinary row wall, with no row-count ceiling.
1649
 
1650
  Join columns may be hidden on the wire. Only identities leave the resolver; the native
1651
  payload builders still apply their complete hidden-field closure before returning cells.
1652
  With traversal_of, the caller must already have established both ends of the declared
1653
  edge named by that key. Only join values leave this intermediate read, never a cell.
 
 
 
 
 
1654
  """
1655
  import core.perm_scope as perm_scope
1656
  import core.relations as relations
1657
 
1658
  columns = set(columns) | {relations.IDENTITY_COL}
1659
  if not _materialised_table(table):
1660
- read = (_compiled_read(table, session, traversal_intermediate=True)
1661
- if traversal_of is not None else _compiled_read(table, session))
1662
  spec = read["spec"]
1663
  bindings = dict(spec["cols"], pid=(spec["id"], int))
1664
  if columns - set(bindings):
@@ -1681,11 +1971,16 @@ def _relation_rows(session, table, columns, *, ids=None, match=None, where=None,
1681
 
1682
  # This is an internal join, never a cell response. Apply the same permanent wall on the
1683
  # complete contract before taking any join value, including a join on a hidden column.
1684
- if traversal_of is None and not perm_scope.may_read(session.user, table, st=session.runtime):
 
1685
  raise err(403, "forbidden_table", "This account cannot open the linked database.")
1686
  if module_cache_source(table) is not None:
1687
  session.require(table)
1688
- fields, rows = perm_scope._read(table, session.user, session.runtime)
 
 
 
 
1689
  rows = perm_scope.apply_row_scope(rows, session.user, table, fields, st=session.runtime)
1690
  if where:
1691
  raise err(409, "relation_target_unavailable", "This relation needs a connected database predicate.")
@@ -1730,6 +2025,11 @@ def _relation_leg_ids(session, source, source_ids, entry):
1730
  # identity. Treating a missing/hidden/blank source value as an empty match would
1731
  # turn "the join input is unavailable" into the business claim "no related rows".
1732
  if source_col != relations.IDENTITY_COL and not values:
 
 
 
 
 
1733
  raise err(400, "unknown_relation", relations.unresolvable_source_refusal(source, entry))
1734
  if (manual and not reverse) or (inverse and reverse):
1735
  values = set().union(*(_relation_cell_ids(row.get(source_col)) for row in source_rows))
@@ -1794,6 +2094,10 @@ def _relation_target_ids(session, target, related):
1794
  if not _may_open(session, leg["target"]):
1795
  raise err(403, "forbidden_table",
1796
  f"the relation ends at '{_relation_label(leg['target'])}', which your account cannot open")
 
 
 
 
1797
  ids = _relation_leg_ids(session, table, ids, leg)
1798
  return ids
1799
 
@@ -2464,7 +2768,8 @@ def _may_see_view(session, vid, store):
2464
 
2465
 
2466
  def _compiled_read(table_key, session, *, filters=None, filterConj="and", sorts=None,
2467
- search=None, tree=_TREE_UNSET, related=None, traversal_intermediate=False):
 
2468
  """EVERYTHING THAT DECIDES **WHICH ROWS** A CONNECTED GRID MAY ANSWER OVER, compiled once.
2469
 
2470
  Returns `{defn, fields, cur, spec, cols, base_where, wall_params, where, params, limits,
@@ -2524,17 +2829,25 @@ def _compiled_read(table_key, session, *, filters=None, filterConj="and", sorts=
2524
  # tenant #0's eight databases is read-through. Left un-applied, an administrator could set a
2525
  # filter in Manage user, see it saved, and the scoped user would meet a 409 where their rows
2526
  # used to be. So the wall is APPLIED here, in SQL, three paragraphs down.
 
 
2527
  if traversal_intermediate:
2528
- defn = _traversal_defn_or_refuse(session, table_key)
2529
  else:
2530
- defn = _defn_or_refuse(session, table_key, defs_only=True, scope_applied=True)
2531
  fields = None
2532
  for _bucket, key, _label, mk_fields in rel.TABLES:
2533
  if key == table_key:
2534
  fields = mk_fields()
2535
  break
2536
  if fields is None:
2537
- raise err(404, "not_connected", "that database is not a connected Odoo grid")
 
 
 
 
 
 
2538
 
2539
  # ⭐⭐ W31-T45 / D-169 β€” THE SECOND WALL, and it asks a question `_defn_or_refuse` above cannot.
2540
  # That wall asks "may this SESSION open this DATABASE"; this one asks "is the file this process
@@ -2549,7 +2862,8 @@ def _compiled_read(table_key, session, *, filters=None, filterConj="and", sorts=
2549
  # turn the loudest refusal in the system into a spinner. The guard therefore runs ABOVE the
2550
  # try, not inside it.
2551
  try:
2552
- cur, mismatch = _mirror_cur(session)
 
2553
  except RuntimeError as e:
2554
  # ⚠ Reached ONLY by the mid-first-sync case: `_mirror_cur` has already consumed the
2555
  # mismatch subclass, which is what makes this broad clause safe to keep here.
@@ -2787,7 +3101,7 @@ def _compiled_read(table_key, session, *, filters=None, filterConj="and", sorts=
2787
 
2788
 
2789
  @contextmanager
2790
- def _materialised_read(table_key, session, query, related):
2791
  """One compiler for native rows and their aggregate sibling, over the walled payload.
2792
 
2793
  The temporary connection contains only this request's already stripped, pinned rows and is
@@ -2806,6 +3120,10 @@ def _materialised_read(table_key, session, query, related):
2806
  else:
2807
  from routes_tables import table_rows
2808
  payload = table_rows(table_key, session=session, related=related)
 
 
 
 
2809
  fields = payload["fields"]
2810
  with duckdb.connect(":memory:") as cur:
2811
  cur.execute("CREATE TABLE pinned_rows AS SELECT value AS payload FROM json_each(?)",
@@ -2882,13 +3200,38 @@ def _group_pin(cols, group_by, group_key):
2882
  return f"({gexpr}) = ?", [str(key)]
2883
 
2884
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2885
  def _materialised_window(table_key, session, offset, limit, filters, filterConj, sorts, search, related,
2886
- groupBy=None, groupKey=None):
2887
  from harness import datastore
2888
 
2889
  query = _selection_query({"filters": _json_arg(filters, "filters"), "filterConj": filterConj,
2890
  "sorts": _json_arg(sorts, "sorts"), "search": search})
2891
- with _materialised_read(table_key, session, query, related) as read:
2892
  spec = read["spec"]
2893
  order = _fs().compile_order_by(query["sorts"], read["cols"], tiebreak_sql=spec["id"])
2894
  where, params = read["where"], list(read["params"])
@@ -2912,6 +3255,7 @@ def _materialised_window(table_key, session, offset, limit, filters, filterConj,
2912
 
2913
 
2914
  @router.get("/odoo-tables/{table_key}/rows")
 
2915
  def odoo_table_rows(table_key: str,
2916
  offset: int = Query(default=0, ge=0),
2917
  limit: int = Query(default=0),
@@ -2922,7 +3266,8 @@ def odoo_table_rows(table_key: str,
2922
  related: str = Query(default=None),
2923
  groupBy: str = Query(default=None),
2924
  groupKey: str = Query(default=None),
2925
- session: Session = Depends(require_session)):
 
2926
  """ONE WINDOW over a connected grid, read straight from the mirror (contract C2).
2927
 
2928
  `{fields, rows, total, totalUnfiltered, offset, limit, limits, identity, recordsMutable}` β€”
@@ -2940,19 +3285,36 @@ def odoo_table_rows(table_key: str,
2940
  argument over: `?related={"table":"customer_data","record":1234,"via":{...}}`. It narrows this
2941
  window to the rows linked to that ONE record, ahead of `filters`, and `total` /
2942
  `totalUnfiltered` count inside the pin rather than around it.
2943
- β›” `routes_connected_tables.connected_table_rows` - the source-neutral ALIAS onto this
2944
- function - forwards its arguments BY NAME and does not forward this one, so a pinned read must
2945
- name THIS route. That file is in no lane's fence this wave; the one-line forward is booked in
2946
- W51-T05's handoff. Degrading to an UNPINNED read is the behaviour that shipped rather than a
2947
- new wrong answer, which is the one direction this gap is allowed to be wrong in.
 
2948
  """
2949
  import aios_grid
2950
  import core.perm_scope as perm_scope
2951
  from harness import datastore
2952
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2953
  if _materialised_table(table_key):
2954
  return _materialised_window(table_key, session, offset, limit, filters, filterConj,
2955
- sorts, search, related, groupBy, groupKey)
2956
 
2957
  # β›”β›” ONE COMPILER, AND THE AGGREGATES DOOR CALLS THE SAME ONE. The permission wall, the
2958
  # hidden-field closure, the user predicate and the search all live in `_compiled_read` above,
@@ -2960,8 +3322,12 @@ def odoo_table_rows(table_key: str,
2960
  # is a page of* - and that is provable only while there is one of it. What stays here is what
2961
  # a WINDOW alone needs: the total order, the projection, the overlay merge and the two
2962
  # sentences a page earns (`window_clamped`, the deep-page walk).
 
 
 
2963
  read = _compiled_read(table_key, session, filters=filters, filterConj=filterConj,
2964
- sorts=sorts, search=search, related=related)
 
2965
  defn, fields, cur, spec = read["defn"], read["fields"], read["cur"], read["spec"]
2966
  cols, limits, frm = read["cols"], read["limits"], read["frm"]
2967
  base_where, wall_params = read["base_where"], read["wall_params"]
@@ -3010,6 +3376,40 @@ def odoo_table_rows(table_key: str,
3010
  rows_src = _pool_from_window(spec, win,
3011
  stamp=mirror_stamp(spec, cur, table_key, defn=defn, rt=
3012
  session.runtime))
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
3013
  overlays = {str(r["pid"]): {k: v for k, v in r.items() if k != "pid"} for r in rows_src}
3014
 
3015
  # ⭐⭐ W30-T28 β€” THE TENANT-WIDE OVERLAY, ON A READ-THROUGH GRID.
@@ -3025,11 +3425,14 @@ def odoo_table_rows(table_key: str,
3025
  # the wall and the predicate, so the argument it demands is the list we just fetched.
3026
  # ⚠ It is NOT a permission wall (its header says so twice); `_defn_or_refuse` above already
3027
  # answered "may this session open this surface".
3028
- shared_defs = _so().fields(table_key, st=session.runtime)
 
 
 
3029
  if shared_defs:
3030
  fields = list(fields) + [dict(f, source="overlay") for f in shared_defs.values()]
3031
  for pid, cells in _so().cells(table_key, [r["pid"] for r in rows_src],
3032
- st=session.runtime).items():
3033
  overlays.setdefault(pid, {}).update(cells)
3034
  # RECOMPUTED ON THE **MERGED** CONTRACT, and that is not belt-and-braces. The closure above
3035
  # ran before `shared_defs` appended this database's tenant-wide overlay columns, so a formula
@@ -3335,6 +3738,7 @@ def odoo_table_aggregates(table_key: str, body: dict = Body(default=None),
3335
  # hand: `filters` a list, `filterConj` in and/or, `search` text under 1,000 characters. A
3336
  # second spelling of "what a predicate on the wire looks like" is how two of them come apart.
3337
  query = _selection_query(payload)
 
3338
  aggs = payload.get("aggs")
3339
  if aggs is None:
3340
  aggs = []
@@ -3348,7 +3752,7 @@ def odoo_table_aggregates(table_key: str, body: dict = Body(default=None),
3348
  raise err(400, "bad_selection", "groupBy must be a column id")
3349
 
3350
  if _materialised_table(table_key):
3351
- with _materialised_read(table_key, session, query, payload.get("related")) as read:
3352
  out = _aggregate_over(read["cur"], read["spec"], read["cols"], read["where"],
3353
  read["params"], read["frm"], aggs, group_by=group_by,
3354
  limits=read["limits"])
@@ -3359,9 +3763,12 @@ def odoo_table_aggregates(table_key: str, body: dict = Body(default=None),
3359
  # who assumed both were bodies would have wired the rows and left the summaries answering over
3360
  # the entire database under a tab that names one record. `_compiled_read` accepts either shape,
3361
  # so the two doors cannot pin differently without this argument being deleted.
 
 
3362
  read = _compiled_read(table_key, session, tree=query["filters"],
3363
  filterConj=query["filterConj"], search=query["search"],
3364
- related=payload.get("related"))
 
3365
  out = _aggregate_over(read["cur"], read["spec"], read["cols"], read["where"],
3366
  read["params"], read["frm"], aggs, group_by=group_by,
3367
  limits=read["limits"])
@@ -3487,7 +3894,7 @@ def population(table_key, cur=None, rt=None):
3487
  from harness import datastore
3488
  if rt is not None:
3489
  rt.assert_datastore_matches() # W31-T45: the same wall the row doors carry
3490
- cur = cur if cur is not None else datastore.ro_con()
3491
  spec = _source_for(cur, table_key)
3492
  if spec is None:
3493
  return None
@@ -3520,9 +3927,9 @@ def whole_pool(table_key, cur=None, rt=None):
3520
  """
3521
  from harness import datastore
3522
  if rt is not None:
3523
- cur = cur if cur is not None else _rt().mirror_cursor(rt)
3524
  rt.assert_datastore_matches() # also covers a cursor the CALLER opened and lent
3525
- cur = cur if cur is not None else datastore.ro_con()
3526
  spec = _source_for(cur, table_key)
3527
  if spec is None:
3528
  raise TooBigToMaterialise(
 
23
  import hmac
24
  import json
25
  import math
26
+ import threading
27
  import time
28
  from contextlib import contextmanager
29
 
 
67
  # takes a mirror cursor with NO tenant gate at all (`is_royal` appears only as an `applicable`
68
  # FIELD in its response), so any authenticated tenant reached tenant #0's file and was served row
69
  # counts from it. That is D-169 firing in production, and it is what `_mirror_cur` closes.
70
+ def _mirror_cur(session, table_key=None):
71
  """A mirror cursor for THIS session's tenant, or a refusal that says which failure it is.
72
 
73
  Returns `(cursor, mismatch_sentence)` β€” exactly one of the two is None, so a caller cannot
 
89
  """
90
  rtm = _rt()
91
  try:
92
+ return _table_cursor(session.runtime, table_key), None
93
  except rtm.DatastoreMismatch as e:
94
  return None, str(e)
95
 
96
 
97
+ def _table_cursor(rt, table_key=None):
98
+ """THE one place a mirror cursor is chosen FOR A TABLE: tenant assertion first, then the gate.
99
+
100
+ Amazon sync is independent of Odoo's `ready()` phases, so a `ut_amzn_*` grid asserts the same
101
+ tenant and then opens the loader's local mirror without that Odoo gate; every other table keeps
102
+ `mirror_cursor`, which refuses until the Odoo backfill is live (partial totals are worse than a
103
+ retry).
104
+ β›” HOTFIX 2026-09-27: this rule lived only in `_mirror_cur`, so the window route served Amazon
105
+ rows while `whole_pool` (every `GET /tables/<key>/rows` and the workspace envelope of a
106
+ read-through grid) still answered `503 store_not_ready` on a staging mirror with 0 of 8 Odoo
107
+ sources ready. Every door now asks here, so the rule cannot hold in one door and not another.
108
+ `rt=None` is the unguarded legacy caller `whole_pool`/`population` already documents.
109
+ """
110
+ from harness import datastore
111
+ if str(table_key or "").startswith("ut_amzn_"):
112
+ if rt is not None:
113
+ rt.assert_datastore_matches()
114
+ return datastore.ro_cursor()
115
+ return _rt().mirror_cursor(rt) if rt is not None else datastore.ro_con()
116
+
117
+
118
  @router.post("/odoo-tables/refresh")
119
  def refresh_odoo_tables(session: Session = Depends(require_session)):
120
  """Rebuild all four locked Odoo databases from the tenant's mirror β€” customers, products,
 
200
  rel = _rel()
201
  if brief:
202
  sources = _sources()
203
+ tables = {key: {"readThrough": key in sources}
204
+ for _bucket, key, _label, _fields in rel.TABLES}
205
+ # S66-T43 (lead, observed 2026-09-25): the grid asks THIS answer which tables read
206
+ # through, and it listed Odoo's roster only, so an Amazon read-through table opened on the
207
+ # ordinary door and painted its empty stub. Every Amazon key the registered provider
208
+ # serves is read-through; none is listed when this container's mirror holds no Amazon rows.
209
+ tables.update({key: {"readThrough": True} for key in sources
210
+ if str(key).startswith("ut_amzn_") and key not in tables})
211
+ return {"ok": True, "tables": tables}
212
 
213
  import core.user_tables as user_tables
214
  # The detailed operator view genuinely needs rows to report counts/freshness, but it needs
 
623
  _SOURCE_PROVIDERS = []
624
  _SOURCE_ANALYSES = {}
625
 
626
+ #: S67-T28: `GRID_SOURCES` is PUBLISHED WHOLE, never seen half built. The first build asks every
627
+ #: provider in turn, and the Amazon one runs count(*) queries; the old loop added each provider's
628
+ #: keys to the shared dict as it went, and returned the dict to any caller that found it
629
+ #: non-empty. So a concurrent request saw Odoo's keys with no Amazon key, answered
630
+ #: `_materialised_table("ut_amzn_orders") == True`, and a 559,363-row read-through target was read
631
+ #: as a stored table: a first-page 500 (lane D, 2026-09-28). Now a build fills a LOCAL dict under
632
+ #: this lock and publishes it in one `dict.update`; a reader waits on the lock until
633
+ #: `_SOURCES_BUILT` says a whole build is visible. An admission (`require_amazon_mirror`, the
634
+ #: Amazon fallback) happens only after a build has been published, so it can never stand in
635
+ #: for one either.
636
+ _SOURCES_LOCK = threading.RLock()
637
+ _SOURCES_BUILT = False
638
+ _SOURCES_BUILDER = None # the thread inside a build, so a re-entrant call cannot recurse
639
+
640
+
641
+ def invalidate_sources():
642
+ """Forget measured bindings after a connector publishes a new mirror schema."""
643
+ global _SOURCES_BUILT
644
+ with _SOURCES_LOCK:
645
+ _SOURCES_BUILT = False
646
+ GRID_SOURCES.clear()
647
+
648
 
649
  def register_source_provider(fn):
650
  """Add a `() -> {table_key: spec}` builder to the connected-grid registry.
 
690
  ⚠ STILL LAZY, for the original reason: `_odoo_sources` names `odoo_relational` constants, and
691
  importing that module at file-import time would drag the Odoo layer into every process that
692
  mounts a router. A provider is a CALLABLE precisely so it stays unimported until first read.
693
+
694
+ ⭐ S67-T28: built into a LOCAL dict under `_SOURCES_LOCK` and published in one step (see the
695
+ lock's comment). An empty registry still means "build" (gates clear it to force a rebuild),
696
+ and so does one that holds only admissions made since the last invalidation.
697
  """
698
+ global _SOURCES_BUILT, _SOURCES_BUILDER
699
+ if _SOURCES_BUILT and GRID_SOURCES:
700
  return GRID_SOURCES
701
+ with _SOURCES_LOCK:
702
+ if (_SOURCES_BUILT and GRID_SOURCES) or _SOURCES_BUILDER == threading.get_ident():
703
+ return GRID_SOURCES
704
+ _SOURCES_BUILDER = threading.get_ident()
705
  try:
706
+ built = {}
707
+ for build in list(_SOURCE_PROVIDERS):
708
+ try:
709
+ extra = build() or {}
710
+ except Exception: # noqa: BLE001
711
+ # β›” ONE PROVIDER'S FAILURE COSTS ITS OWN GRIDS, NEVER THE PAGE. The Odoo
712
+ # grids must not go dark because a newer connector's module is unhappy, and
713
+ # vice versa.
714
+ continue
715
+ if extra:
716
+ built.update(extra)
717
+ _ut().register_connected(*extra)
718
+ # A key admitted since the last invalidation was measured at least as recently as
719
+ # this build, so it stands; everything else comes from the build. ONE update, so a
720
+ # lock-free reader of the dict sees the old entry or the new one, never a gap.
721
+ GRID_SOURCES.update({**built, **GRID_SOURCES})
722
+ _SOURCES_BUILT = True
723
+ finally:
724
+ _SOURCES_BUILDER = None
725
+ return GRID_SOURCES
726
+
727
+
728
+ def _learn_amazon_sources():
729
+ """Publish the Amazon provider's keys into a registry that was built without any of them.
730
+
731
+ HOTFIX 2026-09-27 (moved here by S67-T28 so every door shares it): a registry built at boot
732
+ BEFORE the Amazon mirror restore holds no Amazon key, and only `refresh_state` invalidates
733
+ it. `_source_for` learned them on demand; `_materialised_table` did not, so a read-through
734
+ Amazon LINK TARGET (orders, 559k rows) was dispatched as a stored table and refused as more
735
+ than one window. A registry that already knows one Amazon key is not re-asked.
736
+ """
737
+ _sources()
738
+ if any(k.startswith("ut_amzn_") for k in list(GRID_SOURCES)):
739
+ return
740
+ extra = _amazon_sources()
741
+ if not extra:
742
+ return
743
+ with _SOURCES_LOCK:
744
+ if not any(k.startswith("ut_amzn_") for k in list(GRID_SOURCES)):
745
  GRID_SOURCES.update(extra)
746
  _ut().register_connected(*extra)
 
747
 
748
 
749
  def _odoo_sources():
 
1103
  #: Odoo registers itself, at import, exactly as any other connector does. ⚠ The order providers
1104
  #: are registered in is the order their specs land; keys are namespaced by connector (`ut_odoo_`,
1105
  #: `ut_meta_`), so a later provider cannot shadow an earlier one's grid.
1106
+ def _amazon_sources():
1107
+ try:
1108
+ import amazon_relational
1109
+ return amazon_relational.grid_sources()
1110
+ except Exception:
1111
+ import logging
1112
+ logging.getLogger(__name__).exception("Amazon grid sources could not be loaded")
1113
+ return {}
1114
+
1115
+
1116
+ def require_amazon_mirror(table_key, session, definition=None, st=None):
1117
+ """Refuse a persisted Amazon stub whose mirror has not loaded, even on a cold registry.
1118
+
1119
+ The provider contributes no keys on an Odoo-only mirror. A persisted read-through
1120
+ definition still must refuse there instead of falling through to its empty stored rows.
1121
+ S67-T28: `st` is a definitions snapshot the caller already holds (`_page_defs`), so the
1122
+ window route stops paying a second definitions projection for this one check.
1123
+ """
1124
+ if not str(table_key).startswith("ut_amzn_"):
1125
+ return
1126
+ import amazon_relational as amazon
1127
+ from routes_tables import _defn_or_refuse
1128
+
1129
+ if table_key not in amazon.SOURCE:
1130
+ return
1131
+ if definition is None:
1132
+ definition = _defn_or_refuse(session, table_key, st=st, defs_only=True,
1133
+ scope_applied=True)
1134
+ try:
1135
+ cur, mismatch = _mirror_cur(session, table_key)
1136
+ except RuntimeError as exc:
1137
+ raise err(503, "store_not_ready", str(exc))
1138
+ if mismatch:
1139
+ raise err(409, "cross_tenant_store", mismatch)
1140
+ try:
1141
+ spec = _sources().get(table_key)
1142
+ if (spec is not None or definition.get("readThrough")
1143
+ or table_key in amazon.READ_THROUGH):
1144
+ # Check current table presence even if a binding was cached before a restart.
1145
+ current = amazon.grid_source(cur, table_key)
1146
+ if current.get("missing"):
1147
+ raise err(503, "amazon_mirror_not_loaded", current["missing"])
1148
+ # A dynamically oversized snapshot may have a durable stub before this process
1149
+ # has measured it. Admit its binding without changing another source's registry.
1150
+ # β›” S67-T28: ONLY INTO A PUBLISHED BUILD, and never as the registry's only Amazon
1151
+ # key. Admitted into an empty or Amazon-less registry, this one key used to make
1152
+ # `_sources()` skip its build, so the grid resolved while its link targets
1153
+ # (orders, read-through) were dispatched as stored tables: a first-page 500.
1154
+ if spec is None or spec.get("missing"):
1155
+ _learn_amazon_sources()
1156
+ with _SOURCES_LOCK:
1157
+ _sources()
1158
+ held = GRID_SOURCES.get(table_key)
1159
+ if held is None or held.get("missing"):
1160
+ GRID_SOURCES[table_key] = current
1161
+ finally:
1162
+ cur.close()
1163
+
1164
+
1165
  register_source_provider(_odoo_sources)
1166
+ register_source_provider(_amazon_sources)
1167
  register_source_analysis("odoo", folds=lambda: _fold_reasons(_rel()), spawned=_odoo_spawned)
1168
 
1169
 
 
1392
  if module_cache_source(key) is not None:
1393
  return None
1394
  spec = _sources().get(key)
1395
+ if not spec and key.startswith("ut_amzn_"):
1396
+ # HOTFIX 2026-09-27: a registry built at boot BEFORE the Amazon mirror restore holds no
1397
+ # Amazon key, and only `amazon_relational.refresh_state` invalidates it, which the resync
1398
+ # runs AFTER the Odoo rebuild. On staging that rebuild hung for 14+ minutes, so every
1399
+ # read-through Amazon grid answered "no read-through binding". Ask the Amazon provider
1400
+ # alone, once it can answer; a registry that already knows one Amazon key is not re-asked.
1401
+ _learn_amazon_sources()
1402
+ spec = GRID_SOURCES.get(key)
1403
  if not spec:
1404
  return None
1405
+ if spec.get("missing"):
1406
+ raise err(503, "amazon_mirror_not_loaded", spec["missing"])
1407
  from harness import datastore
1408
  for opt in (spec.get("optional") or ()):
1409
  wants = tuple(opt.get("absent") or ())
 
1430
  for alias, tname in aliases.items():
1431
  cols = datastore.columns_of(tname, cur=cur)
1432
  if not cols:
1433
+ if key.startswith("ut_amzn_"):
1434
+ import amazon_relational
1435
+ raise err(503, "amazon_mirror_not_loaded",
1436
+ amazon_relational.missing_source(key)["missing"])
1437
  return None # the mirror has no such table on this deployment
1438
  have_by_alias[alias] = cols
1439
  excluded = ""
 
1787
 
1788
 
1789
  def _materialised_table(table_key):
1790
+ """Storage dispatch, independent of the browser's fallible status roster.
1791
+
1792
+ β›” S67-T28: an Amazon key the registry does not hold is asked of the Amazon provider first
1793
+ (`_learn_amazon_sources`, the same fallback `_source_for` takes), so a registry built before
1794
+ the Amazon mirror loaded cannot dispatch a read-through Amazon table as a stored one.
1795
+ """
1796
+ if module_cache_source(table_key) is not None:
1797
+ return True
1798
+ if table_key in _sources():
1799
+ return False
1800
+ if str(table_key).startswith("ut_amzn_"):
1801
+ _learn_amazon_sources()
1802
+ return table_key not in GRID_SOURCES
1803
+ return True
1804
+
1805
+
1806
+ #: ⭐ S67-T28 (R38, D-3480 point 1) β€” the tenant's table DEFINITIONS and a stored link target's
1807
+ #: ROWS, each held against the durable `user_tables` revision it was read at. A grouped Amazon page
1808
+ #: used to re-read the definitions projection once for its own wall and once more for EVERY link
1809
+ #: target, plus the stored listings entry, on every page; on staging each of those reads is one
1810
+ #: large JSONB projection (measured 33 to 186 s a page under load, 2026-09-28). Every `put` and
1811
+ #: `update` bumps the revision column in the same statement, so an unchanged token proves the
1812
+ #: bytes unchanged; the token is read BEFORE the value, so a write racing the read can only cache
1813
+ #: newer bytes under the older token (a miss next time, never a stale hit). ⚠ Nothing cached here
1814
+ #: is a permission: the wall (`may_read`, `apply_row_scope`, the compiled predicate) is evaluated
1815
+ #: per request against the session's own user, only the bytes it evaluates over are reused.
1816
+ _REV_MEMO: dict = {}
1817
+ _REV_MEMO_MAX = 64
1818
+ # Route handlers run on a thread pool: the lock makes the evict (`next(iter(...))`) and the
1819
+ # insert one step, so a concurrent insert can never raise "dictionary changed size".
1820
+ _REV_MEMO_LOCK = threading.Lock()
1821
+ #: "No token was handed in, read one." Distinct from None, which means "unreadable, hold nothing".
1822
+ #: It is the `...` literal (a singleton) so a signature can spell its default `...` and still
1823
+ #: define when `verify_grid_ux.py` execs `_relation_rows` on its own, without this module name.
1824
+ _UNREAD = ...
1825
+
1826
+
1827
+ def _user_tables_token(rt):
1828
+ """The durable change token of this tenant's `user_tables` bucket, or None when unreadable."""
1829
+ try:
1830
+ return rt.revision(_ut().STORE_KEY)["token"]
1831
+ except Exception: # noqa: BLE001
1832
+ return None
1833
+
1834
+
1835
+ def _rev_memo(kind, rt, name, read, keep, clone, token=_UNREAD):
1836
+ """`read()`, reused while this tenant's `user_tables` revision is unchanged.
1837
+
1838
+ β›” `clone` runs on the way in AND on every hit, so a caller that edits what it was handed
1839
+ cannot edit the next request's answer. `keep(value)` decides what may be held: an empty
1840
+ answer never is (it is the shape a failed read takes, and caching it would make one outage
1841
+ outlive itself), and neither is the whole-document fallback `all_defs` takes on a failure.
1842
+ `token` is one this REQUEST already read (`_page_defs`), so a page pays one revision round
1843
+ trip, not one per memo. It was read before any value this call reads, which is the only
1844
+ ordering the staleness argument above needs.
1845
+ """
1846
+ if token is _UNREAD:
1847
+ token = _user_tables_token(rt)
1848
+ tenant = getattr(rt, "key", None)
1849
+ slot = (kind, tenant, name)
1850
+ if token is not None and tenant:
1851
+ with _REV_MEMO_LOCK:
1852
+ hit = _REV_MEMO.get(slot)
1853
+ if hit is not None and hit[0] == token:
1854
+ return clone(hit[1])
1855
+ value = read()
1856
+ if token is not None and tenant and keep(value):
1857
+ held = (token, clone(value))
1858
+ with _REV_MEMO_LOCK:
1859
+ if slot not in _REV_MEMO and len(_REV_MEMO) >= _REV_MEMO_MAX:
1860
+ _REV_MEMO.pop(next(iter(_REV_MEMO)), None)
1861
+ _REV_MEMO[slot] = held
1862
+ return value
1863
+
1864
+
1865
+ def _page_defs(session):
1866
+ """ONE lent definitions snapshot for a whole `/rows` page: its own wall and every link target's.
1867
+
1868
+ Returns `(st, token)`: the lent snapshot, and the `user_tables` revision it was checked
1869
+ against (None when unreadable), which the page hands to `_stored_link_rows` so a stored link
1870
+ target does not pay a second revision round trip.
1871
+ The same `lend_defs` the table wall takes for itself (`_defn_or_refuse`, D-213), read once per
1872
+ `user_tables` revision instead of once per question. Kept only when every definition really is
1873
+ a projection (`store.is_projected`), because `all_defs` silently falls back to the whole
1874
+ 28 MB document on a failed read and that must never be held, let alone copied per request.
1875
+ ⚠ The snapshot is a fresh `_Lent` per call, so the `object_shares` and `__shared` buckets it
1876
+ lends lazily live for this one request and are never carried to the next.
1877
+ """
1878
+ import copy
1879
+ import core.store as store
1880
+
1881
+ ut = _ut()
1882
+ rt = session.runtime
1883
+ token = _user_tables_token(rt)
1884
+
1885
+ def keep(v):
1886
+ tables = [d for d in (v or {}).values() if isinstance(d, dict)]
1887
+ return bool(tables) and all(store.is_projected(d) for d in tables)
1888
 
1889
+ defs = _rev_memo("defs", rt, ut.STORE_KEY, lambda: ut.all_defs(rt),
1890
+ keep=keep, clone=copy.deepcopy, token=token)
1891
+ return ut.lend(rt, **{ut.STORE_KEY: defs}), token
1892
 
1893
+
1894
+ def _stored_link_rows(session, table, token=_UNREAD):
1895
+ """`perm_scope._read` for a STORED `ut_*` link target, reused per `user_tables` revision.
1896
+
1897
+ ⚠ Only a table whose rows live in `user_tables` qualifies: a registered row source (a module
1898
+ cache, an app reader) or a read-through binding answers from somewhere this token does not
1899
+ watch, so those read exactly as before. The wall is NOT applied here; the caller applies
1900
+ `apply_row_scope` for its own user on every request. Holding it across users is sound only
1901
+ because `perm_scope._read_user_table` never consults `user` for a `ut_*` key: every
1902
+ user-dependent step (hidden fields, row wall, BU scope) sits above it, in the caller.
1903
+ """
1904
+ import core.perm_scope as perm_scope
1905
+
1906
+ rt = session.runtime
1907
+ if (not str(table).startswith(_ut().KEY_PREFIX) or table in perm_scope._ROW_SOURCES
1908
+ or module_cache_source(table) is not None or not _materialised_table(table)):
1909
+ return perm_scope._read(table, session.user, rt)
1910
+ return _rev_memo("rows", rt, table, lambda: perm_scope._read(table, session.user, rt),
1911
+ keep=lambda v: bool(v[1]),
1912
+ clone=lambda v: ([dict(f) for f in v[0]], [dict(r) for r in v[1]]),
1913
+ token=token)
1914
+
1915
+
1916
+ def _traversal_defn_or_refuse(session, table_key, st=None):
1917
  """R50 / C4: admit an intermediate after the caller has established both end grants.
1918
 
1919
  The session may already open both ends of a declared two-hop link. Its intermediate
 
1921
  read still applies the row wall, hidden-field closure, BU scope and mirror guard.
1922
  """
1923
  ut = _ut()
1924
+ st = st if st is not None else ut.lend_defs(session.runtime)
1925
  defn = ut.get(table_key, st=st)
1926
  if not defn:
1927
  raise err(404, "unknown_table", "that database does not exist")
 
1929
 
1930
 
1931
  def _relation_rows(session, table, columns, *, ids=None, match=None, where=None,
1932
+ traversal_of=None, st=None, rev=...): # `...` IS `_UNREAD`
1933
  """Internal join values, after the ordinary row wall, with no row-count ceiling.
1934
 
1935
  Join columns may be hidden on the wire. Only identities leave the resolver; the native
1936
  payload builders still apply their complete hidden-field closure before returning cells.
1937
  With traversal_of, the caller must already have established both ends of the declared
1938
  edge named by that key. Only join values leave this intermediate read, never a cell.
1939
+ S67-T28: `st` is a lent DEFINITIONS snapshot the caller already holds (`_page_defs`); the wall
1940
+ is asked of it instead of re-reading every definition per target. A stored target's rows come
1941
+ through `_stored_link_rows`, reused per `user_tables` revision; its wall still runs here.
1942
+ `rev` is the revision token `_page_defs` already read for this request; absent, the stored
1943
+ target is read directly, as before S67-T28.
1944
  """
1945
  import core.perm_scope as perm_scope
1946
  import core.relations as relations
1947
 
1948
  columns = set(columns) | {relations.IDENTITY_COL}
1949
  if not _materialised_table(table):
1950
+ read = (_compiled_read(table, session, traversal_intermediate=True, st=st)
1951
+ if traversal_of is not None else _compiled_read(table, session, st=st))
1952
  spec = read["spec"]
1953
  bindings = dict(spec["cols"], pid=(spec["id"], int))
1954
  if columns - set(bindings):
 
1971
 
1972
  # This is an internal join, never a cell response. Apply the same permanent wall on the
1973
  # complete contract before taking any join value, including a join on a hidden column.
1974
+ if traversal_of is None and not perm_scope.may_read(
1975
+ session.user, table, st=st if st is not None else session.runtime):
1976
  raise err(403, "forbidden_table", "This account cannot open the linked database.")
1977
  if module_cache_source(table) is not None:
1978
  session.require(table)
1979
+ # The revision memo serves a caller that handed this request's token (the page's link fills,
1980
+ # S67-T28); any other caller reads exactly as before, which is also what lets
1981
+ # `verify_grid_ux.py` exec this function alone with a stubbed `perm_scope`.
1982
+ fields, rows = (perm_scope._read(table, session.user, session.runtime) if rev is ...
1983
+ else _stored_link_rows(session, table, token=rev))
1984
  rows = perm_scope.apply_row_scope(rows, session.user, table, fields, st=session.runtime)
1985
  if where:
1986
  raise err(409, "relation_target_unavailable", "This relation needs a connected database predicate.")
 
2025
  # identity. Treating a missing/hidden/blank source value as an empty match would
2026
  # turn "the join input is unavailable" into the business claim "no related rows".
2027
  if source_col != relations.IDENTITY_COL and not values:
2028
+ # A visible Amazon row with an empty business key has no related records.
2029
+ # A missing/inaccessible source row must retain the ordinary refusal.
2030
+ if (source.startswith("ut_amzn_") and target.startswith("ut_amzn_")
2031
+ and source_rows and all(source_col in row for row in source_rows)):
2032
+ return set()
2033
  raise err(400, "unknown_relation", relations.unresolvable_source_refusal(source, entry))
2034
  if (manual and not reverse) or (inverse and reverse):
2035
  values = set().union(*(_relation_cell_ids(row.get(source_col)) for row in source_rows))
 
2094
  if not _may_open(session, leg["target"]):
2095
  raise err(403, "forbidden_table",
2096
  f"the relation ends at '{_relation_label(leg['target'])}', which your account cannot open")
2097
+ if not ids and table.startswith("ut_amzn_") and leg["target"].startswith("ut_amzn_"):
2098
+ # An earlier, authorised Amazon hop matched nothing. There is no next
2099
+ # source record to read, and that is an empty relation, not a missing key.
2100
+ continue
2101
  ids = _relation_leg_ids(session, table, ids, leg)
2102
  return ids
2103
 
 
2768
 
2769
 
2770
  def _compiled_read(table_key, session, *, filters=None, filterConj="and", sorts=None,
2771
+ search=None, tree=_TREE_UNSET, related=None, traversal_intermediate=False,
2772
+ st=None):
2773
  """EVERYTHING THAT DECIDES **WHICH ROWS** A CONNECTED GRID MAY ANSWER OVER, compiled once.
2774
 
2775
  Returns `{defn, fields, cur, spec, cols, base_where, wall_params, where, params, limits,
 
2829
  # tenant #0's eight databases is read-through. Left un-applied, an administrator could set a
2830
  # filter in Manage user, see it saved, and the scoped user would meet a 409 where their rows
2831
  # used to be. So the wall is APPLIED here, in SQL, three paragraphs down.
2832
+ # S67-T28: `st` lends a definitions snapshot the caller already holds (`_page_defs`), the
2833
+ # same `st=` `_defn_or_refuse` has always accepted. None keeps its own `lend_defs` read.
2834
  if traversal_intermediate:
2835
+ defn = _traversal_defn_or_refuse(session, table_key, st=st)
2836
  else:
2837
+ defn = _defn_or_refuse(session, table_key, st=st, defs_only=True, scope_applied=True)
2838
  fields = None
2839
  for _bucket, key, _label, mk_fields in rel.TABLES:
2840
  if key == table_key:
2841
  fields = mk_fields()
2842
  break
2843
  if fields is None:
2844
+ if table_key in _sources():
2845
+ fields = defn.get("fields") or []
2846
+ if not fields:
2847
+ raise err(409, "schema_not_loaded",
2848
+ "This connected database has no saved fields yet. Sync its connector first.")
2849
+ else:
2850
+ raise err(404, "not_connected", "that database is not a connected grid")
2851
 
2852
  # ⭐⭐ W31-T45 / D-169 β€” THE SECOND WALL, and it asks a question `_defn_or_refuse` above cannot.
2853
  # That wall asks "may this SESSION open this DATABASE"; this one asks "is the file this process
 
2862
  # turn the loudest refusal in the system into a spinner. The guard therefore runs ABOVE the
2863
  # try, not inside it.
2864
  try:
2865
+ cur, mismatch = (_mirror_cur(session, table_key) if table_key.startswith("ut_amzn_")
2866
+ else _mirror_cur(session))
2867
  except RuntimeError as e:
2868
  # ⚠ Reached ONLY by the mid-first-sync case: `_mirror_cur` has already consumed the
2869
  # mismatch subclass, which is what makes this broad clause safe to keep here.
 
3101
 
3102
 
3103
  @contextmanager
3104
+ def _materialised_read(table_key, session, query, related, ids=None):
3105
  """One compiler for native rows and their aggregate sibling, over the walled payload.
3106
 
3107
  The temporary connection contains only this request's already stripped, pinned rows and is
 
3120
  else:
3121
  from routes_tables import table_rows
3122
  payload = table_rows(table_key, session=session, related=related)
3123
+ if ids is not None:
3124
+ wanted = set(ids)
3125
+ payload = {**payload, "rows": [row for row in payload["rows"]
3126
+ if int(row["pid"]) in wanted]}
3127
  fields = payload["fields"]
3128
  with duckdb.connect(":memory:") as cur:
3129
  cur.execute("CREATE TABLE pinned_rows AS SELECT value AS payload FROM json_each(?)",
 
3200
  return f"({gexpr}) = ?", [str(key)]
3201
 
3202
 
3203
+ def _record_ids(body):
3204
+ """An explicit empty set stays empty. Never truncate a drill's population."""
3205
+ if "ids" not in body:
3206
+ return None
3207
+ ids = body["ids"]
3208
+ if not isinstance(ids, list) or any(type(pid) is not int for pid in ids):
3209
+ raise err(400, "bad_record_ids", "ids must be a list of integer record IDs")
3210
+ if any(pid < -(2 ** 63) or pid >= 2 ** 63 for pid in ids):
3211
+ raise err(400, "bad_record_ids", "Record IDs must fit a signed 64-bit integer")
3212
+ return list(dict.fromkeys(ids))
3213
+
3214
+
3215
+ def _restrict_row_ids(read, ids):
3216
+ """Intersect the walled read with one bound array, independent of the ID count."""
3217
+ if ids is None:
3218
+ return read
3219
+ pin = f'{read["spec"]["id"]} IN (SELECT unnest(?::BIGINT[]))'
3220
+ narrowed = dict(read)
3221
+ for where_key, params_key in (("where", "params"), ("base_where", "wall_params")):
3222
+ where = read[where_key]
3223
+ narrowed[where_key] = f"({where}) AND {pin}" if where else pin
3224
+ narrowed[params_key] = list(read[params_key]) + [ids]
3225
+ return narrowed
3226
+
3227
+
3228
  def _materialised_window(table_key, session, offset, limit, filters, filterConj, sorts, search, related,
3229
+ groupBy=None, groupKey=None, ids=None):
3230
  from harness import datastore
3231
 
3232
  query = _selection_query({"filters": _json_arg(filters, "filters"), "filterConj": filterConj,
3233
  "sorts": _json_arg(sorts, "sorts"), "search": search})
3234
+ with _materialised_read(table_key, session, query, related, ids=ids) as read:
3235
  spec = read["spec"]
3236
  order = _fs().compile_order_by(query["sorts"], read["cols"], tiebreak_sql=spec["id"])
3237
  where, params = read["where"], list(read["params"])
 
3255
 
3256
 
3257
  @router.get("/odoo-tables/{table_key}/rows")
3258
+ @router.post("/odoo-tables/{table_key}/rows")
3259
  def odoo_table_rows(table_key: str,
3260
  offset: int = Query(default=0, ge=0),
3261
  limit: int = Query(default=0),
 
3266
  related: str = Query(default=None),
3267
  groupBy: str = Query(default=None),
3268
  groupKey: str = Query(default=None),
3269
+ session: Session = Depends(require_session),
3270
+ body: dict = Body(default=None)):
3271
  """ONE WINDOW over a connected grid, read straight from the mirror (contract C2).
3272
 
3273
  `{fields, rows, total, totalUnfiltered, offset, limit, limits, identity, recordsMutable}` β€”
 
3285
  argument over: `?related={"table":"customer_data","record":1234,"via":{...}}`. It narrows this
3286
  window to the rows linked to that ONE record, ahead of `filters`, and `total` /
3287
  `totalUnfiltered` count inside the pin rather than around it.
3288
+ S66-T48: POST is the same window with one more narrowing, a JSON body `{"ids": [...]}`: a
3289
+ cell's record set, which can run to thousands of ids and would overflow a URL. Every other
3290
+ part of the question stays in the query string, spelled exactly as GET spells it (one
3291
+ vocabulary, one parser). The set narrows the rows AND their counts before LIMIT/OFFSET, so
3292
+ `total` and the pager describe the cell's records. The connected-tables alias forwards the
3293
+ body.
3294
  """
3295
  import aios_grid
3296
  import core.perm_scope as perm_scope
3297
  from harness import datastore
3298
 
3299
+ # A direct Python caller omits the Body sentinel, and a GET has no body: both read unpinned.
3300
+ ids = None
3301
+ if isinstance(body, dict):
3302
+ # A window part sent in the body would be IGNORED, and an ignored offset or filter is a
3303
+ # wrong page that looks right. Refuse it by name instead.
3304
+ if set(body) - {"ids"}:
3305
+ raise err(400, "bad_window", "Only ids travel in the request body. Send offset, "
3306
+ "limit, filters, sorts and search in the query string.")
3307
+ ids = _record_ids(body)
3308
+
3309
+ # S67-T28: an Amazon grid's page snapshot is taken BEFORE the mirror check, so that check
3310
+ # asks it instead of reading its own definitions projection (one store round trip fewer on
3311
+ # every Amazon page). Not reordered past it: the check may register this grid's binding,
3312
+ # which is what `_materialised_table` below then reads.
3313
+ page = _page_defs(session) if str(table_key).startswith("ut_amzn_") else None
3314
+ require_amazon_mirror(table_key, session, st=page[0] if page else None)
3315
  if _materialised_table(table_key):
3316
  return _materialised_window(table_key, session, offset, limit, filters, filterConj,
3317
+ sorts, search, related, groupBy, groupKey, ids=ids)
3318
 
3319
  # β›”β›” ONE COMPILER, AND THE AGGREGATES DOOR CALLS THE SAME ONE. The permission wall, the
3320
  # hidden-field closure, the user predicate and the search all live in `_compiled_read` above,
 
3322
  # is a page of* - and that is provable only while there is one of it. What stays here is what
3323
  # a WINDOW alone needs: the total order, the projection, the overlay merge and the two
3324
  # sentences a page earns (`window_clamped`, the deep-page walk).
3325
+ # S67-T28 (D-3480 point 1): ONE definitions snapshot answers this page's own wall and every
3326
+ # link target's below, re-read only when `user_tables` has moved (`_page_defs`).
3327
+ page_st, page_rev = page if page is not None else _page_defs(session)
3328
  read = _compiled_read(table_key, session, filters=filters, filterConj=filterConj,
3329
+ sorts=sorts, search=search, related=related, st=page_st)
3330
+ read = _restrict_row_ids(read, ids)
3331
  defn, fields, cur, spec = read["defn"], read["fields"], read["cur"], read["spec"]
3332
  cols, limits, frm = read["cols"], read["limits"], read["frm"]
3333
  base_where, wall_params = read["base_where"], read["wall_params"]
 
3376
  rows_src = _pool_from_window(spec, win,
3377
  stamp=mirror_stamp(spec, cur, table_key, defn=defn, rt=
3378
  session.runtime))
3379
+ if table_key.startswith("ut_amzn_"):
3380
+ import amazon_relational
3381
+ from routes_grid import _may_open
3382
+
3383
+ # β›”β›” HOTFIX 2026-09-28: ONE READ PER STORED TARGET PER REQUEST. FBA customer returns
3384
+ # links listings twice (`seller_sku`, `asin1`); each edge read the whole stored table, so
3385
+ # every page paid two tenant-document reads (~3 s each on staging, /rows 9.5 s against
3386
+ # /aggregates' 1 s) and a burst of group-page clicks starved the store pool until sign-in
3387
+ # failed. A read-through target (orders, 559k rows) keeps its `match=` SQL read per edge:
3388
+ # reading it whole to filter here would be far worse.
3389
+ _opened, _stored = {}, {}
3390
+ _stored_cols = {}
3391
+ for _f, _target, _frm, _on in amazon_relational.link_edges(fields):
3392
+ _stored_cols.setdefault(_target, set()).add(_on)
3393
+
3394
+ def read_amazon_targets(target, column, values):
3395
+ if target not in _opened:
3396
+ _opened[target] = _may_open(session, target, st=page_st)
3397
+ if not _opened[target]:
3398
+ return []
3399
+ if not _materialised_table(target):
3400
+ return _relation_rows(session, target, [column], match=(column, values),
3401
+ st=page_st, rev=page_rev)
3402
+ if target not in _stored:
3403
+ _stored[target] = _relation_rows(
3404
+ session, target, sorted(_stored_cols.get(target, set()) | {column}),
3405
+ st=page_st, rev=page_rev)
3406
+ wanted = {str(value) for value in values}
3407
+ # The same string comparison `_relation_rows`' own `match=` applies to a stored table.
3408
+ return [row for row in _stored[target] if str(row.get(column)) in wanted]
3409
+
3410
+ # Fields are the current reader's contract; target rows cross the same wall
3411
+ # as /related pins. No unscoped SQL subquery can leak a hidden target's id.
3412
+ amazon_relational.fill_link_cells(rows_src, fields, read_amazon_targets)
3413
  overlays = {str(r["pid"]): {k: v for k, v in r.items() if k != "pid"} for r in rows_src}
3414
 
3415
  # ⭐⭐ W30-T28 β€” THE TENANT-WIDE OVERLAY, ON A READ-THROUGH GRID.
 
3425
  # the wall and the predicate, so the argument it demands is the list we just fetched.
3426
  # ⚠ It is NOT a permission wall (its header says so twice); `_defn_or_refuse` above already
3427
  # answered "may this session open this surface".
3428
+ # S67-T28: through `page_st`, which lends a `__shared` bucket lazily ONCE for this request
3429
+ # (`user_tables._Lent`), so `fields` and `cells` share one read of the stratum instead of two.
3430
+ # Both only read it (`shared_overlay._read` copies the top level, `cells` copies each row).
3431
+ shared_defs = _so().fields(table_key, st=page_st)
3432
  if shared_defs:
3433
  fields = list(fields) + [dict(f, source="overlay") for f in shared_defs.values()]
3434
  for pid, cells in _so().cells(table_key, [r["pid"] for r in rows_src],
3435
+ st=page_st).items():
3436
  overlays.setdefault(pid, {}).update(cells)
3437
  # RECOMPUTED ON THE **MERGED** CONTRACT, and that is not belt-and-braces. The closure above
3438
  # ran before `shared_defs` appended this database's tenant-wide overlay columns, so a formula
 
3738
  # hand: `filters` a list, `filterConj` in and/or, `search` text under 1,000 characters. A
3739
  # second spelling of "what a predicate on the wire looks like" is how two of them come apart.
3740
  query = _selection_query(payload)
3741
+ ids = _record_ids(payload)
3742
  aggs = payload.get("aggs")
3743
  if aggs is None:
3744
  aggs = []
 
3752
  raise err(400, "bad_selection", "groupBy must be a column id")
3753
 
3754
  if _materialised_table(table_key):
3755
+ with _materialised_read(table_key, session, query, payload.get("related"), ids=ids) as read:
3756
  out = _aggregate_over(read["cur"], read["spec"], read["cols"], read["where"],
3757
  read["params"], read["frm"], aggs, group_by=group_by,
3758
  limits=read["limits"])
 
3763
  # who assumed both were bodies would have wired the rows and left the summaries answering over
3764
  # the entire database under a tab that names one record. `_compiled_read` accepts either shape,
3765
  # so the two doors cannot pin differently without this argument being deleted.
3766
+ # S67-T28: the same page snapshot `/rows` takes, so opening a grouped view (its group counts
3767
+ # come from here) no longer re-reads every definition while `user_tables` is unchanged.
3768
  read = _compiled_read(table_key, session, tree=query["filters"],
3769
  filterConj=query["filterConj"], search=query["search"],
3770
+ related=payload.get("related"), st=_page_defs(session)[0])
3771
+ read = _restrict_row_ids(read, ids)
3772
  out = _aggregate_over(read["cur"], read["spec"], read["cols"], read["where"],
3773
  read["params"], read["frm"], aggs, group_by=group_by,
3774
  limits=read["limits"])
 
3894
  from harness import datastore
3895
  if rt is not None:
3896
  rt.assert_datastore_matches() # W31-T45: the same wall the row doors carry
3897
+ cur = cur if cur is not None else _table_cursor(rt, table_key)
3898
  spec = _source_for(cur, table_key)
3899
  if spec is None:
3900
  return None
 
3927
  """
3928
  from harness import datastore
3929
  if rt is not None:
3930
+ cur = cur if cur is not None else _table_cursor(rt, table_key)
3931
  rt.assert_datastore_matches() # also covers a cursor the CALLER opened and lent
3932
+ cur = cur if cur is not None else _table_cursor(None, table_key)
3933
  spec = _source_for(cur, table_key)
3934
  if spec is None:
3935
  raise TooBigToMaterialise(
api/routes_platform_admin.py CHANGED
@@ -37,16 +37,60 @@ import subprocess
37
  import sys
38
  import time
39
  from pathlib import Path
 
40
 
41
  from fastapi import APIRouter, Body, Depends, Header, Request
42
 
43
  import core.platform_admin as platform_admin
 
 
44
  import core.store as store
45
  import operator_control
46
  import voice_control_plane as voice
47
  from deps import Session, err, require_session, users
48
 
49
- router = APIRouter(prefix="/api/v1/platform-admin")
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
50
 
51
 
52
  def _now_iso():
@@ -736,6 +780,18 @@ def platform_automations(tenant: str = "", session: Session = Depends(padmin_gat
736
  "resident": resident}}
737
 
738
 
 
 
 
 
 
 
 
 
 
 
 
 
739
  @router.get("/aws")
740
  def platform_aws(days: int = 7, session: Session = Depends(padmin_gate)):
741
  """AWS automation usage, verbatim, or an honest note explaining its absence."""
 
37
  import sys
38
  import time
39
  from pathlib import Path
40
+ from contextlib import asynccontextmanager
41
 
42
  from fastapi import APIRouter, Body, Depends, Header, Request
43
 
44
  import core.platform_admin as platform_admin
45
+ import core.neon_usage as neon_usage
46
+ import core.do_usage as do_usage
47
  import core.store as store
48
  import operator_control
49
  import voice_control_plane as voice
50
  from deps import Session, err, require_session, users
51
 
52
+ @asynccontextmanager
53
+ async def _neon_lifespan(app):
54
+ _start_neon_sampler()
55
+ _start_do_sampler()
56
+ try:
57
+ yield
58
+ finally:
59
+ neon_usage.stop_sampler()
60
+ do_usage.stop_sampler()
61
+
62
+
63
+ router = APIRouter(prefix="/api/v1/platform-admin", lifespan=_neon_lifespan)
64
+
65
+
66
+ def _start_neon_sampler():
67
+ # These callbacks are invoked only by the sampler after its management API
68
+ # awake check. In particular, the card's GET never resolves a store runtime.
69
+ def load():
70
+ from harness import runtime
71
+ return runtime.get_runtime("loopable").get(neon_usage.STORE_KEY)
72
+
73
+ def persist(document):
74
+ from harness import runtime
75
+ runtime.get_runtime("loopable").update(neon_usage.STORE_KEY, lambda _: document, flush="sync")
76
+
77
+ neon_usage.start_sampler(load=load, persist=persist)
78
+
79
+
80
+ def _start_do_sampler():
81
+ def load():
82
+ from harness import runtime
83
+ return runtime.get_runtime("loopable").get(do_usage.STORE_KEY)
84
+
85
+ def persist(document):
86
+ from harness import runtime
87
+ runtime.get_runtime("loopable").update(do_usage.STORE_KEY, lambda _: document, flush="sync")
88
+
89
+ do_usage.start_sampler(load=load, persist=persist)
90
+
91
+
92
+ # FastAPI 0.139 carries a router lifespan into include_router; legacy startup
93
+ # event registration does not fire there. Defining it does no work at import.
94
 
95
 
96
  def _now_iso():
 
780
  "resident": resident}}
781
 
782
 
783
+ @router.get("/neon")
784
+ def platform_neon(session: Session = Depends(padmin_gate)):
785
+ """Project billing totals plus the sampler's bounded, in-memory history."""
786
+ return neon_usage.report()
787
+
788
+
789
+ @router.get("/do")
790
+ def platform_do(session: Session = Depends(padmin_gate)):
791
+ """DigitalOcean management-plane billing, with no database read on this path."""
792
+ return do_usage.report()
793
+
794
+
795
  @router.get("/aws")
796
  def platform_aws(days: int = 7, session: Session = Depends(padmin_gate)):
797
  """AWS automation usage, verbatim, or an honest note explaining its absence."""
api/routes_query.py CHANGED
@@ -410,6 +410,7 @@ def _spec_schema(field_keys):
410
  "filters": {"type": "array", "items": {"type": "object", "properties": {
411
  "colId": col, "op": {"type": "string", "enum": sorted(_grid().FILTER_OPS)},
412
  "value": {"type": "string"},
 
413
  "rhs": {"type": "object", "properties": {
414
  "kind": {"type": "string", "enum": ["field"]}, "colId": col,
415
  }, "required": ["kind", "colId"]},
@@ -897,6 +898,10 @@ def _validate(spec, fields):
897
  for sent, kept in zip(raw_filters, filters):
898
  if sent.get("rhs") and not kept.get("rhs"):
899
  return None, "that column cannot be compared against another column", "rhs_dropped"
 
 
 
 
900
 
901
  aggregation = raw_aggregation if isinstance(raw_aggregation, dict) else {}
902
  op = str(aggregation.get("op") or "").lower()
 
410
  "filters": {"type": "array", "items": {"type": "object", "properties": {
411
  "colId": col, "op": {"type": "string", "enum": sorted(_grid().FILTER_OPS)},
412
  "value": {"type": "string"},
413
+ "values": {"type": "array", "items": {"type": "string"}},
414
  "rhs": {"type": "object", "properties": {
415
  "kind": {"type": "string", "enum": ["field"]}, "colId": col,
416
  }, "required": ["kind", "colId"]},
 
898
  for sent, kept in zip(raw_filters, filters):
899
  if sent.get("rhs") and not kept.get("rhs"):
900
  return None, "that column cannot be compared against another column", "rhs_dropped"
901
+ # D-658: an empty multi-value leaf survives cleaning but narrows no records.
902
+ if sent.get("op") in _grid().MULTI_VALUE_OPS and (
903
+ not sent.get("values") or not kept.get("values")):
904
+ return None, "that filter needs at least one non-blank value", "values_missing"
905
 
906
  aggregation = raw_aggregation if isinstance(raw_aggregation, dict) else {}
907
  op = str(aggregation.get("op") or "").lower()
api/routes_sheet.py CHANGED
@@ -35,6 +35,7 @@ while its surface is a 404.
35
  (standing rule 2), and each names both the cause and the subject.
36
  """
37
 
 
38
  import hashlib
39
  import json
40
  import os
@@ -115,14 +116,67 @@ def _mirror(session: Session):
115
  # ⚠ `TABLES` ITSELF IS UNTOUCHED β€” it has other consumers (`declared_schema_fields`,
116
  # `_odoo_sources`' `register_connected`) that must keep seeing exactly eight rows.
117
  is_registry_grid = fields is None and table_key in rel._REGISTRY_GRID_FIELDS
118
- if fields is None and not is_registry_grid:
119
- raise err(404, "not_connected", f"{table_key} is not a connected Odoo database.")
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
120
  try:
121
- cur, mismatch = rot._mirror_cur(session)
 
 
 
 
 
 
 
 
 
 
122
  except RuntimeError as exc:
123
  raise err(503, "store_not_ready", str(exc))
124
  if mismatch:
125
  raise err(409, "cross_tenant_store", mismatch)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
126
  if is_registry_grid:
127
  # β›”β›” ONE ASSEMBLY PER REQUEST PER (DATABASE, WINDOW), NOT ONE PER CELL. `refresh_sheet`
128
  # resolves a binding CELL BY CELL, and this branch now costs a full `/workspace`
@@ -165,6 +219,8 @@ def _mirror(session: Session):
165
  memo[slot] = (None if want is None else set(want), made)
166
  return made
167
  spec = rot._source_for(cur, table_key)
 
 
168
  if spec is None:
169
  raise err(409, "not_read_through",
170
  f"{table_key} has no read-through binding on this deployment, so a cell cannot "
@@ -179,6 +235,49 @@ def _mirror(session: Session):
179
  return _for
180
 
181
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
182
  def _registry_assembly(session: Session, table_key: str):
183
  """The `g` dict `/workspace` builds for one compiled registry grid, or a NAMED refusal.
184
 
@@ -254,157 +353,144 @@ def _registry_assembly(session: Session, table_key: str):
254
  "shown rather than shown from an older set of columns. Try again in a moment.")
255
 
256
 
257
- def _linked_customer_bridges(database):
258
- """Only declared reverse customer edges with an unconditional, served relation."""
259
- from core.relations import BRIDGES
260
- return [bridge for bridge in BRIDGES if bridge.get("target") == database
261
- and bridge.get("source") == "customer_data" and bridge.get("served")
262
- and bridge.get("cardinality") == "one_to_many" and not bridge.get("where")]
263
 
 
 
 
 
 
 
 
 
 
 
 
264
 
265
- def _linked_customer_fields(session, bridge):
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
266
  from core import perm_scope
267
- _require_database(session, bridge["source"])
268
- g = _registry_assembly(session, bridge["source"])
269
- fields = list(g.get("fields") or [])
270
- hidden = perm_scope.hidden_keys(session.user, bridge["source"], fields, st=session.runtime)
271
- # Only values the server can read per record. A browser formula or linked rollup needs
272
- # another evaluator; offering it here would make a missing value look like a blank value.
273
- fields = [f for f in fields if f.get("key") not in hidden
274
- and f.get("type") not in {"formula", "link", "lookup", "rollup", "metric"}
275
- and not f.get("measure")]
276
- return g, fields
277
 
278
 
279
- def _linked_relation_column(session, bridge, cur, spec, columns):
280
- from core import perm_scope
 
281
  from modules import sheet as sheet_mod
282
- key = str(bridge.get("target_col") or "")
283
- subject = bridge["key"] + ":" + (key or "(missing column)")
284
- hidden = perm_scope.hidden_keys(session.user, bridge["target"], spec.get("fields") or [],
285
- st=session.runtime)
286
- if key in hidden:
287
- raise sheet_mod.SheetRefusal("linked_filter_unreadable", subject,
288
- "the relation column is hidden from this account.")
289
- # The catalogue owns expressions such as sol.order_partner_id and aml.partner_id.
290
- # The declaration supplies the key, and is a fallback only when the catalogue has none.
291
- if (columns.get(key) or {}).get("sql"):
292
- return key
293
- if not re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]*", key):
294
- raise sheet_mod.SheetRefusal("linked_filter_unreadable", subject,
295
- "the link does not declare a readable relation column.")
296
- source = spec.get("from_sql") or spec.get("table")
 
 
 
297
  try:
298
- cur.execute(f'SELECT {key} FROM {source} LIMIT 0')
299
- except Exception:
300
- raise sheet_mod.SheetRefusal("linked_filter_unreadable", subject,
301
- "the declared relation column is unavailable on this database.")
302
- columns[key] = {"sql": key, "type": "int", "aggregate": False}
303
- return key
304
 
305
 
306
  @router.get("/sheets/filter-fields/{database}")
307
  def linked_filter_fields(database: str, session: Session = Depends(require_session)):
308
- """The linked Filter offer, through the same field wall as the related grid."""
309
- from modules import sheet as sheet_mod
310
  database = _require_database(session, database)
311
- bridges = _linked_customer_bridges(database)
312
- if not bridges:
313
- return {"fields": []}
314
- cur, spec, columns = _mirror(session)(database)
315
- columns = dict(columns)
316
- offered = []
317
- for bridge in bridges:
 
 
318
  try:
319
- _linked_relation_column(session, bridge, cur, spec, columns)
320
- _g, fields = _linked_customer_fields(session, bridge)
321
  except HTTPException as exc:
322
- if exc.status_code == 403:
 
 
323
  continue
324
  raise
325
- except sheet_mod.SheetRefusal:
326
  continue
327
- link_label = f'{bridge.get("source_label") or bridge["source"]} via {bridge.get("label") or bridge["key"]}'
 
 
 
 
 
 
 
 
 
 
 
 
328
  for field in fields:
329
- # Copy the existing field vocabulary, including options. No customer values or
330
- # hidden definitions travel on the picker wire.
331
- offered.append({**field,
332
- "key": f'{sheet_mod.LINKED_FILTER_PREFIX}{bridge["key"]}:{field["key"]}',
333
- "label": f'{link_label}: {field.get("label") or field["key"]}'})
334
- return {"fields": offered}
335
-
336
-
337
- def _linked_filter_scope(session, database, nodes, cur, spec, columns, *, today=None):
338
- """Read linked values in Python, then give the relation-blind compiler flat predicates."""
339
- from modules import sheet as sheet_mod
340
- if not sheet_mod.linked_filter_ids(nodes):
341
- return nodes
342
- import aios_grid
343
- from core import perm_scope, shared_overlay, view_templates
344
- from harness import filter_eval, filter_sql
345
-
346
- bridges = {b["key"]: b for b in _linked_customer_bridges(database)}
347
- related = {}
348
-
349
- def resolve(leaf):
350
- subject = leaf["colId"]
351
- bridge_key, field_key = sheet_mod.linked_field_parts(subject)
352
- bridge = bridges.get(bridge_key)
353
- if bridge is None:
354
- raise sheet_mod.SheetRefusal("linked_filter_unreadable", subject,
355
- "this database no longer has that available customer link.")
356
- try:
357
- relation_column = _linked_relation_column(session, bridge, cur, spec, columns)
358
- if bridge_key not in related:
359
- g, fields = _linked_customer_fields(session, bridge)
360
- # Use the rows door's wall, including its shared-field enrichment. The pool's
361
- # identity is pid; it must never be confused with the orders' partner_id.
362
- rows = perm_scope.apply_row_scope(g["rows_src"], session.user, bridge["source"],
363
- aios_grid.FIELDS, st=session.runtime)
364
- related[bridge_key] = (g, fields, rows)
365
- g, fields, source_rows = related[bridge_key]
366
- except HTTPException as exc:
367
- if exc.status_code != 403:
368
- raise
369
- raise sheet_mod.SheetRefusal("linked_filter_unreadable", subject,
370
- "this account cannot read the linked customer database.")
371
- field = next((f for f in fields if f.get("key") == field_key), None)
372
- if field is None:
373
- raise sheet_mod.SheetRefusal("linked_filter_unreadable", subject,
374
- "the field is hidden, removed, or has no readable per-record value.")
375
- local = {**leaf, "colId": field_key}
376
- # Validate before evaluating even an empty pool, so malformed/stale leaves refuse
377
- # rather than looking like a genuine zero. Cell slots were resolved by W60-T06 above.
378
- probe_columns = {field_key: {"sql": "linked_value", "type": sheet_mod.mirror_field_type(field)}}
379
- try:
380
- compiled = sheet_mod.compile_scope([local], "and", probe_columns, today=today)
381
- if (local.get("op") not in filter_sql.FILTER_OPS or compiled is None
382
- or field_key not in compiled.columns_used):
383
- raise ValueError("unusable linked condition")
384
- except (ValueError, sheet_mod.SheetRefusal):
385
- raise sheet_mod.SheetRefusal("linked_filter_unreadable", subject,
386
- "its value or operator cannot be applied to this field.")
387
- storage_key = view_templates.workspace_key(bridge["source"])
388
- try:
389
- definitions = shared_overlay.fields(storage_key, st=session.runtime)
390
- rows = aios_grid.rows_from_pool(source_rows, fields, (g.get("ws") or {}).get("overlays"),
391
- derived=g.get("derived"))
392
- if field_key in definitions:
393
- cells = shared_overlay.cells(storage_key, [r["pid"] for r in source_rows], st=session.runtime)
394
- rows = [{**row, field_key: (cells.get(str(int(row["pid"]))) or {}).get(field_key, "")}
395
- for row in rows]
396
- ctx = filter_eval.EvalCtx(today=today)
397
- ids = [row[bridge["source_col"]] for row in rows
398
- if filter_eval.permits({"conj": "and", "nodes": [local]}, row, fields, ctx)]
399
- return sheet_mod.linked_id_predicate(relation_column, ids)
400
- except Exception:
401
- raise sheet_mod.SheetRefusal("linked_filter_unreadable", subject,
402
- "the linked customer values could not be read. Retry after the data is available.")
403
-
404
- try:
405
- return sheet_mod.rewrite_linked_filters(nodes, resolve)
406
- except sheet_mod.SheetRefusal as exc:
407
- raise err(409, exc.code, exc.message)
408
 
409
 
410
  def _registry_mirror(session: Session, cur, table_key: str, metric_window, needed=None):
@@ -448,6 +534,19 @@ def _registry_mirror(session: Session, cur, table_key: str, metric_window, neede
448
  derived, bounded, limits = _metric_derived(session, table_key, g, metric_window)
449
  rows = aios_grid.rows_from_pool(g["rows_src"], fields, (g.get("ws") or {}).get("overlays"),
450
  derived=derived)
 
 
 
 
 
 
 
 
 
 
 
 
 
451
  # The columns this cell can read, in the assembly's own order. `registry_grid_spec` forces
452
  # `pid` in on its own, because `build_ids_query` needs `spec["id"]` to be a real column.
453
  kept = (fields if needed is None
@@ -815,6 +914,9 @@ def _refuse(exc):
815
  return HTTPException(status_code=409, detail={"error": body})
816
  if exc.code in _FORBIDDEN_CODES:
817
  return err(403, exc.code, exc.message)
 
 
 
818
  status = 404 if exc.code in _MISSING_CODES else 400
819
  return err(status, exc.code, exc.message)
820
 
@@ -1022,17 +1124,43 @@ def _has_cell_inputs(binding):
1022
  related_nodes, _ = _fe_parts(related.get("filter") if isinstance(related, dict) else None)
1023
  nodes = [*nodes, *related_nodes]
1024
  # Linked values and related-table grants can change independently of the orders' cache key.
1025
- if sheet_mod.linked_filter_ids(nodes):
 
1026
  return True
1027
  return any(sheet_mod.filter_cell_nodes(nodes))
1028
 
1029
 
1030
- def _refresh_context(book, binding, memo=None):
1031
- """Authored inputs and relative dates invalidate refresh, but never an open's snapshot."""
 
 
 
 
 
 
 
 
1032
  from core.sheet_binding import fingerprint
1033
  from datetime import date
1034
  payload = {}
1035
- if _has_cell_inputs(binding):
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1036
  if memo is None:
1037
  memo = {}
1038
  if "inputs" not in memo:
@@ -1143,8 +1271,11 @@ def _reresolved_for_reader(session: Session, book, *, endings_only=False):
1143
  # and MCP read agree on stored cells, figures and formats` measured exactly that pair: D1 and
1144
  # F1 fresh 30.0 / 60.0 from MCP against stale-and-blank from the UI. A non-browser consumer has
1145
  # no client engine to cover for it, so the blank is what it would have shown.
 
 
 
1146
  stale_formula = any(
1147
- cell.get("kind") == "formula" and (cell.get("state") != "fresh" or "value" not in cell)
1148
  for tab in (fresh.get("sheets") or []) for cell in (tab.get("cells") or {}).values())
1149
  if changed_values or stale_formula:
1150
  from modules import sheet as sheet_mod
@@ -1760,10 +1891,17 @@ def _recomputed_tab_cells(session: Session, workbook_id: str, tab_ref: str, *, c
1760
  A formula sitting on a tab nobody has open still shows its last answer until that tab is
1761
  opened or the workbook is refreshed, exactly as a stale database-bound cell does.
1762
  """
 
 
 
 
 
1763
  from core import sheets
1764
  from modules import sheet as sheet_mod
1765
  import formula_runtime as fr
 
1766
 
 
1767
  try:
1768
  book = sheets.get_workbook(_store(session), workbook_id)
1769
  found = sheets.find_tab(book, tab_ref)
@@ -1777,17 +1915,43 @@ def _recomputed_tab_cells(session: Session, workbook_id: str, tab_ref: str, *, c
1777
  (found.get("cells") or {}).get(str(addr).strip().upper()) for addr in changes}
1778
  _recompute_formulas(book, fr, sheet_mod,
1779
  cache_key=(session.tenant, session.uname, workbook_id), changes=edits)
1780
- # Date and filter-cell edits also change database totals. Recompute all readers, then formulas.
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1781
  ending_cells = set()
1782
  refreshed_entries = []
1783
  context_memo = {}
1784
- for source in book.get("sheets") or []:
 
 
1785
  for addr, cell in (source.get("cells") or {}).items():
1786
  binding = cell.get("binding") or {}
1787
- if cell.get("kind") != "bound" or not _has_cell_inputs(binding):
 
 
 
1788
  continue
1789
  try:
1790
- result = _resolve_cell_binding(session, binding, book, source.get("id"))
 
1791
  cell.update(value=result["value"], display="" if result["value"] is None else str(result["value"]), state="fresh")
1792
  cell["refreshed_at"] = _now_iso()
1793
  if result.get("limits"):
@@ -1795,6 +1959,8 @@ def _recomputed_tab_cells(session: Session, workbook_id: str, tab_ref: str, *, c
1795
  else:
1796
  cell.pop("limits", None)
1797
  cell.pop("refusal", None)
 
 
1798
  except HTTPException as exc:
1799
  info = ((exc.detail or {}).get("error") or {})
1800
  cell.update(_binding_refusal(
@@ -1804,7 +1970,8 @@ def _recomputed_tab_cells(session: Session, workbook_id: str, tab_ref: str, *, c
1804
  ending_cells.add((source.get("id"), addr))
1805
  refreshed_entries.append((str(source.get("id")) + "!" + addr,
1806
  _binding_scope_key(session, str(binding.get("database") or ""), binding),
1807
- {"tokens": {}, "context": _refresh_context(book, binding, context_memo),
 
1808
  "cell": {k: cell[k] for k in ("value", "display", "state", "refreshed_at", "limits", "refusal")
1809
  if k in cell}}))
1810
  if ending_cells:
@@ -1813,6 +1980,11 @@ def _recomputed_tab_cells(session: Session, workbook_id: str, tab_ref: str, *, c
1813
  {c.get("binding", {}).get("database") for t in book.get("sheets", [])
1814
  for c in t.get("cells", {}).values() if c.get("kind") == "bound"})
1815
  _recompute_formulas(book, fr, sheet_mod)
 
 
 
 
 
1816
  found = sheets.find_tab(book, tab_ref)
1817
  out = {}
1818
  for addr, cell in (found.get("cells") or {}).items():
@@ -1986,12 +2158,25 @@ def unshare_sheet(workbook_id: str, who: str, session: Session = Depends(require
1986
 
1987
 
1988
  @router.post("/sheets/{workbook_id}/refresh")
1989
- def refresh_sheet(workbook_id: str, session: Session = Depends(require_session)):
 
1990
  """C2/R4: Refresh resolves only databases whose token or binding inputs changed.
1991
 
1992
  Viewers refresh their own server cache. Only editors save the workbook document.
1993
  One token is read per distinct database on the unchanged path, regardless of cell count.
 
 
 
 
1994
  """
 
 
 
 
 
 
 
 
1995
  from copy import deepcopy
1996
  from core import sheets, sheet_binding
1997
  from modules import sheet as sheet_mod
@@ -2019,10 +2204,12 @@ def refresh_sheet(workbook_id: str, session: Session = Depends(require_session))
2019
 
2020
  def input_databases(binding, tab_ref):
2021
  nonlocal input_graph
2022
- nodes, _ = _fe_parts((binding.get("scope") or {}).get("filter"))
2023
- related_nodes, _ = _fe_parts((binding.get("related") or {}).get("filter"))
 
 
2024
  refs = [str((node.get(key) or {}).get("ref") or "")
2025
- for node in sheet_mod.filter_cell_nodes([*nodes, *related_nodes])
2026
  for key in ("valueRef", "value2Ref") if key in node]
2027
  period = binding.get("period") or {}
2028
  if period.get("ref"):
@@ -2032,15 +2219,25 @@ def refresh_sheet(workbook_id: str, session: Session = Depends(require_session))
2032
  tabs = book.get("sheets") or []
2033
  names = _tab_scope_names(tabs)
2034
  if input_graph is None:
2035
- documents = {fr.a1_key(addr, name): cell for tab, name in zip(tabs, names)
2036
- for addr, cell in (tab.get("cells") or {}).items()}
2037
- engine = fr.SheetCalculation({key: fr._document_spec(cell, key.rpartition("!")[0])
2038
- for key, cell in documents.items()}, sheets=names,
2039
- named_ranges=book.get("namedRanges"), date_system=book.get("dateSystem", "1900"))
2040
- input_graph = documents, engine
 
 
 
 
 
2041
  documents, engine = input_graph
2042
  home = _resolve_home_tab(book, tabs, names, tab_ref)
2043
- pending, visited, deps = [_asked_key(ref, home, fr) for ref in refs], set(), set()
 
 
 
 
 
2044
  while pending:
2045
  key = pending.pop()
2046
  if key in visited:
@@ -2048,7 +2245,14 @@ def refresh_sheet(workbook_id: str, session: Session = Depends(require_session))
2048
  visited.add(key)
2049
  cell = documents.get(key) or {}
2050
  if cell.get("kind") == "bound":
2051
- deps.add(str((cell.get("binding") or {}).get("database") or ""))
 
 
 
 
 
 
 
2052
  pending.extend(engine.consumed.get(key, ()))
2053
  return deps
2054
 
@@ -2069,16 +2273,15 @@ def refresh_sheet(workbook_id: str, session: Session = Depends(require_session))
2069
 
2070
  def dependencies(db, binding, tab_ref):
2071
  deps = {db}
2072
- if binding.get("related"):
2073
- key = (db, binding["related"].get("via"))
 
 
 
 
2074
  if key not in related_memo:
2075
- related_memo[key] = _binding_related_entry(session, db, binding)
2076
  deps.add(related_memo[key]["target"])
2077
- nodes, _ = _fe_parts((binding.get("scope") or {}).get("filter"))
2078
- linked = sheet_mod.linked_filter_ids(nodes)
2079
- if linked:
2080
- names = {sheet_mod.linked_field_parts(key)[0] for key in linked}
2081
- deps.update(b["source"] for b in _linked_customer_bridges(db) if b["key"] in names)
2082
  deps.update(input_databases(binding, tab_ref))
2083
  return deps
2084
 
@@ -2098,11 +2301,16 @@ def refresh_sheet(workbook_id: str, session: Session = Depends(require_session))
2098
  hit = cache.get("refreshCells", {}).get(str(tab_ref) + "!" + addr, {}).get(scope_key)
2099
  sources = {db: token(db) for db in dependencies(db, binding, tab_ref)}
2100
  if (hit and all(sources.values()) and hit.get("tokens") == sources
2101
- and hit.get("context") == _refresh_context(book, binding, context_memo)
 
2102
  and hit["cell"].get("state") == "fresh"):
2103
  return hit["cell"]
2104
  return None
2105
 
 
 
 
 
2106
  for tab in book.get("sheets") or []:
2107
  for addr, cell in (tab.get("cells") or {}).items():
2108
  if cell.get("kind") != "bound" or _is_view_binding(cell):
@@ -2110,8 +2318,11 @@ def refresh_sheet(workbook_id: str, session: Session = Depends(require_session))
2110
  binding = cell.get("binding") or {}
2111
  db = str(binding.get("database") or "")
2112
  cell_key = str(tab.get("id")) + "!" + addr
 
 
2113
  scope_key = _binding_scope_key(session, db, binding)
2114
- context = _refresh_context(book, binding, context_memo)
 
2115
  hit = cache.get("refreshCells", {}).get(cell_key, {}).get(scope_key)
2116
  source_tokens = {}
2117
  try:
@@ -2131,12 +2342,16 @@ def refresh_sheet(workbook_id: str, session: Session = Depends(require_session))
2131
  status.setdefault(db, "skipped")
2132
  continue
2133
  status[db] = "resolved"
 
2134
  previous = getattr(session, "_sheet_refresh_input", None)
2135
  session._sheet_refresh_input = cached_input
2136
  try:
2137
  out = _resolve_cell_binding(session, binding, book, tab.get("id"), view_memo=view_memo)
2138
  finally:
2139
  session._sheet_refresh_input = previous
 
 
 
2140
  limits = list(out.get("limits") or [])
2141
  if not all(source_tokens.values()):
2142
  limits.append({"code": "change_token_unavailable",
@@ -2147,8 +2362,10 @@ def refresh_sheet(workbook_id: str, session: Session = Depends(require_session))
2147
  "refreshed_at": stamp}
2148
  if limits:
2149
  snapshot["limits"] = limits
2150
- except HTTPException as exc:
2151
- info = ((exc.detail or {}).get("error") or {})
 
 
2152
  snapshot = _binding_refusal(info.get("code", "sheet_unresolved"),
2153
  info.get("message", "This cell could not be refreshed."))
2154
  status[db] = "refused"
@@ -2179,9 +2396,15 @@ def refresh_sheet(workbook_id: str, session: Session = Depends(require_session))
2179
  tab_ref, addr = cell_key.split("!", 1)
2180
  _snapshot_cell(sheets.find_tab(book, tab_ref)["cells"][addr], entry["cell"])
2181
 
 
2182
  started = perf_counter()
2183
  _recompute_formulas(book, fr, sheet_mod)
2184
  recalc_ms = round((perf_counter() - started) * 1000.0, 1)
 
 
 
 
 
2185
  if editable:
2186
  try:
2187
  saved = sheets.put_workbook(store, workbook_id,
@@ -2196,6 +2419,13 @@ def refresh_sheet(workbook_id: str, session: Session = Depends(require_session))
2196
  saved["refreshedFor"] = str(session.uname or "")
2197
  if entries:
2198
  sheet_binding.save_refresh_cache(store, workbook_id, entries, status)
 
 
 
 
 
 
 
2199
  return {"workbook": _response_workbook(saved), "recalc_ms": recalc_ms,
2200
  "binding_refresh": {"databases": status, "token_reads": token_reads}}
2201
 
@@ -2341,6 +2571,10 @@ _METRIC_COLUMN_CREATE_LOCK = threading.RLock()
2341
  def _forget_sheet_database(session, database):
2342
  """A field write invalidates both request-local schema and materialised-column memos."""
2343
  getattr(session, "_sheet_assemblies", {}).pop(database, None)
 
 
 
 
2344
  mirrors = getattr(session, "_sheet_mirrors", {})
2345
  for slot in list(mirrors):
2346
  if slot[0] == database:
@@ -2467,49 +2701,361 @@ def _ending_window(session, binding, *, book=None, tab=None, today=None, seen=()
2467
  return sheet_mod.period_window(period, today, cell_date=date)
2468
 
2469
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2470
  def _read_binding_cell(session, ref, *, book=None, tab=None, today=None, seen=(),
2471
- code="ending_cell_unreadable"):
2472
- """Shared cell read for Ending and Filter, including scoped bound and formula inputs."""
 
 
 
 
 
2473
  from modules import sheet as sheet_mod
2474
  import formula_runtime as fr
2475
 
2476
  def refuse(reason):
2477
  raise sheet_mod.SheetRefusal(code, ref or "(blank)", reason)
2478
 
 
 
 
2479
  if not re.fullmatch(r"(?:(?:'([^']|'')+'|[^'!]+)!)?\$?[A-Za-z]{1,3}\$?[1-9][0-9]{0,6}", ref):
2480
  refuse("its reference is invalid.")
2481
  if book is None:
2482
  refuse("the source workbook is unavailable.")
2483
  tabs = book.get("sheets") or []
2484
- names = _tab_scope_names(tabs)
 
 
 
 
 
2485
  home = _resolve_home_tab(book, tabs, names, tab)
2486
  key = _asked_key(ref, home, fr)
2487
  if key in seen:
2488
  refuse("the reference is circular.")
2489
- cells = {fr.a1_key(addr, name): cell for source, name in zip(tabs, names)
2490
- for addr, cell in (source.get("cells") or {}).items()}
 
2491
  cell = cells.get(key)
2492
  if not isinstance(cell, dict) or cell.get("kind") == "empty":
2493
  refuse("the cell or its sheet is missing or blank.")
2494
  if cell.get("kind") == "bound":
2495
  try:
2496
- lookup = getattr(session, "_sheet_refresh_input", None)
2497
- cached = lookup(key) if lookup is not None else None
2498
- value = (cached["value"] if cached is not None else _resolve_binding(
2499
- session, cell.get("binding"), today=today, book=book,
2500
- tab=key.rpartition("!")[0], ending_seen=(*seen, key))["value"])
2501
  except HTTPException as exc:
2502
  reason = ((exc.detail or {}).get("error") or {}).get("message", "its source is refused.")
2503
  refuse(reason)
2504
  elif cell.get("kind") == "formula":
2505
  # The existing formula graph discovers actual inputs, including dynamic references.
2506
  # A database input starts refused, so a cached value can never bypass this reader's wall.
2507
- bound = {k: c for k, c in cells.items() if c.get("kind") == "bound"}
2508
- specs = {k: ({"refused": "ending_input_pending"} if k in bound
2509
- else fr._document_spec(c, k.rpartition("!")[0])) for k, c in cells.items()}
2510
- engine = fr.SheetCalculation(specs, sheets=names, today=today,
2511
- named_ranges=book.get("namedRanges"), date_system=book.get("dateSystem", "1900"))
2512
- attempted, failures = set(), {}
 
 
 
 
2513
  while True:
2514
  dependencies, pending = set(), [key]
2515
  while pending:
@@ -2525,11 +3071,8 @@ def _read_binding_cell(session, ref, *, book=None, tab=None, today=None, seen=()
2525
  for dependency in needed:
2526
  attempted.add(dependency)
2527
  try:
2528
- lookup = getattr(session, "_sheet_refresh_input", None)
2529
- result = lookup(dependency) if lookup is not None else None
2530
- if result is None:
2531
- result = _resolve_binding(session, bound[dependency].get("binding"), today=today,
2532
- book=book, tab=dependency.rpartition("!")[0], ending_seen=(*seen, key, dependency))
2533
  changes[dependency] = {"value": result["value"]}
2534
  except HTTPException as exc:
2535
  info = ((exc.detail or {}).get("error") or {})
@@ -2551,6 +3094,13 @@ def _read_binding_cell(session, ref, *, book=None, tab=None, today=None, seen=()
2551
  if cell.get("state") == "stale":
2552
  refuse("the cell needs to be refreshed.")
2553
  value = cell.get("value", cell.get("text"))
 
 
 
 
 
 
 
2554
  return value
2555
 
2556
 
@@ -2558,7 +3108,8 @@ def _filter_cell_values(session, nodes, *, book=None, tab=None, today=None, seen
2558
  from modules import sheet as sheet_mod
2559
  try:
2560
  return sheet_mod.resolve_filter_cells(nodes, lambda ref: _read_binding_cell(
2561
- session, ref, book=book, tab=tab, today=today, seen=seen, code="filter_cell_unreadable"))
 
2562
  except sheet_mod.SheetRefusal as exc:
2563
  raise err(400, exc.code, exc.message)
2564
 
@@ -2602,25 +3153,14 @@ def _resolve_cell_binding(session, binding, book, tab, **kwargs):
2602
  return _resolve_binding(session, binding, **kwargs)
2603
 
2604
 
2605
- def _binding_related_entry(session, database, binding):
2606
  """Resolve one authored edge key against the permission-scoped declarations."""
2607
- if "related" not in binding:
2608
- return None
2609
- from core import perm_scope, relations, user_tables
2610
- from routes_grid import _source_fields
2611
-
2612
- fields = _source_fields(session, database)
2613
- defs = {}
2614
- for key, definition in (user_tables.all_defs(st=session.runtime) or {}).items():
2615
- if perm_scope.nav_may_open(session.user, key, st=session.runtime):
2616
- declared = definition.get("fields") or []
2617
- hidden = perm_scope.hidden_keys(session.user, key, declared, st=session.runtime)
2618
- defs[key] = {**definition, "fields": [f for f in declared if f.get("key") not in hidden]}
2619
- entries = relations.related(database, fields=fields, defs=defs)
2620
- name = binding["related"]["via"]
2621
  matches = [entry for entry in entries if entry["key"] == name]
2622
  via = matches[0]["via"] if len(matches) == 1 else {"key": name}
2623
- entry = relations.find(database, via, fields=fields, defs=defs)
 
2624
  if entry is None:
2625
  candidates = [item["key"] for item in relations.via_candidates(entries, via)
2626
  if perm_scope.nav_may_open(session.user, item["target"], st=session.runtime)]
@@ -2640,23 +3180,20 @@ def _binding_related_entry(session, database, binding):
2640
  declaration = next((f for f in defs.get(target, {}).get("fields", [])
2641
  if f.get("key") == entry["via"].get("field")), {})
2642
  link = declaration.get("link") or {}
2643
- if link.get("through") or link.get("selectionMode") == "manual":
 
2644
  raise err(409, "relation_source_unresolvable",
2645
  f"Relationship '{name}' does not declare a direct column join. "
2646
  "Choose a relationship with one readable column on each database.")
2647
  return entry
2648
 
2649
 
2650
- def _binding_related_query(session, binding, entry, cur, spec, columns, *, window=None,
2651
- today=None, book=None, tab=None, ending_seen=()):
2652
  """Materialise the far side on the near cursor, then hand metadata to the compiler."""
2653
- if entry is None:
2654
- return None
2655
  from core import perm_scope, relations
2656
  from modules import sheet as sheet_mod
2657
 
2658
- database, target = binding["database"], entry["target"]
2659
- name = binding["related"]["via"]
2660
  local_key, remote_key = entry["via"].get("from"), entry["via"].get("on")
2661
 
2662
  def map_join(key, table, source_spec, cols, side):
@@ -2682,11 +3219,7 @@ def _binding_related_query(session, binding, entry, cur, spec, columns, *, windo
2682
  wall = ((perm_scope.entry(session.user, target) or {}).get("filter")
2683
  if perm_scope.row_scope_applies(session.user, target) else None)
2684
  wall_nodes, wall_conj = _fe_parts(wall)
2685
- scope_nodes, scope_conj = _fe_parts(binding["related"]["filter"])
2686
- scope_nodes = _filter_cell_values(session, scope_nodes, book=book, tab=tab,
2687
- today=today, seen=ending_seen)
2688
- needed = (sheet_mod.filter_columns_named(scope_nodes)
2689
- | sheet_mod.filter_columns_named(wall_nodes) | {remote_key})
2690
  try:
2691
  rcur, rspec, rcols = _mirror(session)(target, metric_window=window, needed=needed)
2692
  except HTTPException as exc:
@@ -2703,13 +3236,69 @@ def _binding_related_query(session, binding, entry, cur, spec, columns, *, windo
2703
  hidden = perm_scope.hidden_keys(session.user, target, rspec.get("fields") or [],
2704
  st=session.runtime)
2705
  rcols = {key: value for key, value in rcols.items() if key not in hidden}
 
 
 
 
 
 
 
 
 
 
 
 
 
2706
  map_join(remote_key, target, rspec, rcols, "target")
2707
- depends = sheet_mod.filter_columns_named(scope_nodes)
 
 
 
2708
  return {"via": name, "database": target, "local_key": local_key, "remote_key": remote_key,
2709
  "spec": rspec, "columns": rcols, "where": entry.get("where"),
2710
- "scope_nodes": scope_nodes, "scope_conj": scope_conj,
2711
  "wall_nodes": wall_nodes, "wall_conj": wall_conj,
2712
- "limits": [row for key, row in (rspec.get("unbounded") or {}).items() if key in depends]}
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2713
 
2714
 
2715
  def _resolve_binding(session: Session, raw_binding, *, today=None, view_memo=None,
@@ -2760,12 +3349,7 @@ def _resolve_binding(session: Session, raw_binding, *, today=None, view_memo=Non
2760
  wall_tree = ((perm_scope.entry(session.user, database) or {}).get("filter")
2761
  if perm_scope.row_scope_applies(session.user, database) else None)
2762
  wall_nodes, wall_conj = _fe_parts(wall_tree)
2763
- scope = binding["scope"]
2764
- if scope.get("view_id"):
2765
- scope_nodes, scope_conj = _view_filter(session, database, scope["view_id"],
2766
- scope.get("view_name", ""), memo=view_memo)
2767
- else:
2768
- scope_nodes, scope_conj = _fe_parts(scope.get("filter"))
2769
  scope_nodes = _filter_cell_values(session, scope_nodes, book=book, tab=tab,
2770
  today=today, seen=ending_seen)
2771
  needed = (sheet_mod.filter_columns_named(scope_nodes)
@@ -2773,12 +3357,9 @@ def _resolve_binding(session: Session, raw_binding, *, today=None, view_memo=Non
2773
  if binding["aggregate"].get("field"):
2774
  needed.add(str(binding["aggregate"]["field"]))
2775
 
2776
- related_entry = _binding_related_entry(session, database, binding)
2777
- if related_entry:
2778
- needed.add(related_entry["via"].get("from"))
2779
  cur, spec, cols = _mirror(session)(database, metric_window=window, needed=needed)
2780
  cols = dict(cols)
2781
- scope_nodes = _linked_filter_scope(session, database, scope_nodes, cur, spec, cols, today=today)
2782
 
2783
  # β›” THE COLUMN IS CHECKED BEFORE THE SQL IS BUILT (W53-T13). Now that every tenant column is a
2784
  # real column of the materialised table, a `sum` over a text column reaches DuckDB as a binder
@@ -2817,9 +3398,7 @@ def _resolve_binding(session: Session, raw_binding, *, today=None, view_memo=Non
2817
  # spec's column expressions; applying it a second time would double-count the exclusion and
2818
  # break every joined spec, whose partner column is `sol.order_partner_id` or `aml.partner_id`.
2819
  try:
2820
- related = _binding_related_query(
2821
- session, binding, related_entry, cur, spec, cols, window=window, today=today,
2822
- book=book, tab=tab, ending_seen=ending_seen)
2823
  sql, params, limits = sheet_mod.build_aggregate_query(
2824
  spec, binding, cols, scope_nodes=scope_nodes, scope_conj=scope_conj,
2825
  wall_nodes=wall_nodes, wall_conj=wall_conj, today=today, related=related)
@@ -2922,6 +3501,11 @@ def preview_cell(workbook_id: str, addr: str, body: dict = Body(default=None),
2922
  binding editor show a number before anyone commits the cell. R10 may create a missing Metric
2923
  field through the database's ordinary field writer; reusing a column needs no field write.
2924
  """
 
 
 
 
 
2925
  raw = dict(body or {})
2926
  binding = raw.get("binding") or {}
2927
  book = None
@@ -3029,6 +3613,11 @@ def compute_cell(workbook_id: str, addr: str, body: dict = Body(default=None),
3029
  ⚠ `rev` BUMPS, BECAUSE THIS PERSISTS. It goes back in the answer and the client adopts it, or
3030
  its next PATCH would carry a revision the server has already moved past.
3031
  """
 
 
 
 
 
3032
  from core import sheets
3033
 
3034
  try:
@@ -3162,9 +3751,10 @@ def view_rows(workbook_id: str, body: object = Body(default=None),
3162
  if perm_scope.row_scope_applies(session.user, database) else None)
3163
  wall_nodes, wall_conj = _fe_parts(wall_tree)
3164
  scope_nodes, scope_conj = _fe_parts(binding.get("filter"))
 
3165
  # W62 C2: a filter-only field obeys the same grants; linked fields have their own resolver wall.
3166
  for key in sorted(sheet_mod.filter_columns_named(scope_nodes)):
3167
- if not key.startswith(sheet_mod.LINKED_FILTER_PREFIX) and key not in visible:
3168
  return JSONResponse(status_code=400, content={
3169
  "error": "view_field_unknown", "message": f"Field {key} is not available in this database."})
3170
  today = date.today()
@@ -3173,10 +3763,13 @@ def view_rows(workbook_id: str, body: object = Body(default=None),
3173
  window = _ending_window(session, binding, book=book, today=today)
3174
  needed = (set(requested) | sorted_keys | sheet_mod.filter_columns_named(scope_nodes)
3175
  | sheet_mod.filter_columns_named(wall_nodes))
 
3176
  cur, spec, cols = _mirror(session)(database, metric_window=window, needed=needed)
3177
  cols = dict(cols)
3178
- scope_nodes = _linked_filter_scope(session, database, scope_nodes, cur, spec, cols, today=today)
3179
  for key in dict.fromkeys([*requested, *sorted(sorted_keys | sheet_mod.filter_columns_named(scope_nodes))]):
 
 
3180
  if key in hidden or not (cols.get(key) or {}).get("sql"):
3181
  return JSONResponse(status_code=400, content={
3182
  "error": "view_field_unknown", "message": f"Field {key} is not available in this database."})
@@ -3186,7 +3779,8 @@ def view_rows(workbook_id: str, body: object = Body(default=None),
3186
  raise ValueError("unapplied row wall")
3187
  count_sql, params, limits = sheet_mod.build_aggregate_query(
3188
  spec, {"aggregate": {"fn": "count"}}, cols, scope_nodes=scope_nodes,
3189
- scope_conj=scope_conj, wall_nodes=wall_nodes, wall_conj=wall_conj, today=today)
 
3190
  if limits:
3191
  raise ValueError("unapplied view filter")
3192
  except sheet_mod.SheetRefusal as exc:
@@ -3227,8 +3821,20 @@ def view_rows(workbook_id: str, body: object = Body(default=None),
3227
 
3228
 
3229
  @router.get("/sheets/{workbook_id}/cells/{addr}/drill")
3230
- def drill_cell(workbook_id: str, addr: str, session: Session = Depends(require_session)):
3231
- """`{database, view_id?, filter?, record_ids, count}` for `LinkGridModal` (R6, R13, R17).
 
 
 
 
 
 
 
 
 
 
 
 
3232
 
3233
  β›” A FORMULA CELL REFUSES BY NAME AND DOES NOT DRILL TO ONE OPERAND (R17). Picking an input
3234
  would be a guess presented as an answer: `=B4-B8` has two, and neither is what the person
@@ -3249,10 +3855,18 @@ def drill_cell(workbook_id: str, addr: str, session: Session = Depends(require_s
3249
 
3250
  key = str(addr or "").replace("$", "").upper()
3251
  cell = None
3252
- for tab in book.get("sheets") or ():
3253
- if key in (tab.get("cells") or {}):
3254
- cell = tab["cells"][key]
3255
- break
 
 
 
 
 
 
 
 
3256
  if not isinstance(cell, dict):
3257
  raise err(404, "cell_missing", f"Cell {key} is not filled in.")
3258
  if cell.get("kind") == "formula":
@@ -3282,11 +3896,7 @@ def drill_cell(workbook_id: str, addr: str, session: Session = Depends(require_s
3282
  wall_nodes, wall_conj = _fe_parts(wall_tree)
3283
 
3284
  scope = binding["scope"]
3285
- if scope.get("view_id"):
3286
- scope_nodes, scope_conj = _view_filter(session, database, scope["view_id"],
3287
- scope.get("view_name", ""))
3288
- else:
3289
- scope_nodes, scope_conj = _fe_parts(scope.get("filter"))
3290
 
3291
  # ⚠ THE SAME PERIOD THE NUMBER WAS COMPUTED AT (W53-T13). The drill probes `COUNT(*)` and never
3292
  # reads the bound column, but its FILTER may name a Metric column, and a mirror opened at a
@@ -3297,25 +3907,20 @@ def drill_cell(workbook_id: str, addr: str, session: Session = Depends(require_s
3297
  # window is kept UNCONDITIONALLY, exactly as W53-T13 wrote it, and the saving is taken from
3298
  # the column set instead, where the time actually is.
3299
  try:
3300
- scope_nodes = _filter_cell_values(session, scope_nodes, book=book, tab=tab.get("id"))
3301
- window = _ending_window(session, binding, book=book, tab=tab.get("id"))
3302
  except sheet_mod.SheetRefusal as exc:
3303
  raise err(400, exc.code, exc.message)
3304
- related_entry = _binding_related_entry(session, database, binding)
3305
  needed = (sheet_mod.filter_columns_named(scope_nodes)
3306
  | sheet_mod.filter_columns_named(wall_nodes))
3307
- if related_entry:
3308
- needed.add(related_entry["via"].get("from"))
3309
  cur, spec, cols = _mirror(session)(
3310
  database, metric_window=window,
3311
  needed=needed)
3312
  cols = dict(cols)
3313
- scope_nodes = _linked_filter_scope(session, database, scope_nodes, cur, spec, cols)
3314
 
3315
  try:
3316
- related = _binding_related_query(
3317
- session, binding, related_entry, cur, spec, cols, window=window,
3318
- book=book, tab=tab.get("id"))
3319
  count_sql, count_params, limits = sheet_mod.build_aggregate_query(
3320
  spec, {"aggregate": {"fn": "count"}}, cols, scope_nodes=scope_nodes,
3321
  scope_conj=scope_conj, wall_nodes=wall_nodes, wall_conj=wall_conj, related=related)
@@ -3330,10 +3935,21 @@ def drill_cell(workbook_id: str, addr: str, session: Session = Depends(require_s
3330
  "connection. Edit the scope or ask an administrator to update the restriction.")
3331
  total = (cur.execute(count_sql, count_params).fetchone() or [0])[0]
3332
 
 
 
 
 
 
 
 
 
 
 
 
3333
  record_ids = [int(r[0]) for r in cur.execute(ids_sql, ids_params).fetchall() if r and r[0]]
3334
 
3335
  out = {"database": database, "record_ids": record_ids, "count": int(total or 0),
3336
- "limits": limits}
3337
  if scope.get("view_id"):
3338
  out["view_id"] = scope["view_id"]
3339
  else:
@@ -3442,6 +4058,11 @@ def resolve_bindings(workbook_id: str, body: dict = Body(default=None),
3442
  database work, and a batch's cost is dominated by the number of distinct (database, period)
3443
  pairs in it rather than by the number of cells.
3444
  """
 
 
 
 
 
3445
  from core import sheets
3446
  import formula_runtime as fr
3447
 
 
35
  (standing rule 2), and each names both the cause and the subject.
36
  """
37
 
38
+ import contextlib
39
  import hashlib
40
  import json
41
  import os
 
116
  # ⚠ `TABLES` ITSELF IS UNTOUCHED β€” it has other consumers (`declared_schema_fields`,
117
  # `_odoo_sources`' `register_connected`) that must keep seeing exactly eight rows.
118
  is_registry_grid = fields is None and table_key in rel._REGISTRY_GRID_FIELDS
119
+ is_shipments = table_key == rel.SHIPMENTS_KEY
120
+ # ⭐⭐ S66-T43 β€” THE THIRD ROSTER IS THE READ-THROUGH REGISTRY ITSELF. Lane G made every
121
+ # windowed Amazon table a `user_tables` stub served from the mirror through a provider in
122
+ # `routes_odoo_tables._sources()`, the same registry the Data grid's rows door reads. A key
123
+ # found there is exactly as connected as one in `TABLES`, and its FIELDS are the stub's own
124
+ # (`defn["fields"]`), which is where the rows door takes them from too.
125
+ # ⚠ `require_amazon_mirror` runs FIRST for an Amazon key: it admits a stub on a cold
126
+ # registry and refuses 503 with cause and fix when the mirror has not loaded the table,
127
+ # instead of this door calling a real Amazon table "not connected".
128
+ # ⚠ A STORED Amazon snapshot (under `MAX_ROWS`, e.g. listings) is served from the mirror
129
+ # too: its stored ids ARE the mirror's (`amazon_relational._read` stores `grid_source`'s
130
+ # id), and a relation from one Amazon table to another must resolve on ONE cursor
131
+ # (`_binding_related_query` refuses `rcur is not cur`), so both paths agree.
132
+ connector_defn = None
133
+ if fields is None and not is_registry_grid and not is_shipments:
134
+ connector_defn = _connector_definition(session, table_key)
135
+ if connector_defn is None:
136
+ raise err(404, "not_connected", f"{table_key} is not a connected Odoo database.")
137
+ fields = list(connector_defn.get("fields") or [])
138
+ if not fields:
139
+ raise err(409, "schema_not_loaded",
140
+ f"{table_key} has no saved fields yet. Sync its connector first.")
141
  try:
142
+ # The Amazon mirror opens without Odoo's ready() gate, exactly as the rows door does.
143
+ if str(table_key).startswith("ut_amzn_"):
144
+ # ONE Amazon cursor per request: `datastore.ro_cursor()` opens a new one per call,
145
+ # and a related condition joins two Amazon tables on the same cursor.
146
+ cur, mismatch = getattr(session, "_sheet_amzn_cursor", None) or (None, None)
147
+ if cur is None:
148
+ cur, mismatch = rot._mirror_cur(session, table_key)
149
+ if cur is not None:
150
+ session._sheet_amzn_cursor = (cur, None)
151
+ else:
152
+ cur, mismatch = rot._mirror_cur(session)
153
  except RuntimeError as exc:
154
  raise err(503, "store_not_ready", str(exc))
155
  if mismatch:
156
  raise err(409, "cross_tenant_store", mismatch)
157
+ if is_shipments:
158
+ from core import user_tables, perm_scope
159
+ from modules import sheet as sheet_mod
160
+ memo = getattr(session, "_sheet_stored_mirrors", None)
161
+ if memo is None:
162
+ memo = session._sheet_stored_mirrors = {}
163
+ if table_key not in memo:
164
+ stored = (user_tables.get(table_key, st=session.runtime) or {}).get("rows") or {}
165
+ # The store keeps records as {row id: row}. Iterating that dict as a list yields only
166
+ # the ids, which materialises an EMPTY table and answers every cell blank.
167
+ rows = [row for row in (stored.values() if isinstance(stored, dict) else stored)
168
+ if isinstance(row, dict)]
169
+ if not rows:
170
+ raise err(409, "sheet_database_empty",
171
+ f"{table_key} has no stored shipment rows. Sync shipments before binding this cell.")
172
+ fields = rel.shipment_fields()
173
+ hidden = perm_scope.hidden_keys(session.user, table_key, fields, st=session.runtime)
174
+ fields = [field for field in fields if field["key"] not in hidden]
175
+ spec_fields = sheet_mod.mirror_fields(fields)
176
+ spec = rel.registry_grid_spec(cur, table_key, spec_fields, rows,
177
+ id_key="odoo_id", bulk=True)
178
+ memo[table_key] = (cur, {**spec, "fields": fields}, rot._filter_columns(spec, spec_fields))
179
+ return memo[table_key]
180
  if is_registry_grid:
181
  # β›”β›” ONE ASSEMBLY PER REQUEST PER (DATABASE, WINDOW), NOT ONE PER CELL. `refresh_sheet`
182
  # resolves a binding CELL BY CELL, and this branch now costs a full `/workspace`
 
219
  memo[slot] = (None if want is None else set(want), made)
220
  return made
221
  spec = rot._source_for(cur, table_key)
222
+ if spec is None and connector_defn is not None and str(table_key).startswith("ut_amzn_"):
223
+ spec = _amazon_stored_spec(cur, table_key)
224
  if spec is None:
225
  raise err(409, "not_read_through",
226
  f"{table_key} has no read-through binding on this deployment, so a cell cannot "
 
235
  return _for
236
 
237
 
238
+ def _connector_definition(session: Session, table_key: str):
239
+ """The stored stub of a read-through CONNECTOR grid (S66-T43), or None when it is not one.
240
+
241
+ Only a key the read-through registry serves qualifies (`routes_odoo_tables._sources()`), or an
242
+ Amazon table the connector declares (`amazon_relational.SOURCE`, stored or read-through), so an
243
+ ordinary typed database still answers `not_connected`. Memoised on the Session for the reason
244
+ `_mirror` memoises: `resolve_bindings` asks once per CELL and the Session dies with the request.
245
+ ⚠ `scope_applied=True` because every Sheet read compiles the row wall into its own SQL.
246
+ """
247
+ import routes_odoo_tables as rot
248
+ import routes_tables
249
+ memo = getattr(session, "_sheet_connector_defs", None)
250
+ if memo is None:
251
+ memo = session._sheet_connector_defs = {}
252
+ if table_key in memo:
253
+ return memo[table_key]
254
+ key = str(table_key or "")
255
+ if not key.startswith("ut_"):
256
+ return None
257
+ defn = None
258
+ if key.startswith("ut_amzn_"):
259
+ import amazon_relational
260
+ if key not in amazon_relational.SOURCE:
261
+ return None
262
+ defn = routes_tables._defn_or_refuse(session, key, defs_only=True, scope_applied=True)
263
+ rot.require_amazon_mirror(key, session, defn)
264
+ elif key not in rot._sources():
265
+ return None
266
+ if defn is None:
267
+ defn = routes_tables._defn_or_refuse(session, key, defs_only=True, scope_applied=True)
268
+ memo[table_key] = defn
269
+ return defn
270
+
271
+
272
+ def _amazon_stored_spec(cur, table_key):
273
+ """The mirror binding for a STORED Amazon snapshot, or the 503 that names the missing load."""
274
+ import amazon_relational
275
+ spec = amazon_relational.grid_source(cur, table_key)
276
+ if spec.get("missing"):
277
+ raise err(503, "amazon_mirror_not_loaded", spec["missing"])
278
+ return spec
279
+
280
+
281
  def _registry_assembly(session: Session, table_key: str):
282
  """The `g` dict `/workspace` builds for one compiled registry grid, or a NAMED refusal.
283
 
 
353
  "shown rather than shown from an older set of columns. Try again in a moment.")
354
 
355
 
356
+ def _related_declarations(session, database):
357
+ """The same permission-scoped catalogue for reads, rewrites and the picker.
 
 
 
 
358
 
359
+ HOTFIX 2026-09-29: answered once per request inside `_sheet_request_scope`. Every
360
+ `related:` leaf used to re-read `user_tables.all_defs()` and re-scope every definition, once
361
+ per BINDING, so a Refresh of Daily Sales (~470 cells filtered on the customer's pricelist) read
362
+ the whole catalogue ~470 times (SDLC 67 S67-T21 finding).
363
+ """
364
+ memo = getattr(session, "_sheet_read_memo", None)
365
+ slot = ("related", database)
366
+ if isinstance(memo, dict) and slot in memo:
367
+ return memo[slot]
368
+ from core import perm_scope, relations, user_tables
369
+ from routes_grid import _source_fields
370
 
371
+ fields = _source_fields(session, database)
372
+ defs = {}
373
+ for key, definition in (user_tables.all_defs(st=session.runtime) or {}).items():
374
+ if perm_scope.nav_may_open(session.user, key, st=session.runtime):
375
+ declared = definition.get("fields") or []
376
+ hidden = perm_scope.hidden_keys(session.user, key, declared, st=session.runtime)
377
+ defs[key] = {**definition, "fields": [f for f in declared if f.get("key") not in hidden]}
378
+ entries = relations.related(database, fields=fields, defs=defs)
379
+ # S66-T13 / C2: `relations.bridge_entries` lists a bridge only from its SOURCE, so
380
+ # `ut_odoo_orders` has no edge back to `customer_data` although every one-release
381
+ # `linked:bridge_customer_orders:<field>` leaf stands on exactly that edge. Read from the
382
+ # target the columns swap, as `reverse_links` swaps them. A bridge carrying `where` is
383
+ # skipped as `linked:` always skipped it, because that predicate narrows this side's rows.
384
+ seen = {relations.edge_identity(entry) for entry in entries}
385
+ for bridge in relations.BRIDGES:
386
+ if bridge.get("target") != database or bridge.get("where"):
387
+ continue
388
+ entry = {"key": bridge["key"], "label": bridge.get("source_label") or bridge["source"],
389
+ "via": {"direction": "bridge", "key": bridge["key"],
390
+ "on": bridge["source_col"], "from": bridge["target_col"]},
391
+ "target": bridge["source"], "served": bool(bridge.get("served"))}
392
+ if relations.edge_identity(entry) not in seen:
393
+ seen.add(relations.edge_identity(entry))
394
+ entries.append(entry)
395
+ if isinstance(memo, dict):
396
+ memo[slot] = (fields, defs, entries)
397
+ return fields, defs, entries
398
+
399
+
400
+ def _legacy_linked_keys(database, entry):
401
+ """The one-release `linked:` bridge keys an offered relationship replaces."""
402
+ from core import relations
403
+ via = entry.get("via") or {}
404
+ return [bridge["key"] for bridge in relations.BRIDGES
405
+ if bridge.get("target") == database and bridge.get("source") == entry.get("target")
406
+ and bridge.get("target_col") == via.get("from")
407
+ and bridge.get("source_col") == via.get("on") and not bridge.get("where")]
408
+
409
+
410
+ def _related_condition_fields(session, database):
411
  from core import perm_scope
412
+ _require_database(session, database)
413
+ fields = list(_registry_assembly(session, database).get("fields") or [])
414
+ hidden = perm_scope.hidden_keys(session.user, database, fields, st=session.runtime)
415
+ return [field for field in fields if field.get("key") not in hidden
416
+ and field.get("type") not in {"formula", "link", "lookup", "rollup", "metric", "measure"}
417
+ and not field.get("measure")]
 
 
 
 
418
 
419
 
420
+ def _rewrite_related_filters(session, database, nodes):
421
+ """Read the one-release linked spelling as an ordinary related condition."""
422
+ from core import relations
423
  from modules import sheet as sheet_mod
424
+ if not sheet_mod.linked_filter_ids(nodes):
425
+ return nodes
426
+ _fields, _defs, entries = _related_declarations(session, database)
427
+ bridges = {b["key"]: b for b in relations.BRIDGES if b.get("target") == database}
428
+
429
+ def resolve(leaf):
430
+ bridge_key, field_key = sheet_mod.linked_field_parts(leaf["colId"])
431
+ bridge = bridges.get(bridge_key)
432
+ entry = next((entry for entry in entries if bridge
433
+ and entry["target"] == bridge["source"]
434
+ and entry["via"].get("from") == bridge.get("target_col")
435
+ and entry["via"].get("on") == bridge.get("source_col")
436
+ and not entry.get("where")), None)
437
+ if entry is None:
438
+ raise sheet_mod.SheetRefusal("linked_filter_unreadable", leaf["colId"],
439
+ "its declared relationship is unavailable.")
440
+ return {**leaf, "colId": f'related:{entry["key"]}:{field_key}'}
441
+
442
  try:
443
+ return sheet_mod.rewrite_linked_filters(nodes, resolve)
444
+ except sheet_mod.SheetRefusal as exc:
445
+ raise err(409, exc.code, exc.message)
 
 
 
446
 
447
 
448
  @router.get("/sheets/filter-fields/{database}")
449
  def linked_filter_fields(database: str, session: Session = Depends(require_session)):
450
+ """Offer ordinary conditions grouped by their readable related database."""
451
+ from core import perm_scope, relations
452
  database = _require_database(session, database)
453
+ _fields, defs, entries = _related_declarations(session, database)
454
+ offered, databases, seen = [], [], set()
455
+ for candidate in entries:
456
+ if not candidate.get("served") or not perm_scope.nav_may_open(
457
+ session.user, candidate["target"], st=session.runtime):
458
+ continue
459
+ identity = relations.edge_identity(candidate)
460
+ if identity in seen:
461
+ continue
462
  try:
463
+ entry = _binding_related_entry(session, database, candidate["key"])
464
+ fields = _related_condition_fields(session, entry["target"])
465
  except HTTPException as exc:
466
+ code = ((exc.detail or {}).get("error") or {}).get("code") if isinstance(exc.detail, dict) else None
467
+ if exc.status_code == 403 or code in {
468
+ "relation_source_unresolvable", "relation_target_unavailable", "unknown_relation"}:
469
  continue
470
  raise
471
+ if not entry["via"].get("from") or not entry["via"].get("on"):
472
  continue
473
+ seen.add(identity)
474
+ target = entry["target"]
475
+ from core import registry
476
+ from routes_nav import _read_nav_meta
477
+ topic = registry.BY_KEY.get(target) or {}
478
+ label = (_read_nav_meta(session.runtime).get(target, {}).get("name")
479
+ or defs.get(target, {}).get("label")
480
+ or topic.get("label") or target)
481
+ metadata = {"via": entry["key"], "database": target, "databaseLabel": label,
482
+ "relationLabel": entry.get("label") or entry["key"]}
483
+ databases.append(metadata)
484
+ # The panel maps a saved one-release `linked:` leaf onto its related key with these.
485
+ legacy = _legacy_linked_keys(database, entry)
486
  for field in fields:
487
+ field_label = field.get("label") or field["key"]
488
+ offered.append({**field, "key": f'related:{entry["key"]}:{field["key"]}',
489
+ "label": f'{label} via {metadata["relationLabel"]}: {field_label}',
490
+ "related": {**metadata, "field": field["key"], "fieldLabel": field_label,
491
+ "legacyKeys": [f"linked:{key}:{field['key']}"
492
+ for key in legacy]}})
493
+ return {"fields": offered, "databases": databases}
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
494
 
495
 
496
  def _registry_mirror(session: Session, cur, table_key: str, metric_window, needed=None):
 
534
  derived, bounded, limits = _metric_derived(session, table_key, g, metric_window)
535
  rows = aios_grid.rows_from_pool(g["rows_src"], fields, (g.get("ws") or {}).get("overlays"),
536
  derived=derived)
537
+ if table_key == "customer_data" and (needed is None or "agent_id" in needed):
538
+ from core import perm_scope
539
+ agent_field = {"key": "agent_id", "type": "int", "label": "Current agent ID"}
540
+ hidden = perm_scope.hidden_keys(session.user, table_key, [*fields, agent_field], st=session.runtime)
541
+ if "agent_id" not in hidden and "agent" not in hidden:
542
+ # Join only to already-scoped customer rows. pid is the actual res.partner id.
543
+ have = rel.columns(cur, "res_partner")
544
+ if "agent_id" in have:
545
+ agents = dict(cur.execute("SELECT id, agent_id FROM res_partner").fetchall())
546
+ rows = [{**row, "agent_id": agents.get(row.get("pid"))} for row in rows]
547
+ fields = [field for field in fields if field.get("key") != "agent_id"] + [agent_field]
548
+ elif needed is not None and "agent_id" in needed:
549
+ raise err(409, "related_invalid", "customer_data.agent_id is unavailable in the partner mirror.")
550
  # The columns this cell can read, in the assembly's own order. `registry_grid_spec` forces
551
  # `pid` in on its own, because `build_ids_query` needs `spec["id"]` to be a real column.
552
  kept = (fields if needed is None
 
914
  return HTTPException(status_code=409, detail={"error": body})
915
  if exc.code in _FORBIDDEN_CODES:
916
  return err(403, exc.code, exc.message)
917
+ if exc.code == "workbook_unavailable":
918
+ # The store did not answer; the workbook is not gone. 503 tells the client to retry.
919
+ return err(503, exc.code, exc.message)
920
  status = 404 if exc.code in _MISSING_CODES else 400
921
  return err(status, exc.code, exc.message)
922
 
 
1124
  related_nodes, _ = _fe_parts(related.get("filter") if isinstance(related, dict) else None)
1125
  nodes = [*nodes, *related_nodes]
1126
  # Linked values and related-table grants can change independently of the orders' cache key.
1127
+ if (sheet_mod.linked_filter_ids(nodes) or sheet_mod.related_filter_ids(nodes)
1128
+ or related is not None):
1129
  return True
1130
  return any(sheet_mod.filter_cell_nodes(nodes))
1131
 
1132
 
1133
+ def _refresh_context(book, binding, memo=None, *, tab=None, session=None):
1134
+ """Authored inputs and relative dates invalidate refresh, but never an open's snapshot.
1135
+
1136
+ HOTFIX 2026-09-29 (owner: "ONLY when we update ANY cells, its related cells also have to
1137
+ update"): with its `tab` and a request scope, a binding's inputs are the cells it actually
1138
+ READS, transitively (`_input_closure`), not the whole workbook. The whole-workbook fingerprint
1139
+ meant one edit anywhere invalidated every one of Daily Sales' ~470 bindings, so every Refresh
1140
+ after any edit re-resolved all of them. The tab list rides the fingerprint, so a rename or a
1141
+ reorder still invalidates everything. Without a scope the old whole-workbook reading stands.
1142
+ """
1143
  from core.sheet_binding import fingerprint
1144
  from datetime import date
1145
  payload = {}
1146
+ graph = None
1147
+ if session is not None and tab is not None and _has_cell_inputs(binding):
1148
+ import formula_runtime as fr
1149
+ graph = _read_graph(session, book, fr, None)
1150
+ if graph is not None:
1151
+ import formula_runtime as fr
1152
+ tabs = book.get("sheets") or []
1153
+ home = _resolve_home_tab(book, tabs, graph["names"], tab)
1154
+ engine = _read_engine(graph, book, fr, None)[0]
1155
+ closure = _input_closure(graph, engine, fr, _qualified_refs(binding, home, fr, graph))
1156
+ authored = {}
1157
+ for key in sorted(closure):
1158
+ cell = graph["cells"].get(key)
1159
+ authored[key] = None if not isinstance(cell, dict) else {
1160
+ k: cell.get(k) for k in ("kind", "text", "formula", "binding", "value")
1161
+ if k != "value" or cell.get("kind") in ("literal", "empty")}
1162
+ payload["inputs"] = fingerprint([[[t.get("id"), t.get("name")] for t in tabs], authored])
1163
+ elif _has_cell_inputs(binding):
1164
  if memo is None:
1165
  memo = {}
1166
  if "inputs" not in memo:
 
1271
  # and MCP read agree on stored cells, figures and formats` measured exactly that pair: D1 and
1272
  # F1 fresh 30.0 / 60.0 from MCP against stale-and-blank from the UI. A non-browser consumer has
1273
  # no client engine to cover for it, so the blank is what it would have shown.
1274
+ # HOTFIX 2026-09-29: a `refused` formula is a COMPUTED answer (an error is a result), so it no
1275
+ # longer forces a whole-workbook recompute on every tab read. Only a formula with no answer at
1276
+ # all (`stale`, or no value stored) does. One `#DIV/0!` anywhere used to cost every read ~1 s.
1277
  stale_formula = any(
1278
+ cell.get("kind") == "formula" and (cell.get("state") == "stale" or "value" not in cell)
1279
  for tab in (fresh.get("sheets") or []) for cell in (tab.get("cells") or {}).values())
1280
  if changed_values or stale_formula:
1281
  from modules import sheet as sheet_mod
 
1891
  A formula sitting on a tab nobody has open still shows its last answer until that tab is
1892
  opened or the workbook is refreshed, exactly as a stale database-bound cell does.
1893
  """
1894
+ with _sheet_request_scope(session):
1895
+ return _recomputed_tab_cells_scoped(session, workbook_id, tab_ref, changes=changes)
1896
+
1897
+
1898
+ def _recomputed_tab_cells_scoped(session: Session, workbook_id: str, tab_ref: str, *, changes=None):
1899
  from core import sheets
1900
  from modules import sheet as sheet_mod
1901
  import formula_runtime as fr
1902
+ from time import perf_counter
1903
 
1904
+ started = perf_counter()
1905
  try:
1906
  book = sheets.get_workbook(_store(session), workbook_id)
1907
  found = sheets.find_tab(book, tab_ref)
 
1915
  (found.get("cells") or {}).get(str(addr).strip().upper()) for addr in changes}
1916
  _recompute_formulas(book, fr, sheet_mod,
1917
  cache_key=(session.tenant, session.uname, workbook_id), changes=edits)
1918
+ # HOTFIX 2026-09-29 (owner: "we should only update or refresh cells that are affected by an
1919
+ # operation"): a commit re-resolves ONLY the database cells whose inputs read what it changed.
1920
+ # This loop used to resolve EVERY bound cell with a cell input on EVERY commit, whatever was
1921
+ # edited: ~470 SQL resolves (each rebuilding the whole formula graph) per keystroke on Daily
1922
+ # Sales. `_dependent_bound_keys` walks the formula graph forward from the edited cells.
1923
+ graph = _read_graph(session, book, fr, None)
1924
+ targets, mode = None, "all"
1925
+ if edits is not None and graph is not None:
1926
+ try:
1927
+ targets = _dependent_bound_keys(book, graph, fr, set(edits))
1928
+ # A bound cell written by this commit resolves too, as it always did.
1929
+ targets |= {key for key in edits if key in graph["bound"]}
1930
+ mode = "dependents"
1931
+ except Exception as exc: # noqa: BLE001
1932
+ # Unprovable dependents fall back to the old whole-workbook pass: slow, never wrong.
1933
+ try:
1934
+ import harness.telemetry as _tel
1935
+ _tel.error("api:sheet-dependents", exc)
1936
+ except Exception: # noqa: BLE001
1937
+ pass
1938
+ targets, mode = None, "all"
1939
  ending_cells = set()
1940
  refreshed_entries = []
1941
  context_memo = {}
1942
+ view_memo = {}
1943
+ names = _tab_scope_names(book.get("sheets") or [])
1944
+ for source, name in zip(book.get("sheets") or [], names):
1945
  for addr, cell in (source.get("cells") or {}).items():
1946
  binding = cell.get("binding") or {}
1947
+ if cell.get("kind") != "bound" or _is_view_binding(cell) or not _has_cell_inputs(binding):
1948
+ continue
1949
+ key = fr.a1_key(addr, name)
1950
+ if targets is not None and key not in targets:
1951
  continue
1952
  try:
1953
+ result = _resolve_cell_binding(session, binding, book, source.get("id"),
1954
+ view_memo=view_memo)
1955
  cell.update(value=result["value"], display="" if result["value"] is None else str(result["value"]), state="fresh")
1956
  cell["refreshed_at"] = _now_iso()
1957
  if result.get("limits"):
 
1959
  else:
1960
  cell.pop("limits", None)
1961
  cell.pop("refusal", None)
1962
+ if graph is not None:
1963
+ graph["bound_values"][key] = result
1964
  except HTTPException as exc:
1965
  info = ((exc.detail or {}).get("error") or {})
1966
  cell.update(_binding_refusal(
 
1970
  ending_cells.add((source.get("id"), addr))
1971
  refreshed_entries.append((str(source.get("id")) + "!" + addr,
1972
  _binding_scope_key(session, str(binding.get("database") or ""), binding),
1973
+ {"tokens": {}, "context": _refresh_context(book, binding, context_memo,
1974
+ tab=source.get("id"), session=session),
1975
  "cell": {k: cell[k] for k in ("value", "display", "state", "refreshed_at", "limits", "refusal")
1976
  if k in cell}}))
1977
  if ending_cells:
 
1980
  {c.get("binding", {}).get("database") for t in book.get("sheets", [])
1981
  for c in t.get("cells", {}).values() if c.get("kind") == "bound"})
1982
  _recompute_formulas(book, fr, sheet_mod)
1983
+ stats = _read_stats(session) or {}
1984
+ print(f"[sheet] commit wb={workbook_id} tab={tab_ref} edited={len(edits or {})} mode={mode} "
1985
+ f"bound_resolved={len(ending_cells)} engines={stats.get('engines', 0)} "
1986
+ f"input_hits={stats.get('input_hits', 0)} "
1987
+ f"ms={round((perf_counter() - started) * 1000.0)}", flush=True)
1988
  found = sheets.find_tab(book, tab_ref)
1989
  out = {}
1990
  for addr, cell in (found.get("cells") or {}).items():
 
2158
 
2159
 
2160
  @router.post("/sheets/{workbook_id}/refresh")
2161
+ def refresh_sheet(workbook_id: str, body: dict = Body(default=None),
2162
+ session: Session = Depends(require_session)):
2163
  """C2/R4: Refresh resolves only databases whose token or binding inputs changed.
2164
 
2165
  Viewers refresh their own server cache. Only editors save the workbook document.
2166
  One token is read per distinct database on the unchanged path, regardless of cell count.
2167
+
2168
+ HOTFIX 2026-09-29: an optional body `{"cells": ["<tab id>!<ADDR>", ...]}` refreshes ONLY
2169
+ those bound cells (the open-time repair of the few cells this reader has no answer for, and
2170
+ the cells a row or column edit left stale). Every other cell keeps its stored answer.
2171
  """
2172
+ wanted = (body or {}).get("cells") if isinstance(body, dict) else None
2173
+ only = ({str(item) for item in wanted if isinstance(item, str) and "!" in item}
2174
+ if isinstance(wanted, list) else None)
2175
+ with _sheet_request_scope(session):
2176
+ return _refresh_sheet_scoped(workbook_id, session, only=only)
2177
+
2178
+
2179
+ def _refresh_sheet_scoped(workbook_id: str, session: Session, *, only=None):
2180
  from copy import deepcopy
2181
  from core import sheets, sheet_binding
2182
  from modules import sheet as sheet_mod
 
2204
 
2205
  def input_databases(binding, tab_ref):
2206
  nonlocal input_graph
2207
+ binding = dict(binding)
2208
+ if "related" in binding:
2209
+ binding = sheet_mod.clean_binding(binding)
2210
+ nodes, _ = _binding_filter_nodes(session, binding["database"], binding, memo=view_memo)
2211
  refs = [str((node.get(key) or {}).get("ref") or "")
2212
+ for node in sheet_mod.filter_cell_nodes(nodes)
2213
  for key in ("valueRef", "value2Ref") if key in node]
2214
  period = binding.get("period") or {}
2215
  if period.get("ref"):
 
2219
  tabs = book.get("sheets") or []
2220
  names = _tab_scope_names(tabs)
2221
  if input_graph is None:
2222
+ shared = _read_graph(session, book, fr, None)
2223
+ if shared is not None:
2224
+ # The request's one engine (bound cells pending): its edges are the same edges.
2225
+ input_graph = shared["cells"], _read_engine(shared, book, fr, None)[0]
2226
+ else:
2227
+ documents = {fr.a1_key(addr, name): cell for tab, name in zip(tabs, names)
2228
+ for addr, cell in (tab.get("cells") or {}).items()}
2229
+ engine = fr.SheetCalculation({key: fr._document_spec(cell, key.rpartition("!")[0])
2230
+ for key, cell in documents.items()}, sheets=names,
2231
+ named_ranges=book.get("namedRanges"), date_system=book.get("dateSystem", "1900"))
2232
+ input_graph = documents, engine
2233
  documents, engine = input_graph
2234
  home = _resolve_home_tab(book, tabs, names, tab_ref)
2235
+ shared = _read_graph(session, book, fr, None)
2236
+ pending = ([key for ref in refs for key in
2237
+ (_formula_slot_keys(shared, fr, ref, home) if ref.startswith("=")
2238
+ else [_asked_key(ref, home, fr)])] if shared is not None
2239
+ else [_asked_key(ref, home, fr) for ref in refs if not ref.startswith("=")])
2240
+ visited, deps = set(), set()
2241
  while pending:
2242
  key = pending.pop()
2243
  if key in visited:
 
2245
  visited.add(key)
2246
  cell = documents.get(key) or {}
2247
  if cell.get("kind") == "bound":
2248
+ source = dict(cell.get("binding") or {})
2249
+ if "related" in source:
2250
+ source = sheet_mod.clean_binding(source)
2251
+ source_db = str(source.get("database") or "")
2252
+ deps.add(source_db)
2253
+ source_nodes, _ = _binding_filter_nodes(session, source_db, source, memo=view_memo)
2254
+ deps.update(_binding_related_entry(session, source_db, via)["target"]
2255
+ for via in sheet_mod.related_vias(source_nodes))
2256
  pending.extend(engine.consumed.get(key, ()))
2257
  return deps
2258
 
 
2273
 
2274
  def dependencies(db, binding, tab_ref):
2275
  deps = {db}
2276
+ binding = dict(binding)
2277
+ if "related" in binding:
2278
+ binding = sheet_mod.clean_binding(binding)
2279
+ nodes, _ = _binding_filter_nodes(session, db, binding, memo=view_memo)
2280
+ for via in sheet_mod.related_vias(nodes):
2281
+ key = (db, via)
2282
  if key not in related_memo:
2283
+ related_memo[key] = _binding_related_entry(session, db, via)
2284
  deps.add(related_memo[key]["target"])
 
 
 
 
 
2285
  deps.update(input_databases(binding, tab_ref))
2286
  return deps
2287
 
 
2301
  hit = cache.get("refreshCells", {}).get(str(tab_ref) + "!" + addr, {}).get(scope_key)
2302
  sources = {db: token(db) for db in dependencies(db, binding, tab_ref)}
2303
  if (hit and all(sources.values()) and hit.get("tokens") == sources
2304
+ and hit.get("context") == _refresh_context(book, binding, context_memo,
2305
+ tab=tab_ref, session=session)
2306
  and hit["cell"].get("state") == "fresh"):
2307
  return hit["cell"]
2308
  return None
2309
 
2310
+ started = perf_counter()
2311
+ resolved_count = 0
2312
+ scope_names = {id(tab): name for tab, name in
2313
+ zip(book.get("sheets") or [], _tab_scope_names(book.get("sheets") or []))}
2314
  for tab in book.get("sheets") or []:
2315
  for addr, cell in (tab.get("cells") or {}).items():
2316
  if cell.get("kind") != "bound" or _is_view_binding(cell):
 
2318
  binding = cell.get("binding") or {}
2319
  db = str(binding.get("database") or "")
2320
  cell_key = str(tab.get("id")) + "!" + addr
2321
+ if only is not None and cell_key not in only:
2322
+ continue
2323
  scope_key = _binding_scope_key(session, db, binding)
2324
+ context = _refresh_context(book, binding, context_memo, tab=tab.get("id"),
2325
+ session=session)
2326
  hit = cache.get("refreshCells", {}).get(cell_key, {}).get(scope_key)
2327
  source_tokens = {}
2328
  try:
 
2342
  status.setdefault(db, "skipped")
2343
  continue
2344
  status[db] = "resolved"
2345
+ resolved_count += 1
2346
  previous = getattr(session, "_sheet_refresh_input", None)
2347
  session._sheet_refresh_input = cached_input
2348
  try:
2349
  out = _resolve_cell_binding(session, binding, book, tab.get("id"), view_memo=view_memo)
2350
  finally:
2351
  session._sheet_refresh_input = previous
2352
+ shared = _read_graph(session, book, fr, None)
2353
+ if shared is not None:
2354
+ shared["bound_values"][fr.a1_key(addr, scope_names.get(id(tab), ""))] = out
2355
  limits = list(out.get("limits") or [])
2356
  if not all(source_tokens.values()):
2357
  limits.append({"code": "change_token_unavailable",
 
2362
  "refreshed_at": stamp}
2363
  if limits:
2364
  snapshot["limits"] = limits
2365
+ except (HTTPException, sheet_mod.SheetRefusal) as exc:
2366
+ info = ({"code": exc.code, "message": exc.message}
2367
+ if isinstance(exc, sheet_mod.SheetRefusal)
2368
+ else ((exc.detail or {}).get("error") or {}))
2369
  snapshot = _binding_refusal(info.get("code", "sheet_unresolved"),
2370
  info.get("message", "This cell could not be refreshed."))
2371
  status[db] = "refused"
 
2396
  tab_ref, addr = cell_key.split("!", 1)
2397
  _snapshot_cell(sheets.find_tab(book, tab_ref)["cells"][addr], entry["cell"])
2398
 
2399
+ resolve_ms = round((perf_counter() - started) * 1000.0)
2400
  started = perf_counter()
2401
  _recompute_formulas(book, fr, sheet_mod)
2402
  recalc_ms = round((perf_counter() - started) * 1000.0, 1)
2403
+ stats = _read_stats(session) or {}
2404
+ print(f"[sheet] refresh wb={workbook_id} cells={'all' if only is None else len(only)} "
2405
+ f"bound_resolved={resolved_count} engines={stats.get('engines', 0)} "
2406
+ f"input_hits={stats.get('input_hits', 0)} resolve_ms={resolve_ms} "
2407
+ f"recalc_ms={recalc_ms}", flush=True)
2408
  if editable:
2409
  try:
2410
  saved = sheets.put_workbook(store, workbook_id,
 
2419
  saved["refreshedFor"] = str(session.uname or "")
2420
  if entries:
2421
  sheet_binding.save_refresh_cache(store, workbook_id, entries, status)
2422
+ if only is not None:
2423
+ # β›” C5: a targeted answer carries ONLY the cells it resolved for this caller. The rest of
2424
+ # the document holds the owner's stored answers, which a scoped reader must never adopt.
2425
+ return {"workbook": {key: saved.get(key) for key in ("id", "name", "rev", "refreshed_at")},
2426
+ "cells": {cell_key: entry["cell"] for cell_key, _, entry in entries},
2427
+ "recalc_ms": recalc_ms,
2428
+ "binding_refresh": {"databases": status, "token_reads": token_reads}}
2429
  return {"workbook": _response_workbook(saved), "recalc_ms": recalc_ms,
2430
  "binding_refresh": {"databases": status, "token_reads": token_reads}}
2431
 
 
2571
  def _forget_sheet_database(session, database):
2572
  """A field write invalidates both request-local schema and materialised-column memos."""
2573
  getattr(session, "_sheet_assemblies", {}).pop(database, None)
2574
+ memo = getattr(session, "_sheet_read_memo", None)
2575
+ if isinstance(memo, dict):
2576
+ for slot in [slot for slot in memo if isinstance(slot, tuple) and slot[0] == "related"]:
2577
+ memo.pop(slot, None)
2578
  mirrors = getattr(session, "_sheet_mirrors", {})
2579
  for slot in list(mirrors):
2580
  if slot[0] == database:
 
2701
  return sheet_mod.period_window(period, today, cell_date=date)
2702
 
2703
 
2704
+ @contextlib.contextmanager
2705
+ def _sheet_request_scope(session):
2706
+ """HOTFIX 2026-09-29: one memo for one Sheet request, dropped when the request ends.
2707
+
2708
+ ⭐ The Session is built fresh per request (`deps.require_session`), but gates reuse one
2709
+ SimpleNamespace across calls, so the memo is OPT-IN, installed here and removed on exit.
2710
+ Inside it the workbook's authored cells cannot change: every door that opens a scope reads one
2711
+ document and resolves against it, so a cell read or a bound input answered once is the answer
2712
+ for the rest of the request. β›” Never open a scope around code that edits cells between two
2713
+ reads of the same document object (the Daily Sales repair projects in rounds and does not).
2714
+ """
2715
+ previous = getattr(session, "_sheet_read_memo", None)
2716
+ memo = {"stats": {"engines": 0, "resolves": 0, "input_hits": 0}}
2717
+ try:
2718
+ session._sheet_read_memo = memo
2719
+ except Exception: # noqa: BLE001
2720
+ yield None
2721
+ return
2722
+ try:
2723
+ yield memo
2724
+ finally:
2725
+ session._sheet_read_memo = previous
2726
+
2727
+
2728
+ def _request_scoped(route):
2729
+ """Run a route inside `_sheet_request_scope`, keeping its signature for FastAPI and its body in
2730
+ place for the source gates that read it."""
2731
+ import functools
2732
+ import inspect as _inspect
2733
+ signature = _inspect.signature(route)
2734
+
2735
+ @functools.wraps(route)
2736
+ def scoped(*args, **kwargs):
2737
+ session = signature.bind_partial(*args, **kwargs).arguments.get("session")
2738
+ if session is None:
2739
+ return route(*args, **kwargs)
2740
+ with _sheet_request_scope(session):
2741
+ return route(*args, **kwargs)
2742
+ return scoped
2743
+
2744
+
2745
+ def _read_stats(session):
2746
+ memo = getattr(session, "_sheet_read_memo", None)
2747
+ return memo.get("stats") if isinstance(memo, dict) else None
2748
+
2749
+
2750
+ def _read_graph(session, book, fr, today):
2751
+ """This request's cell map and formula graph for `book`, or None outside a request scope."""
2752
+ memo = getattr(session, "_sheet_read_memo", None)
2753
+ if not isinstance(memo, dict) or not isinstance(book, dict):
2754
+ return None
2755
+ slot = ("graph", id(book), str(today or ""))
2756
+ graph = memo.get(slot)
2757
+ if graph is None or graph["book"] is not book:
2758
+ graph = memo[slot] = _new_read_graph(book, fr, memo.get("stats"))
2759
+ return graph
2760
+
2761
+
2762
+ def _new_read_graph(book, fr, stats=None):
2763
+ tabs = book.get("sheets") or []
2764
+ names = _tab_scope_names(tabs)
2765
+ cells = {fr.a1_key(addr, name): cell for source, name in zip(tabs, names)
2766
+ for addr, cell in (source.get("cells") or {}).items() if isinstance(cell, dict)}
2767
+ return {"book": book, "names": names, "cells": cells,
2768
+ "bound": {k: c for k, c in cells.items() if c.get("kind") == "bound"},
2769
+ "engine": None, "attempted": set(), "failures": {}, "bound_values": {},
2770
+ "formulas": {}, "stats": stats}
2771
+
2772
+
2773
+ #: Where a filter formula is evaluated: a private one-formula engine, never a workbook address.
2774
+ _FORMULA_SLOT_ADDR = "XFD1048576"
2775
+
2776
+
2777
+ def _formula_slot(graph, fr, formula, home):
2778
+ """A filter formula, compiled once per request: (slot key, cells it reads, ranges it reads).
2779
+
2780
+ HOTFIX 2026-09-29 (owner: "make formula native inside the Cell database Filter so it can point
2781
+ to an existing date but we can use something like a date function refer to a cell"). A filter
2782
+ value slot holds a cell reference (`F7`) OR a formula (`=DATE($E$24,1,1)`), so a monthly total
2783
+ needs no helper cell holding its first and last day. `cells` is None when it does not compile.
2784
+ """
2785
+ cache = graph.setdefault("formulas", {})
2786
+ slot = (str(home).upper(), formula)
2787
+ if slot not in cache:
2788
+ vkey = fr.a1_key(_FORMULA_SLOT_ADDR, home)
2789
+ book = graph["book"]
2790
+ try:
2791
+ probe = fr.SheetCalculation({vkey: {"formula": formula, "sheet": home}},
2792
+ sheets=graph["names"], named_ranges=book.get("namedRanges"),
2793
+ date_system=book.get("dateSystem", "1900"))
2794
+ compiled = probe.compiled.get(vkey)
2795
+ except Exception: # noqa: BLE001
2796
+ compiled = None
2797
+ cache[slot] = (vkey, set(compiled["cells"]) if compiled else None,
2798
+ list(compiled["ranges"]) if compiled else [])
2799
+ return cache[slot]
2800
+
2801
+
2802
+ def _formula_slot_keys(graph, fr, formula, home):
2803
+ """Every workbook cell a filter formula reads (range members included), or [] if unreadable."""
2804
+ _vkey, reads, ranges = _formula_slot(graph, fr, formula, home)
2805
+ keys = set(reads or ())
2806
+ if ranges:
2807
+ engine = _read_engine(graph, graph["book"], fr, None)[0]
2808
+ for bounds in ranges:
2809
+ keys.update(engine.positions.within(bounds))
2810
+ return sorted(keys)
2811
+
2812
+
2813
+ def _read_binding_formula(session, formula, *, book, tab, today, seen, code, as_date):
2814
+ """A filter / ending value computed by a formula over this workbook's cells, never stored."""
2815
+ from modules import sheet as sheet_mod
2816
+ import formula_runtime as fr
2817
+
2818
+ def refuse(reason):
2819
+ exc = sheet_mod.SheetRefusal(code, formula, reason)
2820
+ exc.message = exc.message.replace("from cell =", "from formula =", 1)
2821
+ exc.args = (exc.message,)
2822
+ raise exc
2823
+
2824
+ graph = _read_graph(session, book, fr, today) or _new_read_graph(book, fr)
2825
+ home = _resolve_home_tab(book, book.get("sheets") or [], graph["names"], tab)
2826
+ vkey, reads, _ranges = _formula_slot(graph, fr, formula, home)
2827
+ if reads is None:
2828
+ refuse("the formula could not be read.")
2829
+ specs = {vkey: {"formula": formula, "sheet": home}}
2830
+ reasons = {}
2831
+ for key in _formula_slot_keys(graph, fr, formula, home):
2832
+ if key == vkey:
2833
+ continue
2834
+ if key in seen:
2835
+ refuse("the reference is circular.")
2836
+ cell = graph["cells"].get(key)
2837
+ if not isinstance(cell, dict) or cell.get("kind") == "empty":
2838
+ if key in reads:
2839
+ # A cell the formula names directly must hold something: a deleted input refuses
2840
+ # by name, as a cell reference does, never a year-zero date. The browser twin is
2841
+ # `sheetState.readFormulaSlot`. A blank range member stays blank.
2842
+ refuse(f"cell {key.rpartition('!')[2]} is blank.")
2843
+ continue
2844
+ try:
2845
+ specs[key] = {"value": _read_binding_cell(session, key, book=book, tab=tab, today=today,
2846
+ seen=seen, code=code),
2847
+ **({"style": cell["style"]} if isinstance(cell.get("style"), dict) else {})}
2848
+ except sheet_mod.SheetRefusal as exc:
2849
+ specs[key] = {"refused": exc.code}
2850
+ reasons[key] = exc.message
2851
+ probe = fr.SheetCalculation(specs, sheets=graph["names"], today=today,
2852
+ named_ranges=book.get("namedRanges"),
2853
+ date_system=book.get("dateSystem", "1900"))
2854
+ if probe.errors.get(vkey):
2855
+ error_code = probe.errors[vkey]
2856
+ refuse(next(iter(reasons.values()), None) or fr._sheet_error_message(error_code))
2857
+ value = probe.values.get(vkey)
2858
+ if value is None or isinstance(value, str) and not value.strip():
2859
+ refuse("its result is blank.")
2860
+ if (as_date and isinstance(value, (int, float)) and not isinstance(value, bool)
2861
+ and sheet_mod.date_formatted_cell({"style": {"format": probe.formats.get(vkey)}})):
2862
+ from harness import windows as W
2863
+ day = W.ending_cell_date(value, book.get("dateSystem", "1900"))
2864
+ if not day:
2865
+ refuse("its value is not a real date.")
2866
+ return day
2867
+ return value
2868
+
2869
+
2870
+ def _read_engine(graph, book, fr, today):
2871
+ """ONE formula engine per request, database inputs pending until a read needs them."""
2872
+ if graph["engine"] is None:
2873
+ specs = {k: ({"refused": "ending_input_pending"} if k in graph["bound"]
2874
+ else fr._document_spec(c, k.rpartition("!")[0]))
2875
+ for k, c in graph["cells"].items()}
2876
+ graph["engine"] = fr.SheetCalculation(
2877
+ specs, sheets=graph["names"], today=today,
2878
+ named_ranges=book.get("namedRanges"), date_system=book.get("dateSystem", "1900"))
2879
+ stats = graph.get("stats")
2880
+ if stats is not None:
2881
+ stats["engines"] += 1
2882
+ return graph["engine"], graph["attempted"], graph["failures"]
2883
+
2884
+
2885
+ def _bound_input(session, key, cell, *, book, today, seen, graph):
2886
+ """A bound cell read as an INPUT (a filter value, an ending date, a formula's operand).
2887
+
2888
+ Refresh's certified cache first, then this request's memo, then one real resolve. A refusal
2889
+ is memoized as the refusal: the same binding over the same document and the same database
2890
+ cannot answer differently twice in one request.
2891
+ """
2892
+ lookup = getattr(session, "_sheet_refresh_input", None)
2893
+ cached = lookup(key) if lookup is not None else None
2894
+ if cached is not None:
2895
+ return cached
2896
+ memo = graph["bound_values"] if graph is not None else None
2897
+ if memo is not None and key in memo:
2898
+ stats = _read_stats(session)
2899
+ if stats is not None:
2900
+ stats["input_hits"] += 1
2901
+ hit = memo[key]
2902
+ if isinstance(hit, HTTPException):
2903
+ raise hit
2904
+ return hit
2905
+ try:
2906
+ result = _resolve_binding(session, cell.get("binding"), today=today, book=book,
2907
+ tab=key.rpartition("!")[0], ending_seen=seen)
2908
+ except HTTPException as exc:
2909
+ if memo is not None:
2910
+ memo[key] = exc
2911
+ raise
2912
+ if memo is not None:
2913
+ memo[key] = result
2914
+ return result
2915
+
2916
+
2917
+ def _binding_input_refs(binding):
2918
+ """Every cell reference a binding reads: filter values at any depth and a period's ref."""
2919
+ from modules import sheet as sheet_mod
2920
+ if not isinstance(binding, dict):
2921
+ return []
2922
+ nodes = []
2923
+ for part in (binding.get("scope"), binding.get("related")):
2924
+ if isinstance(part, dict):
2925
+ found, _ = _fe_parts(part.get("filter"))
2926
+ nodes.extend(found)
2927
+ refs = [str((node.get(slot) or {}).get("ref") or "")
2928
+ for node in sheet_mod.filter_cell_nodes(nodes)
2929
+ for slot in ("valueRef", "value2Ref") if isinstance(node.get(slot), dict)]
2930
+ period = binding.get("period")
2931
+ if isinstance(period, dict) and period.get("ref"):
2932
+ refs.append(str(period["ref"]))
2933
+ return [ref for ref in refs if ref.strip()]
2934
+
2935
+
2936
+ def _qualified_refs(binding, home, fr, graph=None):
2937
+ out = []
2938
+ for ref in _binding_input_refs(binding):
2939
+ if ref.startswith("="):
2940
+ if graph is not None:
2941
+ out.extend(_formula_slot_keys(graph, fr, ref, home))
2942
+ continue
2943
+ try:
2944
+ out.append(_asked_key(ref, home, fr))
2945
+ except Exception: # noqa: BLE001
2946
+ continue
2947
+ return out
2948
+
2949
+
2950
+ def _dependent_bound_keys(book, graph, fr, edited):
2951
+ """HOTFIX 2026-09-29 (owner: "ONLY when we update ANY cells, its related cells also have to
2952
+ update, this includes the cell database"): the bound cells an edit can change, and no other.
2953
+
2954
+ Walks FORWARD from the edited keys over the formula graph's own reverse edges (the same three
2955
+ `SheetCalculation.update` walks: static reads, dynamic reads, range consumers) plus one more
2956
+ edge the formula graph cannot see: a bound cell's filter value / period reference. So editing
2957
+ `F6` reaches `F7 = G6-1`, and every binding filtering on `F7`, and every formula over those.
2958
+ β›” STRUCTURE, NEVER A VALUE DIFF: the stored values are last computed, not current, so an
2959
+ edit reverted to its old value would look unchanged and leave a binding on the edited answer.
2960
+ """
2961
+ from collections import deque
2962
+ engine = _read_engine(graph, book, fr, None)[0]
2963
+ readers = {}
2964
+ for key, cell in graph["bound"].items():
2965
+ if _is_view_binding(cell):
2966
+ continue
2967
+ for ref in _qualified_refs(cell.get("binding"), key.rpartition("!")[0], fr, graph):
2968
+ readers.setdefault(ref, set()).add(key)
2969
+ seen, queue = set(edited), deque(edited)
2970
+ while queue:
2971
+ key = queue.popleft()
2972
+ owners = (set(engine.read_by.get(key, ())) | set(engine.consumed_by.get(key, ()))
2973
+ | set(engine.ranges.at(key, engine.stats)) | readers.get(key, set()))
2974
+ for owner in owners:
2975
+ if owner not in seen:
2976
+ seen.add(owner)
2977
+ queue.append(owner)
2978
+ return {key for key in seen if key in graph["bound"] and key not in edited}
2979
+
2980
+
2981
+ def _input_closure(graph, engine, fr, roots):
2982
+ """Every cell a set of references reads, transitively, through formulas and bound inputs."""
2983
+ seen, pending = set(), list(roots)
2984
+ while pending:
2985
+ key = pending.pop()
2986
+ if key in seen:
2987
+ continue
2988
+ seen.add(key)
2989
+ cell = graph["cells"].get(key)
2990
+ if isinstance(cell, dict) and cell.get("kind") == "bound":
2991
+ pending.extend(_qualified_refs(cell.get("binding"), key.rpartition("!")[0], fr, graph))
2992
+ continue
2993
+ pending.extend(engine.reads.get(key, ()))
2994
+ pending.extend(engine.consumed.get(key, ()))
2995
+ compiled = engine.compiled.get(key)
2996
+ for bounds in (compiled or {}).get("ranges") or ():
2997
+ pending.extend(engine.positions.within(bounds))
2998
+ return seen
2999
+
3000
+
3001
  def _read_binding_cell(session, ref, *, book=None, tab=None, today=None, seen=(),
3002
+ code="ending_cell_unreadable", as_date=False):
3003
+ """Shared cell read for Ending and Filter, including scoped bound and formula inputs.
3004
+
3005
+ S67-T10: with ``as_date`` a date-formatted number (typed, or a formula's result such as
3006
+ 46293) is returned as its ISO day, so a Filter condition compares it as that date for every
3007
+ operator; a plain number with no date format is returned unchanged.
3008
+ """
3009
  from modules import sheet as sheet_mod
3010
  import formula_runtime as fr
3011
 
3012
  def refuse(reason):
3013
  raise sheet_mod.SheetRefusal(code, ref or "(blank)", reason)
3014
 
3015
+ if str(ref or "").strip().startswith("=") and book is not None:
3016
+ return _read_binding_formula(session, str(ref).strip(), book=book, tab=tab, today=today,
3017
+ seen=seen, code=code, as_date=as_date)
3018
  if not re.fullmatch(r"(?:(?:'([^']|'')+'|[^'!]+)!)?\$?[A-Za-z]{1,3}\$?[1-9][0-9]{0,6}", ref):
3019
  refuse("its reference is invalid.")
3020
  if book is None:
3021
  refuse("the source workbook is unavailable.")
3022
  tabs = book.get("sheets") or []
3023
+ # HOTFIX 2026-09-29 (owner: "MAKE IT FAST"): one request reads ONE graph. Before this, every
3024
+ # filter reference of every bound cell rebuilt the whole workbook's cell map and, for a formula
3025
+ # reference, a whole-workbook SheetCalculation (about 0.6 s each): Daily Sales has ~470 bound
3026
+ # cells reading `$E$10` and `F7`, so one commit or one Refresh paid ~900 engine builds.
3027
+ graph = _read_graph(session, book, fr, today)
3028
+ names = graph["names"] if graph is not None else _tab_scope_names(tabs)
3029
  home = _resolve_home_tab(book, tabs, names, tab)
3030
  key = _asked_key(ref, home, fr)
3031
  if key in seen:
3032
  refuse("the reference is circular.")
3033
+ cells = graph["cells"] if graph is not None else {
3034
+ fr.a1_key(addr, name): cell for source, name in zip(tabs, names)
3035
+ for addr, cell in (source.get("cells") or {}).items()}
3036
  cell = cells.get(key)
3037
  if not isinstance(cell, dict) or cell.get("kind") == "empty":
3038
  refuse("the cell or its sheet is missing or blank.")
3039
  if cell.get("kind") == "bound":
3040
  try:
3041
+ value = _bound_input(session, key, cell, book=book, today=today,
3042
+ seen=(*seen, key), graph=graph)["value"]
 
 
 
3043
  except HTTPException as exc:
3044
  reason = ((exc.detail or {}).get("error") or {}).get("message", "its source is refused.")
3045
  refuse(reason)
3046
  elif cell.get("kind") == "formula":
3047
  # The existing formula graph discovers actual inputs, including dynamic references.
3048
  # A database input starts refused, so a cached value can never bypass this reader's wall.
3049
+ bound = graph["bound"] if graph is not None else {
3050
+ k: c for k, c in cells.items() if c.get("kind") == "bound"}
3051
+ if graph is not None:
3052
+ engine, attempted, failures = _read_engine(graph, book, fr, today)
3053
+ else:
3054
+ specs = {k: ({"refused": "ending_input_pending"} if k in bound
3055
+ else fr._document_spec(c, k.rpartition("!")[0])) for k, c in cells.items()}
3056
+ engine = fr.SheetCalculation(specs, sheets=names, today=today,
3057
+ named_ranges=book.get("namedRanges"), date_system=book.get("dateSystem", "1900"))
3058
+ attempted, failures = set(), {}
3059
  while True:
3060
  dependencies, pending = set(), [key]
3061
  while pending:
 
3071
  for dependency in needed:
3072
  attempted.add(dependency)
3073
  try:
3074
+ result = _bound_input(session, dependency, bound[dependency], book=book,
3075
+ today=today, seen=(*seen, key, dependency), graph=graph)
 
 
 
3076
  changes[dependency] = {"value": result["value"]}
3077
  except HTTPException as exc:
3078
  info = ((exc.detail or {}).get("error") or {})
 
3094
  if cell.get("state") == "stale":
3095
  refuse("the cell needs to be refreshed.")
3096
  value = cell.get("value", cell.get("text"))
3097
+ if (as_date and isinstance(value, (int, float)) and not isinstance(value, bool)
3098
+ and sheet_mod.date_formatted_cell(cell)):
3099
+ from harness import windows as W
3100
+ day = W.ending_cell_date(value, book.get("dateSystem", "1900"))
3101
+ if not day:
3102
+ refuse("its value is not a real date.")
3103
+ return day
3104
  return value
3105
 
3106
 
 
3108
  from modules import sheet as sheet_mod
3109
  try:
3110
  return sheet_mod.resolve_filter_cells(nodes, lambda ref: _read_binding_cell(
3111
+ session, ref, book=book, tab=tab, today=today, seen=seen, code="filter_cell_unreadable",
3112
+ as_date=True))
3113
  except sheet_mod.SheetRefusal as exc:
3114
  raise err(400, exc.code, exc.message)
3115
 
 
3153
  return _resolve_binding(session, binding, **kwargs)
3154
 
3155
 
3156
+ def _binding_related_entry(session, database, name):
3157
  """Resolve one authored edge key against the permission-scoped declarations."""
3158
+ from core import perm_scope, relations
3159
+ fields, defs, entries = _related_declarations(session, database)
 
 
 
 
 
 
 
 
 
 
 
 
3160
  matches = [entry for entry in entries if entry["key"] == name]
3161
  via = matches[0]["via"] if len(matches) == 1 else {"key": name}
3162
+ # `pick` over the SCOPED list, which also carries the target-side bridge entries above.
3163
+ entry = relations.pick(entries, via)
3164
  if entry is None:
3165
  candidates = [item["key"] for item in relations.via_candidates(entries, via)
3166
  if perm_scope.nav_may_open(session.user, item["target"], st=session.runtime)]
 
3180
  declaration = next((f for f in defs.get(target, {}).get("fields", [])
3181
  if f.get("key") == entry["via"].get("field")), {})
3182
  link = declaration.get("link") or {}
3183
+ if (link.get("through") or link.get("selectionMode") == "manual"
3184
+ or not entry["via"].get("from") or not entry["via"].get("on")):
3185
  raise err(409, "relation_source_unresolvable",
3186
  f"Relationship '{name}' does not declare a direct column join. "
3187
  "Choose a relationship with one readable column on each database.")
3188
  return entry
3189
 
3190
 
3191
+ def _binding_related_query(session, database, entry, cur, spec, columns, *, fields, window=None):
 
3192
  """Materialise the far side on the near cursor, then hand metadata to the compiler."""
 
 
3193
  from core import perm_scope, relations
3194
  from modules import sheet as sheet_mod
3195
 
3196
+ target, name = entry["target"], entry["key"]
 
3197
  local_key, remote_key = entry["via"].get("from"), entry["via"].get("on")
3198
 
3199
  def map_join(key, table, source_spec, cols, side):
 
3219
  wall = ((perm_scope.entry(session.user, target) or {}).get("filter")
3220
  if perm_scope.row_scope_applies(session.user, target) else None)
3221
  wall_nodes, wall_conj = _fe_parts(wall)
3222
+ needed = set(fields) | sheet_mod.filter_columns_named(wall_nodes) | {remote_key}
 
 
 
 
3223
  try:
3224
  rcur, rspec, rcols = _mirror(session)(target, metric_window=window, needed=needed)
3225
  except HTTPException as exc:
 
3236
  hidden = perm_scope.hidden_keys(session.user, target, rspec.get("fields") or [],
3237
  st=session.runtime)
3238
  rcols = {key: value for key, value in rcols.items() if key not in hidden}
3239
+ declared = {field["key"]: field for field in rspec.get("fields") or []}
3240
+
3241
+ def unknown_field(key):
3242
+ return err(409, "related_field_unknown",
3243
+ f"Field '{key}' on relationship '{name}' is hidden, removed, or has no "
3244
+ "readable per-record value. Choose a readable field on this relationship.")
3245
+
3246
+ for key in sorted(fields):
3247
+ field = declared.get(key) or {}
3248
+ if (key in hidden
3249
+ or field.get("type") in {"formula", "link", "lookup", "rollup", "metric", "measure"}
3250
+ or field.get("measure")):
3251
+ raise unknown_field(key)
3252
  map_join(remote_key, target, rspec, rcols, "target")
3253
+ for key in sorted(fields):
3254
+ expr = (rcols.get(key) or {}).get("sql")
3255
+ if not expr or str(expr).strip().upper() == "NULL":
3256
+ raise unknown_field(key)
3257
  return {"via": name, "database": target, "local_key": local_key, "remote_key": remote_key,
3258
  "spec": rspec, "columns": rcols, "where": entry.get("where"),
 
3259
  "wall_nodes": wall_nodes, "wall_conj": wall_conj,
3260
+ "limits": [row for key, row in (rspec.get("unbounded") or {}).items() if key in fields]}
3261
+
3262
+
3263
+ def _related_leaf_bags(session, database, nodes, needed):
3264
+ """Prepare joins before the local mirror, then materialise their bags on its cursor."""
3265
+ from modules import sheet as sheet_mod
3266
+ entries, fields = {}, {}
3267
+ try:
3268
+ for key in sorted(sheet_mod.related_filter_ids(nodes)):
3269
+ via, field = sheet_mod.related_field_parts(key)
3270
+ if via not in entries:
3271
+ entries[via] = _binding_related_entry(session, database, via)
3272
+ needed.add(entries[via]["via"].get("from"))
3273
+ fields.setdefault(via, set()).add(field)
3274
+ needed.difference_update(sheet_mod.related_filter_ids(nodes))
3275
+ except sheet_mod.SheetRefusal as exc:
3276
+ raise err(409, exc.code, exc.message)
3277
+
3278
+ def materialise(cur, spec, columns, *, window=None):
3279
+ return {via: _binding_related_query(session, database, entry, cur, spec, columns,
3280
+ fields=fields[via], window=window)
3281
+ for via, entry in entries.items()}
3282
+ return materialise
3283
+
3284
+
3285
+ def _binding_filter_nodes(session, database, binding, *, memo=None):
3286
+ """A saved view and an authored extra scope are always intersected."""
3287
+ scope = binding.get("scope") or {}
3288
+ nodes, conj = _fe_parts(scope.get("filter"))
3289
+ from modules import sheet as sheet_mod
3290
+ if sheet_mod.linked_filter_ids(nodes):
3291
+ nodes = _rewrite_related_filters(session, database, nodes)
3292
+ binding["scope"] = {**scope, "filter": {"conj": conj, "nodes": nodes}}
3293
+ if scope.get("view_id"):
3294
+ view_nodes, view_conj = _view_filter(session, database, scope["view_id"],
3295
+ scope.get("view_name", ""), memo=memo)
3296
+ if nodes:
3297
+ nodes, conj = [{"conj": view_conj, "children": view_nodes},
3298
+ {"conj": conj, "children": nodes}], "and"
3299
+ else:
3300
+ nodes, conj = view_nodes, view_conj
3301
+ return _rewrite_related_filters(session, database, nodes), conj
3302
 
3303
 
3304
  def _resolve_binding(session: Session, raw_binding, *, today=None, view_memo=None,
 
3349
  wall_tree = ((perm_scope.entry(session.user, database) or {}).get("filter")
3350
  if perm_scope.row_scope_applies(session.user, database) else None)
3351
  wall_nodes, wall_conj = _fe_parts(wall_tree)
3352
+ scope_nodes, scope_conj = _binding_filter_nodes(session, database, binding, memo=view_memo)
 
 
 
 
 
3353
  scope_nodes = _filter_cell_values(session, scope_nodes, book=book, tab=tab,
3354
  today=today, seen=ending_seen)
3355
  needed = (sheet_mod.filter_columns_named(scope_nodes)
 
3357
  if binding["aggregate"].get("field"):
3358
  needed.add(str(binding["aggregate"]["field"]))
3359
 
3360
+ related_bags = _related_leaf_bags(session, database, scope_nodes, needed)
 
 
3361
  cur, spec, cols = _mirror(session)(database, metric_window=window, needed=needed)
3362
  cols = dict(cols)
 
3363
 
3364
  # β›” THE COLUMN IS CHECKED BEFORE THE SQL IS BUILT (W53-T13). Now that every tenant column is a
3365
  # real column of the materialised table, a `sum` over a text column reaches DuckDB as a binder
 
3398
  # spec's column expressions; applying it a second time would double-count the exclusion and
3399
  # break every joined spec, whose partner column is `sol.order_partner_id` or `aml.partner_id`.
3400
  try:
3401
+ related = related_bags(cur, spec, cols, window=window)
 
 
3402
  sql, params, limits = sheet_mod.build_aggregate_query(
3403
  spec, binding, cols, scope_nodes=scope_nodes, scope_conj=scope_conj,
3404
  wall_nodes=wall_nodes, wall_conj=wall_conj, today=today, related=related)
 
3501
  binding editor show a number before anyone commits the cell. R10 may create a missing Metric
3502
  field through the database's ordinary field writer; reusing a column needs no field write.
3503
  """
3504
+ with _sheet_request_scope(session):
3505
+ return _preview_cell_scoped(workbook_id, addr, body, session)
3506
+
3507
+
3508
+ def _preview_cell_scoped(workbook_id: str, addr: str, body, session: Session):
3509
  raw = dict(body or {})
3510
  binding = raw.get("binding") or {}
3511
  book = None
 
3613
  ⚠ `rev` BUMPS, BECAUSE THIS PERSISTS. It goes back in the answer and the client adopts it, or
3614
  its next PATCH would carry a revision the server has already moved past.
3615
  """
3616
+ with _sheet_request_scope(session):
3617
+ return _compute_cell_scoped(workbook_id, addr, body, session)
3618
+
3619
+
3620
+ def _compute_cell_scoped(workbook_id: str, addr: str, body, session: Session):
3621
  from core import sheets
3622
 
3623
  try:
 
3751
  if perm_scope.row_scope_applies(session.user, database) else None)
3752
  wall_nodes, wall_conj = _fe_parts(wall_tree)
3753
  scope_nodes, scope_conj = _fe_parts(binding.get("filter"))
3754
+ scope_nodes = _rewrite_related_filters(session, database, scope_nodes)
3755
  # W62 C2: a filter-only field obeys the same grants; linked fields have their own resolver wall.
3756
  for key in sorted(sheet_mod.filter_columns_named(scope_nodes)):
3757
+ if not key.startswith(sheet_mod.RELATED_FILTER_PREFIX) and key not in visible:
3758
  return JSONResponse(status_code=400, content={
3759
  "error": "view_field_unknown", "message": f"Field {key} is not available in this database."})
3760
  today = date.today()
 
3763
  window = _ending_window(session, binding, book=book, today=today)
3764
  needed = (set(requested) | sorted_keys | sheet_mod.filter_columns_named(scope_nodes)
3765
  | sheet_mod.filter_columns_named(wall_nodes))
3766
+ related_bags = _related_leaf_bags(session, database, scope_nodes, needed)
3767
  cur, spec, cols = _mirror(session)(database, metric_window=window, needed=needed)
3768
  cols = dict(cols)
3769
+ related = related_bags(cur, spec, cols, window=window)
3770
  for key in dict.fromkeys([*requested, *sorted(sorted_keys | sheet_mod.filter_columns_named(scope_nodes))]):
3771
+ if key.startswith(sheet_mod.RELATED_FILTER_PREFIX):
3772
+ continue
3773
  if key in hidden or not (cols.get(key) or {}).get("sql"):
3774
  return JSONResponse(status_code=400, content={
3775
  "error": "view_field_unknown", "message": f"Field {key} is not available in this database."})
 
3779
  raise ValueError("unapplied row wall")
3780
  count_sql, params, limits = sheet_mod.build_aggregate_query(
3781
  spec, {"aggregate": {"fn": "count"}}, cols, scope_nodes=scope_nodes,
3782
+ scope_conj=scope_conj, wall_nodes=wall_nodes, wall_conj=wall_conj, today=today,
3783
+ related=related)
3784
  if limits:
3785
  raise ValueError("unapplied view filter")
3786
  except sheet_mod.SheetRefusal as exc:
 
3821
 
3822
 
3823
  @router.get("/sheets/{workbook_id}/cells/{addr}/drill")
3824
+ @_request_scoped
3825
+ def drill_cell(workbook_id: str, addr: str, tab: str | None = None,
3826
+ session: Session = Depends(require_session)):
3827
+ """`{database, view_id?, filter?, record_ids, count, id_limit}` for `LinkGridModal` (R6, R13, R17).
3828
+
3829
+ ⭐⭐ S66-T48 β€” `tab` (an id or a name, as `GET /sheets/{id}/tabs/{tab}` accepts) NAMES THE TAB
3830
+ THE CELL IS ON. Without it the address was resolved against the FIRST tab holding it, so a
3831
+ Daily Sales `Summary (SR)` cell drilled `Summary (All)`'s same address. Omitted, the old
3832
+ first-tab rule still answers, so an older client keeps working.
3833
+
3834
+ ⭐⭐ S66-T48 β€” NO CAP ON A CONNECTOR DRILL (standing rule 1). `record_ids` is every matching
3835
+ id; the popup pages them and sends the set in a POST body. Only a database whose records a
3836
+ person types (`records_mutable`) keeps `user_tables.MAX_ROWS`, and then `id_limit` names it so
3837
+ the popup says "showing X of N" rather than presenting X as the whole.
3838
 
3839
  β›” A FORMULA CELL REFUSES BY NAME AND DOES NOT DRILL TO ONE OPERAND (R17). Picking an input
3840
  would be a guess presented as an answer: `=B4-B8` has two, and neither is what the person
 
3855
 
3856
  key = str(addr or "").replace("$", "").upper()
3857
  cell = None
3858
+ home = None
3859
+ if str(tab or "").strip():
3860
+ try:
3861
+ home = sheets.find_tab(book, str(tab).strip())
3862
+ except sheets.SheetRefusal as exc:
3863
+ raise _refuse(exc)
3864
+ cell = (home.get("cells") or {}).get(key)
3865
+ else:
3866
+ for candidate in book.get("sheets") or ():
3867
+ if key in (candidate.get("cells") or {}):
3868
+ home, cell = candidate, candidate["cells"][key]
3869
+ break
3870
  if not isinstance(cell, dict):
3871
  raise err(404, "cell_missing", f"Cell {key} is not filled in.")
3872
  if cell.get("kind") == "formula":
 
3896
  wall_nodes, wall_conj = _fe_parts(wall_tree)
3897
 
3898
  scope = binding["scope"]
3899
+ scope_nodes, scope_conj = _binding_filter_nodes(session, database, binding)
 
 
 
 
3900
 
3901
  # ⚠ THE SAME PERIOD THE NUMBER WAS COMPUTED AT (W53-T13). The drill probes `COUNT(*)` and never
3902
  # reads the bound column, but its FILTER may name a Metric column, and a mirror opened at a
 
3907
  # window is kept UNCONDITIONALLY, exactly as W53-T13 wrote it, and the saving is taken from
3908
  # the column set instead, where the time actually is.
3909
  try:
3910
+ scope_nodes = _filter_cell_values(session, scope_nodes, book=book, tab=home.get("id"))
3911
+ window = _ending_window(session, binding, book=book, tab=home.get("id"))
3912
  except sheet_mod.SheetRefusal as exc:
3913
  raise err(400, exc.code, exc.message)
 
3914
  needed = (sheet_mod.filter_columns_named(scope_nodes)
3915
  | sheet_mod.filter_columns_named(wall_nodes))
3916
+ related_bags = _related_leaf_bags(session, database, scope_nodes, needed)
 
3917
  cur, spec, cols = _mirror(session)(
3918
  database, metric_window=window,
3919
  needed=needed)
3920
  cols = dict(cols)
 
3921
 
3922
  try:
3923
+ related = related_bags(cur, spec, cols, window=window)
 
 
3924
  count_sql, count_params, limits = sheet_mod.build_aggregate_query(
3925
  spec, {"aggregate": {"fn": "count"}}, cols, scope_nodes=scope_nodes,
3926
  scope_conj=scope_conj, wall_nodes=wall_nodes, wall_conj=wall_conj, related=related)
 
3935
  "connection. Edit the scope or ask an administrator to update the restriction.")
3936
  total = (cur.execute(count_sql, count_params).fetchone() or [0])[0]
3937
 
3938
+ # ⭐⭐ S66-T48 β€” THE CAP IS DECIDED BY THE DATA'S SOURCE, NOT BY THE POPUP. A connector's records
3939
+ # (`records_mutable` False) are listed in full: the popup renders one page and the id set
3940
+ # travels in a POST body, which were the two reasons the old 500 id cap gave. A typed database
3941
+ # keeps `MAX_ROWS`, asked only when the count passes it (the predicate reads the store), and
3942
+ # the limit is RETURNED so the popup states it instead of enforcing it silently.
3943
+ from core import user_tables
3944
+ id_limit = None
3945
+ if (int(total or 0) > user_tables.MAX_ROWS
3946
+ and user_tables.records_mutable(database, st=session.runtime)):
3947
+ id_limit = user_tables.MAX_ROWS
3948
+ ids_sql = f"{ids_sql} LIMIT {int(id_limit)}"
3949
  record_ids = [int(r[0]) for r in cur.execute(ids_sql, ids_params).fetchall() if r and r[0]]
3950
 
3951
  out = {"database": database, "record_ids": record_ids, "count": int(total or 0),
3952
+ "id_limit": id_limit, "limits": limits}
3953
  if scope.get("view_id"):
3954
  out["view_id"] = scope["view_id"]
3955
  else:
 
4058
  database work, and a batch's cost is dominated by the number of distinct (database, period)
4059
  pairs in it rather than by the number of cells.
4060
  """
4061
+ with _sheet_request_scope(session):
4062
+ return _resolve_bindings_scoped(workbook_id, body, session)
4063
+
4064
+
4065
+ def _resolve_bindings_scoped(workbook_id: str, body, session: Session):
4066
  from core import sheets
4067
  import formula_runtime as fr
4068
 
api/routes_tables.py CHANGED
@@ -2116,7 +2116,7 @@ def create_table(body: dict = Body(default=None),
2116
  if not key:
2117
  raise err(400, "refused",
2118
  f"could not create it. The name may be empty or this tenant already has "
2119
- f"{ut.MAX_TABLES} databases")
2120
  return {"key": key}
2121
 
2122
 
 
2116
  if not key:
2117
  raise err(400, "refused",
2118
  f"could not create it. The name may be empty or this tenant already has "
2119
+ f"{ut.MAX_TABLES} databases of its own (connected-source databases do not count)")
2120
  return {"key": key}
2121
 
2122
 
api/routes_templates.py CHANGED
@@ -80,8 +80,8 @@ def list_templates(table: str = "", session: Session = Depends(require_session),
80
  `missing_columns`, which is the same function the POST below re-runs).
81
  """
82
  if module:
83
- if module not in ("sheet", "data", "app"):
84
- raise err(400, "unknown_module", "Choose Sheet, Data or App.")
85
  return _module_templates(session)
86
  fields, source = _target_or_refuse(session, table)
87
  vt = _templates()
@@ -137,6 +137,16 @@ def apply_template(key: str, body: dict = Body(default=None),
137
  import core.table_store as table_store
138
  ops = table_store.make(ws_key, st=session.runtime)
139
  applied = []
 
 
 
 
 
 
 
 
 
 
140
  for view in tpl.get('views') or ():
141
  # β›” A COPY PER APPLY, and `createdBy` STAMPED HERE. The registry's dicts are module-level
142
  # β€” mutating one would write this caller's username into the template every other tenant
@@ -145,6 +155,10 @@ def apply_template(key: str, body: dict = Body(default=None),
145
  # side.
146
  payload = dict(view)
147
  payload['config'] = dict(view.get('config') or {})
 
 
 
 
148
  payload['createdBy'] = session.uname
149
  saved = ops.save_view(session.uname, payload)
150
  applied.append({"id": payload['id'], "name": saved.get('name') or payload['name']})
@@ -265,8 +279,68 @@ def _data_template_database(template, definitions):
265
  return ""
266
 
267
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
268
  def _module_templates(session):
269
- from core import sheet_templates, user_tables
270
  from routes_connectors import _entry_types
271
  from routes_keychain import env_odoo_available
272
  from routes_nav import application_templates
@@ -281,9 +355,13 @@ def _module_templates(session):
281
  saved = _saved_sheet_templates(session.runtime)
282
  data_templates = [(row, _data_template_database(row, definitions))
283
  for row in _templates().TEMPLATES]
 
 
 
284
  databases = {row["database_hint"] for row in sheet_templates.TEMPLATES}
285
  databases.update(db for row in saved for db in row["databases"])
286
  databases.update(database for _row, database in data_templates if database)
 
287
  database_states = _database_states(session, definitions, databases)
288
  requirement = lambda database, connector="": _requirement(
289
  database, definitions, have, database_states, connector)
@@ -305,9 +383,25 @@ def _module_templates(session):
305
  app.extend(_catalog_row("finance:" + row["key"], row["label"], row["description"], [],
306
  creation_mode="post")
307
  for row in finance_templates(may_open=lambda permission: _may_open(session, permission)))
308
- return {"sections": [{"module": "sheet", "templates": sheet},
309
- {"module": "data", "templates": data},
310
- {"module": "app", "templates": app}]}
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
311
 
312
 
313
  @router.get("/sheet-templates")
@@ -412,6 +506,9 @@ def create_module_template(module: str, key: str, body: dict = Body(default=None
412
  from urllib.parse import quote
413
 
414
  body = body if isinstance(body, dict) else {}
 
 
 
415
  if module == "app" and key.startswith("finance:"):
416
  import application_registry
417
  from modules.finance_apps import finance_application
@@ -447,6 +544,16 @@ def create_module_template(module: str, key: str, body: dict = Body(default=None
447
  raise err(404, "unknown_template", "That template does not exist.")
448
  if not template["creatable"]:
449
  raise err(409, "template_unavailable", "Connect the required source and switch on its databases.")
 
 
 
 
 
 
 
 
 
 
450
  if module == "sheet":
451
  database = template["requires"][0]["database"] if template["requires"] else ""
452
  result = apply_sheet_template(key, {"database": database, "name": body.get("name", "")}, session)
@@ -455,9 +562,28 @@ def create_module_template(module: str, key: str, body: dict = Body(default=None
455
  database = template["requires"][0]["database"]
456
  result = apply_template(key, {"table": database}, session)
457
  return {"address": "#/" + quote(database, safe=""), **result}
 
 
 
 
 
 
458
  # The existing wizard still owns its POST-then-PATCH share sequence. API clients may
459
  # dispatch creation here; the existing App writer retains schema and permission walls.
460
  result = create_nav_application({**body, "templateKind": key}, session=session)
461
  app = result["application"]
462
  return {"address": "#/app?database=" + quote(app["sourceDatabase"], safe="") + "&application=" + quote(app["id"], safe=""),
463
  **result}
 
 
 
 
 
 
 
 
 
 
 
 
 
 
80
  `missing_columns`, which is the same function the POST below re-runs).
81
  """
82
  if module:
83
+ if module not in ("sheet", "data", "app", "agents"):
84
+ raise err(400, "unknown_module", "Choose Sheet, Data, App or Agents.")
85
  return _module_templates(session)
86
  fields, source = _target_or_refuse(session, table)
87
  vt = _templates()
 
137
  import core.table_store as table_store
138
  ops = table_store.make(ws_key, st=session.runtime)
139
  applied = []
140
+ # S67-T39 (R12, D-2233): a template View bound to an App that no longer exists instantiates as
141
+ # a plain view of its database; the live App ids are read once, and an unreadable registry
142
+ # keeps the reference (the read projection in `grid_events.table_workspace` still masks it).
143
+ try:
144
+ import application_registry
145
+ raw_registry = session.runtime.get(application_registry._KEY)
146
+ live_apps = (set(raw_registry["applications"]) if isinstance(raw_registry, dict)
147
+ and isinstance(raw_registry.get("applications"), dict) else None)
148
+ except Exception:
149
+ live_apps = None
150
  for view in tpl.get('views') or ():
151
  # β›” A COPY PER APPLY, and `createdBy` STAMPED HERE. The registry's dicts are module-level
152
  # β€” mutating one would write this caller's username into the template every other tenant
 
155
  # side.
156
  payload = dict(view)
157
  payload['config'] = dict(view.get('config') or {})
158
+ dead_ref = payload['config'].get('application')
159
+ if (live_apps is not None and isinstance(dead_ref, dict)
160
+ and str(dead_ref.get('id') or '') not in live_apps):
161
+ payload['config'].pop('application')
162
  payload['createdBy'] = session.uname
163
  saved = ops.save_view(session.uname, payload)
164
  applied.append({"id": payload['id'], "name": saved.get('name') or payload['name']})
 
279
  return ""
280
 
281
 
282
+ def _agent_templates():
283
+ """Fresh starter definitions; saving one never arms its schedule."""
284
+ import os
285
+
286
+ base = (os.environ.get("AIOS_PUBLIC_BASE") or os.environ.get("APP_BASE_URL") or "").strip().rstrip("/")
287
+
288
+ def report(instruction, route):
289
+ return {"id": "act_report", "kind": "ai_agent", "config": {
290
+ "url": base + "/#/" + route if base else "",
291
+ "instruction": instruction,
292
+ "field": "", "timeoutMs": 20000, "maxSteps": 6}}
293
+
294
+ def starter(key, name, description, cron, databases, actions):
295
+ return {"key": key, "name": name, "description": description, "databases": databases,
296
+ "definition": {"name": name, "kind": "plain", "trigger": {"key": "schedule"},
297
+ "schedule": {"cron": cron, "enabled": False},
298
+ "flow": {"actions": actions}}}
299
+
300
+ return [
301
+ starter("odoo_sync_check", "Odoo sync check", "Check Odoo database freshness each day.",
302
+ "0 8 * * *", ["ut_odoo_orders"], [
303
+ {"id": "act_find", "kind": "find_records", "config": {
304
+ "table": "ut_odoo_orders", "cond": None, "limit": 25}},
305
+ report("Read the Odoo connection's database sync statuses in this workspace. "
306
+ "Report tables whose last successful sync is older than 24 hours or failed. "
307
+ "Name each table and the observed timestamp or error. If freshness cannot "
308
+ "be read, report it as unknown. Only read; do not sync, change settings, "
309
+ "edit records or send messages.", "connectors")]),
310
+ starter("ar_follow_up_digest", "AR follow-up digest", "Draft a weekly digest of overdue invoices.",
311
+ "0 9 * * 1", ["ut_odoo_invoices"], [
312
+ # The condition vocabulary orders numbers only. Evaluate due dates in the
313
+ # report at run time, never against a date frozen when the template was saved.
314
+ {"id": "act_find", "kind": "find_records", "config": {
315
+ "table": "ut_odoo_invoices",
316
+ "cond": {"field": "residual", "op": ">", "value": 0}, "limit": 100}},
317
+ report("Read open invoices in this workspace's Odoo invoices database. Compare "
318
+ "due_date with today's date and include only overdue invoices with residual "
319
+ "above zero. Draft a follow-up digest grouped by customer, with invoice, "
320
+ "due date, open amount and suggested next action. State any missing data "
321
+ "or incomplete coverage. Only read; do not edit records or send messages.",
322
+ "ut_odoo_invoices")]),
323
+ starter("data_health_watcher", "Data-health watcher", "Check for stale or empty databases each day.",
324
+ "0 7 * * *", [], [
325
+ report("Read the databases visible in this workspace. Flag empty databases and "
326
+ "those whose last successful refresh is older than 24 hours. Report each "
327
+ "database name, observed row count and refresh time when available. "
328
+ "Treat missing freshness information as unknown, never as healthy. "
329
+ "Only read; do not edit records, change settings or send messages.", "data")]),
330
+ ]
331
+
332
+
333
+ def _content_studio():
334
+ """The Content Studio App's own module, or None when it is absent (rule 13: delete-safe)."""
335
+ try:
336
+ import content_studio
337
+ except ImportError:
338
+ return None
339
+ return content_studio
340
+
341
+
342
  def _module_templates(session):
343
+ from core import registry, sheet_templates, user_tables
344
  from routes_connectors import _entry_types
345
  from routes_keychain import env_odoo_available
346
  from routes_nav import application_templates
 
355
  saved = _saved_sheet_templates(session.runtime)
356
  data_templates = [(row, _data_template_database(row, definitions))
357
  for row in _templates().TEMPLATES]
358
+ # S67-T36 (R25): an archived Agents module offers no agent template; the create route stays.
359
+ agent_templates = (_agent_templates() if _may_open(session, "automation")
360
+ and not registry.is_archived("automation") else [])
361
  databases = {row["database_hint"] for row in sheet_templates.TEMPLATES}
362
  databases.update(db for row in saved for db in row["databases"])
363
  databases.update(database for _row, database in data_templates if database)
364
+ databases.update(db for row in agent_templates for db in row["databases"])
365
  database_states = _database_states(session, definitions, databases)
366
  requirement = lambda database, connector="": _requirement(
367
  database, definitions, have, database_states, connector)
 
383
  app.extend(_catalog_row("finance:" + row["key"], row["label"], row["description"], [],
384
  creation_mode="post")
385
  for row in finance_templates(may_open=lambda permission: _may_open(session, permission)))
386
+ sections = [{"module": "sheet", "templates": sheet},
387
+ {"module": "data", "templates": data},
388
+ {"module": "app", "templates": app}]
389
+ # S67-T40 (R18): every tenant's App catalogue offers Content Studio, which creates its own
390
+ # databases. Guarded (rule 13): without the module the catalogue simply omits the row.
391
+ # ⚠ Appended AFTER `sections` (it is the App section's own list, so the order stays Finance,
392
+ # then Studio): `verify_shell_rail.py` scopes its S65-T10 needles to the span from the Finance
393
+ # offer to `sections = [`, and its finance-offer-wizard control mutates the FIRST
394
+ # `creation_mode="post"` in this file. A Studio row inside that span, or above it, masks it.
395
+ studio = _content_studio()
396
+ if studio is not None:
397
+ app.append(_catalog_row(studio.TEMPLATE_KEY, studio.LABEL, studio.DESCRIPTION, [],
398
+ creation_mode="post"))
399
+ if agent_templates:
400
+ sections.append({"module": "agents", "templates": [
401
+ _catalog_row(row["key"], row["name"], row["description"],
402
+ [requirement(db, "odoo") for db in row["databases"]], creation_mode="post")
403
+ for row in agent_templates]})
404
+ return {"sections": sections}
405
 
406
 
407
  @router.get("/sheet-templates")
 
506
  from urllib.parse import quote
507
 
508
  body = body if isinstance(body, dict) else {}
509
+ if module == "agents":
510
+ # A direct call to create_automation does not run its FastAPI dependency.
511
+ session.require("automation")
512
  if module == "app" and key.startswith("finance:"):
513
  import application_registry
514
  from modules.finance_apps import finance_application
 
544
  raise err(404, "unknown_template", "That template does not exist.")
545
  if not template["creatable"]:
546
  raise err(409, "template_unavailable", "Connect the required source and switch on its databases.")
547
+ studio = _content_studio() if module == "app" else None
548
+ if studio is not None and key == studio.TEMPLATE_KEY:
549
+ if not session.runtime.available():
550
+ raise err(503, "store_unavailable", "The tenant store is unavailable. Nothing was created.")
551
+ try:
552
+ return studio.create(session, body.get("name") or "")
553
+ except studio.ContentStudioError as exc:
554
+ raise err(409, "content_studio_refused", str(exc))
555
+ except PermissionError as exc:
556
+ raise err(403, "forbidden", str(exc))
557
  if module == "sheet":
558
  database = template["requires"][0]["database"] if template["requires"] else ""
559
  result = apply_sheet_template(key, {"database": database, "name": body.get("name", "")}, session)
 
562
  database = template["requires"][0]["database"]
563
  result = apply_template(key, {"table": database}, session)
564
  return {"address": "#/" + quote(database, safe=""), **result}
565
+ if module == "agents":
566
+ from routes_automation import create_automation
567
+
568
+ definition = next(row["definition"] for row in _agent_templates() if row["key"] == key)
569
+ result = create_automation(definition, session=session)
570
+ return {"address": "#/automation?agent=" + quote(result["automation"]["id"], safe=""), **result}
571
  # The existing wizard still owns its POST-then-PATCH share sequence. API clients may
572
  # dispatch creation here; the existing App writer retains schema and permission walls.
573
  result = create_nav_application({**body, "templateKind": key}, session=session)
574
  app = result["application"]
575
  return {"address": "#/app?database=" + quote(app["sourceDatabase"], safe="") + "&application=" + quote(app["id"], safe=""),
576
  **result}
577
+
578
+ # S67-T41: Content Studio's own routes (`/api/v1/content-studio/*`) ride this router because the
579
+ # App wires no mount of its own into main.py; GUARDED, so deleting content_studio.py drops the
580
+ # routes and nothing else (rule 13).
581
+ # ⚠ An inline guarded import, NOT a call to `_content_studio()`: this tail belongs to the
582
+ # dispatcher block, which `verify_home.py`'s W61-T23 declaration-order control moves ABOVE that
583
+ # helper's definition, where a call is a NameError that crashes the whole gate.
584
+ try:
585
+ import content_studio as _studio_module
586
+ except ImportError:
587
+ _studio_module = None
588
+ if _studio_module is not None and getattr(_studio_module, "router", None) is not None:
589
+ router.include_router(_studio_module.router)
platform/core/connector_spec.py CHANGED
@@ -278,7 +278,7 @@ AMAZON_TABLES = (
278
  # ⚠ THE MONEY-TO-ORDER BRIDGE β€” what turns "an order happened" into "this is what we were
279
  # actually paid for it". The V2 settlement report carries the order id per line.
280
  links=(
281
- ("amazon_order_id", _AMZN + "orders", "amazon_order_id", "many_to_one"),
282
  ("sku", _AMZN + "listings", "seller_sku", "many_to_one"),
283
  ),
284
  ),
@@ -320,6 +320,309 @@ AMAZON_TABLES = (
320
  # about this product", and a link would be misleading.
321
  links=(),
322
  ),
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
323
  )
324
 
325
 
 
278
  # ⚠ THE MONEY-TO-ORDER BRIDGE β€” what turns "an order happened" into "this is what we were
279
  # actually paid for it". The V2 settlement report carries the order id per line.
280
  links=(
281
+ ("order_id", _AMZN + "orders", "amazon_order_id", "many_to_one"),
282
  ("sku", _AMZN + "listings", "seller_sku", "many_to_one"),
283
  ),
284
  ),
 
320
  # about this product", and a link would be misleading.
321
  links=(),
322
  ),
323
+ ConnectorTableSpec(
324
+ key=_AMZN + "economics", native="analytics_economics_2024_03_15",
325
+ source="amzn_economics", label="Amazon economics", pinned="msku",
326
+ identity=("start_date", "msku", "marketplace_id"),
327
+ links=(("msku", _AMZN + "listings", "seller_sku", "many_to_one"),
328
+ ("child_asin", _AMZN + "listings", "asin1", "many_to_one")),
329
+ ),
330
+ ConnectorTableSpec(
331
+ key=_AMZN + "transactions", native="finances/2024-06-19/transactions",
332
+ source="amzn_transactions", label="Amazon financial transactions", pinned="transaction_id",
333
+ identity=("transaction_id", "item_position"),
334
+ links=(("amazon_order_id", _AMZN + "orders", "amazon_order_id", "many_to_one"),
335
+ ("sku", _AMZN + "listings", "seller_sku", "many_to_one"),
336
+ ("asin", _AMZN + "listings", "asin1", "many_to_one")),
337
+ ),
338
+ ConnectorTableSpec(
339
+ key=_AMZN + "fba_fee_estimates", native="GET_FBA_ESTIMATED_FBA_FEES_TXT_DATA",
340
+ source="amzn_fba_fee_estimates", label="Amazon FBA fee estimates", pinned="sku",
341
+ identity=("_snapshot_date", "sku", "amazon_store"),
342
+ links=(("sku", _AMZN + "listings", "seller_sku", "many_to_one"),
343
+ ("asin", _AMZN + "listings", "asin1", "many_to_one")),
344
+ ),
345
+ ConnectorTableSpec(
346
+ key=_AMZN + "referral_fees", native="GET_REFERRAL_FEE_PREVIEW_REPORT",
347
+ source="amzn_referral_fees", label="Amazon referral fee preview", pinned="seller_sku",
348
+ identity=("_snapshot_date", "seller_sku"),
349
+ links=(("seller_sku", _AMZN + "listings", "seller_sku", "many_to_one"),
350
+ ("asin", _AMZN + "listings", "asin1", "many_to_one")),
351
+ ),
352
+ ConnectorTableSpec(
353
+ key=_AMZN + "storage_fees", native="GET_FBA_STORAGE_FEE_CHARGES_DATA",
354
+ source="amzn_storage_fees", label="Amazon storage fees", pinned="asin",
355
+ identity=("month", "asin", "fnsku", "fulfillment_center"),
356
+ links=(("asin", _AMZN + "listings", "asin1", "many_to_one"),),
357
+ ),
358
+ ConnectorTableSpec(
359
+ key=_AMZN + "search_query_performance", native="GET_BRAND_ANALYTICS_SEARCH_QUERY_PERFORMANCE_REPORT",
360
+ source="amzn_search_query_performance", label="Amazon search query performance",
361
+ pinned="search_query_data_search_query",
362
+ identity=("start_date", "end_date", "asin", "search_query_data_search_query"),
363
+ links=(("asin", _AMZN + "listings", "asin1", "many_to_one"),),
364
+ ),
365
+ ConnectorTableSpec(
366
+ key=_AMZN + "long_term_storage_fees", native="GET_FBA_FULFILLMENT_LONGTERM_STORAGE_FEE_CHARGES_DATA",
367
+ source="amzn_long_term_storage_fees", label="Amazon long term storage fees", pinned="sku",
368
+ identity=("_period_start", "sku", "fnsku", "asin", "_report_row_number"),
369
+ links=(("sku", _AMZN + "listings", "seller_sku", "many_to_one"), ("asin", _AMZN + "listings", "asin1", "many_to_one"),),
370
+ ),
371
+ ConnectorTableSpec(
372
+ key=_AMZN + "storage_overage_fees", native="GET_FBA_OVERAGE_FEE_CHARGES_DATA",
373
+ source="amzn_storage_overage_fees", label="Amazon storage overage fees", pinned="storage_type",
374
+ identity=("_period_start", "storage_type", "_report_row_number"),
375
+ links=(),
376
+ ),
377
+ ConnectorTableSpec(
378
+ key=_AMZN + "reimbursements", native="GET_FBA_REIMBURSEMENTS_DATA",
379
+ source="amzn_reimbursements", label="Amazon reimbursements", pinned="reimbursement_id",
380
+ identity=("_period_start", "reimbursement_id", "sku", "asin", "amazon_order_id", "_report_row_number"),
381
+ links=(("sku", _AMZN + "listings", "seller_sku", "many_to_one"), ("asin", _AMZN + "listings", "asin1", "many_to_one"), ("amazon_order_id", _AMZN + "orders", "amazon_order_id", "many_to_one"),),
382
+ ),
383
+ ConnectorTableSpec(
384
+ key=_AMZN + "fba_customer_returns", native="GET_FBA_FULFILLMENT_CUSTOMER_RETURNS_DATA",
385
+ source="amzn_fba_customer_returns", label="Amazon FBA customer returns", pinned="order_id",
386
+ identity=("_period_start", "order_id", "sku", "fnsku", "asin", "_report_row_number"),
387
+ links=(("order_id", _AMZN + "orders", "amazon_order_id", "many_to_one"), ("sku", _AMZN + "listings", "seller_sku", "many_to_one"), ("asin", _AMZN + "listings", "asin1", "many_to_one"),),
388
+ ),
389
+ ConnectorTableSpec(
390
+ key=_AMZN + "replacements", native="GET_FBA_FULFILLMENT_CUSTOMER_SHIPMENT_REPLACEMENT_DATA",
391
+ source="amzn_replacements", label="Amazon replacements", pinned="replacement_amazon_order_id",
392
+ identity=("_period_start", "replacement_amazon_order_id", "original_amazon_order_id", "sku", "asin", "_report_row_number"),
393
+ links=(("sku", _AMZN + "listings", "seller_sku", "many_to_one"), ("asin", _AMZN + "listings", "asin1", "many_to_one"),),
394
+ ),
395
+ ConnectorTableSpec(
396
+ key=_AMZN + "fba_shipped_sales", native="GET_FBA_FULFILLMENT_CUSTOMER_SHIPMENT_SALES_DATA",
397
+ source="amzn_fba_shipped_sales", label="Amazon FBA shipped sales", pinned="amazon_order_id",
398
+ identity=("_period_start", "amazon_order_id", "sku", "fnsku", "asin", "_report_row_number"),
399
+ links=(("amazon_order_id", _AMZN + "orders", "amazon_order_id", "many_to_one"), ("sku", _AMZN + "listings", "seller_sku", "many_to_one"), ("asin", _AMZN + "listings", "asin1", "many_to_one"),),
400
+ ),
401
+ ConnectorTableSpec(
402
+ key=_AMZN + "fba_shipment_promotions", native="GET_FBA_FULFILLMENT_CUSTOMER_SHIPMENT_PROMOTION_DATA",
403
+ source="amzn_fba_shipment_promotions", label="Amazon FBA shipment promotions", pinned="amazon_order_id",
404
+ identity=("_period_start", "amazon_order_id", "sku", "_report_row_number"),
405
+ links=(("amazon_order_id", _AMZN + "orders", "amazon_order_id", "many_to_one"),),
406
+ ),
407
+ ConnectorTableSpec(
408
+ key=_AMZN + "inventory_ledger_detail", native="GET_LEDGER_DETAIL_VIEW_DATA",
409
+ source="amzn_inventory_ledger_detail", label="Amazon inventory ledger detail", pinned="msku",
410
+ identity=("_period_start", "msku", "fnsku", "asin", "_report_row_number"),
411
+ links=(("msku", _AMZN + "listings", "seller_sku", "many_to_one"), ("asin", _AMZN + "listings", "asin1", "many_to_one"),),
412
+ ),
413
+ ConnectorTableSpec(
414
+ key=_AMZN + "inventory_planning", native="GET_FBA_INVENTORY_PLANNING_DATA",
415
+ source="amzn_inventory_planning", label="Amazon inventory planning", pinned="sku",
416
+ identity=("_snapshot_date", "sku", "fnsku", "asin", "_report_row_number"),
417
+ links=(("sku", _AMZN + "listings", "seller_sku", "many_to_one"), ("asin", _AMZN + "listings", "asin1", "many_to_one"),),
418
+ ),
419
+ ConnectorTableSpec(
420
+ key=_AMZN + "manage_inventory", native="GET_FBA_MYI_UNSUPPRESSED_INVENTORY_DATA",
421
+ source="amzn_manage_inventory", label="Amazon manage inventory", pinned="sku",
422
+ identity=("_snapshot_date", "sku", "fnsku", "asin", "_report_row_number"),
423
+ links=(("sku", _AMZN + "listings", "seller_sku", "many_to_one"), ("asin", _AMZN + "listings", "asin1", "many_to_one"),),
424
+ ),
425
+ ConnectorTableSpec(
426
+ key=_AMZN + "reserved_inventory", native="GET_RESERVED_INVENTORY_DATA",
427
+ source="amzn_reserved_inventory", label="Amazon reserved inventory", pinned="sku",
428
+ identity=("_snapshot_date", "sku", "fnsku", "asin", "_report_row_number"),
429
+ links=(("sku", _AMZN + "listings", "seller_sku", "many_to_one"), ("asin", _AMZN + "listings", "asin1", "many_to_one"),),
430
+ ),
431
+ ConnectorTableSpec(
432
+ key=_AMZN + "restock_recommendations", native="GET_RESTOCK_INVENTORY_RECOMMENDATIONS_REPORT",
433
+ source="amzn_restock_recommendations", label="Amazon restock recommendations", pinned="merchant_sku",
434
+ identity=("_snapshot_date", "merchant_sku", "fnsku", "asin", "_report_row_number"),
435
+ links=(("merchant_sku", _AMZN + "listings", "seller_sku", "many_to_one"),
436
+ ("asin", _AMZN + "listings", "asin1", "many_to_one")),
437
+ ),
438
+ ConnectorTableSpec(
439
+ key=_AMZN + "stranded_inventory", native="GET_STRANDED_INVENTORY_UI_DATA",
440
+ source="amzn_stranded_inventory", label="Amazon stranded inventory", pinned="sku",
441
+ identity=("_snapshot_date", "sku", "fnsku", "asin", "_report_row_number"),
442
+ links=(("sku", _AMZN + "listings", "seller_sku", "many_to_one"), ("asin", _AMZN + "listings", "asin1", "many_to_one"),),
443
+ ),
444
+ ConnectorTableSpec(
445
+ key=_AMZN + "inbound_noncompliance", native="GET_FBA_FULFILLMENT_INBOUND_NONCOMPLIANCE_DATA",
446
+ source="amzn_inbound_noncompliance", label="Amazon inbound noncompliance", pinned="shipment_id",
447
+ identity=("_period_start", "shipment_id", "sku", "asin", "_report_row_number"),
448
+ links=(("sku", _AMZN + "listings", "seller_sku", "many_to_one"), ("asin", _AMZN + "listings", "asin1", "many_to_one"),),
449
+ ),
450
+ ConnectorTableSpec(
451
+ key=_AMZN + "inventory_expiration_dates", native="GET_FBA_FULFILLMENT_INVENTORY_EXPIRATION_DATE_DATA",
452
+ source="amzn_inventory_expiration_dates", label="Amazon inventory expiration dates", pinned="sku",
453
+ identity=("_snapshot_date", "sku", "fnsku", "_report_row_number"),
454
+ links=(("sku", _AMZN + "listings", "seller_sku", "many_to_one"),),
455
+ ),
456
+ ConnectorTableSpec(
457
+ key=_AMZN + "recommended_removals", native="GET_FBA_RECOMMENDED_REMOVAL_DATA",
458
+ source="amzn_recommended_removals", label="Amazon recommended removals", pinned="sku",
459
+ identity=("_snapshot_date", "sku", "fnsku", "asin", "_report_row_number"),
460
+ links=(("sku", _AMZN + "listings", "seller_sku", "many_to_one"), ("asin", _AMZN + "listings", "asin1", "many_to_one"),),
461
+ ),
462
+ ConnectorTableSpec(
463
+ key=_AMZN + "removal_orders", native="GET_FBA_FULFILLMENT_REMOVAL_ORDER_DETAIL_DATA",
464
+ source="amzn_removal_orders", label="Amazon removal orders", pinned="order_id",
465
+ identity=("_period_start", "order_id", "sku", "fnsku", "_report_row_number"),
466
+ links=(("sku", _AMZN + "listings", "seller_sku", "many_to_one"),),
467
+ ),
468
+ ConnectorTableSpec(
469
+ key=_AMZN + "removal_shipments", native="GET_FBA_FULFILLMENT_REMOVAL_SHIPMENT_DETAIL_DATA",
470
+ source="amzn_removal_shipments", label="Amazon removal shipments", pinned="order_id",
471
+ identity=("_period_start", "order_id", "sku", "fnsku", "_report_row_number"),
472
+ links=(("sku", _AMZN + "listings", "seller_sku", "many_to_one"),),
473
+ ),
474
+ ConnectorTableSpec(
475
+ key=_AMZN + "subscribe_save_forecast", native="GET_FBA_SNS_FORECAST_DATA",
476
+ source="amzn_subscribe_save_forecast", label="Amazon Subscribe and Save forecast", pinned="sku",
477
+ identity=("_snapshot_date", "sku", "asin", "_report_row_number"),
478
+ links=(("sku", _AMZN + "listings", "seller_sku", "many_to_one"), ("asin", _AMZN + "listings", "asin1", "many_to_one"),),
479
+ ),
480
+ ConnectorTableSpec(
481
+ key=_AMZN + "subscribe_save_performance", native="GET_FBA_SNS_PERFORMANCE_DATA",
482
+ source="amzn_subscribe_save_performance", label="Amazon Subscribe and Save performance", pinned="sku",
483
+ identity=("_period_start", "sku", "asin", "_report_row_number"),
484
+ links=(("sku", _AMZN + "listings", "seller_sku", "many_to_one"), ("asin", _AMZN + "listings", "asin1", "many_to_one"),),
485
+ ),
486
+ ConnectorTableSpec(
487
+ key=_AMZN + "remote_fulfillment_eligibility", native="GET_REMOTE_FULFILLMENT_ELIGIBILITY",
488
+ source="amzn_remote_fulfillment_eligibility", label="Amazon remote fulfillment eligibility", pinned="sku",
489
+ identity=("_snapshot_date", "sku", "asin", "_report_row_number"),
490
+ links=(), # Measured header has no product key; do not declare a guessed join.
491
+ ),
492
+ ConnectorTableSpec(
493
+ key=_AMZN + "search_catalog_performance", native="GET_BRAND_ANALYTICS_SEARCH_CATALOG_PERFORMANCE_REPORT",
494
+ source="amzn_search_catalog_performance", label="Amazon search catalog performance", pinned="asin",
495
+ identity=("_period_start", "asin", "_report_row_number"),
496
+ links=(("asin", _AMZN + "listings", "asin1", "many_to_one"),),
497
+ ),
498
+ ConnectorTableSpec(
499
+ key=_AMZN + "market_basket", native="GET_BRAND_ANALYTICS_MARKET_BASKET_REPORT",
500
+ source="amzn_market_basket", label="Amazon market basket", pinned="asin",
501
+ identity=("_period_start", "asin", "purchased_with_asin", "_report_row_number"),
502
+ links=(("asin", _AMZN + "listings", "asin1", "many_to_one"),),
503
+ ),
504
+ ConnectorTableSpec(
505
+ key=_AMZN + "repeat_purchase", native="GET_BRAND_ANALYTICS_REPEAT_PURCHASE_REPORT",
506
+ source="amzn_repeat_purchase", label="Amazon repeat purchase", pinned="asin",
507
+ identity=("_period_start", "asin", "_report_row_number"),
508
+ links=(("asin", _AMZN + "listings", "asin1", "many_to_one"),),
509
+ ),
510
+ ConnectorTableSpec(
511
+ key=_AMZN + "promotion_performance", native="GET_PROMOTION_PERFORMANCE_REPORT",
512
+ source="amzn_promotion_performance", label="Amazon promotion performance", pinned="promotion_id",
513
+ identity=("_period_start", "promotion_id", "_report_row_number"),
514
+ links=(),
515
+ ),
516
+ ConnectorTableSpec(
517
+ key=_AMZN + "coupon_performance", native="GET_COUPON_PERFORMANCE_REPORT",
518
+ source="amzn_coupon_performance", label="Amazon coupon performance", pinned="coupon_id",
519
+ identity=("_period_start", "coupon_id", "_report_row_number"),
520
+ links=(),
521
+ ),
522
+ ConnectorTableSpec(
523
+ key=_AMZN + "account_health", native="GET_V2_SELLER_PERFORMANCE_REPORT",
524
+ source="amzn_account_health", label="Amazon account health", pinned="marketplace_id",
525
+ identity=("_snapshot_date", "marketplace_id", "_json_root", "_report_row_number"),
526
+ links=(),
527
+ ),
528
+ ConnectorTableSpec(
529
+ key=_AMZN + "inactive_listings", native="GET_MERCHANT_LISTINGS_INACTIVE_DATA",
530
+ source="amzn_inactive_listings", label="Amazon inactive listings", pinned="seller_sku",
531
+ identity=("_snapshot_date", "seller_sku", "asin1", "_report_row_number"),
532
+ links=(("seller_sku", _AMZN + "listings", "seller_sku", "many_to_one"),
533
+ ("asin1", _AMZN + "listings", "asin1", "many_to_one")),
534
+ ),
535
+ ConnectorTableSpec(
536
+ key=_AMZN + "cancelled_listings", native="GET_MERCHANT_CANCELLED_LISTINGS_DATA",
537
+ source="amzn_cancelled_listings", label="Amazon cancelled listings", pinned="seller_sku",
538
+ identity=("_snapshot_date", "seller_sku", "asin1", "_report_row_number"),
539
+ links=(("seller_sku", _AMZN + "listings", "seller_sku", "many_to_one"),
540
+ ("asin1", _AMZN + "listings", "asin1", "many_to_one")),
541
+ ),
542
+ ConnectorTableSpec(
543
+ key=_AMZN + "suppressed_listings", native="GET_MERCHANTS_LISTINGS_FYP_REPORT",
544
+ source="amzn_suppressed_listings", label="Amazon suppressed listings", pinned="sku",
545
+ identity=("_snapshot_date", "sku", "asin", "_report_row_number"),
546
+ links=(("sku", _AMZN + "listings", "seller_sku", "many_to_one"), ("asin", _AMZN + "listings", "asin1", "many_to_one"),),
547
+ ),
548
+ ConnectorTableSpec(
549
+ key=_AMZN + "listing_defects", native="GET_MERCHANT_LISTINGS_DEFECT_DATA",
550
+ source="amzn_listing_defects", label="Amazon listing defects", pinned="sku",
551
+ identity=("_snapshot_date", "sku", "_report_row_number"),
552
+ links=(("sku", _AMZN + "listings", "seller_sku", "many_to_one"),),
553
+ ),
554
+ ConnectorTableSpec(
555
+ key=_AMZN + "sold_listings", native="GET_CONVERGED_FLAT_FILE_SOLD_LISTINGS_DATA",
556
+ source="amzn_sold_listings", label="Amazon sold listings", pinned="sku",
557
+ identity=("_snapshot_date", "sku", "_report_row_number"),
558
+ links=(("sku", _AMZN + "listings", "seller_sku", "many_to_one"),),
559
+ ),
560
+ ConnectorTableSpec(
561
+ key=_AMZN + "mfn_returns", native="GET_FLAT_FILE_RETURNS_DATA_BY_RETURN_DATE",
562
+ source="amzn_mfn_returns", label="Amazon MFN returns", pinned="order_id",
563
+ identity=("_period_start", "order_id", "sku", "asin", "_report_row_number"),
564
+ links=(("order_id", _AMZN + "orders", "amazon_order_id", "many_to_one"), ("merchant_sku", _AMZN + "listings", "seller_sku", "many_to_one"), ("asin", _AMZN + "listings", "asin1", "many_to_one"),),
565
+ ),
566
+ ConnectorTableSpec(
567
+ key=_AMZN + "b2b_recommended", native="GET_B2B_PRODUCT_OPPORTUNITIES_RECOMMENDED_FOR_YOU",
568
+ source="amzn_b2b_recommended", label="Amazon B2B product opportunities", pinned="asin",
569
+ identity=("_snapshot_date", "asin", "_report_row_number"),
570
+ links=(("asin", _AMZN + "listings", "asin1", "many_to_one"),),
571
+ ),
572
+ ConnectorTableSpec(
573
+ key=_AMZN + "b2b_not_yet_on_amazon", native="GET_B2B_PRODUCT_OPPORTUNITIES_NOT_YET_ON_AMAZON",
574
+ source="amzn_b2b_not_yet_on_amazon", label="Amazon B2B products not yet on Amazon", pinned="ean",
575
+ identity=("_snapshot_date", "ean", "upc", "part_number", "_report_row_number"),
576
+ links=(),
577
+ ),
578
+ ConnectorTableSpec(
579
+ key=_AMZN + "geo_opportunities", native="GET_FLAT_FILE_GEO_OPPORTUNITIES",
580
+ source="amzn_geo_opportunities", label="Amazon geo opportunities", pinned="asin",
581
+ identity=("_snapshot_date", "asin", "_report_row_number"),
582
+ links=(("asin", _AMZN + "listings", "asin1", "many_to_one"),),
583
+ ),
584
+ ConnectorTableSpec(
585
+ key=_AMZN + "out_of_stock_mfn", native="GET_OUT_OF_STOCK_MFN_INFORMATION",
586
+ source="amzn_out_of_stock_mfn", label="Amazon out of stock MFN listings", pinned="sku",
587
+ identity=("_snapshot_date", "sku", "asin", "_report_row_number"),
588
+ links=(("sku", _AMZN + "listings", "seller_sku", "many_to_one"), ("asin", _AMZN + "listings", "asin1", "many_to_one"),),
589
+ ),
590
+ ConnectorTableSpec(
591
+ key=_AMZN + "b2b_quotes", native="RFQD_BULK_DOWNLOAD",
592
+ source="amzn_b2b_quotes", label="Amazon B2B quantity discounts", pinned="sku",
593
+ identity=("_snapshot_date", "sku", "asin", "_report_row_number"),
594
+ links=(("sku", _AMZN + "listings", "seller_sku", "many_to_one"), ("asin", _AMZN + "listings", "asin1", "many_to_one"),),
595
+ ),
596
+ ConnectorTableSpec(
597
+ key=_AMZN + "fee_discounts", native="FEE_DISCOUNTS_REPORT",
598
+ source="amzn_fee_discounts", label="Amazon fee discounts", pinned="sku",
599
+ identity=("_snapshot_date", "sku", "asin", "_report_row_number"),
600
+ links=(), # The measured empty report supplies metadata only.
601
+ ),
602
+ ConnectorTableSpec(
603
+ key=_AMZN + "order_metrics", native="sales/v1/orderMetrics",
604
+ source="amzn_order_metrics", label="Amazon order metrics", pinned="interval",
605
+ identity=("_period_start", "interval", "_report_row_number"),
606
+ links=(),
607
+ ),
608
+ ConnectorTableSpec(
609
+ key=_AMZN + "fba_inventory", native="fba/inventory/v1/summaries",
610
+ source="amzn_fba_inventory", label="Amazon FBA inventory summaries", pinned="seller_sku",
611
+ identity=("_snapshot_date", "seller_sku", "fn_sku", "asin", "_report_row_number"),
612
+ links=(("seller_sku", _AMZN + "listings", "seller_sku", "many_to_one"), ("asin", _AMZN + "listings", "asin1", "many_to_one"),),
613
+ ),
614
+ ConnectorTableSpec(
615
+ key=_AMZN + "listing_items", native="listings/2021-08-01/items/{sellerId}",
616
+ source="amzn_listing_items", label="Amazon listing status and issues", pinned="sku",
617
+ identity=("_snapshot_date", "sku", "_report_row_number"),
618
+ links=(("sku", _AMZN + "listings", "seller_sku", "many_to_one"),),
619
+ ),
620
+ ConnectorTableSpec(
621
+ key=_AMZN + "catalog_items", native="catalog/2022-04-01/items",
622
+ source="amzn_catalog_items", label="Amazon catalog attributes", pinned="asin",
623
+ identity=("_snapshot_date", "asin", "_report_row_number"),
624
+ links=(("asin", _AMZN + "listings", "asin1", "many_to_one"),),
625
+ ),
626
  )
627
 
628
 
platform/core/daily_sales_repair.py CHANGED
@@ -40,6 +40,39 @@ _SUMMARY_RATIO_CASCADES = {
40
  "R71": "=R69-R70", "R72": "=R71/R70", "R73": "=R72-Q72",
41
  }
42
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
43
 
44
  def operation_for(workbook_id):
45
  """Bind a repair operation to the workbook the caller names."""
@@ -141,10 +174,27 @@ def _validate_binding(name, binding, *, window_kind=None):
141
  # Accept the same explicit, nonempty conjunction used by rows 108-113. Requiring
142
  # both referenced date bounds keeps a missing period from becoming an unbounded sum.
143
  nodes = tree.get("nodes") if isinstance(tree, dict) else None
144
- has_filter = (isinstance(nodes, list) and bool(nodes) and tree.get("conj") == "and"
145
- and all(isinstance(n, dict) and n.get("colId") and n.get("op")
146
- and ("value" in n or isinstance(n.get("valueRef"), dict))
147
- for n in nodes))
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
148
  if not has_view and not has_filter:
149
  raise RepairRefusal(
150
  "repair_binding_scope_required",
@@ -158,7 +208,7 @@ def _validate_binding(name, binding, *, window_kind=None):
158
  period = binding.get("period")
159
  if period is None and has_filter and not has_view:
160
  bounds = {}
161
- for node in nodes:
162
  slot = node.get("valueRef")
163
  if slot is not None:
164
  if not isinstance(slot, dict) or set(slot) != {"ref"}:
@@ -281,6 +331,70 @@ def check_summary_formula_errors(book):
281
  "negative_control": {"planted": len(expected), "found": len(found)}}
282
 
283
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
284
  def _formula_error_comparison(before, after, edits, reported_errors=()):
285
  """Reject new errors anywhere and every residual error in explicitly planned cells.
286
 
@@ -331,6 +445,21 @@ def _cell_map(book):
331
  for address, cell in (tab.get("cells") or {}).items()}
332
 
333
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
334
  def _checked_projection(projected, edits, formula_check):
335
  """Make formula output part of the plan, never an unpersisted preflight side effect."""
336
  if not callable(formula_check):
@@ -351,6 +480,10 @@ def _checked_projection(projected, edits, formula_check):
351
  if before.get(slot) == after.get(slot):
352
  continue
353
  row = planned.get(slot)
 
 
 
 
354
  if row is None or row["kind"] == "recalculated":
355
  previous, recalculated = before.get(slot), after.get(slot)
356
  # Recalculation may refresh a healthy formula's result, but may not repair
@@ -365,7 +498,8 @@ def _checked_projection(projected, edits, formula_check):
365
  != {key: recalculated[key] for key in _VISUAL_CELL_KEYS if key in recalculated}:
366
  raise RepairRefusal(
367
  "repair_unplanned_formula_change",
368
- "Formula validation would change a cell outside the supplied source diff.")
 
369
  desired = sheets.clean_cell(_preserve_visual(previous, recalculated), slot[1])
370
  if row is None:
371
  sheet = sheet_names[slot[0]]
@@ -382,7 +516,8 @@ def _checked_projection(projected, edits, formula_check):
382
  or row["after"].get("kind") != "formula":
383
  raise RepairRefusal(
384
  "repair_unplanned_formula_change",
385
- "Formula validation would change a cell outside the supplied source diff.")
 
386
  recalculated = after.get(slot)
387
  if not isinstance(recalculated, dict) \
388
  or (row["kind"] == "source"
@@ -441,10 +576,10 @@ def _summary_date(value):
441
  "Every summary date must be a real ISO date from the sheet or mirror.") from exc
442
 
443
 
444
- def _resolved(binding, book, resolve_binding):
445
  if not callable(resolve_binding):
446
  raise RepairRefusal("repair_binding_resolver_required", "The summary needs its scoped source resolver.")
447
- result = resolve_binding(copy.deepcopy(binding), book, _tab(book, SUMMARY_TAB))
448
  if not isinstance(result, dict) or "value" not in result or result.get("value") is None:
449
  raise RepairRefusal("repair_binding_unreadable", "A summary source did not return a value.")
450
  if result.get("limits"):
@@ -599,9 +734,9 @@ def _build_summary_plan(current, request, resolve_binding, formula_check):
599
  "scope": {"filter": {"conj": "and", "nodes": [
600
  {"colId": "invoice_date", "op": "within", "dateWindow": {"kind": kind},
601
  "valueRef": {"ref": end_ref}},
602
- {"colId": "move_type", "op": "eq", "value": "out_invoice"}]}},
603
- "related": {"via": "customer_link", "filter": {"conj": "and", "nodes": [
604
- {"colId": "pricelist", "op": "eq", "valueRef": {"ref": scope_ref}}]}}}
605
  _validate_binding(col + str(row_number), binding)
606
  value = _resolved(binding, projected, resolve_binding)
607
  if type(value) not in (int, float) or not math.isfinite(value):
@@ -685,6 +820,919 @@ def validate(book, *, windows, mirror_as_of, partner_ids, invoice_validate, form
685
  return checks
686
 
687
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
688
  def _build_plan(current, request, resolve_binding):
689
  if not isinstance(request.get("source_cells"), list) or not request["source_cells"]:
690
  raise RepairRefusal(
@@ -877,15 +1925,62 @@ def execute(st, backup_store, request, *, target_workbook_id=TARGET_WORKBOOK_ID,
877
  raise RepairRefusal("repair_revision_mismatch",
878
  "The named workbook is not at the expected repair revision.")
879
  digest = sheets.workbook_digest(current)
 
 
 
880
  summary_evidence = None
881
  summary_errors_before = None
882
- if "summary_block" in request:
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
883
  summary_errors_before = check_summary_formula_errors(current)
884
  edits, summary_evidence = _build_summary_plan(current, request, resolve_binding, formula_check)
885
  else:
886
  edits = _build_plan(current, request, resolve_binding)
887
  projected = _project(current, edits)
888
  projected, projected_errors = _checked_projection(projected, edits, formula_check)
 
 
 
 
889
  comparison = _formula_error_comparison(current, projected, edits, projected_errors)
890
  summary_errors_after = (check_summary_formula_errors(projected)
891
  if summary_evidence is not None else None)
@@ -896,7 +1991,11 @@ def execute(st, backup_store, request, *, target_workbook_id=TARGET_WORKBOOK_ID,
896
  "The projected repair introduces a formula error or leaves one in its repair scope, "
897
  "so no repair was applied.")
898
  visual_before = visual_metadata(current)
899
- if visual_metadata(projected) != visual_before:
 
 
 
 
900
  raise RepairRefusal("repair_visual_metadata_changed",
901
  "The projected repair changes Daily Sales visual metadata.")
902
 
@@ -928,9 +2027,13 @@ def execute(st, backup_store, request, *, target_workbook_id=TARGET_WORKBOOK_ID,
928
  or _cell_map(checked_readback) != _cell_map(readback):
929
  raise RepairRefusal("repair_readback_formula_state",
930
  "The repaired workbook did not read back calculation-current.")
931
- if visual_metadata(readback) != visual_before:
932
  raise RepairRefusal("repair_readback_visual_metadata",
933
  "Daily Sales visual metadata changed during repair.")
 
 
 
 
934
  if summary_evidence is not None:
935
  tab = _tab(readback, SUMMARY_TAB)
936
  if any((tab.get("cells", {}).get(col + str(row)) or {}).get("kind")
@@ -963,8 +2066,36 @@ def execute(st, backup_store, request, *, target_workbook_id=TARGET_WORKBOOK_ID,
963
  "bindings": bindings,
964
  "formula_errors": formula_error_census(readback),
965
  "formula_error_comparison": readback_comparison,
 
966
  "visual_metadata_preserved": True,
967
  }
968
  if summary_evidence is not None:
969
  result["summary_block"] = summary_evidence
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
970
  return result
 
40
  "R71": "=R69-R70", "R72": "=R71/R70", "R73": "=R72-Q72",
41
  }
42
 
43
+ # Exact authored formulas at the staging snapshot; an owner edit is never an oracle.
44
+ # Q64 deliberately keeps its formula but participates in the recalculation/write plan.
45
+ _FORMULA_REPAIRS = {
46
+ ("Summary (SR)", "A1", '=CONCATENATE(Print!C6," Sales Report")'):
47
+ ('=CONCATENATE(C6," Sales Report")', "resolved", "C6 holds this tab's sales representative."),
48
+ ("Summary (SR)", "W8", "=Print!C6"):
49
+ ("=C6", "resolved", "C6 holds this tab's sales representative."),
50
+ ("Summary (SR)", "Q39", '=_xlfn.XLOOKUP(Print!$C$6,Print!C14:C30,Print!E14:E30,"")'):
51
+ ('=""', "deliberate blank", "The Print target table is absent from this workbook."),
52
+ ("Summary (SR)", "O43", "=Q39-O41"):
53
+ ('=IFERROR(Q39-O41,"")', "deliberate blank", "The target in Q39 is unavailable."),
54
+ ("Summary (SR)", "R64", '=COUNTIF(Uncollectible!E7:E1000,"*")'):
55
+ ('=""', "deliberate blank", "The Uncollectible tab is absent from this workbook."),
56
+ ("Summary (SR)", "R65", '=COUNTIF(Closed!E7:E1000,"*")'):
57
+ ('=""', "deliberate blank", "The Closed tab is absent from this workbook."),
58
+ ("Summary (SR)", "Q64", "=R64"):
59
+ ("=R64", "deliberate blank", "The unchanged reference follows the deliberate blank in R64."),
60
+ ("Summary (SR)", "F71", "=(F77/F66) * (365 / (F$8 - DATE(YEAR(F$8),1,1) + 1))"):
61
+ ('=IFERROR((F77/F66) * (365 / (F$8 - DATE(YEAR(F$8),1,1) + 1)),"")',
62
+ "deliberate blank", "The ratio is blank while its denominator is unavailable."),
63
+ ("Summary (SR)", "F72", "=(F78/F69) * (365 / (F$8 - DATE(YEAR(F$8),1,1) + 1))"):
64
+ ('=IFERROR((F78/F69) * (365 / (F$8 - DATE(YEAR(F$8),1,1) + 1)),"")',
65
+ "deliberate blank", "The ratio is blank while its denominator is unavailable."),
66
+ ("Summary (SR)", "F73", "=F71-F72"):
67
+ ('=IFERROR(F71-F72,"")', "deliberate blank", "The difference is blank while either ratio is unavailable."),
68
+ ("Summary (SR)", "Q63", "=R63"):
69
+ ('=IF(ISBLANK(R63),"",R63)', "deliberate blank", "An empty R63 has no figure to report."),
70
+ ("Summary (SR)", "R68", "=R67-Q67"):
71
+ ('=IFERROR(R67-Q67,"")', "deliberate blank", "The difference is blank while R67 is unavailable."),
72
+ (SUMMARY_TAB, "F52", "=G51-F51"):
73
+ ('=IFERROR(G51-F51,"")', "deliberate blank", "The difference is blank while F51 is unavailable."),
74
+ }
75
+
76
 
77
  def operation_for(workbook_id):
78
  """Bind a repair operation to the workbook the caller names."""
 
174
  # Accept the same explicit, nonempty conjunction used by rows 108-113. Requiring
175
  # both referenced date bounds keeps a missing period from becoming an unbounded sum.
176
  nodes = tree.get("nodes") if isinstance(tree, dict) else None
177
+ def conjunction_leaves(items):
178
+ if not isinstance(items, list) or not items:
179
+ return None
180
+ leaves = []
181
+ for node in items:
182
+ if not isinstance(node, dict):
183
+ return None
184
+ if "children" in node:
185
+ children = conjunction_leaves(node["children"])
186
+ if node.get("conj", "and") != "and" or children is None:
187
+ return None
188
+ leaves.extend(children)
189
+ elif node.get("colId") and node.get("op") and (
190
+ "value" in node or isinstance(node.get("valueRef"), dict)):
191
+ leaves.append(node)
192
+ else:
193
+ return None
194
+ return leaves
195
+
196
+ leaves = conjunction_leaves(nodes)
197
+ has_filter = leaves is not None and tree.get("conj") == "and"
198
  if not has_view and not has_filter:
199
  raise RepairRefusal(
200
  "repair_binding_scope_required",
 
208
  period = binding.get("period")
209
  if period is None and has_filter and not has_view:
210
  bounds = {}
211
+ for node in leaves:
212
  slot = node.get("valueRef")
213
  if slot is not None:
214
  if not isinstance(slot, dict) or set(slot) != {"ref"}:
 
331
  "negative_control": {"planted": len(expected), "found": len(found)}}
332
 
333
 
334
+ def _recalculated_formula_cycles(book, formula_check):
335
+ checked = copy.deepcopy(book)
336
+ try:
337
+ result = formula_check(checked)
338
+ except Exception as exc:
339
+ raise RepairRefusal("repair_cycle_check_failed",
340
+ "The formula engine could not complete the cycle check.") from exc
341
+ if isinstance(result, dict) and isinstance(result.get("book"), dict):
342
+ checked = result["book"]
343
+ # The engine gives all members of a cyclic component the same origin. Preserve
344
+ # authored tab names in the report; engine lookup keys are upper case.
345
+ groups = {}
346
+ for tab in checked.get("sheets") or ():
347
+ for address, cell in (tab.get("cells") or {}).items():
348
+ if not isinstance(cell, dict) or cell.get("kind") != "formula":
349
+ continue
350
+ error = cell.get("error") or (cell.get("refusal") if cell.get("state") == "refused" else None)
351
+ if not isinstance(error, dict) or error.get("code") != "circular_reference":
352
+ continue
353
+ label = str(tab.get("name") or "") + "!" + address
354
+ origin = str(error.get("origin") or label).upper()
355
+ groups.setdefault(origin, set()).add(label)
356
+ return sorted(sorted(addresses) for addresses in groups.values())
357
+
358
+
359
+ def formula_cycles(book, formula_check):
360
+ """Recalculate a copy and report the engine's circular references by component.
361
+
362
+ The injected checker is the same API-side engine used by the repair. Every call
363
+ first proves that both a two-cell cycle and a self-loop are detectable, without
364
+ trusting stored errors or changing the supplied workbook.
365
+ """
366
+ if not callable(formula_check):
367
+ raise RepairRefusal("repair_cycle_check_required",
368
+ "The cycle check requires a formula engine.")
369
+ probe = {"sheets": [{"id": "cycle-probe", "name": "Cycle probe", "cells": {
370
+ address: {"kind": "formula", "state": "stale", "formula": formula}
371
+ for address, formula in (("A1", "=B1"), ("B1", "=A1"), ("C1", "=C1"))
372
+ }}]}
373
+ expected = [["Cycle probe!A1", "Cycle probe!B1"], ["Cycle probe!C1"]]
374
+ found = _recalculated_formula_cycles(probe, formula_check)
375
+ if found != expected:
376
+ raise RepairRefusal("repair_cycle_census_blind",
377
+ "The cycle check failed to find its planted formula cycles.")
378
+ cycles = _recalculated_formula_cycles(book, formula_check)
379
+ return {"count": len(cycles), "cycles": cycles,
380
+ "negative_control": {"planted": len(expected), "found": len(found), "cycles": found}}
381
+
382
+
383
+ def _formula_cycle_comparison(before, after, formula_check):
384
+ """Diagnostic evidence only: keep execute's existing acceptance/refusal contract.
385
+
386
+ Legacy callers may omit the engine. A missing or blind checker is explicitly
387
+ reported as a named refusal, never represented as a successful zero census.
388
+ """
389
+ def report(book):
390
+ try:
391
+ return formula_cycles(book, formula_check)
392
+ except RepairRefusal as exc:
393
+ return {"refusal": {"code": exc.code, "message": exc.message}}
394
+
395
+ return {"before": report(before), "after": report(after)}
396
+
397
+
398
  def _formula_error_comparison(before, after, edits, reported_errors=()):
399
  """Reject new errors anywhere and every residual error in explicitly planned cells.
400
 
 
445
  for address, cell in (tab.get("cells") or {}).items()}
446
 
447
 
448
+ def _same_unplanned_error(previous, recalculated):
449
+ """True when recalculation leaves an erroring formula with the same formula and error code."""
450
+ def code(cell):
451
+ error = cell.get("error") or (cell.get("refusal") if cell.get("state") == "refused" else None)
452
+ return error.get("code") if isinstance(error, dict) else None
453
+
454
+ return (isinstance(previous, dict) and isinstance(recalculated, dict)
455
+ and previous.get("kind") == recalculated.get("kind") == "formula"
456
+ and isinstance(previous.get("formula"), str)
457
+ and previous.get("formula") == recalculated.get("formula")
458
+ and code(previous) is not None and code(previous) == code(recalculated)
459
+ and {key: previous[key] for key in _VISUAL_CELL_KEYS if key in previous}
460
+ == {key: recalculated[key] for key in _VISUAL_CELL_KEYS if key in recalculated})
461
+
462
+
463
  def _checked_projection(projected, edits, formula_check):
464
  """Make formula output part of the plan, never an unpersisted preflight side effect."""
465
  if not callable(formula_check):
 
480
  if before.get(slot) == after.get(slot):
481
  continue
482
  row = planned.get(slot)
483
+ if row is None and _same_unplanned_error(before.get(slot), after.get(slot)):
484
+ # An unrelated formula that was an error and recalculates to the SAME error is
485
+ # neither repaired nor broken by this plan, so it stays exactly as stored.
486
+ continue
487
  if row is None or row["kind"] == "recalculated":
488
  previous, recalculated = before.get(slot), after.get(slot)
489
  # Recalculation may refresh a healthy formula's result, but may not repair
 
498
  != {key: recalculated[key] for key in _VISUAL_CELL_KEYS if key in recalculated}:
499
  raise RepairRefusal(
500
  "repair_unplanned_formula_change",
501
+ "Formula validation would change " + str(sheet_names.get(slot[0]) or slot[0])
502
+ + "!" + slot[1] + ", a cell outside the supplied source diff.")
503
  desired = sheets.clean_cell(_preserve_visual(previous, recalculated), slot[1])
504
  if row is None:
505
  sheet = sheet_names[slot[0]]
 
516
  or row["after"].get("kind") != "formula":
517
  raise RepairRefusal(
518
  "repair_unplanned_formula_change",
519
+ "Formula validation would change " + str(sheet_names.get(slot[0]) or slot[0])
520
+ + "!" + slot[1] + ", a cell outside the supplied source diff.")
521
  recalculated = after.get(slot)
522
  if not isinstance(recalculated, dict) \
523
  or (row["kind"] == "source"
 
576
  "Every summary date must be a real ISO date from the sheet or mirror.") from exc
577
 
578
 
579
+ def _resolved(binding, book, resolve_binding, *, tab_name=SUMMARY_TAB):
580
  if not callable(resolve_binding):
581
  raise RepairRefusal("repair_binding_resolver_required", "The summary needs its scoped source resolver.")
582
+ result = resolve_binding(copy.deepcopy(binding), book, _tab(book, tab_name))
583
  if not isinstance(result, dict) or "value" not in result or result.get("value") is None:
584
  raise RepairRefusal("repair_binding_unreadable", "A summary source did not return a value.")
585
  if result.get("limits"):
 
734
  "scope": {"filter": {"conj": "and", "nodes": [
735
  {"colId": "invoice_date", "op": "within", "dateWindow": {"kind": kind},
736
  "valueRef": {"ref": end_ref}},
737
+ {"colId": "move_type", "op": "eq", "value": "out_invoice"},
738
+ {"colId": "related:customer_link:pricelist", "op": "eq",
739
+ "valueRef": {"ref": scope_ref}}]}}}
740
  _validate_binding(col + str(row_number), binding)
741
  value = _resolved(binding, projected, resolve_binding)
742
  if type(value) not in (int, float) or not math.isfinite(value):
 
820
  return checks
821
 
822
 
823
+ def _binding_ref_slots(value):
824
+ """Yield both operand slots at any depth without changing other binding metadata."""
825
+ if isinstance(value, dict):
826
+ for key, child in value.items():
827
+ if key in {"valueRef", "value2Ref"} and isinstance(child, dict):
828
+ yield child
829
+ yield from _binding_ref_slots(child)
830
+ elif isinstance(value, list):
831
+ for child in value:
832
+ yield from _binding_ref_slots(child)
833
+
834
+
835
+ def _has_legacy_related(value):
836
+ def linked(node):
837
+ if isinstance(node, dict):
838
+ key = node.get("colId")
839
+ return ((isinstance(key, str) and key.startswith("linked:"))
840
+ or any(linked(child) for child in node.values()))
841
+ return isinstance(node, list) and any(linked(child) for child in node)
842
+
843
+ return isinstance(value, dict) and ("related" in value or linked(value))
844
+
845
+
846
+ def _remap_cell(book, tab_name, ref):
847
+ """Read an explicitly supplied A1 ref, relative to the migrated cell's tab."""
848
+ match = re.fullmatch(r"(?:(?:'((?:[^']|'')+)'|([^'!]+))!)?(\$?[A-Za-z]{1,3}\$?[1-9][0-9]{0,6})",
849
+ ref) if isinstance(ref, str) else None
850
+ if not match:
851
+ raise RepairRefusal("repair_ref_remap_invalid", "Invalid remap reference " + str(ref) + ".")
852
+ name = (match[1].replace("''", "'") if match[1] else match[2]) or tab_name
853
+ try:
854
+ return (_tab(book, name).get("cells") or {}).get(_addr(match[3])) or {}
855
+ except RepairRefusal as exc:
856
+ raise RepairRefusal("repair_ref_remap_invalid",
857
+ "Remap reference " + ref + " cannot be read.") from exc
858
+
859
+
860
+ def _build_related_migration_plan(current, request, resolve_binding, formula_check=None):
861
+ """Migrate by content, proving explicit moved helpers and every stored answer first."""
862
+ from core.sheet_binding import BindingMetadataRefusal, rewrite_legacy_related
863
+
864
+ migration = request["related_migration"]
865
+ if not isinstance(migration, dict) or not isinstance(migration.get("ref_remap", {}), dict):
866
+ raise RepairRefusal("repair_ref_remap_invalid", "Supply a reference remap object for the migration.")
867
+ if any(key in request for key in ("summary_block", "source_cells", "binding_cells")):
868
+ raise RepairRefusal("repair_migration_plan_conflict", "Run the related migration as its own repair.")
869
+ remap = migration.get("ref_remap", {})
870
+ edits = []
871
+ for tab in current.get("sheets") or ():
872
+ for address, before in (tab.get("cells") or {}).items():
873
+ if not isinstance(before, dict) or before.get("kind") != "bound" \
874
+ or not isinstance(before.get("binding"), dict) \
875
+ or not _has_legacy_related(before["binding"]):
876
+ continue
877
+ try:
878
+ binding = rewrite_legacy_related(before["binding"], linked=True)
879
+ except BindingMetadataRefusal as exc:
880
+ raise RepairRefusal(exc.code, exc.message) from exc
881
+ if _has_legacy_related(binding):
882
+ raise RepairRefusal("related_invalid", "A nested legacy relationship cannot be migrated.")
883
+ edits.append({"kind": "binding", "name": tab["name"] + "!" + address,
884
+ "sheet": tab["name"], "address": address, "before": copy.deepcopy(before),
885
+ "after": {**copy.deepcopy(before), "binding": binding}})
886
+ # A retry with the same remap must be a no-op, not an unused-remap refusal.
887
+ if not edits:
888
+ return [], []
889
+ if not str(request.get("refreshed_at") or "").strip():
890
+ raise RepairRefusal("repair_timestamp_required", "The migration needs an explicit refresh timestamp.")
891
+ edits.sort(key=lambda row: (row["sheet"].casefold(), row["address"]))
892
+ recalculated = []
893
+
894
+ def current_after_recalculation(tab_name, ref, stored):
895
+ # A structure change leaves a helper formula `stale` until the next recalculation, so the
896
+ # formula engine, not the stored flag, decides whether the stored text is still its answer.
897
+ if not callable(formula_check):
898
+ return False
899
+ if not recalculated:
900
+ checked = formula_check(copy.deepcopy(current))
901
+ recalculated.append(checked.get("book") if isinstance(checked, dict) else None)
902
+ if not isinstance(recalculated[0], dict):
903
+ return False
904
+ again = _remap_cell(recalculated[0], tab_name, ref)
905
+ return (again.get("kind") == "formula" and again.get("state") == "fresh"
906
+ and not again.get("error") and not again.get("refusal")
907
+ and again.get("value") == stored.get("value"))
908
+
909
+ used = set()
910
+ for row in edits:
911
+ for slot in _binding_ref_slots(row["after"]["binding"]):
912
+ ref = slot.get("ref")
913
+ if not isinstance(ref, str) or ref not in remap:
914
+ continue
915
+ target = remap[ref]
916
+ source_cell = _remap_cell(current, row["sheet"], ref)
917
+ target_cell = _remap_cell(current, row["sheet"], target)
918
+ source_empty = (source_cell.get("kind", "empty") in {"empty", "literal"}
919
+ and source_cell.get("value") in (None, "")
920
+ and not any(source_cell.get(key) for key in ("formula", "binding", "text")))
921
+ current_text = target_cell.get("state") == "fresh" or (
922
+ target_cell.get("state") == "stale"
923
+ and current_after_recalculation(row["sheet"], target, target_cell))
924
+ if not source_empty or target_cell.get("kind") != "formula" \
925
+ or not str(target_cell.get("formula") or "").startswith("=") \
926
+ or not current_text \
927
+ or target_cell.get("error") or target_cell.get("refusal") \
928
+ or not isinstance(target_cell.get("value"), str) or not target_cell["value"].strip():
929
+ raise RepairRefusal(
930
+ "repair_ref_remap_invalid",
931
+ "Remap " + ref + " to " + str(target) + " on " + row["sheet"]
932
+ + " requires an empty source and a current formula with a non-empty string value.")
933
+ used.add(ref)
934
+ slot["ref"] = target
935
+ unused = set(remap) - used
936
+ if unused:
937
+ raise RepairRefusal("repair_ref_remap_unused",
938
+ "No migrated binding uses remap reference " + ", ".join(sorted(map(str, unused))) + ".")
939
+
940
+ projected = _project(current, edits)
941
+ migrated = []
942
+ for row in edits:
943
+ value = _resolved(row["after"]["binding"], projected, resolve_binding, tab_name=row["sheet"])
944
+ before_value = row["before"].get("value")
945
+ numeric = type(value) in (int, float) and type(before_value) in (int, float)
946
+ equal = (math.isfinite(value) and math.isfinite(before_value)
947
+ and math.isclose(value, before_value, rel_tol=1e-9, abs_tol=1e-6)) if numeric else (
948
+ type(value) is type(before_value) and value == before_value)
949
+ if not equal:
950
+ raise RepairRefusal("repair_migration_value_changed",
951
+ row["name"] + " would change from " + repr(before_value)
952
+ + " to " + repr(value) + "; no migration was applied.")
953
+ desired = _preserve_visual(row["before"], {
954
+ **row["after"], "kind": "bound", "state": "fresh", "value": value,
955
+ "display": str(value), "refreshed_at": request["refreshed_at"]})
956
+ cleaned = sheets.clean_cell(desired, row["address"])
957
+ if cleaned != desired:
958
+ raise RepairRefusal("repair_migration_metadata_changed",
959
+ row["name"] + " cannot be migrated without changing its saved metadata.")
960
+ row["after"] = cleaned
961
+ _tab(projected, row["sheet"])["cells"][row["address"]] = copy.deepcopy(cleaned)
962
+ migrated.append({"sheet": row["sheet"], "address": row["address"],
963
+ "before_value": before_value, "after_value": value})
964
+ return edits, migrated
965
+
966
+
967
+ def _formula_repair_census(book):
968
+ errors = formula_error_census(book)
969
+ return {name: {"count": sum(row["sheet"] == name for row in errors),
970
+ "errors": [row for row in errors if row["sheet"] == name]}
971
+ for name in (SUMMARY_TAB, "Summary (SR)")}
972
+
973
+
974
+ def _check_formula_repair_errors(book):
975
+ """All codes count, including blank, missing tabs and circular references."""
976
+ for name, address, _before in _FORMULA_REPAIRS:
977
+ cell = (_tab(book, name).get("cells") or {}).get(address) or {}
978
+ if cell.get("kind") != "formula" or cell.get("state") != "fresh" \
979
+ or cell.get("error") or cell.get("refusal") or cell.get("value") is None:
980
+ raise RepairRefusal("repair_error_cell_unresolved",
981
+ name + "!" + address + " did not recalculate to a non-error value.")
982
+ census = _formula_repair_census(book)
983
+ remaining = [row for tab in census.values() for row in tab["errors"]]
984
+ if remaining:
985
+ raise RepairRefusal("repair_formula_errors_remain",
986
+ "Formula errors remain: " + json.dumps(remaining, sort_keys=True) + ".")
987
+ return census
988
+
989
+
990
+ def _build_formula_repair_plan(current, request, formula_check):
991
+ """Restore the two stored marker masters and the exact, enumerated error formulas."""
992
+ from openpyxl.formula.translate import Translator
993
+
994
+ if not isinstance(request["formula_repair"], dict) or any(
995
+ key in request for key in ("related_migration", "summary_block", "source_cells", "binding_cells")):
996
+ raise RepairRefusal("repair_formula_plan_conflict", "Run the formula repair as its own repair.")
997
+ if not callable(formula_check):
998
+ raise RepairRefusal("repair_formula_check_required", "The formula repair requires a formula engine.")
999
+ edits, evidence = [], []
1000
+
1001
+ def append(name, address, formula):
1002
+ before = copy.deepcopy((_tab(current, name).get("cells") or {}).get(address))
1003
+ desired = None
1004
+ if formula is not None:
1005
+ desired = {**copy.deepcopy(before or {}), "kind": "formula",
1006
+ "formula": formula, "state": "stale", "value": None}
1007
+ for key in ("error", "refusal", "display"):
1008
+ desired.pop(key, None)
1009
+ after = _preserve_visual(before, desired)
1010
+ if after is not None:
1011
+ after = sheets.clean_cell(after, address)
1012
+ # Even unchanged formula text must be planned: Q64's old error is repaired by R64.
1013
+ if before != after:
1014
+ edits.append({"kind": "derived", "name": name + "!" + address,
1015
+ "sheet": name, "address": address, "before": before, "after": after})
1016
+
1017
+ def empty(cell):
1018
+ return (cell.get("kind", "empty") in {"empty", "literal"}
1019
+ and cell.get("value") in (None, "")
1020
+ and not any(cell.get(key) for key in ("formula", "binding", "text")))
1021
+
1022
+ cached_markers = []
1023
+ for name, marker_row, last_copy in ((SUMMARY_TAB, 4, 6), ("Summary (SR)", 5, 7)):
1024
+ cells = _tab(current, name).get("cells") or {}
1025
+ source, origin = "E" + str(marker_row), "F" + str(marker_row)
1026
+ stored = cells.get(source) or {}
1027
+ master = stored.get("formula") if stored.get("kind") == "formula" else None
1028
+ retry = empty(stored)
1029
+ if retry:
1030
+ # A completed repair retains the verbatim master in F, after clearing E.
1031
+ held = cells.get(origin) or {}
1032
+ master = held.get("formula") if held.get("kind") == "formula" else None
1033
+ if not isinstance(master, str) or not master.startswith('=IF(' + source + '="",'):
1034
+ raise RepairRefusal("repair_marker_master_missing",
1035
+ name + "!" + source + " must hold its self-referencing marker master.")
1036
+ translated = {origin: master}
1037
+ translated.update({chr(col) + str(marker_row): Translator(master, origin=origin).translate_formula(
1038
+ chr(col) + str(marker_row)) for col in range(ord("G"), ord("Q") + 1)})
1039
+ if retry and (not all(empty(cells.get("E" + str(row)) or {})
1040
+ for row in range(3, last_copy + 1)) or not all(
1041
+ (cells.get(address) or {}).get("kind") == "formula"
1042
+ and cells[address].get("formula") == formula for address, formula in translated.items())):
1043
+ raise RepairRefusal("repair_marker_master_missing",
1044
+ name + "!" + source + " is missing and the restored marker row is incomplete.")
1045
+ for address, formula in translated.items():
1046
+ before = cells.get(address) or {}
1047
+ exact_formula = before.get("kind") == "formula" and before.get("formula") == formula
1048
+ # Excel's cached month-end markers (H4 2026-06-30, M4 2026-07-31, Q4 2026-08-31 on
1049
+ # staging) are typed ISO dates; one is accepted only if the restored formula
1050
+ # recalculates to exactly it (checked after projection, below), never by address.
1051
+ cached_date = (before.get("kind") == "literal" and isinstance(before.get("value"), str)
1052
+ and re.fullmatch(r"\d{4}-\d{2}-\d{2}", before["value"]) is not None)
1053
+ if not (empty(before) or cached_date or exact_formula):
1054
+ raise RepairRefusal("repair_marker_row_occupied",
1055
+ name + "!" + address + " contains an owner value outside the marker repair.")
1056
+ if cached_date:
1057
+ cached_markers.append((name, address, before["value"]))
1058
+ append(name, address, formula)
1059
+ for row in range(3, last_copy + 1):
1060
+ append(name, "E" + str(row), None)
1061
+
1062
+ for (name, address, expected), (formula, resolution, reason) in _FORMULA_REPAIRS.items():
1063
+ before = (_tab(current, name).get("cells") or {}).get(address) or {}
1064
+ if before.get("kind") != "formula" or before.get("formula") not in (expected, formula):
1065
+ raise RepairRefusal("repair_error_cell_changed",
1066
+ name + "!" + address + " no longer holds the expected error formula or its repair.")
1067
+ evidence.append({"tab": name, "address": address, "before": before["formula"],
1068
+ "after": formula, "resolution": resolution, "reason": reason})
1069
+ append(name, address, formula)
1070
+
1071
+ def checked(book):
1072
+ result = formula_check(book)
1073
+ if not isinstance(result, dict) or not isinstance(result.get("book"), dict):
1074
+ raise RepairRefusal("repair_formula_check_required",
1075
+ "The formula repair requires a recalculated workbook from its engine.")
1076
+ # Name unresolved targets before the generic unplanned-change guard can mask them.
1077
+ _check_formula_repair_errors(result["book"])
1078
+ return result
1079
+
1080
+ projected, _errors = _checked_projection(_project(current, edits), edits, checked)
1081
+ for name, address, cached in cached_markers:
1082
+ got = ((_tab(projected, name).get("cells") or {}).get(address) or {}).get("value")
1083
+ if got != cached:
1084
+ raise RepairRefusal("repair_marker_row_occupied",
1085
+ name + "!" + address + " holds " + cached + " but its restored marker "
1086
+ "recalculates to " + json.dumps(got) + ", so it is an owner value.")
1087
+ cycles = formula_cycles(projected, formula_check)
1088
+ if cycles["count"]:
1089
+ raise RepairRefusal("repair_formula_errors_remain",
1090
+ "Formula cycles remain: " + json.dumps(cycles["cycles"]) + ".")
1091
+ return [row for row in edits if row["before"] != row["after"]], evidence
1092
+
1093
+
1094
+ def _daily_sales_formula_checker(formula_check):
1095
+ """Keep number-format inference from repainting this content-only repair."""
1096
+ if not callable(formula_check):
1097
+ raise RepairRefusal("repair_formula_check_required", "Daily Sales requires a formula engine.")
1098
+
1099
+ def checked(book):
1100
+ before = _cell_map(book)
1101
+ result = formula_check(book)
1102
+ if not isinstance(result, dict) or not isinstance(result.get("book"), dict):
1103
+ raise RepairRefusal("repair_formula_check_required",
1104
+ "Daily Sales requires a recalculated workbook from its engine.")
1105
+ for tab in result["book"].get("sheets") or ():
1106
+ for address, cell in (tab.get("cells") or {}).items():
1107
+ prior = before.get((str(tab.get("name") or "").casefold(), address)) or {}
1108
+ inferred = (cell.get("formulaMeta") or {}).get("inferredFormat")
1109
+ if cell.get("kind") != "formula" or not isinstance(inferred, dict):
1110
+ continue
1111
+ style, old_style = cell.get("style") or {}, prior.get("style") or {}
1112
+ if inferred.get("format") == style.get("format") and {
1113
+ k: v for k, v in style.items() if k != "format"} == {
1114
+ k: v for k, v in old_style.items() if k != "format"}:
1115
+ if "style" in prior:
1116
+ cell["style"] = copy.deepcopy(prior["style"])
1117
+ else:
1118
+ cell.pop("style", None)
1119
+ meta = cell.get("formulaMeta") or {}
1120
+ old_inferred = (prior.get("formulaMeta") or {}).get("inferredFormat")
1121
+ if old_inferred:
1122
+ meta["inferredFormat"] = copy.deepcopy(old_inferred)
1123
+ else:
1124
+ meta.pop("inferredFormat", None)
1125
+ if not meta:
1126
+ cell.pop("formulaMeta", None)
1127
+ return result
1128
+ return checked
1129
+
1130
+
1131
+ def _daily_sales_invariants(book, formula_check):
1132
+ census = _formula_repair_census(book)
1133
+ if any(row["count"] for row in census.values()):
1134
+ raise RepairRefusal("repair_formula_errors_remain",
1135
+ "Daily Sales formula errors remain: " + json.dumps(census, sort_keys=True))
1136
+ for name in (SUMMARY_TAB, "Summary (SR)"):
1137
+ for address, cell in (_tab(book, name).get("cells") or {}).items():
1138
+ if cell.get("kind") == "formula" and cell.get("state") != "fresh":
1139
+ raise RepairRefusal("repair_formula_check_required", name + "!" + address
1140
+ + " has not been recalculated.")
1141
+ cycles = formula_cycles(book, formula_check)
1142
+ if cycles["count"]:
1143
+ raise RepairRefusal("repair_formula_errors_remain", "Daily Sales formula cycles remain.")
1144
+ return census
1145
+
1146
+
1147
+ def _build_daily_sales_bindings_plan(current, request, resolve_binding, formula_check):
1148
+ """R18-R21: locate the authored report by labels and resolve every proposed binding."""
1149
+ from calendar import month_name, monthrange
1150
+ from openpyxl.formula.translate import Translator
1151
+ from openpyxl.utils.cell import column_index_from_string, get_column_letter
1152
+
1153
+ if not isinstance(request["daily_sales_bindings"], dict) or any(key in request for key in (
1154
+ "formula_repair", "related_migration", "summary_block", "source_cells", "binding_cells")):
1155
+ raise RepairRefusal("repair_daily_sales_plan_conflict", "Run Daily Sales bindings as their own repair.")
1156
+ if not callable(formula_check) or not callable(resolve_binding):
1157
+ raise RepairRefusal("repair_binding_resolver_required",
1158
+ "Daily Sales bindings require the resolver and formula engine.")
1159
+ if not str(request.get("refreshed_at") or "").strip():
1160
+ raise RepairRefusal("repair_timestamp_required", "Daily Sales bindings require a refresh timestamp.")
1161
+ tab = _tab(current, SUMMARY_TAB)
1162
+ _tab(current, "Summary (SR)")
1163
+ cells = tab.get("cells") or {}
1164
+
1165
+ def label(text, column="D", lo=0, hi=sheets.DOCUMENT_MAX_ROWS + 1):
1166
+ found = [address for address, cell in cells.items()
1167
+ if re.fullmatch(column + r"[1-9][0-9]*", address)
1168
+ and lo < int(address[len(column):]) < hi
1169
+ and str(cell.get("value") or "").strip() == text]
1170
+ if len(found) != 1:
1171
+ raise RepairRefusal("repair_label_missing", SUMMARY_TAB + " label '" + text
1172
+ + "' must occur exactly once in its report block.")
1173
+ return found[0]
1174
+
1175
+ def row(text, **kwargs):
1176
+ return int(re.search(r"\d+$", label(text, **kwargs))[0])
1177
+
1178
+ ending = row("End. Date")
1179
+ monthly = row("Monthly metrics")
1180
+ allocated = row("Allocated client summary")
1181
+ frequency = row("Average order frequency", lo=allocated)
1182
+ orders = row("Number of Orders", lo=frequency)
1183
+ aov = row("Avg. Order Value", lo=orders)
1184
+ delivery = row("Average Delivery Time", lo=aov)
1185
+ sales = row("Total Sales", hi=monthly)
1186
+ blocks = {}
1187
+ for name, start, end in (("sales", sales, monthly), ("frequency", frequency, orders),
1188
+ ("orders", orders, aov), ("aov", aov, delivery),
1189
+ ("delivery", delivery, sheets.DOCUMENT_MAX_ROWS + 1)):
1190
+ blocks[name] = [row(text, lo=start, hi=end) for text in YTD_NAMES]
1191
+ allocated_row = row("No. of allocated client", lo=allocated, hi=frequency)
1192
+ purchasing_row = row("No. of purchasing client", lo=allocated, hi=frequency)
1193
+ previous_row = row("Previous value", lo=allocated, hi=frequency)
1194
+ percent_row = row("% of purchasing client", lo=allocated, hi=frequency)
1195
+ helpers = [address for address, cell in cells.items() if cell.get("kind") == "formula"
1196
+ and str(cell.get("formula") or "").startswith('=IF($D$9="RI","Royal 1 (USD)"')]
1197
+ if len(helpers) != 1:
1198
+ raise RepairRefusal("repair_label_missing", "Daily Sales BU helper formula must occur exactly once.")
1199
+ helper = helpers[0]
1200
+ columns = []
1201
+ for address, cell in cells.items():
1202
+ match = re.fullmatch(r"([A-Z]+)([1-9][0-9]*)", address)
1203
+ if match and int(match[2]) == ending and column_index_from_string(match[1]) >= 6:
1204
+ try:
1205
+ _summary_date(cell.get("value"))
1206
+ except RepairRefusal:
1207
+ continue
1208
+ columns.append(match[1])
1209
+ columns.sort(key=column_index_from_string)
1210
+ if tuple(columns) != SUMMARY_COLUMNS:
1211
+ raise RepairRefusal("repair_label_missing", "End. Date requires one date for each F through R column.")
1212
+ edits, observed = [], []
1213
+
1214
+ def cache_or_empty(cell):
1215
+ return (cell.get("kind", "empty") in {"literal", "empty"}
1216
+ and (cell.get("value") in (None, "") or type(cell.get("value")) in (int, float))
1217
+ and not any(cell.get(key) for key in ("formula", "binding", "text")))
1218
+
1219
+ def append(address, desired, kind):
1220
+ before = copy.deepcopy(cells.get(address))
1221
+ after = sheets.clean_cell(_preserve_visual(before, desired), address)
1222
+ edits.append({"kind": kind, "name": SUMMARY_TAB + "!" + address,
1223
+ "sheet": SUMMARY_TAB, "address": address, "before": before, "after": after})
1224
+
1225
+ def formula_fill(origin, targets):
1226
+ master = (cells.get(origin) or {}).get("formula")
1227
+ if not isinstance(master, str) or not master.startswith("="):
1228
+ raise RepairRefusal("repair_formula_master_missing", SUMMARY_TAB + "!" + origin
1229
+ + " must hold its authored formula master.")
1230
+ for address in targets:
1231
+ text = Translator(master, origin=origin).translate_formula(address)
1232
+ before = cells.get(address) or {}
1233
+ if not (cache_or_empty(before) or before.get("kind") == "formula"
1234
+ and before.get("formula") == text):
1235
+ raise RepairRefusal("repair_owner_value", SUMMARY_TAB + "!" + address
1236
+ + " contains an owner value outside this formula fill.")
1237
+ append(address, {"kind": "formula", "state": "stale", "formula": text}, "derived")
1238
+
1239
+ def bind(address, database, fn, nodes, field=None):
1240
+ aggregate = {"fn": fn, **({"field": field} if field else {})}
1241
+ binding = sheets.clean_cell({"kind": "bound", "binding": {
1242
+ "database": database, "aggregate": aggregate,
1243
+ "scope": {"filter": {"conj": "and", "nodes": nodes}}}}, address)["binding"]
1244
+ before = cells.get(address) or {}
1245
+ if not (cache_or_empty(before) or before.get("kind") == "bound" and before.get("binding") == binding):
1246
+ raise RepairRefusal("repair_owner_value", SUMMARY_TAB + "!" + address
1247
+ + " contains an owner value outside this binding plan.")
1248
+ try:
1249
+ result = resolve_binding(copy.deepcopy(binding), copy.deepcopy(current), copy.deepcopy(tab))
1250
+ except Exception as exc:
1251
+ detail = getattr(exc, "detail", None)
1252
+ info = (detail.get("error") or {}) if isinstance(detail, dict) else {}
1253
+ code = getattr(exc, "code", None) or info.get("code") or "repair_binding_unreadable"
1254
+ raise RepairRefusal(code, SUMMARY_TAB + "!" + address + " binding resolution refused.") from exc
1255
+ if not isinstance(result, dict) or "value" not in result or result.get("refusal") \
1256
+ or result.get("error") or result.get("state") not in (None, "fresh"):
1257
+ raise RepairRefusal("repair_binding_unreadable", SUMMARY_TAB + "!" + address
1258
+ + " did not resolve a fresh bound value.")
1259
+ if result.get("limits"):
1260
+ raise RepairRefusal("repair_binding_limits", SUMMARY_TAB + "!" + address
1261
+ + " has unapplied or limited binding conditions.")
1262
+ value = result["value"]
1263
+ if not (value is None and fn == "avg") and (
1264
+ type(value) not in (int, float) or not math.isfinite(value)):
1265
+ raise RepairRefusal("repair_binding_unreadable", SUMMARY_TAB + "!" + address
1266
+ + " must resolve a finite number.")
1267
+ prior = before.get("value")
1268
+ if before.get("kind") == "bound" and before.get("binding") == binding \
1269
+ and type(prior) in (int, float) and type(value) in (int, float) \
1270
+ and math.isclose(prior, value, rel_tol=1e-9, abs_tol=1e-9):
1271
+ # A float SUM/AVG can differ in its last bits between runs (parallel summation
1272
+ # order: 168809.64999999997 then 168809.65). It is the same number, so the stored
1273
+ # one is kept and a rerun changes nothing.
1274
+ value = prior
1275
+ desired = {"kind": "bound", "state": "fresh", "binding": binding, "value": value,
1276
+ "display": "" if value is None else str(value), "refreshed_at": request["refreshed_at"]}
1277
+ if all(before.get(key) == desired[key] for key in ("kind", "state", "binding", "value", "display")) \
1278
+ and not any(before.get(key) for key in ("limits", "refusal", "error")):
1279
+ desired["refreshed_at"] = before.get("refreshed_at", request["refreshed_at"])
1280
+ append(address, desired, "binding")
1281
+ observed.append({"address": address, "binding": binding, "value": value})
1282
+
1283
+ def bu(via=""):
1284
+ return {"colId": ("related:" + via + ":" if via else "") + "pricelist",
1285
+ "op": "eq", "valueRef": {"ref": helper}}
1286
+
1287
+ def window(field, col, kind):
1288
+ return {"colId": field, "op": "within", "valueRef": {"ref": col + str(ending)},
1289
+ "dateWindow": {"kind": kind}}
1290
+
1291
+ for half, label_column, months in (("H1", "D", range(1, 7)), ("H2", "I", range(7, 13))):
1292
+ header = label(half, column=label_column, lo=monthly, hi=allocated)
1293
+ header_row = int(re.search(r"\d+$", header)[0])
1294
+ base_col = column_index_from_string(label_column)
1295
+ month_rows = [row(month_name[month], column=label_column, lo=header_row, hi=allocated)
1296
+ for month in months]
1297
+ for offset in (1, 2):
1298
+ value_col = get_column_letter(base_col + offset)
1299
+ year = (cells.get(value_col + str(header_row)) or {}).get("value")
1300
+ numeric_year = (type(year) in (int, float) and math.isfinite(year) and int(year) == year)
1301
+ if not (numeric_year or isinstance(year, str) and year.isdigit()) \
1302
+ or not 1900 <= int(year) <= 9999:
1303
+ raise RepairRefusal("repair_label_missing", half + " requires a calendar year at "
1304
+ + value_col + str(header_row) + ".")
1305
+ year = int(year)
1306
+ for month, month_row in zip(months, month_rows):
1307
+ bind(value_col + str(month_row), "ut_odoo_invoices", "sum", [
1308
+ {"colId": "invoice_date", "op": "between", "value": f"{year:04d}-{month:02d}-01",
1309
+ "value2": f"{year:04d}-{month:02d}-{monthrange(year, month)[1]:02d}"},
1310
+ {"colId": "move_type", "op": "eq", "value": "out_invoice"}, bu("customer_link")],
1311
+ "amount_untaxed")
1312
+ growth = get_column_letter(base_col + 3)
1313
+ formula_fill(growth + str(month_rows[0]), [growth + str(r) for r in month_rows])
1314
+
1315
+ for col in columns:
1316
+ bind(col + str(allocated_row), "customer_data", "count", [bu(), {"conj": "and", "children": [
1317
+ {"colId": "related:invoices:agent_id", "op": "eq", "rhs": {"colId": "agent_id"}},
1318
+ {"colId": "related:invoices:invoice_date", "op": "lte", "valueRef": {"ref": col + str(ending)}},
1319
+ {"colId": "related:invoices:move_type", "op": "eq", "value": "out_invoice"}]}])
1320
+ for i, kind in enumerate(YTD_NAMES.values()):
1321
+ bind(col + str((purchasing_row, previous_row)[i]), "customer_data", "count", [bu(),
1322
+ {"conj": "and", "children": [window("related:invoices:invoice_date", col, kind),
1323
+ {"colId": "related:invoices:move_type", "op": "eq", "value": "out_invoice"}]}])
1324
+ bind(col + str(blocks["orders"][i]), "ut_odoo_orders", "count", [
1325
+ window("order_date", col, kind), {"colId": "state", "op": "isAnyOf", "values": ["sale", "done"]},
1326
+ bu("bridge_customer_orders")])
1327
+ bind(col + str(blocks["delivery"][i]), "ut_odoo_shipments", "avg", [
1328
+ {"colId": "state", "op": "eq", "value": "done"}, window("date_done", col, kind),
1329
+ bu("customer_link")], "delivery_days")
1330
+ for target_row in [percent_row, *blocks["frequency"], *blocks["aov"]]:
1331
+ formula_fill("F" + str(target_row), [col + str(target_row) for col in columns])
1332
+ projected, _errors = _checked_projection(_project(current, edits), edits, formula_check)
1333
+ _daily_sales_invariants(projected, formula_check)
1334
+ return [edit for edit in edits if edit["before"] != edit["after"]], {
1335
+ "helper": helper, "ending_row": ending, "observed": observed}
1336
+
1337
+
1338
+ # ═════════════════════════════════════════════════════════════════════════════════════════════
1339
+ # ⭐⭐ S67-T21 / R34 β€” EVERY METRIC COMPUTED; ONLY THE OWNER'S OWN INPUTS STAY TYPED
1340
+ # ═════════════════════════════════════════════════════════════════════════════════════════════
1341
+ #
1342
+ # The owner's weekly scorecard read pasted numbers: the weekly block and the middle of a dozen
1343
+ # rows were values copied once and never recomputed. This mode applies an explicit, fully named
1344
+ # cell plan (each cell a formula, a Cell Database binding or a blank), resolved and recalculated
1345
+ # BEFORE a byte moves, and REFUSES unless the projected workbook keeps no hardcoded number but the
1346
+ # owner's inputs. The plan itself is built by the operator from the Excel reference; this module is
1347
+ # what makes it safe to apply: owner inputs and labels are untouchable, every binding resolves to
1348
+ # a fresh value with no limits, the formula census stays empty, and the backup lands first.
1349
+
1350
+ SR_TAB = "Summary (SR)"
1351
+
1352
+ #: R34 (owner, 2026-09-28), verbatim cells. Summary (All)'s six are the owner's inputs; Summary
1353
+ #: (SR)'s are the only numbers the Excel file itself types on that tab (its targets and goals).
1354
+ OWNER_INPUTS = {
1355
+ SUMMARY_TAB: frozenset({"Q25", "T20", "T37", "T38", "T51", "T55"}),
1356
+ SR_TAB: frozenset({"T24", "T25", "T34", "T67", "T71", "T105", "T112"}),
1357
+ }
1358
+ _PLANNED_KINDS = ("formula", "bound", "empty")
1359
+ #: The tab-level visual keys a plan may DECLARE (O42: match the Excel file's freeze). Anything else
1360
+ #: visual still refuses, exactly as every content-only repair above does.
1361
+ _PLANNED_TAB_VISUAL_KEYS = ("freeze",)
1362
+
1363
+
1364
+ def hardcoded_numbers(book):
1365
+ """`{tab name: {address: number}}` for every literal numeric cell, PURE. R34's census."""
1366
+ out = {}
1367
+ for tab in book.get("sheets") or ():
1368
+ found = {}
1369
+ for address, cell in (tab.get("cells") or {}).items():
1370
+ value = (cell or {}).get("value")
1371
+ if (cell or {}).get("kind") == "literal" and type(value) in (int, float):
1372
+ found[address] = value
1373
+ out[str(tab.get("name") or "")] = found
1374
+ return out
1375
+
1376
+
1377
+ def _bound_value(value, fn):
1378
+ """A resolved binding's value as the document stores it, or a refusal.
1379
+
1380
+ ⭐ A `min`/`max` over a date column answers a DATE ("Latest data" is `max(invoice_date)`), which
1381
+ is stored as its ISO text, the shape every formula date in this workbook already has.
1382
+ """
1383
+ import datetime as _dtm
1384
+ if isinstance(value, (_dtm.date, _dtm.datetime)):
1385
+ if fn not in ("min", "max"):
1386
+ raise RepairRefusal("repair_binding_unreadable", "Only min or max may answer a date.")
1387
+ return value.isoformat()[:10]
1388
+ if isinstance(value, str) and fn in ("min", "max") and re.fullmatch(r"\d{4}-\d{2}-\d{2}.*", value):
1389
+ return value[:10]
1390
+ if value is None and fn in ("avg", "min", "max"):
1391
+ return None
1392
+ if type(value) not in (int, float) or not math.isfinite(value):
1393
+ raise RepairRefusal("repair_binding_unreadable", "A binding must resolve a finite number.")
1394
+ return value
1395
+
1396
+
1397
+ def _visual_without(book, planned_notes, keys=("note",)):
1398
+ """`visual_metadata` with each planned cell's declared visual `keys` removed (a note for
1399
+ T21, a sparkline for T23), so a declared change is the only visual change a plan may make."""
1400
+ shaped = copy.deepcopy(book)
1401
+ for tab in shaped.get("sheets") or ():
1402
+ name = str(tab.get("name") or "").casefold()
1403
+ for address, cell in (tab.get("cells") or {}).items():
1404
+ if (name, address) in planned_notes and isinstance(cell, dict):
1405
+ for key in keys:
1406
+ cell.pop(key, None)
1407
+ return visual_metadata(shaped)
1408
+
1409
+
1410
+ def _build_computed_metrics_plan(current, request, resolve_binding, formula_check):
1411
+ """R34: validate and resolve an explicit per-cell plan; refuse unless it leaves no hardcode."""
1412
+ spec = request["computed_metrics"]
1413
+ if not isinstance(spec, dict) or any(key in request for key in (
1414
+ "formula_repair", "related_migration", "summary_block", "source_cells",
1415
+ "binding_cells", "daily_sales_bindings")):
1416
+ raise RepairRefusal("repair_metrics_plan_conflict",
1417
+ "Run the computed metrics plan as its own repair.")
1418
+ if not callable(formula_check) or not callable(resolve_binding):
1419
+ raise RepairRefusal("repair_binding_resolver_required",
1420
+ "Computed metrics require the resolver and the formula engine.")
1421
+ refreshed_at = str(request.get("refreshed_at") or "").strip()
1422
+ if not refreshed_at:
1423
+ raise RepairRefusal("repair_timestamp_required", "Computed metrics require a refresh timestamp.")
1424
+ rows = spec.get("cells")
1425
+ if not isinstance(rows, list) or not rows:
1426
+ raise RepairRefusal("repair_metrics_plan_empty", "The computed metrics plan names no cell.")
1427
+ edits, observed, seen, planned_notes = [], [], set(), set()
1428
+ # ⭐ A binding may read a cell this same plan writes (a month's helper start/end dates, a
1429
+ # year header), so bindings resolve against the workbook WITH the planned formulas and blanks
1430
+ # applied and recalculated, never against the stale current one.
1431
+ staged = []
1432
+ for row in rows:
1433
+ want = row.get("cell") if isinstance(row, dict) else None
1434
+ if isinstance(want, dict) and want.get("kind") in ("formula", "empty"):
1435
+ tab = _tab(current, str(row.get("sheet") or ""))
1436
+ address = _addr(row.get("address"))
1437
+ before = copy.deepcopy((tab.get("cells") or {}).get(address))
1438
+ cell = ({"kind": "formula", "state": "stale", "formula": str(want.get("formula") or "")}
1439
+ if want["kind"] == "formula" else {"kind": "empty", "state": "typed"})
1440
+ staged.append({"sheet": str(tab.get("name")), "address": address, "before": before,
1441
+ "after": sheets.clean_cell(_preserve_visual(before, cell) or cell, address)})
1442
+ def recalculated(extra):
1443
+ projected = _project(current, staged + extra)
1444
+ checked = formula_check(copy.deepcopy(projected))
1445
+ return checked["book"] if isinstance(checked, dict) and isinstance(
1446
+ checked.get("book"), dict) else projected
1447
+
1448
+ def resolve_row(row, book):
1449
+ sheet = str(_tab(current, str(row.get("sheet") or "")).get("name"))
1450
+ address = _addr(row.get("address"))
1451
+ binding = sheets.clean_cell({"kind": "bound", "binding": row["cell"].get("binding")},
1452
+ address)["binding"]
1453
+ try:
1454
+ result = resolve_binding(copy.deepcopy(binding), copy.deepcopy(book),
1455
+ copy.deepcopy(_tab(book, sheet)))
1456
+ except Exception as exc: # noqa: BLE001
1457
+ detail = getattr(exc, "detail", None)
1458
+ info = (detail.get("error") or {}) if isinstance(detail, dict) else {}
1459
+ code = getattr(exc, "code", None) or info.get("code") or "repair_binding_unreadable"
1460
+ return None, RepairRefusal(code, sheet + "!" + address + " binding resolution refused.")
1461
+ return (binding, result), None
1462
+
1463
+ # ⭐ ROUNDS, because a binding may read a date another binding produces through formulas
1464
+ # (Summary (SR)'s week axis runs E12 `max(invoice_date)` -> E15 -> R8 -> F8 -> F31's window).
1465
+ # Each round resolves what it can read, the next projection carries those values, and only a
1466
+ # cell whose filter input was not yet readable waits; no progress in a round refuses by name.
1467
+ resolved, pending = {}, [row for row in rows if isinstance(row, dict)
1468
+ and isinstance(row.get("cell"), dict)
1469
+ and row["cell"].get("kind") == "bound"]
1470
+ for _round in range(6):
1471
+ if not pending:
1472
+ break
1473
+ extra = []
1474
+ for (sheet, address), (binding, result) in resolved.items():
1475
+ value = result.get("value") if isinstance(result, dict) else None
1476
+ try:
1477
+ value = _bound_value(value, (binding.get("aggregate") or {}).get("fn"))
1478
+ except RepairRefusal:
1479
+ value = None
1480
+ extra.append({"sheet": sheet, "address": address, "before": None,
1481
+ "after": sheets.clean_cell({"kind": "bound", "state": "fresh",
1482
+ "binding": binding, "value": value,
1483
+ "display": "" if value is None else str(value)},
1484
+ address)})
1485
+ book = recalculated(extra)
1486
+ waiting, first_error = [], None
1487
+ for row in pending:
1488
+ got, error = resolve_row(row, book)
1489
+ if got is not None:
1490
+ resolved[(str(_tab(current, str(row["sheet"])).get("name")),
1491
+ _addr(row.get("address")))] = got
1492
+ elif error.code == "filter_cell_unreadable":
1493
+ waiting.append(row)
1494
+ first_error = first_error or error
1495
+ else:
1496
+ raise error
1497
+ if len(waiting) == len(pending):
1498
+ raise first_error
1499
+ pending = waiting
1500
+ if pending:
1501
+ raise RepairRefusal("filter_cell_unreadable", "A binding's filter cell never became readable.")
1502
+ for row in rows:
1503
+ if not isinstance(row, dict):
1504
+ raise RepairRefusal("repair_metrics_cell_invalid", "Every planned cell must be an object.")
1505
+ sheet_name = str(row.get("sheet") or "").strip()
1506
+ address = _addr(row.get("address"))
1507
+ tab = _tab(current, sheet_name)
1508
+ sheet_name = str(tab.get("name") or sheet_name)
1509
+ slot = (sheet_name.casefold(), address)
1510
+ if slot in seen:
1511
+ raise RepairRefusal("repair_edit_duplicate", "A planned cell may be named only once.")
1512
+ seen.add(slot)
1513
+ if address in OWNER_INPUTS.get(sheet_name, ()):
1514
+ raise RepairRefusal("repair_owner_input", sheet_name + "!" + address
1515
+ + " is one of the owner's typed inputs and stays typed.")
1516
+ cells = tab.get("cells") or {}
1517
+ before = copy.deepcopy(cells.get(address))
1518
+ prior_value = (before or {}).get("value")
1519
+ want = row.get("cell")
1520
+ if not isinstance(want, dict) or want.get("kind") not in (*_PLANNED_KINDS, "note_only"):
1521
+ raise RepairRefusal("repair_metrics_cell_invalid", sheet_name + "!" + address
1522
+ + " must be planned as a formula, a binding, a blank or a note.")
1523
+ kind = want["kind"]
1524
+ # A typed ISO date is a value (Summary (SR)'s frozen E13/E15), never a label.
1525
+ if (before or {}).get("kind") == "literal" and isinstance(prior_value, str) \
1526
+ and prior_value.strip() and kind != "note_only" \
1527
+ and not re.fullmatch(r"\d{4}-\d{2}-\d{2}", prior_value.strip()):
1528
+ raise RepairRefusal("repair_label_protected", sheet_name + "!" + address
1529
+ + " holds a label, which a metrics plan never replaces.")
1530
+ if kind == "note_only":
1531
+ desired = copy.deepcopy(before) if before else {"kind": "empty", "state": "typed"}
1532
+ elif kind == "formula":
1533
+ text = str(want.get("formula") or "")
1534
+ if not text.startswith("="):
1535
+ raise RepairRefusal("repair_metrics_cell_invalid", sheet_name + "!" + address
1536
+ + " needs a formula that starts with =.")
1537
+ desired = _preserve_visual(before, {"kind": "formula", "state": "stale", "formula": text})
1538
+ elif kind == "empty":
1539
+ desired = _preserve_visual(before, {"kind": "empty", "state": "typed"}) \
1540
+ or {"kind": "empty", "state": "typed"}
1541
+ else:
1542
+ binding, result = resolved[(sheet_name, address)]
1543
+ fn = (binding.get("aggregate") or {}).get("fn")
1544
+ if not isinstance(result, dict) or "value" not in result or result.get("refusal") \
1545
+ or result.get("error") or result.get("state") not in (None, "fresh"):
1546
+ raise RepairRefusal("repair_binding_unreadable", sheet_name + "!" + address
1547
+ + " did not resolve a fresh bound value.")
1548
+ if result.get("limits"):
1549
+ raise RepairRefusal("repair_binding_limits", sheet_name + "!" + address
1550
+ + " has unapplied or limited binding conditions.")
1551
+ value = _bound_value(result["value"], fn)
1552
+ if (before or {}).get("kind") == "bound" and before.get("binding") == binding \
1553
+ and type(prior_value) in (int, float) and type(value) in (int, float) \
1554
+ and math.isclose(prior_value, value, rel_tol=1e-9, abs_tol=1e-9):
1555
+ value = prior_value # the same number; a rerun changes nothing
1556
+ desired = _preserve_visual(before, {
1557
+ "kind": "bound", "state": "fresh", "binding": binding, "value": value,
1558
+ "display": "" if value is None else str(value), "refreshed_at": refreshed_at})
1559
+ observed.append({"sheet": sheet_name, "address": address, "value": value})
1560
+ if "note" in row:
1561
+ note = str(row.get("note") or "").strip()
1562
+ if note:
1563
+ desired["note"] = note
1564
+ else:
1565
+ desired.pop("note", None)
1566
+ planned_notes.add(slot)
1567
+ after = sheets.clean_cell(desired, address)
1568
+ # A planned formula is `derived`: the recalculation below is what fills its value, and
1569
+ # `_checked_projection` accepts a recalculated planned cell only under that kind.
1570
+ edit_kind = {"formula": "derived", "bound": "binding"}.get(kind, "metric")
1571
+ edits.append({"kind": edit_kind, "name": sheet_name + "!" + address, "sheet": sheet_name,
1572
+ "address": address, "before": before, "after": after})
1573
+ tab_visual = spec.get("tab_visual") or {}
1574
+ if not isinstance(tab_visual, dict):
1575
+ raise RepairRefusal("repair_metrics_visual_invalid", "Tab visual changes must be an object.")
1576
+ for sheet_name, keys in tab_visual.items():
1577
+ _tab(current, sheet_name)
1578
+ if not isinstance(keys, dict) or not keys or set(keys) - set(_PLANNED_TAB_VISUAL_KEYS):
1579
+ raise RepairRefusal("repair_metrics_visual_invalid",
1580
+ "A metrics plan may declare only a tab's freeze.")
1581
+ projected, _errors = _checked_projection(_project(current, edits), edits, formula_check)
1582
+ _daily_sales_invariants(projected, formula_check)
1583
+ remaining = {name: sorted(set(found) - OWNER_INPUTS.get(name, frozenset()))
1584
+ for name, found in hardcoded_numbers(projected).items()
1585
+ if name in OWNER_INPUTS}
1586
+ if any(remaining.values()):
1587
+ raise RepairRefusal("repair_hardcoded_remain", "Hardcoded numbers would remain: "
1588
+ + json.dumps(remaining, sort_keys=True))
1589
+ return [edit for edit in edits if edit["before"] != edit["after"]], {
1590
+ "observed": observed, "planned_notes": sorted(f"{s}!{a}" for s, a in planned_notes),
1591
+ "tab_visual": copy.deepcopy(tab_visual),
1592
+ "hardcoded_before": hardcoded_numbers(current), "hardcoded_after": hardcoded_numbers(projected)}
1593
+
1594
+
1595
+ def _apply_tab_visual(st, workbook_id, tab_visual, *, actor="", actor_name=""):
1596
+ """Write the DECLARED tab-level visual keys through the ordinary document door, then prove
1597
+ nothing else moved. Returns the readback snapshot."""
1598
+ doc = sheets.workbook_snapshot(st, workbook_id)
1599
+ before_cells = {str(t.get("name")): copy.deepcopy(t.get("cells") or {}) for t in doc["sheets"]}
1600
+ before_visual = visual_metadata(doc)
1601
+ for sheet_name, keys in tab_visual.items():
1602
+ tab = _tab(doc, sheet_name)
1603
+ for key, value in keys.items():
1604
+ tab[key] = copy.deepcopy(value)
1605
+ sheets.put_workbook(st, workbook_id, doc, expect_rev=int(doc.get("rev") or 0),
1606
+ actor=actor, actor_name=actor_name, lossless=True)
1607
+ readback = sheets.workbook_snapshot(st, workbook_id)
1608
+ for sheet_name, keys in tab_visual.items():
1609
+ tab = _tab(readback, sheet_name)
1610
+ for key, value in keys.items():
1611
+ if _canonical(tab.get(key)) != _canonical(value):
1612
+ raise RepairRefusal("repair_visual_readback", sheet_name + " " + key
1613
+ + " did not read back as declared.")
1614
+ if {str(t.get("name")): t.get("cells") or {} for t in readback["sheets"]} != before_cells:
1615
+ raise RepairRefusal("repair_visual_readback", "A declared visual change moved cell content.")
1616
+ expected = copy.deepcopy(before_visual)
1617
+ for item in expected:
1618
+ for key, value in (tab_visual.get(item.get("name")) or {}).items():
1619
+ item[key] = copy.deepcopy(value)
1620
+ if visual_metadata(readback) != _canonical(expected):
1621
+ raise RepairRefusal("repair_visual_readback", "An undeclared visual property changed.")
1622
+ return readback
1623
+
1624
+
1625
+ # ═════════════════════════════════════════════════════════════════════════════════════════════
1626
+ # ⭐⭐ S67-T22 / S67-T23 β€” DECLARED VISUAL REPAIRS: Excel's rule colours and its 3M sparklines
1627
+ # ═════════════════════════════════════════════════════════════════════════════════════════════
1628
+ #
1629
+ # Every repair above REFUSES a visual change, on purpose. These two tickets ARE visual changes, so
1630
+ # they get their own mode that names exactly what may move: a tab's `rules` (T22, O31) and a cell's
1631
+ # `sparkline` (T23, O32). Cell content, every other visual key and every other cell must read back
1632
+ # unchanged; the backup lands before anything is written.
1633
+
1634
+ _VISUAL_REPAIR_TAB_KEYS = ("rules",)
1635
+
1636
+
1637
+ def _build_visual_repair_plan(current, request):
1638
+ spec = request["visual_repair"]
1639
+ if not isinstance(spec, dict) or any(key in request for key in (
1640
+ "formula_repair", "related_migration", "summary_block", "source_cells",
1641
+ "binding_cells", "daily_sales_bindings", "computed_metrics")):
1642
+ raise RepairRefusal("repair_visual_plan_conflict", "Run the visual repair as its own repair.")
1643
+ tab_visual = copy.deepcopy(spec.get("tab_visual") or {})
1644
+ for sheet_name, keys in tab_visual.items():
1645
+ _tab(current, sheet_name)
1646
+ if not isinstance(keys, dict) or not keys or set(keys) - set(_VISUAL_REPAIR_TAB_KEYS):
1647
+ raise RepairRefusal("repair_visual_invalid", "A visual repair may declare only a tab's rules.")
1648
+ keys["rules"] = sheets._clean_rules(keys["rules"], lossless=True)
1649
+ edits, seen = [], set()
1650
+ for row in spec.get("cells") or ():
1651
+ if not isinstance(row, dict) or "sparkline" not in row:
1652
+ raise RepairRefusal("repair_visual_invalid", "A visual repair cell names only its sparkline.")
1653
+ tab = _tab(current, str(row.get("sheet") or ""))
1654
+ sheet_name, address = str(tab.get("name")), _addr(row.get("address"))
1655
+ if (sheet_name.casefold(), address) in seen:
1656
+ raise RepairRefusal("repair_edit_duplicate", "A repaired cell may be named only once.")
1657
+ seen.add((sheet_name.casefold(), address))
1658
+ before = copy.deepcopy((tab.get("cells") or {}).get(address))
1659
+ desired = copy.deepcopy(before) if before else {"kind": "empty", "state": "typed"}
1660
+ if row["sparkline"] is None:
1661
+ desired.pop("sparkline", None)
1662
+ else:
1663
+ clean = sheets._clean_sparkline(row["sparkline"])
1664
+ if clean is None:
1665
+ raise RepairRefusal("repair_visual_invalid", sheet_name + "!" + address
1666
+ + " names a sparkline that cannot be drawn.")
1667
+ desired["sparkline"] = clean
1668
+ after = sheets.clean_cell(desired, address)
1669
+ content = ("kind", "state", "value", "display", "formula", "binding", "text")
1670
+ if any((before or {}).get(key) != after.get(key) for key in content if key != "state") \
1671
+ and not (before is None and after.get("kind") == "empty"):
1672
+ raise RepairRefusal("repair_visual_content_changed", sheet_name + "!" + address
1673
+ + " would change content, which a visual repair never does.")
1674
+ edits.append({"kind": "visual", "name": sheet_name + "!" + address, "sheet": sheet_name,
1675
+ "address": address, "before": before, "after": after})
1676
+ return [e for e in edits if e["before"] != e["after"]], {
1677
+ "tab_visual": tab_visual, "sparkline_cells": sorted(e["name"] for e in edits)}
1678
+
1679
+ def _execute_visual_repair(st, backup_store, request, current, revision, digest, workbook_id,
1680
+ *, actor="", actor_name=""):
1681
+ """Back up, then write ONLY the declared sparklines (exact cell edits) and tab rules."""
1682
+ edits, evidence = _build_visual_repair_plan(current, request)
1683
+ if not edits and not evidence["tab_visual"]:
1684
+ return {"workbook_id": workbook_id, "revision_before": revision, "revision_after": revision,
1685
+ "applied": 0, "noop": True, "visual_repair": evidence}
1686
+ planned = {(row["sheet"].casefold(), row["address"]) for row in edits}
1687
+ projected = _project(current, edits)
1688
+ if _visual_without(projected, planned, ("sparkline",)) != \
1689
+ _visual_without(current, planned, ("sparkline",)):
1690
+ raise RepairRefusal("repair_visual_metadata_changed",
1691
+ "The projected repair changes a visual property it did not declare.")
1692
+ exact = [{key: copy.deepcopy(row[key]) for key in ("sheet", "address", "before", "after")}
1693
+ for row in edits]
1694
+ diff_digest = sheets.workbook_digest({"edits": exact, "tab_visual": evidence["tab_visual"]})
1695
+ backup_key = _write_backup(backup_store, current, digest, diff_digest, workbook_id)
1696
+ revision_after = revision
1697
+ if exact:
1698
+ sheets.apply_exact_cell_edits(st, workbook_id, exact, revision, digest,
1699
+ actor=actor, actor_name=actor_name)
1700
+ readback = sheets.workbook_snapshot(st, workbook_id)
1701
+ if int(readback.get("rev") or 0) != revision + 1:
1702
+ raise RepairRefusal("repair_readback_revision",
1703
+ "The repaired workbook revision did not read back as expected.")
1704
+ for row in edits:
1705
+ actual = (_tab(readback, row["sheet"]).get("cells") or {}).get(row["address"])
1706
+ if actual != sheets.clean_cell(row["after"], row["address"]):
1707
+ raise RepairRefusal("repair_readback_cell",
1708
+ row["sheet"] + "!" + row["address"] + " did not read back exactly.")
1709
+ revision_after = int(readback["rev"])
1710
+ if evidence["tab_visual"]:
1711
+ revision_after = int(_apply_tab_visual(st, workbook_id, evidence["tab_visual"],
1712
+ actor=actor, actor_name=actor_name)["rev"])
1713
+ return {"workbook_id": workbook_id, "revision_before": revision,
1714
+ "revision_after": revision_after, "applied": len(exact), "backup_key": backup_key,
1715
+ "backup_digest": digest, "diff_digest": diff_digest, "visual_repair": evidence}
1716
+
1717
+
1718
+ def restore_backup(st, backup_store, backup_key, *, workbook_id, actor="", actor_name=""):
1719
+ """NC for every mode here: put a backup's workbook back and prove it round-trips."""
1720
+ record = backup_store.get(backup_key)
1721
+ if not isinstance(record, dict) or record.get("workbook_id") != workbook_id:
1722
+ raise RepairRefusal("repair_backup_missing", "That backup does not exist for this workbook.")
1723
+ saved = copy.deepcopy(record["workbook"])
1724
+ current = sheets.workbook_snapshot(st, workbook_id)
1725
+ sheets.put_workbook(st, workbook_id, saved, expect_rev=int(current.get("rev") or 0),
1726
+ actor=actor, actor_name=actor_name, lossless=True)
1727
+ readback = sheets.workbook_snapshot(st, workbook_id)
1728
+ same = ({str(t.get("name")): t.get("cells") or {} for t in readback["sheets"]}
1729
+ == {str(t.get("name")): t.get("cells") or {} for t in saved["sheets"]}
1730
+ and visual_metadata(readback) == visual_metadata(saved))
1731
+ if not same:
1732
+ raise RepairRefusal("repair_restore_unverified", "The restored workbook did not round-trip.")
1733
+ return readback
1734
+
1735
+
1736
  def _build_plan(current, request, resolve_binding):
1737
  if not isinstance(request.get("source_cells"), list) or not request["source_cells"]:
1738
  raise RepairRefusal(
 
1925
  raise RepairRefusal("repair_revision_mismatch",
1926
  "The named workbook is not at the expected repair revision.")
1927
  digest = sheets.workbook_digest(current)
1928
+ if "visual_repair" in request:
1929
+ return _execute_visual_repair(st, backup_store, request, current, revision, digest,
1930
+ target_workbook_id, actor=actor, actor_name=actor_name)
1931
  summary_evidence = None
1932
  summary_errors_before = None
1933
+ migrated = None
1934
+ formula_evidence = None
1935
+ daily_evidence = None
1936
+ metrics_evidence = None
1937
+ if "computed_metrics" in request:
1938
+ formula_check = _daily_sales_formula_checker(formula_check)
1939
+ edits, metrics_evidence = _build_computed_metrics_plan(
1940
+ current, request, resolve_binding, formula_check)
1941
+ if not edits and not metrics_evidence["tab_visual"]:
1942
+ return {"workbook_id": target_workbook_id, "revision_before": revision,
1943
+ "revision_after": revision, "applied": 0, "noop": True,
1944
+ "computed_metrics": metrics_evidence}
1945
+ elif "daily_sales_bindings" in request:
1946
+ formula_check = _daily_sales_formula_checker(formula_check)
1947
+ edits, daily_evidence = _build_daily_sales_bindings_plan(
1948
+ current, request, resolve_binding, formula_check)
1949
+ if not edits:
1950
+ census = _daily_sales_invariants(current, formula_check)
1951
+ return {"workbook_id": target_workbook_id, "revision_before": revision,
1952
+ "revision_after": revision, "applied": 0, "noop": True,
1953
+ "daily_sales_bindings": daily_evidence, "formula_errors": [],
1954
+ "formula_error_census": {"before": census, "after": copy.deepcopy(census)},
1955
+ "formula_cycles": _formula_cycle_comparison(current, current, formula_check),
1956
+ "visual_metadata_preserved": True}
1957
+ elif "formula_repair" in request:
1958
+ edits, formula_evidence = _build_formula_repair_plan(current, request, formula_check)
1959
+ if not edits:
1960
+ census = _formula_repair_census(current)
1961
+ return {"workbook_id": target_workbook_id, "revision_before": revision,
1962
+ "revision_after": revision, "applied": 0, "noop": True,
1963
+ "formula_repair": formula_evidence, "formula_errors": formula_error_census(current),
1964
+ "formula_error_census": {"before": census, "after": copy.deepcopy(census)},
1965
+ "formula_cycles": _formula_cycle_comparison(current, current, formula_check),
1966
+ "visual_metadata_preserved": True}
1967
+ elif "related_migration" in request:
1968
+ edits, migrated = _build_related_migration_plan(
1969
+ current, request, resolve_binding, formula_check)
1970
+ if not edits:
1971
+ return {"workbook_id": target_workbook_id, "revision_before": revision,
1972
+ "revision_after": revision, "applied": 0, "migrated": [], "noop": True}
1973
+ elif "summary_block" in request:
1974
  summary_errors_before = check_summary_formula_errors(current)
1975
  edits, summary_evidence = _build_summary_plan(current, request, resolve_binding, formula_check)
1976
  else:
1977
  edits = _build_plan(current, request, resolve_binding)
1978
  projected = _project(current, edits)
1979
  projected, projected_errors = _checked_projection(projected, edits, formula_check)
1980
+ if formula_evidence is not None:
1981
+ _check_formula_repair_errors(projected)
1982
+ if daily_evidence is not None:
1983
+ _daily_sales_invariants(projected, formula_check)
1984
  comparison = _formula_error_comparison(current, projected, edits, projected_errors)
1985
  summary_errors_after = (check_summary_formula_errors(projected)
1986
  if summary_evidence is not None else None)
 
1991
  "The projected repair introduces a formula error or leaves one in its repair scope, "
1992
  "so no repair was applied.")
1993
  visual_before = visual_metadata(current)
1994
+ planned_notes = set()
1995
+ if metrics_evidence is not None:
1996
+ planned_notes = {tuple(item.split("!", 1)) for item in metrics_evidence["planned_notes"]}
1997
+ planned_notes = {(sheet.casefold(), address) for sheet, address in planned_notes}
1998
+ if _visual_without(projected, planned_notes) != _visual_without(current, planned_notes):
1999
  raise RepairRefusal("repair_visual_metadata_changed",
2000
  "The projected repair changes Daily Sales visual metadata.")
2001
 
 
2027
  or _cell_map(checked_readback) != _cell_map(readback):
2028
  raise RepairRefusal("repair_readback_formula_state",
2029
  "The repaired workbook did not read back calculation-current.")
2030
+ if _visual_without(readback, planned_notes) != _visual_without(current, planned_notes):
2031
  raise RepairRefusal("repair_readback_visual_metadata",
2032
  "Daily Sales visual metadata changed during repair.")
2033
+ if formula_evidence is not None:
2034
+ _check_formula_repair_errors(readback)
2035
+ if daily_evidence is not None:
2036
+ _daily_sales_invariants(readback, formula_check)
2037
  if summary_evidence is not None:
2038
  tab = _tab(readback, SUMMARY_TAB)
2039
  if any((tab.get("cells", {}).get(col + str(row)) or {}).get("kind")
 
2066
  "bindings": bindings,
2067
  "formula_errors": formula_error_census(readback),
2068
  "formula_error_comparison": readback_comparison,
2069
+ "formula_cycles": _formula_cycle_comparison(current, readback, formula_check),
2070
  "visual_metadata_preserved": True,
2071
  }
2072
  if summary_evidence is not None:
2073
  result["summary_block"] = summary_evidence
2074
+ if formula_evidence is not None:
2075
+ result["formula_repair"] = formula_evidence
2076
+ result["formula_error_census"] = {
2077
+ "before": _formula_repair_census(current), "after": _formula_repair_census(readback)}
2078
+ if migrated is not None:
2079
+ result["migrated"] = [
2080
+ {**row, "after_value": _tab(readback, row["sheet"])["cells"][row["address"]]["value"]}
2081
+ for row in migrated]
2082
+ if daily_evidence is not None:
2083
+ result["daily_sales_bindings"] = daily_evidence
2084
+ result["formula_error_census"] = {
2085
+ "before": _formula_repair_census(current), "after": _formula_repair_census(readback)}
2086
+ if metrics_evidence is not None:
2087
+ _daily_sales_invariants(readback, formula_check)
2088
+ remaining = {name: sorted(set(found) - OWNER_INPUTS.get(name, frozenset()))
2089
+ for name, found in hardcoded_numbers(readback).items() if name in OWNER_INPUTS}
2090
+ if any(remaining.values()):
2091
+ raise RepairRefusal("repair_hardcoded_remain", "Hardcoded numbers remain after the write: "
2092
+ + json.dumps(remaining, sort_keys=True))
2093
+ if metrics_evidence["tab_visual"]:
2094
+ visual_readback = _apply_tab_visual(st, target_workbook_id, metrics_evidence["tab_visual"],
2095
+ actor=actor, actor_name=actor_name)
2096
+ result["revision_after"] = visual_readback["rev"]
2097
+ metrics_evidence["hardcoded_readback"] = hardcoded_numbers(readback)
2098
+ result["computed_metrics"] = metrics_evidence
2099
+ result["formula_error_census"] = {
2100
+ "before": _formula_repair_census(current), "after": _formula_repair_census(readback)}
2101
  return result
platform/core/data_binding.py CHANGED
@@ -167,6 +167,69 @@ def local_override():
167
  return (not is_space()) and _env(_LOCAL_OPT_IN_VAR) == '1'
168
 
169
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
170
  def default_store():
171
  """What `OS_DATA_REPO` resolves to when it is unset.
172
 
@@ -221,7 +284,7 @@ def write_refusal(repo):
221
 
222
  def check_write(repo, operation='write'):
223
  """Raise `StoreWriteRefused` unless this deployment may write `repo`. The one enforcement door."""
224
- reason = write_refusal(repo)
225
  if reason:
226
  raise StoreWriteRefused(f'{operation} to {repo!r} refused: {reason}')
227
 
 
167
  return (not is_space()) and _env(_LOCAL_OPT_IN_VAR) == '1'
168
 
169
 
170
+ #: W-219. Inherited by every child a gate process starts, so a subprocess a gate spawns is a gate too.
171
+ _GATE_PROCESS_VAR = 'AIOS_GATE_PROCESS'
172
+
173
+ #: The directories the product gate battery lives in (`aios-web/api`, `aios-web/web`, `aios-web`,
174
+ #: `platform`). `ops/verify_*.py` are OPERATOR tools, some of which read or probe a live store on
175
+ #: purpose, so they are not gates here.
176
+ _GATE_DIRS = frozenset({'api', 'web', 'aios-web', 'platform'})
177
+
178
+
179
+ def _argv_gate():
180
+ """The `verify_*.py` gate script this process was started as, or `''`."""
181
+ import sys
182
+ argv0 = str((getattr(sys, 'argv', None) or [''])[0] or '')
183
+ name = os.path.basename(argv0).lower()
184
+ if not (name.startswith('verify_') and name.endswith('.py')):
185
+ return ''
186
+ parent = os.path.basename(os.path.dirname(os.path.abspath(argv0))).lower()
187
+ return name if parent in _GATE_DIRS else ''
188
+
189
+
190
+ def gate_process():
191
+ """The gate this process is, or `''`. W-219: a gate writes FIXTURE stores only, never a real one.
192
+
193
+ β›”β›” WHY THIS LIVES AT THE STORE DOOR AND NOT IN EACH GATE. A lane container carries the exact
194
+ isolated staging posture `store_pg.write_refusal` is written to ALLOW (`STORE_BACKEND=pg`, the
195
+ staging `DATABASE_URL`, `AIOS_DEPLOYMENT_ID=fsanyoto/loopable`, `AIOS_SANDBOX_TENANTS=
196
+ royal-imports`), and `_fake_store.installed` swaps module functions a bound `PgStore` never
197
+ calls. So `verify_meta.py` run in SDLC 61 wrote staging's `t_royal_imports` through the real
198
+ app, and a pin at the top of each gate (W-219, W-240) only covers the gates somebody pinned:
199
+ `verify_amazon` and `verify_odoo_relational` carry none. Asking the PROCESS covers every gate,
200
+ including one written tomorrow.
201
+
202
+ Two ways to be one: started as `verify_*.py` from a battery directory (every battery row and
203
+ every lane runs `python <dir>/verify_<x>.py`), or `AIOS_GATE_PROCESS=1` inherited from one.
204
+ β›” There is no way OUT through the environment, because a lane container exports whatever it
205
+ was handed: the only off switch is an in-process patch of this function, which is what a gate
206
+ testing the PRODUCT rules does.
207
+ """
208
+ name = _argv_gate()
209
+ if name:
210
+ return name
211
+ return 'a process started by a gate' if _env(_GATE_PROCESS_VAR) == '1' else ''
212
+
213
+
214
+ if _argv_gate():
215
+ os.environ[_GATE_PROCESS_VAR] = '1' # W-219: a child this gate spawns is a gate too
216
+
217
+
218
+ def gate_write_refusal(repo):
219
+ """`None` unless this is a gate process writing a PRODUCTION store, else why not. W-219.
220
+
221
+ Consulted by `check_write` BEFORE `write_refusal`, so neither `AIOS_ALLOW_PRODUCTION_DATA=1`
222
+ on a laptop nor a production identity opens a production store to a gate.
223
+ """
224
+ who = gate_process()
225
+ rid = str(repo or '').strip()
226
+ if not who or rid not in PRODUCTION_STORES:
227
+ return None
228
+ return (f'this process is a verification gate ({who}), and a gate never writes {rid}, which '
229
+ f'holds real customer data. A gate writes fixture stores only: empty HF_TOKEN and '
230
+ f'point OS_DATA_REPO at a fixture store (W-219).')
231
+
232
+
233
  def default_store():
234
  """What `OS_DATA_REPO` resolves to when it is unset.
235
 
 
284
 
285
  def check_write(repo, operation='write'):
286
  """Raise `StoreWriteRefused` unless this deployment may write `repo`. The one enforcement door."""
287
+ reason = gate_write_refusal(repo) or write_refusal(repo)
288
  if reason:
289
  raise StoreWriteRefused(f'{operation} to {repo!r} refused: {reason}')
290
 
platform/core/do_usage.py ADDED
@@ -0,0 +1,583 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """DigitalOcean billing observations, never SQL or a database wake-up probe.
2
+
3
+ The card uses GETs and memory only. The optional sampler receives storage
4
+ callbacks from the host and shares Neon's management-plane store-awake guard.
5
+ """
6
+ from __future__ import annotations
7
+
8
+ import copy
9
+ import json
10
+ import math
11
+ import os
12
+ import re
13
+ import threading
14
+ from datetime import datetime, timezone
15
+ from urllib.error import HTTPError, URLError
16
+ from urllib.request import HTTPRedirectHandler, Request, build_opener
17
+
18
+ from core.neon_usage import send_telegram, store_is_awake
19
+
20
+
21
+ API_BASE = "https://api.digitalocean.com/v2"
22
+ HTTP_TIMEOUT = 5
23
+ DEFAULT_CLUSTER_ID = "0c4049b4-8541-43a7-ab6e-a69c34a8c604"
24
+ STORE_KEY = "platform_do_usage_samples"
25
+ MAX_SAMPLES = 240
26
+ FIXED_PRICES = {"db-s-1vcpu-1gb": {"usd": 15.15, "source": "R28 fixed monthly price per node"}}
27
+ CLUSTER_SOURCE = "DigitalOcean databases API"
28
+ BALANCE_SOURCE = "DigitalOcean balance API (daily snapshot, account-wide)"
29
+ BILLING_SOURCE = "DigitalOcean billing history API"
30
+
31
+
32
+ class UsageReadError(Exception):
33
+ """Fixed, credential-free cause safe to show to an operator."""
34
+
35
+
36
+ class _NoRedirect(HTTPRedirectHandler):
37
+ def redirect_request(self, req, fp, code, msg, headers, newurl):
38
+ return None
39
+
40
+
41
+ def _get(path, token):
42
+ try:
43
+ request = Request(API_BASE + path, headers={"Authorization": "Bearer " + token}, method="GET")
44
+ with build_opener(_NoRedirect()).open(request, timeout=HTTP_TIMEOUT) as response:
45
+ payload = json.loads(response.read(1_000_001))
46
+ if not isinstance(payload, dict):
47
+ raise ValueError()
48
+ return payload
49
+ except HTTPError as exc:
50
+ raise UsageReadError(f"DigitalOcean API returned HTTP {exc.code}.") from None
51
+ except TimeoutError:
52
+ raise UsageReadError("DigitalOcean API timed out.") from None
53
+ except URLError as exc:
54
+ cause = "timed out" if isinstance(exc.reason, TimeoutError) else "could not be reached"
55
+ raise UsageReadError(f"DigitalOcean API {cause}.") from None
56
+ except Exception:
57
+ # Do not return the URL, response body or exception text: any can contain credentials.
58
+ raise UsageReadError("DigitalOcean API returned an unreadable response.") from None
59
+
60
+
61
+ def _now():
62
+ return datetime.now(timezone.utc).isoformat(timespec="seconds")
63
+
64
+
65
+ def _stamp(value):
66
+ try:
67
+ parsed = datetime.fromisoformat(str(value).replace("Z", "+00:00"))
68
+ return parsed.astimezone(timezone.utc) if parsed.tzinfo else None
69
+ except (ValueError, TypeError):
70
+ return None
71
+
72
+
73
+ def _text(value):
74
+ if not isinstance(value, str) or not value.strip():
75
+ raise ValueError()
76
+ return value.replace("\u2014", "; ").replace("\u2013", " to ")[:512]
77
+
78
+
79
+ def _money(value):
80
+ if isinstance(value, bool) or value is None:
81
+ raise ValueError()
82
+ number = float(value)
83
+ if not math.isfinite(number):
84
+ raise ValueError()
85
+ return number
86
+
87
+
88
+ def read_account():
89
+ """Independently read each API block; allowlist fields to exclude connection secrets."""
90
+ read_at = _now()
91
+ cluster = {"available": False, "cause": None, "source": CLUSTER_SOURCE, "read_at": read_at,
92
+ **dict.fromkeys(("id", "name", "size", "num_nodes", "region", "status", "version", "storage_size_mib"))}
93
+ balance = {"available": False, "cause": None, "source": BALANCE_SOURCE, "read_at": read_at,
94
+ **dict.fromkeys(("month_to_date_usage", "account_balance", "month_to_date_balance", "generated_at"))}
95
+ billing = {"available": False, "cause": None, "source": BILLING_SOURCE, "read_at": read_at}
96
+ price = {"monthly_usd": None, "source": None, "read_at": read_at, "cause": None}
97
+ result = {"available": False, "cause": None, "read_at": read_at, "cluster": cluster,
98
+ "balance": balance, "billing": billing, "billing_history": [], "fixed_price": price}
99
+ token = os.environ.get("AIOS_DO_USAGE_TOKEN") or os.environ.get("DIGITALOCEAN_TOKEN", "")
100
+ if not token:
101
+ cause = "No DigitalOcean token is configured on this deployment, so usage cannot be read."
102
+ result["cause"] = price["cause"] = cause
103
+ for block in (cluster, balance, billing):
104
+ block["cause"] = cause
105
+ return result
106
+ selected = os.environ.get("AIOS_DO_CLUSTER_ID", DEFAULT_CLUSTER_ID)
107
+
108
+ try:
109
+ if not re.fullmatch(r"[a-zA-Z0-9-]{1,128}", selected):
110
+ raise UsageReadError("AIOS_DO_CLUSTER_ID is invalid; the cluster could not be read.")
111
+ database = _get("/databases/" + selected, token).get("database")
112
+ if not isinstance(database, dict) or database.get("id") != selected:
113
+ raise ValueError()
114
+ fields = {key: _text(database.get(key)) for key in ("id", "name", "size", "region", "status", "version")}
115
+ nodes = database.get("num_nodes")
116
+ if type(nodes) is not int or nodes < 1:
117
+ raise ValueError()
118
+ storage = database.get("storage_size_mib")
119
+ fields.update(num_nodes=nodes, storage_size_mib=storage if type(storage) is int and storage >= 0 else None)
120
+ cluster.update(fields, available=True, read_at=_now())
121
+ quoted = FIXED_PRICES.get(fields["size"])
122
+ if quoted:
123
+ price.update(monthly_usd=round(quoted["usd"] * nodes, 2), source=quoted["source"], read_at=cluster["read_at"])
124
+ else:
125
+ price["cause"] = f"Fixed price UNKNOWN for size {fields['size']}; no fixed price is configured."
126
+ except UsageReadError as exc:
127
+ cluster["cause"] = str(exc)
128
+ except Exception:
129
+ cluster["cause"] = "DigitalOcean cluster response is missing valid cluster details."
130
+ if not cluster["available"]:
131
+ price["cause"] = cluster["cause"]
132
+
133
+ try:
134
+ payload = _get("/customers/my/balance", token)
135
+ amounts = {key: _money(payload.get(key)) for key in
136
+ ("month_to_date_usage", "account_balance", "month_to_date_balance")}
137
+ generated_at = payload.get("generated_at")
138
+ if amounts["month_to_date_usage"] < 0 or not _stamp(generated_at):
139
+ raise ValueError()
140
+ balance.update(amounts, generated_at=generated_at, available=True, read_at=_now())
141
+ except UsageReadError as exc:
142
+ balance["cause"] = str(exc)
143
+ except Exception:
144
+ balance["cause"] = "DigitalOcean balance response is missing valid amounts or its snapshot time."
145
+
146
+ try:
147
+ payload = _get("/customers/my/billing_history", token)
148
+ rows = payload.get("billing_history")
149
+ # A new account's empty history arrives as the bare list envelope {"links": {}, "meta": {}}
150
+ # with the key omitted (read 2026-09-25): an empty history, not an unreadable one.
151
+ if rows is None and isinstance(payload, dict) and "meta" in payload:
152
+ rows = []
153
+ if not isinstance(rows, list):
154
+ raise ValueError()
155
+ history = []
156
+ for row in rows:
157
+ date = row.get("date")
158
+ if not _stamp(date):
159
+ raise ValueError()
160
+ history.append({"date": date, "description": _text(row.get("description")),
161
+ "amount": _money(row.get("amount")), "type": _text(row.get("type"))})
162
+ # Report a bounded page honestly; never follow an API-provided URL with a bearer token.
163
+ pages = (payload.get("links") or {}).get("pages") or {}
164
+ billing.update(available=True, read_at=_now(), has_more=bool(pages.get("next")))
165
+ result["billing_history"] = history
166
+ except UsageReadError as exc:
167
+ billing["cause"] = str(exc)
168
+ except Exception:
169
+ billing["cause"] = "DigitalOcean billing history response could not be read."
170
+ result["available"] = any(block["available"] for block in (cluster, balance, billing))
171
+ if not result["available"]:
172
+ result["cause"] = " ".join(f"{name}: {block['cause']}" for name, block in
173
+ (("Cluster", cluster), ("Balance", balance), ("Billing history", billing)))
174
+ return result
175
+
176
+
177
+ def evaluate_alarm(account):
178
+ """Account-wide MTD against the sum of the configured cluster's fixed node prices."""
179
+ balance, price = account["balance"], account["fixed_price"]
180
+ if not balance["available"]:
181
+ return None, "Alert evaluation deferred. " + balance["cause"]
182
+ expected = price["monthly_usd"]
183
+ if expected is None:
184
+ return None, "Alert evaluation deferred. " + price["cause"]
185
+ override = os.environ.get("AIOS_DO_ALARM_USD")
186
+ try:
187
+ threshold = expected if override is None else _money(override)
188
+ except (ValueError, TypeError, OverflowError):
189
+ return None, "AIOS_DO_ALARM_USD is invalid; alert evaluation is deferred."
190
+ if balance["month_to_date_usage"] <= threshold:
191
+ return None, None
192
+ test = override is not None
193
+ cluster = account["cluster"]
194
+ message = ("TEST " if test else "") + (
195
+ f"DigitalOcean month-to-date spend USD {balance['month_to_date_usage']:.2f}, "
196
+ f"as of {balance['generated_at']} ({balance['source']}; read {balance['read_at']}). "
197
+ f"Expected fixed price USD {expected:.2f} a month ({price['source']}; read {price['read_at']}). "
198
+ f"Cluster {cluster['name']} ({cluster['size']}, {cluster['num_nodes']} "
199
+ f"node{'' if cluster['num_nodes'] == 1 else 's'}, {cluster['region']}).")
200
+ if test:
201
+ message += f" TEST threshold USD {threshold:.2f} (AIOS_DO_ALARM_USD; read {account['read_at']})."
202
+ return {"level": "red", "message": message, "test": test, "threshold": threshold,
203
+ "period": _stamp(balance["generated_at"]).strftime("%Y-%m")}, None
204
+
205
+
206
+ _lock = threading.RLock()
207
+ _sample_lock = threading.Lock()
208
+ _thread = None
209
+ _stop = threading.Event()
210
+ _samples = []
211
+ _alerted = {}
212
+ _loaded = False
213
+ _cluster_id = None
214
+ _status = {"every_s": 0, "last_at": None, "persisted": None,
215
+ "cause": "Scheduled sampling is disabled; AIOS_DO_SAMPLE_SECONDS is unset or zero."}
216
+
217
+
218
+ def snapshot():
219
+ with _lock:
220
+ return copy.deepcopy(_samples), dict(_status), dict(_alerted)
221
+
222
+
223
+ def report():
224
+ """The route performs management GETs only; it never loads storage or sends alerts."""
225
+ account = read_account()
226
+ samples, status, alerted = snapshot()
227
+ alarm, cause = evaluate_alarm(account)
228
+ return {**account, "samples": samples, "sampler": status, "alarm": alarm,
229
+ "alarm_cause": cause, "alerted": alerted}
230
+
231
+
232
+ def _clean_samples(samples):
233
+ clean = {}
234
+ for row in samples if isinstance(samples, list) else []:
235
+ try:
236
+ if not _stamp(row.get("at")) or not _stamp(row.get("generated_at")):
237
+ continue
238
+ usage = _money(row.get("month_to_date_usage"))
239
+ if usage < 0:
240
+ continue
241
+ clean[row["at"]] = {"at": row["at"], "generated_at": row["generated_at"],
242
+ "month_to_date_usage": usage, "source": BALANCE_SOURCE}
243
+ except (AttributeError, TypeError, ValueError, OverflowError):
244
+ continue
245
+ return sorted(clean.values(), key=lambda row: _stamp(row["at"]))[-MAX_SAMPLES:]
246
+
247
+
248
+ def _clean_alerted(alerted):
249
+ if not isinstance(alerted, dict):
250
+ raise ValueError()
251
+ for month, at in alerted.items():
252
+ if not isinstance(month, str) or not re.fullmatch(r"\d{4}-(0[1-9]|1[0-2])", month) or not _stamp(at):
253
+ raise ValueError()
254
+ return {key: alerted[key] for key in sorted(alerted)[-2:]}
255
+
256
+
257
+ def _sample_once(load, persist, *, sender=None):
258
+ # Serialize manual and scheduled ticks as well as delivery marker changes.
259
+ with _sample_lock:
260
+ _sample_tick(load, persist, sender=sender)
261
+
262
+
263
+ def _sample_tick(load, persist, *, sender=None):
264
+ global _loaded, _samples, _alerted, _cluster_id
265
+ account = read_account()
266
+ balance = account["balance"]
267
+ alarm, alarm_cause = evaluate_alarm(account)
268
+ if not balance["available"]:
269
+ with _lock:
270
+ _status["cause"] = alarm_cause
271
+ return
272
+ selected = os.environ.get("AIOS_DO_CLUSTER_ID", DEFAULT_CLUSTER_ID)
273
+ with _lock:
274
+ if _cluster_id != selected:
275
+ _cluster_id, _loaded, _samples, _alerted = selected, False, [], {}
276
+ _status["persisted"] = None
277
+ _samples = _clean_samples([*_samples, {"at": balance["read_at"],
278
+ "generated_at": balance["generated_at"], "month_to_date_usage": balance["month_to_date_usage"]}])
279
+ _status["last_at"] = balance["read_at"]
280
+ awake, cause = store_is_awake()
281
+ if not awake:
282
+ with _lock:
283
+ _status["cause"] = cause
284
+ return
285
+ try:
286
+ if not _loaded:
287
+ saved = load()
288
+ saved = {} if saved is None else saved
289
+ matching = saved.get("cluster_id") == selected
290
+ history = _clean_samples(saved.get("samples", [])) if matching else []
291
+ markers = _clean_alerted(saved.get("alerted", {})) if matching else {}
292
+ with _lock:
293
+ _samples = _clean_samples([*history, *_samples])
294
+ _alerted, _loaded = markers, True
295
+ except Exception:
296
+ with _lock:
297
+ _status["cause"] = "Sample history and alert markers could not be loaded; alerts deferred."
298
+ return
299
+ sent = False
300
+ try:
301
+ def save():
302
+ awake, cause = store_is_awake()
303
+ if not awake:
304
+ raise UsageReadError(cause)
305
+ samples, _, markers = snapshot()
306
+ persist({"cluster_id": selected, "samples": samples, "alerted": markers})
307
+ with _lock:
308
+ _status.update(persisted=_now(), cause=alarm_cause)
309
+
310
+ # Flush any successful delivery marker before considering another send.
311
+ save()
312
+ if not alarm or alarm["period"] in _alerted:
313
+ return
314
+ try:
315
+ ok, delivery = (sender or send_telegram)(alarm["message"])
316
+ except Exception:
317
+ ok, delivery = False, "Telegram alert could not be sent."
318
+ if not ok:
319
+ with _lock:
320
+ _status["cause"] = delivery
321
+ return
322
+ sent = True
323
+ with _lock:
324
+ _alerted = _clean_alerted({**_alerted, alarm["period"]: _now()})
325
+ save()
326
+ except Exception:
327
+ with _lock:
328
+ _status["cause"] = (
329
+ "Telegram alert sent; its marker remains pending in memory because storage is unavailable."
330
+ if sent else "Sample history could not be stored; alerts deferred until storage is available.")
331
+
332
+
333
+ def start_sampler(*, load=None, persist=None, sender=None):
334
+ """Off by default; at most one sampler per process, with a one-minute floor."""
335
+ global _thread
336
+ try:
337
+ every_s = int(os.environ.get("AIOS_DO_SAMPLE_SECONDS", "0"))
338
+ except ValueError:
339
+ every_s = 0
340
+ if every_s <= 0 or load is None or persist is None:
341
+ with _lock:
342
+ _status.update(every_s=0, cause="Scheduled sampling is disabled; AIOS_DO_SAMPLE_SECONDS is unset, invalid or zero, or storage callbacks are unavailable.")
343
+ return False
344
+ every_s = max(60, every_s)
345
+ with _lock:
346
+ if _thread is not None and _thread.is_alive():
347
+ return True
348
+ _status.update(every_s=every_s, cause="Waiting for the first scheduled sample.")
349
+ _stop.clear()
350
+
351
+ def run():
352
+ while not _stop.is_set():
353
+ try:
354
+ _sample_once(load, persist, sender=sender)
355
+ except Exception:
356
+ with _lock:
357
+ _status["cause"] = "The scheduled sample could not be completed; it will retry."
358
+ if _stop.wait(every_s):
359
+ break
360
+
361
+ _thread = threading.Thread(target=run, name="do-usage-sampler", daemon=True)
362
+ _thread.start()
363
+ return True
364
+
365
+
366
+ def stop_sampler():
367
+ _stop.set()
368
+
369
+
370
+ def _selftest():
371
+ """Run without credentials, sockets, runtimes, files or a real Telegram sender."""
372
+ import sys
373
+ from unittest.mock import patch
374
+ from core import neon_usage
375
+
376
+ module = sys.modules[__name__]
377
+ at = "2026-09-25T12:00:00Z"
378
+ snapshot_at = "2026-09-25T00:00:00Z"
379
+ database = {"id": DEFAULT_CLUSTER_ID, "name": "fixture-cluster", "size": "db-s-1vcpu-1gb",
380
+ "num_nodes": 1, "region": "nyc3", "status": "online", "version": "18",
381
+ "storage_size_mib": 10240, "connection": {"password": "fixture-secret"}}
382
+ responses = {
383
+ "/databases/" + DEFAULT_CLUSTER_ID: {"database": database},
384
+ "/customers/my/balance": {"month_to_date_usage": "20.00", "account_balance": "-1.00",
385
+ "month_to_date_balance": "19.00", "generated_at": snapshot_at},
386
+ "/customers/my/billing_history": {"billing_history": []},
387
+ }
388
+ calls, sends, documents = [], [], []
389
+
390
+ def fake_get(path, token):
391
+ calls.append(path)
392
+ return copy.deepcopy(responses[path])
393
+
394
+ def fake_sender(message):
395
+ sends.append(message)
396
+ return True, len(sends)
397
+
398
+ def persist(document):
399
+ documents.append(copy.deepcopy(document))
400
+
401
+ def reset():
402
+ global _loaded, _samples, _alerted, _cluster_id
403
+ _loaded, _samples, _alerted, _cluster_id = False, [], {}, None
404
+ sends.clear()
405
+ documents.clear()
406
+
407
+ # Double protection: all requests are fake, including if a test forgets sender=.
408
+ with patch.dict(os.environ, {}, clear=True), patch.object(module, "_get", side_effect=fake_get), \
409
+ patch.object(module, "send_telegram", side_effect=fake_sender), \
410
+ patch.object(neon_usage, "send_telegram", side_effect=fake_sender), \
411
+ patch.object(module, "_now", return_value=at):
412
+ missing = read_account()
413
+ assert not missing["available"] and missing["cause"] and not calls
414
+ assert missing["balance"]["month_to_date_usage"] is None
415
+ assert missing["fixed_price"]["monthly_usd"] is None
416
+ assert not start_sampler(load=lambda: {}, persist=persist, sender=fake_sender)
417
+ assert "disabled" in report()["sampler"]["cause"]
418
+ os.environ["DIGITALOCEAN_TOKEN"] = "fixture-fallback"
419
+ account = read_account()
420
+ assert account["fixed_price"]["monthly_usd"] == 15.15
421
+ assert account["balance"]["account_balance"] == -1
422
+ assert account["billing"]["available"] and account["billing_history"] == []
423
+ # The real empty history omits the key: {"links": {}, "meta": {}} is empty, not unreadable.
424
+ responses["/customers/my/billing_history"] = {"links": {}, "meta": {}}
425
+ enveloped = read_account()
426
+ assert enveloped["billing"]["available"] and enveloped["billing_history"] == []
427
+ responses["/customers/my/billing_history"] = {"links": {}}
428
+ assert not read_account()["billing"]["available"]
429
+ responses["/customers/my/billing_history"] = {"billing_history": []}
430
+ assert "fixture-secret" not in json.dumps(account)
431
+ assert account["balance"]["source"] == BALANCE_SOURCE and account["balance"]["generated_at"] == snapshot_at
432
+ os.environ["AIOS_DO_USAGE_TOKEN"] = "fixture-preferred"
433
+ def preferred_get(path, token):
434
+ assert token == "fixture-preferred"
435
+ return fake_get(path, token)
436
+ with patch.object(module, "_get", side_effect=preferred_get):
437
+ assert read_account()["available"]
438
+
439
+ for failed_path, failed_block in (("/databases/" + DEFAULT_CLUSTER_ID, "cluster"),
440
+ ("/customers/my/balance", "balance"),
441
+ ("/customers/my/billing_history", "billing")):
442
+ def fail_one(path, token):
443
+ if path == failed_path:
444
+ raise UsageReadError("DigitalOcean API returned HTTP 403.")
445
+ return fake_get(path, token)
446
+ with patch.object(module, "_get", side_effect=fail_one):
447
+ partial = read_account()
448
+ assert partial["available"] and not partial[failed_block]["available"]
449
+ assert "403" in partial[failed_block]["cause"]
450
+ assert all(partial[key]["available"] for key in ("cluster", "balance", "billing") if key != failed_block)
451
+ if failed_block == "balance":
452
+ assert partial["balance"]["month_to_date_usage"] is None
453
+ assert evaluate_alarm(partial)[0] is None and evaluate_alarm(partial)[1]
454
+ _sample_once(lambda: {}, persist, sender=fake_sender)
455
+ assert not sends and "403" in snapshot()[1]["cause"]
456
+
457
+ original_balance = copy.deepcopy(responses["/customers/my/balance"])
458
+ for invalid in (None, True, "NaN", "Infinity", "unreadable"):
459
+ responses["/customers/my/balance"]["month_to_date_usage"] = invalid
460
+ malformed = read_account()
461
+ assert not malformed["balance"]["available"] and malformed["balance"]["cause"]
462
+ assert malformed["balance"]["month_to_date_usage"] is None
463
+ assert evaluate_alarm(malformed)[0] is None
464
+ responses["/customers/my/balance"] = original_balance
465
+
466
+ database["size"] = "unknown-size"
467
+ unknown = read_account()
468
+ assert unknown["fixed_price"]["monthly_usd"] is None
469
+ assert "UNKNOWN" in unknown["fixed_price"]["cause"] and "unknown-size" in unknown["fixed_price"]["cause"]
470
+ assert evaluate_alarm(unknown)[0] is None
471
+ database["size"] = "db-s-1vcpu-1gb"
472
+ database["num_nodes"] = 2
473
+ assert read_account()["fixed_price"]["monthly_usd"] == 30.30
474
+ database["num_nodes"] = 1
475
+
476
+ reset()
477
+ os.environ["AIOS_DO_ALARM_USD"] = "1"
478
+ _sample_once(lambda: {}, persist, sender=fake_sender)
479
+ _sample_once(lambda: documents[-1], persist, sender=fake_sender)
480
+ assert len(sends) == 1 and sends[0].startswith("TEST ")
481
+ assert all(word in sends[0] for word in ("20.00", "15.15", snapshot_at, "fixture-cluster", "R28"))
482
+ saved = copy.deepcopy(documents[-1])
483
+ reset()
484
+ _sample_once(lambda: saved, persist, sender=fake_sender)
485
+ assert not sends and "2026-09" in documents[-1]["alerted"]
486
+ responses["/customers/my/balance"]["generated_at"] = "2026-10-01T00:00:00Z"
487
+ _sample_once(lambda: saved, persist, sender=fake_sender)
488
+ assert len(sends) == 1 # A new billing month permits one new delivery.
489
+ responses["/customers/my/balance"]["generated_at"] = snapshot_at
490
+
491
+ reset()
492
+ del os.environ["AIOS_DO_ALARM_USD"]
493
+ responses["/customers/my/balance"]["month_to_date_usage"] = "15.15"
494
+ _sample_once(lambda: {}, persist, sender=fake_sender)
495
+ assert not sends
496
+ responses["/customers/my/balance"]["month_to_date_usage"] = "0.00"
497
+ zero = report()
498
+ assert zero["balance"]["month_to_date_usage"] == 0 and not zero["alarm"]
499
+ _sample_once(lambda: {}, persist, sender=fake_sender)
500
+ assert not sends
501
+ os.environ["AIOS_DO_ALARM_USD"] = "-1"
502
+ _sample_once(lambda: {}, persist, sender=fake_sender)
503
+ assert len(sends) == 1 and sends[0].startswith("TEST ")
504
+ os.environ["AIOS_DO_ALARM_USD"] = "NaN"
505
+ assert report()["alarm"] is None and "invalid" in report()["alarm_cause"]
506
+
507
+ reset()
508
+ os.environ["AIOS_DO_ALARM_USD"] = "-1"
509
+ os.environ["STORE_BACKEND"] = "pg"
510
+ os.environ["DATABASE_URL"] = "postgresql://fixture@ep-fixture.neon.tech/app"
511
+ with patch.object(neon_usage, "read_project", return_value={"available": True, "id": "fixture"}), \
512
+ patch.object(neon_usage, "_get", return_value={"endpoints": [
513
+ {"id": "ep-fixture", "current_state": "idle"}]}):
514
+ def forbidden(*args):
515
+ raise AssertionError("Idle compute must never call storage")
516
+ _sample_once(forbidden, forbidden, sender=fake_sender)
517
+ assert not sends and "idle" in snapshot()[1]["cause"]
518
+ assert report()["balance"]["available"]
519
+ del os.environ["STORE_BACKEND"]
520
+ del os.environ["DATABASE_URL"]
521
+ with patch.object(neon_usage, "read_project", side_effect=AssertionError("Unexpected Neon read")):
522
+ assert store_is_awake(backend="pg", database_url="postgresql://fixture@fixture.db.ondigitalocean.com/app")[0]
523
+ assert not store_is_awake(backend="pg", database_url="")[0]
524
+ assert not store_is_awake({"available": False, "cause": "offline"}, backend="pg",
525
+ database_url="postgresql://fixture@ep-fixture.neon.tech/app")[0]
526
+ reset()
527
+ def failed_load():
528
+ raise RuntimeError("fixture-secret")
529
+ _sample_once(failed_load, persist, sender=fake_sender)
530
+ assert not sends and "loaded" in snapshot()[1]["cause"]
531
+ reset()
532
+ def failed_persist(document):
533
+ raise RuntimeError("fixture-secret")
534
+ _sample_once(lambda: {}, failed_persist, sender=fake_sender)
535
+ assert not sends and "stored" in snapshot()[1]["cause"]
536
+ assert "fixture-secret" not in json.dumps(report())
537
+ reset()
538
+ def fail_after_delivery(document):
539
+ if sends:
540
+ raise RuntimeError("fixture-secret")
541
+ persist(document)
542
+ _sample_once(lambda: {}, fail_after_delivery, sender=fake_sender)
543
+ assert len(sends) == 1 and "pending in memory" in snapshot()[1]["cause"]
544
+ _sample_once(lambda: {}, persist, sender=fake_sender)
545
+ assert len(sends) == 1 and "2026-09" in documents[-1]["alerted"]
546
+ rows = [{"at": f"2026-09-{1 + n // 24:02}T{n % 24:02}:00:00Z", "generated_at": snapshot_at,
547
+ "month_to_date_usage": n} for n in range(MAX_SAMPLES + 1)]
548
+ assert len(_clean_samples(rows)) == MAX_SAMPLES
549
+
550
+ # Exercise the actual HTTP boundary with a fake opener, including credential-bearing errors.
551
+ class Response:
552
+ def __enter__(self):
553
+ return self
554
+ def __exit__(self, *args):
555
+ pass
556
+ def read(self, limit):
557
+ return b'{"ok": true}'
558
+
559
+ class Opener:
560
+ def open(self, request, *, timeout):
561
+ assert request.get_method() == "GET" and timeout == 5
562
+ return Response()
563
+
564
+ with patch.object(module, "build_opener", return_value=Opener()) as opener:
565
+ assert _get("/fixture", "fixture-token") == {"ok": True}
566
+ assert isinstance(opener.call_args.args[0], _NoRedirect)
567
+ assert opener.call_args.args[0].redirect_request(None, None, 302, None, None, None) is None
568
+ for error in (HTTPError("fixture-private-url", 403, "fixture-token", None, None),
569
+ URLError("fixture-private-url"), TimeoutError("fixture-token"), ValueError("fixture-token")):
570
+ with patch.object(module, "build_opener", side_effect=error):
571
+ try:
572
+ _get("/fixture", "fixture-token")
573
+ raise AssertionError("The HTTP error must fail closed")
574
+ except UsageReadError as exc:
575
+ assert "fixture-" not in str(exc)
576
+ print("PASS DigitalOcean usage selftest: isolated API failures, pricing, snapshot provenance, monthly alerts, restart markers, idle-store guard and safe HTTP errors; fake senders only.")
577
+
578
+
579
+ if __name__ == "__main__":
580
+ import sys
581
+ if sys.argv[1:] != ["--selftest"]:
582
+ raise SystemExit("Usage: python -m core.do_usage --selftest")
583
+ _selftest()
platform/core/external_tools.py CHANGED
@@ -9,7 +9,7 @@ import json
9
  from dataclasses import dataclass
10
  from typing import Callable, Optional
11
 
12
- from core import sheets, sheet_structure, sheet_templates, write_execution
13
 
14
 
15
  @dataclass(frozen=True)
@@ -42,12 +42,19 @@ _STRING = {"type": "string"}
42
  _ADDRESS = {"type": "string", "description": "An A1 cell address on the named tab."}
43
  _FORMAT = _object({"kind": _STRING, "code": _STRING, "decimals": {"type": "integer"},
44
  "currency": _STRING, "thousands": {"type": "boolean"}}, ("kind",))
 
 
 
 
 
 
 
45
  _STYLE = _object({
46
  **{key: {"type": "boolean"} for key in ("bold", "italic", "underline", "strike", "wrap")},
47
  **{key: _STRING for key in ("fill", "color", "font", "align", "valign")},
48
  "size": {"type": ["number", "string"]}, "indent": {"type": "integer"},
49
  "textRotation": {"type": "number"}, "format": _FORMAT,
50
- "border": _object({side: {"type": "boolean"} for side in ("top", "right", "bottom", "left")}),
51
  })
52
  _VALIDATION = _object({"list": {"type": "array", "items": _STRING}}, ("list",))
53
  _FILTER = _object({"conj": {"enum": ["and", "or"]}, "nodes": {"type": "array", "items": _object({
@@ -62,15 +69,25 @@ _BINDING = _object({
62
  "period": _object({"unit": _STRING, "start": _STRING}),
63
  "ending": {"type": "object"},
64
  }, ("database", "aggregate", "scope"))
65
- _CELL = _object({"kind": {"enum": ["literal", "formula", "bound", "empty"]},
66
- "value": {}, "text": _STRING, "formula": _STRING, "state": _STRING,
67
- "style": _STYLE, "validation": _VALIDATION, "binding": _BINDING}, ("kind",))
68
  _RANGE = {"oneOf": [_STRING, _object({key: {"type": "integer", "minimum": 0}
69
  for key in ("col0", "row0", "col1", "row1")},
70
  ("col0", "row0", "col1", "row1"))]}
 
 
 
 
 
 
 
 
 
 
 
71
  _RULE = _object({"id": _STRING, "range": _RANGE, "kind": _STRING, "op": _STRING,
72
  "value": {}, "value2": {}, "formula": _STRING, "style": _STYLE,
73
- "scale": {"type": "object"}}, ("range", "op"))
 
 
74
  _CHART = _object({"id": _STRING, "kind": _STRING, "range": _RANGE, "title": _STRING,
75
  "stacking": _STRING, "source": {"type": "object"},
76
  "series": {"type": "array", "items": {"type": "object"}},
@@ -166,6 +183,16 @@ def _add_tab(ctx, args):
166
  return _put(ctx, args, book, tab["id"])
167
 
168
 
 
 
 
 
 
 
 
 
 
 
169
  def _save_template(ctx, args):
170
  book = ctx.project_book(_book(ctx, args))
171
  record = sheet_templates.save_sheet_template(
@@ -269,7 +296,7 @@ def _structure(ctx, args):
269
  return _put(ctx, args, shifted, tab["id"])
270
 
271
 
272
- def _tool(description, schema, handler, *, write=False, grants=("analyst", "sheets")):
273
  return {"description": description, "inputSchema": schema, "write": write,
274
  "grants": grants, "handler": handler}
275
 
@@ -328,6 +355,9 @@ TOOLS = {
328
  _object({"name": _STRING}, ("name",), additionalProperties=False), _create, write=True),
329
  "sheet_add_tab": _tool("Add a blank tab to a workbook.",
330
  _schema({"name": _STRING}, ("name",), tab=False), _add_tab, write=True),
 
 
 
331
  "sheet_read_cells": _tool("Read a tab with styles, geometry and figures under the maker's grants.",
332
  _schema({"addresses": {"type": "array", "items": _ADDRESS}}), _read_cells),
333
  "sheet_write_cells": _tool("Write cell documents with values or formulas; null deletes a cell.",
@@ -367,10 +397,10 @@ TOOLS = {
367
  ("workbook_id", "key", "name", "description"), additionalProperties=False),
368
  _save_template, write=True),
369
  "list_databases": _tool("List the databases permitted or visible to the key's maker.",
370
- _object(additionalProperties=False), lambda ctx, args: ctx.list_databases(), grants=("analyst",)),
371
  "describe_database": _tool("Read a database's field schema through the strict Assistant scope wall.",
372
  _object({"database": _STRING}, ("database",), additionalProperties=False),
373
- lambda ctx, args: ctx.describe_database(args["database"]), grants=("analyst",)),
374
  "query_database": _tool(
375
  "Query currently permitted records with bounded filters, stable sorts, cursor pagination and aggregates.",
376
  _object({
@@ -389,7 +419,7 @@ TOOLS = {
389
  }, ("database",), additionalProperties=False),
390
  lambda ctx, args: ctx.query_database(args["database"], {
391
  key: value for key, value in args.items() if key != "database"
392
- }), grants=("analyst",)),
393
  "view_upsert": _tool(
394
  "Create or update a saved View using the grid's filters, sorts, groupBy, visible fields and display config.",
395
  _object({"database": _STRING, "view": _object({
@@ -398,7 +428,7 @@ TOOLS = {
398
  "config": _VIEW_CONFIG, "note": _STRING}, ("id", "name", "config"),
399
  additionalProperties=False)},
400
  ("database", "view"), additionalProperties=False),
401
- lambda ctx, args: ctx.view_event("view_upsert", args), write=True, grants=("analyst",)),
402
  "view_share_preview": _tool(
403
  "Preview a personal draft View's C8 audience and field inheritance without sharing or mutating it.",
404
  _object({
@@ -408,21 +438,21 @@ TOOLS = {
408
  "user": _STRING, "role": {"enum": ["view", "edit"]},
409
  }, ("user", "role"), additionalProperties=False)},
410
  }, ("database", "view_id", "entries"), additionalProperties=False),
411
- lambda ctx, args: ctx.view_share_preview(args), grants=("analyst",)),
412
  "view_delete": _tool("Delete a saved View through its ordinary owner and database permission walls.",
413
  _object({"database": _STRING, "view_id": _STRING}, ("database", "view_id"),
414
  additionalProperties=False),
415
- lambda ctx, args: ctx.view_event("view_delete", args), write=True, grants=("analyst",)),
416
  "app_upsert": _tool(
417
  "Create an App with name, type, compatibleDatabases and config, or update it by app_id. Identity fields are immutable.",
418
  _object(_APPLICATION, additionalProperties=False,
419
  anyOf=[{"required": ["app_id"]}, {"required": ["name", "type", "config"]},
420
  {"required": ["name", "templateKind", "sourceDatabase", "fieldMap", "navPlacement"]}]),
421
- lambda ctx, args: ctx.app_upsert(args), write=True, grants=("analyst",)),
422
  "app_delete": _tool("Delete an App. confirmName must match its current name exactly.",
423
  _object({"app_id": _STRING, "confirmName": _STRING}, ("app_id", "confirmName"),
424
  additionalProperties=False),
425
- lambda ctx, args: ctx.app_delete(args), write=True, grants=("analyst",)),
426
  "database_object": _tool(
427
  "Discover/query a database or preview-confirm record and field CRUD through its live UI walls.",
428
  _object({
@@ -435,7 +465,7 @@ TOOLS = {
435
  "query": {"type": "object"},
436
  }, ("action",), additionalProperties=False),
437
  lambda ctx, args: _callback(ctx, "database_object", args), write=True,
438
- grants=("analyst",)),
439
  "view_object": _tool(
440
  "Read or preview-confirm a personal View operation through the ordinary grid writer.",
441
  _object({
@@ -452,7 +482,7 @@ TOOLS = {
452
  ctx.view_share_preview({"database": args["database"],
453
  "view_id": args["view_id"],
454
  "entries": args.get("entries") or []})
455
- ), write=True, grants=("analyst",)),
456
  **{name: _object_tool(name, actions)
457
  for name, actions in _SHEET_OBJECT_ACTIONS.items()},
458
  "validated_batch": _tool(
@@ -464,12 +494,12 @@ TOOLS = {
464
  additionalProperties=False)}},
465
  ("operations",), additionalProperties=False),
466
  lambda ctx, args: _callback(ctx, "validated_batch", args), write=True,
467
- grants=("analyst",)),
468
  "confirm_change": _tool(
469
  "Confirm one unexpired preview. Destructive/schema changes require its exact typed phrase.",
470
  _object({"confirmation_token": _STRING, "confirmation_phrase": _STRING},
471
  ("confirmation_token",), additionalProperties=False),
472
- lambda ctx, args: _callback(ctx, "confirm_change", args), grants=("analyst",)),
473
  }
474
 
475
 
@@ -506,7 +536,12 @@ def dispatch(ctx, name, arguments):
506
  if tool is None:
507
  raise sheets.SheetRefusal("tool_not_found", "The requested tool is not available.")
508
  for grant in tool["grants"]:
509
- ctx.session.require(grant)
 
 
 
 
 
510
  schema = tool["inputSchema"]
511
  if not isinstance(arguments, dict):
512
  raise sheets.SheetRefusal("tool_arguments_invalid", "Tool arguments must be an object.")
 
9
  from dataclasses import dataclass
10
  from typing import Callable, Optional
11
 
12
+ from core import perm_scope, sheets, sheet_structure, sheet_templates, write_execution
13
 
14
 
15
  @dataclass(frozen=True)
 
42
  _ADDRESS = {"type": "string", "description": "An A1 cell address on the named tab."}
43
  _FORMAT = _object({"kind": _STRING, "code": _STRING, "decimals": {"type": "integer"},
44
  "currency": _STRING, "thousands": {"type": "boolean"}}, ("kind",))
45
+ # S67-T14 / contract C2: a border side is the legacy `true` (read as a thin black line) or
46
+ # `{style, color}` with one of Excel's seven line styles and a six digit hex colour; `false` or an
47
+ # absent side is no side. `sheets.py::_clean_border_side` refuses anything else by code, and this
48
+ # publishes the same shape so an AI client can draw a styled border at all.
49
+ _BORDER_SIDE = {"oneOf": [{"type": "boolean"}, _object({
50
+ "style": {"enum": list(sheets.CELL_BORDER_STYLES)},
51
+ "color": {"type": "string", "pattern": "^#[0-9A-Fa-f]{6}$"}}, ("style", "color"))]}
52
  _STYLE = _object({
53
  **{key: {"type": "boolean"} for key in ("bold", "italic", "underline", "strike", "wrap")},
54
  **{key: _STRING for key in ("fill", "color", "font", "align", "valign")},
55
  "size": {"type": ["number", "string"]}, "indent": {"type": "integer"},
56
  "textRotation": {"type": "number"}, "format": _FORMAT,
57
+ "border": _object({side: _BORDER_SIDE for side in ("top", "right", "bottom", "left")}),
58
  })
59
  _VALIDATION = _object({"list": {"type": "array", "items": _STRING}}, ("list",))
60
  _FILTER = _object({"conj": {"enum": ["and", "or"]}, "nodes": {"type": "array", "items": _object({
 
69
  "period": _object({"unit": _STRING, "start": _STRING}),
70
  "ending": {"type": "object"},
71
  }, ("database", "aggregate", "scope"))
 
 
 
72
  _RANGE = {"oneOf": [_STRING, _object({key: {"type": "integer", "minimum": 0}
73
  for key in ("col0", "row0", "col1", "row1")},
74
  ("col0", "row0", "col1", "row1"))]}
75
+ # S67-T18: an in-cell sparkline, `{kind, range}` exactly as `sheets.py::_clean_sparkline` stores
76
+ # it; the range is a rectangle on the SAME tab (an A1 range such as "B2:M2", or the dict).
77
+ _SPARKLINE = _object({"kind": {"enum": list(sheets.SPARKLINE_KINDS)}, "range": _RANGE},
78
+ ("kind", "range"))
79
+ _CELL = _object({"kind": {"enum": ["literal", "formula", "bound", "empty"]},
80
+ "value": {}, "text": _STRING, "formula": _STRING, "state": _STRING,
81
+ "style": _STYLE, "validation": _VALIDATION, "binding": _BINDING,
82
+ "sparkline": _SPARKLINE}, ("kind",))
83
+ # S67-T13: a colour scale's two ends and what each end means. `sheets.py::_clean_rules` refuses
84
+ # a type outside `SHEET_RULE_SCALE_TYPES`; the enum publishes the same list to the caller.
85
+ _SCALE_TYPE = {"enum": list(sheets.SHEET_RULE_SCALE_TYPES)}
86
  _RULE = _object({"id": _STRING, "range": _RANGE, "kind": _STRING, "op": _STRING,
87
  "value": {}, "value2": {}, "formula": _STRING, "style": _STYLE,
88
+ "scale": _object({"low": _STRING, "high": _STRING,
89
+ "minType": _SCALE_TYPE, "maxType": _SCALE_TYPE})},
90
+ ("range", "op"))
91
  _CHART = _object({"id": _STRING, "kind": _STRING, "range": _RANGE, "title": _STRING,
92
  "stacking": _STRING, "source": {"type": "object"},
93
  "series": {"type": "array", "items": {"type": "object"}},
 
183
  return _put(ctx, args, book, tab["id"])
184
 
185
 
186
+ def _delete_tab(ctx, args):
187
+ book = _book(ctx, args, write=True)
188
+ tab = sheets.find_tab(book, args["tab"])
189
+ if len(book["sheets"]) <= 1:
190
+ raise sheets.SheetRefusal(
191
+ "tab_last", "A workbook keeps at least one sheet, so this one cannot be deleted.")
192
+ book["sheets"] = [row for row in book["sheets"] if row["id"] != tab["id"]]
193
+ return _put(ctx, args, book)
194
+
195
+
196
  def _save_template(ctx, args):
197
  book = ctx.project_book(_book(ctx, args))
198
  record = sheet_templates.save_sheet_template(
 
296
  return _put(ctx, args, shifted, tab["id"])
297
 
298
 
299
+ def _tool(description, schema, handler, *, write=False, grants=("ai_apps", "sheets")):
300
  return {"description": description, "inputSchema": schema, "write": write,
301
  "grants": grants, "handler": handler}
302
 
 
355
  _object({"name": _STRING}, ("name",), additionalProperties=False), _create, write=True),
356
  "sheet_add_tab": _tool("Add a blank tab to a workbook.",
357
  _schema({"name": _STRING}, ("name",), tab=False), _add_tab, write=True),
358
+ "sheet_delete_tab": _tool(
359
+ "Preview deleting a tab by id or name; requires typed confirmation. The last tab cannot be deleted.",
360
+ _schema({}), _delete_tab, write=True),
361
  "sheet_read_cells": _tool("Read a tab with styles, geometry and figures under the maker's grants.",
362
  _schema({"addresses": {"type": "array", "items": _ADDRESS}}), _read_cells),
363
  "sheet_write_cells": _tool("Write cell documents with values or formulas; null deletes a cell.",
 
397
  ("workbook_id", "key", "name", "description"), additionalProperties=False),
398
  _save_template, write=True),
399
  "list_databases": _tool("List the databases permitted or visible to the key's maker.",
400
+ _object(additionalProperties=False), lambda ctx, args: ctx.list_databases(), grants=("ai_apps",)),
401
  "describe_database": _tool("Read a database's field schema through the strict Assistant scope wall.",
402
  _object({"database": _STRING}, ("database",), additionalProperties=False),
403
+ lambda ctx, args: ctx.describe_database(args["database"]), grants=("ai_apps",)),
404
  "query_database": _tool(
405
  "Query currently permitted records with bounded filters, stable sorts, cursor pagination and aggregates.",
406
  _object({
 
419
  }, ("database",), additionalProperties=False),
420
  lambda ctx, args: ctx.query_database(args["database"], {
421
  key: value for key, value in args.items() if key != "database"
422
+ }), grants=("ai_apps",)),
423
  "view_upsert": _tool(
424
  "Create or update a saved View using the grid's filters, sorts, groupBy, visible fields and display config.",
425
  _object({"database": _STRING, "view": _object({
 
428
  "config": _VIEW_CONFIG, "note": _STRING}, ("id", "name", "config"),
429
  additionalProperties=False)},
430
  ("database", "view"), additionalProperties=False),
431
+ lambda ctx, args: ctx.view_event("view_upsert", args), write=True, grants=("ai_apps",)),
432
  "view_share_preview": _tool(
433
  "Preview a personal draft View's C8 audience and field inheritance without sharing or mutating it.",
434
  _object({
 
438
  "user": _STRING, "role": {"enum": ["view", "edit"]},
439
  }, ("user", "role"), additionalProperties=False)},
440
  }, ("database", "view_id", "entries"), additionalProperties=False),
441
+ lambda ctx, args: ctx.view_share_preview(args), grants=("ai_apps",)),
442
  "view_delete": _tool("Delete a saved View through its ordinary owner and database permission walls.",
443
  _object({"database": _STRING, "view_id": _STRING}, ("database", "view_id"),
444
  additionalProperties=False),
445
+ lambda ctx, args: ctx.view_event("view_delete", args), write=True, grants=("ai_apps",)),
446
  "app_upsert": _tool(
447
  "Create an App with name, type, compatibleDatabases and config, or update it by app_id. Identity fields are immutable.",
448
  _object(_APPLICATION, additionalProperties=False,
449
  anyOf=[{"required": ["app_id"]}, {"required": ["name", "type", "config"]},
450
  {"required": ["name", "templateKind", "sourceDatabase", "fieldMap", "navPlacement"]}]),
451
+ lambda ctx, args: ctx.app_upsert(args), write=True, grants=("ai_apps",)),
452
  "app_delete": _tool("Delete an App. confirmName must match its current name exactly.",
453
  _object({"app_id": _STRING, "confirmName": _STRING}, ("app_id", "confirmName"),
454
  additionalProperties=False),
455
+ lambda ctx, args: ctx.app_delete(args), write=True, grants=("ai_apps",)),
456
  "database_object": _tool(
457
  "Discover/query a database or preview-confirm record and field CRUD through its live UI walls.",
458
  _object({
 
465
  "query": {"type": "object"},
466
  }, ("action",), additionalProperties=False),
467
  lambda ctx, args: _callback(ctx, "database_object", args), write=True,
468
+ grants=("ai_apps",)),
469
  "view_object": _tool(
470
  "Read or preview-confirm a personal View operation through the ordinary grid writer.",
471
  _object({
 
482
  ctx.view_share_preview({"database": args["database"],
483
  "view_id": args["view_id"],
484
  "entries": args.get("entries") or []})
485
+ ), write=True, grants=("ai_apps",)),
486
  **{name: _object_tool(name, actions)
487
  for name, actions in _SHEET_OBJECT_ACTIONS.items()},
488
  "validated_batch": _tool(
 
494
  additionalProperties=False)}},
495
  ("operations",), additionalProperties=False),
496
  lambda ctx, args: _callback(ctx, "validated_batch", args), write=True,
497
+ grants=("ai_apps",)),
498
  "confirm_change": _tool(
499
  "Confirm one unexpired preview. Destructive/schema changes require its exact typed phrase.",
500
  _object({"confirmation_token": _STRING, "confirmation_phrase": _STRING},
501
  ("confirmation_token",), additionalProperties=False),
502
+ lambda ctx, args: _callback(ctx, "confirm_change", args), grants=("ai_apps",)),
503
  }
504
 
505
 
 
536
  if tool is None:
537
  raise sheets.SheetRefusal("tool_not_found", "The requested tool is not available.")
538
  for grant in tool["grants"]:
539
+ if grant == perm_scope.AI_APPS:
540
+ reason = perm_scope.ai_apps_refusal(ctx.session.user, ctx.session.runtime)
541
+ if reason:
542
+ raise sheets.SheetRefusal("ai_apps_off", reason)
543
+ else:
544
+ ctx.session.require(grant)
545
  schema = tool["inputSchema"]
546
  if not isinstance(arguments, dict):
547
  raise sheets.SheetRefusal("tool_arguments_invalid", "Tool arguments must be an object.")
platform/core/grid_events.py CHANGED
@@ -632,6 +632,57 @@ def _grant_marker(ctx, ws):
632
  return _has_grants
633
 
634
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
635
  def table_workspace(ctx, allowed_pids=None, consume_corrections=True, capture_document=False):
636
  """Durable Airtable-style view/schema/overlay state, with an honest session fallback.
637
 
@@ -707,6 +758,8 @@ def table_workspace(ctx, allowed_pids=None, consume_corrections=True, capture_do
707
  merged.update(ws.get('views') or {}) # own views win the id collision
708
  ws['views'] = {vid: scope_view(v, uname, allowed_pids)
709
  for vid, v in merged.items()}
 
 
710
  return ws
711
  # Store down. Streamlit degrades to its captioned session workspace exactly as before; an
712
  # API caller has no durable dict and must be told, not quietly served memory.
@@ -1584,7 +1637,13 @@ def handle_one(event, ctx):
1584
  _refuse(ctx, 'invalid_application_ref',
1585
  'that Application reference is not a valid typed identity', view_id)
1586
  return False
1587
- config['application'] = application
 
 
 
 
 
 
1588
  # Wave-6 item 10: a direct non-grid display mode rides the view config, validated
1589
  # structurally (mode whitelist, refs must be this table's fields). A bound Application's
1590
  # config remains in the registry and is resolved on read, so an offered clone is dropped.
 
632
  return _has_grants
633
 
634
 
635
+ #: S67-T39 / R36: the Application registry's store key. READ here, never imported:
636
+ #: `application_registry` lives in the api layer and core never imports up (ARCHITECTURE).
637
+ _APPLICATIONS_KEY = 'applications_v1'
638
+
639
+
640
+ def _live_application_ids(ctx):
641
+ """The set of App ids the tenant registry holds, or None when that is not KNOWN.
642
+
643
+ None (read failed, or no registry document with an `applications` map) means "strip
644
+ nothing": an unknown answer must never turn a live App binding into a plain view.
645
+ """
646
+ try:
647
+ doc = _store_of(ctx).get(_APPLICATIONS_KEY)
648
+ except Exception as exc:
649
+ _tel.error('applications:live', exc)
650
+ return None
651
+ apps = doc.get('applications') if isinstance(doc, dict) else None
652
+ return {str(key) for key in apps} if isinstance(apps, dict) else None
653
+
654
+
655
+ def _application_ref_id(view):
656
+ config = view.get('config') if isinstance(view, dict) else None
657
+ ref = config.get('application') if isinstance(config, dict) else None
658
+ return str(ref.get('id') or '') if isinstance(ref, dict) else None
659
+
660
+
661
+ def _drop_dead_application_refs(ctx, views):
662
+ """S67-T39 / R36 / D-2233: a View bound to a DELETED App is served as a plain view of its
663
+ database (the `application` member dropped, everything else kept), so it opens with no
664
+ error. Projection only, and it never raises: any failure serves the views unchanged.
665
+ The registry is read only when some View actually carries a reference.
666
+ """
667
+ try:
668
+ if not any(_application_ref_id(v) is not None for v in (views or {}).values()):
669
+ return views
670
+ live = _live_application_ids(ctx)
671
+ if live is None:
672
+ return views
673
+ out = {}
674
+ for vid, view in views.items():
675
+ ref_id = _application_ref_id(view)
676
+ if ref_id is not None and ref_id not in live:
677
+ config = {k: v for k, v in view['config'].items() if k != 'application'}
678
+ view = {**view, 'config': config}
679
+ out[vid] = view
680
+ return out
681
+ except Exception as exc:
682
+ _tel.error('applications:dead_refs', exc)
683
+ return views
684
+
685
+
686
  def table_workspace(ctx, allowed_pids=None, consume_corrections=True, capture_document=False):
687
  """Durable Airtable-style view/schema/overlay state, with an honest session fallback.
688
 
 
758
  merged.update(ws.get('views') or {}) # own views win the id collision
759
  ws['views'] = {vid: scope_view(v, uname, allowed_pids)
760
  for vid, v in merged.items()}
761
+ if ws.get('views'):
762
+ ws['views'] = _drop_dead_application_refs(ctx, ws['views'])
763
  return ws
764
  # Store down. Streamlit degrades to its captioned session workspace exactly as before; an
765
  # API caller has no durable dict and must be told, not quietly served memory.
 
1637
  _refuse(ctx, 'invalid_application_ref',
1638
  'that Application reference is not a valid typed identity', view_id)
1639
  return False
1640
+ # S67-T39 / R36: a reference to a DELETED App is dropped, not stored, so the View
1641
+ # saves as a plain view of its database. Unknown registry state keeps the ref.
1642
+ _live = _live_application_ids(ctx)
1643
+ if _live is not None and application['id'] not in _live:
1644
+ application = None
1645
+ else:
1646
+ config['application'] = application
1647
  # Wave-6 item 10: a direct non-grid display mode rides the view config, validated
1648
  # structurally (mode whitelist, refs must be this table's fields). A bound Application's
1649
  # config remains in the registry and is resolved on read, so an offered clone is dropped.
platform/core/neon_usage.py ADDED
@@ -0,0 +1,590 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Neon management-plane usage. Importing or reading a card never opens a database.
2
+
3
+ Storage is injected by the API host and called only after an awake observation.
4
+ Unchanged counters are stale observations, not evidence of zero consumption.
5
+ """
6
+ from __future__ import annotations
7
+
8
+ import copy
9
+ import json
10
+ import math
11
+ import os
12
+ import re
13
+ import threading
14
+ from datetime import datetime, timezone
15
+ from urllib.error import HTTPError, URLError
16
+ from urllib.parse import urlencode, urlsplit
17
+ from urllib.request import HTTPRedirectHandler, Request, build_opener
18
+
19
+
20
+ # Launch pricing: AutoWiki/research/neon-usage-samples/2026-09-25-sample1.json
21
+ # notes[2] and AutoWiki/research/neon-alternative-feasibility.md. The compute
22
+ # allowance is a meter/alert threshold; ALL CU-hours are billed, including it.
23
+ LAUNCH_PLAN = {
24
+ "allowance": {"transfer_gb": 500.0, "compute_hours": 300.0, "storage_gib": 10.0},
25
+ "usd": {"transfer_per_gb": 0.10, "compute_per_hour": 0.106,
26
+ "storage_per_gb_month": 0.35},
27
+ }
28
+ API_BASE = "https://console.neon.tech/api/v2"
29
+ HTTP_TIMEOUT = 5
30
+ DEFAULT_SAMPLE_SECONDS = 21600
31
+ MAX_SAMPLES = 240
32
+ DEFAULT_ALARM_SHARE = 0.70
33
+ STORE_KEY = "platform_neon_usage_samples"
34
+ _FIELDS = ("data_transfer_bytes", "compute_time_seconds", "synthetic_storage_size")
35
+ _METRIC_LABELS = {"transfer_gb": "Transfer", "compute_hours": "Compute", "storage_gib": "Storage"}
36
+ _ALERT_KEYS = {f"{metric}:{kind}" for metric in _METRIC_LABELS
37
+ for kind in ("share", "projected_overage")}
38
+ _guards = None
39
+
40
+
41
+ def set_guards(guards):
42
+ """The API host publishes its startup checks; this layer never imports the API."""
43
+ global _guards
44
+ _guards = copy.deepcopy(guards)
45
+
46
+
47
+ class UsageReadError(Exception):
48
+ """Contains only a fixed, credential-free explanation suitable for the card."""
49
+
50
+
51
+ class _NoRedirect(HTTPRedirectHandler):
52
+ def redirect_request(self, req, fp, code, msg, headers, newurl):
53
+ return None
54
+
55
+
56
+ def _get(path, key):
57
+ request = Request(API_BASE + path, headers={"Authorization": "Bearer " + key}, method="GET")
58
+ try:
59
+ with build_opener(_NoRedirect()).open(request, timeout=HTTP_TIMEOUT) as response:
60
+ payload = json.loads(response.read(1_000_001))
61
+ if not isinstance(payload, dict):
62
+ raise ValueError()
63
+ return payload
64
+ except HTTPError as exc:
65
+ raise UsageReadError(f"Neon management API returned HTTP {exc.code}.") from None
66
+ except TimeoutError:
67
+ raise UsageReadError("Neon management API timed out.") from None
68
+ except URLError as exc:
69
+ cause = "timed out" if isinstance(exc.reason, TimeoutError) else "could not be reached"
70
+ raise UsageReadError(f"Neon management API {cause}.") from None
71
+ except (ValueError, OSError):
72
+ raise UsageReadError("Neon management API returned an unreadable response.") from None
73
+
74
+
75
+ def _identifier(value):
76
+ return isinstance(value, str) and re.fullmatch(r"[A-Za-z0-9_-]{1,128}", value) is not None
77
+
78
+
79
+ def _pages(path, field, key, params=None):
80
+ """Bound discovery, and refuse partial discovery rather than guessing uniqueness."""
81
+ params, seen = dict(params or {}), set()
82
+ for _ in range(10):
83
+ payload = _get(path + ("?" + urlencode(params) if params else ""), key)
84
+ rows = payload.get(field)
85
+ if not isinstance(rows, list) or any(not isinstance(row, dict) for row in rows):
86
+ raise UsageReadError("Neon project discovery returned an unreadable response.")
87
+ # Neon echoes the last cursor even on the terminal empty project page.
88
+ if not rows:
89
+ return
90
+ yield from rows
91
+ cursor = (payload.get("pagination") or {}).get("cursor")
92
+ if not cursor:
93
+ return
94
+ if cursor in seen:
95
+ break
96
+ seen.add(cursor)
97
+ params["cursor"] = cursor
98
+ raise UsageReadError("Neon project discovery is incomplete; set NEON_PROJECT_ID.")
99
+
100
+
101
+ def resolve_project_id(api_key=None, project_id=None):
102
+ """Explicit id first; otherwise exactly one project across the visible orgs."""
103
+ key = os.environ.get("NEON_API_KEY", "") if api_key is None else api_key
104
+ if not key:
105
+ raise UsageReadError("NEON_API_KEY is absent; Neon usage cannot be read.")
106
+ selected = os.environ.get("NEON_PROJECT_ID", "") if project_id is None else project_id
107
+ if selected:
108
+ if not _identifier(selected):
109
+ raise UsageReadError("NEON_PROJECT_ID is invalid.")
110
+ return selected
111
+ projects = set()
112
+ for org in _pages("/users/me/organizations", "organizations", key):
113
+ if not _identifier(org.get("id")):
114
+ raise UsageReadError("Neon organization discovery returned no usable id.")
115
+ for project in _pages("/projects", "projects", key, {"org_id": org["id"], "limit": 100}):
116
+ if not _identifier(project.get("id")):
117
+ raise UsageReadError("Neon project discovery returned no usable id.")
118
+ projects.add(project["id"])
119
+ if len(projects) > 1:
120
+ raise UsageReadError("Multiple Neon projects are visible; set NEON_PROJECT_ID.")
121
+ if len(projects) != 1:
122
+ raise UsageReadError("No Neon project is visible; set NEON_PROJECT_ID.")
123
+ return projects.pop()
124
+
125
+
126
+ def _now():
127
+ return datetime.now(timezone.utc).isoformat(timespec="seconds")
128
+
129
+
130
+ def _stamp(value):
131
+ try:
132
+ parsed = datetime.fromisoformat(str(value).replace("Z", "+00:00"))
133
+ return parsed if parsed.tzinfo else None
134
+ except (ValueError, TypeError):
135
+ return None
136
+
137
+
138
+ def _number(value):
139
+ return (isinstance(value, (int, float)) and not isinstance(value, bool)
140
+ and math.isfinite(value) and value >= 0)
141
+
142
+
143
+ def read_project(api_key=None, project_id=None):
144
+ """Consumption comes ONLY from GET /projects/{id}, never branch sums or SQL."""
145
+ result = {"available": False, "cause": None, "read_at": _now(), "id": None, "name": None,
146
+ "consumption_period_start": None, "consumption_period_end": None,
147
+ **dict.fromkeys(_FIELDS)}
148
+ try:
149
+ key = os.environ.get("NEON_API_KEY", "") if api_key is None else api_key
150
+ selected = resolve_project_id(key, project_id)
151
+ project = _get("/projects/" + selected, key).get("project")
152
+ if not isinstance(project, dict) or any(not _number(project.get(k)) for k in _FIELDS):
153
+ raise UsageReadError("Neon project response is missing valid usage totals.")
154
+ start, end = (_stamp(project.get("consumption_period_" + k)) for k in ("start", "end"))
155
+ if not start or not end or end <= start:
156
+ raise UsageReadError("Neon project response is missing a valid billing period.")
157
+ if project.get("id") != selected:
158
+ raise UsageReadError("Neon project response did not match the requested project.")
159
+ result.update({k: project[k] for k in _FIELDS})
160
+ result.update(available=True, id=selected, name=project.get("name") or selected,
161
+ consumption_period_start=project["consumption_period_start"],
162
+ consumption_period_end=project["consumption_period_end"], read_at=_now())
163
+ except UsageReadError as exc:
164
+ result["cause"] = str(exc)
165
+ except Exception:
166
+ # Never return exception text: HTTP clients and credentials can occur in it.
167
+ result["cause"] = "Neon usage could not be read because of an unexpected response."
168
+ return result
169
+
170
+
171
+ def _clean_samples(samples):
172
+ clean = {}
173
+ for item in samples if isinstance(samples, list) else []:
174
+ if not isinstance(item, dict) or not _stamp(item.get("at")) or not _stamp(item.get("period_start")):
175
+ continue
176
+ if any(not _number(item.get(k)) for k in ("transfer_bytes", "compute_seconds", "storage_bytes")):
177
+ continue
178
+ row = {k: item[k] for k in ("at", "transfer_bytes", "compute_seconds", "storage_bytes", "period_start")}
179
+ clean[(row["at"], row["period_start"])] = row
180
+ return sorted(clean.values(), key=lambda row: _stamp(row["at"]))[-MAX_SAMPLES:]
181
+
182
+
183
+ def _rate(samples, field, divisor, *, gauge=False):
184
+ result = {"per_day": None, "cause": "Waiting for two samples with different values; Neon totals refresh slowly.",
185
+ "from_at": None, "to_at": None}
186
+ if len(samples) < 2:
187
+ return result
188
+ latest = samples[-1]
189
+ for prior in reversed(samples[:-1]):
190
+ days = (_stamp(latest["at"]) - _stamp(prior["at"])).total_seconds() / 86400
191
+ delta = latest[field] - prior[field]
192
+ if days <= 0 or delta == 0:
193
+ continue
194
+ if delta < 0 and not gauge:
195
+ result["cause"] = "The counter decreased; waiting for a new pair of increasing samples."
196
+ return result
197
+ return {"per_day": delta / divisor / days, "cause": None,
198
+ "from_at": prior["at"], "to_at": latest["at"]}
199
+ return result
200
+
201
+
202
+ def _alarm_threshold():
203
+ raw = os.environ.get("AIOS_NEON_ALARM_SHARE")
204
+ if raw is None:
205
+ return DEFAULT_ALARM_SHARE, None
206
+ try:
207
+ threshold = float(raw)
208
+ if not math.isfinite(threshold) or not 0 < threshold <= 1.5:
209
+ raise ValueError()
210
+ return threshold, None
211
+ except ValueError:
212
+ return DEFAULT_ALARM_SHARE, "AIOS_NEON_ALARM_SHARE is invalid; using the default 70% threshold."
213
+
214
+
215
+ def _alarm_message(summary, crossings, test):
216
+ metrics = []
217
+ for metric, label in _METRIC_LABELS.items():
218
+ rows = [row for row in crossings if row["metric"] == metric]
219
+ if rows:
220
+ kinds = ", ".join("share limit" if row["kind"] == "share" else "projected overage"
221
+ for row in rows)
222
+ metrics.append(f"{label} {rows[0]['share']:.1%} used ({kinds})")
223
+ total = summary.get("projection", {}).get("total")
224
+ bill = f"USD {total:,.2f}" if _number(total) else "unavailable"
225
+ return ("TEST " if test else "") + "Neon usage: " + "; ".join(metrics) + f". Projected bill {bill}."
226
+
227
+
228
+ def evaluate_alarm(summary, share_threshold=DEFAULT_ALARM_SHARE):
229
+ """Pure threshold evaluation, independent of delivery or saved alert markers."""
230
+ if not summary.get("available"):
231
+ return None
232
+ if not _number(share_threshold) or not 0 < share_threshold <= 1.5:
233
+ share_threshold = DEFAULT_ALARM_SHARE
234
+ crossings = []
235
+ projection = summary.get("projection", {})
236
+ bill = projection.get("total")
237
+ for metric in _METRIC_LABELS:
238
+ share = summary.get("share", {}).get(metric)
239
+ if not _number(share):
240
+ continue
241
+ projected = projection.get("usage", {}).get(metric)
242
+ allowance = summary.get("allowance", {}).get(metric)
243
+ for kind, crossed in (
244
+ ("share", share >= share_threshold),
245
+ ("projected_overage", _number(projected) and _number(allowance) and projected > allowance),
246
+ ):
247
+ if crossed:
248
+ crossings.append({"metric": metric, "kind": kind, "share": share,
249
+ "projected_bill": bill if _number(bill) else None})
250
+ if not crossings:
251
+ return None
252
+ test = share_threshold != DEFAULT_ALARM_SHARE
253
+ return {"level": "red", "message": _alarm_message(summary, crossings, test),
254
+ "crossings": crossings, "threshold": share_threshold, "test": test}
255
+
256
+
257
+ def send_telegram(text):
258
+ """Bounded POST; return only a message id or a fixed, credential-free cause."""
259
+ token = os.environ.get("TELEGRAM_BOT_TOKEN", "")
260
+ chat_id = os.environ.get("TELEGRAM_CHAT_ID", "")
261
+ missing = [name for name, value in (("TELEGRAM_BOT_TOKEN", token), ("TELEGRAM_CHAT_ID", chat_id))
262
+ if not value]
263
+ if missing:
264
+ return False, "Telegram alert cannot be sent: missing " + ", ".join(missing) + "."
265
+ try:
266
+ body = json.dumps({"chat_id": chat_id, "text": text[:4096]}).encode("utf-8")
267
+ request = Request("https://api.telegram.org/bot" + token + "/sendMessage", data=body,
268
+ headers={"Content-Type": "application/json"}, method="POST")
269
+ with build_opener(_NoRedirect()).open(request, timeout=HTTP_TIMEOUT) as response:
270
+ payload = json.loads(response.read(65_537))
271
+ if not isinstance(payload, dict):
272
+ return False, "Telegram returned an unreadable alert response."
273
+ result = payload.get("result")
274
+ if (payload.get("ok") is True and isinstance(result, dict)
275
+ and type(result.get("message_id")) is int):
276
+ return True, result["message_id"]
277
+ # Telegram descriptions can echo request data. Never return them.
278
+ code = payload.get("error_code")
279
+ suffix = f" (HTTP {code})" if type(code) is int and 100 <= code <= 599 else ""
280
+ return False, "Telegram rejected the alert" + suffix + "."
281
+ except HTTPError as exc:
282
+ code = exc.code
283
+ suffix = f" HTTP {code}" if type(code) is int and 100 <= code <= 599 else " an HTTP error"
284
+ return False, "Telegram returned" + suffix + "."
285
+ except TimeoutError:
286
+ return False, "Telegram alert timed out."
287
+ except URLError as exc:
288
+ cause = "timed out" if isinstance(exc.reason, TimeoutError) else "could not be reached"
289
+ return False, f"Telegram {cause}."
290
+ except (ValueError, OSError):
291
+ return False, "Telegram returned an unreadable alert response."
292
+ except Exception:
293
+ return False, "Telegram alert could not be sent."
294
+
295
+
296
+ def summarise(project, samples, *, share_threshold=None):
297
+ """Calculate usage and alarms; earlier periods remain in history, never in rates."""
298
+ if share_threshold is None:
299
+ share_threshold, _ = _alarm_threshold()
300
+ samples = _clean_samples(samples)
301
+ allowance = dict(LAUNCH_PLAN["allowance"])
302
+ result = {"available": project["available"], "cause": project.get("cause"),
303
+ "read_at": project["read_at"], "project": {k: project.get(k) for k in ("id", "name")},
304
+ "period": {k: project.get("consumption_period_" + k) for k in ("start", "end")},
305
+ "usage": dict.fromkeys(allowance), "allowance": allowance,
306
+ "share": dict.fromkeys(allowance), "rate": {},
307
+ "projection": {"transfer": None, "compute": None, "storage": None, "total": None,
308
+ "currency": "USD", "basis": {}, "cause": project.get("cause")},
309
+ "samples": samples, "alarm": None, "guards": copy.deepcopy(_guards)}
310
+ if not project["available"]:
311
+ for metric in allowance:
312
+ result["rate"][metric] = {"per_day": None, "cause": project.get("cause"),
313
+ "from_at": None, "to_at": None}
314
+ return result
315
+ usage = dict(zip(allowance, (project[_FIELDS[0]] / 1e9, project[_FIELDS[1]] / 3600,
316
+ project[_FIELDS[2]] / 2**30)))
317
+ result["usage"] = usage
318
+ result["share"] = {key: usage[key] / value for key, value in allowance.items()}
319
+ start, end, now = (_stamp(value) for value in (result["period"]["start"], result["period"]["end"],
320
+ result["read_at"]))
321
+ current = [row for row in samples if _stamp(row["period_start"]) == start
322
+ and start <= _stamp(row["at"]) <= min(now, end)]
323
+ rates = {"transfer_gb": _rate(current, "transfer_bytes", 1e9),
324
+ "compute_hours": _rate(current, "compute_seconds", 3600),
325
+ "storage_gib": _rate(current, "storage_bytes", 2**30, gauge=True)}
326
+ result["rate"] = rates
327
+ elapsed = (min(now, end) - start).total_seconds() / 86400
328
+ remaining = max(0, (end - now).total_seconds() / 86400)
329
+ period_days = (end - start).total_seconds() / 86400
330
+ if elapsed <= 0:
331
+ result["projection"]["cause"] = "The billing period has not started; an average is not available."
332
+ result["alarm"] = evaluate_alarm(result, share_threshold)
333
+ return result
334
+ projected, basis = {}, {}
335
+ for metric in ("transfer_gb", "compute_hours"):
336
+ rate = rates[metric]["per_day"]
337
+ basis[metric] = "Two changed samples" if rate is not None else "Period-to-date average"
338
+ projected[metric] = usage[metric] + remaining * (rate if rate is not None else usage[metric] / elapsed)
339
+ # Storage is a gauge, not a cumulative counter: dividing its size by elapsed
340
+ # days would invent growth. Integrate the observed sizes over the period,
341
+ # carrying the first size back to its start when no older observation exists.
342
+ cursor, size, size_days = start, (current[0]["storage_bytes"] / 2**30 if current else usage["storage_gib"]), 0.0
343
+ for row in current:
344
+ at = _stamp(row["at"])
345
+ size_days += size * (at - cursor).total_seconds() / 86400
346
+ cursor, size = at, row["storage_bytes"] / 2**30
347
+ size_days += size * (min(now, end) - cursor).total_seconds() / 86400
348
+ storage_average = size_days / elapsed
349
+ storage_rate = rates["storage_gib"]["per_day"]
350
+ projected["storage_gib"] = (max(0, usage["storage_gib"] + remaining * storage_rate)
351
+ if storage_rate is not None else storage_average)
352
+ future_average = ((usage["storage_gib"] + projected["storage_gib"]) / 2
353
+ if storage_rate is not None else storage_average)
354
+ billed_storage_gb = (size_days + remaining * future_average) / period_days * 2**30 / 1e9
355
+ basis["storage_gib"] = ("Two changed samples; estimated average storage" if storage_rate is not None
356
+ else "Period-to-date average storage size; held constant")
357
+ prices = LAUNCH_PLAN["usd"]
358
+ costs = {"transfer": max(0, projected["transfer_gb"] - allowance["transfer_gb"]) * prices["transfer_per_gb"],
359
+ "compute": projected["compute_hours"] * prices["compute_per_hour"],
360
+ "storage": billed_storage_gb * prices["storage_per_gb_month"]}
361
+ result["projection"] = {**costs, "total": sum(costs.values()), "currency": "USD",
362
+ "basis": basis, "usage": projected, "cause": None}
363
+ result["alarm"] = evaluate_alarm(result, share_threshold)
364
+ return result
365
+
366
+
367
+ def store_is_awake(project=None, database_url=None, backend=None):
368
+ """No SQL probe. Unknown host/state fails closed, including history loading."""
369
+ backend = os.environ.get("STORE_BACKEND", "hf") if backend is None else backend
370
+ if backend != "pg":
371
+ return True, None
372
+ try:
373
+ host = urlsplit(os.environ.get("DATABASE_URL", "") if database_url is None else database_url).hostname
374
+ except ValueError:
375
+ host = None
376
+ if not host:
377
+ return False, "Sample history is pending: the store host is not configured."
378
+ if not (host.endswith(".neon.tech") or host.endswith(".neon.build")):
379
+ return True, None
380
+ # Other management-plane samplers share this guard without reading Neon
381
+ # at all when their history store is hosted elsewhere.
382
+ if project is None:
383
+ project = read_project()
384
+ if not project["available"]:
385
+ return False, project["cause"]
386
+ try:
387
+ endpoints = _get("/projects/" + project["id"] + "/endpoints", os.environ.get("NEON_API_KEY", ""))["endpoints"]
388
+ for endpoint in endpoints:
389
+ endpoint_id = endpoint.get("id")
390
+ if _identifier(endpoint_id) and (host.startswith(endpoint_id + ".") or host.startswith(endpoint_id + "-pooler.")):
391
+ if endpoint.get("current_state") == "active":
392
+ return True, None
393
+ return False, "Sample history is pending while the store compute is idle."
394
+ return False, "Sample history is pending: the store endpoint was not found in this project."
395
+ except UsageReadError as exc:
396
+ return False, str(exc)
397
+ except Exception:
398
+ return False, "Sample history is pending: the store compute state could not be read."
399
+
400
+
401
+ _lock = threading.RLock()
402
+ _thread = None
403
+ _stop = threading.Event()
404
+ _samples = []
405
+ _loaded = False
406
+ _project_id = None
407
+ _alerted = {}
408
+ _status = {"every_s": 0, "last_at": None, "pending": 0, "persisted": None,
409
+ "cause": "Scheduled sampling is disabled; AIOS_NEON_SAMPLE_SECONDS is unset or zero."}
410
+
411
+
412
+ def snapshot():
413
+ with _lock:
414
+ return copy.deepcopy(_samples), dict(_status)
415
+
416
+
417
+ def _sampler_cause(cause=None):
418
+ _, config_cause = _alarm_threshold()
419
+ if config_cause and config_cause not in (cause or ""):
420
+ return " ".join(item for item in (cause, config_cause) if item)
421
+ return cause or None
422
+
423
+
424
+ def _period_key(value):
425
+ stamp = _stamp(value)
426
+ if stamp is None:
427
+ raise ValueError("Invalid alert period")
428
+ return stamp.astimezone(timezone.utc).isoformat(timespec="seconds")
429
+
430
+
431
+ def _clean_alerted(alerted, period_start, samples):
432
+ """At most six keys each for the current and previous monthly billing period."""
433
+ if not isinstance(alerted, dict):
434
+ raise ValueError("Unreadable alert markers")
435
+ current = _period_key(period_start)
436
+ clean = {}
437
+ for period, keys in alerted.items():
438
+ period = _period_key(period)
439
+ if not isinstance(keys, list) or any(not isinstance(key, str) or key not in _ALERT_KEYS for key in keys):
440
+ raise ValueError("Unreadable alert markers")
441
+ clean.setdefault(period, set()).update(keys)
442
+ periods = set(clean) | {_period_key(row["period_start"]) for row in samples}
443
+ start = _stamp(current)
444
+ month = start.year * 12 + start.month
445
+ previous = max((period for period in periods if period < current
446
+ and month - (_stamp(period).year * 12 + _stamp(period).month) <= 1), default=None)
447
+ return {period: sorted(clean.get(period, set())) for period in (previous, current) if period}
448
+
449
+
450
+ def report():
451
+ """Card path: management API plus memory only, never the store callbacks."""
452
+ project = read_project()
453
+ samples, status = snapshot()
454
+ status["cause"] = _sampler_cause(status["cause"])
455
+ return {**summarise(project, samples), "sampler": status}
456
+
457
+
458
+ def _sample_once(load, persist, *, sender=None):
459
+ global _loaded, _samples, _alerted, _project_id
460
+ project = read_project()
461
+ if not project["available"]:
462
+ with _lock:
463
+ _status["cause"] = _sampler_cause(project["cause"])
464
+ return
465
+ sample = {"at": project["read_at"], "transfer_bytes": project[_FIELDS[0]],
466
+ "compute_seconds": project[_FIELDS[1]], "storage_bytes": project[_FIELDS[2]],
467
+ "period_start": project["consumption_period_start"]}
468
+ with _lock:
469
+ if _project_id != project["id"]:
470
+ _project_id, _loaded, _samples, _alerted = project["id"], False, [], {}
471
+ _status.update(pending=0, persisted=None)
472
+ _samples = _clean_samples([*_samples, sample])
473
+ _status.update(last_at=sample["at"], pending=min(MAX_SAMPLES, _status["pending"] + 1))
474
+ awake, cause = store_is_awake(project)
475
+ if not awake:
476
+ with _lock:
477
+ _status["cause"] = _sampler_cause(cause)
478
+ return
479
+ try:
480
+ if not _loaded:
481
+ saved = load()
482
+ if saved is None:
483
+ saved = {}
484
+ with _lock:
485
+ # Never mix a saved ring from a different configured project.
486
+ matching = saved.get("project_id") == project["id"]
487
+ history = saved.get("samples", []) if matching else []
488
+ _samples = _clean_samples([*history, *_samples])
489
+ _alerted = _clean_alerted(saved.get("alerted", {}) if matching else {},
490
+ sample["period_start"], _samples)
491
+ _loaded = True
492
+ except Exception:
493
+ with _lock:
494
+ _status["cause"] = _sampler_cause("Sample history and alert markers could not be loaded; alerts deferred.")
495
+ return
496
+ sent = False
497
+ try:
498
+ # Loading/initializing the runtime may take time. Re-observe immediately
499
+ # before the write rather than relying on its earlier active state.
500
+ awake, cause = store_is_awake(project)
501
+ if not awake:
502
+ with _lock:
503
+ _status["cause"] = _sampler_cause(cause)
504
+ return
505
+ samples, _ = snapshot()
506
+ with _lock:
507
+ _alerted = _clean_alerted(_alerted, sample["period_start"], samples)
508
+ alerted = copy.deepcopy(_alerted)
509
+ # Confirm storage works before sending. This also flushes any markers
510
+ # left in memory by a successful send whose subsequent save failed.
511
+ persist({"project_id": project["id"], "samples": samples, "alerted": alerted})
512
+ with _lock:
513
+ _status.update(pending=0, persisted=_now(), cause=_sampler_cause())
514
+ summary = summarise(project, samples)
515
+ alarm = summary["alarm"]
516
+ period = _period_key(sample["period_start"])
517
+ crossings = [row for row in alarm["crossings"]
518
+ if f"{row['metric']}:{row['kind']}" not in alerted[period]] if alarm else []
519
+ if not crossings:
520
+ return
521
+ text = (_alarm_message(summary, crossings, alarm["test"])
522
+ + f"\nPeriod {period} to {_period_key(summary['period']['end'])} (end exclusive).")
523
+ try:
524
+ ok, delivery = (sender or send_telegram)(text)
525
+ except Exception:
526
+ ok, delivery = False, "Telegram alert could not be sent."
527
+ if not ok:
528
+ with _lock:
529
+ _status["cause"] = _sampler_cause(delivery)
530
+ return
531
+ sent = True
532
+ with _lock:
533
+ _alerted[period] = sorted(set(_alerted[period])
534
+ | {f"{row['metric']}:{row['kind']}" for row in crossings})
535
+ alerted = copy.deepcopy(_alerted)
536
+ # Keep successful deliveries in memory even if this save fails. The
537
+ # next tick flushes them before considering another message.
538
+ awake, cause = store_is_awake(project)
539
+ if not awake:
540
+ with _lock:
541
+ _status["cause"] = _sampler_cause("Telegram alert sent; alert markers await storage. " + (cause or ""))
542
+ return
543
+ persist({"project_id": project["id"], "samples": samples, "alerted": alerted})
544
+ with _lock:
545
+ _status.update(persisted=_now(), cause=_sampler_cause())
546
+ except Exception:
547
+ with _lock:
548
+ _status["cause"] = _sampler_cause(
549
+ "Telegram alert sent; alert markers could not be stored and remain pending in memory." if sent
550
+ else "Sample history could not be stored; samples and alert markers remain pending in memory.")
551
+
552
+
553
+ def start_sampler(*, load=None, persist=None, sender=None):
554
+ """One daemon per app process; unset/zero is OFF in imports, gates and tests.
555
+
556
+ Deploy explicitly installs DEFAULT_SAMPLE_SECONDS. Callbacks are mandatory
557
+ so this module never imports a database implementation or a tenant runtime.
558
+ """
559
+ global _thread
560
+ try:
561
+ every_s = int(os.environ.get("AIOS_NEON_SAMPLE_SECONDS", "0"))
562
+ except ValueError:
563
+ every_s = 0
564
+ if every_s <= 0 or load is None or persist is None:
565
+ return False
566
+ # Never turn a typo into a tight management-API polling loop.
567
+ every_s = max(60, every_s)
568
+ with _lock:
569
+ if _thread is not None and _thread.is_alive():
570
+ return True
571
+ _status.update(every_s=every_s, cause=_sampler_cause("Waiting for the first scheduled sample."))
572
+ _stop.clear()
573
+
574
+ def run():
575
+ while not _stop.is_set():
576
+ try:
577
+ _sample_once(load, persist, sender=sender)
578
+ except Exception:
579
+ with _lock:
580
+ _status["cause"] = _sampler_cause("The scheduled sample could not be completed; it will retry.")
581
+ if _stop.wait(every_s):
582
+ break
583
+
584
+ _thread = threading.Thread(target=run, name="neon-usage-sampler", daemon=True)
585
+ _thread.start()
586
+ return True
587
+
588
+
589
+ def stop_sampler():
590
+ _stop.set()
platform/core/perm_scope.py CHANGED
@@ -51,6 +51,9 @@ import core.perms as perms
51
  _FORMULA_REF = re.compile(r"\{([^{}]*)\}")
52
 
53
  PERMS_VERSION = 1
 
 
 
54
 
55
 
56
  def _rec(user):
@@ -118,6 +121,32 @@ def may_metrics(user, module):
118
  return not (isinstance(e, dict) and e.get('metrics') is False)
119
 
120
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
121
  def nav_may_open(user, key, st=None):
122
  """May this account SEE `key` β€” in the nav, and at the route gate? The ADMISSION question.
123
 
@@ -136,6 +165,8 @@ def nav_may_open(user, key, st=None):
136
  1. an admin sees everything (break-glass, as everywhere else);
137
  2. an EXPLICIT `access: false` DENIES, on any key β€” an administrator unticking a box must
138
  take the row off the nav, and before this it did not;
 
 
139
  3. a `ut_*` key with no explicit deny defers to `user_tables.may_open` β€” creator, admin or
140
  a `core.shares` grant β€” and **an `access: true` entry may NEVER widen past it**. The
141
  editor must not become a way to hand somebody another user's private table;
@@ -149,11 +180,12 @@ def nav_may_open(user, key, st=None):
149
  registry topic and fatal for exactly one key family. Every door that can be handed a
150
  `ut_*` key now threads its session's own runtime: `deps.Session.require`,
151
  `routes_charts._require_database`, `routes_nav.nav` and `routes_tables` (both lending
152
- the rows-free projection) and `routes_admin.get_perms`. The two callers that still omit
153
- `st` are `perms.nav_pages`, which iterates `core.registry.REGISTRY` (a compiled literal
154
  carrying no `ut_` key β€” `verify_perm_scope.py` section K pins that, so the fence can go
155
- red rather than merely being asserted here), and `routes_admin`'s surface DEFAULT, which
156
- its own `m.get("surface")` test fences to registry rows. Neither can reach this leg;
 
157
 
158
  4. a REGISTRY TOPIC with an explicit entry takes that entry, grant included. Here the
159
  editor IS the authority: `_clean_perms` writes an entry for every governed topic on
@@ -169,8 +201,11 @@ def nav_may_open(user, key, st=None):
169
  if perms.is_admin(user):
170
  return True
171
  e = entry(user, key)
172
- if e is not None and not bool(e.get('access', True)):
 
173
  return False
 
 
174
  if str(key or '').startswith(_ut().KEY_PREFIX):
175
  return bool(_ut().may_open(key, (user or {}).get('username'), False, st=st))
176
  if e is not None:
@@ -186,12 +221,39 @@ def assistant_entry(user, module):
186
  that reader must be able to name the migrated grant that admitted a database. In particular,
187
  a store-outage admin identity with no ``perms`` document is not an unresolved permission that
188
  may be widened into data access.
 
 
 
 
 
189
  """
190
  e = entry(user, module)
191
- if (not is_migrated(user) or not isinstance(e, dict)
192
- or not bool(e.get('access', True)) or not may_access(user, module)):
193
- return None
194
- return e
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
195
 
196
 
197
  # ── FIELDS ───────────────────────────────────────────────────────────────────────────────────
@@ -1672,7 +1734,9 @@ def _read_user_table(table_key, st, want_rows=True):
1672
  'reading a read-through database, or read it through the API'))
1673
  keys = {f['key'] for f in fields if f.get('key')}
1674
  return fields, list(_MIRROR_READER(table_key, keys, st) or ())
1675
- whole = ut.get(table_key, st=st)
 
 
1676
  if whole is None:
1677
  # Deleted between the wall and here. The same refusal, not an empty table.
1678
  raise UnknownTable(f"no database named '{table_key}'")
 
51
  _FORMULA_REF = re.compile(r"\{([^{}]*)\}")
52
 
53
  PERMS_VERSION = 1
54
+ AI_APPS = "ai_apps"
55
+ AI_APPS_OFF = ("AI apps (MCP) is switched off for your account. "
56
+ "An administrator can switch it on in Settings > Manage users.")
57
 
58
 
59
  def _rec(user):
 
121
  return not (isinstance(e, dict) and e.get('metrics') is False)
122
 
123
 
124
+ def may_ai_apps(user):
125
+ """R3/C6: AI apps are a separate capability, granted unless explicitly switched off.
126
+
127
+ Like may_metrics, this deliberately differs from may_access: no existing migrated or
128
+ legacy account could declare this new key. Absence must not revoke everybody's gateway
129
+ access. Only access: False denies; the in-app Assistant still uses analyst.
130
+ """
131
+ if perms.is_admin(user):
132
+ return True
133
+ e = entry(user, AI_APPS)
134
+ return not (e is not None and e.get('access') is False)
135
+
136
+
137
+ def ai_apps_refusal(user, runtime):
138
+ """Return the account or tenant refusal, or None. Shared by OAuth and every tool call.
139
+
140
+ Do not route this through Session.require: its literal tenant-list check cannot recognise
141
+ the existing analyst tenant entitlement that tenant_has_module preserves for this gateway.
142
+ """
143
+ if not may_ai_apps(user):
144
+ return AI_APPS_OFF
145
+ if not perms.tenant_has_module(runtime, AI_APPS):
146
+ return "AI apps (MCP) is not available in this workspace."
147
+ return None
148
+
149
+
150
  def nav_may_open(user, key, st=None):
151
  """May this account SEE `key` β€” in the nav, and at the route gate? The ADMISSION question.
152
 
 
165
  1. an admin sees everything (break-glass, as everywhere else);
166
  2. an EXPLICIT `access: false` DENIES, on any key β€” an administrator unticking a box must
167
  take the row off the nav, and before this it did not;
168
+ R3/C6's ai_apps capability then grants by absence, including migrated and legacy-list
169
+ accounts. It is an editor surface, not a registry page or a rail entry;
170
  3. a `ut_*` key with no explicit deny defers to `user_tables.may_open` β€” creator, admin or
171
  a `core.shares` grant β€” and **an `access: true` entry may NEVER widen past it**. The
172
  editor must not become a way to hand somebody another user's private table;
 
180
  registry topic and fatal for exactly one key family. Every door that can be handed a
181
  `ut_*` key now threads its session's own runtime: `deps.Session.require`,
182
  `routes_charts._require_database`, `routes_nav.nav` and `routes_tables` (both lending
183
+ the rows-free projection) and `routes_admin.get_perms`. The caller that still omits
184
+ `st` is `perms.nav_pages`, which iterates `core.registry.REGISTRY` (a compiled literal
185
  carrying no `ut_` key β€” `verify_perm_scope.py` section K pins that, so the fence can go
186
+ red rather than merely being asserted here). It cannot reach this leg. The editor's
187
+ surface default now also threads its runtime; its catalogue includes R3/C6's gateway
188
+ capability as well as registry pages;
189
 
190
  4. a REGISTRY TOPIC with an explicit entry takes that entry, grant included. Here the
191
  editor IS the authority: `_clean_perms` writes an entry for every governed topic on
 
201
  if perms.is_admin(user):
202
  return True
203
  e = entry(user, key)
204
+ if e is not None and (e.get('access') is False if key == AI_APPS
205
+ else not bool(e.get('access', True))):
206
  return False
207
+ if key == AI_APPS:
208
+ return True
209
  if str(key or '').startswith(_ut().KEY_PREFIX):
210
  return bool(_ut().may_open(key, (user or {}).get('username'), False, st=st))
211
  if e is not None:
 
221
  that reader must be able to name the migrated grant that admitted a database. In particular,
222
  a store-outage admin identity with no ``perms`` document is not an unresolved permission that
223
  may be widened into data access.
224
+
225
+ ⭐ S67-T34 / D-276: ONE NAMED EXCEPTION, the grant the NAVIGATION uses. For the two registry
226
+ topics in ``ASSISTANT_NAV_TOPICS`` an administrator account is admitted unfiltered (unless
227
+ explicitly unticked), because ``nav_may_open`` leg 1 opens them to it; the AI list must equal
228
+ the nav (R33). See the comment at that leg for why the outage identity no longer reaches it.
229
  """
230
  e = entry(user, module)
231
+ if (is_migrated(user) and isinstance(e, dict)
232
+ and bool(e.get('access', True)) and may_access(user, module)):
233
+ return e
234
+ # ⭐⭐ S67-T34 / D-276 / R26: THE ADMINISTRATOR THE NAVIGATION ADMITS. `nav_may_open` leg 1
235
+ # opens every registry topic to an administrator, and `_clean_perms` writes no per-topic
236
+ # entry for one, so an administrator saw `customer_data` and `product_data` (Odoo products,
237
+ # R26's Inventory source) in the nav and had them refused here. MEASURED on staging
238
+ # 2026-09-28: aaron (admin, no `perms_v`) had both in the nav and neither in the AI list. The
239
+ # answer is the grant the nav already uses, admitted for exactly these topics: an account
240
+ # whose ROLE is admin, minus an explicit `access: false` (narrower than the nav, never wider).
241
+ # Unfiltered, as the nav's break-glass is.
242
+ # ⚠ THE STORE-OUTAGE IDENTITY the docstring names is no longer built by `deps._user_for`
243
+ # (it admits a stored record or the QA runner, nothing else), so the role is read off a
244
+ # stored account here. If a synthetic admin identity is ever reintroduced, it must be kept
245
+ # out of this leg.
246
+ # β›” NOT A MEMBER. A non-administrator still needs the explicit migrated grant above, so a
247
+ # user without it does not see the database (the NC).
248
+ if (module in ASSISTANT_NAV_TOPICS and perms.is_admin(user)
249
+ and not (isinstance(e, dict) and e.get('access') is False)):
250
+ return dict(e) if isinstance(e, dict) else {'access': True, 'filter': None}
251
+ return None
252
+
253
+
254
+ #: S67-T34: the registry topics the navigation opens to an administrator by break-glass, which
255
+ #: `assistant_entry` therefore admits for a migrated administrator without a per-topic entry.
256
+ ASSISTANT_NAV_TOPICS = frozenset({'customer_data', 'product_data'})
257
 
258
 
259
  # ── FIELDS ───────────────────────────────────────────────────────────────────────────────────
 
1734
  'reading a read-through database, or read it through the API'))
1735
  keys = {f['key'] for f in fields if f.get('key')}
1736
  return fields, list(_MIRROR_READER(table_key, keys, st) or ())
1737
+ # HOTFIX 2026-09-28: this table's entry alone where the store can (`get_one`), not a deep
1738
+ # copy of every table in the tenant. Amazon link cells read listings here on every page.
1739
+ whole = ut.get_one(table_key, st=st)
1740
  if whole is None:
1741
  # Deleted between the wall and here. The same refusal, not an empty table.
1742
  raise UnknownTable(f"no database named '{table_key}'")
platform/core/perms.py CHANGED
@@ -253,6 +253,10 @@ def tenant_has_module(runtime, key):
253
  allowed = tenant_modules(runtime)
254
  if allowed is None or key in allowed:
255
  return True
 
 
 
 
256
  parent = registry.BY_KEY.get(key, {}).get('parent')
257
  return parent in allowed if parent else False
258
 
@@ -336,17 +340,22 @@ def governable_surface_modules():
336
  filter or hiddenFields against an empty vocabulary, so a client cannot store a rule nothing
337
  would apply.
338
 
339
- ⭐ DERIVED, NEVER LISTED: every non-archived, non-`group_only` registry row with NO
340
  `topic` that `nav_pages` would offer (not `validate_only`, not `api_surface: False`, not
341
  `settings`). Today that is `analyst` (Assistant) and `automation` (Agents); a surface added
342
  to the registry tomorrow is governable with no edit here β€” the same rule
343
  `tenant_governable_modules` states for databases, applied to the other kind of row.
 
 
 
 
 
344
  """
345
  return [{'key': m['key'], 'label': m['label']}
346
  for m in registry.REGISTRY
347
  if not m.get('archived') and not m.get('group_only') and not m.get('topic')
348
  and not m.get('validate_only') and m.get('api_surface') is not False
349
- and m['key'] != 'settings']
350
 
351
 
352
  #: β›” `enforced_module_keys` IS DELETED (W36-T22 / C2), NOT NEUTERED TO "every governable key".
 
253
  allowed = tenant_modules(runtime)
254
  if allowed is None or key in allowed:
255
  return True
256
+ # R3/C6: tenants provisioned with Assistant already had the external gateway. Its new
257
+ # account-level grant must not remove that entitlement from existing tenant catalogues.
258
+ if key == 'ai_apps':
259
+ return 'analyst' in allowed
260
  parent = registry.BY_KEY.get(key, {}).get('parent')
261
  return parent in allowed if parent else False
262
 
 
340
  filter or hiddenFields against an empty vocabulary, so a client cannot store a rule nothing
341
  would apply.
342
 
343
+ Page surfaces are DERIVED: every non-archived, non-`group_only` registry row with NO
344
  `topic` that `nav_pages` would offer (not `validate_only`, not `api_surface: False`, not
345
  `settings`). Today that is `analyst` (Assistant) and `automation` (Agents); a surface added
346
  to the registry tomorrow is governable with no edit here β€” the same rule
347
  `tenant_governable_modules` states for databases, applied to the other kind of row.
348
+
349
+ R3/C6 adds one listed capability: ai_apps, labelled AI apps (MCP). It governs the external
350
+ gateway, not a page, so it belongs in the account editor without entering REGISTRY or the
351
+ navigation rail. registry.py is outside this ticket's fence. Both consumers here are editor
352
+ helpers (the catalogue and empty field schema); neither turns this row into navigation.
353
  """
354
  return [{'key': m['key'], 'label': m['label']}
355
  for m in registry.REGISTRY
356
  if not m.get('archived') and not m.get('group_only') and not m.get('topic')
357
  and not m.get('validate_only') and m.get('api_surface') is not False
358
+ and m['key'] != 'settings'] + [{'key': 'ai_apps', 'label': 'AI apps (MCP)'}]
359
 
360
 
361
  #: β›” `enforced_module_keys` IS DELETED (W36-T22 / C2), NOT NEUTERED TO "every governable key".
platform/core/registry.py CHANGED
@@ -203,7 +203,12 @@ REGISTRY = [
203
  'proof). No nav, no page. Wave 17 R8 ("I don\'t even know what it does. Delete '
204
  'them") made that literal: `validate_only` takes it out of the account menu too, '
205
  'which is the last place it was still visible. The proof survives; the door does not.'},
 
 
 
 
206
  {'key': 'automation', 'label': 'Agents', 'brand': False, 'hq': True, 'validate': False,
 
207
  'nav': True,
208
  'note': 'Wave 18 (C-AUTONAV): scheduled jobs that create and refresh user databases '
209
  '(website scrape-to-DB, the Instagram field). React-only surface (no PAGE_FUNCS '
@@ -280,6 +285,11 @@ def brand_filterable(key) -> bool:
280
  return bool(BY_KEY.get(key, {}).get('brand'))
281
 
282
 
 
 
 
 
 
283
  # ═════════════════════════════════════════════════════════════════════════════════════════════
284
  # ⭐⭐ W33-T41 / item 12a β€” ONE SUBJECT, ONE DATABASE
285
  # ═════════════════════════════════════════════════════════════════════════════════════════════
 
203
  'proof). No nav, no page. Wave 17 R8 ("I don\'t even know what it does. Delete '
204
  'them") made that literal: `validate_only` takes it out of the account menu too, '
205
  'which is the last place it was still visible. The proof survives; the door does not.'},
206
+ # S67-T36 (owner instr 24, R25, 2026-09-28): Agents is TABLED. `archived` hides the UI only:
207
+ # no rail row, no Settings module toggle, no Home section, no agent templates. The
208
+ # `/automations` APIs, stored automations and their schedules are untouched and keep running.
209
+ # ⭐ RESTORING IS THIS ONE LINE: delete `'archived': True` below and every door comes back.
210
  {'key': 'automation', 'label': 'Agents', 'brand': False, 'hq': True, 'validate': False,
211
+ 'archived': True,
212
  'nav': True,
213
  'note': 'Wave 18 (C-AUTONAV): scheduled jobs that create and refresh user databases '
214
  '(website scrape-to-DB, the Instagram field). React-only surface (no PAGE_FUNCS '
 
285
  return bool(BY_KEY.get(key, {}).get('brand'))
286
 
287
 
288
+ def is_archived(key) -> bool:
289
+ """True when the registry row `key` is archived (tabled for everyone). Unknown keys: False."""
290
+ return bool(BY_KEY.get(key, {}).get('archived'))
291
+
292
+
293
  # ═════════════════════════════════════════════════════════════════════════════════════════════
294
  # ⭐⭐ W33-T41 / item 12a β€” ONE SUBJECT, ONE DATABASE
295
  # ═════════════════════════════════════════════════════════════════════════════════════════════
platform/core/sheet_binding.py CHANGED
@@ -163,11 +163,13 @@ class BindingMetadataRefusal(Exception):
163
 
164
 
165
  def clean_related(raw):
166
- """One declared edge key and a structurally valid, losslessly copied filter tree.
167
 
168
  Absence is handled by the caller. An explicitly supplied null is malformed, as are
169
  extra relationship bags and nested relationships. Operator support belongs to the
170
  query compiler, which reports unpushable conditions rather than dropping metadata.
 
 
171
  """
172
  def refuse():
173
  raise BindingMetadataRefusal("related_invalid")
@@ -214,6 +216,63 @@ def clean_related(raw):
214
  return {"via": via.strip(), "filter": deepcopy(tree)}
215
 
216
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
217
  def clean_period(raw):
218
  """Return one canonical Sheet period, or ``None`` when no window was named."""
219
  if not isinstance(raw, dict):
 
163
 
164
 
165
  def clean_related(raw):
166
+ """Read the legacy edge bag for its one-release migration to ordinary filter leaves.
167
 
168
  Absence is handled by the caller. An explicitly supplied null is malformed, as are
169
  extra relationship bags and nested relationships. Operator support belongs to the
170
  query compiler, which reports unpushable conditions rather than dropping metadata.
171
+ The binding cleaner qualifies this copied tree and intersects it with the local scope;
172
+ neither this reader nor that rewrite mutates the saved document.
173
  """
174
  def refuse():
175
  raise BindingMetadataRefusal("related_invalid")
 
216
  return {"via": via.strip(), "filter": deepcopy(tree)}
217
 
218
 
219
+ def rewrite_legacy_related(binding, *, linked=False):
220
+ """Copy a binding, moving its legacy bag into the ordinary filter tree.
221
+
222
+ This is a lossless metadata rewrite, not a binding cleaner or a relationship lookup.
223
+ The scoped resolver still proves every named edge. Migration may also replace the
224
+ linked spelling with its explicit bridge identity; normal reads leave those aliases
225
+ to the API's permission-scoped declaration lookup.
226
+ """
227
+ out = deepcopy(binding)
228
+ if "related" in out:
229
+ legacy = clean_related(out.pop("related"))
230
+
231
+ def parts(tree):
232
+ if isinstance(tree, dict):
233
+ nodes = tree.get("nodes")
234
+ return (nodes if isinstance(nodes, list) else []), (
235
+ "or" if tree.get("conj") == "or" else "and")
236
+ return (tree if isinstance(tree, list) else []), "and"
237
+
238
+ def qualify(node):
239
+ if isinstance(node.get("children"), list):
240
+ return {**node, "children": [qualify(child) for child in node["children"]]}
241
+ leaf = {**node, "colId": f"related:{legacy['via']}:{node['colId']}"}
242
+ if isinstance(node.get("rhs"), dict) and node["rhs"].get("colId"):
243
+ leaf["rhs"] = {**node["rhs"],
244
+ "colId": f"related:{legacy['via']}:{node['rhs']['colId']}"}
245
+ return leaf
246
+
247
+ scope = out.setdefault("scope", {})
248
+ if not isinstance(scope, dict):
249
+ raise BindingMetadataRefusal("related_invalid")
250
+ nodes, conj = parts(legacy["filter"])
251
+ original, original_conj = parts(scope.get("filter"))
252
+ original = ([{"conj": original_conj, "children": original}]
253
+ if original and original_conj == "or" else list(original))
254
+ scope["filter"] = {"conj": "and", "nodes": [*original,
255
+ {"conj": conj, "children": [qualify(node) for node in nodes]}]}
256
+
257
+ def rewrite_links(value):
258
+ if isinstance(value, dict):
259
+ key = value.get("colId")
260
+ if isinstance(key, str) and key.startswith("linked:"):
261
+ parts = key.split(":")
262
+ if len(parts) != 3 or not all(parts[1:]):
263
+ raise BindingMetadataRefusal("related_invalid", key)
264
+ value["colId"] = "related:" + key[len("linked:"):]
265
+ for child in value.values():
266
+ rewrite_links(child)
267
+ elif isinstance(value, list):
268
+ for child in value:
269
+ rewrite_links(child)
270
+
271
+ if linked:
272
+ rewrite_links(out)
273
+ return out
274
+
275
+
276
  def clean_period(raw):
277
  """Return one canonical Sheet period, or ``None`` when no window was named."""
278
  if not isinstance(raw, dict):
platform/core/sheet_import.py CHANGED
@@ -43,9 +43,11 @@ from __future__ import annotations
43
  import re
44
  import zipfile
45
  import hashlib
 
46
  import json
47
  import math
48
  import os
 
49
  import tempfile
50
  import xml.etree.ElementTree as ET
51
  from datetime import date, datetime, time, timezone
@@ -392,12 +394,11 @@ def style_of(cell, notes):
392
  if border is not None:
393
  sides = {}
394
  for side in sheets._BORDER_SIDES:
395
- edge = getattr(border, side, None)
396
- if edge is not None and getattr(edge, "style", None):
397
- sides[side] = True
398
  if sides:
399
  out["border"] = sides
400
- notes.add("a border keeps its side but not its weight, style or colour")
401
 
402
  fmt, note = classify_format(getattr(cell, "number_format", ""))
403
  if fmt:
@@ -407,6 +408,47 @@ def style_of(cell, notes):
407
  return out
408
 
409
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
410
  # ── values ───────────────────────────────────────────────────────────────────────────────────
411
 
412
  def json_value(value):
@@ -1037,6 +1079,134 @@ def guard_import_archive(path):
1037
  "That file could not be read as a workbook. Save it again as .xlsx and try once more.") from exc
1038
 
1039
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1040
  def _print_of(ws):
1041
  """Store print setup and inclusive print areas; export executes these settings later."""
1042
  setup = {key: getattr(ws.page_setup, key) for key in ws.page_setup.__attrs__
@@ -1148,6 +1318,7 @@ def import_workbook(path, *, source="", progress=None):
1148
 
1149
  counts = {"cells": 0, "formulas": 0}
1150
  book_notes = set()
 
1151
  authored = _authored_chart_document(path, book_notes)
1152
  chart_tabs = {tab["name"]: tab for tab in authored["sheets"]} if authored else {}
1153
  if authored and len(chart_tabs) != len(authored["sheets"]):
@@ -1209,6 +1380,8 @@ def import_workbook(path, *, source="", progress=None):
1209
  cells = _cells_of(ws, cached, notes, counts, excel)
1210
  _say("formats", 0.7 + 0.25 * (index / total))
1211
  _validations_of(ws, cells, notes)
 
 
1212
  cols, rows, widths, freeze, hidden_rows, hidden_cols, heights = _geometry_of(ws, notes)
1213
  tab = {"id": "", "name": name[:sheets._MAX_NAME], "cols": cols, "rows": rows,
1214
  "cells": cells, "usedExtent": sheets._used_extent(cells, excel["blankStyles"]),
 
43
  import re
44
  import zipfile
45
  import hashlib
46
+ import io
47
  import json
48
  import math
49
  import os
50
+ import posixpath
51
  import tempfile
52
  import xml.etree.ElementTree as ET
53
  from datetime import date, datetime, time, timezone
 
394
  if border is not None:
395
  sides = {}
396
  for side in sheets._BORDER_SIDES:
397
+ line = border_line(getattr(border, side, None), notes)
398
+ if line is not None:
399
+ sides[side] = line
400
  if sides:
401
  out["border"] = sides
 
402
 
403
  fmt, note = classify_format(getattr(cell, "number_format", ""))
404
  if fmt:
 
408
  return out
409
 
410
 
411
+ #: S67-T14: the OOXML line styles that are not one of C2's seven, each drawn as its nearest.
412
+ #: The dash and dot variants keep their PATTERN over their weight, because a patterned line drawn
413
+ #: solid reads as a different border while a medium dash drawn at 1px still reads as dashed.
414
+ _BORDER_NEAREST = {
415
+ "mediumDashed": "dashed", "dashDot": "dashed", "dashDotDot": "dashed",
416
+ "mediumDashDot": "dashed", "mediumDashDotDot": "dashed", "slantDashDot": "dashed",
417
+ }
418
+
419
+
420
+ def border_line(edge, notes):
421
+ """One openpyxl border side as C2's `{style, color}`, or None when the side draws nothing.
422
+
423
+ The seven Excel names C2 shares pass through unchanged; the dash dot variants become their
424
+ nearest with a note naming the approximation. The colour is the explicit RGB when there is one;
425
+ an automatic colour is black, which is what Excel draws; a theme or indexed colour is drawn in
426
+ black too, with a note, because a colour nobody chose is never guessed (see `argb_of`).
427
+ """
428
+ style = str(getattr(edge, "style", None) or "") if edge is not None else ""
429
+ if not style or style == "none":
430
+ return None
431
+ if style not in sheets.CELL_BORDER_STYLES:
432
+ nearest = _BORDER_NEAREST.get(style, "thin")
433
+ notes.add("a " + style + " border is drawn as " + nearest)
434
+ style = nearest
435
+ colour = getattr(edge, "color", None)
436
+ argb = argb_of(colour)
437
+ if argb:
438
+ hex_colour = "#" + argb.lower()
439
+ else:
440
+ hex_colour = "#000000"
441
+ kind = getattr(colour, "type", "") if colour is not None else ""
442
+ # ⚠ `type`, never `.auto`: openpyxl answers an unset descriptor with a (truthy) error
443
+ # STRING, so `colour.auto` reads as set on every theme colour.
444
+ automatic = (colour is None or kind == "auto"
445
+ or (kind == "indexed" and getattr(colour, "indexed", None) == 64)
446
+ or (kind == "rgb" and str(getattr(colour, "rgb", "")).upper() == "00000000"))
447
+ if not automatic:
448
+ notes.add("a theme or indexed border colour was drawn in black")
449
+ return {"style": style, "color": hex_colour}
450
+
451
+
452
  # ── values ───────────────────────────────────────────────────────────────────────────────────
453
 
454
  def json_value(value):
 
1079
  "That file could not be read as a workbook. Save it again as .xlsx and try once more.") from exc
1080
 
1081
 
1082
+ # ── sparklines (S67-T18) ─────────────────────────────────────────────────────────────────────
1083
+
1084
+ #: ⭐ S67-T18: Excel keeps sparkline groups in the worksheet's x14 extension, which openpyxl drops,
1085
+ #: so they are read from the worksheet PART itself. A group with no `type` draws lines, `column`
1086
+ #: draws columns and `stacked` draws win/loss bars (Excel's name for `sheets.SPARKLINE_KINDS`'s
1087
+ #: `winloss`). The part was already admitted by `guard_import_archive`, so this second read adds
1088
+ #: no new exposure.
1089
+ _X14 = "{http://schemas.microsoft.com/office/spreadsheetml/2009/9/main}"
1090
+ _XM = "{http://schemas.microsoft.com/office/excel/2006/main}"
1091
+ _MAIN_NS = "{http://schemas.openxmlformats.org/spreadsheetml/2006/main}"
1092
+ _DOC_REL_NS = "{http://schemas.openxmlformats.org/officeDocument/2006/relationships}"
1093
+ _PKG_REL_NS = "{http://schemas.openxmlformats.org/package/2006/relationships}"
1094
+ _SPARKLINE_KIND_OF = {"line": "line", "column": "column", "stacked": "winloss"}
1095
+
1096
+
1097
+ def _worksheet_parts(archive):
1098
+ """`{tab name: worksheet part}` through `xl/workbook.xml` and its relationships, never by index."""
1099
+ names = set(archive.namelist())
1100
+ if "xl/workbook.xml" not in names or "xl/_rels/workbook.xml.rels" not in names:
1101
+ return {}
1102
+ targets = {}
1103
+ for rel in ET.fromstring(archive.read("xl/_rels/workbook.xml.rels")).iter(_PKG_REL_NS + "Relationship"):
1104
+ target = str(rel.get("Target") or "")
1105
+ targets[rel.get("Id")] = (target.lstrip("/") if target.startswith("/")
1106
+ else posixpath.normpath("xl/" + target))
1107
+ out = {}
1108
+ for sheet in ET.fromstring(archive.read("xl/workbook.xml")).iter(_MAIN_NS + "sheet"):
1109
+ part = targets.get(sheet.get(_DOC_REL_NS + "id"))
1110
+ if part in names:
1111
+ out[str(sheet.get("name") or "")] = part
1112
+ return out
1113
+
1114
+
1115
+ def _sparkline_groups(stream):
1116
+ """`[(type, [(formula, sqref), ...]), ...]` for one worksheet part's x14 sparkline groups."""
1117
+ groups = []
1118
+ inside = 0 # a group's subtree is kept until the group itself ends
1119
+ for event, element in ET.iterparse(stream, events=("start", "end")):
1120
+ if element.tag == _X14 + "sparklineGroup":
1121
+ if event == "start":
1122
+ inside += 1
1123
+ continue
1124
+ inside -= 1
1125
+ entries = [((spark.findtext(_XM + "f") or "").strip(),
1126
+ (spark.findtext(_XM + "sqref") or "").strip())
1127
+ for spark in element.iter(_X14 + "sparkline")]
1128
+ groups.append((str(element.get("type") or "line"), entries))
1129
+ element.clear()
1130
+ elif event == "end" and not inside:
1131
+ element.clear()
1132
+ return groups
1133
+
1134
+
1135
+ def _sparklines_of(path, notes):
1136
+ """`{tab name: groups}` for every worksheet that carries x14 sparklines; `{}` for the rest.
1137
+
1138
+ A part whose bytes never mention a sparkline group is not parsed at all, so a workbook with
1139
+ no sparklines imports exactly as it did before S67-T18.
1140
+ """
1141
+ out = {}
1142
+ try:
1143
+ with zipfile.ZipFile(path) as archive:
1144
+ for name, part in _worksheet_parts(archive).items():
1145
+ raw = archive.read(part)
1146
+ if b"sparklineGroup" not in raw:
1147
+ continue
1148
+ groups = _sparkline_groups(io.BytesIO(raw))
1149
+ if groups:
1150
+ out[name] = groups
1151
+ except (KeyError, OSError, ET.ParseError, zipfile.BadZipFile):
1152
+ notes.add("the workbook's sparklines could not be read, so none were imported")
1153
+ return {}
1154
+ return out
1155
+
1156
+
1157
+ def _count_phrase(count, one, many):
1158
+ return (one if count == 1 else str(count) + " " + many)
1159
+
1160
+
1161
+ def _apply_sparklines(tab_name, cells, groups, blank_styles, notes):
1162
+ """Write `cell.sparkline {kind, range}` on each location cell of this tab's groups.
1163
+
1164
+ `sheets._clean_sparkline` stores a rectangle LOCAL to its tab, so a sparkline drawn from
1165
+ another tab's data cannot be carried: it is left out and NAMED, never dropped in silence
1166
+ (the module note). A location that is not one cell, or data that is not a range, is named too.
1167
+ A blank location becomes an `empty` cell holding only the sparkline, which is exactly what
1168
+ `clean_cell`'s empty branch keeps; its compact blank-style reference moves onto it.
1169
+ """
1170
+ other_tab = unreadable = 0
1171
+ for kind_name, entries in groups:
1172
+ kind = _SPARKLINE_KIND_OF.get(kind_name)
1173
+ if kind is None:
1174
+ unreadable += len(entries)
1175
+ continue
1176
+ for formula, sqref in entries:
1177
+ source = sheet_of_ref(formula)
1178
+ if source and source.casefold() != tab_name.casefold():
1179
+ other_tab += 1
1180
+ continue
1181
+ rect = rect_of(formula)
1182
+ places = sqref.replace("$", "").upper().split()
1183
+ if (rect is None or len(places) != 1 or ":" in places[0]
1184
+ or not _address_in_document(places[0])):
1185
+ unreadable += 1
1186
+ continue
1187
+ addr = places[0]
1188
+ spark = {"kind": kind, "range": rect}
1189
+ cell = cells.get(addr)
1190
+ if cell is None:
1191
+ if len(cells) >= sheets._MAX_CELLS_PER_SHEET:
1192
+ unreadable += 1
1193
+ continue
1194
+ cell = {"kind": "empty", "sparkline": spark}
1195
+ if addr in blank_styles:
1196
+ cell["excel"] = {"styleId": blank_styles.pop(addr)}
1197
+ cells[addr] = cell
1198
+ else:
1199
+ cell["sparkline"] = spark
1200
+ if other_tab:
1201
+ notes.add(_count_phrase(other_tab, "a sparkline", "sparklines")
1202
+ + " drawn from another tab's data " + ("was" if other_tab == 1 else "were")
1203
+ + " left out")
1204
+ if unreadable:
1205
+ notes.add(_count_phrase(unreadable, "a sparkline", "sparklines")
1206
+ + " with a location or data range that could not be read "
1207
+ + ("was" if unreadable == 1 else "were") + " left out")
1208
+
1209
+
1210
  def _print_of(ws):
1211
  """Store print setup and inclusive print areas; export executes these settings later."""
1212
  setup = {key: getattr(ws.page_setup, key) for key in ws.page_setup.__attrs__
 
1318
 
1319
  counts = {"cells": 0, "formulas": 0}
1320
  book_notes = set()
1321
+ sparklines = _sparklines_of(path, book_notes) # S67-T18
1322
  authored = _authored_chart_document(path, book_notes)
1323
  chart_tabs = {tab["name"]: tab for tab in authored["sheets"]} if authored else {}
1324
  if authored and len(chart_tabs) != len(authored["sheets"]):
 
1380
  cells = _cells_of(ws, cached, notes, counts, excel)
1381
  _say("formats", 0.7 + 0.25 * (index / total))
1382
  _validations_of(ws, cells, notes)
1383
+ if name in sparklines: # S67-T18, before usedExtent
1384
+ _apply_sparklines(name, cells, sparklines[name], excel["blankStyles"], notes)
1385
  cols, rows, widths, freeze, hidden_rows, hidden_cols, heights = _geometry_of(ws, notes)
1386
  tab = {"id": "", "name": name[:sheets._MAX_NAME], "cols": cols, "rows": rows,
1387
  "cells": cells, "usedExtent": sheets._used_extent(cells, excel["blankStyles"]),
platform/core/sheet_structure.py CHANGED
@@ -11,7 +11,8 @@ import math
11
  import re
12
  from collections.abc import Callable
13
 
14
- from core.sheets import SheetRefusal
 
15
 
16
 
17
  REF_SOURCE = r"(?:(?:'([^']{1,64})'|([A-Za-z_][A-Za-z0-9_]{0,63}))!)?(\$?)([A-Za-z]{1,3})(\$?)([1-9][0-9]{0,6})"
@@ -162,8 +163,7 @@ def shift_for_structure(text: str, shift: dict, own_sheet: str, *, bare: bool =
162
 
163
  return format_ref(at_index(first, move(start))) + separator + format_ref(at_index(last, move(end)))
164
 
165
- # Insertions keep the existing per-reference scan; range handling is deletion-only.
166
- return rewrite_refs(text, rewrite, rewrite_range if shift["delta"] < 0 else None)
167
 
168
 
169
  def _moved_index(value: int, at: int, delta: int) -> int | None:
@@ -185,6 +185,11 @@ def _reanchor_binding(value, shift: dict, own_sheet: str) -> bool:
185
  if key in BINDING_REF_KEYS and isinstance(child, str) and parse_ref(child):
186
  value[key] = shift_for_structure(child, shift, own_sheet, bare=True)
187
  changed = value[key] != child or changed
 
 
 
 
 
188
  elif isinstance(child, (dict, list)):
189
  changed = _reanchor_binding(child, shift, own_sheet) or changed
190
  return changed
@@ -206,7 +211,6 @@ def _reanchor_cells(tab: dict, shift: dict) -> None:
206
  href = link.get("href")
207
  if isinstance(href, str) and parse_ref(href):
208
  link["href"] = shift_for_structure(href, shift, tab["name"], bare=True)
209
- changed = link["href"] != href or changed
210
  if changed and cell.get("kind") in ("formula", "bound"):
211
  cell["state"] = "stale"
212
 
@@ -239,6 +243,28 @@ def _move_tab(tab: dict, op: dict) -> None:
239
  continue
240
  moved = _moved_index(ref["row"] if axis == "row" else col_to_index(ref["col"]), at, delta)
241
  if moved is not None:
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
242
  ref[axis] = moved if axis == "row" else index_to_col(moved)
243
  cells[format_ref(ref)] = cell
244
  tab["cells"] = cells
@@ -262,26 +288,78 @@ def _move_tab(tab: dict, op: dict) -> None:
262
  if "rules" in tab:
263
  rules = []
264
  for rule in tab["rules"]:
265
- rect = _move_range(rule["range"], op, drop_deleted=True)
 
 
 
 
 
 
266
  if rect is not None:
267
  rules.append({**rule, "range": rect})
268
  tab["rules"] = rules
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
269
 
270
 
271
- def _move_charts(tab: dict, tab_id: str, op: dict) -> None:
272
  for chart in tab.get("charts", []):
273
  source = chart.get("source") or {}
 
 
 
 
 
 
 
 
 
 
 
 
 
274
  if source.get("sheetId", tab["id"]) == tab_id:
275
- chart["range"] = _move_range(chart["range"], op)
 
 
 
 
276
  if "ranges" in source:
277
- source["ranges"] = [_move_range(rect, op) for rect in source["ranges"]]
278
- if tab["id"] == tab_id:
279
- # Stored geometry anchors are zero-based too; pixel offsets and sizes stay intact.
280
- for key in ("geometry", "anchor"):
281
- geometry = chart.get(key)
282
- if isinstance(geometry, dict) and op["axis"] in geometry:
283
- moved = shift_index(geometry[op["axis"]] + 1, op["at"], op["delta"])
284
- geometry[op["axis"]] = (op["at"] if moved is None else moved) - 1
 
 
 
 
 
 
285
 
286
 
287
  def apply_structure(book: dict, tab_id: str, op: dict) -> dict:
@@ -308,9 +386,31 @@ def apply_structure(book: dict, tab_id: str, op: dict) -> dict:
308
  if not op["delta"]:
309
  return result
310
  shift = {**op, "sheet": target["name"]}
 
 
311
  for tab in result["sheets"]:
312
  _reanchor_cells(tab, shift)
313
- _move_charts(tab, tab_id, op)
 
 
 
 
 
314
  if tab["id"] == tab_id:
315
  _move_tab(tab, op)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
316
  return result
 
11
  import re
12
  from collections.abc import Callable
13
 
14
+ from core.sheets import (DEFAULT_SHEET_COLS, DEFAULT_SHEET_ROWS, MAX_SHEET_COLS,
15
+ MAX_SHEET_ROWS, SheetRefusal)
16
 
17
 
18
  REF_SOURCE = r"(?:(?:'([^']{1,64})'|([A-Za-z_][A-Za-z0-9_]{0,63}))!)?(\$?)([A-Za-z]{1,3})(\$?)([1-9][0-9]{0,6})"
 
163
 
164
  return format_ref(at_index(first, move(start))) + separator + format_ref(at_index(last, move(end)))
165
 
166
+ return rewrite_refs(text, rewrite, rewrite_range)
 
167
 
168
 
169
  def _moved_index(value: int, at: int, delta: int) -> int | None:
 
185
  if key in BINDING_REF_KEYS and isinstance(child, str) and parse_ref(child):
186
  value[key] = shift_for_structure(child, shift, own_sheet, bare=True)
187
  changed = value[key] != child or changed
188
+ elif key in BINDING_REF_KEYS and isinstance(child, str) and child.strip().startswith("="):
189
+ # HOTFIX 2026-09-29: a filter value slot may hold a FORMULA (`=DATE($E$24,1,1)`);
190
+ # its references follow a row or column edit exactly as a formula cell's do.
191
+ value[key] = shift_for_structure(child, shift, own_sheet)
192
+ changed = value[key] != child or changed
193
  elif isinstance(child, (dict, list)):
194
  changed = _reanchor_binding(child, shift, own_sheet) or changed
195
  return changed
 
211
  href = link.get("href")
212
  if isinstance(href, str) and parse_ref(href):
213
  link["href"] = shift_for_structure(href, shift, tab["name"], bare=True)
 
214
  if changed and cell.get("kind") in ("formula", "bound"):
215
  cell["state"] = "stale"
216
 
 
243
  continue
244
  moved = _moved_index(ref["row"] if axis == "row" else col_to_index(ref["col"]), at, delta)
245
  if moved is not None:
246
+ binding = cell.get("binding") or {}
247
+ if (cell.get("kind") == "bound" and binding.get("mode") == "view"
248
+ and binding.get("viewType") != "grid" and isinstance(binding.get("block"), dict)):
249
+ block = binding["block"]
250
+ size = {key: value if type(value := block.get(key)) is int and value > 0 else fallback
251
+ for key, fallback in (("cols", 8), ("rows", 20))}
252
+ key = "rows" if axis == "row" else "cols"
253
+ value = ref["row"] if axis == "row" else col_to_index(ref["col"])
254
+ end = value + size[key]
255
+ change = (delta if value < at < end else 0) if delta > 0 else -max(
256
+ 0, min(end, at - delta) - max(value, at))
257
+ if change:
258
+ binding["block"] = {**size, key: max(1, size[key] + change)}
259
+ child = cell.get("viewChild")
260
+ origin = parse_ref(child.get("origin")) if isinstance(child, dict) else None
261
+ if origin:
262
+ index = _moved_index(origin["row"] if axis == "row" else col_to_index(origin["col"]), at, delta)
263
+ if index is None:
264
+ cell.pop("viewChild", None)
265
+ else:
266
+ origin[axis] = index if axis == "row" else index_to_col(index)
267
+ child["origin"] = format_ref(origin)
268
  ref[axis] = moved if axis == "row" else index_to_col(moved)
269
  cells[format_ref(ref)] = cell
270
  tab["cells"] = cells
 
288
  if "rules" in tab:
289
  rules = []
290
  for rule in tab["rules"]:
291
+ rect = rule["range"]
292
+ if isinstance(rect, str):
293
+ rect = shift_for_structure(rect, {**op, "sheet": tab["name"]}, tab["name"])
294
+ if rect == "#REF!":
295
+ continue
296
+ else:
297
+ rect = _move_range(rect, op, drop_deleted=True)
298
  if rect is not None:
299
  rules.append({**rule, "range": rect})
300
  tab["rules"] = rules
301
+ if axis == "row" and "rowGroups" in tab:
302
+ groups = []
303
+ for group in tab["rowGroups"]:
304
+ rect = _move_range({"row0": group["start"] - 1, "row1": group["end"] - 1},
305
+ op, drop_deleted=True)
306
+ if rect is not None:
307
+ groups.append({**group, "start": rect["row0"] + 1, "end": rect["row1"] + 1})
308
+ if groups:
309
+ tab["rowGroups"] = groups
310
+ else:
311
+ tab.pop("rowGroups", None)
312
+
313
+
314
+ def _move_chart_range(rect: dict, op: dict, extent: dict) -> dict:
315
+ """Match types.ts::chartRangeAfterStructure, including its viewport clamp."""
316
+ start, end = op["axis"] + "0", op["axis"] + "1"
317
+ lo, hi = max(0, min(rect[start], rect[end])), max(0, max(rect[start], rect[end]))
318
+ moved = _move_range({**rect, start: lo, end: hi}, op)
319
+ lo, hi = moved[start], moved[end]
320
+ ceiling = max(0, extent["rows" if op["axis"] == "row" else "cols"] - 1)
321
+ if hi > ceiling:
322
+ lo, hi = max(0, lo - (hi - ceiling)), ceiling
323
+ return {**moved, start: min(lo, ceiling), end: min(max(lo, hi), ceiling)}
324
 
325
 
326
+ def _move_charts(tab: dict, tab_id: str, op: dict, extent: dict) -> None:
327
  for chart in tab.get("charts", []):
328
  source = chart.get("source") or {}
329
+ previous = chart.get("geometry") or {}
330
+ geometry = {"col": previous.get("col", min(chart["range"]["col0"], chart["range"]["col1"])),
331
+ "row": previous.get("row", min(chart["range"]["row0"], chart["range"]["row1"])),
332
+ "width": previous.get("width", 640), "height": previous.get("height", 320)}
333
+ if tab["id"] == tab_id:
334
+ point = _move_chart_range({"col0": geometry["col"], "col1": geometry["col"],
335
+ "row0": geometry["row"], "row1": geometry["row"]}, op, extent)
336
+ if point["col0"] != geometry["col"] or point["row0"] != geometry["row"]:
337
+ chart["geometry"] = {**geometry, "col": point["col0"], "row": point["row0"]}
338
+ anchor = chart.get("anchor")
339
+ if isinstance(anchor, dict) and op["axis"] in anchor:
340
+ moved = shift_index(anchor[op["axis"]] + 1, op["at"], op["delta"])
341
+ anchor[op["axis"]] = (op["at"] if moved is None else moved) - 1
342
  if source.get("sheetId", tab["id"]) == tab_id:
343
+ rect = _move_chart_range(chart["range"], op, extent)
344
+ if (tab["id"] != tab_id and "geometry" not in chart
345
+ and (rect["col0"] != chart["range"]["col0"] or rect["row0"] != chart["range"]["row0"])):
346
+ chart["geometry"] = geometry
347
+ chart["range"] = rect
348
  if "ranges" in source:
349
+ source["ranges"] = [_move_chart_range(rect, op, extent) for rect in source["ranges"]]
350
+ presentation = chart.get("presentation") or {}
351
+ if "series" in presentation:
352
+ series = []
353
+ for setting in presentation["series"]:
354
+ match = re.fullmatch(r"(column|row):([A-Z]+|\d+)", setting["key"])
355
+ if match and ("col" if match[1] == "column" else "row") == op["axis"]:
356
+ index = col_to_index(match[2]) if match[1] == "column" else int(match[2])
357
+ moved = _moved_index(index, op["at"], op["delta"])
358
+ if moved is None:
359
+ continue
360
+ setting["key"] = match[1] + ":" + (index_to_col(moved) if match[1] == "column" else str(moved))
361
+ series.append(setting)
362
+ presentation["series"] = series
363
 
364
 
365
  def apply_structure(book: dict, tab_id: str, op: dict) -> dict:
 
386
  if not op["delta"]:
387
  return result
388
  shift = {**op, "sheet": target["name"]}
389
+ extent = {"cols": min(MAX_SHEET_COLS, max(DEFAULT_SHEET_COLS, target.get("cols", DEFAULT_SHEET_COLS))),
390
+ "rows": min(MAX_SHEET_ROWS, max(DEFAULT_SHEET_ROWS, target.get("rows", DEFAULT_SHEET_ROWS)))}
391
  for tab in result["sheets"]:
392
  _reanchor_cells(tab, shift)
393
+ _move_charts(tab, tab_id, op, extent)
394
+ for rule in tab.get("rules", []):
395
+ if rule.get("kind") == "formula":
396
+ for key in ("value", "value2"):
397
+ if isinstance(rule.get(key), str):
398
+ rule[key] = shift_for_structure(rule[key], shift, tab["name"])
399
  if tab["id"] == tab_id:
400
  _move_tab(tab, op)
401
+ names = result.get("namedRanges") or []
402
+ # The lossless document accepts the older name -> ref dictionary as well as named rows.
403
+ if isinstance(names, dict):
404
+ for name, value in names.items():
405
+ if isinstance(value, str):
406
+ names[name] = shift_for_structure(value, shift, result["sheets"][0]["name"])
407
+ names = [value for value in names.values() if isinstance(value, dict)]
408
+ for named_range in names:
409
+ scope = named_range.get("sheet")
410
+ own_sheet = next((tab["name"] for tab in result["sheets"]
411
+ if tab["id"] == scope or tab["name"] == scope),
412
+ scope if scope is not None else result["sheets"][0]["name"])
413
+ for key in ("ref", "base"):
414
+ if isinstance(named_range.get(key), str):
415
+ named_range[key] = shift_for_structure(named_range[key], shift, own_sheet)
416
  return result
platform/core/sheets.py CHANGED
@@ -152,6 +152,14 @@ MAX_FONT_PT = 96
152
  _STYLE_MARKS = ("bold", "italic", "underline")
153
  _BORDER_SIDES = ("top", "right", "bottom", "left")
154
 
 
 
 
 
 
 
 
 
155
  #: R8's indent, in steps rather than pixels: a stored pixel count would be a layout decision that
156
  #: outlives every layout change, exactly as a stored hex outlives a palette.
157
  _MAX_INDENT = 8
@@ -204,6 +212,12 @@ SHEET_RULE_OPS = ("gt", "lt", "gte", "lte", "eq", "neq", "between", "contains",
204
  #: every rule written before this ticket already is, so writing the word onto them would make a
205
  #: reload compare unequal to the save. Mirrored in `web/src/sheet/types.ts::SHEET_RULE_KINDS`.
206
  SHEET_RULE_KINDS = ("cell", "scale", "bar", "formula")
 
 
 
 
 
 
207
  # The client compiler accepts 500 characters per condition.
208
  MAX_RULE_FORMULA_LENGTH = 500
209
 
@@ -267,10 +281,8 @@ _MAX_UNSUPPORTED = 200
267
  CELL_COLOUR_PATTERN = r"^(--lp-[a-z0-9-]{1,40}|#[0-9a-f]{6})$"
268
  _CELL_COLOUR = re.compile(CELL_COLOUR_PATTERN)
269
 
270
- #: R2 / C1: CELL `color` / `fill` use CELL_COLOUR_PATTERN above. This older guard stays on
271
- #: `_clean_rule_style` and `_clean_scale`: their RULE_FILLS / RULE_INKS galleries remain token
272
- #: only. Its existing storage allowance is unchanged; a cell fill's custom door does not widen
273
- #: the conditional-format palettes or make either guard dead.
274
  _COLOUR = re.compile(r"^[A-Za-z0-9#_-]{1,40}$")
275
 
276
  _MAX_DECIMALS = 10
@@ -654,13 +666,44 @@ def _clean_style(raw):
654
 
655
  border = raw.get("border")
656
  if isinstance(border, dict):
657
- sides = {side: True for side in _BORDER_SIDES if border.get(side) is True}
 
 
 
 
658
  if sides:
659
  out["border"] = sides
660
 
661
  return out or None
662
 
663
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
664
  # ── the cell ─────────────────────────────────────────────────────────────────────────────────
665
 
666
  def _clean_validation(raw):
@@ -1100,9 +1143,22 @@ def _clean_rule_style(raw):
1100
  return {}
1101
  out = {}
1102
  for key in _RULE_STYLE_COLOURS:
1103
- token = _text(raw.get(key), 40)
1104
- if token and _COLOUR.match(token):
1105
- out[key] = token
 
 
 
 
 
 
 
 
 
 
 
 
 
1106
  for mark in _RULE_STYLE_MARKS:
1107
  if raw.get(mark) is True:
1108
  out[mark] = True
@@ -1114,21 +1170,37 @@ def _clean_rule_scale(raw):
1114
 
1115
  β›” BOTH ENDS OR NEITHER. One end alone names no ramp at all, and storing half of one would
1116
  leave the client looking up a pair that cannot match and falling back to a default scale the
1117
- person never chose. The tokens go through the same `_COLOUR` guard `_clean_rule_style` uses,
1118
- so nothing free form reaches a canvas `fillStyle` from a stored document.
1119
 
1120
  ⚠ THE SCALE'S BOUNDS ARE NOT HERE. They ride `value` and `value2` on the rule itself, which
1121
  `_clean_rules` already persists for every op that takes operands, so a colour scale needed no
1122
  new numeric slot at all.
 
 
 
 
 
1123
  """
1124
  if not isinstance(raw, dict):
1125
  return None
 
 
 
 
 
 
 
 
 
 
1126
  out = {}
1127
  for key in ("low", "high"):
1128
  token = _text(raw.get(key), 40)
1129
  if not token or not _COLOUR.match(token):
1130
  return None
1131
  out[key] = token
 
1132
  return out
1133
 
1134
 
@@ -1699,10 +1771,31 @@ def _append_write_log(tab, rev, addrs, *, actor="", actor_name=""):
1699
  tab["writes"] = writes[-WRITE_LOG_REVS:]
1700
 
1701
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1702
  def get_workbook(st, workbook_id):
1703
  """One workbook, or a named refusal when the document is not there."""
1704
  wid = _slug_id(workbook_id)
1705
- doc = _read(st, workbook_key(wid))
1706
  if not doc or not doc.get("id"):
1707
  raise SheetRefusal("workbook_missing",
1708
  "That workbook no longer exists.")
 
152
  _STYLE_MARKS = ("bold", "italic", "underline")
153
  _BORDER_SIDES = ("top", "right", "bottom", "left")
154
 
155
+ #: ⭐⭐ S67-T14 / CONTRACT C2: Excel's seven border line styles, name for name. Mirrored in
156
+ #: `aios-web/web/src/sheet/types.ts::CELL_BORDER_STYLES`, and `verify_sheet.py` compares the two
157
+ #: lists. A side is either the legacy `True` (read as a thin black line and stored unchanged, so a
158
+ #: document written before S67 reads back byte for byte) or `{"style": <one of these>,
159
+ #: "color": "#rrggbb"}`.
160
+ CELL_BORDER_STYLES = ("thin", "medium", "thick", "dashed", "dotted", "double", "hair")
161
+ _BORDER_COLOUR = re.compile(r"^#[0-9a-f]{6}$")
162
+
163
  #: R8's indent, in steps rather than pixels: a stored pixel count would be a layout decision that
164
  #: outlives every layout change, exactly as a stored hex outlives a palette.
165
  _MAX_INDENT = 8
 
212
  #: every rule written before this ticket already is, so writing the word onto them would make a
213
  #: reload compare unequal to the save. Mirrored in `web/src/sheet/types.ts::SHEET_RULE_KINDS`.
214
  SHEET_RULE_KINDS = ("cell", "scale", "bar", "formula")
215
+ #: ⭐ S67-T13: Excel's colour scale point types, one list for both ends. `lowest` / `highest` are
216
+ #: measured from the range at paint, so the scale re-spreads as the data changes; the other three
217
+ #: read their number from the rule's `value` (low end) or `value2` (high end). A type outside this
218
+ #: list is REFUSED by name, never dropped: a dropped type reads back as a different scale. Mirrored
219
+ #: in `web/src/sheet/types.ts::SheetRuleScaleType`.
220
+ SHEET_RULE_SCALE_TYPES = ("lowest", "highest", "number", "percent", "percentile")
221
  # The client compiler accepts 500 characters per condition.
222
  MAX_RULE_FORMULA_LENGTH = 500
223
 
 
281
  CELL_COLOUR_PATTERN = r"^(--lp-[a-z0-9-]{1,40}|#[0-9a-f]{6})$"
282
  _CELL_COLOUR = re.compile(CELL_COLOUR_PATTERN)
283
 
284
+ #: Cell and rule `color` / `fill` use CELL_COLOUR_PATTERN above. This older guard now serves
285
+ #: `_clean_rule_scale` only; the scale's existing storage allowance is unchanged.
 
 
286
  _COLOUR = re.compile(r"^[A-Za-z0-9#_-]{1,40}$")
287
 
288
  _MAX_DECIMALS = 10
 
666
 
667
  border = raw.get("border")
668
  if isinstance(border, dict):
669
+ sides = {}
670
+ for side in _BORDER_SIDES:
671
+ line = _clean_border_side(border.get(side))
672
+ if line is not None:
673
+ sides[side] = line
674
  if sides:
675
  out["border"] = sides
676
 
677
  return out or None
678
 
679
 
680
+ def _clean_border_side(raw):
681
+ """One C2 border side: `True`, a `{style, color}` dict, or None when the side is absent.
682
+
683
+ β›”β›” THE SECOND DOCUMENTED EXCEPTION TO "UNKNOWN KEYS ARE DROPPED IN SILENCE". A border side
684
+ that names a line style outside the seven, or a colour that is not a six digit hex, is
685
+ REFUSED by name. Dropping it would store a cell with no line where the person drew one, and
686
+ the caller could not learn its request went nowhere: `_clean_format`'s reason, one key over.
687
+ A side that is `False`, None or absent is simply no side. A legacy `True` stays `True`.
688
+ """
689
+ if raw is None or raw is False:
690
+ return None
691
+ if raw is True:
692
+ return True
693
+ if not isinstance(raw, dict):
694
+ return None
695
+ style = raw.get("style")
696
+ colour = raw.get("color")
697
+ if isinstance(colour, str):
698
+ colour = colour.strip().lower()
699
+ if (not isinstance(style, str) or style not in CELL_BORDER_STYLES
700
+ or not isinstance(colour, str) or not _BORDER_COLOUR.match(colour)):
701
+ raise SheetRefusal(
702
+ "cell_border_invalid",
703
+ "A border needs a line style such as thin and a six digit hex colour such as #000000.")
704
+ return {"style": style, "color": colour}
705
+
706
+
707
  # ── the cell ─────────────────────────────────────────────────────────────────────────────────
708
 
709
  def _clean_validation(raw):
 
1143
  return {}
1144
  out = {}
1145
  for key in _RULE_STYLE_COLOURS:
1146
+ token = raw.get(key)
1147
+ if token is None or token == "":
1148
+ continue
1149
+ if isinstance(token, str):
1150
+ token = token.strip()
1151
+ if not token:
1152
+ continue
1153
+ if token.startswith("#"):
1154
+ token = token.lower()
1155
+ # Validate the whole value before normalising: neither truncation nor the cell
1156
+ # cleaner's three digit import rescue may turn an invalid rule colour into a choice.
1157
+ if not isinstance(token, str) or not _CELL_COLOUR.fullmatch(token):
1158
+ raise SheetRefusal(
1159
+ "sheet_rule_colour_invalid",
1160
+ "A rule colour must be a palette colour or a six digit hex colour such as #c8a24b.")
1161
+ out[key] = _normalise_colour(token)
1162
  for mark in _RULE_STYLE_MARKS:
1163
  if raw.get(mark) is True:
1164
  out[mark] = True
 
1170
 
1171
  β›” BOTH ENDS OR NEITHER. One end alone names no ramp at all, and storing half of one would
1172
  leave the client looking up a pair that cannot match and falling back to a default scale the
1173
+ person never chose. The tokens keep their existing `_COLOUR` guard, so nothing free form
1174
+ reaches a canvas `fillStyle` from a stored document.
1175
 
1176
  ⚠ THE SCALE'S BOUNDS ARE NOT HERE. They ride `value` and `value2` on the rule itself, which
1177
  `_clean_rules` already persists for every op that takes operands, so a colour scale needed no
1178
  new numeric slot at all.
1179
+
1180
+ ⭐ S67-T13: `minType` / `maxType` say what each end MEANS (`SHEET_RULE_SCALE_TYPES`). Each is
1181
+ kept only when present and never injected, so a stored boundless scale reads back byte for
1182
+ byte, and input with no types returns exactly the `{low, high}` it always did. An unknown type
1183
+ is refused by name.
1184
  """
1185
  if not isinstance(raw, dict):
1186
  return None
1187
+ types = {}
1188
+ for key in ("minType", "maxType"):
1189
+ kind = raw.get(key)
1190
+ if kind is None or kind == "":
1191
+ continue
1192
+ if kind not in SHEET_RULE_SCALE_TYPES:
1193
+ raise SheetRefusal(
1194
+ "sheet_rule_scale_type_unknown",
1195
+ "A colour scale end must be one of " + ", ".join(SHEET_RULE_SCALE_TYPES) + ".")
1196
+ types[key] = kind
1197
  out = {}
1198
  for key in ("low", "high"):
1199
  token = _text(raw.get(key), 40)
1200
  if not token or not _COLOUR.match(token):
1201
  return None
1202
  out[key] = token
1203
+ out.update(types)
1204
  return out
1205
 
1206
 
 
1771
  tab["writes"] = writes[-WRITE_LOG_REVS:]
1772
 
1773
 
1774
+ def _read_workbook_doc(st, key):
1775
+ """A workbook document, `{}` when it is absent, and a REFUSAL when the store could not answer.
1776
+
1777
+ HOTFIX 2026-09-29: `_read` swallows every store error into `{}`, so an overloaded database
1778
+ (staging right after a restart: a pool timeout, a 503 on the next request) was reported as
1779
+ "That workbook no longer exists." and the browser dropped the owner's open tab. A store that
1780
+ fails is not a workbook that is gone; the reader is told to try again and nothing is dropped.
1781
+ """
1782
+ try:
1783
+ projected = getattr(st, "get_projection", None)
1784
+ if callable(projected):
1785
+ return dict(projected(key, drop=()) or {})
1786
+ return dict(st.get(key) or {})
1787
+ except (KeyError, FileNotFoundError):
1788
+ return {}
1789
+ except Exception as exc: # noqa: BLE001
1790
+ raise SheetRefusal("workbook_unavailable",
1791
+ "This workbook could not be read just now, so nothing was changed. "
1792
+ "Try again in a moment.") from exc
1793
+
1794
+
1795
  def get_workbook(st, workbook_id):
1796
  """One workbook, or a named refusal when the document is not there."""
1797
  wid = _slug_id(workbook_id)
1798
+ doc = _read_workbook_doc(st, workbook_key(wid))
1799
  if not doc or not doc.get("id"):
1800
  raise SheetRefusal("workbook_missing",
1801
  "That workbook no longer exists.")
platform/core/store_pg.py CHANGED
@@ -36,16 +36,25 @@ worth hiding: a connection failure RAISES from `get()` here. Callers that must n
36
  should catch it β€” silently returning `{}` from a database that is simply unreachable is how an
37
  empty dict gets written back over real data.
38
  """
 
 
 
39
  import json
40
  import os
 
 
41
  import threading
 
 
42
  from urllib.parse import urlparse
43
 
44
  import core.data_binding as data_binding
 
45
 
46
  _URL_ENV = 'DATABASE_URL'
47
  _LOCK = threading.RLock()
48
  _POOL = {'pool': None, 'url': None, 'schema': None}
 
49
 
50
 
51
  def _url():
@@ -149,15 +158,25 @@ def sandbox_tenants():
149
 
150
 
151
  def isolated_staging_database():
152
- """Whether this process is staging on the exact isolated database host it was handed."""
153
  if data_binding.deployment_id() != _STAGING_SPACE:
154
  return False
155
- allowed_host = str(os.environ.get(_ISOLATED_DATABASE_HOST_VAR) or '').strip().lower()
156
  try:
157
- current_host = (urlparse(str(os.environ.get('DATABASE_URL') or '')).hostname or '').lower()
 
 
 
 
 
158
  except ValueError:
159
  return False
160
- return bool(allowed_host and current_host == allowed_host)
 
 
 
 
 
161
 
162
 
163
  def write_refusal(tenant_slug=None):
@@ -172,7 +191,7 @@ def write_refusal(tenant_slug=None):
172
  4. anything else -> REFUSED
173
 
174
  Rule 2 permits one narrow exception: the exact staging Space, an explicitly handed tenant,
175
- and a DATABASE_URL host matching the isolated host stamped by the deploy must all agree.
176
  """
177
  slug = _effective_slug(tenant_slug)
178
  if data_binding.is_production_deployment():
@@ -202,6 +221,29 @@ def write_refusal(tenant_slug=None):
202
  f'intended.')
203
 
204
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
205
  def check_write(tenant_slug=None, operation='write'):
206
  """Raise `StoreWriteRefused` unless this deployment may write this tenant. The pg door.
207
 
@@ -212,12 +254,44 @@ def check_write(tenant_slug=None, operation='write'):
212
  a session-scoped fallback and tell the user their change was saved. A new type would have had
213
  to re-earn both properties, and would have failed the second one silently.
214
  """
215
- reason = write_refusal(tenant_slug)
216
  if reason:
217
  raise data_binding.StoreWriteRefused(
218
  f'{operation} to tenant {_effective_slug(tenant_slug)!r} refused: {reason}')
219
 
220
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
221
  def _pool():
222
  """The psycopg connection pool, opened once per process.
223
 
@@ -229,7 +303,10 @@ def _pool():
229
  if not url:
230
  raise RuntimeError(
231
  f'{_URL_ENV} is not set; the Postgres store backend has nothing to connect to.')
232
- if _POOL['pool'] is not None and _POOL['url'] == url:
 
 
 
233
  return _POOL['pool']
234
  try:
235
  from psycopg_pool import ConnectionPool
@@ -249,9 +326,11 @@ def _pool():
249
  # real requests, not be held awake by this process's idle pool for psycopg's 10-minute
250
  # default. Thirty seconds still reuses a normal request burst without creating a resident
251
  # database client.
 
252
  _POOL['pool'] = ConnectionPool(url, min_size=0, max_size=8, open=True, timeout=10.0,
253
- max_idle=30.0)
254
  _POOL['url'] = url
 
255
  return _POOL['pool']
256
 
257
 
@@ -389,6 +468,22 @@ def _get_projections(names, drop=(), tenant_slug=None, timeout_s=None):
389
  }
390
 
391
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
392
  def _get_many(names, tenant_slug=None):
393
  """Several ordinary documents in one round trip, keyed by requested store key."""
394
  keys = list(dict.fromkeys(str(name) for name in (names or ()) if str(name)))
@@ -435,6 +530,32 @@ def exists(name, tenant_slug=None):
435
  DOCUMENT_CEILING_BYTES = 128 * 1024 * 1024
436
 
437
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
438
  class StoreDocumentTooLarge(RuntimeError):
439
  """A write refused because the document would grow past `DOCUMENT_CEILING_BYTES`."""
440
 
@@ -733,6 +854,10 @@ class PgStore:
733
  """Bound multi-key read used by request-scoped read snapshots."""
734
  return _get_many(names, tenant_slug=self.tenant_slug)
735
 
 
 
 
 
736
  def _read_strict(self, name):
737
  """Interface parity with the HF Store. There, `get` is lenient (swallows a transient
738
  failure) and `_read_strict` opts out so a failed read ABORTS a read-modify-write instead
 
36
  should catch it β€” silently returning `{}` from a database that is simply unreachable is how an
37
  empty dict gets written back over real data.
38
  """
39
+ import atexit
40
+ import base64
41
+ import binascii
42
  import json
43
  import os
44
+ import ssl
45
+ import tempfile
46
  import threading
47
+ from functools import lru_cache
48
+ from pathlib import Path
49
  from urllib.parse import urlparse
50
 
51
  import core.data_binding as data_binding
52
+ from core.user_tables import enforce_byte_budget as _BYTE_BUDGET_ENFORCER
53
 
54
  _URL_ENV = 'DATABASE_URL'
55
  _LOCK = threading.RLock()
56
  _POOL = {'pool': None, 'url': None, 'schema': None}
57
+ _DATABASE_CA_ENV = 'AIOS_DATABASE_CA_PEM_B64'
58
 
59
 
60
  def _url():
 
158
 
159
 
160
  def isolated_staging_database():
161
+ """Match the exact staging database; legacy host stamps are valid only on port 5432."""
162
  if data_binding.deployment_id() != _STAGING_SPACE:
163
  return False
164
+ stamp = str(os.environ.get(_ISOLATED_DATABASE_HOST_VAR) or '').strip()
165
  try:
166
+ current = _database_endpoint(_url() or '')
167
+ if not stamp or not current[0]:
168
+ return False
169
+ if '/' in stamp or ':' in stamp:
170
+ return current == _database_endpoint('postgresql://' + stamp)
171
+ return current[1] == 5432 and current[0] == stamp.lower()
172
  except ValueError:
173
  return False
174
+
175
+
176
+ def _database_endpoint(url):
177
+ parsed = urlparse(url)
178
+ return ((parsed.hostname or '').lower(), parsed.port or 5432,
179
+ parsed.path.lstrip('/').split('/', 1)[0])
180
 
181
 
182
  def write_refusal(tenant_slug=None):
 
191
  4. anything else -> REFUSED
192
 
193
  Rule 2 permits one narrow exception: the exact staging Space, an explicitly handed tenant,
194
+ and a DATABASE_URL identity matching the isolated database stamped by the deploy must agree.
195
  """
196
  slug = _effective_slug(tenant_slug)
197
  if data_binding.is_production_deployment():
 
221
  f'intended.')
222
 
223
 
224
+ def gate_write_refusal(tenant_slug=None):
225
+ """`None` unless a GATE process is writing a production tenant, else why not. W-219.
226
+
227
+ β›”β›” CONSULTED BEFORE `write_refusal`, AND IT IS THE HALF THAT CLOSES W-219. A lane container is
228
+ the exact isolated staging posture `write_refusal` rule 2 ALLOWS for tenant #0, so a gate run
229
+ there wrote staging's `t_royal_imports` (SDLC 61: `odoo_connector_config` `syncEvery=manual`,
230
+ `ut_odoo_invoices` disabled, four fixture keychain credentials). A gate writes fixture tenants
231
+ (`qa-b`, `qa-pg`) or nothing, on every deployment, whatever the environment grants: see
232
+ `data_binding.gate_process` for who counts as a gate and why the environment cannot opt out.
233
+ `write_refusal` itself is unchanged, so the product rule stays testable on its own.
234
+ """
235
+ who = data_binding.gate_process()
236
+ if not who:
237
+ return None
238
+ slug = _effective_slug(tenant_slug)
239
+ if slug not in PRODUCTION_TENANTS:
240
+ return None
241
+ return (f'this process is a verification gate ({who}), and a gate never writes tenant '
242
+ f'{slug!r} ({_schema_for(slug)}), which holds real customer data, not even on isolated '
243
+ f'staging. A gate writes fixture tenants only: run it with STORE_BACKEND=hf and no '
244
+ f'DATABASE_URL (W-219).')
245
+
246
+
247
  def check_write(tenant_slug=None, operation='write'):
248
  """Raise `StoreWriteRefused` unless this deployment may write this tenant. The pg door.
249
 
 
254
  a session-scoped fallback and tell the user their change was saved. A new type would have had
255
  to re-earn both properties, and would have failed the second one silently.
256
  """
257
+ reason = gate_write_refusal(tenant_slug) or write_refusal(tenant_slug)
258
  if reason:
259
  raise data_binding.StoreWriteRefused(
260
  f'{operation} to tenant {_effective_slug(tenant_slug)!r} refused: {reason}')
261
 
262
 
263
+ class DatabaseCAError(RuntimeError):
264
+ """The configured database CA cannot be used; TLS must never silently fall back."""
265
+
266
+
267
+ @lru_cache(maxsize=1)
268
+ def _database_ca_file(encoded):
269
+ """Decode a configured CA once per process and retain its private file for libpq."""
270
+ try:
271
+ pem = base64.b64decode(encoded, validate=True)
272
+ if not (pem.strip().startswith(b'-----BEGIN CERTIFICATE-----')
273
+ and pem.strip().endswith(b'-----END CERTIFICATE-----')):
274
+ raise ValueError
275
+ ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT).load_verify_locations(cadata=pem.decode('ascii'))
276
+ except (ValueError, binascii.Error, ssl.SSLError):
277
+ raise DatabaseCAError(
278
+ f'{_DATABASE_CA_ENV} must be base64-encoded PEM certificate data') from None
279
+ try:
280
+ fd, name = tempfile.mkstemp(prefix='aios-database-ca-', suffix='.crt')
281
+ path = Path(name)
282
+ try:
283
+ with os.fdopen(fd, 'wb') as target:
284
+ os.chmod(path, 0o600)
285
+ target.write(pem)
286
+ except BaseException:
287
+ path.unlink(missing_ok=True)
288
+ raise
289
+ except OSError:
290
+ raise DatabaseCAError(f'{_DATABASE_CA_ENV} could not create a private CA file') from None
291
+ atexit.register(path.unlink, missing_ok=True)
292
+ return str(path)
293
+
294
+
295
  def _pool():
296
  """The psycopg connection pool, opened once per process.
297
 
 
303
  if not url:
304
  raise RuntimeError(
305
  f'{_URL_ENV} is not set; the Postgres store backend has nothing to connect to.')
306
+ ca = os.environ.get('AIOS_DATABASE_CA_PEM_B64')
307
+ sslrootcert = _database_ca_file(ca) if ca is not None else None
308
+ if (_POOL['pool'] is not None and _POOL['url'] == url
309
+ and _POOL.get('sslrootcert') == sslrootcert):
310
  return _POOL['pool']
311
  try:
312
  from psycopg_pool import ConnectionPool
 
326
  # real requests, not be held awake by this process's idle pool for psycopg's 10-minute
327
  # default. Thirty seconds still reuses a normal request burst without creating a resident
328
  # database client.
329
+ options = {'kwargs': {'sslrootcert': sslrootcert}} if sslrootcert is not None else {}
330
  _POOL['pool'] = ConnectionPool(url, min_size=0, max_size=8, open=True, timeout=10.0,
331
+ max_idle=30.0, **options)
332
  _POOL['url'] = url
333
+ _POOL['sslrootcert'] = sslrootcert
334
  return _POOL['pool']
335
 
336
 
 
468
  }
469
 
470
 
471
+ def _get_entry(name, entry, tenant_slug=None):
472
+ """ONE top-level entry of document `name` (`value -> entry`), or None when either is absent.
473
+
474
+ HOTFIX 2026-09-28. Private for `_get_projections`' reason: an optimization offered by a bound
475
+ `PgStore` handle, never a store operation, so callers keep their whole-read fallback. A
476
+ stored `ut_*` table read this way crosses the wire as that table alone, not the whole
477
+ `user_tables` document (~60-100 MB on staging, ~3 s per read, holding a pool connection).
478
+ """
479
+ with _pool().connection() as con:
480
+ row = con.execute(
481
+ _q('SELECT value -> %s FROM {tbl} WHERE key = %s', 'store_kv', tenant_slug),
482
+ (str(entry), str(name))).fetchone()
483
+ value = row[0] if row else None
484
+ return dict(value) if isinstance(value, dict) else None
485
+
486
+
487
  def _get_many(names, tenant_slug=None):
488
  """Several ordinary documents in one round trip, keyed by requested store key."""
489
  keys = list(dict.fromkeys(str(name) for name in (names or ()) if str(name)))
 
530
  DOCUMENT_CEILING_BYTES = 128 * 1024 * 1024
531
 
532
 
533
+ def byte_budget_status():
534
+ """Inspect the shipped limits and enforcer without reading or writing a store."""
535
+ from core import user_tables as ut
536
+
537
+ reasons = []
538
+ limits = (
539
+ ("MAX_TABLE_BYTES", ut.MAX_TABLE_BYTES, 48 * 1024 * 1024),
540
+ ("MAX_DOCUMENT_BYTES", ut.MAX_DOCUMENT_BYTES, 128 * 1024 * 1024),
541
+ ("DOCUMENT_CEILING_BYTES", DOCUMENT_CEILING_BYTES, 128 * 1024 * 1024),
542
+ )
543
+ for name, value, shipped in limits:
544
+ # bool is an int subclass, but is not a byte limit.
545
+ if type(value) is not int or value <= 0:
546
+ reasons.append(f"{name} must be a positive integer")
547
+ elif value > shipped:
548
+ reasons.append(f"{name} exceeds the shipped {shipped // 1048576} MB limit")
549
+ if (ut.enforce_byte_budget is not _BYTE_BUDGET_ENFORCER
550
+ or getattr(_BYTE_BUDGET_ENFORCER, "__module__", None) != "core.user_tables"
551
+ or getattr(_BYTE_BUDGET_ENFORCER, "__name__", None) != "enforce_byte_budget"):
552
+ reasons.append("user_tables.enforce_byte_budget is not the original enforcer")
553
+ return {"armed": not reasons, "reasons": reasons,
554
+ "max_table_bytes": ut.MAX_TABLE_BYTES,
555
+ "max_document_bytes": ut.MAX_DOCUMENT_BYTES,
556
+ "document_ceiling_bytes": DOCUMENT_CEILING_BYTES}
557
+
558
+
559
  class StoreDocumentTooLarge(RuntimeError):
560
  """A write refused because the document would grow past `DOCUMENT_CEILING_BYTES`."""
561
 
 
854
  """Bound multi-key read used by request-scoped read snapshots."""
855
  return _get_many(names, tenant_slug=self.tenant_slug)
856
 
857
+ def get_entry(self, name, entry):
858
+ """Bound single-entry read; None means absent, and the caller falls back to `get`."""
859
+ return _get_entry(name, entry, tenant_slug=self.tenant_slug)
860
+
861
  def _read_strict(self, name):
862
  """Interface parity with the HF Store. There, `get` is lenient (swallows a transient
863
  failure) and `_read_strict` opts out so a failed read ABORTS a read-modify-write instead
platform/core/user_tables.py CHANGED
@@ -46,6 +46,7 @@ inherits C1 by construction β€” there is no list to add it to, which is R6's act
46
  """
47
  import datetime as _dt
48
  import hashlib
 
49
  import json as _json
50
  import math
51
  import re
@@ -278,6 +279,11 @@ def is_connected(table_key, st=None):
278
  table = all_defs(st).get(key) or {}
279
  if not table:
280
  return False
 
 
 
 
 
281
  if table.get('connected') is True:
282
  return True
283
  return (any(key.startswith(p) for p in _CONNECTED_PREFIXES)
@@ -285,6 +291,26 @@ def is_connected(table_key, st=None):
285
  and table.get('recordMode') == AUTOMATION_RECORD_MODE)
286
 
287
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
288
  #: ⭐⭐ W30-T31 (D-87) β€” table keys whose rows DO NOT LIVE HERE AT ALL.
289
  #:
290
  #: A subset of `_CONNECTED`, and the distinction is the whole ticket. *Connected* says where the
@@ -1572,6 +1598,28 @@ def get(table_key, st=None):
1572
  return all_tables(st).get(str(table_key))
1573
 
1574
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1575
  # --- DURABLE MAP ZONES -------------------------------------------------------
1576
  # Zones intentionally do not live in a view config: a view is a person's presentation of rows,
1577
  # while a named geographic boundary is tenant data used by several views and routes. Keeping a
@@ -1807,7 +1855,7 @@ def create(label, username, fields=None, source=None, st=None, *, managed_by=Non
1807
  if not label:
1808
  return None
1809
  existing = all_tables(st)
1810
- if len(existing) >= MAX_TABLES:
1811
  return None
1812
  key = KEY_PREFIX + _slug(label)
1813
  from core import route_system
@@ -5177,6 +5225,23 @@ def unread_note_subjects(subjects, viewer, *, st=None):
5177
  return out
5178
 
5179
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
5180
  def nav_entries(viewer=None, is_admin=False, st=None):
5181
  """Registry-SHAPED dicts so the nav can render user tables beside real modules with no
5182
  special-casing: the same keys the flat nav reads (`key`, `label`, `source`).
@@ -5206,8 +5271,7 @@ def nav_entries(viewer=None, is_admin=False, st=None):
5206
  if route_system.is_managed(t) or route_system.is_managed(key):
5207
  continue
5208
  source = str(t.get('source') or BLANK_SOURCE)
5209
- source_group = ('Odoo' if source.lower() == 'odoo' or key.startswith('ut_odoo_')
5210
- else 'Custom' if source == BLANK_SOURCE else source)
5211
  out.append({'key': key, 'label': t.get('label') or key,
5212
  'source': source, 'sourceGroup': source_group,
5213
  'user_table': True, 'nav': True, 'validate': False})
 
46
  """
47
  import datetime as _dt
48
  import hashlib
49
+ import inspect
50
  import json as _json
51
  import math
52
  import re
 
279
  table = all_defs(st).get(key) or {}
280
  if not table:
281
  return False
282
+ return _connected_definition(key, table)
283
+
284
+
285
+ def _connected_definition(key, table):
286
+ """`is_connected`'s answers 2 and 3, read off a definition the caller already holds."""
287
  if table.get('connected') is True:
288
  return True
289
  return (any(key.startswith(p) for p in _CONNECTED_PREFIXES)
 
291
  and table.get('recordMode') == AUTOMATION_RECORD_MODE)
292
 
293
 
294
+ def counts_toward_table_cap(table_key, definition=None):
295
+ """Does this database take one of the tenant's `MAX_TABLES` slots?
296
+
297
+ ⭐⭐ OWNER RULING 2026-09-27 (D-3284): CONNECTOR-OWNED DATABASES DO NOT. The cap bounds what
298
+ PEOPLE create. A connector spawns one database per report (Amazon alone about sixty), and
299
+ counting those hid ten Amazon databases behind the cap with their rows sitting unreachable in
300
+ the mirror. The same answers as `is_connected`, off a definition the caller holds, because a
301
+ spawn's updater holds the raw document rather than a store.
302
+ """
303
+ key = str(table_key or '').strip()
304
+ if key in _CONNECTED:
305
+ return False
306
+ return not _connected_definition(key, definition if isinstance(definition, dict) else {})
307
+
308
+
309
+ def capped_table_count(tables):
310
+ """How many `MAX_TABLES` slots `{key: definition}` uses. Connector databases take none."""
311
+ return sum(1 for k, d in (tables or {}).items() if counts_toward_table_cap(k, d))
312
+
313
+
314
  #: ⭐⭐ W30-T31 (D-87) β€” table keys whose rows DO NOT LIVE HERE AT ALL.
315
  #:
316
  #: A subset of `_CONNECTED`, and the distinction is the whole ticket. *Connected* says where the
 
1598
  return all_tables(st).get(str(table_key))
1599
 
1600
 
1601
+ def get_one(table_key, st=None):
1602
+ """`get`, reading ONLY this table's entry when the bound store can (HOTFIX 2026-09-28).
1603
+
1604
+ `get` deep-copies the whole tenant document to answer for one table. On a Postgres-backed
1605
+ runtime `get_entry` reads `value -> table_key` instead, so a 1,168-row stored table no longer
1606
+ costs a ~60-100 MB read. β›” ONLY A CLASS THAT DEFINES `get_entry` QUALIFIES (`getattr_static`),
1607
+ never one that FORWARDS it: `_Lent`, the external-API draft runtime and the other snapshot
1608
+ wrappers serve documents of their own and pass the rest through `__getattr__`, so a plain
1609
+ `getattr` would read the live store past the snapshot (or the drafted rows) they promised.
1610
+ Anything else (no capability, an absent entry, a failed read) falls back to `get` unchanged.
1611
+ """
1612
+ if (st is not None and not isinstance(st, _Lent)
1613
+ and callable(inspect.getattr_static(type(st), 'get_entry', None))):
1614
+ try:
1615
+ one = st.get_entry(STORE_KEY, str(table_key))
1616
+ except Exception: # noqa: BLE001
1617
+ one = None
1618
+ if isinstance(one, dict):
1619
+ return one
1620
+ return get(table_key, st)
1621
+
1622
+
1623
  # --- DURABLE MAP ZONES -------------------------------------------------------
1624
  # Zones intentionally do not live in a view config: a view is a person's presentation of rows,
1625
  # while a named geographic boundary is tenant data used by several views and routes. Keeping a
 
1855
  if not label:
1856
  return None
1857
  existing = all_tables(st)
1858
+ if capped_table_count(existing) >= MAX_TABLES: # connector databases take no slot (D-3284)
1859
  return None
1860
  key = KEY_PREFIX + _slug(label)
1861
  from core import route_system
 
5225
  return out
5226
 
5227
 
5228
+ def database_source_group(key, source=None):
5229
+ """Default origin folder; a user's placement is applied by the Data index consumer."""
5230
+ from core.connector_spec import connector_of
5231
+
5232
+ connector = connector_of(key)
5233
+ if connector is None:
5234
+ # Connector registries can lag newly synced tables. Monid's ut_amazon_ keys
5235
+ # are Loopable-made, so only the connector-owned ut_amzn_ prefix belongs here.
5236
+ connector = next((name for prefix, name in (
5237
+ ('ut_amzn_', 'amazon'), ('ut_meta_', 'meta'), ('ut_odoo_', 'odoo'),
5238
+ ) if key.startswith(prefix)), None)
5239
+ if connector is None and (key in {'customer_data', 'product_data'}
5240
+ or str(source or '').lower() == 'odoo'):
5241
+ connector = 'odoo'
5242
+ return {'amazon': 'Amazon', 'meta': 'Meta Ads', 'odoo': 'Odoo'}.get(connector, 'Loopable')
5243
+
5244
+
5245
  def nav_entries(viewer=None, is_admin=False, st=None):
5246
  """Registry-SHAPED dicts so the nav can render user tables beside real modules with no
5247
  special-casing: the same keys the flat nav reads (`key`, `label`, `source`).
 
5271
  if route_system.is_managed(t) or route_system.is_managed(key):
5272
  continue
5273
  source = str(t.get('source') or BLANK_SOURCE)
5274
+ source_group = database_source_group(key, source)
 
5275
  out.append({'key': key, 'label': t.get('label') or key,
5276
  'source': source, 'sourceGroup': source_group,
5277
  'user_table': True, 'nav': True, 'validate': False})
platform/core/users.py CHANGED
@@ -566,6 +566,15 @@ def registry():
566
  return store.get('users')
567
 
568
 
 
 
 
 
 
 
 
 
 
569
  def ensure_bootstrap():
570
  """Seed an 'admin' account from APP_PASSWORD ONLY on a truly fresh store (no users file yet).
571
 
@@ -700,6 +709,7 @@ def verify(username, pw):
700
 
701
  ``APP_PASSWORD`` may have seeded the first admin record, but it is never consulted here. A
702
  missing/unreachable registry therefore fails closed instead of minting an MFA-exempt identity.
 
703
  """
704
  username = (username or '').strip().lower()
705
  if not username or not pw:
@@ -707,8 +717,8 @@ def verify(username, pw):
707
  try:
708
  # read fresh so accounts created moments ago (UI or out-of-band) are recognised at once
709
  reg = store.get('users', fresh=True)
710
- except Exception:
711
- reg = {}
712
  username, u = _resolve_identifier(reg, username)
713
  # β›” KEY FIRST, EMAIL SECOND, AND SINCE WAVE 37 THAT ORDER IS A DECISION RATHER THAN AN ACCIDENT.
714
  # R8 lets a username BE an email address, so a typed string can now match a registry KEY and a
 
566
  return store.get('users')
567
 
568
 
569
+ class RegistryUnavailable(RuntimeError):
570
+ """The account registry could not be READ, which is not the same fact as "no such account".
571
+
572
+ HOTFIX 2026-09-28: `verify` used to fold a failed read into `{}`, so a busy store (a
573
+ `PoolTimeout` on staging) answered a CORRECT password with "that username and password do
574
+ not match" and charged the attempt to the lockout budget. Still fail-closed: the caller
575
+ answers 503, it never admits anyone."""
576
+
577
+
578
  def ensure_bootstrap():
579
  """Seed an 'admin' account from APP_PASSWORD ONLY on a truly fresh store (no users file yet).
580
 
 
709
 
710
  ``APP_PASSWORD`` may have seeded the first admin record, but it is never consulted here. A
711
  missing/unreachable registry therefore fails closed instead of minting an MFA-exempt identity.
712
+ An unreadable registry raises `RegistryUnavailable` rather than reading as an unknown user.
713
  """
714
  username = (username or '').strip().lower()
715
  if not username or not pw:
 
717
  try:
718
  # read fresh so accounts created moments ago (UI or out-of-band) are recognised at once
719
  reg = store.get('users', fresh=True)
720
+ except Exception as exc:
721
+ raise RegistryUnavailable(f'{type(exc).__name__}: {exc}') from exc
722
  username, u = _resolve_identifier(reg, username)
723
  # β›” KEY FIRST, EMAIL SECOND, AND SINCE WAVE 37 THAT ORDER IS A DECISION RATHER THAN AN ACCIDENT.
724
  # R8 lets a username BE an email address, so a typed string can now match a registry KEY and a
platform/harness/amazon_client.py CHANGED
@@ -21,19 +21,19 @@ disregards AWS SigV4 entirely and authorizes on the LWA access token alone: a pl
21
  signing requests is describing a dead API.
22
  Β· https://developer-docs.amazon.com/sp-api/changelog/sp-api-will-no-longer-require-aws-iam-or-aws-signature-version-4
23
 
24
- β›” **READ-ONLY, ENFORCED HERE, WITH EXACTLY TWO NAMED EXCEPTIONS.** `_request` refuses any method but
25
  GET before it builds a request. SP-API can create feeds, send buyer messages and adjust listings;
26
  none of that is reachable from this module, and the refusal sits below every route and payload so no
27
  caller can opt out.
28
 
29
- ⚠ **The exceptions are `_create_report` and `_competitive_summary_batch`, and both are declared
30
  rather than hidden.** Asking Amazon to BUILD a report is a POST, and Product Pricing's
31
- `getCompetitiveSummary` is a POST-wrapped batch of read-only GET subrequests. Both functions have
32
  their path as a module constant rather than a parameter, so no caller can redirect either one. They
33
  cannot alter a listing, price, order or inventory level, and cannot send a buyer anything. Full
34
  reasoning sits on their path constants.
35
 
36
- β›” **THE BULK PATH IS THE REPORTS API, NEVER PAGE-BY-PAGE.** SP-API is token-bucket rate limited per
37
  operation per selling-partner/application pair, and Amazon treats throttling as evidence of a wrong
38
  call pattern rather than a wrong ceiling. Standing rule 1 forbids a silent cap on a connector's
39
  records, and its second half is what applies: a limit that genuinely cannot be removed must be
@@ -47,8 +47,10 @@ from __future__ import annotations
47
  import contextvars
48
  import json
49
  import random
 
50
  import threading
51
  import time
 
52
  import urllib.error
53
  import urllib.parse
54
  import urllib.request
@@ -100,6 +102,62 @@ _report_pacer = {
100
  _entity_pacers = {
101
  "competitive_summary": {"last": None, "lock": threading.Lock()},
102
  "awd_inventory": {"last": None, "lock": threading.Lock()},
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
103
  }
104
 
105
  #: ⚠ MUST MATCH `core.keychain.ENTRY_TYPES` OR `add_entry` 400s with a message about our data
@@ -179,6 +237,9 @@ CREDENTIAL_FIELDS = ("client_id", "client_secret", "refresh_token", "seller_id",
179
  #: production host, whereas a production credential wrongly marked sandbox would quietly read mocks.
180
  OPTIONAL_FIELDS = ("sandbox", "region")
181
 
 
 
 
182
  #: ⭐ THE SCHEMA SPINE (owner rulings R4 + R14: seven tables, and NOTHING ELSE). Each row names
183
  #: the table, the bulk report
184
  #: type that fills it, and the summary aggregate its `validate()` reconciles against.
@@ -230,6 +291,7 @@ REPORT_TYPES = {
230
  },
231
  "listings": {
232
  "report": "GET_MERCHANT_LISTINGS_ALL_DATA",
 
233
  "validates_against": "listing report row count vs our listing rows",
234
  "role": "Product Listing",
235
  },
@@ -254,6 +316,8 @@ REPORT_TYPES = {
254
  },
255
  "inventory": {
256
  "report": "GET_LEDGER_SUMMARY_VIEW_DATA",
 
 
257
  "entity_read": "AWD v2024-05-09 listInventory (all 200-row pages)",
258
  "validates_against": "FBA ledger summary vs detail and AWD API items vs stored AWD rows",
259
  "role": "Amazon Fulfillment / Amazon Warehousing and Distribution",
@@ -273,6 +337,7 @@ REPORT_TYPES = {
273
  # page loop over a full catalogue is precisely what rule 1 forbids.
274
  "pricing": {
275
  "report": "GET_SALES_AND_TRAFFIC_REPORT",
 
276
  "report_covers": "buy box SHARE only; PRICE has no bulk path",
277
  "price_path": "Product Pricing v2022-05-01 getCompetitiveSummary (batch 20, 0.033/sec)",
278
  "validates_against": "sales and traffic report totals and competitive ASIN responses",
@@ -283,11 +348,630 @@ REPORT_TYPES = {
283
  # wave's one UNPROVEN ticket with its exit condition written in advance: returns >0 rows
284
  # for any registered brand, else demote to a declared-but-unsynced stub.
285
  "report": "GET_BRAND_ANALYTICS_SEARCH_TERMS_REPORT",
 
286
  "validates_against": "report period totals vs our search-term rows",
287
  "role": "Brand Analytics",
288
  },
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
289
  }
290
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
291
  #: Eight of the ten roles Amazon approved on 2026-07-22. Buyer Communication and Buyer Solicitation
292
  #: are REFUSED (owner ruling R7): both are write-shaped and PII-bearing, and it is precisely those
293
  #: restricted roles that trigger Phase 1 verification, a Phase 2 data-security assessment and an
@@ -312,13 +996,14 @@ REPORT_ROLES = {
312
  }
313
  _REPORT_ROLE_INDEX = {"inventory": 0, "pricing": 1}
314
  _BULK_REPORT_ROLES = tuple(dict.fromkeys(
315
- roles[_REPORT_ROLE_INDEX.get(table, 0)] for table, roles in REPORT_ROLES.items()))
 
316
  _APPROVED_REPORT_TYPES = tuple(dict.fromkeys(
317
  spec[key] for spec in REPORT_TYPES.values() for key in ("report", "report_incremental")
318
  if spec.get(key)))
319
 
320
  #: Exact method/path pairs. Braced ids match ONE opaque path segment, never a suffix, query,
321
- #: escaped slash or traversal. The two POSTs only request a report or wrap fixed read-only GETs.
322
  #: LWA, presigned documents and the schema module's public GitHub refresh are not SP-API
323
  #: operations. The gate enumerates those transports separately; they confer no operation scope.
324
  OPERATIONS = {
@@ -353,6 +1038,22 @@ OPERATIONS = {
353
  "name": "listInventory", "roles": ("Amazon Warehousing and Distribution",),
354
  "method": "GET", "path": "awd/2024-05-09/inventory",
355
  },
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
356
  }
357
 
358
  #: Answer "what does this approved role let this connector read?" directly from data.
@@ -444,6 +1145,26 @@ class AmazonError(RuntimeError):
444
  """Any refusal from LWA or SP-API, already stripped of anything secret."""
445
 
446
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
447
  class AmazonOperationRefused(AmazonError):
448
  """A local scope refusal, never a throttle and never eligible for throttling retries."""
449
 
@@ -497,6 +1218,8 @@ def operation_for(method, path):
497
 
498
  def require_operation(method, path, *, report_type=None):
499
  """Fail closed before constructing any URL or touching a transport."""
 
 
500
  operation = operation_for(method, path)
501
  if operation is None:
502
  raise AmazonOperationRefused(f"{str(method).upper()} {_safe(path)}")
@@ -564,6 +1287,252 @@ def _pace_entity(name, interval, *, sleep=time.sleep, now=time.monotonic):
564
  return wait
565
 
566
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
567
  def _post_form(url, data, timeout=_TIMEOUT):
568
  # LWA exchanges credentials at api.amazon.com, not an SP-API operation. Its sole caller fixes
569
  # LWA_TOKEN_URL; the connector-wide gate checks that call separately from OPERATIONS.
@@ -850,6 +1819,9 @@ def _create_report(token, report_type, marketplace_ids, region=DEFAULT_REGION,
850
  is that this function can reach exactly one operation and a caller cannot redirect it.
851
  """
852
  require_operation("POST", _REPORTS_CREATE_PATH, report_type=str(report_type))
 
 
 
853
 
854
  def attempt():
855
  """One raw attempt at the declared report-creation POST."""
@@ -877,10 +1849,12 @@ def _create_report(token, report_type, marketplace_ids, region=DEFAULT_REGION,
877
  except urllib.error.HTTPError as e:
878
  detail = ""
879
  try:
880
- errs = (json.loads((e.read() or b"").decode("utf-8") or "{}").get("errors") or [])
881
- detail = str((errs[0] or {}).get("message") or "")[:200] if errs else ""
 
 
882
  except Exception: # noqa: BLE001
883
- detail = ""
884
  if e.code == 429:
885
  raise AmazonThrottled(
886
  f"{_safe(url)} was throttled (429) creating a {report_type} report. Report "
@@ -891,8 +1865,11 @@ def _create_report(token, report_type, marketplace_ids, region=DEFAULT_REGION,
891
  raise AmazonError(
892
  f"{_safe(url)} refused a {report_type} report (403). That is a ROLE answer: this "
893
  "report needs a role the application was not granted") from None
 
 
 
894
  raise AmazonError(f"{_safe(url)} answered {e.code}"
895
- + (f": {detail}" if detail else "")) from None
896
  except Exception as e: # noqa: BLE001
897
  raise AmazonError(f"{_safe(url)} unreachable: {type(e).__name__}") from None
898
  rid = str(got.get("reportId") or "")
@@ -954,7 +1931,41 @@ def report_document(fields, report_type, start=None, end=None, options=None,
954
  doc_id = str(got.get("reportDocumentId") or "")
955
  break
956
  if state in _REPORT_DEAD:
957
- raise AmazonError(f"Amazon ended the {report_type} report as {state} (reportId {rid})")
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
958
  time.sleep(poll_seconds)
959
  waited += poll_seconds
960
  # β›” The token is ~1 hour and a long report can outlive it. Re-mint rather than 403.
@@ -973,6 +1984,8 @@ def report_document(fields, report_type, start=None, end=None, options=None,
973
  text = _download_document(url, str(meta.get("compressionAlgorithm") or ""))
974
  say(f" {report_type}: {len(text):,} chars")
975
  return text, {"reportId": rid, "reportDocumentId": doc_id,
 
 
976
  "compression": meta.get("compressionAlgorithm")}
977
 
978
 
@@ -1039,6 +2052,31 @@ def settlement_document(fields, document_id, *, timeout=_TIMEOUT):
1039
  return _download_document(url, str(meta.get("compressionAlgorithm") or ""))
1040
 
1041
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1042
  def _download_document(url, compression="", timeout=120):
1043
  """Fetch one presigned report document and decode it. -> text.
1044
 
@@ -1057,12 +2095,20 @@ def _download_document(url, compression="", timeout=120):
1057
  raw = r.read()
1058
  except Exception as e: # noqa: BLE001
1059
  raise AmazonError(f"{_safe(url)} unreachable: {type(e).__name__}") from None
 
 
 
 
 
 
1060
  if str(compression).upper() == "GZIP":
1061
  import gzip
1062
  try:
1063
  raw = gzip.decompress(raw)
1064
  except Exception as e: # noqa: BLE001
1065
  raise AmazonError(f"the report document did not decompress: {type(e).__name__}") from None
 
 
1066
  for enc in ("utf-8", "cp1252"):
1067
  try:
1068
  return raw.decode(enc)
 
21
  signing requests is describing a dead API.
22
  Β· https://developer-docs.amazon.com/sp-api/changelog/sp-api-will-no-longer-require-aws-iam-or-aws-signature-version-4
23
 
24
+ β›” **READ-ONLY, ENFORCED HERE, WITH EXACTLY THREE NAMED EXCEPTIONS.** `_request` refuses any method but
25
  GET before it builds a request. SP-API can create feeds, send buyer messages and adjust listings;
26
  none of that is reachable from this module, and the refusal sits below every route and payload so no
27
  caller can opt out.
28
 
29
+ ⚠ **The exceptions are `_create_report`, `_competitive_summary_batch` and `_create_query`, all declared
30
  rather than hidden.** Asking Amazon to BUILD a report is a POST, and Product Pricing's
31
+ `getCompetitiveSummary` wraps read-only GETs, and Data Kiosk generates a fixed economics query. These functions have
32
  their path as a module constant rather than a parameter, so no caller can redirect either one. They
33
  cannot alter a listing, price, order or inventory level, and cannot send a buyer anything. Full
34
  reasoning sits on their path constants.
35
 
36
+ β›” **REPORTS AND DATA KIOSK ARE THE BULK PATHS; FINANCES FOLLOWS EVERY PAGE.** SP-API is token-bucket rate limited per
37
  operation per selling-partner/application pair, and Amazon treats throttling as evidence of a wrong
38
  call pattern rather than a wrong ceiling. Standing rule 1 forbids a silent cap on a connector's
39
  records, and its second half is what applies: a limit that genuinely cannot be removed must be
 
47
  import contextvars
48
  import json
49
  import random
50
+ import re
51
  import threading
52
  import time
53
+ from datetime import date, datetime, timezone, timedelta
54
  import urllib.error
55
  import urllib.parse
56
  import urllib.request
 
102
  _entity_pacers = {
103
  "competitive_summary": {"last": None, "lock": threading.Lock()},
104
  "awd_inventory": {"last": None, "lock": threading.Lock()},
105
+ "transactions": {"last": None, "lock": threading.Lock()},
106
+ "getOrderMetrics": {"last": None, "lock": threading.Lock()},
107
+ "getInventorySummaries": {"last": None, "lock": threading.Lock()},
108
+ "searchListingsItems": {"last": None, "lock": threading.Lock()},
109
+ "searchCatalogItems": {"last": None, "lock": threading.Lock()},
110
+ }
111
+
112
+ # Data Kiosk has its own quota, independent of createReport.
113
+ QUERY_BURST = 15
114
+ QUERY_SUSTAINED_SECONDS = 60.0
115
+ _query_pacer = {"tokens": float(QUERY_BURST), "updated": None, "lock": threading.Lock()}
116
+ _QUERIES_CREATE_PATH = "dataKiosk/2023-11-15/queries"
117
+ _TRANSACTIONS_PATH = "finances/2024-06-19/transactions"
118
+ TRANSACTIONS_INTERVAL = 2.0
119
+
120
+ # Selection pinned to Amazon's published analytics_economics_2024_03_15.graphql,
121
+ # read 2026-09-25. RANGE avoids MONTH's automatic rounding of partial months.
122
+ # Only dates, a validated marketplace ID and one of two fixed grains are substituted.
123
+ _ECONOMICS_QUERY = '''query {
124
+ analytics_economics_2024_03_15 {
125
+ economics(startDate: "__START__", endDate: "__END__",
126
+ marketplaceIds: ["__MARKETPLACE__"], aggregateBy: {date: __DATE__, productId: __PRODUCT__}) {
127
+ startDate endDate marketplaceId msku fnsku childAsin parentAsin
128
+ sales {
129
+ orderedProductSales { amount currencyCode } unitsOrdered
130
+ refundedProductSales { amount currencyCode } unitsRefunded
131
+ netProductSales { amount currencyCode } netUnitsSold
132
+ averageSellingPrice { amount currencyCode }
133
+ }
134
+ fees { feeTypeName charges { startDate endDate identifier aggregatedDetail {
135
+ amount { amount currencyCode } totalAmount { amount currencyCode }
136
+ amountPerUnit { amount currencyCode } amountPerUnitDelta { amount currencyCode }
137
+ promotionAmount { amount currencyCode } taxAmount { amount currencyCode } quantity
138
+ } } }
139
+ ads { adTypeName charge {
140
+ amount { amount currencyCode } totalAmount { amount currencyCode }
141
+ amountPerUnit { amount currencyCode } promotionAmount { amount currencyCode }
142
+ taxAmount { amount currencyCode } quantity
143
+ } }
144
+ netProceeds { total { amount currencyCode } perUnit { amount currencyCode } }
145
+ cost {
146
+ costOfGoodsSold { amount currencyCode } miscellaneousCost { amount currencyCode }
147
+ fbaCost { shippingToAmazonCost { amount currencyCode } }
148
+ mfnCost { fulfillmentCost { amount currencyCode } storageCost { amount currencyCode } }
149
+ }
150
+ }
151
+ }
152
+ }'''
153
+ _ECONOMICS_TEMPLATES = {
154
+ False: _ECONOMICS_QUERY.replace("__DATE__", "DAY").replace("__PRODUCT__", "MSKU"),
155
+ # S66-T40 (lead, measured live 2026-09-25, window 2026-09-20..24): RANGE x MSKU equals the
156
+ # sum of our DAY x MSKU rows to the cent (fees 38,009.94, net 38,474.335), while ANY
157
+ # PARENT_ASIN grain carries 37.75 of fees Amazon attributes to no MSKU. The reconciliation
158
+ # checks OUR date roll-up at the SAME product grain; comparing across grains refused a
159
+ # correct transform.
160
+ True: _ECONOMICS_QUERY.replace("__DATE__", "RANGE").replace("__PRODUCT__", "MSKU"),
161
  }
162
 
163
  #: ⚠ MUST MATCH `core.keychain.ENTRY_TYPES` OR `add_entry` 400s with a message about our data
 
237
  #: production host, whereas a production credential wrongly marked sandbox would quietly read mocks.
238
  OPTIONAL_FIELDS = ("sandbox", "region")
239
 
240
+ SQP_ASIN_OPTION_MAX_CHARS = 200
241
+ SQP_PERIOD = "MONTH"
242
+
243
  #: ⭐ THE SCHEMA SPINE (owner rulings R4 + R14: seven tables, and NOTHING ELSE). Each row names
244
  #: the table, the bulk report
245
  #: type that fills it, and the summary aggregate its `validate()` reconciles against.
 
291
  },
292
  "listings": {
293
  "report": "GET_MERCHANT_LISTINGS_ALL_DATA",
294
+ "snapshot": True,
295
  "validates_against": "listing report row count vs our listing rows",
296
  "role": "Product Listing",
297
  },
 
316
  },
317
  "inventory": {
318
  "report": "GET_LEDGER_SUMMARY_VIEW_DATA",
319
+ "snapshot": True,
320
+ "options": {"aggregateByTimePeriod": "DAILY", "aggregateByLocation": "COUNTRY"},
321
  "entity_read": "AWD v2024-05-09 listInventory (all 200-row pages)",
322
  "validates_against": "FBA ledger summary vs detail and AWD API items vs stored AWD rows",
323
  "role": "Amazon Fulfillment / Amazon Warehousing and Distribution",
 
337
  # page loop over a full catalogue is precisely what rule 1 forbids.
338
  "pricing": {
339
  "report": "GET_SALES_AND_TRAFFIC_REPORT",
340
+ "json_root": "salesAndTrafficByAsin",
341
  "report_covers": "buy box SHARE only; PRICE has no bulk path",
342
  "price_path": "Product Pricing v2022-05-01 getCompetitiveSummary (batch 20, 0.033/sec)",
343
  "validates_against": "sales and traffic report totals and competitive ASIN responses",
 
348
  # wave's one UNPROVEN ticket with its exit condition written in advance: returns >0 rows
349
  # for any registered brand, else demote to a declared-but-unsynced stub.
350
  "report": "GET_BRAND_ANALYTICS_SEARCH_TERMS_REPORT",
351
+ "json_root": "dataByDepartmentAndSearchTerm",
352
  "validates_against": "report period totals vs our search-term rows",
353
  "role": "Brand Analytics",
354
  },
355
+ "economics": {
356
+ "api": "createQuery",
357
+ "validates_against": "DAY x MSKU sales, fees and net proceeds vs Amazon RANGE x MSKU totals per currency",
358
+ "role": "Selling Partner Insights",
359
+ },
360
+ "transactions": {
361
+ "api": "listTransactions",
362
+ "validates_against": "Amazon transaction count and totalAmount per currency vs stored items, and breakdowns vs transaction totals",
363
+ "role": "Finance and Accounting",
364
+ },
365
+ # snapshot forbids dataStartTime/dataEndTime; min_interval_s is a successful
366
+ # refresh floor (also enforced on manual sync). Monthly history is separate.
367
+ "fba_fee_estimates": {
368
+ "report": "GET_FBA_ESTIMATED_FBA_FEES_TXT_DATA", "snapshot": True,
369
+ "min_interval_s": 86400,
370
+ "validates_against": "Amazon document data-row count vs stored rows for its latest snapshot date",
371
+ "role": "Amazon Fulfillment",
372
+ },
373
+ "referral_fees": {
374
+ "report": "GET_REFERRAL_FEE_PREVIEW_REPORT", "snapshot": True,
375
+ "min_interval_s": 86400,
376
+ "validates_against": "Amazon document data-row count vs stored rows for its latest snapshot date",
377
+ "role": "Finance and Accounting",
378
+ },
379
+ "storage_fees": {
380
+ "report": "GET_FBA_STORAGE_FEE_CHARGES_DATA", "monthly": True,
381
+ "min_interval_s": 30 * 86400,
382
+ "validates_against": "Amazon document data-row count vs stored rows for its latest calendar month",
383
+ "role": "Amazon Fulfillment",
384
+ },
385
+ "search_query_performance": {
386
+ "report": "GET_BRAND_ANALYTICS_SEARCH_QUERY_PERFORMANCE_REPORT",
387
+ "role": "Brand Analytics",
388
+ "validates_against": "Amazon dataByAsin row count and ASIN purchases per batch vs stored rows for the latest month",
389
+ "json_root": "dataByAsin", "monthly": True,
390
+ "period": SQP_PERIOD, "asin_option_max_chars": SQP_ASIN_OPTION_MAX_CHARS,
391
+ },
392
+ }
393
+
394
+ # S66-T42: inventory sections 2.2 and 2.3. Fields are measured at sync, including
395
+ # JSON documents. Row roots were read from amzn/selling-partner-api-models/schemas/reports
396
+ # on 2026-09-25 at commit 713565ff394d136629a342120872e65cb073d162.
397
+ # json_schema names the public source, without changing the pinned proto.
398
+ GROUP_A_TABLES = {
399
+ "long_term_storage_fees": {
400
+ "label": "Amazon long term storage fees",
401
+ "report": "GET_FBA_FULFILLMENT_LONGTERM_STORAGE_FEE_CHARGES_DATA",
402
+ "role": "Amazon Fulfillment",
403
+ "min_interval_s": 14400,
404
+ "cadence_note": "Complete calendar months; refresh no sooner than 14400 seconds.",
405
+ "monthly": True,
406
+ "validates_against": "Amazon document or API data-row count vs stored rows for the latest period",
407
+ },
408
+ "storage_overage_fees": {
409
+ "label": "Amazon storage overage fees",
410
+ "report": "GET_FBA_OVERAGE_FEE_CHARGES_DATA",
411
+ "role": "Amazon Fulfillment",
412
+ "min_interval_s": 14400,
413
+ "cadence_note": "Complete calendar months; refresh no sooner than 14400 seconds.",
414
+ "monthly": True,
415
+ "validates_against": "Amazon document or API data-row count vs stored rows for the latest period",
416
+ },
417
+ "reimbursements": {
418
+ "label": "Amazon reimbursements",
419
+ "report": "GET_FBA_REIMBURSEMENTS_DATA",
420
+ "role": "Amazon Fulfillment",
421
+ "min_interval_s": 14400,
422
+ "cadence_note": "Date windows; refresh no sooner than 14400 seconds.",
423
+ "validates_against": "Amazon document or API data-row count vs stored rows for the latest period",
424
+ },
425
+ "fba_customer_returns": {
426
+ "label": "Amazon FBA customer returns",
427
+ "report": "GET_FBA_FULFILLMENT_CUSTOMER_RETURNS_DATA",
428
+ "role": "Amazon Fulfillment",
429
+ "min_interval_s": 14400,
430
+ "cadence_note": "Date windows; refresh no sooner than 14400 seconds.",
431
+ "validates_against": "Amazon document or API data-row count vs stored rows for the latest period",
432
+ },
433
+ "replacements": {
434
+ "label": "Amazon replacements",
435
+ "report": "GET_FBA_FULFILLMENT_CUSTOMER_SHIPMENT_REPLACEMENT_DATA",
436
+ "role": "Amazon Fulfillment",
437
+ "min_interval_s": 14400,
438
+ "cadence_note": "Date windows; refresh no sooner than 14400 seconds.",
439
+ "validates_against": "Amazon document or API data-row count vs stored rows for the latest period",
440
+ },
441
+ "fba_shipped_sales": {
442
+ "label": "Amazon FBA shipped sales",
443
+ "report": "GET_FBA_FULFILLMENT_CUSTOMER_SHIPMENT_SALES_DATA",
444
+ "role": "Amazon Fulfillment",
445
+ "min_interval_s": 1800,
446
+ "cadence_note": "Date windows; refresh no sooner than 1800 seconds.",
447
+ "validates_against": "Amazon document or API data-row count vs stored rows for the latest period",
448
+ },
449
+ "fba_shipment_promotions": {
450
+ "label": "Amazon FBA shipment promotions",
451
+ "report": "GET_FBA_FULFILLMENT_CUSTOMER_SHIPMENT_PROMOTION_DATA",
452
+ "role": "Amazon Fulfillment",
453
+ "min_interval_s": 14400,
454
+ "cadence_note": "Date windows; refresh no sooner than 14400 seconds.",
455
+ "validates_against": "Amazon document or API data-row count vs stored rows for the latest period",
456
+ },
457
+ "inventory_ledger_detail": {
458
+ "label": "Amazon inventory ledger detail",
459
+ "report": "GET_LEDGER_DETAIL_VIEW_DATA",
460
+ "role": "Amazon Fulfillment",
461
+ "min_interval_s": 1800,
462
+ "cadence_note": "Date windows; refresh no sooner than 1800 seconds.",
463
+ "validates_against": "Amazon document or API data-row count vs stored rows for the latest period",
464
+ },
465
+ "inventory_planning": {
466
+ "label": "Amazon inventory planning",
467
+ "report": "GET_FBA_INVENTORY_PLANNING_DATA",
468
+ "role": "Amazon Fulfillment",
469
+ "min_interval_s": 14400,
470
+ "cadence_note": "Current snapshot; refresh no sooner than 14400 seconds.",
471
+ "snapshot": True,
472
+ "validates_against": "Amazon document or API data-row count vs stored rows for the latest period",
473
+ },
474
+ "manage_inventory": {
475
+ "label": "Amazon manage inventory",
476
+ "report": "GET_FBA_MYI_UNSUPPRESSED_INVENTORY_DATA",
477
+ "role": "Amazon Fulfillment",
478
+ "min_interval_s": 1800,
479
+ "cadence_note": "Current snapshot; refresh no sooner than 1800 seconds.",
480
+ "snapshot": True,
481
+ "validates_against": "Amazon document or API data-row count vs stored rows for the latest period",
482
+ },
483
+ "reserved_inventory": {
484
+ "label": "Amazon reserved inventory",
485
+ "report": "GET_RESERVED_INVENTORY_DATA",
486
+ "role": "Amazon Fulfillment",
487
+ "min_interval_s": 1800,
488
+ "cadence_note": "Current snapshot; refresh no sooner than 1800 seconds.",
489
+ "snapshot": True,
490
+ "validates_against": "Amazon document or API data-row count vs stored rows for the latest period",
491
+ },
492
+ "restock_recommendations": {
493
+ "label": "Amazon restock recommendations",
494
+ "report": "GET_RESTOCK_INVENTORY_RECOMMENDATIONS_REPORT",
495
+ "role": "Amazon Fulfillment",
496
+ "min_interval_s": 1800,
497
+ "cadence_note": "Current snapshot; refresh no sooner than 1800 seconds.",
498
+ "snapshot": True,
499
+ "validates_against": "Amazon document or API data-row count vs stored rows for the latest period",
500
+ },
501
+ "stranded_inventory": {
502
+ "label": "Amazon stranded inventory",
503
+ "report": "GET_STRANDED_INVENTORY_UI_DATA",
504
+ "role": "Amazon Fulfillment",
505
+ "min_interval_s": 1800,
506
+ "cadence_note": "Current snapshot; refresh no sooner than 1800 seconds.",
507
+ "snapshot": True,
508
+ "validates_against": "Amazon document or API data-row count vs stored rows for the latest period",
509
+ },
510
+ "inbound_noncompliance": {
511
+ "label": "Amazon inbound noncompliance",
512
+ "report": "GET_FBA_FULFILLMENT_INBOUND_NONCOMPLIANCE_DATA",
513
+ "role": "Amazon Fulfillment",
514
+ "min_interval_s": 14400,
515
+ "cadence_note": "Date windows; refresh no sooner than 14400 seconds.",
516
+ "validates_against": "Amazon document or API data-row count vs stored rows for the latest period",
517
+ },
518
+ "inventory_expiration_dates": {
519
+ "label": "Amazon inventory expiration dates",
520
+ "report": "GET_FBA_FULFILLMENT_INVENTORY_EXPIRATION_DATE_DATA",
521
+ "role": "Amazon Fulfillment",
522
+ "min_interval_s": 14400,
523
+ "cadence_note": "Current snapshot; refresh no sooner than 14400 seconds.",
524
+ "snapshot": True,
525
+ "validates_against": "Amazon document or API data-row count vs stored rows for the latest period",
526
+ },
527
+ "recommended_removals": {
528
+ "label": "Amazon recommended removals",
529
+ "report": "GET_FBA_RECOMMENDED_REMOVAL_DATA",
530
+ "role": "Amazon Fulfillment",
531
+ "min_interval_s": 14400,
532
+ "cadence_note": "Current snapshot; refresh no sooner than 14400 seconds.",
533
+ "snapshot": True,
534
+ "validates_against": "Amazon document or API data-row count vs stored rows for the latest period",
535
+ },
536
+ "removal_orders": {
537
+ "label": "Amazon removal orders",
538
+ "report": "GET_FBA_FULFILLMENT_REMOVAL_ORDER_DETAIL_DATA",
539
+ "role": "Amazon Fulfillment",
540
+ "min_interval_s": 1800,
541
+ "cadence_note": "Date windows; refresh no sooner than 1800 seconds.",
542
+ "validates_against": "Amazon document or API data-row count vs stored rows for the latest period",
543
+ },
544
+ "removal_shipments": {
545
+ "label": "Amazon removal shipments",
546
+ "report": "GET_FBA_FULFILLMENT_REMOVAL_SHIPMENT_DETAIL_DATA",
547
+ "role": "Amazon Fulfillment",
548
+ "min_interval_s": 1800,
549
+ "cadence_note": "Date windows; refresh no sooner than 1800 seconds.",
550
+ "validates_against": "Amazon document or API data-row count vs stored rows for the latest period",
551
+ },
552
+ "subscribe_save_forecast": {
553
+ "label": "Amazon Subscribe and Save forecast",
554
+ "report": "GET_FBA_SNS_FORECAST_DATA",
555
+ "role": "Amazon Fulfillment",
556
+ "min_interval_s": 14400,
557
+ "cadence_note": "Current snapshot; refresh no sooner than 14400 seconds.",
558
+ "snapshot": True,
559
+ "validates_against": "Amazon document or API data-row count vs stored rows for the latest period",
560
+ },
561
+ "subscribe_save_performance": {
562
+ "label": "Amazon Subscribe and Save performance",
563
+ "report": "GET_FBA_SNS_PERFORMANCE_DATA",
564
+ "role": "Amazon Fulfillment",
565
+ "min_interval_s": 14400,
566
+ "cadence_note": "Date windows; refresh no sooner than 14400 seconds.",
567
+ "validates_against": "Amazon document or API data-row count vs stored rows for the latest period",
568
+ },
569
+ "remote_fulfillment_eligibility": {
570
+ "label": "Amazon remote fulfillment eligibility",
571
+ "report": "GET_REMOTE_FULFILLMENT_ELIGIBILITY",
572
+ "role": "Amazon Fulfillment",
573
+ "min_interval_s": 86400,
574
+ "cadence_note": "Current snapshot; refresh no sooner than 86400 seconds.",
575
+ "snapshot": True,
576
+ "validates_against": "Amazon document or API data-row count vs stored rows for the latest period",
577
+ },
578
+ "search_catalog_performance": {
579
+ "label": "Amazon search catalog performance",
580
+ "report": "GET_BRAND_ANALYTICS_SEARCH_CATALOG_PERFORMANCE_REPORT",
581
+ "role": "Brand Analytics",
582
+ "min_interval_s": 86400,
583
+ "cadence_note": "Complete calendar months; refresh no sooner than 86400 seconds.",
584
+ "monthly": True,
585
+ "json_root": "dataByAsin",
586
+ "json_schema": "sellingPartnerSearchCatalogPerformanceReport.json",
587
+ "period": "MONTH",
588
+ "validates_against": "Amazon document or API data-row count vs stored rows for the latest period",
589
+ },
590
+ "market_basket": {
591
+ "label": "Amazon market basket",
592
+ "report": "GET_BRAND_ANALYTICS_MARKET_BASKET_REPORT",
593
+ "role": "Brand Analytics",
594
+ "min_interval_s": 86400,
595
+ "cadence_note": "Complete calendar months; refresh no sooner than 86400 seconds.",
596
+ "monthly": True,
597
+ "json_root": "dataByAsin",
598
+ "json_schema": "sellingPartnerMarketBasketAnalysisReport.json",
599
+ "period": "MONTH",
600
+ "validates_against": "Amazon document or API data-row count vs stored rows for the latest period",
601
+ },
602
+ "repeat_purchase": {
603
+ "label": "Amazon repeat purchase",
604
+ "report": "GET_BRAND_ANALYTICS_REPEAT_PURCHASE_REPORT",
605
+ "role": "Brand Analytics",
606
+ "min_interval_s": 86400,
607
+ "cadence_note": "Complete calendar months; refresh no sooner than 86400 seconds.",
608
+ "monthly": True,
609
+ "json_root": "dataByAsin",
610
+ "json_schema": "sellingPartnerRepeatPurchaseReport.json",
611
+ "period": "MONTH",
612
+ "validates_against": "Amazon document or API data-row count vs stored rows for the latest period",
613
+ },
614
+ "promotion_performance": {
615
+ "label": "Amazon promotion performance",
616
+ "report": "GET_PROMOTION_PERFORMANCE_REPORT",
617
+ "role": "Selling Partner Insights",
618
+ "min_interval_s": 86400,
619
+ "cadence_note": "Date windows; refresh no sooner than 86400 seconds.",
620
+ "json_root": "promotions",
621
+ "json_schema": "promotionReport.json",
622
+ "date_options": ["promotionStartDateFrom","promotionStartDateTo"],
623
+ "validates_against": "Amazon document or API data-row count vs stored rows for the latest period",
624
+ },
625
+ "coupon_performance": {
626
+ "label": "Amazon coupon performance",
627
+ "report": "GET_COUPON_PERFORMANCE_REPORT",
628
+ "role": "Brand Analytics",
629
+ "min_interval_s": 86400,
630
+ "cadence_note": "Date windows; refresh no sooner than 86400 seconds.",
631
+ "json_root": "coupons",
632
+ "json_schema": "sellerCouponReport.json",
633
+ "date_options": ["couponStartDateFrom","couponStartDateTo"],
634
+ "validates_against": "Amazon document or API data-row count vs stored rows for the latest period",
635
+ },
636
+ "account_health": {
637
+ "label": "Amazon account health",
638
+ "report": "GET_V2_SELLER_PERFORMANCE_REPORT",
639
+ "role": "Selling Partner Insights",
640
+ "min_interval_s": 86400,
641
+ "cadence_note": "Current snapshot; refresh no sooner than 86400 seconds.",
642
+ "snapshot": True,
643
+ "json_root": "performanceMetrics",
644
+ "json_schema": "accountHealthReport-2020-11-18.json",
645
+ "json_extra_roots": ["accountStatuses"],
646
+ "validates_against": "Amazon document or API data-row count vs stored rows for the latest period",
647
+ },
648
+ "inactive_listings": {
649
+ "label": "Amazon inactive listings",
650
+ "report": "GET_MERCHANT_LISTINGS_INACTIVE_DATA",
651
+ "role": "Inventory and Order Tracking",
652
+ "min_interval_s": 14400,
653
+ "cadence_note": "Current snapshot; refresh no sooner than 14400 seconds.",
654
+ "snapshot": True,
655
+ "validates_against": "Amazon document or API data-row count vs stored rows for the latest period",
656
+ },
657
+ "cancelled_listings": {
658
+ "label": "Amazon cancelled listings",
659
+ "report": "GET_MERCHANT_CANCELLED_LISTINGS_DATA",
660
+ "role": "Inventory and Order Tracking",
661
+ "min_interval_s": 14400,
662
+ "cadence_note": "Current snapshot; refresh no sooner than 14400 seconds.",
663
+ "snapshot": True,
664
+ "validates_against": "Amazon document or API data-row count vs stored rows for the latest period",
665
+ },
666
+ "suppressed_listings": {
667
+ "label": "Amazon suppressed listings",
668
+ "report": "GET_MERCHANTS_LISTINGS_FYP_REPORT",
669
+ "role": "Product Listing",
670
+ "min_interval_s": 14400,
671
+ "cadence_note": "Current snapshot; refresh no sooner than 14400 seconds.",
672
+ "snapshot": True,
673
+ "validates_against": "Amazon document or API data-row count vs stored rows for the latest period",
674
+ },
675
+ "listing_defects": {
676
+ "label": "Amazon listing defects",
677
+ "report": "GET_MERCHANT_LISTINGS_DEFECT_DATA",
678
+ "role": "Inventory and Order Tracking",
679
+ "min_interval_s": 14400,
680
+ "cadence_note": "Current snapshot; refresh no sooner than 14400 seconds.",
681
+ "snapshot": True,
682
+ "validates_against": "Amazon document or API data-row count vs stored rows for the latest period",
683
+ },
684
+ "sold_listings": {
685
+ "label": "Amazon sold listings",
686
+ "report": "GET_CONVERGED_FLAT_FILE_SOLD_LISTINGS_DATA",
687
+ "role": "Inventory and Order Tracking",
688
+ "min_interval_s": 14400,
689
+ "cadence_note": "Current snapshot; refresh no sooner than 14400 seconds.",
690
+ "snapshot": True,
691
+ "validates_against": "Amazon document or API data-row count vs stored rows for the latest period",
692
+ },
693
+ "mfn_returns": {
694
+ "label": "Amazon MFN returns",
695
+ "report": "GET_FLAT_FILE_RETURNS_DATA_BY_RETURN_DATE",
696
+ "role": "Inventory and Order Tracking",
697
+ "min_interval_s": 14400,
698
+ "cadence_note": "Date windows; refresh no sooner than 14400 seconds.",
699
+ "max_window_days": 60,
700
+ "validates_against": "Amazon document or API data-row count vs stored rows for the latest period",
701
+ },
702
+ "b2b_recommended": {
703
+ "label": "Amazon B2B product opportunities",
704
+ "report": "GET_B2B_PRODUCT_OPPORTUNITIES_RECOMMENDED_FOR_YOU",
705
+ "role": "Product Listing",
706
+ "min_interval_s": 604800,
707
+ "cadence_note": "Current snapshot; refresh no sooner than 604800 seconds.",
708
+ "snapshot": True,
709
+ "json_root": "recommendations",
710
+ "json_schema": "b2bProductOpportunitiesRecommendedForYouReport-2020-11-19.json",
711
+ "validates_against": "Amazon document or API data-row count vs stored rows for the latest period",
712
+ },
713
+ "b2b_not_yet_on_amazon": {
714
+ "label": "Amazon B2B products not yet on Amazon",
715
+ "report": "GET_B2B_PRODUCT_OPPORTUNITIES_NOT_YET_ON_AMAZON",
716
+ "role": "Product Listing",
717
+ "min_interval_s": 604800,
718
+ "cadence_note": "Current snapshot; refresh no sooner than 604800 seconds.",
719
+ "snapshot": True,
720
+ "json_root": "recommendations",
721
+ "json_schema": "b2bProductOpportunitiesNotYetOnAmazonReport-2020-11-19.json",
722
+ "validates_against": "Amazon document or API data-row count vs stored rows for the latest period",
723
+ },
724
+ "geo_opportunities": {
725
+ "label": "Amazon geo opportunities",
726
+ "report": "GET_FLAT_FILE_GEO_OPPORTUNITIES",
727
+ "role": "Inventory and Order Tracking",
728
+ "min_interval_s": 86400,
729
+ "cadence_note": "Current snapshot; refresh no sooner than 86400 seconds.",
730
+ "snapshot": True,
731
+ "validates_against": "Amazon document or API data-row count vs stored rows for the latest period",
732
+ },
733
+ "out_of_stock_mfn": {
734
+ "label": "Amazon out of stock MFN listings",
735
+ "report": "GET_OUT_OF_STOCK_MFN_INFORMATION",
736
+ "role": "Inventory and Order Tracking",
737
+ "min_interval_s": 14400,
738
+ "cadence_note": "Current snapshot; refresh no sooner than 14400 seconds.",
739
+ "snapshot": True,
740
+ "validates_against": "Amazon document or API data-row count vs stored rows for the latest period",
741
+ },
742
+ "b2b_quotes": {
743
+ "label": "Amazon B2B quantity discounts",
744
+ "report": "RFQD_BULK_DOWNLOAD",
745
+ "role": "Product Listing",
746
+ "min_interval_s": 86400,
747
+ "cadence_note": "Current snapshot; refresh no sooner than 86400 seconds.",
748
+ "snapshot": True,
749
+ "validates_against": "Amazon document or API data-row count vs stored rows for the latest period",
750
+ },
751
+ "fee_discounts": {
752
+ "label": "Amazon fee discounts",
753
+ "report": "FEE_DISCOUNTS_REPORT",
754
+ "role": "Product Listing",
755
+ "min_interval_s": 86400,
756
+ "cadence_note": "Current snapshot; refresh no sooner than 86400 seconds.",
757
+ "snapshot": True,
758
+ "validates_against": "Amazon document or API data-row count vs stored rows for the latest period",
759
+ },
760
+ "order_metrics": {
761
+ "label": "Amazon order metrics",
762
+ "api": "getOrderMetrics",
763
+ "role": "Inventory and Order Tracking",
764
+ "min_interval_s": 1800,
765
+ "cadence_note": "Date windows; refresh no sooner than 1800 seconds.",
766
+ "validates_against": "Amazon daily order, unit and sales totals vs stored metrics; order counts independently reconcile orders",
767
+ },
768
+ "fba_inventory": {
769
+ "label": "Amazon FBA inventory summaries",
770
+ "api": "getInventorySummaries",
771
+ "role": "Amazon Fulfillment",
772
+ "min_interval_s": 1800,
773
+ "cadence_note": "Current snapshot; refresh no sooner than 1800 seconds.",
774
+ "snapshot": True,
775
+ "validates_against": "Amazon document or API data-row count vs stored rows for the latest period",
776
+ },
777
+ "listing_items": {
778
+ "label": "Amazon listing status and issues",
779
+ "api": "searchListingsItems",
780
+ "role": "Product Listing",
781
+ "min_interval_s": 1800,
782
+ "cadence_note": "Current snapshot; refresh no sooner than 1800 seconds.",
783
+ "snapshot": True,
784
+ "validates_against": "Amazon document or API data-row count vs stored rows for the latest period",
785
+ },
786
+ "catalog_items": {
787
+ "label": "Amazon catalog attributes",
788
+ "api": "searchCatalogItems",
789
+ "role": "Product Listing",
790
+ "min_interval_s": 1800,
791
+ "cadence_note": "Current snapshot; refresh no sooner than 1800 seconds.",
792
+ "snapshot": True,
793
+ "validates_against": "Amazon document or API data-row count vs stored rows for the latest period",
794
+ },
795
+ }
796
+ REPORT_TYPES.update(GROUP_A_TABLES)
797
+
798
+ # These are visible declarations, never enablement keys or request allow-list entries.
799
+ DECLARED_DATASETS = {
800
+ "manage_inventory_variants": {
801
+ "label": "Amazon manage inventory alternatives",
802
+ "report_types": ["GET_FBA_MYI_ALL_INVENTORY_DATA","GET_AFN_INVENTORY_DATA"],
803
+ "role": "Amazon Fulfillment",
804
+ "cadence_note": "Not requested by this connector.",
805
+ "reason": "The connector uses the unsuppressed manage inventory report for this dataset.",
806
+ },
807
+ "stranded_inventory_loader": {
808
+ "label": "Amazon stranded inventory loader",
809
+ "report_types": ["GET_STRANDED_INVENTORY_LOADER_DATA"],
810
+ "role": "Amazon Fulfillment",
811
+ "cadence_note": "Not requested by this connector.",
812
+ "reason": "The connector uses the stranded inventory UI report for this dataset.",
813
+ },
814
+ "listing_variants": {
815
+ "label": "Amazon listing report variants",
816
+ "report_types": ["GET_MERCHANT_LISTINGS_DATA","GET_MERCHANT_LISTINGS_DATA_BACK_COMPAT","GET_MERCHANT_LISTINGS_DATA_LITE","GET_MERCHANT_LISTINGS_DATA_LITER","GET_FLAT_FILE_OPEN_LISTINGS_DATA"],
817
+ "role": "Product Listing",
818
+ "cadence_note": "Not requested by this connector.",
819
+ "reason": "These listing variants overlap Amazon listings, which reads the all listings report.",
820
+ },
821
+ "pending_orders": {
822
+ "label": "Amazon pending order reports",
823
+ "report_types": ["GET_FLAT_FILE_PENDING_ORDERS_DATA","GET_PENDING_ORDERS_DATA","GET_CONVERGED_FLAT_FILE_PENDING_ORDERS_DATA"],
824
+ "role": "Inventory and Order Tracking",
825
+ "cadence_note": "Not requested by this connector.",
826
+ "reason": "Pending orders are covered by Amazon orders.",
827
+ },
828
+ "archived_orders": {
829
+ "label": "Amazon archived orders",
830
+ "report_types": ["GET_FLAT_FILE_ARCHIVED_ORDERS_DATA_BY_ORDER_DATE"],
831
+ "role": "Inventory and Order Tracking",
832
+ "cadence_note": "Not requested by this connector.",
833
+ "reason": "Archived orders before the normal history window are not built in this release.",
834
+ },
835
+ "xml_orders": {
836
+ "label": "Amazon XML order reports",
837
+ "report_types": ["GET_XML_ALL_ORDERS_DATA_BY_ORDER_DATE_GENERAL","GET_XML_ALL_ORDERS_DATA_BY_LAST_UPDATE_GENERAL"],
838
+ "role": "Inventory and Order Tracking",
839
+ "cadence_note": "Not requested by this connector.",
840
+ "reason": "The XML order reports duplicate the flat file reports used by Amazon orders.",
841
+ },
842
+ "mfn_return_variants": {
843
+ "label": "Amazon MFN return report variants",
844
+ "report_types": ["GET_CSV_MFN_PRIME_RETURNS_REPORT","GET_XML_MFN_PRIME_RETURNS_REPORT","GET_FLAT_FILE_MFN_SKU_RETURN_ATTRIBUTES_REPORT","GET_XML_MFN_SKU_RETURN_ATTRIBUTES_REPORT"],
845
+ "role": "Inventory and Order Tracking",
846
+ "cadence_note": "Not requested by this connector.",
847
+ "reason": "Prime return, return attribute and attribute feed record variants are not built; Amazon MFN returns reads returns by return date.",
848
+ },
849
+ "browse_tree": {
850
+ "label": "Amazon browse tree",
851
+ "report_types": ["GET_XML_BROWSE_TREE_DATA"],
852
+ "role": "Product Listing",
853
+ "cadence_note": "Not requested by this connector.",
854
+ "reason": "The XML browse tree is category reference data and is not built in this release.",
855
+ },
856
+ "sales_traffic_kiosk": {
857
+ "label": "Amazon sales and traffic query",
858
+ "operations": ["analytics_salesAndTraffic_2024_04_24"],
859
+ "role": "Brand Analytics",
860
+ "cadence_note": "Not requested by this connector.",
861
+ "reason": "Same rows as Amazon pricing and Buy Box, which reads the report twin.",
862
+ },
863
+ "financial_events": {
864
+ "label": "Amazon legacy financial events",
865
+ "operations": ["listFinancialEvents","listFinancialEventGroups"],
866
+ "role": "Finance and Accounting",
867
+ "cadence_note": "Not requested by this connector.",
868
+ "reason": "Superseded by Amazon financial transactions.",
869
+ },
870
+ "payouts": {
871
+ "label": "Amazon payouts",
872
+ "operations": ["listPayouts","listExpectedPayouts"],
873
+ "role": "Finance and Accounting",
874
+ "cadence_note": "Not requested by this connector.",
875
+ "reason": "Payout timing from the Transfers API is not built in this release.",
876
+ },
877
+ "fee_estimates_batch": {
878
+ "label": "Amazon batch fee estimates",
879
+ "operations": ["getMyFeesEstimates"],
880
+ "role": "Pricing",
881
+ "cadence_note": "Not requested by this connector.",
882
+ "reason": "Current price fee estimates are covered by Amazon FBA fee estimates and referral fee preview; hypothetical prices are not built.",
883
+ },
884
+ "offer_prices": {
885
+ "label": "Amazon offer price alternatives",
886
+ "operations": ["getItemOffersBatch","getListingOffersBatch","getFeaturedOfferExpectedPriceBatch"],
887
+ "role": "Pricing",
888
+ "cadence_note": "Not requested by this connector.",
889
+ "reason": "Amazon pricing and Buy Box reads competitive summaries; other offer price operations are not built in this release.",
890
+ },
891
+ "inbound_shipments": {
892
+ "label": "Amazon inbound shipments",
893
+ "operations": ["listInboundPlans","listShipmentItems","listInboundShipments"],
894
+ "role": "Amazon Fulfillment",
895
+ "cadence_note": "Not requested by this connector.",
896
+ "reason": "FBA and AWD inbound shipment plans and items are not built in this release.",
897
+ },
898
+ "replenishment": {
899
+ "label": "Amazon replenishment metrics",
900
+ "operations": ["listOfferMetrics","getSellingPartnerMetrics"],
901
+ "role": "Brand Analytics",
902
+ "cadence_note": "Not requested by this connector.",
903
+ "reason": "Only returns rows for Subscribe and Save offers; API offer metrics are not built in this release.",
904
+ },
905
+ "review_topics": {
906
+ "label": "Amazon review topics and trends",
907
+ "operations": ["getItemReviewTopics","getItemReviewTrends","getBrowseNodeReturnTopics","getBrowseNodeReturnTrends"],
908
+ "role": "Selling Partner Insights",
909
+ "cadence_note": "Not requested by this connector.",
910
+ "reason": "Per ASIN review and return topics are not built in this release; the limit is Amazon's per ASIN request rate.",
911
+ },
912
+ "promotions": {
913
+ "label": "Amazon promotion calendar",
914
+ "operations": ["searchPromotions","getPromotion"],
915
+ "role": "Pricing",
916
+ "cadence_note": "Not requested by this connector.",
917
+ "reason": "The Promotions API calendar is not built in this release; promotion results are in Amazon promotion performance.",
918
+ },
919
+ "aplus_content": {
920
+ "label": "Amazon A plus content",
921
+ "operations": ["searchContentDocuments"],
922
+ "role": "Brand Analytics",
923
+ "cadence_note": "Not requested by this connector.",
924
+ "reason": "A plus content reference data is not built in this release.",
925
+ },
926
+ "listing_restrictions": {
927
+ "label": "Amazon listing restrictions",
928
+ "operations": ["getListingsRestrictions"],
929
+ "role": "Product Listing",
930
+ "cadence_note": "Not requested by this connector.",
931
+ "reason": "Listing restriction reference data is not built in this release.",
932
+ },
933
+ "product_types": {
934
+ "label": "Amazon product type definitions",
935
+ "operations": ["searchDefinitionsProductTypes","getDefinitionsProductType"],
936
+ "role": "Product Listing",
937
+ "cadence_note": "Not requested by this connector.",
938
+ "reason": "Product type definition reference data is not built in this release.",
939
+ },
940
+ "supply_sources": {
941
+ "label": "Amazon supply sources",
942
+ "operations": ["getSupplySources"],
943
+ "role": "Selling Partner Insights",
944
+ "cadence_note": "Not requested by this connector.",
945
+ "reason": "Supply source reference data is not built in this release.",
946
+ },
947
  }
948
 
949
+ # Include the existing seven base tables and T40/T41 datasets in the same inventory.
950
+ GROUP_A = {
951
+ key: {"key": key, "label": spec.get("label") or "Amazon " + key.replace("_", " "),
952
+ "report_types": tuple(spec[k] for k in ("report", "report_incremental", "report_discover") if spec.get(k)),
953
+ "operation": spec.get("api"), "role": spec["role"],
954
+ "cadence_note": spec.get("cadence_note") or "Uses the existing connector cadence."}
955
+ for key, spec in REPORT_TYPES.items()
956
+ }
957
+ GROUP_A.update({key: {"key": key, **spec} for key, spec in DECLARED_DATASETS.items()})
958
+
959
+
960
+ # Inventory section 2.5: immutable refusal even if a future report roster admits one.
961
+ EXCLUDED_REPORT_TYPES = frozenset({
962
+ "GET_ORDER_REPORT_DATA_SHIPPING", "GET_ORDER_REPORT_DATA_TAX", "GET_ORDER_REPORT_DATA_ALL",
963
+ "GET_FLAT_FILE_ORDER_REPORT_DATA_SHIPPING", "GET_FLAT_FILE_ORDER_REPORT_DATA_TAX",
964
+ "GET_FLAT_FILE_ORDERS_RECONCILIATION_DATA_SHIPPING", "GET_FLAT_FILE_ORDERS_RECONCILIATION_DATA_TAX",
965
+ "GET_FLAT_FILE_ACTIONABLE_ORDER_DATA_SHIPPING", "GET_ORDERS_DATA",
966
+ "GET_XML_ALL_ORDERS_DATA_BY_LAST_UPDATE_ALL", "GET_AMAZON_FULFILLED_SHIPMENTS_DATA_TAX",
967
+ "GET_FLAT_FILE_SALES_TAX_DATA", "GET_FLAT_FILE_VAT_INVOICE_DATA_REPORT",
968
+ "GET_XML_VAT_INVOICE_DATA_REPORT", "GET_VAT_TRANSACTION_DATA",
969
+ "GET_GST_MTR_B2B_CUSTOM", "GET_GST_MTR_B2C_CUSTOM", "GET_EASYSHIP_DOCUMENTS",
970
+ "GET_EASYSHIP_DOCUMENTS_PICKEDUP", "GET_EASYSHIP_DOCUMENTS_WAITING_FOR_PICKUP",
971
+ "GET_MONTHLY_WARRANTIES", "WARRANTY_BASE_ITEM_INFO",
972
+ "GET_AMAZON_FULFILLED_SHIPMENTS_DATA_GENERAL", "GET_SELLER_FEEDBACK_DATA",
973
+ })
974
+
975
  #: Eight of the ten roles Amazon approved on 2026-07-22. Buyer Communication and Buyer Solicitation
976
  #: are REFUSED (owner ruling R7): both are write-shaped and PII-bearing, and it is precisely those
977
  #: restricted roles that trigger Phase 1 verification, a Phase 2 data-security assessment and an
 
996
  }
997
  _REPORT_ROLE_INDEX = {"inventory": 0, "pricing": 1}
998
  _BULK_REPORT_ROLES = tuple(dict.fromkeys(
999
+ roles[_REPORT_ROLE_INDEX.get(table, 0)] for table, roles in REPORT_ROLES.items()
1000
+ if REPORT_TYPES[table].get("report") or REPORT_TYPES[table].get("report_discover")))
1001
  _APPROVED_REPORT_TYPES = tuple(dict.fromkeys(
1002
  spec[key] for spec in REPORT_TYPES.values() for key in ("report", "report_incremental")
1003
  if spec.get(key)))
1004
 
1005
  #: Exact method/path pairs. Braced ids match ONE opaque path segment, never a suffix, query,
1006
+ #: escaped slash or traversal. The three POSTs generate reports/queries or wrap fixed read-only GETs.
1007
  #: LWA, presigned documents and the schema module's public GitHub refresh are not SP-API
1008
  #: operations. The gate enumerates those transports separately; they confer no operation scope.
1009
  OPERATIONS = {
 
1038
  "name": "listInventory", "roles": ("Amazon Warehousing and Distribution",),
1039
  "method": "GET", "path": "awd/2024-05-09/inventory",
1040
  },
1041
+ "createQuery": {"name": "createQuery", "roles": ("Selling Partner Insights",),
1042
+ "method": "POST", "path": _QUERIES_CREATE_PATH},
1043
+ "getQuery": {"name": "getQuery", "roles": ("Selling Partner Insights",),
1044
+ "method": "GET", "path": "dataKiosk/2023-11-15/queries/{queryId}"},
1045
+ "getDocument": {"name": "getDocument", "roles": ("Selling Partner Insights",),
1046
+ "method": "GET", "path": "dataKiosk/2023-11-15/documents/{documentId}"},
1047
+ "listTransactions": {"name": "listTransactions", "roles": ("Finance and Accounting",),
1048
+ "method": "GET", "path": _TRANSACTIONS_PATH},
1049
+ "getOrderMetrics": {"name": "getOrderMetrics", "roles": ("Inventory and Order Tracking",),
1050
+ "method": "GET", "path": "sales/v1/orderMetrics"},
1051
+ "getInventorySummaries": {"name": "getInventorySummaries", "roles": ("Amazon Fulfillment",),
1052
+ "method": "GET", "path": "fba/inventory/v1/summaries"},
1053
+ "searchListingsItems": {"name": "searchListingsItems", "roles": ("Product Listing",),
1054
+ "method": "GET", "path": "listings/2021-08-01/items/{sellerId}"},
1055
+ "searchCatalogItems": {"name": "searchCatalogItems", "roles": ("Product Listing",),
1056
+ "method": "GET", "path": "catalog/2022-04-01/items"},
1057
  }
1058
 
1059
  #: Answer "what does this approved role let this connector read?" directly from data.
 
1145
  """Any refusal from LWA or SP-API, already stripped of anything secret."""
1146
 
1147
 
1148
+ class AmazonReportError(AmazonError):
1149
+ """Vendor report refusal, with its complete reason available to the batch coordinator."""
1150
+
1151
+ def __init__(self, message, reason, status):
1152
+ self.vendor_reason = str(reason).replace("\u2014", "; ").replace("\u2013", " to ")
1153
+ self.status = status
1154
+ super().__init__(f"{message}: {self.vendor_reason}")
1155
+
1156
+
1157
+ def sqp_refused_asins(error, asins):
1158
+ """Only Amazon's named brand refusals permit removing an ASIN from one retry."""
1159
+ if not isinstance(error, AmazonReportError) or error.status not in (400, "FATAL"):
1160
+ return []
1161
+ reason = error.vendor_reason
1162
+ if not re.search(r"not\s+(?:part|associated|owned|eligible|authorized|belong|registered)|invalid\s+asin|outside.*brand", reason, re.I):
1163
+ return []
1164
+ named = set(re.findall(r"\b[A-Z0-9]{10}\b", reason.upper()))
1165
+ return [asin for asin in asins if asin in named]
1166
+
1167
+
1168
  class AmazonOperationRefused(AmazonError):
1169
  """A local scope refusal, never a throttle and never eligible for throttling retries."""
1170
 
 
1218
 
1219
  def require_operation(method, path, *, report_type=None):
1220
  """Fail closed before constructing any URL or touching a transport."""
1221
+ if report_type in EXCLUDED_REPORT_TYPES:
1222
+ raise AmazonOperationRefused(f"excluded report {report_type}")
1223
  operation = operation_for(method, path)
1224
  if operation is None:
1225
  raise AmazonOperationRefused(f"{str(method).upper()} {_safe(path)}")
 
1287
  return wait
1288
 
1289
 
1290
+ def _pace_query_create(*, sleep=time.sleep, now=time.monotonic):
1291
+ """Reserve from Data Kiosk's independent burst-15, one-per-minute bucket."""
1292
+ with _query_pacer["lock"]:
1293
+ stamp = now()
1294
+ updated = _query_pacer["updated"]
1295
+ elapsed = max(0.0, stamp - updated) if updated is not None else 0.0
1296
+ tokens = min(QUERY_BURST, _query_pacer["tokens"] + elapsed / QUERY_SUSTAINED_SECONDS)
1297
+ wait = max(0.0, (1.0 - tokens) * QUERY_SUSTAINED_SECONDS)
1298
+ if wait:
1299
+ sleep(wait)
1300
+ issued = max(stamp + wait, now())
1301
+ _query_pacer.update(tokens=min(QUERY_BURST, tokens +
1302
+ (issued - stamp) / QUERY_SUSTAINED_SECONDS) - 1.0, updated=issued)
1303
+ return wait
1304
+
1305
+
1306
+ def economics_query(start, end, marketplace_id, *, summary=False):
1307
+ """Build only the pinned economics query; never accept caller GraphQL."""
1308
+ for value in (start, end):
1309
+ if not isinstance(value, str) or not re.fullmatch(r"[0-9]{4}-[0-9]{2}-[0-9]{2}", value):
1310
+ raise AmazonError("Economics dates must be YYYY-MM-DD")
1311
+ try:
1312
+ date.fromisoformat(value)
1313
+ except ValueError:
1314
+ raise AmazonError("Economics dates must be valid YYYY-MM-DD dates") from None
1315
+ if start > end or type(summary) is not bool:
1316
+ raise AmazonError("Economics needs an ordered date range and a boolean summary selection")
1317
+ if not isinstance(marketplace_id, str) or not re.fullmatch(r"[A-Z0-9]+", marketplace_id):
1318
+ raise AmazonError("Economics needs an alphanumeric marketplace ID")
1319
+ return (_ECONOMICS_TEMPLATES[summary].replace("__START__", start)
1320
+ .replace("__END__", end).replace("__MARKETPLACE__", marketplace_id))
1321
+
1322
+
1323
+ def _create_query(token, start, end, marketplace_id, *, summary=False,
1324
+ region=DEFAULT_REGION, timeout=_TIMEOUT, sandbox=False):
1325
+ """The third declared POST wrapper, fixed to a read-only economics query."""
1326
+ require_operation("POST", _QUERIES_CREATE_PATH)
1327
+ body = {"query": economics_query(start, end, marketplace_id, summary=summary)}
1328
+
1329
+ def attempt():
1330
+ base = (SANDBOX_ENDPOINTS if sandbox else ENDPOINTS).get(str(region).lower())
1331
+ if not base:
1332
+ raise AmazonError(f"unknown region `{region}`")
1333
+ url = f"{base}/{_QUERIES_CREATE_PATH}"
1334
+ req = urllib.request.Request(url, data=json.dumps(body).encode("utf-8"), method="POST")
1335
+ req.add_header("x-amz-access-token", token)
1336
+ req.add_header("Content-Type", "application/json")
1337
+ req.add_header("User-Agent", _UA)
1338
+ try:
1339
+ with urllib.request.urlopen(req, timeout=timeout) as response:
1340
+ got = json.loads(response.read().decode("utf-8") or "{}")
1341
+ except urllib.error.HTTPError as exc:
1342
+ if exc.code == 429:
1343
+ raise AmazonThrottled("createQuery throttled (429); Data Kiosk permits a burst of "
1344
+ "15, then one query per 60 seconds",
1345
+ retry_after=(exc.headers or {}).get("Retry-After")) from None
1346
+ if exc.code == 403:
1347
+ raise AmazonError("createQuery answered 403: needs the Selling Partner Insights role") from None
1348
+ raise AmazonError(f"createQuery answered {exc.code}") from None
1349
+ except Exception as exc:
1350
+ raise AmazonError(f"createQuery unreachable: {type(exc).__name__}") from None
1351
+ if not got.get("queryId"):
1352
+ raise AmazonError("createQuery returned no queryId")
1353
+ return str(got["queryId"])
1354
+
1355
+ _pace_query_create()
1356
+ return _retry(attempt, _QUERIES_CREATE_PATH,
1357
+ exhausted_fix="reduce Data Kiosk query frequency and retry the window")
1358
+
1359
+
1360
+ def _kiosk_document(token, document_id, *, region, sandbox, timeout):
1361
+ meta = _request(f"dataKiosk/2023-11-15/documents/{document_id}", token,
1362
+ region=region, sandbox=sandbox, timeout=timeout)
1363
+ url = str(meta.get("documentUrl") or "")
1364
+ if not url:
1365
+ raise AmazonError("Data Kiosk document returned no documentUrl")
1366
+ text = _download_document(url, str(meta.get("compressionAlgorithm") or ""))
1367
+ return text
1368
+
1369
+
1370
+ def economics_document(fields, start, end, *, summary=False, max_wait=900,
1371
+ poll_seconds=15, timeout=_TIMEOUT, sleep=time.sleep):
1372
+ """Request, poll and download JSONL, including the vendor error document on failure."""
1373
+ marketplace = str(fields.get("marketplace_id") or US_MARKETPLACE_ID)
1374
+ economics_query(start, end, marketplace, summary=summary) # Validate before LWA.
1375
+ token = access_token(fields, timeout=timeout)
1376
+ region, sandbox = str(fields.get("region") or DEFAULT_REGION), is_sandbox(fields)
1377
+ query_id = _create_query(token, start, end, marketplace, summary=summary,
1378
+ region=region, sandbox=sandbox, timeout=timeout)
1379
+ waited = 0
1380
+ while waited <= max_wait:
1381
+ got = _request(f"dataKiosk/2023-11-15/queries/{query_id}", token,
1382
+ region=region, sandbox=sandbox, timeout=timeout)
1383
+ state = got.get("processingStatus")
1384
+ if got.get("errorDocumentId"):
1385
+ detail = _kiosk_document(token, got["errorDocumentId"], region=region,
1386
+ sandbox=sandbox, timeout=timeout)
1387
+ detail = detail.replace("\u2014", "; ").replace("\u2013", " to ")
1388
+ raise AmazonError(f"Economics query {query_id} {state}: {detail[:1200]}")
1389
+ if state == "DONE":
1390
+ doc_id = got.get("dataDocumentId")
1391
+ if not doc_id:
1392
+ raise AmazonError(f"Economics query {query_id} returned no dataDocumentId")
1393
+ return (_kiosk_document(token, doc_id, region=region, sandbox=sandbox, timeout=timeout),
1394
+ {"queryId": query_id, "reportDocumentId": doc_id})
1395
+ if state in ("FATAL", "CANCELLED"):
1396
+ raise AmazonError(f"Economics query {query_id} ended as {state}")
1397
+ sleep(poll_seconds)
1398
+ waited += poll_seconds
1399
+ if waited and waited % 1800 == 0:
1400
+ token = access_token(fields, timeout=timeout)
1401
+ raise AmazonError(f"Economics query {query_id} still pending after {max_wait}s; "
1402
+ "increase the query wait before retrying the window")
1403
+
1404
+
1405
+ def transactions(fields, start, end, *, timeout=_TIMEOUT, sleep=time.sleep, now=time.monotonic):
1406
+ """Follow every Finances page at 0.5/s, including empty pages carrying a nextToken."""
1407
+ token, issued = access_token(fields, timeout=timeout), now()
1408
+ following, seen, rows = None, set(), []
1409
+ while True:
1410
+ _pace_entity("transactions", TRANSACTIONS_INTERVAL, sleep=sleep, now=now)
1411
+ if now() - issued >= 3000:
1412
+ token, issued = access_token(fields, timeout=timeout), now()
1413
+ got = _request(_TRANSACTIONS_PATH, token,
1414
+ region=str(fields.get("region") or DEFAULT_REGION),
1415
+ sandbox=is_sandbox(fields), timeout=timeout,
1416
+ postedAfter=start, postedBefore=end,
1417
+ marketplaceId=fields.get("marketplace_id") or US_MARKETPLACE_ID,
1418
+ nextToken=following,
1419
+ throttle_fix="resume the same Finances window at its 0.5 requests/second rate")
1420
+ payload = got.get("payload") or {}
1421
+ page = payload.get("transactions")
1422
+ if not isinstance(page, list):
1423
+ raise AmazonError("listTransactions returned no transactions list")
1424
+ rows.extend(page)
1425
+ following = payload.get("nextToken")
1426
+ if not following:
1427
+ return rows
1428
+ if following in seen:
1429
+ raise AmazonError("listTransactions repeated a nextToken; cannot prove complete pagination; "
1430
+ "retry the window after Amazon corrects the cursor")
1431
+ seen.add(following)
1432
+
1433
+
1434
+ ENTITY_INTERVALS = {"getOrderMetrics": 2.0, "getInventorySummaries": 0.5,
1435
+ "searchListingsItems": 0.2, "searchCatalogItems": 0.5}
1436
+
1437
+
1438
+ def group_a_records(fields, operation, *, start=None, end=None, asins=(),
1439
+ timeout=_TIMEOUT, sleep=time.sleep, now=time.monotonic):
1440
+ """Fixed GET readers, every page and every own-ASIN batch; no record ceiling."""
1441
+ if operation not in ENTITY_INTERVALS:
1442
+ raise AmazonOperationRefused("unapproved group (a) reader")
1443
+ marketplace = fields.get("marketplace_id") or US_MARKETPLACE_ID
1444
+ path = OPERATIONS[operation]["path"]
1445
+ params = {"marketplaceIds": marketplace}
1446
+ if operation == "getOrderMetrics":
1447
+ params.update(interval=f"{start}--{end}", granularity="Day", granularityTimeZone="UTC")
1448
+ root, payload_key, next_key = None, "payload", None
1449
+ elif operation == "getInventorySummaries":
1450
+ params.update(details="true", granularityType="Marketplace", granularityId=marketplace)
1451
+ root, payload_key, next_key = "inventorySummaries", "payload", "nextToken"
1452
+ elif operation == "searchListingsItems":
1453
+ seller = str(fields.get("seller_id") or "")
1454
+ if not re.fullmatch(r"[A-Za-z0-9]+", seller):
1455
+ raise AmazonError("Listing status needs the seller ID saved with the connection")
1456
+ path = path.replace("{sellerId}", seller)
1457
+ params.update(includedData="summaries,issues,offers,fulfillmentAvailability,attributes", pageSize=20)
1458
+ root, payload_key, next_key = "items", None, "pageToken"
1459
+ else:
1460
+ # `relationships` (S67-T29, O23): same Catalog Items endpoint and Product Listing role;
1461
+ # it carries VARIATION parentAsins so the mirror can map every own ASIN to its parent.
1462
+ params.update(identifiersType="ASIN",
1463
+ includedData="summaries,salesRanks,dimensions,attributes,relationships",
1464
+ pageSize=20)
1465
+ root, payload_key, next_key = "items", None, "pageToken"
1466
+ batches = [None]
1467
+ if operation == "searchCatalogItems":
1468
+ identifiers = sorted(set(asins))
1469
+ if any(not re.fullmatch(r"[A-Z0-9]{10}", str(a)) for a in identifiers):
1470
+ raise AmazonError("Catalog contains an invalid ASIN; correct the listings or pricing mirror and retry")
1471
+ batches = [identifiers[i:i + 20] for i in range(0, len(identifiers), 20)]
1472
+ if not batches:
1473
+ raise AmazonError("Catalog attributes are waiting for our ASINs; sync Amazon listings first")
1474
+ token, issued = access_token(fields, timeout=timeout), now()
1475
+ rows = []
1476
+ listing_end = datetime.now(timezone.utc)
1477
+ # Date partitions bypass Listings search's 1,000-item pagination ceiling.
1478
+ # Inclusive boundaries are deduplicated by SKU after every partition has finished.
1479
+ if operation == "searchListingsItems":
1480
+ batches = [(datetime(1970, 1, 1, tzinfo=timezone.utc), listing_end)]
1481
+ for batch in batches:
1482
+ query, seen = dict(params), set()
1483
+ if operation == "searchListingsItems":
1484
+ lower, upper = batch
1485
+ query.update(createdAfter=lower.isoformat(), createdBefore=upper.isoformat())
1486
+ elif batch is not None:
1487
+ query["identifiers"] = ",".join(batch)
1488
+ page_rows, expected = [], None
1489
+ while True:
1490
+ _pace_entity(operation, ENTITY_INTERVALS[operation], sleep=sleep, now=now)
1491
+ if now() - issued >= 3000:
1492
+ token, issued = access_token(fields, timeout=timeout), now()
1493
+ got = _request(path, token, region=str(fields.get("region") or DEFAULT_REGION),
1494
+ sandbox=is_sandbox(fields), timeout=timeout, **query,
1495
+ throttle_fix=f"continue every page at Amazon's {1 / ENTITY_INTERVALS[operation]:g} requests per second")
1496
+ body = got.get(payload_key) if payload_key else got
1497
+ if operation == "searchListingsItems" and isinstance(body, dict):
1498
+ count = body.get("numberOfResults")
1499
+ if isinstance(count, int) and count > 1000:
1500
+ midpoint = lower + (upper - lower) / 2
1501
+ if (upper - lower) <= timedelta(milliseconds=1):
1502
+ raise AmazonError("Amazon Listings search exceeds its 1,000 SKU limit within one millisecond. "
1503
+ "No partial snapshot was stored. Ask Amazon to raise this search limit or use a SKU-filtered export.")
1504
+ batches.extend(((lower, midpoint), (midpoint, upper)))
1505
+ page_rows = []
1506
+ break
1507
+ page = body.get(root) if root and isinstance(body, dict) else body
1508
+ if not isinstance(page, list) or any(not isinstance(row, dict) for row in page):
1509
+ raise AmazonError(f"{operation} returned no valid row list; retry after Amazon restores its response")
1510
+ page_rows.extend(page)
1511
+ if isinstance(body, dict) and type(body.get("numberOfResults")) is int:
1512
+ expected = max(expected or 0, body["numberOfResults"])
1513
+ following = (got.get("pagination") or {}).get("nextToken")
1514
+ if isinstance(body, dict):
1515
+ following = following or (body.get("pagination") or {}).get("nextToken") or body.get("nextToken")
1516
+ if not following:
1517
+ if expected is not None and len(page_rows) != expected:
1518
+ raise AmazonError(f"{operation} reported {expected} rows but returned {len(page_rows)}. "
1519
+ "No partial snapshot was stored; retry after Amazon restores complete pagination.")
1520
+ break
1521
+ if not next_key or following in seen:
1522
+ raise AmazonError(f"{operation} repeated or cannot follow its nextToken; retry after Amazon fixes pagination")
1523
+ seen.add(following)
1524
+ query[next_key] = following
1525
+ rows.extend(page_rows)
1526
+ if operation == "searchListingsItems":
1527
+ unique = {}
1528
+ for row in rows:
1529
+ if not row.get("sku"):
1530
+ raise AmazonError("Amazon listing search omitted a SKU; cannot reconcile date partition boundaries")
1531
+ unique[row["sku"]] = row
1532
+ rows = list(unique.values())
1533
+ return rows
1534
+
1535
+
1536
  def _post_form(url, data, timeout=_TIMEOUT):
1537
  # LWA exchanges credentials at api.amazon.com, not an SP-API operation. Its sole caller fixes
1538
  # LWA_TOKEN_URL; the connector-wide gate checks that call separately from OPERATIONS.
 
1819
  is that this function can reach exactly one operation and a caller cannot redirect it.
1820
  """
1821
  require_operation("POST", _REPORTS_CREATE_PATH, report_type=str(report_type))
1822
+ if any(s.get("snapshot") and s.get("report") == report_type for s in REPORT_TYPES.values()):
1823
+ if start is not None or end is not None:
1824
+ raise AmazonError("Snapshot reports cannot take a data window")
1825
 
1826
  def attempt():
1827
  """One raw attempt at the declared report-creation POST."""
 
1849
  except urllib.error.HTTPError as e:
1850
  detail = ""
1851
  try:
1852
+ detail = (e.read() or b"").decode("utf-8", errors="replace")
1853
+ errs = json.loads(detail or "{}").get("errors") or []
1854
+ detail = " ".join(str(err.get(part) or "") for err in errs
1855
+ for part in ("message", "details") if err.get(part)) or detail
1856
  except Exception: # noqa: BLE001
1857
+ pass
1858
  if e.code == 429:
1859
  raise AmazonThrottled(
1860
  f"{_safe(url)} was throttled (429) creating a {report_type} report. Report "
 
1865
  raise AmazonError(
1866
  f"{_safe(url)} refused a {report_type} report (403). That is a ROLE answer: this "
1867
  "report needs a role the application was not granted") from None
1868
+ if e.code == 400 and report_type == REPORT_TYPES["search_query_performance"]["report"]:
1869
+ raise AmazonReportError("Amazon refused Search Query Performance (400)",
1870
+ detail, 400) from None
1871
  raise AmazonError(f"{_safe(url)} answered {e.code}"
1872
+ + (f": {detail[:200]}" if detail else "")) from None
1873
  except Exception as e: # noqa: BLE001
1874
  raise AmazonError(f"{_safe(url)} unreachable: {type(e).__name__}") from None
1875
  rid = str(got.get("reportId") or "")
 
1931
  doc_id = str(got.get("reportDocumentId") or "")
1932
  break
1933
  if state in _REPORT_DEAD:
1934
+ if state == "FATAL" and report_type == REPORT_TYPES["search_query_performance"]["report"]:
1935
+ error_id = got.get("reportDocumentId") or got.get("errorDocumentId")
1936
+ reason = str(got.get("errorMessage") or "No error document was supplied by Amazon")
1937
+ if error_id:
1938
+ meta = _request(f"reports/2021-06-30/documents/{error_id}", token,
1939
+ region=region, timeout=timeout, sandbox=sbx)
1940
+ if meta.get("url"):
1941
+ reason = _download_document(meta["url"], str(meta.get("compressionAlgorithm") or ""))
1942
+ raise AmazonReportError(f"Amazon ended Search Query Performance as FATAL (reportId {rid})",
1943
+ reason, state)
1944
+ if state == "CANCELLED":
1945
+ # S66-T42 (lead, observed live 2026-09-25): Amazon documents CANCELLED as also
1946
+ # meaning "the report had no data to return". Raising turned an empty period into
1947
+ # a role-shaped refusal on screen and left the window to retry forever. An empty
1948
+ # document lets the window complete and the connector detail name the empty period.
1949
+ say(f" {report_type}: Amazon cancelled report {rid}; no data for the requested period")
1950
+ return "", {"reportId": rid, "reportDocumentId": None, "cancelled": True,
1951
+ "dataStartTime": got.get("dataStartTime"),
1952
+ "dataEndTime": got.get("dataEndTime"),
1953
+ "createdTime": got.get("createdTime"), "compression": None}
1954
+ # Every FATAL report now carries Amazon's OWN reason when it supplied an error document,
1955
+ # so the connector detail names why, never only a generic role hint.
1956
+ detail = ""
1957
+ error_id = got.get("reportDocumentId") or got.get("errorDocumentId")
1958
+ if error_id:
1959
+ try:
1960
+ meta = _request(f"reports/2021-06-30/documents/{error_id}", token,
1961
+ region=region, timeout=timeout, sandbox=sbx)
1962
+ if meta.get("url"):
1963
+ detail = _download_document(meta["url"], str(meta.get("compressionAlgorithm") or ""))
1964
+ except AmazonError:
1965
+ detail = ""
1966
+ detail = " ".join(str(detail).replace("\u2014", "; ").replace("\u2013", " to ").split())[:240]
1967
+ raise AmazonError(f"Amazon ended the {report_type} report as {state} (reportId {rid})"
1968
+ + (f": {detail}" if detail else ""))
1969
  time.sleep(poll_seconds)
1970
  waited += poll_seconds
1971
  # β›” The token is ~1 hour and a long report can outlive it. Re-mint rather than 403.
 
1984
  text = _download_document(url, str(meta.get("compressionAlgorithm") or ""))
1985
  say(f" {report_type}: {len(text):,} chars")
1986
  return text, {"reportId": rid, "reportDocumentId": doc_id,
1987
+ "dataStartTime": got.get("dataStartTime"),
1988
+ "dataEndTime": got.get("dataEndTime"), "createdTime": got.get("createdTime"),
1989
  "compression": meta.get("compressionAlgorithm")}
1990
 
1991
 
 
2052
  return _download_document(url, str(meta.get("compressionAlgorithm") or ""))
2053
 
2054
 
2055
+ def _workbook_as_tsv(raw):
2056
+ """An Excel workbook document, as the tab-separated text every flat-file parser reads.
2057
+
2058
+ S66-T42 (lead, observed live 2026-09-25): Amazon delivers some reports (FEE_DISCOUNTS_REPORT,
2059
+ GET_REMOTE_FULFILLMENT_ELIGIBILITY) as an .xlsx workbook, a ZIP container. Decoded as text it
2060
+ became garbage column names carrying NUL bytes, and Postgres refused the whole Amazon write.
2061
+ The first sheet is the report; its first row is the header, like a flat file. Tabs and line
2062
+ breaks inside a cell become spaces so the TSV shape holds.
2063
+ """
2064
+ import io
2065
+ try:
2066
+ import openpyxl
2067
+ book = openpyxl.load_workbook(io.BytesIO(raw), read_only=True, data_only=True)
2068
+ except Exception as e: # noqa: BLE001
2069
+ raise AmazonError(f"the report document is a workbook that did not open: {type(e).__name__}") from None
2070
+ sheet = book.worksheets[0] if book.worksheets else None
2071
+ lines = []
2072
+ for values in (sheet.iter_rows(values_only=True) if sheet is not None else ()):
2073
+ cells = ["" if v is None else " ".join(str(v).replace("\x00", "").split()) for v in values]
2074
+ if any(cells):
2075
+ lines.append("\t".join(cells))
2076
+ book.close()
2077
+ return "\n".join(lines)
2078
+
2079
+
2080
  def _download_document(url, compression="", timeout=120):
2081
  """Fetch one presigned report document and decode it. -> text.
2082
 
 
2095
  raw = r.read()
2096
  except Exception as e: # noqa: BLE001
2097
  raise AmazonError(f"{_safe(url)} unreachable: {type(e).__name__}") from None
2098
+ # S66-T40 (lead, measured live 2026-09-25): Data Kiosk's getDocument answers only
2099
+ # `documentId` and `documentUrl`, yet the JSONL it points at IS gzip. With no declared
2100
+ # algorithm the bytes were decoded as text and every economics window failed to parse. The
2101
+ # gzip magic number is unambiguous (no text report starts with 0x1f 0x8b), so it decides.
2102
+ if not compression and raw[:2] == b"\x1f\x8b":
2103
+ compression = "GZIP"
2104
  if str(compression).upper() == "GZIP":
2105
  import gzip
2106
  try:
2107
  raw = gzip.decompress(raw)
2108
  except Exception as e: # noqa: BLE001
2109
  raise AmazonError(f"the report document did not decompress: {type(e).__name__}") from None
2110
+ if raw[:4] == b"PK\x03\x04":
2111
+ return _workbook_as_tsv(raw)
2112
  for enc in ("utf-8", "cp1252"):
2113
  try:
2114
  return raw.decode(enc)
platform/harness/amazon_schema.py CHANGED
@@ -53,6 +53,9 @@ from pathlib import Path
53
 
54
  _HERE = Path(__file__).resolve().parent
55
  CATALOGUE = _HERE / "proto" / "amazon-report-schemas.json"
 
 
 
56
 
57
  #: The vendor repository, pinned. ⚠ `main` is what `--refresh` reads, and the resulting file
58
  #: records the exact commit and per-file blob shas it saw, so a later refresh that changes a column
@@ -81,14 +84,24 @@ PROVENANCE = {
81
  "inventory": "measured_at_sync",
82
  "pricing": "schema_artifact",
83
  "brand_analytics": "schema_artifact",
 
 
 
 
 
 
84
  }
85
 
 
 
 
86
  #: Human copy for the disclosure screen, per provenance. β›” NO EM DASH OR EN DASH: this reaches a
87
  #: screen and standing rule 2 governs it.
88
  PROVENANCE_COPY = {
 
89
  "schema_artifact": "Columns come from Amazon's published report schema, pinned below.",
90
- "measured_at_sync": ("Amazon publishes no schema for this report. Its columns are read from "
91
- "the report's own header row the first time it syncs."),
92
  }
93
 
94
  _JSON_SCALAR = {"string": "VARCHAR", "integer": "BIGINT", "number": "DOUBLE", "boolean": "VARCHAR"}
@@ -254,9 +267,23 @@ def refresh(log=print):
254
  def catalogue():
255
  """The pinned catalogue off disk, or an empty shell when it has never been refreshed."""
256
  try:
257
- return json.loads(CATALOGUE.read_text(encoding="utf-8"))
258
  except Exception: # noqa: BLE001
259
- return {"repo": REPO, "commit": None, "fetched_utc": None, "tables": {}}
 
 
 
 
 
 
 
 
 
 
 
 
 
 
260
 
261
 
262
  def columns_for(table):
@@ -282,7 +309,7 @@ def _table_spec(table, identity, relationships=(), *, grain="entity", line_discr
282
  "provenance": PROVENANCE[table],
283
  "assumption": (
284
  "A-QI14: This wave declares from REPORT_TYPES and the pinned artifacts. "
285
- "The five measured_at_sync tables' columns are re-measured against the first "
286
  "production sync during the deferred QI13 hotfix (intake RDA6)."),
287
  },
288
  "row_identity": identity,
@@ -358,9 +385,122 @@ TABLE_SPECS = {
358
  "brand_analytics": _table_spec("brand_analytics", (
359
  "department_name", "search_term", "clicked_asin", "click_share_rank",
360
  )),
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
361
  }
362
 
363
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
364
  def row_identity_for(table, row):
365
  """Evaluate the declared full composite. Missing lineage is an error, never a parent ID.
366
 
@@ -469,7 +609,8 @@ def selftest(log=print):
469
  ck("NC-AS6 every schema_artifact table has an artifact", not orphan, orphan)
470
 
471
  # NC-AS7 β€” the seven tables of R4, no more and no fewer.
472
- ck("NC-AS7 seven tables are declared", len(PROVENANCE) == 7, sorted(PROVENANCE))
 
473
 
474
  # NC-AS8 β€” standing rule 2 governs PROVENANCE_COPY: it reaches the disclosure screen.
475
  dashed = [k for k, v in PROVENANCE_COPY.items() if "β€”" in v or "–" in v]
@@ -477,8 +618,8 @@ def selftest(log=print):
477
 
478
  # NC-AS9 β€” disclosure() names every table and carries the sync stamp key even when unpinned.
479
  d = disclosure()
480
- ck("NC-AS9 disclosure covers all seven and carries a synced field",
481
- len(d["tables"]) == 7 and "synced" in d and all(r["table"].startswith("ut_amzn_")
482
  for r in d["tables"]))
483
 
484
  # NC-AS10 β€” a measured_at_sync table must ship ZERO columns. An invented column is the defect.
 
53
 
54
  _HERE = Path(__file__).resolve().parent
55
  CATALOGUE = _HERE / "proto" / "amazon-report-schemas.json"
56
+ if str(_HERE.parent) not in sys.path:
57
+ sys.path.insert(0, str(_HERE.parent))
58
+ from harness.amazon_client import GROUP_A_TABLES, DECLARED_DATASETS
59
 
60
  #: The vendor repository, pinned. ⚠ `main` is what `--refresh` reads, and the resulting file
61
  #: records the exact commit and per-file blob shas it saw, so a later refresh that changes a column
 
84
  "inventory": "measured_at_sync",
85
  "pricing": "schema_artifact",
86
  "brand_analytics": "schema_artifact",
87
+ "economics": "measured_at_sync",
88
+ "transactions": "measured_at_sync",
89
+ "fba_fee_estimates": "measured_at_sync",
90
+ "referral_fees": "measured_at_sync",
91
+ "storage_fees": "measured_at_sync",
92
+ "search_query_performance": "measured_at_sync",
93
  }
94
 
95
+ PROVENANCE.update(dict.fromkeys(GROUP_A_TABLES, "measured_at_sync"))
96
+ PROVENANCE.update(dict.fromkeys(DECLARED_DATASETS, "declared"))
97
+
98
  #: Human copy for the disclosure screen, per provenance. β›” NO EM DASH OR EN DASH: this reaches a
99
  #: screen and standing rule 2 governs it.
100
  PROVENANCE_COPY = {
101
+ "declared": "This dataset is listed for completeness; its named reason explains why no rows are loaded.",
102
  "schema_artifact": "Columns come from Amazon's published report schema, pinned below.",
103
+ "measured_at_sync": ("Columns are measured from the document header or JSON record keys "
104
+ "when this database syncs; no column artifact is pinned here."),
105
  }
106
 
107
  _JSON_SCALAR = {"string": "VARCHAR", "integer": "BIGINT", "number": "DOUBLE", "boolean": "VARCHAR"}
 
267
  def catalogue():
268
  """The pinned catalogue off disk, or an empty shell when it has never been refreshed."""
269
  try:
270
+ result = json.loads(CATALOGUE.read_text(encoding="utf-8"))
271
  except Exception: # noqa: BLE001
272
+ result = {"repo": REPO, "commit": None, "fetched_utc": None, "tables": {}}
273
+ from core.connector_spec import AMAZON_TABLES
274
+ descriptors = {row.key: row for row in AMAZON_TABLES}
275
+ for name, provenance in PROVENANCE.items():
276
+ key = "ut_amzn_" + name
277
+ table = result.setdefault("tables", {}).setdefault(name, {})
278
+ table["provenance"] = provenance
279
+ if name in DECLARED_DATASETS:
280
+ declared = DECLARED_DATASETS[name]
281
+ label = declared["label"]
282
+ native = ", ".join(declared.get("report_types") or declared.get("operations") or ())
283
+ else:
284
+ label, native = descriptors[key].label, descriptors[key].native
285
+ table[key] = {"label": label, "native": native}
286
+ return result
287
 
288
 
289
  def columns_for(table):
 
309
  "provenance": PROVENANCE[table],
310
  "assumption": (
311
  "A-QI14: This wave declares from REPORT_TYPES and the pinned artifacts. "
312
+ "Measured-at-sync columns are re-measured against the first "
313
  "production sync during the deferred QI13 hotfix (intake RDA6)."),
314
  },
315
  "row_identity": identity,
 
385
  "brand_analytics": _table_spec("brand_analytics", (
386
  "department_name", "search_term", "clicked_asin", "click_share_rank",
387
  )),
388
+ "economics": _table_spec("economics", ("start_date", "msku", "marketplace_id"), (
389
+ ("msku", "listings", "seller_sku", "many_to_one"),
390
+ ("child_asin", "listings", "asin1", "many_to_one"),
391
+ ), grain="day_msku"),
392
+ "transactions": _table_spec("transactions", ("transaction_id", "item_position"), (
393
+ ("amazon_order_id", "orders", "amazon_order_id", "many_to_one"),
394
+ ("sku", "listings", "seller_sku", "many_to_one"),
395
+ ), grain="transaction_item"),
396
+ # S66-T40 (lead, measured live): the fee estimate snapshot has one row per SKU per Amazon
397
+ # store, and the referral fee preview spells its SKU `seller_sku`.
398
+ "fba_fee_estimates": _table_spec("fba_fee_estimates", ("_snapshot_date", "sku", "amazon_store"), (
399
+ ("sku", "listings", "seller_sku", "many_to_one"),
400
+ ), grain="snapshot_sku"),
401
+ "referral_fees": _table_spec("referral_fees", ("_snapshot_date", "seller_sku"), (
402
+ ("seller_sku", "listings", "seller_sku", "many_to_one"),
403
+ ), grain="snapshot_sku"),
404
+ "storage_fees": _table_spec("storage_fees", ("month", "asin", "fnsku", "fulfillment_center"), (
405
+ ("asin", "listings", "asin1", "many_to_one"),
406
+ ), grain="month_product_center"),
407
+ "search_query_performance": _table_spec("search_query_performance", (
408
+ "start_date", "end_date", "asin", "search_query_data_search_query"), (
409
+ ("asin", "listings", "asin1", "many_to_one"),
410
+ ), grain="month_asin_search_query"),
411
  }
412
 
413
 
414
+ TABLE_SPECS.update({
415
+ "long_term_storage_fees": _table_spec("long_term_storage_fees", ("_period_start", "sku", "fnsku", "asin", "_report_row_number"),
416
+ (("sku", "listings", "seller_sku", "many_to_one"), ("asin", "listings", "asin1", "many_to_one"),), grain="month_row"),
417
+ "storage_overage_fees": _table_spec("storage_overage_fees", ("_period_start", "storage_type", "_report_row_number"),
418
+ (), grain="month_row"),
419
+ "reimbursements": _table_spec("reimbursements", ("_period_start", "reimbursement_id", "sku", "asin", "amazon_order_id", "_report_row_number"),
420
+ (("sku", "listings", "seller_sku", "many_to_one"), ("asin", "listings", "asin1", "many_to_one"), ("amazon_order_id", "orders", "amazon_order_id", "many_to_one"),), grain="window_row"),
421
+ "fba_customer_returns": _table_spec("fba_customer_returns", ("_period_start", "order_id", "sku", "fnsku", "asin", "_report_row_number"),
422
+ (("order_id", "orders", "amazon_order_id", "many_to_one"), ("sku", "listings", "seller_sku", "many_to_one"), ("asin", "listings", "asin1", "many_to_one"),), grain="window_row"),
423
+ "replacements": _table_spec("replacements", ("_period_start", "replacement_amazon_order_id", "original_amazon_order_id", "sku", "asin", "_report_row_number"),
424
+ (("sku", "listings", "seller_sku", "many_to_one"), ("asin", "listings", "asin1", "many_to_one"),), grain="window_row"),
425
+ "fba_shipped_sales": _table_spec("fba_shipped_sales", ("_period_start", "amazon_order_id", "sku", "fnsku", "asin", "_report_row_number"),
426
+ (("amazon_order_id", "orders", "amazon_order_id", "many_to_one"), ("sku", "listings", "seller_sku", "many_to_one"), ("asin", "listings", "asin1", "many_to_one"),), grain="window_row"),
427
+ "fba_shipment_promotions": _table_spec("fba_shipment_promotions", ("_period_start", "amazon_order_id", "sku", "_report_row_number"),
428
+ (("amazon_order_id", "orders", "amazon_order_id", "many_to_one"), ("sku", "listings", "seller_sku", "many_to_one"),), grain="window_row"),
429
+ "inventory_ledger_detail": _table_spec("inventory_ledger_detail", ("_period_start", "msku", "fnsku", "asin", "_report_row_number"),
430
+ (("msku", "listings", "seller_sku", "many_to_one"), ("asin", "listings", "asin1", "many_to_one"),), grain="window_row"),
431
+ "inventory_planning": _table_spec("inventory_planning", ("_snapshot_date", "sku", "fnsku", "asin", "_report_row_number"),
432
+ (("sku", "listings", "seller_sku", "many_to_one"), ("asin", "listings", "asin1", "many_to_one"),), grain="snapshot_row"),
433
+ "manage_inventory": _table_spec("manage_inventory", ("_snapshot_date", "sku", "fnsku", "asin", "_report_row_number"),
434
+ (("sku", "listings", "seller_sku", "many_to_one"), ("asin", "listings", "asin1", "many_to_one"),), grain="snapshot_row"),
435
+ "reserved_inventory": _table_spec("reserved_inventory", ("_snapshot_date", "sku", "fnsku", "asin", "_report_row_number"),
436
+ (("sku", "listings", "seller_sku", "many_to_one"), ("asin", "listings", "asin1", "many_to_one"),), grain="snapshot_row"),
437
+ "restock_recommendations": _table_spec("restock_recommendations", ("_snapshot_date", "merchant_sku", "fnsku", "asin", "_report_row_number"),
438
+ (("asin", "listings", "asin1", "many_to_one"),), grain="snapshot_row"),
439
+ "stranded_inventory": _table_spec("stranded_inventory", ("_snapshot_date", "sku", "fnsku", "asin", "_report_row_number"),
440
+ (("sku", "listings", "seller_sku", "many_to_one"), ("asin", "listings", "asin1", "many_to_one"),), grain="snapshot_row"),
441
+ "inbound_noncompliance": _table_spec("inbound_noncompliance", ("_period_start", "shipment_id", "sku", "asin", "_report_row_number"),
442
+ (("sku", "listings", "seller_sku", "many_to_one"), ("asin", "listings", "asin1", "many_to_one"),), grain="window_row"),
443
+ "inventory_expiration_dates": _table_spec("inventory_expiration_dates", ("_snapshot_date", "sku", "fnsku", "_report_row_number"),
444
+ (("sku", "listings", "seller_sku", "many_to_one"),), grain="snapshot_row"),
445
+ "recommended_removals": _table_spec("recommended_removals", ("_snapshot_date", "sku", "fnsku", "asin", "_report_row_number"),
446
+ (("sku", "listings", "seller_sku", "many_to_one"), ("asin", "listings", "asin1", "many_to_one"),), grain="snapshot_row"),
447
+ "removal_orders": _table_spec("removal_orders", ("_period_start", "order_id", "sku", "fnsku", "_report_row_number"),
448
+ (("sku", "listings", "seller_sku", "many_to_one"),), grain="window_row"),
449
+ "removal_shipments": _table_spec("removal_shipments", ("_period_start", "order_id", "sku", "fnsku", "_report_row_number"),
450
+ (("sku", "listings", "seller_sku", "many_to_one"),), grain="window_row"),
451
+ "subscribe_save_forecast": _table_spec("subscribe_save_forecast", ("_snapshot_date", "sku", "asin", "_report_row_number"),
452
+ (("sku", "listings", "seller_sku", "many_to_one"), ("asin", "listings", "asin1", "many_to_one"),), grain="snapshot_row"),
453
+ "subscribe_save_performance": _table_spec("subscribe_save_performance", ("_period_start", "sku", "asin", "_report_row_number"),
454
+ (("sku", "listings", "seller_sku", "many_to_one"), ("asin", "listings", "asin1", "many_to_one"),), grain="window_row"),
455
+ "remote_fulfillment_eligibility": _table_spec("remote_fulfillment_eligibility", ("_snapshot_date", "sku", "asin", "_report_row_number"),
456
+ (("sku", "listings", "seller_sku", "many_to_one"), ("asin", "listings", "asin1", "many_to_one"),), grain="snapshot_row"),
457
+ "search_catalog_performance": _table_spec("search_catalog_performance", ("_period_start", "asin", "_report_row_number"),
458
+ (("asin", "listings", "asin1", "many_to_one"),), grain="month_row"),
459
+ "market_basket": _table_spec("market_basket", ("_period_start", "asin", "purchased_with_asin", "_report_row_number"),
460
+ (("asin", "listings", "asin1", "many_to_one"),), grain="month_row"),
461
+ "repeat_purchase": _table_spec("repeat_purchase", ("_period_start", "asin", "_report_row_number"),
462
+ (("asin", "listings", "asin1", "many_to_one"),), grain="month_row"),
463
+ "promotion_performance": _table_spec("promotion_performance", ("_period_start", "promotion_id", "_report_row_number"),
464
+ (), grain="window_row"),
465
+ "coupon_performance": _table_spec("coupon_performance", ("_period_start", "coupon_id", "_report_row_number"),
466
+ (), grain="window_row"),
467
+ "account_health": _table_spec("account_health", ("_snapshot_date", "marketplace_id", "_json_root", "_report_row_number"),
468
+ (), grain="snapshot_row"),
469
+ "inactive_listings": _table_spec("inactive_listings", ("_snapshot_date", "seller_sku", "asin1", "_report_row_number"),
470
+ (("seller_sku", "listings", "seller_sku", "many_to_one"),), grain="snapshot_row"),
471
+ "cancelled_listings": _table_spec("cancelled_listings", ("_snapshot_date", "seller_sku", "asin1", "_report_row_number"),
472
+ (("seller_sku", "listings", "seller_sku", "many_to_one"),), grain="snapshot_row"),
473
+ "suppressed_listings": _table_spec("suppressed_listings", ("_snapshot_date", "sku", "asin", "_report_row_number"),
474
+ (("sku", "listings", "seller_sku", "many_to_one"), ("asin", "listings", "asin1", "many_to_one"),), grain="snapshot_row"),
475
+ "listing_defects": _table_spec("listing_defects", ("_snapshot_date", "sku", "_report_row_number"),
476
+ (("sku", "listings", "seller_sku", "many_to_one"),), grain="snapshot_row"),
477
+ "sold_listings": _table_spec("sold_listings", ("_snapshot_date", "sku", "_report_row_number"),
478
+ (("sku", "listings", "seller_sku", "many_to_one"),), grain="snapshot_row"),
479
+ "mfn_returns": _table_spec("mfn_returns", ("_period_start", "order_id", "sku", "asin", "_report_row_number"),
480
+ (("order_id", "orders", "amazon_order_id", "many_to_one"), ("sku", "listings", "seller_sku", "many_to_one"), ("asin", "listings", "asin1", "many_to_one"),), grain="window_row"),
481
+ "b2b_recommended": _table_spec("b2b_recommended", ("_snapshot_date", "asin", "_report_row_number"),
482
+ (("asin", "listings", "asin1", "many_to_one"),), grain="snapshot_row"),
483
+ "b2b_not_yet_on_amazon": _table_spec("b2b_not_yet_on_amazon", ("_snapshot_date", "ean", "upc", "part_number", "_report_row_number"),
484
+ (), grain="snapshot_row"),
485
+ "geo_opportunities": _table_spec("geo_opportunities", ("_snapshot_date", "asin", "_report_row_number"),
486
+ (("asin", "listings", "asin1", "many_to_one"),), grain="snapshot_row"),
487
+ "out_of_stock_mfn": _table_spec("out_of_stock_mfn", ("_snapshot_date", "sku", "asin", "_report_row_number"),
488
+ (("sku", "listings", "seller_sku", "many_to_one"), ("asin", "listings", "asin1", "many_to_one"),), grain="snapshot_row"),
489
+ "b2b_quotes": _table_spec("b2b_quotes", ("_snapshot_date", "sku", "asin", "_report_row_number"),
490
+ (("sku", "listings", "seller_sku", "many_to_one"), ("asin", "listings", "asin1", "many_to_one"),), grain="snapshot_row"),
491
+ "fee_discounts": _table_spec("fee_discounts", ("_snapshot_date", "sku", "asin", "_report_row_number"),
492
+ (("sku", "listings", "seller_sku", "many_to_one"), ("asin", "listings", "asin1", "many_to_one"),), grain="snapshot_row"),
493
+ "order_metrics": _table_spec("order_metrics", ("_period_start", "interval", "_report_row_number"),
494
+ (), grain="window_row"),
495
+ "fba_inventory": _table_spec("fba_inventory", ("_snapshot_date", "seller_sku", "fn_sku", "asin", "_report_row_number"),
496
+ (("seller_sku", "listings", "seller_sku", "many_to_one"), ("asin", "listings", "asin1", "many_to_one"),), grain="snapshot_row"),
497
+ "listing_items": _table_spec("listing_items", ("_snapshot_date", "sku", "_report_row_number"),
498
+ (("sku", "listings", "seller_sku", "many_to_one"),), grain="snapshot_row"),
499
+ "catalog_items": _table_spec("catalog_items", ("_snapshot_date", "asin", "_report_row_number"),
500
+ (("asin", "listings", "asin1", "many_to_one"),), grain="snapshot_row"),
501
+ })
502
+
503
+
504
  def row_identity_for(table, row):
505
  """Evaluate the declared full composite. Missing lineage is an error, never a parent ID.
506
 
 
609
  ck("NC-AS6 every schema_artifact table has an artifact", not orphan, orphan)
610
 
611
  # NC-AS7 β€” the seven tables of R4, no more and no fewer.
612
+ ck("NC-AS7 table and declared dataset keys cover group (a)",
613
+ set(PROVENANCE) == set(TABLE_SPECS) | set(DECLARED_DATASETS), sorted(PROVENANCE))
614
 
615
  # NC-AS8 β€” standing rule 2 governs PROVENANCE_COPY: it reaches the disclosure screen.
616
  dashed = [k for k, v in PROVENANCE_COPY.items() if "β€”" in v or "–" in v]
 
618
 
619
  # NC-AS9 β€” disclosure() names every table and carries the sync stamp key even when unpinned.
620
  d = disclosure()
621
+ ck("NC-AS9 disclosure covers all datasets and carries a synced field",
622
+ len(d["tables"]) == len(PROVENANCE) and "synced" in d and all(r["table"].startswith("ut_amzn_")
623
  for r in d["tables"]))
624
 
625
  # NC-AS10 β€” a measured_at_sync table must ship ZERO columns. An invented column is the defect.
platform/harness/amazon_store.py CHANGED
The diff for this file is too large to render. See raw diff
 
platform/harness/datastore.py CHANGED
@@ -797,13 +797,63 @@ def _published_content_sha(tok, data=None):
797
  ("sha256", "contentSha", "content_sha", "sha"))
798
 
799
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
800
  def ensure_seed():
801
  """Fresh/ephemeral deployment (HF Space disks reset on every rebuild): hydrate the store
802
  from the PRIVATE dataset's seed snapshot β€” Analyst-ready in ~a minute instead of a 20-40 min
803
  XML-RPC backfill; the auto-sync then closes the gap via write_date cursors. Also restores
804
  the pilot views/routines when absent. Fail-quiet: no seed/token β†’ normal backfill."""
805
  import shutil
806
- if DB_PATH.exists():
807
  return False
808
  tok = os.environ.get("HF_TOKEN")
809
  if not tok:
@@ -812,11 +862,10 @@ def ensure_seed():
812
  from huggingface_hub import hf_hub_download
813
  DB_PATH.parent.mkdir(parents=True, exist_ok=True)
814
  p = hf_hub_download(SEED_DATASET, SEED_FILE, repo_type="dataset", token=tok)
815
- # Atomic install: copy to a temp path then os.replace β€” a reader that guards on
816
- # DB_PATH.exists() must never bind a half-written file into the shared instance.
817
  tmp = DB_PATH.with_name(DB_PATH.name + ".tmp")
818
  shutil.copyfile(p, tmp)
819
- os.replace(tmp, DB_PATH)
820
  # ⭐ W50-T29 β€” RECORD WHICH SEED THIS IS, because nothing did and the status surface could
821
  # therefore only ever say "there is a mirror". Three fields: the dataset, the commit sha of
822
  # the exact file downloaded, and the date the DATA was cut, which comes from the seed's own
 
797
  ("sha256", "contentSha", "content_sha", "sha"))
798
 
799
 
800
+ def _seedable():
801
+ """Is there no usable mirror to keep? True for a missing file, or for one an early reader
802
+ CREATED empty (`_instance()` makes the file) before the boot seed landed: tables none but
803
+ `_sync_state`, and that one without a row. A file holding anything else is kept."""
804
+ if not DB_PATH.exists():
805
+ return True
806
+ try:
807
+ cur = ro_cursor()
808
+ try:
809
+ names = {r[0] for r in cur.execute(
810
+ "SELECT table_name FROM information_schema.tables "
811
+ "WHERE table_schema = 'main'").fetchall()}
812
+ rows = (cur.execute("SELECT count(*) FROM _sync_state").fetchone()[0]
813
+ if "_sync_state" in names else 0)
814
+ finally:
815
+ cur.close()
816
+ except Exception: # noqa: BLE001
817
+ return False # cannot tell: keep the file, the pre-hotfix behaviour
818
+ return not (names - {"_sync_state"}) and not rows
819
+
820
+
821
+ def _install_seed(tmp):
822
+ """Swap the downloaded seed in, closing any instance a reader opened on the old file.
823
+
824
+ β›”β›” HOTFIX 2026-09-27 (D-3030). A reader that reached `_instance()` during the seed download
825
+ CREATED an empty royal.duckdb and kept it open; a bare `os.replace` then orphaned that handle
826
+ (Linux keeps the unlinked inode), so the whole process read an EMPTY mirror, 0 of 8 Odoo
827
+ phases, while the seed on disk had all 8 `live`. Staging's Odoo cadence is manual, so nothing
828
+ ever refilled it: every Odoo grid, measure column and Sheet cell refused "first sync".
829
+ Under `_INSTANCE_LOCK` (a plain Lock, so `use_path` is inlined, not called) no reader can
830
+ reopen the old file between the close and the swap; the close checkpoints its WAL, and any
831
+ leftover WAL is removed so it cannot be replayed onto the seed.
832
+ """
833
+ with _INSTANCE_LOCK:
834
+ con = _INSTANCE.get("con")
835
+ if con is not None:
836
+ try:
837
+ con.close()
838
+ except Exception: # noqa: BLE001
839
+ pass
840
+ _INSTANCE["con"] = None
841
+ wal = DB_PATH.with_name(DB_PATH.name + ".wal")
842
+ if wal.exists():
843
+ wal.unlink()
844
+ os.replace(tmp, DB_PATH)
845
+ _RO_TLS.__dict__.pop("cur", None)
846
+ _READY["ok"] = False
847
+ _GENERATION[0] += 1 # every other thread's cached cursor is now stale
848
+
849
+
850
  def ensure_seed():
851
  """Fresh/ephemeral deployment (HF Space disks reset on every rebuild): hydrate the store
852
  from the PRIVATE dataset's seed snapshot β€” Analyst-ready in ~a minute instead of a 20-40 min
853
  XML-RPC backfill; the auto-sync then closes the gap via write_date cursors. Also restores
854
  the pilot views/routines when absent. Fail-quiet: no seed/token β†’ normal backfill."""
855
  import shutil
856
+ if not _seedable():
857
  return False
858
  tok = os.environ.get("HF_TOKEN")
859
  if not tok:
 
862
  from huggingface_hub import hf_hub_download
863
  DB_PATH.parent.mkdir(parents=True, exist_ok=True)
864
  p = hf_hub_download(SEED_DATASET, SEED_FILE, repo_type="dataset", token=tok)
865
+ # Atomic install: copy to a temp path then swap it in under the instance lock.
 
866
  tmp = DB_PATH.with_name(DB_PATH.name + ".tmp")
867
  shutil.copyfile(p, tmp)
868
+ _install_seed(tmp)
869
  # ⭐ W50-T29 β€” RECORD WHICH SEED THIS IS, because nothing did and the status surface could
870
  # therefore only ever say "there is a mirror". Three fields: the dataset, the commit sha of
871
  # the exact file downloaded, and the date the DATA was cut, which comes from the seed's own
platform/harness/runtime.py CHANGED
@@ -204,6 +204,14 @@ class TenantRuntime:
204
  answer = reader([store_key for _name, store_key in keyed])
205
  return {name: answer.get(store_key, {}) for name, store_key in keyed}
206
 
 
 
 
 
 
 
 
 
207
  def revision(self, name):
208
  """Return the tiny durable change token for this tenant's addressed bucket."""
209
  return self._store().revision(self.store_key(name))
 
204
  answer = reader([store_key for _name, store_key in keyed])
205
  return {name: answer.get(store_key, {}) for name, store_key in keyed}
206
 
207
+ def get_entry(self, name, entry):
208
+ """One top-level entry of `name` when the bound store offers it, else None (HOTFIX
209
+ 2026-09-28). None also means absent; either way the caller falls back to `get`."""
210
+ reader = getattr(self._store(), "get_entry", None)
211
+ if not callable(reader):
212
+ return None
213
+ return reader(self.store_key(name), entry)
214
+
215
  def revision(self, name):
216
  """Return the tiny durable change token for this tenant's addressed bucket."""
217
  return self._store().revision(self.store_key(name))
platform/harness/telemetry.py CHANGED
@@ -70,8 +70,13 @@ def error(where, exc=None, **fields):
70
  ERR_PATH.parent.mkdir(parents=True, exist_ok=True)
71
  with ERR_PATH.open("a", encoding="utf-8") as f:
72
  f.write(json.dumps(rec, default=str) + "\n")
73
- print(f"[app-error] {rec['ts']} {rec['where']}: "
74
- f"{rec.get('type', '')} {rec.get('msg', '')}", file=sys.stderr, flush=True)
 
 
 
 
 
75
  except Exception:
76
  pass
77
 
 
70
  ERR_PATH.parent.mkdir(parents=True, exist_ok=True)
71
  with ERR_PATH.open("a", encoding="utf-8") as f:
72
  f.write(json.dumps(rec, default=str) + "\n")
73
+ # SDLC 67 sweep: an exc-less call (routes_activity forwarding a client error) left
74
+ # `type`/`msg` unset, so this mirror printed an empty ": " on every one -- the JSONL line
75
+ # kept the real `name`/`message`/`source` fields, only the container-log mirror lost them.
76
+ detail = f"{rec.get('type', '')} {rec.get('msg', '')}".strip()
77
+ if not detail:
78
+ detail = " ".join(f"{k}={v}" for k, v in fields.items() if v) or "(no detail)"
79
+ print(f"[app-error] {rec['ts']} {rec['where']}: {detail}", file=sys.stderr, flush=True)
80
  except Exception:
81
  pass
82