fsanyoto commited on
Commit
c1278e4
·
verified ·
1 Parent(s): dd84471

Deploy AIOS web (React glide grid + FastAPI slice)

Browse files
This view is limited to 50 files because it contains too many changes.   See raw diff
Files changed (50) hide show
  1. RELEASES.json +13 -1
  2. VERSION +1 -1
  3. api/ai_review.py +0 -255
  4. api/automation_engine.py +0 -0
  5. api/connectors_tt.py +568 -568
  6. api/deps.py +1 -8
  7. api/main.py +0 -0
  8. api/odoo_relational.py +0 -0
  9. api/routes_admin.py +0 -0
  10. api/routes_alerts.py +537 -664
  11. api/routes_automation.py +0 -0
  12. api/routes_changes.py +1 -85
  13. api/routes_connectors.py +13 -42
  14. api/routes_customers.py +0 -11
  15. api/routes_keychain.py +0 -0
  16. api/routes_oauth.py +114 -114
  17. api/routes_odoo_tables.py +0 -0
  18. api/routes_query.py +693 -713
  19. api/routes_records.py +164 -211
  20. api/routes_shares.py +385 -393
  21. api/routes_tables.py +0 -0
  22. jobs/web_agent_job.py +2 -117
  23. platform/aios_grid.py +0 -0
  24. platform/aios_grid_fields.json +464 -474
  25. platform/core/alerts.py +2 -21
  26. platform/core/grid_events.py +0 -0
  27. platform/core/perms.py +0 -110
  28. platform/core/registry.py +2 -115
  29. platform/core/shares.py +4 -26
  30. platform/core/store.py +817 -838
  31. platform/core/table_store.py +618 -618
  32. platform/core/user_tables.py +0 -0
  33. platform/core/users.py +25 -45
  34. platform/harness/datastore.py +2 -23
  35. platform/harness/meta_store.py +521 -521
  36. platform/harness/tools.py +9 -38
  37. platform/modules/product_data.py +797 -804
  38. platform/modules/products.py +728 -737
  39. web/public/sample_customers.json +545 -558
  40. web/src/alerts/alertsModel.ts +365 -418
  41. web/src/automation/AutomationBuilder.tsx +0 -0
  42. web/src/automation/AutomationDetail.tsx +1 -26
  43. web/src/automation/AutomationSurface.tsx +782 -806
  44. web/src/automation/automationApi.ts +0 -11
  45. web/src/connectors/ConnectorsPage.tsx +0 -14
  46. web/src/connectors/brandMarks.tsx +338 -411
  47. web/src/customer-grid/ColumnMenu.tsx +0 -0
  48. web/src/customer-grid/CustomerGrid.tsx +0 -0
  49. web/src/customer-grid/ImportDialog.css +154 -154
  50. web/src/customer-grid/ImportDialog.tsx +299 -299
RELEASES.json CHANGED
@@ -1,6 +1,18 @@
1
  {
2
- "current": "6a642e1",
3
  "releases": [
 
 
 
 
 
 
 
 
 
 
 
 
4
  {
5
  "version": "v24",
6
  "sha": "514d829",
 
1
  {
2
+ "current": "v25 (bc24829)",
3
  "releases": [
4
+ {
5
+ "version": "v25",
6
+ "sha": "bc24829",
7
+ "date": "2026-08-14",
8
+ "subject": "v25 - wave 32's LIVE build (deployed 2026-08-13, bc24829)"
9
+ },
10
+ {
11
+ "version": "v26",
12
+ "sha": "6a642e1",
13
+ "date": "2026-08-14",
14
+ "subject": "v26 - wave 33's build (deployed 2026-08-14, 6a642e1)"
15
+ },
16
  {
17
  "version": "v24",
18
  "sha": "514d829",
VERSION CHANGED
@@ -1 +1 @@
1
- 6a642e1
 
1
+ v25 (bc24829)
api/ai_review.py CHANGED
@@ -205,258 +205,3 @@ def decide(*, prompt, options, row, fields=(), label="Review", timeout=None):
205
  "problem": reason or "the model did not choose one of the stages"}
206
  return choice, {"provider": p["name"], "model": model, "reason": reason}
207
  return "", {"problem": "; ".join(problems)[:300] or "no provider answered"}
208
-
209
-
210
- # ══════════════════════════════════════════════════════════════════ the FLOW WRITER
211
- # ⭐⭐ WAVE 33 · W33-T54/T55 (owner item 7, ruling R3) — A PROMPT BECOMES A DRAFT AUTOMATION.
212
- #
213
- # Owner, verbatim: *"lay out the foundation with custom tools we provide"* — and the tools are the
214
- # action catalog, which has been machine-readable since wave 23. So this writes no new vocabulary:
215
- # it hands the model the SAME `ACTION_CATALOG` the menu paints, the SAME `ACTION_REQUIRED` the
216
- # runner blocks on, and the SAME trigger keys `clean_trigger` accepts, and asks for one JSON object
217
- # in that vocabulary.
218
- #
219
- # ⛔ WHY IT LIVES IN `ai_review.py` RATHER THAN A NEW FILE. Two reasons, both about ownership. This
220
- # is the one module in the API package that already holds an LLM ladder and its credentials, so a
221
- # second one would be a second place a rotated key has to be noticed; and a new `routes_*.py` needs
222
- # a `main.py` line from another lane (contract C2) to be reachable at all, which is how wave 23
223
- # shipped three finished routers 404-dead behind green gates. The DOOR is a route on
224
- # `routes_automation.py`, which is already mounted.
225
- #
226
- # ⛔⛔ CONTRACT C7 IS THE DESIGN, NOT A CHECK AT THE END: *"the flow writer may emit only kinds
227
- # present in `ACTION_CATALOG`, and its output must pass `clean_actions` unchanged."* Both halves are
228
- # enforced mechanically rather than asked for politely — the kind list is an `enum` in the schema
229
- # the model fills, and the caller runs `clean_actions` and DIFFS the result. That diff matters
230
- # because `clean_actions` has no disclosure channel (D-75): it drops a key it does not recognise
231
- # and answers 200, so a draft accepted without the diff would show a person a flow that is not the
232
- # flow the model described, with nothing anywhere saying so.
233
- #
234
- # ⚠ `chat` IS INJECTABLE, exactly as `routes_query._call_model`'s is, and it is the reason this door
235
- # can be proven end to end with no API key and no spend. A gate that can only run where a
236
- # credential exists is a gate that never runs.
237
-
238
- #: Cerebras first, NOT the cheap-first order `ladder()` uses. Same reason `routes_query` inverts it:
239
- #: this contract includes a REFUSAL, and refusing honestly ("I cannot build that from the steps you
240
- #: have") is a model property. A cheaper rung answers an impossible request with a plausible flow,
241
- #: which is worse than no flow. Override with `AIOS_FLOW_PROVIDER`.
242
- FLOW_PROVIDER_ORDER = ("cerebras", "groq", "openrouter", "anthropic")
243
- MAX_DRAFT_ACTIONS = 12 # a draft a person reads in one screen; the engine's own cap is 25
244
- MAX_PROMPT_CHARS = 2000
245
-
246
-
247
- def flow_providers():
248
- """The rungs usable here, in THIS module's refusal-first order. Empty = the feature is off."""
249
- pin = (os.environ.get("AIOS_FLOW_PROVIDER") or "").strip().lower()
250
- by_name = {p["name"]: p for p in PROVIDERS}
251
- order = [pin] if pin else list(FLOW_PROVIDER_ORDER)
252
- return [by_name[n] for n in order
253
- if n in by_name and (os.environ.get(by_name[n]["env"]) or "").strip()]
254
-
255
-
256
- def flow_schema(kinds, trigger_keys, table_keys):
257
- """The tool schema the model fills — the catalog's OWN key lists as enums.
258
-
259
- ⛔ `kind` AND `trigger` ARE ENUMS, not free strings, and that is C7's first half enforced by the
260
- transport rather than by a check afterwards. A model asked for "any action name" invents
261
- `send_slack_message` on a deployment that has no such kind, and the failure then surfaces as a
262
- 400 from `clean_actions` carrying a sentence about a word the person never typed.
263
- ⚠ `required` IS `["kind"]` ALONE, deliberately: a REFUSAL carries no name and no actions, and a
264
- schema demanding them turns an honest refusal into a provider-side 400 that reads exactly like a
265
- transport failure (`routes_query._spec_schema` carries the same note, for the same measured
266
- reason).
267
- """
268
- return {
269
- "type": "object",
270
- "properties": {
271
- "kind": {"type": "string", "enum": ["flow", "refused"],
272
- "description": "refused = this cannot be built from the steps available"},
273
- "refusal": {"type": "string",
274
- "description": "when kind=refused: ONE plain sentence naming what is "
275
- "missing, in the words a non-technical person would use"},
276
- "name": {"type": "string", "description": "a short title for the automation"},
277
- "trigger": {"type": "string", "enum": sorted(trigger_keys),
278
- "description": "what starts this automation"},
279
- "table": {"type": "string", "enum": sorted(table_keys),
280
- "description": "the database whose records this flow walks, if any"},
281
- "actions": {
282
- "type": "array",
283
- "maxItems": MAX_DRAFT_ACTIONS,
284
- "items": {
285
- "type": "object",
286
- "properties": {
287
- "kind": {"type": "string", "enum": sorted(kinds)},
288
- "why": {"type": "string",
289
- "description": "one short sentence: why this step is here"},
290
- "config": {"type": "object", "additionalProperties": True,
291
- "description": "the step's settings, using ONLY the keys named "
292
- "for that kind in the system message"},
293
- },
294
- "required": ["kind"],
295
- },
296
- },
297
- },
298
- "required": ["kind"],
299
- }
300
-
301
-
302
- def flow_system_prompt(catalog, required, triggers, tables):
303
- """What the model is told it may build with — DERIVED, never written down twice.
304
-
305
- Every list here is the server's own: the catalog rows the menu paints, the required-key table
306
- the runner blocks on, the trigger keys `clean_trigger` accepts, and this tenant's real databases
307
- with their real columns. Nothing about the vocabulary is restated by hand, so a kind added to
308
- the catalog is offered here on the same deploy and a kind removed stops being offered.
309
- """
310
- lines = ["You build small automations for a business tool. You are given the EXACT set of "
311
- "steps this tool can perform. You may use nothing else.",
312
- "", "THE STEPS YOU MAY USE:"]
313
- # ⚠ BOTH HALVES, `key (phrase)`. The KEY is what the model must write into `config` and the
314
- # PHRASE is the only human wording of that requirement anywhere — the one the runner's own
315
- # refusal sentence is built from. Keys alone leave the model guessing what `field` means on a
316
- # `web_read` (it is a column to write into, not a form field); phrases alone leave it guessing
317
- # what to call the setting. Handing it one and hoping is how a draft comes back configured
318
- # against a key the validator drops.
319
- req = {str(k): [f"{key} ({phrase})" for phrase, key in v] for k, v in (required or {}).items()}
320
- for row in catalog or []:
321
- if not row.get("ready"):
322
- continue
323
- kind = str(row.get("kind") or "")
324
- need = req.get(kind) or []
325
- lines.append(f"- {kind}: {row.get('label')} - {row.get('detail') or ''}"
326
- + (f" REQUIRED settings: {', '.join(need)}" if need else ""))
327
- lines += ["", "WHAT CAN START AN AUTOMATION:"]
328
- for t in triggers or []:
329
- if t.get("planned") or not t.get("ready", True):
330
- continue
331
- lines.append(f"- {t.get('key')}: {t.get('label')}")
332
- lines += ["", "THE DATABASES THIS PERSON HAS:"]
333
- for t in (tables or [])[:40]:
334
- cols = ", ".join(str(f.get("key")) for f in (t.get("fields") or [])[:25])
335
- lines.append(f"- {t.get('key')} ({t.get('label')}): {cols or 'no columns yet'}")
336
- if not tables:
337
- lines.append("- (none - do not name a database)")
338
- lines += [
339
- "",
340
- "RULES:",
341
- "1. Use ONLY the step kinds listed above. If what is asked needs a step that is not there, "
342
- "answer kind=refused and say plainly which capability is missing.",
343
- "2. Fill in every REQUIRED setting you can from what the person told you. Leave one blank "
344
- "rather than inventing a web address, a CSS selector or a column name.",
345
- "3. A web address must start with http:// or https://.",
346
- "4. To use a value from the record the flow is walking, write it as {{Column name}}.",
347
- "5. Name a database only from the list above, by its key.",
348
- "6. Keep it short. Fewer steps that work beat more steps that guess.",
349
- ]
350
- return "\n".join(lines)
351
-
352
-
353
- def _flow_from_tool_call(obj):
354
- """The model's tool arguments -> `(draft, refusal)`. A shape error is a refusal, never a crash."""
355
- if not isinstance(obj, dict):
356
- return None, "the assistant's answer could not be read"
357
- if str(obj.get("kind") or "") == "refused":
358
- return None, (str(obj.get("refusal") or "").strip()
359
- or "the assistant could not build this from the steps available")
360
- acts = obj.get("actions")
361
- if not isinstance(acts, list) or not acts:
362
- return None, ("the assistant did not produce any steps - try describing what should "
363
- "happen, one action at a time")
364
- out = []
365
- for a in acts[:MAX_DRAFT_ACTIONS]:
366
- if not isinstance(a, dict) or not str(a.get("kind") or "").strip():
367
- continue
368
- cfg = a.get("config")
369
- out.append({"kind": str(a["kind"]).strip(),
370
- "config": cfg if isinstance(cfg, dict) else {},
371
- "why": str(a.get("why") or "").strip()[:200]})
372
- if not out:
373
- return None, "the assistant's steps could not be read"
374
- return {"name": str(obj.get("name") or "").strip()[:80] or "New automation",
375
- "trigger": str(obj.get("trigger") or "").strip(),
376
- "table": str(obj.get("table") or "").strip(),
377
- # ⛔⛔ THE TRUNCATION IS REPORTED, and it is a STANDING RULE that it must be (owner,
378
- # 2026-08-12, W30/R6 second sentence): *"if there is lag or it can't be done, you need
379
- # to explicitly tell me why and recommend a fix"* — **a silent truncation IS the
380
- # violation, not the limit.** `acts[:MAX_DRAFT_ACTIONS]` above dropped everything past
381
- # the twelfth and said nothing, so a model that answered with a twenty-step journey had
382
- # eight steps deleted between the answer and the screen, with no key anywhere in the
383
- # response naming them. `MAX_ACTIONS` is 20, so those steps were STORABLE — this
384
- # ceiling is the draft door's own, which makes reporting it the whole obligation.
385
- # ⚠ Found by the verifier that checked this ticket, not by a gate.
386
- "asked": len([a for a in acts if isinstance(a, dict)]),
387
- "actions": out}, ""
388
-
389
-
390
- def draft_flow(*, prompt, catalog, required, triggers, tables, chat=None, timeout=None):
391
- """A sentence -> `(draft, refusal_sentence, provider)`. ⛔ NOTHING IS SAVED HERE.
392
-
393
- Exactly one of `draft` and `refusal_sentence` is truthy — the same contract
394
- `web_agent.run_step` keeps, so a caller has no third case to get wrong.
395
- """
396
- text = str(prompt or "").strip()[:MAX_PROMPT_CHARS]
397
- if not text:
398
- return None, "type what you want the automation to do", None
399
- kinds = sorted({str(r.get("kind")) for r in (catalog or []) if r.get("ready")})
400
- if not kinds:
401
- return None, "this deployment offers no automation steps to build with", None
402
- trigger_keys = sorted({str(t.get("key")) for t in (triggers or [])
403
- if t.get("key") and not t.get("planned")}) or ["manual"]
404
- table_keys = sorted({str(t.get("key")) for t in (tables or []) if t.get("key")}) or [""]
405
- tools = [{"type": "function", "function": {
406
- "name": "build_automation",
407
- "description": "Emit the automation, or refuse.",
408
- "parameters": flow_schema(kinds, trigger_keys, table_keys)}}]
409
- messages = [{"role": "system",
410
- "content": flow_system_prompt(catalog, required, triggers, tables)},
411
- {"role": "user", "content": text}]
412
-
413
- if chat is not None:
414
- # ⚠ THE INJECTED PATH IS THE PROVEN PATH. It runs the SAME parse and the SAME refusal
415
- # branches as a live call; only the transport is replaced.
416
- draft, refusal = _flow_from_tool_call(chat(messages, tools))
417
- return draft, refusal, "injected"
418
-
419
- provs = flow_providers()
420
- if not provs:
421
- # ⛔ SAY SO. An AI feature that silently does nothing is indistinguishable from one that was
422
- # never built [[flag-shipped-without-its-writer]].
423
- return None, ("the assistant is not configured on this deployment, so an automation cannot "
424
- "be drafted from a description yet"), None
425
- tmo = float(timeout or TIMEOUT_SECONDS)
426
- problems = []
427
- for p in provs:
428
- if p["shape"] != "openai":
429
- # Anthropic's tool transport differs; it stays on the ladder for `decide()` and is
430
- # SKIPPED here rather than half-supported. Named in `problems` so it is not invisible.
431
- problems.append(f"{p['name']}: tool calls are not wired for this shape")
432
- continue
433
- try:
434
- r = requests.post(p["url"], timeout=tmo,
435
- headers={"Authorization": f"Bearer {os.environ[p['env']].strip()}",
436
- "Content-Type": "application/json"},
437
- json={"model": p["model"], "messages": messages, "tools": tools,
438
- "tool_choice": "required", "temperature": 0.1,
439
- "max_tokens": 1500})
440
- except Exception as e: # noqa: BLE001
441
- problems.append(f"{p['name']}: {type(e).__name__}")
442
- continue
443
- if r.status_code != 200:
444
- problems.append(f"{p['name']}: HTTP {r.status_code}")
445
- continue
446
- try:
447
- calls = (((r.json().get("choices") or [{}])[0].get("message") or {})
448
- .get("tool_calls") or [])
449
- args = json.loads(calls[0]["function"]["arguments"]) if calls else None
450
- except Exception as e: # noqa: BLE001
451
- problems.append(f"{p['name']}: unreadable answer ({type(e).__name__})")
452
- continue
453
- draft, refusal = _flow_from_tool_call(args)
454
- if draft is None and isinstance(args, dict) and str(args.get("kind")) == "refused":
455
- # ⛔ A REFUSAL IS AN ANSWER, NOT A FAULT, so it does NOT fall through to a more
456
- # expensive rung — the model has just said this cannot be built. `decide()` takes the
457
- # same posture for the same reason.
458
- return None, refusal, p["name"]
459
- if draft is not None:
460
- return draft, "", p["name"]
461
- problems.append(f"{p['name']}: {refusal}")
462
- return None, ("; ".join(problems)[:300] or "no provider answered"), None
 
205
  "problem": reason or "the model did not choose one of the stages"}
206
  return choice, {"provider": p["name"], "model": model, "reason": reason}
207
  return "", {"problem": "; ".join(problems)[:300] or "no provider answered"}
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
api/automation_engine.py CHANGED
The diff for this file is too large to render. See raw diff
 
api/connectors_tt.py CHANGED
@@ -1,568 +1,568 @@
1
- """connectors_tt.py — the TIKTOK connector (wave 29 · item 7 · DEBT D-9 · rulings R1 + R2).
2
-
3
- Everything in this file knows what a VENDOR's TikTok row looks like. Nothing in it knows what an
4
- automation is. That split is `connectors_ig.py`'s (wave 27 item 23) and it is the reason this file
5
- exists at all rather than another thousand lines inside the engine.
6
-
7
- ⛔ **EVERY VENDOR FIELD NAME HERE WAS PROBED, NOT GUESSED.** The whole schema — 40 profile / 43
8
- post / 17 comment fields, each with the vendor's own type, description and `pii` flag — was read
9
- live from `GET /datasets/{id}/metadata` for **$0.00** and written down in
10
- `.claude/wiki/waves/wave29/proto/tiktok-schema.md` (promoted to `tiktok-capture.md` at
11
- close-out). That document is the AUTHORITY: do not re-probe it, and do not invent a key. Where a
12
- name below reads through a candidate list it is because the vendor has two names for one fact
13
- (`biography`/`signature`, `region`/`country`), never because the name is uncertain.
14
-
15
- ⛔ **NOTHING HERE EVER AUTHENTICATES TO TIKTOK.** No login, no cookie, no account to get banned —
16
- public data through a supplier, exactly the rail `connectors_ig.py` states for Instagram. The
17
- vendor key is a key to a SUPPLIER.
18
-
19
- ⭐ **THE TRANSPORT IS `connectors_bd.py` — SHARED, VENDOR-NAMED, AND NO LONGER BORROWED FROM THE
20
- OTHER PLATFORM'S CONNECTOR** (WAVE 30 · T09, DEBT D-128). `bd_call`, `bd_scrape` and
21
- `bd_filter_start` take the dataset id as a PARAMETER — they are Bright Data's wire, not
22
- Instagram's — and re-implementing them here would be a second copy of the deferral handling, the
23
- truncation guard, the SSRF rail and the snapshot-progress reader, i.e. five places for one bug.
24
- Until wave 30 the code was right and the NAME was wrong: this file imported thirteen symbols from
25
- `connectors_ig`, which read as a dependency on Instagram and was really a dependency on a supplier.
26
- ⚠ **This file now imports ZERO names from `connectors_ig`, and a gate check asserts that**, because
27
- the sentence above is the kind that quietly stops being true.
28
-
29
- ⚠ **WHAT $0 COULD NOT BUY, so nobody reads this file as more measured than it is:**
30
- 1. the real ROW shape — `/metadata` describes a DATASET, and Instagram's rows carry undeclared
31
- envelope keys (`timestamp`, `input`) that no metadata call mentions;
32
- 2. that a declared field POPULATES — Bright Data's Instagram Reels *declares* `views: number`
33
- and delivers an account-grain wrong number (§4e). **Declared is not delivered**, and the one
34
- TikTok claim that matters most (`play_count`) is exactly a declaration.
35
- """
36
- from __future__ import annotations
37
-
38
- import automation_engine as engine
39
- # ⚠ T09 — FOUR NAMES CAME OFF THIS LIST AND NOTHING BROKE, which is the point of deriving an
40
- # import block from the AST both ways. `bd_call`, `bd_filter_start`, `bd_key` and `bd_ready` were
41
- # imported here under a comment claiming they were *"re-exported for the runners"*; no runner ever
42
- # read them off this module (the engine imports them from the transport itself), so they were four
43
- # lines of dependency nobody was paying for. `[[artifact-with-no-importer]]` in its smallest form.
44
- from connectors_bd import (
45
- _bd_first_url,
46
- _bd_flag,
47
- _bd_list,
48
- _bd_source_payload,
49
- _first,
50
- _ig_int,
51
- bd_scrape,
52
- )
53
-
54
- # ---------------------------------------------------------------------------------------------
55
- # THE DATASETS
56
- # ---------------------------------------------------------------------------------------------
57
- # ⚠ CATALOGUE PRESENCE IS NOT ENTITLEMENT — the same finding Instagram produced. `GET
58
- # /datasets/list` returned 1,735 rows of which 12 are TikTok; the three below answer 200 with a
59
- # full field list, and the two DISCOVERY halves answer **404 for our key**:
60
- # `gd_lj71gn6l68bz7y9hc` (posts by profile) and `gd_lilwhto81z415d9mdl` (posts by keyword).
61
- # ⇒ TikTok discovery routes through the PROFILES dataset's corpus filter, exactly as Instagram's
62
- # does. A ticket that reaches for a by-keyword endpoint is reaching for a 404.
63
- TT_DS_PROFILES = "gd_l1villgoiiidt09ci" # TikTok - Profiles. 40 fields, 152,000,000 records
64
- TT_DS_POSTS = "gd_lu702nij2f790tmv9h" # TikTok - Posts. 43 fields
65
- TT_DS_COMMENTS = "gd_lkf2st302ap89utw5k" # TikTok - Comments. 17 fields
66
-
67
- #: The vendor's two post-type tokens, verbatim from the dataset's own `ai_description`
68
- #: (*"strictly these two"*, video = 99.5% of rows). Ours are `image`/`video`/`carousel`.
69
- TT_POST_TYPE_VIDEO = "video"
70
- TT_POST_TYPE_CONTENT = "content"
71
-
72
- #: What a TikTok profile URL looks like, for the runner that has a handle and needs a URL. Kept
73
- #: beside the dataset ids because it is the same class of vendor fact.
74
- #: ⚠ ONE SPELLING. `platform/core/user_tables.profile_url(handle, 'tiktok')` builds the identical
75
- #: string from `_PROFILE_RULES` (contract C2, E's half) — this exists for the connector's own
76
- #: batch calls, and the gate asserts the two agree rather than trusting that they do.
77
- TT_PROFILE_URL = "https://www.tiktok.com/@{handle}"
78
-
79
-
80
- def tt_profile_url(handle):
81
- """`nurilab` → `https://www.tiktok.com/@nurilab`. `''` for a blank handle, never a bare `@`."""
82
- h = str(handle or "").strip().lstrip("@")
83
- return TT_PROFILE_URL.format(handle=h) if h else ""
84
-
85
-
86
- # ---------------------------------------------------------------------------------------------
87
- # THE FIELD MAPS
88
- # ---------------------------------------------------------------------------------------------
89
- # Each function turns ONE vendor row into the cell dict for one of the `ut_tt_*` schemas declared
90
- # in `automation_engine`. The rules they all obey, stated once:
91
- #
92
- # * **BLANK MEANS NOT READ, NEVER "THEY HAVE NONE".** A key the vendor did not send is OMITTED,
93
- # so a later, richer pull fills it instead of being overwritten by this one's silence. `_first`
94
- # returns `None` (never 0) when nothing matches, which is what makes that possible.
95
- # * **A zero from the vendor is a MEASUREMENT and survives.** (Instagram's `_ig_zero_is_blank`
96
- # rule is scoped to a paid rung whose zeros were proven fictional; nothing here has earned it.)
97
- # * **Every unpromoted vendor key stays whole in `source_payload`.** A schema addition on the
98
- # vendor's side is preserved rather than silently discarded while our column model catches up.
99
- # * **Nothing here writes a session token.** `tt_chain_token`, `secu_id` (~85% null), `short_id`
100
- # (100% null in the sample), `ftc` (100% null) and `relation` are deliberately unmapped; they
101
- # ride in `source_payload` where they make no claim.
102
-
103
-
104
- def _tt_str(node, *names):
105
- """The first non-empty string among `names`, or None when the vendor sent nothing.
106
-
107
- ⛔ `None`, NOT `""`. The callers below drop `None` keys, which is what keeps a blank honest —
108
- an empty string written into a cell claims "we looked and it is empty".
109
- """
110
- v = _first(node, *names)
111
- if v is None:
112
- return None
113
- s = str(v).strip()
114
- return s or None
115
-
116
-
117
- def _tt_pct(node, *names):
118
- """A vendor 0–1 engagement fraction → our stored 0–100 percentage, or None.
119
-
120
- ⛔ THE ×100 IS NOT COSMETIC (wave 26, amendment C1-a). Our `pct` renderer appends the sign to
121
- the STORED number, so writing the vendor's raw 0.0656 would print a 6.6% creator as `0.0%` —
122
- measured on the Instagram side, and TikTok sends the same shape on all three of its rates.
123
- """
124
- v = _first(node, *names)
125
- if v is None:
126
- return None
127
- out = engine._pct100(v)
128
- return out or None
129
-
130
-
131
- def _tt_day(node, *names):
132
- """A vendor stamp → `YYYY-MM-DD`, or None. Our `date` columns store a day."""
133
- v = _first(node, *names)
134
- if v is None:
135
- return None
136
- return engine._day(v) or None
137
-
138
-
139
- def _drop_blanks(row):
140
- """The one place a mapped row loses its `None`s — see the BLANK MEANS NOT READ rule above."""
141
- return {k: v for k, v in row.items() if v is not None and v != ""}
142
-
143
-
144
- def normalize_profile(node, handle=""):
145
- """A TikTok Profiles row → the `ut_tt_profile` / `ut_tt_snapshots` cell shape.
146
-
147
- ⚠ TWO FIELDS READ THROUGH A CANDIDATE PAIR, and both pairs are the vendor's, not a guess:
148
- * `biography` is PRIMARY and `signature` the FALLBACK — the probe measured `signature`
149
- populated on 85% of rows and they carry the same text;
150
- * `region` is PRIMARY and `country` the FALLBACK — `region` is the one with a documented
151
- two-letter-ISO description, `country` has no description at all.
152
- ⚠ `videos_count` is mapped to `posts_count` with a caveat recorded rather than hidden: its
153
- `ai_description` ranges 1-89, so it may be a WINDOW rather than a lifetime total. It is the
154
- only count of its kind the dataset offers.
155
- """
156
- node = node if isinstance(node, dict) else {}
157
- account = _tt_str(node, "account_id") or str(handle or "").strip().lstrip("@")
158
- return _drop_blanks({
159
- "platform": engine.PLATFORM_TIKTOK,
160
- "handle": account,
161
- "full_name": _tt_str(node, "nickname"),
162
- "tt_id": _tt_str(node, "id"),
163
- "profile_url": _tt_str(node, "url") or (tt_profile_url(account) or None),
164
- "bio": _tt_str(node, "biography", "signature"),
165
- "external_url": _bd_first_url(_first(node, "bio_link")),
166
- "verified": _bd_flag(node, "is_verified"),
167
- "is_private": _bd_flag(node, "is_private"),
168
- # ⚠ APPROXIMATE, and the word is the vendor's: `is_commerce_user` has *"many null values"*
169
- # on their own description. It is the closest thing TikTok has to Instagram's
170
- # `is_business_account`, and `_bd_flag` writes nothing at all when the key is absent — so
171
- # the approximation only ever fills a cell the vendor actually answered.
172
- "is_business": _bd_flag(node, "is_commerce_user"),
173
- "followers": _ig_int(_first(node, "followers")),
174
- "following": _ig_int(_first(node, "following")),
175
- "posts_count": _ig_int(_first(node, "videos_count")),
176
- # ⚠ `likes` on a PROFILE row is likes RECEIVED across the account's videos (18-110,200, no
177
- # nulls). It is not a post-level number and it is not our `likes` column, which is why it
178
- # is stored under a different name.
179
- "likes_received": _ig_int(_first(node, "likes")),
180
- "avg_engagement": _tt_pct(node, "awg_engagement_rate"),
181
- "like_engagement": _tt_pct(node, "like_engagement_rate"),
182
- "comment_engagement": _tt_pct(node, "comment_engagement_rate"),
183
- "country_code": _tt_str(node, "region", "country"),
184
- "region": _tt_str(node, "region"),
185
- "predicted_lang": _tt_str(node, "predicted_lang"),
186
- # ⚠ ACCOUNT AGE, NOT A MEASUREMENT STAMP — `create_time` on a profile is when the ACCOUNT
187
- # was made. TikTok stamps nothing with "when this number was true", exactly like Instagram,
188
- # which is why the append law dates a snapshot by when WE read it.
189
- "account_created_at": _tt_day(node, "create_time"),
190
- "source_payload": _bd_source_payload(node),
191
- })
192
-
193
-
194
- def tt_post_type(node):
195
- """A TikTok post row → one of OUR three type options, or None.
196
-
197
- ⛔ THE FIRST OF THE PROBE DOC'S TWO NAMED BLOCKERS. The vendor's vocabulary is `"video"` /
198
- `"content"`; ours is `image` / `video` / `carousel` and has no `"content"`. Writing the
199
- vendor's token would fail `_clean_field`'s option check on the way in and would put an
200
- untranslated API word in front of a user on the way out.
201
-
202
- So: `video` is `video`, and `content` — TikTok's photo-mode post — is `image`, EXCEPT when the
203
- row carries more than one `carousel_images` entry, which is what a carousel IS on either
204
- network. ⚠ The multi-image branch is decided from the IMAGES, never from the type token: the
205
- token cannot express it, so inferring `carousel` from the word would be inventing a fact.
206
- ⚠ An UNKNOWN token returns None rather than defaulting to `video` (99.5% of rows are video, and
207
- that is exactly what would make the wrong default invisible).
208
- """
209
- raw = str((node or {}).get("post_type") or "").strip().lower()
210
- if raw == TT_POST_TYPE_VIDEO:
211
- return "video"
212
- if raw != TT_POST_TYPE_CONTENT:
213
- return None
214
- images = (node or {}).get("carousel_images")
215
- return "carousel" if isinstance(images, list) and len(images) > 1 else "image"
216
-
217
-
218
- def normalize_post(node):
219
- """A TikTok Posts row → the `ut_tt_posts` cell shape. None when it carries no identity.
220
-
221
- ⛔ THE SECOND NAMED BLOCKER, RESOLVED HERE AND NOWHERE ELSE: `play_count` is ONE number and
222
- Instagram's schema has TWO columns for it (`plays` and `views`). On TikTok they are the same
223
- fact — `play_count` IS the count TikTok displays under a video — so it maps to `views` and
224
- `ut_tt_posts` HAS NO `plays` COLUMN. Copying one vendor number into two of our columns would
225
- manufacture a second measurement that a rollup could average or double-count, which is a worse
226
- outcome than the missing column it would paper over.
227
-
228
- ⚠ `shortcode` reads `shortcode` then `post_id`: both are 19-digit numerics on this dataset and
229
- the probe measured them as the same shape. That equality is what lets the comments→posts link
230
- join with NO normaliser, which the Instagram side never had.
231
- ⚠ `num_share_count` (a number) is preferred over `share_count` (typed TEXT by the vendor).
232
- ⚠ `commerce_info` is a business/commerce LOCATION per its own description — cities and
233
- countries. It is NOT a paid-partnership flag, and nothing in this dataset is: TikTok declares
234
- no equivalent, so `paid_partnership`/`partner` have no column on this family at all.
235
- """
236
- node = node if isinstance(node, dict) else {}
237
- shortcode = _tt_str(node, "shortcode", "post_id")
238
- if not shortcode:
239
- return None
240
- return _drop_blanks({
241
- "platform": engine.PLATFORM_TIKTOK,
242
- "shortcode": shortcode,
243
- # ⛔⛔ `account_id`, NOT `profile_username` — MEASURED on a real Posts row 2026-08-12.
244
- # The vendor's `profile_username` is the DISPLAY NAME (`"Dina"`), while `account_id` is the
245
- # @handle (`"d1na_th"`) — the same field `normalize_profile` already reads for `handle`, so
246
- # one name means one thing across both corpora. Reading the display name silently broke the
247
- # only join this table has: `ut_tt_posts.influencer_key` -> `ut_tt_profile.handle` matched
248
- # NOTHING, so a person could not filter posts by creator and a rollup would count zero.
249
- # ⚠ NO FALLBACK TO `profile_username`, deliberately. It is not a degraded handle, it is a
250
- # different fact, and filling a join key with it is worse than leaving it blank — a blank
251
- # is visibly missing, a display name looks like an answer [[one-question-two-normalizers]].
252
- # The URL is the honest second source: it carries the handle by construction.
253
- "influencer_key": (_tt_str(node, "account_id")
254
- or tt_handle(_tt_str(node, "profile_url", "url") or "") or None),
255
- "posted_at": _tt_day(node, "create_time"),
256
- "type": tt_post_type(node),
257
- "caption": _tt_str(node, "description"),
258
- "url": _tt_str(node, "url"),
259
- "hashtags": _bd_list(node, "hashtags"),
260
- "tagged_location": _tt_str(node, "commerce_info"),
261
- "views": _ig_int(_first(node, "play_count")),
262
- "likes": _ig_int(_first(node, "digg_count")),
263
- "comments": _ig_int(_first(node, "comment_count")),
264
- "shares": _ig_int(_first(node, "num_share_count")),
265
- "saves": _ig_int(_first(node, "collect_count")),
266
- "video_duration": _ig_int(_first(node, "video_duration")),
267
- "source_payload": _bd_source_payload(node),
268
- })
269
-
270
-
271
- def normalize_comment(node):
272
- """A TikTok Comments row → the `ut_tt_comments` cell shape. None without a comment id.
273
-
274
- ⚠ THE NAME COLLISION, RESOLVED: TikTok's `replies` is an ARRAY of reply objects and OUR
275
- `replies` column is an INT count. The count comes from `num_replies`; the array stays whole in
276
- `source_payload`. Reading the array's length instead would be a second, disagreeing answer to
277
- a question the vendor already answers — and it would disagree, because a page of replies is not
278
- all of them.
279
- ⚠ `date_created` is typed `date` by this vendor, unlike Instagram's `comment_date` which is
280
- text and needs defensive parsing. It still goes through `_tt_day` — one date path, so a vendor
281
- that changes its mind cannot change ours.
282
- ⚠ The comment TEXT and every identifiable commenter field (`commenter_user_name` is flagged
283
- PII) stay in `source_payload` and are promoted to no column, which is the same posture the
284
- Instagram comment schema takes for the same D-24 reason.
285
- """
286
- node = node if isinstance(node, dict) else {}
287
- comment_key = _tt_str(node, "comment_id")
288
- if not comment_key:
289
- return None
290
- return _drop_blanks({
291
- "platform": engine.PLATFORM_TIKTOK,
292
- "comment_key": comment_key,
293
- "shortcode": _tt_str(node, "post_id"),
294
- # ⭐⭐ OWNER RULING 2026-08-12: the comment's CONTENT gets a column. `comment_text` is the
295
- # vendor's own key and `comment_text_only` its stripped variant (the probe recorded both);
296
- # primary first, so a row carrying the rich form is not silently served the plain one.
297
- # ⛔ The commenter's identity is deliberately NOT promoted — see `TT_COMMENT_FIELDS`.
298
- "text": _tt_str(node, "comment_text", "comment_text_only"),
299
- "commented_at": _tt_day(node, "date_created"),
300
- "likes": _ig_int(_first(node, "num_likes")),
301
- "replies": _ig_int(_first(node, "num_replies")),
302
- "source_payload": _bd_source_payload(node),
303
- })
304
-
305
-
306
- #: ⭐ The three maps, addressable by name — so a gate (and the runners in T04-T06) can walk them
307
- #: rather than naming three functions, and so adding a fourth dataset is one entry.
308
- TT_NORMALIZERS = {
309
- "tt_profile": normalize_profile,
310
- "tt_post_metrics": normalize_post,
311
- "tt_comments": normalize_comment,
312
- }
313
-
314
-
315
- # ---------------------------------------------------------------------------------------------
316
- # THE FETCH — wave 30 · W30-T08 (carrying wave-29's dropped T05)
317
- # ---------------------------------------------------------------------------------------------
318
-
319
- def tt_handle(url):
320
- """A TikTok profile URL **or** a bare handle → the handle. `''` when it is neither.
321
-
322
- Deliberately permissive about the input and strict about the output, because the two callers
323
- hand it different things: an automation stores whatever a person typed in the profile column
324
- (`@nurilab`, `nurilab`, or the full URL), while the discovery runner already holds a clean
325
- `account_id`. One normaliser, so a row found by discovery and a row typed by hand cannot
326
- resolve to two different handles.
327
- """
328
- s = str(url or "").strip()
329
- if not s:
330
- return ""
331
- if "tiktok.com" in s.lower():
332
- # Everything after the first `@`, up to the next path segment or query.
333
- tail = s.split("@", 1)[1] if "@" in s else ""
334
- s = tail.split("/")[0].split("?")[0].split("#")[0]
335
- s = s.strip().lstrip("@").strip()
336
- # A handle is the vendor's `account_id` shape: alphanumerics, dots and underscores.
337
- return s if s and all(c.isalnum() or c in "._" for c in s) else ""
338
-
339
-
340
- def tt_post_urls(node, limit=0):
341
- """⭐ WAVE 30 · T10 — the profile row's own post permalinks, newest-first as the vendor sends.
342
-
343
- ⛔ THIS IS WHY TIKTOK POST CAPTURE COSTS NO EXTRA DISCOVERY. `top_videos` rides the PROFILE row
344
- we have already bought, so the posts read is a scrape of links we hold, never a search for them.
345
- The two TikTok DISCOVERY datasets (posts-by-profile, posts-by-keyword) are **404 for our key**,
346
- so a design that reached for either would not merely be dearer, it would not work.
347
-
348
- ⛔⛔ CORRECTED 2026-08-12 — THIS DOCSTRING USED TO SAY *"the probe MEASURED `top_videos` as an
349
- array of video permalinks, NO empties"*, AND THAT SENTENCE IS WHAT SHIPPED THE BUG. The probe
350
- read `/datasets/{id}/metadata` — a DATASET description — and the phrase quoted was the field's
351
- `ai_description`, not an observation of a row. The real row sends **dicts keyed `video_url`**
352
- (measured below), so the reader built against the quoted sentence found nothing, forever, in
353
- silence. ⭐ The transferable half: *"the probe measured X"* and *"the probe read a declaration
354
- of X"* are different claims, and prose cannot be told apart by a reader downstream — which is
355
- why the correction names the method, not just the value.
356
-
357
- ⚠ `top_posts_data` is deliberately NOT read: the probe calls it *"a thin dup of `top_videos`"*,
358
- and preferring whichever happened to be longer is how one creator's window silently differs
359
- from another's.
360
- ⚠ `limit <= 0` means "everything the row carried". The CAP IS THE CALLER'S — `config.maxPosts`,
361
- validated 1..12 — and it is applied here rather than after the scrape so an unwanted post is
362
- never bought. [[a-constant-two-features-share]]: the 12 is the vendor's measured profile window,
363
- not a number this function may invent.
364
- """
365
- raw = (node or {}).get("top_videos")
366
- out = []
367
- for item in raw if isinstance(raw, list) else []:
368
- # ⛔⛔ MEASURED ON A REAL ROW 2026-08-12, AND IT IS NOT WHAT THE SCHEMA SAID.
369
- # `top_videos` is NOT an array of permalink strings. One paid TikTok Profiles scrape of a
370
- # live handle returns **19 DICTS**, keyed
371
- # `video_url · video_id · playcount · diggcount · commentcount · share_count ·
372
- # favorites_count · create_date · cover_image`.
373
- # The docstring above cites the $0 probe as having "measured" permalinks — it had not, and
374
- # could not: `/datasets/{id}/metadata` describes a DATASET, and the probe's own verdict says
375
- # so in terms (*"what $0 cannot buy … the real ROW shape … that a declared field
376
- # POPULATES"*, `wave29/proto/tiktok-schema.md`). This is the SECOND time this vendor's
377
- # declaration has diverged from its delivery on this exact axis; BD's IG Reels `views` was
378
- # the first. [[reachable-is-not-the-same-as-built]]
379
- # ⇒ The consequence, live: `TT_DS_POSTS` was never reached, because this returned an EMPTY
380
- # list on every real profile — post capture could not have worked for anybody, and the
381
- # T10 gate stayed green because its canned fixture encoded the DECLARED shape. A fixture
382
- # written from a schema tests the schema.
383
- # ⚠ `video_url` is FIRST because it is the key the vendor actually sends; `url` is kept
384
- # because it costs nothing and is what a future corpus revision would most likely use. The
385
- # bare-string branch stays for the same reason — this widens what is ACCEPTED and invents
386
- # nothing: a shape that yields no `http…` value still degrades to "no posts", exactly as
387
- # before, rather than to a URL built out of a guess.
388
- # ⚠ `top_posts_data` is STILL not read (it carries `post_url` and would work): preferring
389
- # whichever array happened to be longer is how one creator's window silently differs from
390
- # another's, and that reasoning is unchanged by this correction.
391
- if isinstance(item, dict):
392
- u = str(item.get("video_url") or item.get("url") or "").strip()
393
- else:
394
- u = str(item or "").strip()
395
- if u.lower().startswith("http") and u not in out:
396
- out.append(u)
397
- return out[:limit] if limit and limit > 0 else out
398
-
399
-
400
- def pull_posts_tt(post_urls, log=print, deferred=None):
401
- """The TikTok Posts dataset for a list of permalinks → `(rows, note)`, already normalised.
402
-
403
- ⚠ ONE CALL FOR THE WHOLE WINDOW. `bd_scrape` has always taken a list, and the Instagram side
404
- measured what happens when a caller forgets: 25 records, one billed snapshot each, a walk still
405
- running at 67 minutes. Nothing here loops per URL.
406
- """
407
- urls = [str(u) for u in (post_urls or []) if str(u or "").strip()]
408
- if not urls:
409
- return [], ""
410
- rows, note = bd_scrape(TT_DS_POSTS, urls, deferred=deferred)
411
- if note:
412
- log(f"[aios-tt] posts: {note}")
413
- return [], note
414
- out = [r for r in (normalize_post(n) for n in rows) if r]
415
- return out, ""
416
-
417
-
418
- def pull_comments_tt(post_urls, log=print, deferred=None):
419
- """The TikTok Comments dataset for a list of POST permalinks → `(rows, note)`, normalised.
420
-
421
- ⛔ THE MOST EXPENSIVE THING THIS PRODUCT BUYS, and the reason `commentMetrics` defaults OFF on
422
- both networks: a comments scrape ingests identifiable third parties who never entered anybody's
423
- list (D-24). The mapper already keeps every commenter field in `source_payload` and promotes
424
- none of them to a column; this function adds no new exposure, it just has to be asked for.
425
- """
426
- urls = [str(u) for u in (post_urls or []) if str(u or "").strip()]
427
- if not urls:
428
- return [], ""
429
- rows, note = bd_scrape(TT_DS_COMMENTS, urls, deferred=deferred)
430
- if note:
431
- log(f"[aios-tt] comments: {note}")
432
- return [], note
433
- out = [r for r in (normalize_comment(n) for n in rows) if r]
434
- return out, ""
435
-
436
-
437
- #: ⭐⭐ WAVE 30 · D-156 — THE MEDIA DATASETS, AS A SET, SO THE HAND-OFF CAN FILTER ON IDENTITY.
438
- #: `_media_deferrals` uses this to lift ONLY posts/comments snapshots out of the local deferral
439
- #: list. That is what makes it structurally impossible to file a PROFILE snapshot in the engine's
440
- #: metric queue — the defect a draft of T10 shipped and A-39 booked as "the wrong fix is worse
441
- #: than the gap". A membership test cannot be got wrong by a later edit the way `if` order can.
442
- TT_MEDIA_DATASETS = (TT_DS_POSTS, TT_DS_COMMENTS)
443
-
444
-
445
- def _media_deferrals(deferred):
446
- """The POSTS/COMMENTS entries of a `bd_scrape` deferral list — never the profile's.
447
-
448
- ⚠ The engine, not this module, decides what a deferral MEANS: it stamps `kind` and the
449
- handle and files it. TikTok needs no `_tag_metric_deferrals` twin because one dataset is one
450
- kind here, so the id already carries everything a mapper choice depends on — and importing
451
- Instagram's tagger is not available anyway (W30-T09 gates ZERO `from connectors_ig` lines).
452
- """
453
- out = []
454
- for d in deferred or []:
455
- if isinstance(d, dict) and str(d.get("datasetId") or "") in TT_MEDIA_DATASETS:
456
- out.append(dict(d))
457
- return out
458
-
459
-
460
- def pull_profile_tt(url, log=print, pending_profile=None, prefetch=None,
461
- max_posts=0, post_metrics=False, comment_metrics=False):
462
- """ONE TikTok profile from the vendor. Same return contract as `pull_profile`.
463
-
464
- `{state, profile, posts, comments, via, note}` with `state ∈ ok | partial | blocked | error`,
465
- so the engine's enrich branch treats every network identically and no caller learns a new
466
- shape.
467
-
468
- ⭐ WAVE 30 · T10 — POSTS AND COMMENTS ARE REAL NOW, AND BOTH DEFAULT OFF, exactly as Instagram's
469
- do. `post_metrics` scrapes the profile row's own `top_videos` permalinks (see `tt_post_urls` —
470
- no discovery call, because both TikTok discovery datasets 404 for our key); `comment_metrics`
471
- then scrapes the comments of the posts that came back. ⚠ COMMENTS REQUIRE POSTS by construction
472
- rather than by a rule: their input IS a post permalink, so asking for comments with post capture
473
- off is a request with no subject, and it returns none instead of quietly buying posts nobody
474
- asked for.
475
-
476
- ⛔ `partial` IS THE SUCCESS STATE WHENEVER NO MEDIA WAS READ, and that is deliberate rather than
477
- pessimistic. The Instagram contract reads `ok` only when identity AND media both landed
478
- (`pull_profile_bd`: *"identity without media is still partial ... a run that wrote a follower
479
- count and no posts must not paint green over a posts table that did not grow"*). So: posts not
480
- ASKED for → `partial`, saying so; posts asked for and landed → `ok`; asked for and none came →
481
- `partial` with the vendor's reason. The state answers "did this pull deliver what it went for",
482
- never "did the function finish".
483
-
484
- ⚠ **NO FREE RUNG, AND NO FALLBACK CHAIN.** Instagram's `pull_profile` drops to Apify when the
485
- paid rung refuses; `providers.DEFAULT_CHAINS["tt_profile"]` is deliberately single-provider,
486
- with its own note explaining that a multi-provider chain is a promise something walks it and
487
- that nothing walks Instagram's second name today either. So a refusal here is final, and it
488
- says so instead of implying a retry somewhere.
489
- """
490
- handle = tt_handle(url)
491
- if not handle:
492
- return {"state": "error", "profile": {}, "posts": [], "comments": [], "via": "",
493
- "note": f"{url!r} is not a TikTok profile URL or handle"}
494
-
495
- # ⭐ THE BATCH FAST PATH, same shape as the Instagram side: `prefetch` is `{handle: node}` from
496
- # one multi-URL scrape covering a whole selection. A hit is a vendor round trip that does not
497
- # happen; a miss falls through to the single-URL call below.
498
- cached = prefetch.get(handle) if isinstance(prefetch, dict) else None
499
- _deferred = []
500
- if isinstance(cached, dict) and cached:
501
- rows, note = [cached], ""
502
- else:
503
- rows, note = bd_scrape(TT_DS_PROFILES, [tt_profile_url(handle)], deferred=_deferred)
504
-
505
- node = rows[0] if rows else {}
506
- profile = normalize_profile(node, handle) if node else {}
507
- # ⛔ THE READABILITY TEST IS `followers`/`following`, NOT "did we get a dict". `normalize_profile`
508
- # drops blanks, so an unreadable row still returns `{"platform": …, "handle": …}` — truthy, and
509
- # carrying nothing anybody asked for. The Instagram rung tests exactly this pair for exactly
510
- # this reason, and answering "0 followers" instead is the failure it exists to prevent.
511
- unreadable = profile.get("followers") is None and profile.get("following") is None
512
- if note or unreadable:
513
- # ⭐ THE DEFERRAL IS HANDED OVER RATHER THAN DISCARDED. A snapshot the vendor is still
514
- # building HAS ALREADY BEEN PAID FOR; dropping its id bills again on the next run for the
515
- # same record. That was live on the Instagram profile path until 2026-08-09 — measured on
516
- # nurilab as two runs, two fresh snapshots, both abandoned — and it is not being
517
- # reintroduced here by omission.
518
- if isinstance(pending_profile, list):
519
- for d in _deferred:
520
- pending_profile.append({**d, "kind": "profile", "influencer": handle})
521
- why = note or ("the scrape answered, but no follower/following counts were readable in it "
522
- "(the field names may have moved - see tiktok-capture.md)")
523
- return {"state": "blocked", "profile": {}, "posts": [], "comments": [], "via": "brightdata",
524
- "deferredProfile": [d.get("snapshotId") for d in _deferred],
525
- "note": why}
526
-
527
- # --- W30-T10: THE MEDIA, ONLY WHEN IT WAS ASKED FOR. ------------------------------------
528
- if not post_metrics:
529
- return {"state": "partial", "profile": profile, "posts": [], "comments": [],
530
- "via": "brightdata",
531
- "note": note or "profile read; post capture is off for this step"}
532
- urls = tt_post_urls(node, limit=max_posts)
533
- if not urls:
534
- # ⚠ NOT AN ERROR AND NOT A RETRY. A creator with no `top_videos` has nothing to buy, and
535
- # saying so is what stops the next run paying to be told the same thing.
536
- return {"state": "partial", "profile": profile, "posts": [], "comments": [],
537
- "via": "brightdata",
538
- "note": note or "profile read; this account's row carried no post links"}
539
- posts, p_note = pull_posts_tt(urls, log=log, deferred=_deferred)
540
- comments, c_note = ([], "")
541
- if comment_metrics and posts:
542
- # The comments dataset is keyed on a POST permalink, so it reads the posts we just bought —
543
- # `url` from the mapper, never the profile's raw array, so a post the posts scrape refused
544
- # is not silently asked about again one rung later.
545
- comments, c_note = pull_comments_tt([p.get("url") for p in posts if p.get("url")],
546
- log=log, deferred=_deferred)
547
- # ⭐⭐ WAVE 30 · D-156 — THE MEDIA DEFERRALS ARE HANDED BACK, and the shape of the hand-off is
548
- # the whole lesson. An earlier draft of T10 appended every `_deferred` entry to
549
- # `pending_profile` tagged `kind: "profile"`. By the time control reaches here a PROFILE
550
- # deferral is impossible — the profile branch above returns `blocked` on any note — so **every
551
- # id fanned out that way was a POSTS or COMMENTS snapshot in the PROFILE queue**, whose
552
- # collector writes preset profile cells onto somebody's record from post rows. The engine keeps
553
- # the two queues apart deliberately (`_pending_profile_tasks` vs `_pending_metric_tasks`).
554
- # ⇒ So this returns them under their OWN key, filtered by dataset identity
555
- # (`_media_deferrals`), and the engine files them in the metric queue with the handle it
556
- # already holds. Returning rather than appending also keeps the queue's vocabulary out of a
557
- # connector: this module knows which CORPUS deferred, never what the engine calls it.
558
- # ⚠ `deferredMedia` rides BOTH returns on purpose. The empty-posts case is the one that
559
- # matters most — that is exactly the run where the vendor took too long, so a caller reading
560
- # the ids only from the success path would lose every batch it actually paid for.
561
- deferred_media = _media_deferrals(_deferred)
562
- if not posts:
563
- return {"state": "partial", "profile": profile, "posts": [], "comments": [],
564
- "via": "brightdata", "deferredMedia": deferred_media,
565
- "note": p_note or note or "profile read; the post source returned nothing"}
566
- return {"state": "ok", "profile": profile, "posts": posts, "comments": comments,
567
- "via": "brightdata", "deferredMedia": deferred_media,
568
- "note": c_note or note or ""}
 
1
+ """connectors_tt.py — the TIKTOK connector (wave 29 · item 7 · DEBT D-9 · rulings R1 + R2).
2
+
3
+ Everything in this file knows what a VENDOR's TikTok row looks like. Nothing in it knows what an
4
+ automation is. That split is `connectors_ig.py`'s (wave 27 item 23) and it is the reason this file
5
+ exists at all rather than another thousand lines inside the engine.
6
+
7
+ ⛔ **EVERY VENDOR FIELD NAME HERE WAS PROBED, NOT GUESSED.** The whole schema — 40 profile / 43
8
+ post / 17 comment fields, each with the vendor's own type, description and `pii` flag — was read
9
+ live from `GET /datasets/{id}/metadata` for **$0.00** and written down in
10
+ `.claude/wiki/waves/wave29/proto/tiktok-schema.md` (promoted to `tiktok-capture.md` at
11
+ close-out). That document is the AUTHORITY: do not re-probe it, and do not invent a key. Where a
12
+ name below reads through a candidate list it is because the vendor has two names for one fact
13
+ (`biography`/`signature`, `region`/`country`), never because the name is uncertain.
14
+
15
+ ⛔ **NOTHING HERE EVER AUTHENTICATES TO TIKTOK.** No login, no cookie, no account to get banned —
16
+ public data through a supplier, exactly the rail `connectors_ig.py` states for Instagram. The
17
+ vendor key is a key to a SUPPLIER.
18
+
19
+ ⭐ **THE TRANSPORT IS `connectors_bd.py` — SHARED, VENDOR-NAMED, AND NO LONGER BORROWED FROM THE
20
+ OTHER PLATFORM'S CONNECTOR** (WAVE 30 · T09, DEBT D-128). `bd_call`, `bd_scrape` and
21
+ `bd_filter_start` take the dataset id as a PARAMETER — they are Bright Data's wire, not
22
+ Instagram's — and re-implementing them here would be a second copy of the deferral handling, the
23
+ truncation guard, the SSRF rail and the snapshot-progress reader, i.e. five places for one bug.
24
+ Until wave 30 the code was right and the NAME was wrong: this file imported thirteen symbols from
25
+ `connectors_ig`, which read as a dependency on Instagram and was really a dependency on a supplier.
26
+ ⚠ **This file now imports ZERO names from `connectors_ig`, and a gate check asserts that**, because
27
+ the sentence above is the kind that quietly stops being true.
28
+
29
+ ⚠ **WHAT $0 COULD NOT BUY, so nobody reads this file as more measured than it is:**
30
+ 1. the real ROW shape — `/metadata` describes a DATASET, and Instagram's rows carry undeclared
31
+ envelope keys (`timestamp`, `input`) that no metadata call mentions;
32
+ 2. that a declared field POPULATES — Bright Data's Instagram Reels *declares* `views: number`
33
+ and delivers an account-grain wrong number (§4e). **Declared is not delivered**, and the one
34
+ TikTok claim that matters most (`play_count`) is exactly a declaration.
35
+ """
36
+ from __future__ import annotations
37
+
38
+ import automation_engine as engine
39
+ # ⚠ T09 — FOUR NAMES CAME OFF THIS LIST AND NOTHING BROKE, which is the point of deriving an
40
+ # import block from the AST both ways. `bd_call`, `bd_filter_start`, `bd_key` and `bd_ready` were
41
+ # imported here under a comment claiming they were *"re-exported for the runners"*; no runner ever
42
+ # read them off this module (the engine imports them from the transport itself), so they were four
43
+ # lines of dependency nobody was paying for. `[[artifact-with-no-importer]]` in its smallest form.
44
+ from connectors_bd import (
45
+ _bd_first_url,
46
+ _bd_flag,
47
+ _bd_list,
48
+ _bd_source_payload,
49
+ _first,
50
+ _ig_int,
51
+ bd_scrape,
52
+ )
53
+
54
+ # ---------------------------------------------------------------------------------------------
55
+ # THE DATASETS
56
+ # ---------------------------------------------------------------------------------------------
57
+ # ⚠ CATALOGUE PRESENCE IS NOT ENTITLEMENT — the same finding Instagram produced. `GET
58
+ # /datasets/list` returned 1,735 rows of which 12 are TikTok; the three below answer 200 with a
59
+ # full field list, and the two DISCOVERY halves answer **404 for our key**:
60
+ # `gd_lj71gn6l68bz7y9hc` (posts by profile) and `gd_lilwhto81z415d9mdl` (posts by keyword).
61
+ # ⇒ TikTok discovery routes through the PROFILES dataset's corpus filter, exactly as Instagram's
62
+ # does. A ticket that reaches for a by-keyword endpoint is reaching for a 404.
63
+ TT_DS_PROFILES = "gd_l1villgoiiidt09ci" # TikTok - Profiles. 40 fields, 152,000,000 records
64
+ TT_DS_POSTS = "gd_lu702nij2f790tmv9h" # TikTok - Posts. 43 fields
65
+ TT_DS_COMMENTS = "gd_lkf2st302ap89utw5k" # TikTok - Comments. 17 fields
66
+
67
+ #: The vendor's two post-type tokens, verbatim from the dataset's own `ai_description`
68
+ #: (*"strictly these two"*, video = 99.5% of rows). Ours are `image`/`video`/`carousel`.
69
+ TT_POST_TYPE_VIDEO = "video"
70
+ TT_POST_TYPE_CONTENT = "content"
71
+
72
+ #: What a TikTok profile URL looks like, for the runner that has a handle and needs a URL. Kept
73
+ #: beside the dataset ids because it is the same class of vendor fact.
74
+ #: ⚠ ONE SPELLING. `platform/core/user_tables.profile_url(handle, 'tiktok')` builds the identical
75
+ #: string from `_PROFILE_RULES` (contract C2, E's half) — this exists for the connector's own
76
+ #: batch calls, and the gate asserts the two agree rather than trusting that they do.
77
+ TT_PROFILE_URL = "https://www.tiktok.com/@{handle}"
78
+
79
+
80
+ def tt_profile_url(handle):
81
+ """`nurilab` → `https://www.tiktok.com/@nurilab`. `''` for a blank handle, never a bare `@`."""
82
+ h = str(handle or "").strip().lstrip("@")
83
+ return TT_PROFILE_URL.format(handle=h) if h else ""
84
+
85
+
86
+ # ---------------------------------------------------------------------------------------------
87
+ # THE FIELD MAPS
88
+ # ---------------------------------------------------------------------------------------------
89
+ # Each function turns ONE vendor row into the cell dict for one of the `ut_tt_*` schemas declared
90
+ # in `automation_engine`. The rules they all obey, stated once:
91
+ #
92
+ # * **BLANK MEANS NOT READ, NEVER "THEY HAVE NONE".** A key the vendor did not send is OMITTED,
93
+ # so a later, richer pull fills it instead of being overwritten by this one's silence. `_first`
94
+ # returns `None` (never 0) when nothing matches, which is what makes that possible.
95
+ # * **A zero from the vendor is a MEASUREMENT and survives.** (Instagram's `_ig_zero_is_blank`
96
+ # rule is scoped to a paid rung whose zeros were proven fictional; nothing here has earned it.)
97
+ # * **Every unpromoted vendor key stays whole in `source_payload`.** A schema addition on the
98
+ # vendor's side is preserved rather than silently discarded while our column model catches up.
99
+ # * **Nothing here writes a session token.** `tt_chain_token`, `secu_id` (~85% null), `short_id`
100
+ # (100% null in the sample), `ftc` (100% null) and `relation` are deliberately unmapped; they
101
+ # ride in `source_payload` where they make no claim.
102
+
103
+
104
+ def _tt_str(node, *names):
105
+ """The first non-empty string among `names`, or None when the vendor sent nothing.
106
+
107
+ ⛔ `None`, NOT `""`. The callers below drop `None` keys, which is what keeps a blank honest —
108
+ an empty string written into a cell claims "we looked and it is empty".
109
+ """
110
+ v = _first(node, *names)
111
+ if v is None:
112
+ return None
113
+ s = str(v).strip()
114
+ return s or None
115
+
116
+
117
+ def _tt_pct(node, *names):
118
+ """A vendor 0–1 engagement fraction → our stored 0–100 percentage, or None.
119
+
120
+ ⛔ THE ×100 IS NOT COSMETIC (wave 26, amendment C1-a). Our `pct` renderer appends the sign to
121
+ the STORED number, so writing the vendor's raw 0.0656 would print a 6.6% creator as `0.0%` —
122
+ measured on the Instagram side, and TikTok sends the same shape on all three of its rates.
123
+ """
124
+ v = _first(node, *names)
125
+ if v is None:
126
+ return None
127
+ out = engine._pct100(v)
128
+ return out or None
129
+
130
+
131
+ def _tt_day(node, *names):
132
+ """A vendor stamp → `YYYY-MM-DD`, or None. Our `date` columns store a day."""
133
+ v = _first(node, *names)
134
+ if v is None:
135
+ return None
136
+ return engine._day(v) or None
137
+
138
+
139
+ def _drop_blanks(row):
140
+ """The one place a mapped row loses its `None`s — see the BLANK MEANS NOT READ rule above."""
141
+ return {k: v for k, v in row.items() if v is not None and v != ""}
142
+
143
+
144
+ def normalize_profile(node, handle=""):
145
+ """A TikTok Profiles row → the `ut_tt_profile` / `ut_tt_snapshots` cell shape.
146
+
147
+ ⚠ TWO FIELDS READ THROUGH A CANDIDATE PAIR, and both pairs are the vendor's, not a guess:
148
+ * `biography` is PRIMARY and `signature` the FALLBACK — the probe measured `signature`
149
+ populated on 85% of rows and they carry the same text;
150
+ * `region` is PRIMARY and `country` the FALLBACK — `region` is the one with a documented
151
+ two-letter-ISO description, `country` has no description at all.
152
+ ⚠ `videos_count` is mapped to `posts_count` with a caveat recorded rather than hidden: its
153
+ `ai_description` ranges 1-89, so it may be a WINDOW rather than a lifetime total. It is the
154
+ only count of its kind the dataset offers.
155
+ """
156
+ node = node if isinstance(node, dict) else {}
157
+ account = _tt_str(node, "account_id") or str(handle or "").strip().lstrip("@")
158
+ return _drop_blanks({
159
+ "platform": engine.PLATFORM_TIKTOK,
160
+ "handle": account,
161
+ "full_name": _tt_str(node, "nickname"),
162
+ "tt_id": _tt_str(node, "id"),
163
+ "profile_url": _tt_str(node, "url") or (tt_profile_url(account) or None),
164
+ "bio": _tt_str(node, "biography", "signature"),
165
+ "external_url": _bd_first_url(_first(node, "bio_link")),
166
+ "verified": _bd_flag(node, "is_verified"),
167
+ "is_private": _bd_flag(node, "is_private"),
168
+ # ⚠ APPROXIMATE, and the word is the vendor's: `is_commerce_user` has *"many null values"*
169
+ # on their own description. It is the closest thing TikTok has to Instagram's
170
+ # `is_business_account`, and `_bd_flag` writes nothing at all when the key is absent — so
171
+ # the approximation only ever fills a cell the vendor actually answered.
172
+ "is_business": _bd_flag(node, "is_commerce_user"),
173
+ "followers": _ig_int(_first(node, "followers")),
174
+ "following": _ig_int(_first(node, "following")),
175
+ "posts_count": _ig_int(_first(node, "videos_count")),
176
+ # ⚠ `likes` on a PROFILE row is likes RECEIVED across the account's videos (18-110,200, no
177
+ # nulls). It is not a post-level number and it is not our `likes` column, which is why it
178
+ # is stored under a different name.
179
+ "likes_received": _ig_int(_first(node, "likes")),
180
+ "avg_engagement": _tt_pct(node, "awg_engagement_rate"),
181
+ "like_engagement": _tt_pct(node, "like_engagement_rate"),
182
+ "comment_engagement": _tt_pct(node, "comment_engagement_rate"),
183
+ "country_code": _tt_str(node, "region", "country"),
184
+ "region": _tt_str(node, "region"),
185
+ "predicted_lang": _tt_str(node, "predicted_lang"),
186
+ # ⚠ ACCOUNT AGE, NOT A MEASUREMENT STAMP — `create_time` on a profile is when the ACCOUNT
187
+ # was made. TikTok stamps nothing with "when this number was true", exactly like Instagram,
188
+ # which is why the append law dates a snapshot by when WE read it.
189
+ "account_created_at": _tt_day(node, "create_time"),
190
+ "source_payload": _bd_source_payload(node),
191
+ })
192
+
193
+
194
+ def tt_post_type(node):
195
+ """A TikTok post row → one of OUR three type options, or None.
196
+
197
+ ⛔ THE FIRST OF THE PROBE DOC'S TWO NAMED BLOCKERS. The vendor's vocabulary is `"video"` /
198
+ `"content"`; ours is `image` / `video` / `carousel` and has no `"content"`. Writing the
199
+ vendor's token would fail `_clean_field`'s option check on the way in and would put an
200
+ untranslated API word in front of a user on the way out.
201
+
202
+ So: `video` is `video`, and `content` — TikTok's photo-mode post — is `image`, EXCEPT when the
203
+ row carries more than one `carousel_images` entry, which is what a carousel IS on either
204
+ network. ⚠ The multi-image branch is decided from the IMAGES, never from the type token: the
205
+ token cannot express it, so inferring `carousel` from the word would be inventing a fact.
206
+ ⚠ An UNKNOWN token returns None rather than defaulting to `video` (99.5% of rows are video, and
207
+ that is exactly what would make the wrong default invisible).
208
+ """
209
+ raw = str((node or {}).get("post_type") or "").strip().lower()
210
+ if raw == TT_POST_TYPE_VIDEO:
211
+ return "video"
212
+ if raw != TT_POST_TYPE_CONTENT:
213
+ return None
214
+ images = (node or {}).get("carousel_images")
215
+ return "carousel" if isinstance(images, list) and len(images) > 1 else "image"
216
+
217
+
218
+ def normalize_post(node):
219
+ """A TikTok Posts row → the `ut_tt_posts` cell shape. None when it carries no identity.
220
+
221
+ ⛔ THE SECOND NAMED BLOCKER, RESOLVED HERE AND NOWHERE ELSE: `play_count` is ONE number and
222
+ Instagram's schema has TWO columns for it (`plays` and `views`). On TikTok they are the same
223
+ fact — `play_count` IS the count TikTok displays under a video — so it maps to `views` and
224
+ `ut_tt_posts` HAS NO `plays` COLUMN. Copying one vendor number into two of our columns would
225
+ manufacture a second measurement that a rollup could average or double-count, which is a worse
226
+ outcome than the missing column it would paper over.
227
+
228
+ ⚠ `shortcode` reads `shortcode` then `post_id`: both are 19-digit numerics on this dataset and
229
+ the probe measured them as the same shape. That equality is what lets the comments→posts link
230
+ join with NO normaliser, which the Instagram side never had.
231
+ ⚠ `num_share_count` (a number) is preferred over `share_count` (typed TEXT by the vendor).
232
+ ⚠ `commerce_info` is a business/commerce LOCATION per its own description — cities and
233
+ countries. It is NOT a paid-partnership flag, and nothing in this dataset is: TikTok declares
234
+ no equivalent, so `paid_partnership`/`partner` have no column on this family at all.
235
+ """
236
+ node = node if isinstance(node, dict) else {}
237
+ shortcode = _tt_str(node, "shortcode", "post_id")
238
+ if not shortcode:
239
+ return None
240
+ return _drop_blanks({
241
+ "platform": engine.PLATFORM_TIKTOK,
242
+ "shortcode": shortcode,
243
+ # ⛔⛔ `account_id`, NOT `profile_username` — MEASURED on a real Posts row 2026-08-12.
244
+ # The vendor's `profile_username` is the DISPLAY NAME (`"Dina"`), while `account_id` is the
245
+ # @handle (`"d1na_th"`) — the same field `normalize_profile` already reads for `handle`, so
246
+ # one name means one thing across both corpora. Reading the display name silently broke the
247
+ # only join this table has: `ut_tt_posts.influencer_key` -> `ut_tt_profile.handle` matched
248
+ # NOTHING, so a person could not filter posts by creator and a rollup would count zero.
249
+ # ⚠ NO FALLBACK TO `profile_username`, deliberately. It is not a degraded handle, it is a
250
+ # different fact, and filling a join key with it is worse than leaving it blank — a blank
251
+ # is visibly missing, a display name looks like an answer [[one-question-two-normalizers]].
252
+ # The URL is the honest second source: it carries the handle by construction.
253
+ "influencer_key": (_tt_str(node, "account_id")
254
+ or tt_handle(_tt_str(node, "profile_url", "url") or "") or None),
255
+ "posted_at": _tt_day(node, "create_time"),
256
+ "type": tt_post_type(node),
257
+ "caption": _tt_str(node, "description"),
258
+ "url": _tt_str(node, "url"),
259
+ "hashtags": _bd_list(node, "hashtags"),
260
+ "tagged_location": _tt_str(node, "commerce_info"),
261
+ "views": _ig_int(_first(node, "play_count")),
262
+ "likes": _ig_int(_first(node, "digg_count")),
263
+ "comments": _ig_int(_first(node, "comment_count")),
264
+ "shares": _ig_int(_first(node, "num_share_count")),
265
+ "saves": _ig_int(_first(node, "collect_count")),
266
+ "video_duration": _ig_int(_first(node, "video_duration")),
267
+ "source_payload": _bd_source_payload(node),
268
+ })
269
+
270
+
271
+ def normalize_comment(node):
272
+ """A TikTok Comments row → the `ut_tt_comments` cell shape. None without a comment id.
273
+
274
+ ⚠ THE NAME COLLISION, RESOLVED: TikTok's `replies` is an ARRAY of reply objects and OUR
275
+ `replies` column is an INT count. The count comes from `num_replies`; the array stays whole in
276
+ `source_payload`. Reading the array's length instead would be a second, disagreeing answer to
277
+ a question the vendor already answers — and it would disagree, because a page of replies is not
278
+ all of them.
279
+ ⚠ `date_created` is typed `date` by this vendor, unlike Instagram's `comment_date` which is
280
+ text and needs defensive parsing. It still goes through `_tt_day` — one date path, so a vendor
281
+ that changes its mind cannot change ours.
282
+ ⚠ The comment TEXT and every identifiable commenter field (`commenter_user_name` is flagged
283
+ PII) stay in `source_payload` and are promoted to no column, which is the same posture the
284
+ Instagram comment schema takes for the same D-24 reason.
285
+ """
286
+ node = node if isinstance(node, dict) else {}
287
+ comment_key = _tt_str(node, "comment_id")
288
+ if not comment_key:
289
+ return None
290
+ return _drop_blanks({
291
+ "platform": engine.PLATFORM_TIKTOK,
292
+ "comment_key": comment_key,
293
+ "shortcode": _tt_str(node, "post_id"),
294
+ # ⭐⭐ OWNER RULING 2026-08-12: the comment's CONTENT gets a column. `comment_text` is the
295
+ # vendor's own key and `comment_text_only` its stripped variant (the probe recorded both);
296
+ # primary first, so a row carrying the rich form is not silently served the plain one.
297
+ # ⛔ The commenter's identity is deliberately NOT promoted — see `TT_COMMENT_FIELDS`.
298
+ "text": _tt_str(node, "comment_text", "comment_text_only"),
299
+ "commented_at": _tt_day(node, "date_created"),
300
+ "likes": _ig_int(_first(node, "num_likes")),
301
+ "replies": _ig_int(_first(node, "num_replies")),
302
+ "source_payload": _bd_source_payload(node),
303
+ })
304
+
305
+
306
+ #: ⭐ The three maps, addressable by name — so a gate (and the runners in T04-T06) can walk them
307
+ #: rather than naming three functions, and so adding a fourth dataset is one entry.
308
+ TT_NORMALIZERS = {
309
+ "tt_profile": normalize_profile,
310
+ "tt_post_metrics": normalize_post,
311
+ "tt_comments": normalize_comment,
312
+ }
313
+
314
+
315
+ # ---------------------------------------------------------------------------------------------
316
+ # THE FETCH — wave 30 · W30-T08 (carrying wave-29's dropped T05)
317
+ # ---------------------------------------------------------------------------------------------
318
+
319
+ def tt_handle(url):
320
+ """A TikTok profile URL **or** a bare handle → the handle. `''` when it is neither.
321
+
322
+ Deliberately permissive about the input and strict about the output, because the two callers
323
+ hand it different things: an automation stores whatever a person typed in the profile column
324
+ (`@nurilab`, `nurilab`, or the full URL), while the discovery runner already holds a clean
325
+ `account_id`. One normaliser, so a row found by discovery and a row typed by hand cannot
326
+ resolve to two different handles.
327
+ """
328
+ s = str(url or "").strip()
329
+ if not s:
330
+ return ""
331
+ if "tiktok.com" in s.lower():
332
+ # Everything after the first `@`, up to the next path segment or query.
333
+ tail = s.split("@", 1)[1] if "@" in s else ""
334
+ s = tail.split("/")[0].split("?")[0].split("#")[0]
335
+ s = s.strip().lstrip("@").strip()
336
+ # A handle is the vendor's `account_id` shape: alphanumerics, dots and underscores.
337
+ return s if s and all(c.isalnum() or c in "._" for c in s) else ""
338
+
339
+
340
+ def tt_post_urls(node, limit=0):
341
+ """⭐ WAVE 30 · T10 — the profile row's own post permalinks, newest-first as the vendor sends.
342
+
343
+ ⛔ THIS IS WHY TIKTOK POST CAPTURE COSTS NO EXTRA DISCOVERY. `top_videos` rides the PROFILE row
344
+ we have already bought, so the posts read is a scrape of links we hold, never a search for them.
345
+ The two TikTok DISCOVERY datasets (posts-by-profile, posts-by-keyword) are **404 for our key**,
346
+ so a design that reached for either would not merely be dearer, it would not work.
347
+
348
+ ⛔⛔ CORRECTED 2026-08-12 — THIS DOCSTRING USED TO SAY *"the probe MEASURED `top_videos` as an
349
+ array of video permalinks, NO empties"*, AND THAT SENTENCE IS WHAT SHIPPED THE BUG. The probe
350
+ read `/datasets/{id}/metadata` — a DATASET description — and the phrase quoted was the field's
351
+ `ai_description`, not an observation of a row. The real row sends **dicts keyed `video_url`**
352
+ (measured below), so the reader built against the quoted sentence found nothing, forever, in
353
+ silence. ⭐ The transferable half: *"the probe measured X"* and *"the probe read a declaration
354
+ of X"* are different claims, and prose cannot be told apart by a reader downstream — which is
355
+ why the correction names the method, not just the value.
356
+
357
+ ⚠ `top_posts_data` is deliberately NOT read: the probe calls it *"a thin dup of `top_videos`"*,
358
+ and preferring whichever happened to be longer is how one creator's window silently differs
359
+ from another's.
360
+ ⚠ `limit <= 0` means "everything the row carried". The CAP IS THE CALLER'S — `config.maxPosts`,
361
+ validated 1..12 — and it is applied here rather than after the scrape so an unwanted post is
362
+ never bought. [[a-constant-two-features-share]]: the 12 is the vendor's measured profile window,
363
+ not a number this function may invent.
364
+ """
365
+ raw = (node or {}).get("top_videos")
366
+ out = []
367
+ for item in raw if isinstance(raw, list) else []:
368
+ # ⛔⛔ MEASURED ON A REAL ROW 2026-08-12, AND IT IS NOT WHAT THE SCHEMA SAID.
369
+ # `top_videos` is NOT an array of permalink strings. One paid TikTok Profiles scrape of a
370
+ # live handle returns **19 DICTS**, keyed
371
+ # `video_url · video_id · playcount · diggcount · commentcount · share_count ·
372
+ # favorites_count · create_date · cover_image`.
373
+ # The docstring above cites the $0 probe as having "measured" permalinks — it had not, and
374
+ # could not: `/datasets/{id}/metadata` describes a DATASET, and the probe's own verdict says
375
+ # so in terms (*"what $0 cannot buy … the real ROW shape … that a declared field
376
+ # POPULATES"*, `wave29/proto/tiktok-schema.md`). This is the SECOND time this vendor's
377
+ # declaration has diverged from its delivery on this exact axis; BD's IG Reels `views` was
378
+ # the first. [[reachable-is-not-the-same-as-built]]
379
+ # ⇒ The consequence, live: `TT_DS_POSTS` was never reached, because this returned an EMPTY
380
+ # list on every real profile — post capture could not have worked for anybody, and the
381
+ # T10 gate stayed green because its canned fixture encoded the DECLARED shape. A fixture
382
+ # written from a schema tests the schema.
383
+ # ⚠ `video_url` is FIRST because it is the key the vendor actually sends; `url` is kept
384
+ # because it costs nothing and is what a future corpus revision would most likely use. The
385
+ # bare-string branch stays for the same reason — this widens what is ACCEPTED and invents
386
+ # nothing: a shape that yields no `http…` value still degrades to "no posts", exactly as
387
+ # before, rather than to a URL built out of a guess.
388
+ # ⚠ `top_posts_data` is STILL not read (it carries `post_url` and would work): preferring
389
+ # whichever array happened to be longer is how one creator's window silently differs from
390
+ # another's, and that reasoning is unchanged by this correction.
391
+ if isinstance(item, dict):
392
+ u = str(item.get("video_url") or item.get("url") or "").strip()
393
+ else:
394
+ u = str(item or "").strip()
395
+ if u.lower().startswith("http") and u not in out:
396
+ out.append(u)
397
+ return out[:limit] if limit and limit > 0 else out
398
+
399
+
400
+ def pull_posts_tt(post_urls, log=print, deferred=None):
401
+ """The TikTok Posts dataset for a list of permalinks → `(rows, note)`, already normalised.
402
+
403
+ ⚠ ONE CALL FOR THE WHOLE WINDOW. `bd_scrape` has always taken a list, and the Instagram side
404
+ measured what happens when a caller forgets: 25 records, one billed snapshot each, a walk still
405
+ running at 67 minutes. Nothing here loops per URL.
406
+ """
407
+ urls = [str(u) for u in (post_urls or []) if str(u or "").strip()]
408
+ if not urls:
409
+ return [], ""
410
+ rows, note = bd_scrape(TT_DS_POSTS, urls, deferred=deferred)
411
+ if note:
412
+ log(f"[aios-tt] posts: {note}")
413
+ return [], note
414
+ out = [r for r in (normalize_post(n) for n in rows) if r]
415
+ return out, ""
416
+
417
+
418
+ def pull_comments_tt(post_urls, log=print, deferred=None):
419
+ """The TikTok Comments dataset for a list of POST permalinks → `(rows, note)`, normalised.
420
+
421
+ ⛔ THE MOST EXPENSIVE THING THIS PRODUCT BUYS, and the reason `commentMetrics` defaults OFF on
422
+ both networks: a comments scrape ingests identifiable third parties who never entered anybody's
423
+ list (D-24). The mapper already keeps every commenter field in `source_payload` and promotes
424
+ none of them to a column; this function adds no new exposure, it just has to be asked for.
425
+ """
426
+ urls = [str(u) for u in (post_urls or []) if str(u or "").strip()]
427
+ if not urls:
428
+ return [], ""
429
+ rows, note = bd_scrape(TT_DS_COMMENTS, urls, deferred=deferred)
430
+ if note:
431
+ log(f"[aios-tt] comments: {note}")
432
+ return [], note
433
+ out = [r for r in (normalize_comment(n) for n in rows) if r]
434
+ return out, ""
435
+
436
+
437
+ #: ⭐⭐ WAVE 30 · D-156 — THE MEDIA DATASETS, AS A SET, SO THE HAND-OFF CAN FILTER ON IDENTITY.
438
+ #: `_media_deferrals` uses this to lift ONLY posts/comments snapshots out of the local deferral
439
+ #: list. That is what makes it structurally impossible to file a PROFILE snapshot in the engine's
440
+ #: metric queue — the defect a draft of T10 shipped and A-39 booked as "the wrong fix is worse
441
+ #: than the gap". A membership test cannot be got wrong by a later edit the way `if` order can.
442
+ TT_MEDIA_DATASETS = (TT_DS_POSTS, TT_DS_COMMENTS)
443
+
444
+
445
+ def _media_deferrals(deferred):
446
+ """The POSTS/COMMENTS entries of a `bd_scrape` deferral list — never the profile's.
447
+
448
+ ⚠ The engine, not this module, decides what a deferral MEANS: it stamps `kind` and the
449
+ handle and files it. TikTok needs no `_tag_metric_deferrals` twin because one dataset is one
450
+ kind here, so the id already carries everything a mapper choice depends on — and importing
451
+ Instagram's tagger is not available anyway (W30-T09 gates ZERO `from connectors_ig` lines).
452
+ """
453
+ out = []
454
+ for d in deferred or []:
455
+ if isinstance(d, dict) and str(d.get("datasetId") or "") in TT_MEDIA_DATASETS:
456
+ out.append(dict(d))
457
+ return out
458
+
459
+
460
+ def pull_profile_tt(url, log=print, pending_profile=None, prefetch=None,
461
+ max_posts=0, post_metrics=False, comment_metrics=False):
462
+ """ONE TikTok profile from the vendor. Same return contract as `pull_profile`.
463
+
464
+ `{state, profile, posts, comments, via, note}` with `state ∈ ok | partial | blocked | error`,
465
+ so the engine's enrich branch treats every network identically and no caller learns a new
466
+ shape.
467
+
468
+ ⭐ WAVE 30 · T10 — POSTS AND COMMENTS ARE REAL NOW, AND BOTH DEFAULT OFF, exactly as Instagram's
469
+ do. `post_metrics` scrapes the profile row's own `top_videos` permalinks (see `tt_post_urls` —
470
+ no discovery call, because both TikTok discovery datasets 404 for our key); `comment_metrics`
471
+ then scrapes the comments of the posts that came back. ⚠ COMMENTS REQUIRE POSTS by construction
472
+ rather than by a rule: their input IS a post permalink, so asking for comments with post capture
473
+ off is a request with no subject, and it returns none instead of quietly buying posts nobody
474
+ asked for.
475
+
476
+ ⛔ `partial` IS THE SUCCESS STATE WHENEVER NO MEDIA WAS READ, and that is deliberate rather than
477
+ pessimistic. The Instagram contract reads `ok` only when identity AND media both landed
478
+ (`pull_profile_bd`: *"identity without media is still partial ... a run that wrote a follower
479
+ count and no posts must not paint green over a posts table that did not grow"*). So: posts not
480
+ ASKED for → `partial`, saying so; posts asked for and landed → `ok`; asked for and none came →
481
+ `partial` with the vendor's reason. The state answers "did this pull deliver what it went for",
482
+ never "did the function finish".
483
+
484
+ ⚠ **NO FREE RUNG, AND NO FALLBACK CHAIN.** Instagram's `pull_profile` drops to Apify when the
485
+ paid rung refuses; `providers.DEFAULT_CHAINS["tt_profile"]` is deliberately single-provider,
486
+ with its own note explaining that a multi-provider chain is a promise something walks it and
487
+ that nothing walks Instagram's second name today either. So a refusal here is final, and it
488
+ says so instead of implying a retry somewhere.
489
+ """
490
+ handle = tt_handle(url)
491
+ if not handle:
492
+ return {"state": "error", "profile": {}, "posts": [], "comments": [], "via": "",
493
+ "note": f"{url!r} is not a TikTok profile URL or handle"}
494
+
495
+ # ⭐ THE BATCH FAST PATH, same shape as the Instagram side: `prefetch` is `{handle: node}` from
496
+ # one multi-URL scrape covering a whole selection. A hit is a vendor round trip that does not
497
+ # happen; a miss falls through to the single-URL call below.
498
+ cached = prefetch.get(handle) if isinstance(prefetch, dict) else None
499
+ _deferred = []
500
+ if isinstance(cached, dict) and cached:
501
+ rows, note = [cached], ""
502
+ else:
503
+ rows, note = bd_scrape(TT_DS_PROFILES, [tt_profile_url(handle)], deferred=_deferred)
504
+
505
+ node = rows[0] if rows else {}
506
+ profile = normalize_profile(node, handle) if node else {}
507
+ # ⛔ THE READABILITY TEST IS `followers`/`following`, NOT "did we get a dict". `normalize_profile`
508
+ # drops blanks, so an unreadable row still returns `{"platform": …, "handle": …}` — truthy, and
509
+ # carrying nothing anybody asked for. The Instagram rung tests exactly this pair for exactly
510
+ # this reason, and answering "0 followers" instead is the failure it exists to prevent.
511
+ unreadable = profile.get("followers") is None and profile.get("following") is None
512
+ if note or unreadable:
513
+ # ⭐ THE DEFERRAL IS HANDED OVER RATHER THAN DISCARDED. A snapshot the vendor is still
514
+ # building HAS ALREADY BEEN PAID FOR; dropping its id bills again on the next run for the
515
+ # same record. That was live on the Instagram profile path until 2026-08-09 — measured on
516
+ # nurilab as two runs, two fresh snapshots, both abandoned — and it is not being
517
+ # reintroduced here by omission.
518
+ if isinstance(pending_profile, list):
519
+ for d in _deferred:
520
+ pending_profile.append({**d, "kind": "profile", "influencer": handle})
521
+ why = note or ("the scrape answered, but no follower/following counts were readable in it "
522
+ "(the field names may have moved - see tiktok-capture.md)")
523
+ return {"state": "blocked", "profile": {}, "posts": [], "comments": [], "via": "brightdata",
524
+ "deferredProfile": [d.get("snapshotId") for d in _deferred],
525
+ "note": why}
526
+
527
+ # --- W30-T10: THE MEDIA, ONLY WHEN IT WAS ASKED FOR. ------------------------------------
528
+ if not post_metrics:
529
+ return {"state": "partial", "profile": profile, "posts": [], "comments": [],
530
+ "via": "brightdata",
531
+ "note": note or "profile read; post capture is off for this step"}
532
+ urls = tt_post_urls(node, limit=max_posts)
533
+ if not urls:
534
+ # ⚠ NOT AN ERROR AND NOT A RETRY. A creator with no `top_videos` has nothing to buy, and
535
+ # saying so is what stops the next run paying to be told the same thing.
536
+ return {"state": "partial", "profile": profile, "posts": [], "comments": [],
537
+ "via": "brightdata",
538
+ "note": note or "profile read; this account's row carried no post links"}
539
+ posts, p_note = pull_posts_tt(urls, log=log, deferred=_deferred)
540
+ comments, c_note = ([], "")
541
+ if comment_metrics and posts:
542
+ # The comments dataset is keyed on a POST permalink, so it reads the posts we just bought —
543
+ # `url` from the mapper, never the profile's raw array, so a post the posts scrape refused
544
+ # is not silently asked about again one rung later.
545
+ comments, c_note = pull_comments_tt([p.get("url") for p in posts if p.get("url")],
546
+ log=log, deferred=_deferred)
547
+ # ⭐⭐ WAVE 30 · D-156 — THE MEDIA DEFERRALS ARE HANDED BACK, and the shape of the hand-off is
548
+ # the whole lesson. An earlier draft of T10 appended every `_deferred` entry to
549
+ # `pending_profile` tagged `kind: "profile"`. By the time control reaches here a PROFILE
550
+ # deferral is impossible — the profile branch above returns `blocked` on any note — so **every
551
+ # id fanned out that way was a POSTS or COMMENTS snapshot in the PROFILE queue**, whose
552
+ # collector writes preset profile cells onto somebody's record from post rows. The engine keeps
553
+ # the two queues apart deliberately (`_pending_profile_tasks` vs `_pending_metric_tasks`).
554
+ # ⇒ So this returns them under their OWN key, filtered by dataset identity
555
+ # (`_media_deferrals`), and the engine files them in the metric queue with the handle it
556
+ # already holds. Returning rather than appending also keeps the queue's vocabulary out of a
557
+ # connector: this module knows which CORPUS deferred, never what the engine calls it.
558
+ # ⚠ `deferredMedia` rides BOTH returns on purpose. The empty-posts case is the one that
559
+ # matters most — that is exactly the run where the vendor took too long, so a caller reading
560
+ # the ids only from the success path would lose every batch it actually paid for.
561
+ deferred_media = _media_deferrals(_deferred)
562
+ if not posts:
563
+ return {"state": "partial", "profile": profile, "posts": [], "comments": [],
564
+ "via": "brightdata", "deferredMedia": deferred_media,
565
+ "note": p_note or note or "profile read; the post source returned nothing"}
566
+ return {"state": "ok", "profile": profile, "posts": posts, "comments": comments,
567
+ "via": "brightdata", "deferredMedia": deferred_media,
568
+ "note": c_note or note or ""}
api/deps.py CHANGED
@@ -110,14 +110,7 @@ def _user_for(claims):
110
  # (login validated the slug's existence, never the membership) — with
111
  # AIOS_ENABLE_QA_TENANT=1 that was a working cross-tenant session. A signature proves
112
  # the cookie is ours; THIS proves the account belongs where the cookie says it does.
113
- # ⚠ WAVE 33 (W33-T37) — the default was DELETED, not changed. It read
114
- # `pub.get("tenant", "royal-imports")` and could never fire: `users._public` normalises a
115
- # missing tenant to `royal-imports` before this line ever sees it. A dead default that
116
- # names a real tenant reads like a live fallback, and the cross-tenant audit spent a row
117
- # proving it was not one. Bare `.get` is also the safer shape if that ever stops being
118
- # true: `None != claim_tenant` denies, where the old default would have ADMITTED a
119
- # tenant-less record into tenant #0's session.
120
- if claim_tenant != pub.get("tenant"):
121
  return None
122
  return pub
123
  if (uname == "admin" and os.environ.get("APP_PASSWORD", "")
 
110
  # (login validated the slug's existence, never the membership) — with
111
  # AIOS_ENABLE_QA_TENANT=1 that was a working cross-tenant session. A signature proves
112
  # the cookie is ours; THIS proves the account belongs where the cookie says it does.
113
+ if claim_tenant != pub.get("tenant", "royal-imports"):
 
 
 
 
 
 
 
114
  return None
115
  return pub
116
  if (uname == "admin" and os.environ.get("APP_PASSWORD", "")
api/main.py CHANGED
The diff for this file is too large to render. See raw diff
 
api/odoo_relational.py CHANGED
The diff for this file is too large to render. See raw diff
 
api/routes_admin.py CHANGED
The diff for this file is too large to render. See raw diff
 
api/routes_alerts.py CHANGED
@@ -1,664 +1,537 @@
1
- """routes_alerts.py — the Alerts module (wave 20, owner item 25, contract C-ALERT).
2
-
3
- GET /api/v1/alerts -> {alerts:[...]}
4
- POST /api/v1/alerts <- {viewId, topic, label?}
5
- DELETE /api/v1/alerts/{alert_id}
6
- POST /api/v1/alerts/{alert_id}/run -> evaluate now (the pane's manual refresh)
7
- GET /api/v1/notifications -> {unread, items:[...]}
8
- POST /api/v1/notifications/read <- {ids:[...]|null, read?:bool}
9
-
10
- The semantics — an alert is a view plus a remembered matched set, a notification is a NEW
11
- ENTRANT, and the first evaluation seeds silently — live in `core.alerts` with the reasoning.
12
- This file owns the two things a route must: WHO may do it, and HOW the view gets evaluated.
13
-
14
- ⭐ **THE EVALUATION RUNS AS THE ALERT'S OWNER, NOT AS THE CALLER.** `_run_alert` builds the pool
15
- for `rec['owner']`, never for whoever tripped the write hook. Any other choice leaks: a
16
- full-access admin editing a cell would otherwise evaluate a BU-scoped user's alert over the whole
17
- book, and the notification would name customers that user may not see — a permission leak wearing
18
- a notification's clothes. The owner's own scope is the only correct basis for their alert.
19
-
20
- ⚠ **AN ALERT IS NOT A SECOND READ PATH.** It resolves rows through the same
21
- `routes_customers.grid_assembly` / `routes_tables.ut_assembly` the grid uses, so a row that an
22
- alert can see is by construction a row its owner could open. Re-implementing the filter here
23
- would be a second definition of "matches", and those two would drift.
24
- """
25
- import re
26
-
27
- from fastapi import APIRouter, Body, Depends
28
-
29
- import core.alerts as alerts
30
- from deps import Session, err, require_session
31
-
32
- router = APIRouter(prefix="/api/v1")
33
-
34
- #: The alert-bearing surfaces. `ut_` tables are admitted by prefix, like everywhere else.
35
- _TOPICS = ("customer", "product")
36
-
37
- # ── ⭐⭐ WAVE 32 · T20 · CONTRACT C3 — THE INBOX SHAPE, DERIVED ON READ ────────────────────────
38
- #
39
- # `GET /notifications` gains `subject`, `kind` and `target` per item (`read` was always there).
40
- #
41
- # ⛔ DERIVED, NEVER STORED, AND THAT IS THE WHOLE OF WHY THIS WAVE EXISTS. Stamping the three
42
- # keys onto the record at write time would give them to notifications minted AFTER the deploy and
43
- # to nothing else — every notification already sitting in every tenant's inbox would open nothing,
44
- # and the feature would be correct in the source and absent from the product
45
- # ([[a-migration-that-runs-on-the-next-write]], D-201). A read-side derivation reaches a
46
- # notification queued last month. It also keeps the store shape out of `core/alerts.py`, which is
47
- # another lane's file this wave — but that is the convenience, not the reason.
48
- #
49
- # ⚠ TWO PRODUCERS WRITE TWO SHAPES into one inbox, and the vocabulary below is what tells them
50
- # apart. `_queue` (a record ENTERED a watched view) sets `topic`+`viewId`. `notify()` sets
51
- # `topic='automation'` and puts the producer's key in `alertId`, leaving `viewId` empty. Deciding
52
- # here means the client branches on ONE field instead of re-deriving the same split.
53
- #
54
- # ⛔ **D-101 IS CLOSED HERE, BY SUBTRACTION.** There was a THIRD shape — `kind='automation_review'`
55
- # + `autoId`, a card arriving at a review stage — and its producer `notify_review` was deleted by
56
- # W27/R3 with the review lanes. `automation_engine.py`'s own tombstone (search `notify_review`)
57
- # records the 2026-08-12 sweep: **no `.py` file anywhere produces one**, while the client branch,
58
- # its route and three gate legs stayed fully alive. D-101's exit condition is *"the client review
59
- # branch is deleted in the same change as any remaining residue, OR `notify_review` gains its real
60
- # caller"* — the residue is zero, so the branch goes. It is not carried into the Inbox: a stored
61
- # review notification (if any survives in a tenant from the wave-23 era) derives as an ordinary
62
- # `alert` with no target, i.e. an honest unclickable row, which is correct — the board it pointed
63
- # at was deleted two waves ago.
64
-
65
- #: C3's `kind` vocabulary. Plain strings on the wire — the client must never union over them
66
- #: (alertsModel's wave-9 law: a client union turns "the server grew a kind" into a dropped row).
67
- NOTIF_KIND_ALERT = "alert"
68
- NOTIF_KIND_AUTOMATION = "automation"
69
- NOTIF_KIND_SHARE = "share"
70
-
71
- #: C3's `target.module` vocabulary, and the automation sub-selection.
72
- TARGET_MODULE_DATABASE = "database"
73
- TARGET_MODULE_AUTOMATION = "automation"
74
- TARGET_TAB_RUNS = "runs"
75
-
76
- #: The topic `notify()` carries for a SHARE (W32-T28 writes it; nothing does yet, and a kind with
77
- #: no producer is a string that reads as a feature — the reason this constant is named here and
78
- #: cited from `routes_shares` rather than typed twice).
79
- SHARE_TOPIC = "share"
80
-
81
- #: `core.alerts.notify`'s default topic for a run outcome. Mirrors `inboxModel.AUTOMATION_TOPIC`.
82
- AUTOMATION_TOPIC = "automation"
83
-
84
- _UT_TOPIC = re.compile(r"ut_[A-Za-z0-9_]+\Z")
85
-
86
-
87
- def route_for_topic(topic):
88
- """A grid SCOPE key -> the registry route that renders it, or None.
89
-
90
- ⛔ THE SAME TABLE AS `alertsModel.routeForTopic`, and the parity is GATED
91
- (`verify_alerts.py`'s vocabulary scan) rather than trusted. The two built-ins are the only
92
- pair that differ — the registry names the surface (`customer_data`) while the grid names the
93
- scope (`customer`) — so a topic passed through as a route sends every click to a page that
94
- does not exist. `None` for anything else: a target this product cannot resolve must be ABSENT
95
- rather than plausible, because an absent target renders as a row that does not pretend to be
96
- clickable, and a wrong one renders as a click that silently goes nowhere.
97
- """
98
- t = str(topic or "").strip()
99
- if t == "customer":
100
- return "customer_data"
101
- if t == "product":
102
- return "product_data"
103
- if _UT_TOPIC.match(t):
104
- return t
105
- return None
106
-
107
-
108
- def _refusal_code(exc):
109
- """The `error.code` an `HTTPException` raised by `deps.err()` carries, or `""`.
110
-
111
- ⭐ W32-T22. Four refusals travel up the assembly chain — `unknown_table` (404), `forbidden`
112
- (403), `window_required` (409) and `store_not_ready` (503) — and each already names its own
113
- cause. Anything that reduces all four to one word is throwing away the only information the
114
- reader could have acted on. Returns `""` for a plain exception, so a caller can tell
115
- "refused, and here is why" apart from "broke, and we do not know why".
116
- """
117
- detail = getattr(exc, "detail", None)
118
- if isinstance(detail, dict):
119
- inner = detail.get("error")
120
- if isinstance(inner, dict):
121
- return str(inner.get("code") or "")
122
- return ""
123
-
124
-
125
- def notification_view(item):
126
- """One STORED notification -> the shape the Inbox renders. PURE, and total.
127
-
128
- Never raises and never drops a row: an item it cannot classify comes back as an `alert` with
129
- no `target`, which the client renders as an unclickable row rather than hiding. An inbox that
130
- silently omits what it does not understand is the one failure a reader cannot detect.
131
- """
132
- if not isinstance(item, dict):
133
- return item
134
- topic = str(item.get("topic") or "").strip()
135
- alert_id = str(item.get("alertId") or "").strip()
136
-
137
- # ⛔ THE ID TEST IS HALF OF EVERY BRANCH, and it is the load-bearing half. A row whose topic
138
- # says `automation` but whose producer key never arrived (a truncated payload, a server
139
- # mid-deploy) would otherwise be handed a target naming NOTHING — a click that appears to work
140
- # and silently does not, which is this repo's most-repeated failure shape. Failing the test
141
- # drops it to the `alert` branch, where `route_for_topic` refuses out loud by answering None.
142
- if topic == AUTOMATION_TOPIC and alert_id:
143
- kind = NOTIF_KIND_AUTOMATION
144
- target = {"module": TARGET_MODULE_AUTOMATION, "id": alert_id, "tab": TARGET_TAB_RUNS}
145
- elif topic == SHARE_TOPIC and alert_id:
146
- # ⭐ W32-T28: the sharer writes `key=<the ROUTE to open>` and, for a shared VIEW,
147
- # `row_id=<the view to select>`.
148
- #
149
- # ⛔ `key` IS ALREADY A ROUTE, NOT A RAW OBJECT ID, and the first version of this got it
150
- # wrong in a way worth recording: a shared VIEW put the VIEW's id in `alertId`, so the
151
- # target read `{module: "database", id: "view_42"}` — an instruction to open a database
152
- # called `view_42`. It looked right in the payload and would have opened nothing. The
153
- # producer resolves the object to its topic and hands over the route; this branch only
154
- # shapes what it is given.
155
- kind = NOTIF_KIND_SHARE
156
- row_id = str(item.get("rowId") or "").strip()
157
- target = {"module": TARGET_MODULE_DATABASE, "id": alert_id,
158
- **({"tab": row_id} if row_id else {})}
159
- else:
160
- kind = NOTIF_KIND_ALERT
161
- route = route_for_topic(topic)
162
- view_id = str(item.get("viewId") or "").strip()
163
- target = None if route is None else (
164
- {"module": TARGET_MODULE_DATABASE, "id": route,
165
- **({"tab": view_id} if view_id else {})})
166
-
167
- # The email split: `subject` is the HEADER (what this is about — the alert, the automation,
168
- # the database), `label` stays the BODY (what happened — the record that entered, the run
169
- # summary). They were one field, which is why a notification read as a sentence with no
170
- # sender and the pane could not be laid out like mail.
171
- subject = str(item.get("alertLabel") or "").strip() or str(item.get("label") or "").strip()
172
- # ⚠ `kind` is OVERWRITTEN, not merged. There was one stored value (`automation_review`) and it
173
- # is D-101's dead one; leaving it through would give the client two vocabularies for one
174
- # question, which is the defect this wave's item 6 is about in a different file.
175
- # ⭐⭐ W33-T28 (owner: "the Inbox reads like email") — THE SENDER, WHICH DID NOT EXIST.
176
- #
177
- # ⛔ A `verifier` reading the finished wave-32 surface found that the row's sender POSITION was
178
- # occupied by `kindLabel(n.kind)` — the literals "Alert" / "Automation" / "Shared with you" —
179
- # i.e. a CATEGORY standing where a who belongs, and no sender field anywhere on the wire, in
180
- # the model or in the markup. Mail has a from. This is it.
181
- #
182
- # ⚠ IT IS DERIVED HERE, NOT STORED, FOR EVERY KIND BUT ONE — and the exception is the point.
183
- # An alert firing and an automation landing rows have no person behind them; their honest
184
- # sender is the machine that did it, named as the thing the reader recognises. A SHARE has a
185
- # real person, and only the producer knows who: `routes_shares.py` writes it as `actor` and
186
- # this reads it back. ⛔ It is NOT parsed out of the body prose ("<name> shared this with
187
- # you") — a sender recovered by regexing a sentence breaks the first time the sentence is
188
- # reworded, and it would break silently, in the header.
189
- #
190
- # ⚠ FALLS BACK, NEVER BLANK. A share queued BEFORE `actor` existed has none, and a row with an
191
- # empty from column reads as a broken inbox rather than as an old notification.
192
- actor = str(item.get("actor") or "").strip()
193
- if kind == NOTIF_KIND_SHARE:
194
- sender = actor or "A teammate"
195
- elif kind == NOTIF_KIND_AUTOMATION:
196
- sender = actor or "Automation"
197
- else:
198
- sender = actor or "Alerts"
199
- out = {**item, "read": bool(item.get("read")), "kind": kind,
200
- "subject": subject or "Notification", "sender": sender}
201
- if target is not None:
202
- out["target"] = target
203
- return out
204
-
205
-
206
- def inbox_view(box):
207
- """`core.alerts.inbox()`'s answer, with every item put through {@link notification_view}.
208
-
209
- ⚠ `unread` IS NOT RECOUNTED. It is the ACCOUNT's number and `items` is one page of it; a
210
- recount here would make the badge a function of whatever this page happened to include, which
211
- is the exact defect `alertsModel.parseInbox`'s own header records from the other side.
212
- """
213
- if not isinstance(box, dict):
214
- return box
215
- items = box.get("items")
216
- if not isinstance(items, list):
217
- return box
218
- # ⭐⭐ W33-T28 / D-208 — THE SERVER'S CLOCK RIDES WITH THE PAGE, and it is what lets the client
219
- # render a mail-shaped stamp ("09:41" today, "Aug 12" beyond) instead of `2026-08-13 09:41`.
220
- #
221
- # ⛔ THE CLIENT MUST NOT READ ITS OWN CLOCK, which is D-208's exit condition word for word and
222
- # is why this key exists rather than a `new Date()` in the browser. `at` is sent as UTC WITH
223
- # its offset (D-18) precisely so every reader sees the same instant; deciding "is this today?"
224
- # against a browser clock would re-introduce the drift the offset exists to remove — a reader a
225
- # day ahead being told an event happened tomorrow [[date-window-vocabulary]]. Both operands
226
- # now come from the same machine.
227
- # ⚠ Same funnel as the enrichment, so the read door and the mark-read door cannot disagree —
228
- # the note two lines up records what happened last time only one of them was enriched.
229
- return {**box, "now": alerts._now_iso(),
230
- "items": [notification_view(n) for n in items]}
231
-
232
-
233
- def _view_by_id(g, view_id):
234
- """One saved view out of an assembly, by id. `None` when there is no such view.
235
-
236
- ⛔⛔ W33-T29 (owner: *"Alert me about new records"* answering "Something went wrong") — THIS
237
- FUNCTION EXISTS BECAUSE TWO CALL SITES BOTH WROTE `(g.get("views") or {}).get(view_id)`, AND
238
- `g["views"] IS A LIST`. `aios_grid.views_from_defs` returns `[{...}]`, `workspace_wire` passes
239
- it straight out and both `ut_assembly` and `grid_assembly` return it unchanged — so `.get` on
240
- it raises `AttributeError`, and `views_from_defs` always returns at least one element, so the
241
- `or {}` never fires. **It raised on EVERY call, on every topic, since wave 20.**
242
-
243
- ⛔ AND THE TWO SITES FAILED DIFFERENTLY, WHICH IS WHY ONLY ONE WAS EVER REPORTED. In
244
- `_require_filtered_view` the raise lands ABOVE the handler's own `try`, so it leaves as a bare
245
- FastAPI 500 and the client's `errorMessage` turns any 5xx into *"Something went wrong on our
246
- side"* — the exact sentence the owner reported (D-107's shape, again: an attribute error above
247
- the guard arrives as plain text rather than as our envelope). In `_evaluate` the identical
248
- line is swallowed by `/notifications`' `except Exception: continue`, so **every stored
249
- view-alert was silently dropped from the Inbox** and nobody had anything to report at all.
250
- One expression, one loud symptom and one silent one.
251
-
252
- ⚠ SO IT IS A FUNCTION, NOT TWO FIXED LINES. Two copies of "find the view" is what let one site
253
- be discussed for three waves while its twin went unnoticed [[one-question-two-normalizers]].
254
-
255
- ⚠ It accepts a dict too, and that is not defensive noise: `verify_alerts`' door fixture was
256
- keyed `{id: view}` — which is precisely why the gate was green while production raised on
257
- every call. The fixture is moving to the production shape in this same change, and tolerating
258
- both here means a caller that legitimately holds one cannot resurrect the bug.
259
- """
260
- want = str(view_id or "")
261
- if not want:
262
- return None
263
- views = (g or {}).get("views")
264
- if isinstance(views, dict):
265
- found = views.get(want)
266
- return found if isinstance(found, dict) else None
267
- if not isinstance(views, list):
268
- return None
269
- for v in views:
270
- if isinstance(v, dict) and str(v.get("id") or "") == want:
271
- return v
272
- return None
273
-
274
-
275
- def _topic_or_400(raw):
276
- topic = str(raw or "").strip().lower()
277
- if topic.startswith("ut_") or topic in _TOPICS:
278
- return topic
279
- raise err(400, "bad_topic", f"topic must be one of {', '.join(_TOPICS)} or a ut_ table")
280
-
281
-
282
- def _owner_session(session: Session, owner: str):
283
- """A `Session` for the alert's OWNER (see the module note on why the owner, not the caller).
284
-
285
- ⚠ `Session` exposes `uname`/`admin` as PROPERTIES derived from `user`, not as fields — so an
286
- owner session is built by swapping the `user` RECORD and letting both derive themselves. An
287
- earlier version passed `uname=`/`admin=` to the constructor, which would have raised on the
288
- first write hook of the wave; the properties are the single definition of who a session is,
289
- and going around them is how a session with an admin flag and a non-admin record exists.
290
-
291
- Returns None when the owner is gone or deactivated — their alerts then stop evaluating rather
292
- than evaluating as somebody else, which is the fail-closed direction.
293
- """
294
- import core.users as users
295
-
296
- if str(owner) == str(session.uname):
297
- return session
298
- rec = (users.registry() or {}).get(str(owner))
299
- if not isinstance(rec, dict) or not rec.get("active", True):
300
- return None
301
- # `_public` is THE definition of what a session may know about its own account (never a hash
302
- # or a salt) — the same one `routes_auth` uses. Building the dict by hand here would be a
303
- # second definition, and the one that leaks is always the copy.
304
- return Session(tenant=session.tenant, user=users._public(str(owner), rec),
305
- claims=session.claims, runtime=session.runtime)
306
-
307
-
308
- def _evaluate(session: Session, rec: dict, assemblies=None):
309
- """Resolve `rec`'s view over its topic AS THE ALERT'S OWNER, then fold the result in.
310
-
311
- ⭐⭐ W31-T24 — `assemblies` IS A PER-REQUEST MEMO, KEYED `(topic, owner)`, and it is the whole
312
- of this ticket's server half. `/notifications` re-evaluates EVERY alert inline on read and each
313
- one built a FULL assembly — the pool, the workspace, `rows_from_pool` over every row. Two
314
- alerts on one view built that table twice; ten built it ten times. Nothing dedupes them,
315
- because each `_evaluate` was a closed call.
316
- ⚠ `(topic, owner)` and not `topic`: the assembly is built as the alert's OWNER (see the module
317
- note — evaluating a BU-scoped user's alert on a full-access admin's pool is a permission leak
318
- wearing a notification's clothes), so two owners on one topic are two DIFFERENT tables and
319
- must never share an entry. Getting that key wrong is the one way this optimisation could leak.
320
- ⚠ Passing nothing keeps the old behaviour exactly, which is what the create/run doors want:
321
- they evaluate ONE alert and a memo for a single call is pure overhead.
322
- """
323
- import aios_grid
324
- from harness import filter_eval
325
-
326
- owner_sess = _owner_session(session, rec.get("owner"))
327
- if owner_sess is None:
328
- return {"skipped": "owner_unavailable"}
329
- topic = str(rec.get("topic") or "")
330
- memo_key = (topic, str(owner_sess.uname))
331
- g = assemblies.get(memo_key) if isinstance(assemblies, dict) else None
332
- if g is None:
333
- try:
334
- if topic.startswith("ut_"):
335
- from routes_tables import ut_assembly
336
- # ⛔ `consume_corrections=False`, and the default was a REAL BUG, not a tidy-up.
337
- # `ut_assembly` defaults it True, so every `/notifications` read CONSUMED the
338
- # one-shot field-name correction acks for every `ut_` topic that has an alert —
339
- # taking them from the `/workspace` refresh that exists to show them to the person
340
- # who made the edit. The customer branch below has always passed False; this one
341
- # inherited a default nobody re-read. An inbox poll must never consume a one-shot.
342
- g = ut_assembly(owner_sess, topic,
343
- storage_key=f"{owner_sess.tenant}:{topic}:{owner_sess.uname}",
344
- consume_corrections=False)
345
- else:
346
- from routes_customers import grid_assembly
347
- g = grid_assembly(owner_sess, scope=topic, consume_corrections=False)
348
- except Exception as e: # noqa: BLE001
349
- # ⭐ W32-T22 — SKIPPING IS FINE HERE; SKIPPING ANONYMOUSLY IS NOT. This one must not
350
- # raise (one bad alert cannot empty an inbox), so unlike `_require_filtered_view` it
351
- # keeps a blanket catch — but it now reports the refusal's OWN code where there is
352
- # one. `type(e).__name__` said `HTTPException` for four different causes, and
353
- # `lastError` is the only place a user ever learns why an alert stopped firing.
354
- #
355
- # ⚠ `with_rows=True` STAYS on this path, deliberately: unlike the create door, an
356
- # evaluation genuinely needs the rows to run the filter over. So an alert on a
357
- # read-through grid is created (T22) and then skips at evaluation with
358
- # `window_required` naming why — which is D-184's remaining half, and it is a
359
- # SENTENCE now rather than silence.
360
- return {"skipped": _refusal_code(e) or "unavailable", "detail": type(e).__name__}
361
- if isinstance(assemblies, dict):
362
- assemblies[memo_key] = g
363
-
364
- view = _view_by_id(g, rec.get("viewId"))
365
- if not isinstance(view, dict):
366
- # Deleted, or un-shared out from under the alert. Say so on the RECORD rather than
367
- # deleting the alert: an alert that silently vanishes is indistinguishable from one that
368
- # never fires, and the user cannot debug what is not there.
369
- return {"skipped": "view_missing"}
370
-
371
- # The SAME row build the grid and `/customers` use — `rows_from_pool` is what puts derived
372
- # and overlay values on a row. Evaluating a filter against raw pool dicts would silently
373
- # never match any condition on a user-created or measure column.
374
- #
375
- # ⛔⛔ AND "THE SAME ROW BUILD" WAS NOT TRUE, WHICH MADE EVERY ALERT ON A `ut_*` DATABASE BLIND
376
- # TO IMPORTED DATA. Found by a verifier driving one real assembly through both paths.
377
- #
378
- # `routes_tables.table_rows` — the grid the person is looking at — merges the DEFINITION rows
379
- # underneath the overlay ("base first, overlay wins"; that merge is itself the fix for owner
380
- # item 3, *"it all got reseted"*). `_evaluate` is a second copy of that read and never got it:
381
- # it handed `ws['overlays']` to `rows_from_pool` raw, so for a `ut_*` table every base cell
382
- # evaluated as BLANK. Measured on one assembly, same view, same rows:
383
- # rows_src state='unpaid' / 'paid'
384
- # _evaluate saw state='' / '' ⇐ every base cell blank
385
- # the GRID saw state='unpaid' / 'paid'
386
- # so `state eq unpaid` matched NOTHING while the view showed one row, and `state isEmpty`
387
- # matched EVERYTHING while the view showed none. **The alert did not merely miss rows — it
388
- # inverted.** End to end: a row whose value arrived by import, automation, paste or the create
389
- # door never fired; only a value typed as a hand EDIT did.
390
- #
391
- # ⚠ Scope, so nobody widens the fix past its cause: materialised `ut_*` tables are hit;
392
- # `customer`/`product` are not (their fields are `source: "odoo"` and read off `rows_src`);
393
- # `ut_odoo_*` never reaches here (`with_rows=True` refuses first and returns
394
- # `skipped: window_required`).
395
- # ⛔ ORDER IS LOAD-BEARING AND IS THE GRID'S: base underneath, overlay ON TOP. Inverting it
396
- # would let a stale definition value shadow an edit the user has just made — the same defect
397
- # `table_rows`' own note records, arriving from the other side.
398
- _ov = (g.get("ws") or {}).get("overlays") or {}
399
- _merged = {}
400
- for _r in g["rows_src"]:
401
- _pid = str(_r.get("pid"))
402
- _cells = {k: v for k, v in _r.items() if k != "pid"}
403
- _o = _ov.get(_pid)
404
- if isinstance(_o, dict):
405
- _cells.update(_o)
406
- _merged[_pid] = _cells
407
- rows = aios_grid.rows_from_pool(g["rows_src"], g["fields"], _merged,
408
- derived=g.get("derived"))
409
- config = view.get("config") or view
410
- ctx = filter_eval.EvalCtx(
411
- cohort_sets={str(k): {str(p) for p in (v.get("memberPids") or ())}
412
- for k, v in (g.get("lists") or {}).items() if isinstance(v, dict)},
413
- measure_sets=g.get("measure_sets") or {},
414
- today=g.get("today"))
415
- pids = filter_eval.visible_pids(config.get("filters") or [], rows, g["fields"], ctx,
416
- member_pids=config.get("memberPids"))
417
- labels = {str(r.get("pid")): str(r.get("name") or r.get("pid")) for r in rows}
418
- return alerts.evaluate(rec.get("id"), [str(p) for p in pids],
419
- labels=labels, partial=False, st=session.runtime)
420
-
421
-
422
- @router.get("/alerts")
423
- def list_alerts(session: Session = Depends(require_session)):
424
- return {"alerts": alerts.list_alerts(user=session.uname, is_admin=session.admin,
425
- st=session.runtime)}
426
-
427
-
428
- @router.post("/alerts")
429
- def create_alert(body: dict = Body(default=None), session: Session = Depends(require_session)):
430
- body = body or {}
431
- view_id = str(body.get("viewId") or "").strip()
432
- if not view_id:
433
- raise err(400, "bad_view", "an alert needs the id of the view it watches")
434
- topic = _topic_or_400(body.get("topic"))
435
- _require_filtered_view(session, topic, view_id)
436
- import uuid
437
- aid = f"al_{uuid.uuid4().hex[:12]}"
438
- rec = alerts.create(aid, view_id=view_id, topic=topic, owner=session.uname,
439
- label=body.get("label") or "", st=session.runtime)
440
- # SEED IMMEDIATELY, so the alert starts from "everything currently matching is old news".
441
- # Deferring this to the first write hook would mean the next edit announces the whole view.
442
- outcome = _evaluate(session, rec)
443
- return {"alert": {**rec, "seeded": True}, "first": outcome}
444
-
445
-
446
- def _require_filtered_view(session: Session, topic: str, view_id: str):
447
- """400 unless `view_id` exists on `topic` AND actually narrows something.
448
-
449
- ⛔ AN ALERT ON AN UNFILTERED VIEW IS SILENTLY INCAPABLE OF ALERTING, which is worse than one
450
- that is refused. `filter_eval` treats an inactive tree as "no narrowing, every row shows"
451
- (`visible_pids`'s own rule), so such an alert seeds with the entire table and can never see an
452
- entrant again — there is nothing left to enter. The owner's words are *"when a Record gets
453
- into that Filter's criteria"*: no criteria, no alert, and said at creation rather than
454
- discovered by never being notified.
455
-
456
- `is_rule_active` is the SAME activeness predicate the engine and the column tints use — a
457
- half-typed rule is not a filter, and this must agree with what actually narrows or it would
458
- accept a view whose one rule the engine then ignores.
459
-
460
- ⭐⭐ WAVE 32 · T22 (owner item 17) — THIS FUNCTION WAS THE ERROR. Two defects, stacked, and
461
- the second one hid the first.
462
-
463
- (1) **IT ASKED FOR EVERY ROW OF A TABLE IT NEVER LOOKS AT.** The only thing read below is
464
- `g["views"]`. `ut_assembly` defaults `with_rows=True`, so creating an alert on a
465
- read-through grid built the whole pool — and `scoped_pool` refuses that with
466
- `409 window_required` over 963,783 rows, exactly as it is supposed to. `with_rows=False`
467
- (W31-T20's flag, built for precisely this) answers the same question with `scoped_pids`,
468
- runs the SAME `_defn_or_refuse` wall, and does not refuse. **That is D-184's create half,
469
- closed** — an alert on a read-through grid can now be made at all.
470
- (2) **A BLANKET `except Exception` TURNED EVERY NAMED REFUSAL INTO A 503.** `HTTPException`
471
- is an `Exception`, so `404 unknown_table`, `403 forbidden`, `409 window_required` and
472
- `503 store_not_ready` — four refusals that each say what is wrong — were all replaced by
473
- *"the table is unavailable — try again in a moment"*. ⛔ AND THAT SENTENCE NEVER REACHED
474
- A USER EITHER: `alertsApi.errorMessage` discards the text of any status ≥ 500 by design
475
- (a 5xx body is the server's internals), substituting *"Something went wrong on our
476
- side."* — which is the owner's screenshot, word for word. A knowable cause returned as a
477
- 5xx is invisible by construction, so re-wording the 503 could never have fixed this.
478
- ⚠ The except is narrowed, not deleted: an UNEXPECTED failure is still a 503, because that is
479
- honest. What it may no longer do is catch a refusal that already knows its own name.
480
- """
481
- from fastapi import HTTPException
482
-
483
- from harness import filter_eval
484
-
485
- try:
486
- if topic.startswith("ut_"):
487
- from routes_tables import ut_assembly
488
- # ⚠ `consume_corrections=False` — the customer branch has always passed it and this
489
- # one inherited a default nobody re-read. Creating an alert must not eat the one-shot
490
- # field-name correction acks belonging to the `/workspace` refresh that exists to show
491
- # them to the person who made the edit. Same defect `_evaluate`'s header records.
492
- g = ut_assembly(session, topic,
493
- storage_key=f"{session.tenant}:{topic}:{session.uname}",
494
- consume_corrections=False, with_rows=False)
495
- else:
496
- from routes_customers import grid_assembly
497
- g = grid_assembly(session, scope=topic, consume_corrections=False)
498
- except HTTPException:
499
- raise # it already names its own cause
500
- except Exception as e: # noqa: BLE001
501
- # Genuinely unexpected. Still a 503, and now it carries the exception TYPE — without it,
502
- # the one path that reaches this branch is also the one path with nothing to debug from.
503
- raise err(503, "unavailable",
504
- f"the table could not be read ({type(e).__name__}) — try again in a moment")
505
- view = _view_by_id(g, view_id)
506
- if not isinstance(view, dict):
507
- raise err(404, "no_view", "that view does not exist on this table")
508
- nodes, _conj = filter_eval.tree_parts((view.get("config") or view).get("filters") or [])
509
-
510
- # ⛔⛔ WAVE 33 · T29 — **CORRECTION: THE BLOCK BELOW IS TRUE ABOUT THE CODE AND FALSE ABOUT
511
- # PRODUCTION, AND IT MUST BE READ SECOND.** It claims the missing-argument `TypeError` "IS
512
- # owner item 17" — the owner's *"Something went wrong"*. It was not, and it could not have
513
- # been: at `cbcf005`, the build the owner was using, the dict-read on `views` sat ~10 lines
514
- # ABOVE this call and raised `AttributeError` on EVERY request, so the walk never reached the
515
- # leaf and the arity bug was unreachable. `_view_by_id`'s own header records that fix.
516
- #
517
- # ⚠ WHY THE STALE PARAGRAPH STAYS RATHER THAN GETTING DELETED: the arity bug was real, the
518
- # fix was right, and the three reasons it hid are the most transferable thing in this file.
519
- # What was wrong is only its CLAIM TO BE THE CAUSE. Two comment blocks in one function each
520
- # naming themselves as the origin of the same screenshot are mutually exclusive, and the next
521
- # reader believes whichever they meet first — which is why this correction sits above rather
522
- # than below. Caught by a verifier that read the SHIPPED file at the deployed commit instead
523
- # of the working tree. [[grep-output-is-not-source]]
524
- #
525
- # ⛔⛔ WAVE 32 · T22 — **THE CALL BELOW WAS MISSING AN ARGUMENT** (and wave 32 believed, wrongly,
526
- # that this was owner item 17 — see the correction directly above).
527
- #
528
- # `is_rule_active(rule, columns)` takes TWO parameters (`harness/filter_sql.py`; every other
529
- # caller in the repo passes both). This one passed ONE, so the moment the walk reached a LEAF
530
- # rule it raised `TypeError: is_rule_active() missing 1 required positional argument`.
531
- #
532
- # ⚠ READ WHAT THAT MEANS BEFORE FIXING ANYTHING ELSE: the walk only reaches a leaf when the
533
- # view HAS a condition — and a view with a condition is the only kind an alert is allowed on.
534
- # A view with no filters yields an empty `nodes`, so `_any_active` returns False without ever
535
- # calling this, and the reader gets the honest 400 `no_filter`. **So the only path that
536
- # worked was the refusal path: "Alert me about new records" had never once created an alert
537
- # on a filtered view.** ⛔ And the raise lands OUTSIDE the `try` above, so it was not even the
538
- # 503 — it was a bare FastAPI 500, which `alertsApi.errorMessage` renders as *"Something went
539
- # wrong on our side. Try again in a moment."*, the owner's screenshot word for word.
540
- #
541
- # ⚠ THREE THINGS HID IT, and they are worth more than the fix. (1) Python does not check
542
- # arity until the line RUNS, and this line runs only on the success path of a feature whose
543
- # every test exercised its refusals. (2) The `no_filter` 400 above it is a real, correct,
544
- # well-tested refusal, so the door looked alive. (3) `verify_alerts.py` asserts the refusal
545
- # (`no_filter` reaches the user) and the transport — never a creation. A gate can be green,
546
- # thorough and honest about everything except the one path the feature exists for.
547
- #
548
- # `_columns_map` is the DEFINITION of fields -> the membership set `is_rule_active` looks a
549
- # column up in; building a second dict here would be a second answer to one question, which
550
- # is this wave's other headline defect in a different file. Its leading underscore is a real
551
- # smell and is BOOKED (PENDING, mailbox/C.md) rather than worked around.
552
- columns = filter_eval._columns_map(g.get("fields") or [])
553
-
554
- def _any_active(ns):
555
- for n in ns or ():
556
- if isinstance(n, dict) and isinstance(n.get("children"), list):
557
- if _any_active(n["children"]):
558
- return True
559
- elif filter_eval.is_rule_active(n, columns):
560
- return True
561
- return False
562
-
563
- if not _any_active(nodes):
564
- raise err(400, "no_filter",
565
- "this view has no active filter, so no record can ever ENTER it — add a "
566
- "condition to the view first, then create the alert")
567
-
568
-
569
- @router.delete("/alerts/{alert_id}")
570
- def delete_alert(alert_id: str, session: Session = Depends(require_session)):
571
- rec = next((r for r in alerts.list_alerts(st=session.runtime)
572
- if str(r.get("id")) == str(alert_id)), None)
573
- if rec is None:
574
- raise err(404, "no_alert", "that alert does not exist")
575
- if str(rec.get("owner")) != str(session.uname) and not session.admin:
576
- raise err(403, "not_yours", "only the alert's owner (or an administrator) can delete it")
577
- alerts.delete(alert_id, st=session.runtime)
578
- return {"ok": True}
579
-
580
-
581
- @router.post("/alerts/{alert_id}/run")
582
- def run_alert(alert_id: str, session: Session = Depends(require_session)):
583
- rec = next((r for r in alerts.list_alerts(user=session.uname, is_admin=session.admin,
584
- st=session.runtime)
585
- if str(r.get("id")) == str(alert_id)), None)
586
- if rec is None:
587
- raise err(404, "no_alert", "that alert does not exist")
588
- return _evaluate(session, rec)
589
-
590
-
591
- @router.get("/notifications")
592
- def notifications(session: Session = Depends(require_session)):
593
- """The inbox — RE-EVALUATED on read, which is a deliberate design choice.
594
-
595
- ⭐ A-S1-2 RESOLVED THE OTHER WAY, and the reason is structural rather than a shortcut. The
596
- plan was a push hook: the automation engine calls `after_write` when it lands rows. But
597
- `run_async` runs on a BACKGROUND THREAD with no `Session` in scope, and an alert must be
598
- evaluated as its OWNER (see `_evaluate`) — so a push hook would have to mint a session inside
599
- a worker thread from a tenant runtime, which is exactly the kind of ad-hoc identity
600
- construction that leaks scope.
601
-
602
- Pulling on read has none of that: the caller IS a session, the assemblies are already
603
- scope-cached, and the user cannot observe the difference — an inbox is only ever read by
604
- someone opening it. The cost is that a notification is minted when you LOOK rather than when
605
- the row landed, so the `at` stamp is detection time, not arrival time.
606
-
607
- `after_write` stays exported for the day the engine can hand over a real identity.
608
-
609
- ⭐⭐ W31-T24 — ONE ASSEMBLY PER (TOPIC, OWNER), NOT ONE PER ALERT.
610
- ⛔ MEASURED FIRST, AND THE MEASUREMENT CORRECTS AN EARLIER READING OF IT. This route is
611
- **20 ms in-process and 3,280 ms live** on tenant #0 — but tenant #0 has **ZERO alerts**
612
- (censused 2026-08-12), so the 20 ms is an EMPTY LOOP and says nothing at all about what the
613
- re-evaluation costs. The live 3,280 ms is the two store reads either side of that loop. So the
614
- body below is not slow today; it is UNEXERCISED, and every alert a tenant creates adds a whole
615
- grid assembly to an inbox poll. The memo turns O(alerts) into O(distinct topic × owner), which
616
- is the difference between "fine" and "three seconds per alert" the day somebody uses the
617
- feature. ⚠ Making the read cheap by evaluating LESS is the obvious wrong fix and is not what
618
- this does: every alert is still evaluated, against the same rows, in the same order.
619
- """
620
- assemblies = {}
621
- for rec in alerts.list_alerts(user=session.uname, is_admin=False, st=session.runtime):
622
- try:
623
- _evaluate(session, rec, assemblies=assemblies)
624
- except Exception: # noqa: BLE001
625
- continue # one bad alert must not empty the pane
626
- # ⭐ W32-T20 (C3): every item leaves through `inbox_view`, so a notification queued before
627
- # this wave carries a `target` too. See `notification_view`'s header for why it is derived.
628
- return inbox_view(alerts.inbox(session.uname, st=session.runtime))
629
-
630
-
631
- @router.post("/notifications/read")
632
- def read_notifications(body: dict = Body(default=None),
633
- session: Session = Depends(require_session)):
634
- body = body or {}
635
- ids = body.get("ids")
636
- if ids is not None and not isinstance(ids, list):
637
- raise err(400, "bad_ids", "ids must be a list, or null to mark every notification")
638
- # ⚠ THE SAME ENRICHMENT ON BOTH DOORS. `mark_read` returns a fresh inbox, and the Inbox
639
- # module re-renders from it — an un-enriched answer here would strip `target` off every row
640
- # the moment somebody marked one read, i.e. the feature would work until first use.
641
- return inbox_view(alerts.mark_read(session.uname, ids, read=bool(body.get("read", True)),
642
- st=session.runtime))
643
-
644
-
645
- def after_write(session: Session, topic_key: str):
646
- """THE WRITE HOOK — call after a write that could change what a view matches.
647
-
648
- Exported as a plain function (not a route) so `core.grid_events`' callers and S2's automation
649
- upserts reach it the same way. It never raises: an alert evaluation failing must not fail the
650
- edit that triggered it.
651
-
652
- ⭐ W31-T24 — it shares `/notifications`' memo shape for the same reason: a write that changes
653
- one view can trip several alerts on the SAME topic, and each would otherwise rebuild the table.
654
- ⚠ STILL ZERO PRODUCTION CALLERS (W31-T24 confirmed it; the route docstring above says why the
655
- push hook was resolved the other way). Booked rather than wired: minting a session inside the
656
- engine's worker thread is the ad-hoc identity construction this file exists to avoid.
657
- """
658
- try:
659
- assemblies = {}
660
- return alerts.after_write(topic_key, st=session.runtime,
661
- runner=lambda rec: _evaluate(session, rec,
662
- assemblies=assemblies))
663
- except Exception: # noqa: BLE001
664
- return {"evaluated": 0}
 
1
+ """routes_alerts.py — the Alerts module (wave 20, owner item 25, contract C-ALERT).
2
+
3
+ GET /api/v1/alerts -> {alerts:[...]}
4
+ POST /api/v1/alerts <- {viewId, topic, label?}
5
+ DELETE /api/v1/alerts/{alert_id}
6
+ POST /api/v1/alerts/{alert_id}/run -> evaluate now (the pane's manual refresh)
7
+ GET /api/v1/notifications -> {unread, items:[...]}
8
+ POST /api/v1/notifications/read <- {ids:[...]|null, read?:bool}
9
+
10
+ The semantics — an alert is a view plus a remembered matched set, a notification is a NEW
11
+ ENTRANT, and the first evaluation seeds silently — live in `core.alerts` with the reasoning.
12
+ This file owns the two things a route must: WHO may do it, and HOW the view gets evaluated.
13
+
14
+ ⭐ **THE EVALUATION RUNS AS THE ALERT'S OWNER, NOT AS THE CALLER.** `_run_alert` builds the pool
15
+ for `rec['owner']`, never for whoever tripped the write hook. Any other choice leaks: a
16
+ full-access admin editing a cell would otherwise evaluate a BU-scoped user's alert over the whole
17
+ book, and the notification would name customers that user may not see — a permission leak wearing
18
+ a notification's clothes. The owner's own scope is the only correct basis for their alert.
19
+
20
+ ⚠ **AN ALERT IS NOT A SECOND READ PATH.** It resolves rows through the same
21
+ `routes_customers.grid_assembly` / `routes_tables.ut_assembly` the grid uses, so a row that an
22
+ alert can see is by construction a row its owner could open. Re-implementing the filter here
23
+ would be a second definition of "matches", and those two would drift.
24
+ """
25
+ import re
26
+
27
+ from fastapi import APIRouter, Body, Depends
28
+
29
+ import core.alerts as alerts
30
+ from deps import Session, err, require_session
31
+
32
+ router = APIRouter(prefix="/api/v1")
33
+
34
+ #: The alert-bearing surfaces. `ut_` tables are admitted by prefix, like everywhere else.
35
+ _TOPICS = ("customer", "product")
36
+
37
+ # ── ⭐⭐ WAVE 32 · T20 · CONTRACT C3 — THE INBOX SHAPE, DERIVED ON READ ────────────────────────
38
+ #
39
+ # `GET /notifications` gains `subject`, `kind` and `target` per item (`read` was always there).
40
+ #
41
+ # ⛔ DERIVED, NEVER STORED, AND THAT IS THE WHOLE OF WHY THIS WAVE EXISTS. Stamping the three
42
+ # keys onto the record at write time would give them to notifications minted AFTER the deploy and
43
+ # to nothing else — every notification already sitting in every tenant's inbox would open nothing,
44
+ # and the feature would be correct in the source and absent from the product
45
+ # ([[a-migration-that-runs-on-the-next-write]], D-201). A read-side derivation reaches a
46
+ # notification queued last month. It also keeps the store shape out of `core/alerts.py`, which is
47
+ # another lane's file this wave — but that is the convenience, not the reason.
48
+ #
49
+ # ⚠ TWO PRODUCERS WRITE TWO SHAPES into one inbox, and the vocabulary below is what tells them
50
+ # apart. `_queue` (a record ENTERED a watched view) sets `topic`+`viewId`. `notify()` sets
51
+ # `topic='automation'` and puts the producer's key in `alertId`, leaving `viewId` empty. Deciding
52
+ # here means the client branches on ONE field instead of re-deriving the same split.
53
+ #
54
+ # ⛔ **D-101 IS CLOSED HERE, BY SUBTRACTION.** There was a THIRD shape — `kind='automation_review'`
55
+ # + `autoId`, a card arriving at a review stage — and its producer `notify_review` was deleted by
56
+ # W27/R3 with the review lanes. `automation_engine.py`'s own tombstone (search `notify_review`)
57
+ # records the 2026-08-12 sweep: **no `.py` file anywhere produces one**, while the client branch,
58
+ # its route and three gate legs stayed fully alive. D-101's exit condition is *"the client review
59
+ # branch is deleted in the same change as any remaining residue, OR `notify_review` gains its real
60
+ # caller"* — the residue is zero, so the branch goes. It is not carried into the Inbox: a stored
61
+ # review notification (if any survives in a tenant from the wave-23 era) derives as an ordinary
62
+ # `alert` with no target, i.e. an honest unclickable row, which is correct — the board it pointed
63
+ # at was deleted two waves ago.
64
+
65
+ #: C3's `kind` vocabulary. Plain strings on the wire — the client must never union over them
66
+ #: (alertsModel's wave-9 law: a client union turns "the server grew a kind" into a dropped row).
67
+ NOTIF_KIND_ALERT = "alert"
68
+ NOTIF_KIND_AUTOMATION = "automation"
69
+ NOTIF_KIND_SHARE = "share"
70
+
71
+ #: C3's `target.module` vocabulary, and the automation sub-selection.
72
+ TARGET_MODULE_DATABASE = "database"
73
+ TARGET_MODULE_AUTOMATION = "automation"
74
+ TARGET_TAB_RUNS = "runs"
75
+
76
+ #: The topic `notify()` carries for a SHARE (W32-T28 writes it; nothing does yet, and a kind with
77
+ #: no producer is a string that reads as a feature — the reason this constant is named here and
78
+ #: cited from `routes_shares` rather than typed twice).
79
+ SHARE_TOPIC = "share"
80
+
81
+ #: `core.alerts.notify`'s default topic for a run outcome. Mirrors `inboxModel.AUTOMATION_TOPIC`.
82
+ AUTOMATION_TOPIC = "automation"
83
+
84
+ _UT_TOPIC = re.compile(r"ut_[A-Za-z0-9_]+\Z")
85
+
86
+
87
+ def route_for_topic(topic):
88
+ """A grid SCOPE key -> the registry route that renders it, or None.
89
+
90
+ ⛔ THE SAME TABLE AS `alertsModel.routeForTopic`, and the parity is GATED
91
+ (`verify_alerts.py`'s vocabulary scan) rather than trusted. The two built-ins are the only
92
+ pair that differ — the registry names the surface (`customer_data`) while the grid names the
93
+ scope (`customer`) — so a topic passed through as a route sends every click to a page that
94
+ does not exist. `None` for anything else: a target this product cannot resolve must be ABSENT
95
+ rather than plausible, because an absent target renders as a row that does not pretend to be
96
+ clickable, and a wrong one renders as a click that silently goes nowhere.
97
+ """
98
+ t = str(topic or "").strip()
99
+ if t == "customer":
100
+ return "customer_data"
101
+ if t == "product":
102
+ return "product_data"
103
+ if _UT_TOPIC.match(t):
104
+ return t
105
+ return None
106
+
107
+
108
+ def _refusal_code(exc):
109
+ """The `error.code` an `HTTPException` raised by `deps.err()` carries, or `""`.
110
+
111
+ ⭐ W32-T22. Four refusals travel up the assembly chain — `unknown_table` (404), `forbidden`
112
+ (403), `window_required` (409) and `store_not_ready` (503) — and each already names its own
113
+ cause. Anything that reduces all four to one word is throwing away the only information the
114
+ reader could have acted on. Returns `""` for a plain exception, so a caller can tell
115
+ "refused, and here is why" apart from "broke, and we do not know why".
116
+ """
117
+ detail = getattr(exc, "detail", None)
118
+ if isinstance(detail, dict):
119
+ inner = detail.get("error")
120
+ if isinstance(inner, dict):
121
+ return str(inner.get("code") or "")
122
+ return ""
123
+
124
+
125
+ def notification_view(item):
126
+ """One STORED notification -> the shape the Inbox renders. PURE, and total.
127
+
128
+ Never raises and never drops a row: an item it cannot classify comes back as an `alert` with
129
+ no `target`, which the client renders as an unclickable row rather than hiding. An inbox that
130
+ silently omits what it does not understand is the one failure a reader cannot detect.
131
+ """
132
+ if not isinstance(item, dict):
133
+ return item
134
+ topic = str(item.get("topic") or "").strip()
135
+ alert_id = str(item.get("alertId") or "").strip()
136
+
137
+ # ⛔ THE ID TEST IS HALF OF EVERY BRANCH, and it is the load-bearing half. A row whose topic
138
+ # says `automation` but whose producer key never arrived (a truncated payload, a server
139
+ # mid-deploy) would otherwise be handed a target naming NOTHING — a click that appears to work
140
+ # and silently does not, which is this repo's most-repeated failure shape. Failing the test
141
+ # drops it to the `alert` branch, where `route_for_topic` refuses out loud by answering None.
142
+ if topic == AUTOMATION_TOPIC and alert_id:
143
+ kind = NOTIF_KIND_AUTOMATION
144
+ target = {"module": TARGET_MODULE_AUTOMATION, "id": alert_id, "tab": TARGET_TAB_RUNS}
145
+ elif topic == SHARE_TOPIC and alert_id:
146
+ # ⭐ W32-T28: the sharer writes `key=<the ROUTE to open>` and, for a shared VIEW,
147
+ # `row_id=<the view to select>`.
148
+ #
149
+ # ⛔ `key` IS ALREADY A ROUTE, NOT A RAW OBJECT ID, and the first version of this got it
150
+ # wrong in a way worth recording: a shared VIEW put the VIEW's id in `alertId`, so the
151
+ # target read `{module: "database", id: "view_42"}` — an instruction to open a database
152
+ # called `view_42`. It looked right in the payload and would have opened nothing. The
153
+ # producer resolves the object to its topic and hands over the route; this branch only
154
+ # shapes what it is given.
155
+ kind = NOTIF_KIND_SHARE
156
+ row_id = str(item.get("rowId") or "").strip()
157
+ target = {"module": TARGET_MODULE_DATABASE, "id": alert_id,
158
+ **({"tab": row_id} if row_id else {})}
159
+ else:
160
+ kind = NOTIF_KIND_ALERT
161
+ route = route_for_topic(topic)
162
+ view_id = str(item.get("viewId") or "").strip()
163
+ target = None if route is None else (
164
+ {"module": TARGET_MODULE_DATABASE, "id": route,
165
+ **({"tab": view_id} if view_id else {})})
166
+
167
+ # The email split: `subject` is the HEADER (what this is about — the alert, the automation,
168
+ # the database), `label` stays the BODY (what happened — the record that entered, the run
169
+ # summary). They were one field, which is why a notification read as a sentence with no
170
+ # sender and the pane could not be laid out like mail.
171
+ subject = str(item.get("alertLabel") or "").strip() or str(item.get("label") or "").strip()
172
+ # ⚠ `kind` is OVERWRITTEN, not merged. There was one stored value (`automation_review`) and it
173
+ # is D-101's dead one; leaving it through would give the client two vocabularies for one
174
+ # question, which is the defect this wave's item 6 is about in a different file.
175
+ out = {**item, "read": bool(item.get("read")), "kind": kind,
176
+ "subject": subject or "Notification"}
177
+ if target is not None:
178
+ out["target"] = target
179
+ return out
180
+
181
+
182
+ def inbox_view(box):
183
+ """`core.alerts.inbox()`'s answer, with every item put through {@link notification_view}.
184
+
185
+ ⚠ `unread` IS NOT RECOUNTED. It is the ACCOUNT's number and `items` is one page of it; a
186
+ recount here would make the badge a function of whatever this page happened to include, which
187
+ is the exact defect `alertsModel.parseInbox`'s own header records from the other side.
188
+ """
189
+ if not isinstance(box, dict):
190
+ return box
191
+ items = box.get("items")
192
+ if not isinstance(items, list):
193
+ return box
194
+ return {**box, "items": [notification_view(n) for n in items]}
195
+
196
+
197
+ def _topic_or_400(raw):
198
+ topic = str(raw or "").strip().lower()
199
+ if topic.startswith("ut_") or topic in _TOPICS:
200
+ return topic
201
+ raise err(400, "bad_topic", f"topic must be one of {', '.join(_TOPICS)} or a ut_ table")
202
+
203
+
204
+ def _owner_session(session: Session, owner: str):
205
+ """A `Session` for the alert's OWNER (see the module note on why the owner, not the caller).
206
+
207
+ ⚠ `Session` exposes `uname`/`admin` as PROPERTIES derived from `user`, not as fields — so an
208
+ owner session is built by swapping the `user` RECORD and letting both derive themselves. An
209
+ earlier version passed `uname=`/`admin=` to the constructor, which would have raised on the
210
+ first write hook of the wave; the properties are the single definition of who a session is,
211
+ and going around them is how a session with an admin flag and a non-admin record exists.
212
+
213
+ Returns None when the owner is gone or deactivated — their alerts then stop evaluating rather
214
+ than evaluating as somebody else, which is the fail-closed direction.
215
+ """
216
+ import core.users as users
217
+
218
+ if str(owner) == str(session.uname):
219
+ return session
220
+ rec = (users.registry() or {}).get(str(owner))
221
+ if not isinstance(rec, dict) or not rec.get("active", True):
222
+ return None
223
+ # `_public` is THE definition of what a session may know about its own account (never a hash
224
+ # or a salt) — the same one `routes_auth` uses. Building the dict by hand here would be a
225
+ # second definition, and the one that leaks is always the copy.
226
+ return Session(tenant=session.tenant, user=users._public(str(owner), rec),
227
+ claims=session.claims, runtime=session.runtime)
228
+
229
+
230
+ def _evaluate(session: Session, rec: dict, assemblies=None):
231
+ """Resolve `rec`'s view over its topic AS THE ALERT'S OWNER, then fold the result in.
232
+
233
+ ⭐⭐ W31-T24 — `assemblies` IS A PER-REQUEST MEMO, KEYED `(topic, owner)`, and it is the whole
234
+ of this ticket's server half. `/notifications` re-evaluates EVERY alert inline on read and each
235
+ one built a FULL assembly — the pool, the workspace, `rows_from_pool` over every row. Two
236
+ alerts on one view built that table twice; ten built it ten times. Nothing dedupes them,
237
+ because each `_evaluate` was a closed call.
238
+ ⚠ `(topic, owner)` and not `topic`: the assembly is built as the alert's OWNER (see the module
239
+ note — evaluating a BU-scoped user's alert on a full-access admin's pool is a permission leak
240
+ wearing a notification's clothes), so two owners on one topic are two DIFFERENT tables and
241
+ must never share an entry. Getting that key wrong is the one way this optimisation could leak.
242
+ ⚠ Passing nothing keeps the old behaviour exactly, which is what the create/run doors want:
243
+ they evaluate ONE alert and a memo for a single call is pure overhead.
244
+ """
245
+ import aios_grid
246
+ from harness import filter_eval
247
+
248
+ owner_sess = _owner_session(session, rec.get("owner"))
249
+ if owner_sess is None:
250
+ return {"skipped": "owner_unavailable"}
251
+ topic = str(rec.get("topic") or "")
252
+ memo_key = (topic, str(owner_sess.uname))
253
+ g = assemblies.get(memo_key) if isinstance(assemblies, dict) else None
254
+ if g is None:
255
+ try:
256
+ if topic.startswith("ut_"):
257
+ from routes_tables import ut_assembly
258
+ # ⛔ `consume_corrections=False`, and the default was a REAL BUG, not a tidy-up.
259
+ # `ut_assembly` defaults it True, so every `/notifications` read CONSUMED the
260
+ # one-shot field-name correction acks for every `ut_` topic that has an alert —
261
+ # taking them from the `/workspace` refresh that exists to show them to the person
262
+ # who made the edit. The customer branch below has always passed False; this one
263
+ # inherited a default nobody re-read. An inbox poll must never consume a one-shot.
264
+ g = ut_assembly(owner_sess, topic,
265
+ storage_key=f"{owner_sess.tenant}:{topic}:{owner_sess.uname}",
266
+ consume_corrections=False)
267
+ else:
268
+ from routes_customers import grid_assembly
269
+ g = grid_assembly(owner_sess, scope=topic, consume_corrections=False)
270
+ except Exception as e: # noqa: BLE001
271
+ # ⭐ W32-T22 — SKIPPING IS FINE HERE; SKIPPING ANONYMOUSLY IS NOT. This one must not
272
+ # raise (one bad alert cannot empty an inbox), so unlike `_require_filtered_view` it
273
+ # keeps a blanket catch — but it now reports the refusal's OWN code where there is
274
+ # one. `type(e).__name__` said `HTTPException` for four different causes, and
275
+ # `lastError` is the only place a user ever learns why an alert stopped firing.
276
+ #
277
+ # ⚠ `with_rows=True` STAYS on this path, deliberately: unlike the create door, an
278
+ # evaluation genuinely needs the rows to run the filter over. So an alert on a
279
+ # read-through grid is created (T22) and then skips at evaluation with
280
+ # `window_required` naming why — which is D-184's remaining half, and it is a
281
+ # SENTENCE now rather than silence.
282
+ return {"skipped": _refusal_code(e) or "unavailable", "detail": type(e).__name__}
283
+ if isinstance(assemblies, dict):
284
+ assemblies[memo_key] = g
285
+
286
+ view = (g.get("views") or {}).get(str(rec.get("viewId")))
287
+ if not isinstance(view, dict):
288
+ # Deleted, or un-shared out from under the alert. Say so on the RECORD rather than
289
+ # deleting the alert: an alert that silently vanishes is indistinguishable from one that
290
+ # never fires, and the user cannot debug what is not there.
291
+ return {"skipped": "view_missing"}
292
+
293
+ # The SAME row build the grid and `/customers` use — `rows_from_pool` is what puts derived
294
+ # and overlay values on a row. Evaluating a filter against raw pool dicts would silently
295
+ # never match any condition on a user-created or measure column.
296
+ rows = aios_grid.rows_from_pool(g["rows_src"], g["fields"], g["ws"].get("overlays"),
297
+ derived=g.get("derived"))
298
+ config = view.get("config") or view
299
+ ctx = filter_eval.EvalCtx(
300
+ cohort_sets={str(k): {str(p) for p in (v.get("memberPids") or ())}
301
+ for k, v in (g.get("lists") or {}).items() if isinstance(v, dict)},
302
+ measure_sets=g.get("measure_sets") or {},
303
+ today=g.get("today"))
304
+ pids = filter_eval.visible_pids(config.get("filters") or [], rows, g["fields"], ctx,
305
+ member_pids=config.get("memberPids"))
306
+ labels = {str(r.get("pid")): str(r.get("name") or r.get("pid")) for r in rows}
307
+ return alerts.evaluate(rec.get("id"), [str(p) for p in pids],
308
+ labels=labels, partial=False, st=session.runtime)
309
+
310
+
311
+ @router.get("/alerts")
312
+ def list_alerts(session: Session = Depends(require_session)):
313
+ return {"alerts": alerts.list_alerts(user=session.uname, is_admin=session.admin,
314
+ st=session.runtime)}
315
+
316
+
317
+ @router.post("/alerts")
318
+ def create_alert(body: dict = Body(default=None), session: Session = Depends(require_session)):
319
+ body = body or {}
320
+ view_id = str(body.get("viewId") or "").strip()
321
+ if not view_id:
322
+ raise err(400, "bad_view", "an alert needs the id of the view it watches")
323
+ topic = _topic_or_400(body.get("topic"))
324
+ _require_filtered_view(session, topic, view_id)
325
+ import uuid
326
+ aid = f"al_{uuid.uuid4().hex[:12]}"
327
+ rec = alerts.create(aid, view_id=view_id, topic=topic, owner=session.uname,
328
+ label=body.get("label") or "", st=session.runtime)
329
+ # SEED IMMEDIATELY, so the alert starts from "everything currently matching is old news".
330
+ # Deferring this to the first write hook would mean the next edit announces the whole view.
331
+ outcome = _evaluate(session, rec)
332
+ return {"alert": {**rec, "seeded": True}, "first": outcome}
333
+
334
+
335
+ def _require_filtered_view(session: Session, topic: str, view_id: str):
336
+ """400 unless `view_id` exists on `topic` AND actually narrows something.
337
+
338
+ ⛔ AN ALERT ON AN UNFILTERED VIEW IS SILENTLY INCAPABLE OF ALERTING, which is worse than one
339
+ that is refused. `filter_eval` treats an inactive tree as "no narrowing, every row shows"
340
+ (`visible_pids`'s own rule), so such an alert seeds with the entire table and can never see an
341
+ entrant again — there is nothing left to enter. The owner's words are *"when a Record gets
342
+ into that Filter's criteria"*: no criteria, no alert, and said at creation rather than
343
+ discovered by never being notified.
344
+
345
+ `is_rule_active` is the SAME activeness predicate the engine and the column tints use — a
346
+ half-typed rule is not a filter, and this must agree with what actually narrows or it would
347
+ accept a view whose one rule the engine then ignores.
348
+
349
+ ⭐⭐ WAVE 32 · T22 (owner item 17) — THIS FUNCTION WAS THE ERROR. Two defects, stacked, and
350
+ the second one hid the first.
351
+
352
+ (1) **IT ASKED FOR EVERY ROW OF A TABLE IT NEVER LOOKS AT.** The only thing read below is
353
+ `g["views"]`. `ut_assembly` defaults `with_rows=True`, so creating an alert on a
354
+ read-through grid built the whole pool — and `scoped_pool` refuses that with
355
+ `409 window_required` over 963,783 rows, exactly as it is supposed to. `with_rows=False`
356
+ (W31-T20's flag, built for precisely this) answers the same question with `scoped_pids`,
357
+ runs the SAME `_defn_or_refuse` wall, and does not refuse. **That is D-184's create half,
358
+ closed** — an alert on a read-through grid can now be made at all.
359
+ (2) **A BLANKET `except Exception` TURNED EVERY NAMED REFUSAL INTO A 503.** `HTTPException`
360
+ is an `Exception`, so `404 unknown_table`, `403 forbidden`, `409 window_required` and
361
+ `503 store_not_ready` — four refusals that each say what is wrong — were all replaced by
362
+ *"the table is unavailable — try again in a moment"*. ⛔ AND THAT SENTENCE NEVER REACHED
363
+ A USER EITHER: `alertsApi.errorMessage` discards the text of any status ≥ 500 by design
364
+ (a 5xx body is the server's internals), substituting *"Something went wrong on our
365
+ side."* — which is the owner's screenshot, word for word. A knowable cause returned as a
366
+ 5xx is invisible by construction, so re-wording the 503 could never have fixed this.
367
+ ⚠ The except is narrowed, not deleted: an UNEXPECTED failure is still a 503, because that is
368
+ honest. What it may no longer do is catch a refusal that already knows its own name.
369
+ """
370
+ from fastapi import HTTPException
371
+
372
+ from harness import filter_eval
373
+
374
+ try:
375
+ if topic.startswith("ut_"):
376
+ from routes_tables import ut_assembly
377
+ # ⚠ `consume_corrections=False` — the customer branch has always passed it and this
378
+ # one inherited a default nobody re-read. Creating an alert must not eat the one-shot
379
+ # field-name correction acks belonging to the `/workspace` refresh that exists to show
380
+ # them to the person who made the edit. Same defect `_evaluate`'s header records.
381
+ g = ut_assembly(session, topic,
382
+ storage_key=f"{session.tenant}:{topic}:{session.uname}",
383
+ consume_corrections=False, with_rows=False)
384
+ else:
385
+ from routes_customers import grid_assembly
386
+ g = grid_assembly(session, scope=topic, consume_corrections=False)
387
+ except HTTPException:
388
+ raise # it already names its own cause
389
+ except Exception as e: # noqa: BLE001
390
+ # Genuinely unexpected. Still a 503, and now it carries the exception TYPE — without it,
391
+ # the one path that reaches this branch is also the one path with nothing to debug from.
392
+ raise err(503, "unavailable",
393
+ f"the table could not be read ({type(e).__name__}) — try again in a moment")
394
+ view = (g.get("views") or {}).get(str(view_id))
395
+ if not isinstance(view, dict):
396
+ raise err(404, "no_view", "that view does not exist on this table")
397
+ nodes, _conj = filter_eval.tree_parts((view.get("config") or view).get("filters") or [])
398
+
399
+ # ⛔⛔ WAVE 32 · T22 — **THE CALL BELOW WAS MISSING AN ARGUMENT, AND THAT IS OWNER ITEM 17.**
400
+ #
401
+ # `is_rule_active(rule, columns)` takes TWO parameters (`harness/filter_sql.py`; every other
402
+ # caller in the repo passes both). This one passed ONE, so the moment the walk reached a LEAF
403
+ # rule it raised `TypeError: is_rule_active() missing 1 required positional argument`.
404
+ #
405
+ # ⚠ READ WHAT THAT MEANS BEFORE FIXING ANYTHING ELSE: the walk only reaches a leaf when the
406
+ # view HAS a condition — and a view with a condition is the only kind an alert is allowed on.
407
+ # A view with no filters yields an empty `nodes`, so `_any_active` returns False without ever
408
+ # calling this, and the reader gets the honest 400 `no_filter`. **So the only path that
409
+ # worked was the refusal path: "Alert me about new records" had never once created an alert
410
+ # on a filtered view.** ⛔ And the raise lands OUTSIDE the `try` above, so it was not even the
411
+ # 503 — it was a bare FastAPI 500, which `alertsApi.errorMessage` renders as *"Something went
412
+ # wrong on our side. Try again in a moment."*, the owner's screenshot word for word.
413
+ #
414
+ # ⚠ THREE THINGS HID IT, and they are worth more than the fix. (1) Python does not check
415
+ # arity until the line RUNS, and this line runs only on the success path of a feature whose
416
+ # every test exercised its refusals. (2) The `no_filter` 400 above it is a real, correct,
417
+ # well-tested refusal, so the door looked alive. (3) `verify_alerts.py` asserts the refusal
418
+ # (`no_filter` reaches the user) and the transport — never a creation. A gate can be green,
419
+ # thorough and honest about everything except the one path the feature exists for.
420
+ #
421
+ # `_columns_map` is the DEFINITION of fields -> the membership set `is_rule_active` looks a
422
+ # column up in; building a second dict here would be a second answer to one question, which
423
+ # is this wave's other headline defect in a different file. Its leading underscore is a real
424
+ # smell and is BOOKED (PENDING, mailbox/C.md) rather than worked around.
425
+ columns = filter_eval._columns_map(g.get("fields") or [])
426
+
427
+ def _any_active(ns):
428
+ for n in ns or ():
429
+ if isinstance(n, dict) and isinstance(n.get("children"), list):
430
+ if _any_active(n["children"]):
431
+ return True
432
+ elif filter_eval.is_rule_active(n, columns):
433
+ return True
434
+ return False
435
+
436
+ if not _any_active(nodes):
437
+ raise err(400, "no_filter",
438
+ "this view has no active filter, so no record can ever ENTER it — add a "
439
+ "condition to the view first, then create the alert")
440
+
441
+
442
+ @router.delete("/alerts/{alert_id}")
443
+ def delete_alert(alert_id: str, session: Session = Depends(require_session)):
444
+ rec = next((r for r in alerts.list_alerts(st=session.runtime)
445
+ if str(r.get("id")) == str(alert_id)), None)
446
+ if rec is None:
447
+ raise err(404, "no_alert", "that alert does not exist")
448
+ if str(rec.get("owner")) != str(session.uname) and not session.admin:
449
+ raise err(403, "not_yours", "only the alert's owner (or an administrator) can delete it")
450
+ alerts.delete(alert_id, st=session.runtime)
451
+ return {"ok": True}
452
+
453
+
454
+ @router.post("/alerts/{alert_id}/run")
455
+ def run_alert(alert_id: str, session: Session = Depends(require_session)):
456
+ rec = next((r for r in alerts.list_alerts(user=session.uname, is_admin=session.admin,
457
+ st=session.runtime)
458
+ if str(r.get("id")) == str(alert_id)), None)
459
+ if rec is None:
460
+ raise err(404, "no_alert", "that alert does not exist")
461
+ return _evaluate(session, rec)
462
+
463
+
464
+ @router.get("/notifications")
465
+ def notifications(session: Session = Depends(require_session)):
466
+ """The inbox — RE-EVALUATED on read, which is a deliberate design choice.
467
+
468
+ ⭐ A-S1-2 RESOLVED THE OTHER WAY, and the reason is structural rather than a shortcut. The
469
+ plan was a push hook: the automation engine calls `after_write` when it lands rows. But
470
+ `run_async` runs on a BACKGROUND THREAD with no `Session` in scope, and an alert must be
471
+ evaluated as its OWNER (see `_evaluate`) — so a push hook would have to mint a session inside
472
+ a worker thread from a tenant runtime, which is exactly the kind of ad-hoc identity
473
+ construction that leaks scope.
474
+
475
+ Pulling on read has none of that: the caller IS a session, the assemblies are already
476
+ scope-cached, and the user cannot observe the difference — an inbox is only ever read by
477
+ someone opening it. The cost is that a notification is minted when you LOOK rather than when
478
+ the row landed, so the `at` stamp is detection time, not arrival time.
479
+
480
+ `after_write` stays exported for the day the engine can hand over a real identity.
481
+
482
+ ⭐⭐ W31-T24 — ONE ASSEMBLY PER (TOPIC, OWNER), NOT ONE PER ALERT.
483
+ ⛔ MEASURED FIRST, AND THE MEASUREMENT CORRECTS AN EARLIER READING OF IT. This route is
484
+ **20 ms in-process and 3,280 ms live** on tenant #0 — but tenant #0 has **ZERO alerts**
485
+ (censused 2026-08-12), so the 20 ms is an EMPTY LOOP and says nothing at all about what the
486
+ re-evaluation costs. The live 3,280 ms is the two store reads either side of that loop. So the
487
+ body below is not slow today; it is UNEXERCISED, and every alert a tenant creates adds a whole
488
+ grid assembly to an inbox poll. The memo turns O(alerts) into O(distinct topic × owner), which
489
+ is the difference between "fine" and "three seconds per alert" the day somebody uses the
490
+ feature. ⚠ Making the read cheap by evaluating LESS is the obvious wrong fix and is not what
491
+ this does: every alert is still evaluated, against the same rows, in the same order.
492
+ """
493
+ assemblies = {}
494
+ for rec in alerts.list_alerts(user=session.uname, is_admin=False, st=session.runtime):
495
+ try:
496
+ _evaluate(session, rec, assemblies=assemblies)
497
+ except Exception: # noqa: BLE001
498
+ continue # one bad alert must not empty the pane
499
+ # ⭐ W32-T20 (C3): every item leaves through `inbox_view`, so a notification queued before
500
+ # this wave carries a `target` too. See `notification_view`'s header for why it is derived.
501
+ return inbox_view(alerts.inbox(session.uname, st=session.runtime))
502
+
503
+
504
+ @router.post("/notifications/read")
505
+ def read_notifications(body: dict = Body(default=None),
506
+ session: Session = Depends(require_session)):
507
+ body = body or {}
508
+ ids = body.get("ids")
509
+ if ids is not None and not isinstance(ids, list):
510
+ raise err(400, "bad_ids", "ids must be a list, or null to mark every notification")
511
+ # ⚠ THE SAME ENRICHMENT ON BOTH DOORS. `mark_read` returns a fresh inbox, and the Inbox
512
+ # module re-renders from it — an un-enriched answer here would strip `target` off every row
513
+ # the moment somebody marked one read, i.e. the feature would work until first use.
514
+ return inbox_view(alerts.mark_read(session.uname, ids, read=bool(body.get("read", True)),
515
+ st=session.runtime))
516
+
517
+
518
+ def after_write(session: Session, topic_key: str):
519
+ """THE WRITE HOOK — call after a write that could change what a view matches.
520
+
521
+ Exported as a plain function (not a route) so `core.grid_events`' callers and S2's automation
522
+ upserts reach it the same way. It never raises: an alert evaluation failing must not fail the
523
+ edit that triggered it.
524
+
525
+ ⭐ W31-T24 — it shares `/notifications`' memo shape for the same reason: a write that changes
526
+ one view can trip several alerts on the SAME topic, and each would otherwise rebuild the table.
527
+ ⚠ STILL ZERO PRODUCTION CALLERS (W31-T24 confirmed it; the route docstring above says why the
528
+ push hook was resolved the other way). Booked rather than wired: minting a session inside the
529
+ engine's worker thread is the ad-hoc identity construction this file exists to avoid.
530
+ """
531
+ try:
532
+ assemblies = {}
533
+ return alerts.after_write(topic_key, st=session.runtime,
534
+ runner=lambda rec: _evaluate(session, rec,
535
+ assemblies=assemblies))
536
+ except Exception: # noqa: BLE001
537
+ return {"evaluated": 0}
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
api/routes_automation.py CHANGED
The diff for this file is too large to render. See raw diff
 
api/routes_changes.py CHANGED
@@ -55,7 +55,7 @@ So: a valid session, a scope sanitised to a bucket-safe shape, and — for a `ut
55
  cached verdict that this session may open it.
56
  """
57
  import time
58
- from fastapi import APIRouter, Depends, Request
59
 
60
  import core.shared_overlay as shared_overlay
61
  import core.store as store
@@ -224,87 +224,3 @@ def changes(scope: str = "", session: Session = Depends(require_session)):
224
  # a token frozen at 0 would promise liveness the backend is not delivering.
225
  tokens[label] = rev.get("token") if isinstance(rev, dict) else None
226
  return {"scope": _clean_scope(scope), "tokens": tokens, "backend": store.backend()}
227
-
228
-
229
- # ── ⭐⭐ D-144 (W33-T32) — A CAUGHT RENDER ERROR TELLS SOMEBODY ────────────────────────────────
230
- #
231
- # ⛔ THE DEBT, in its own words: *"A caught render error tells nobody."* W30-T39 gave the app its
232
- # first error boundary — measured at the time, `componentDidCatch|getDerivedStateFromError` was
233
- # **0 hits** across all of `web/src`, so any throw anywhere blanked the whole product. The
234
- # boundary CONTAINS the failure; it does not REPORT it, so the next user complaint still arrives
235
- # as a sentence rather than as a stack. That is why owner item 2's crash could not be diagnosed
236
- # and had to be fixed at both candidate causes.
237
- #
238
- # ⚠ IT LIVES HERE, IN THE CHANGE-TOKEN ROUTER, AND THE REASON IS WORTH ONE LINE. It is not about
239
- # change tokens. It is here because this router is already mounted, already in one lane's fence,
240
- # and a NEW `routes_*.py` costs a cross-fence mount (contract C2) that this wave has already spent
241
- # three times — and a fourth finished-but-unmounted router is precisely the failure wave 23 shipped
242
- # three of. Subject purity would have been bought with the exact risk the contract exists to
243
- # prevent. ⚠ It performs no `store.get()`, so this file's zero-deep-copy claim (and
244
- # `verify_live_workspace`'s count of it) is untouched: `telemetry.log` appends one line to a
245
- # container-local JSONL.
246
- #
247
- # ⛔⛔ AND IT CARRIES NO ROW DATA, WHICH IS THE DEBT'S OWN HARD CONDITION. The body is rebuilt KEY
248
- # BY KEY from exactly two strings; there is no `**body` anywhere below. A crash reporter that
249
- # forwarded props or state would quietly ship a customer's cell values into a log — the one way
250
- # this feature could do more harm than the silence it replaces.
251
- _CLIENT_ERR_MAX = 4 * 1024
252
- _CLIENT_ERR_WINDOW_S = 60
253
- _CLIENT_ERR_PER_WINDOW = 12
254
- _CLIENT_ERR_HITS: dict = {}
255
-
256
-
257
- @router.post("/client-error")
258
- async def client_error(request: Request, session: Session = Depends(require_session)):
259
- """One caught render error, forwarded to telemetry. `{message, componentStack}`.
260
-
261
- ⚠ SESSION-GATED, deliberately. A public crash sink is an unauthenticated append to a log file,
262
- which is a disk-filling primitive with a URL. Only signed-in accounts can crash this app in a
263
- way we need to hear about.
264
- ⚠ RATE-LIMITED PER ACCOUNT, because the failure mode is a render LOOP: a boundary that catches,
265
- resets and re-throws would otherwise post several times a second, for ever, from every open
266
- tab. Beyond the window the report is DROPPED and the request still answers 200 — the client
267
- must never learn to retry a report.
268
- """
269
- now = time.time()
270
- who = session.uname or "?"
271
- seen = [t for t in _CLIENT_ERR_HITS.get(who, ()) if now - t < _CLIENT_ERR_WINDOW_S]
272
- seen.append(now)
273
- _CLIENT_ERR_HITS[who] = seen
274
- if len(_CLIENT_ERR_HITS) > 2048:
275
- for k in [k for k, v in _CLIENT_ERR_HITS.items()
276
- if not v or now - v[-1] > _CLIENT_ERR_WINDOW_S]:
277
- _CLIENT_ERR_HITS.pop(k, None)
278
- if len(seen) > _CLIENT_ERR_PER_WINDOW:
279
- return {"ok": True, "recorded": False}
280
-
281
- # The body, BOUNDED BY THE STREAM. `Body(...)` would already have been read and parsed before
282
- # this line ran, and `content-length` is caller-supplied — the same argument `routes_forms`
283
- # and `routes_publish` both carry.
284
- size, chunks = 0, []
285
- async for chunk in request.stream():
286
- size += len(chunk)
287
- if size > _CLIENT_ERR_MAX:
288
- raise err(413, "body_too_large", "that report is too large")
289
- chunks.append(chunk)
290
- import json as _json
291
- try:
292
- body = _json.loads(b"".join(chunks) or b"{}")
293
- except ValueError:
294
- body = {}
295
- if not isinstance(body, dict):
296
- body = {}
297
-
298
- # ⛔ TWO KEYS, BOTH TRUNCATED, NOTHING ELSE. See the block note above.
299
- message = str(body.get("message") or "")[:500]
300
- stack = str(body.get("componentStack") or "")[:4000]
301
- if not message and not stack:
302
- return {"ok": True, "recorded": False}
303
- try:
304
- import harness.telemetry as _tel
305
- _tel.log("client_error", user=session.uname, tenant=session.tenant,
306
- message=message, componentStack=stack)
307
- except Exception: # noqa: BLE001
308
- # Telemetry may never break a request — `deps.py` carries the same rule at its own site.
309
- return {"ok": True, "recorded": False}
310
- return {"ok": True, "recorded": True}
 
55
  cached verdict that this session may open it.
56
  """
57
  import time
58
+ from fastapi import APIRouter, Depends
59
 
60
  import core.shared_overlay as shared_overlay
61
  import core.store as store
 
224
  # a token frozen at 0 would promise liveness the backend is not delivering.
225
  tokens[label] = rev.get("token") if isinstance(rev, dict) else None
226
  return {"scope": _clean_scope(scope), "tokens": tokens, "backend": store.backend()}
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
api/routes_connectors.py CHANGED
@@ -30,7 +30,7 @@ from deps import Session, require_session
30
  #: one question, and this module used to answer it with an unguarded `os.environ.get("ODOO_URL")`
31
  #: while `routes_keychain` answered it correctly. Importing the correct one is the fix; writing a
32
  #: second correct one here would leave the same two-copies shape that produced the defect.
33
- from routes_keychain import TENANT_WIDE_TYPES, env_odoo_available
34
 
35
  # ⛔ THE PREFIX IS LOAD-BEARING AND IT WAS MISSING (found at the wave-23 close-out audit).
36
  # This router declared a bare `APIRouter()`, so its one path was served at `/connectors/directory`
@@ -396,31 +396,15 @@ def directory(rt, uname, is_admin=False, incomplete=None):
396
  # reader. A token connector is the case R4 is really about: a member's own Stripe
397
  # key is theirs, and the card must say which kind of connection it is looking at.
398
  "scope": scopes.get(t["entry"]) or "business",
399
- # ⭐⭐ W33-T66 / D-204 — THE MEMBER'S DOOR, AND IT IS A THIRD WORD RATHER THAN A
400
- # WIDENED ONE. This route's own docstring has promised since wave 23 that a
401
- # non-admin may *"connect their OWN per-user accounts"*, and the next sentence
402
- # then handed them an empty `manage` — a contradiction inside one paragraph, and
403
- # D-204's subject. W32-T11 opened the ROOM (`admin_gate` is gone from
404
- # `routes_keychain`; a member may POST `scope: "personal"`), so the only thing
405
- # still missing was a way to reach it from the map.
406
- #
407
- # ⛔ `keychain` DOES NOT WIDEN, and that is deliberate rather than timid.
408
- # `verify_automation`'s NC asserts a non-admin is offered no `manage == "keychain"`
409
- # row, and its own comment says why in the right words: *"the line is TENANT-WIDE
410
- # vs PER-USER, not admin vs everyone … what they must never be offered is the
411
- # KEYCHAIN door, which edits credentials the whole tenant uses."* That assertion
412
- # is CORRECT and stays green. A member's personal connection is a different door
413
- # into the same room, so it gets its own word.
414
- #
415
- # ⚠ THE PREDICATE IS THE CREDENTIAL'S OWN RULE, IMPORTED, NOT RE-STATED HERE.
416
- # `TENANT_WIDE_TYPES` is what `routes_keychain` REFUSES a personal scope for, so
417
- # offering a personal door for one of those types would be offering a button that
418
- # 400s — the exact failure R6 removed from the Google card. Today that means
419
- # Stripe and Shopify open for a member and Meta Ads does not
420
- # ([[one-question-two-normalizers]]).
421
- "manage": ("keychain" if is_admin
422
- else "" if t["entry"] in TENANT_WIDE_TYPES
423
- else "keychain_personal"),
424
  # Said once, on the row, rather than in a paragraph somewhere (R13): storing the
425
  # key is real and useful, and it is not the same as the data being on screen.
426
  "note": "Storing the key connects the account; reading its data lands in a later "
@@ -582,22 +566,9 @@ def connectors_directory(session: Session = Depends(require_session)):
582
  """C11: every connector this platform knows about, with its state for THIS reader.
583
 
584
  Session-gated rather than admin-gated on purpose: a non-admin should be able to see what the
585
- workspace is connected to, and connect their OWN per-user accounts — which is the whole point
586
- of R7's per-user slots.
587
-
588
- ⭐⭐ W33-T66 / D-204 — AND THAT SECOND CLAUSE IS NOW TRUE. This docstring used to promise it and
589
- then say, one sentence later, that `manage` *"is empty for a non-admin"* — a contradiction
590
- inside a single paragraph, which is how the promise survived three waves without a door behind
591
- it. `manage` carries WHICH ROOM, and there are three answers for a keychain-backed row:
592
-
593
- · `keychain` — the tenant-wide credential door. **Admins only**, and it must stay
594
- that way: it edits the key the whole workspace reads through.
595
- · `keychain_personal` — the same room, for a connection that is this reader's own. Offered
596
- to a non-admin **only** where a personal credential is actually
597
- allowed (`routes_keychain.TENANT_WIDE_TYPES` is the refusal list, so
598
- Odoo and Meta Ads never get it).
599
- · `""` — nothing this reader can do here; the client renders no control
600
- rather than a door that would 403.
601
  """
602
  # ⚠ `session.admin`, NOT `is_admin`. A `getattr(session, "is_admin", False)` here reads as
603
  # careful and is silently False for every admin on earth — the manage actions would simply
 
30
  #: one question, and this module used to answer it with an unguarded `os.environ.get("ODOO_URL")`
31
  #: while `routes_keychain` answered it correctly. Importing the correct one is the fix; writing a
32
  #: second correct one here would leave the same two-copies shape that produced the defect.
33
+ from routes_keychain import env_odoo_available
34
 
35
  # ⛔ THE PREFIX IS LOAD-BEARING AND IT WAS MISSING (found at the wave-23 close-out audit).
36
  # This router declared a bare `APIRouter()`, so its one path was served at `/connectors/directory`
 
396
  # reader. A token connector is the case R4 is really about: a member's own Stripe
397
  # key is theirs, and the card must say which kind of connection it is looking at.
398
  "scope": scopes.get(t["entry"]) or "business",
399
+ # ⚠ `manage` STAYS ADMIN-ONLY HERE, deliberately, and it is a smaller door than
400
+ # R4 allows. A member may now hold a PERSONAL connection — the room for it is
401
+ # Settings → Keychains, which W32-T11 opens to them — but this card's action word
402
+ # has been "the tenant-wide credential door" since wave 23 and a sibling gate
403
+ # asserts a member is offered none. Widening it is a one-word change once that
404
+ # assertion is retargeted; booked rather than done, because reddening another
405
+ # lane's gate to add a second door to a room that is already reachable is a bad
406
+ # trade.
407
+ "manage": "keychain" if is_admin else "",
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
408
  # Said once, on the row, rather than in a paragraph somewhere (R13): storing the
409
  # key is real and useful, and it is not the same as the data being on screen.
410
  "note": "Storing the key connects the account; reading its data lands in a later "
 
566
  """C11: every connector this platform knows about, with its state for THIS reader.
567
 
568
  Session-gated rather than admin-gated on purpose: a non-admin should be able to see what the
569
+ workspace is connected to (and connect their OWN per-user accounts, which is the whole point
570
+ of R7's per-user slots). `manage` is what carries admin-ness — it is empty for a non-admin,
571
+ so the client renders the card without an action rather than offering a door that 403s.
 
 
 
 
 
 
 
 
 
 
 
 
 
572
  """
573
  # ⚠ `session.admin`, NOT `is_admin`. A `getattr(session, "is_admin", False)` here reads as
574
  # careful and is silently False for every admin on earth — the manage actions would simply
api/routes_customers.py CHANGED
@@ -209,17 +209,6 @@ def grid_assembly(session: Session, scope: str = "customer", storage_key: str =
209
  # The derived channel: cohort membership cells + measure column values, ONE dict — the
210
  # same read-only channel the embed host hands to rows_from_pool.
211
  derived = aios_grid.cohort_cells(lists)
212
- # ⭐ W33-T43 / owner item 12 ("One unique ID per database always"). The customer grid carried
213
- # NO Odoo id column at all, while its retiring twin `ut_odoo_customers` carried `partner_id`
214
- # as its join key — so the merge would have lost the one value every Odoo document joins on.
215
- #
216
- # ⛔ IT IS DERIVED, NOT A POOL COLUMN, AND THAT IS THE WHOLE POINT: a customer row's `pid` IS
217
- # the `res.partner` id (`modules/customer_data.pool` mints it that way and the identity is
218
- # asserted against Odoo in that module). Adding it to the pool would be a SECOND source for
219
- # one fact, which is the class of defect item 12 is about. This channel exists for exactly
220
- # this — a value the host knows per render and the pool has no business storing.
221
- for pid in pids:
222
- derived.setdefault(pid, {})["partner_id"] = pid
223
  for pid, cells in measure_resolve.column_values(
224
  fields, team_id, pids, today, stamp, rt.measure_memo,
225
  on_error=_measure_err).items():
 
209
  # The derived channel: cohort membership cells + measure column values, ONE dict — the
210
  # same read-only channel the embed host hands to rows_from_pool.
211
  derived = aios_grid.cohort_cells(lists)
 
 
 
 
 
 
 
 
 
 
 
212
  for pid, cells in measure_resolve.column_values(
213
  fields, team_id, pids, today, stamp, rt.measure_memo,
214
  on_error=_measure_err).items():
api/routes_keychain.py CHANGED
The diff for this file is too large to render. See raw diff
 
api/routes_oauth.py CHANGED
@@ -1,114 +1,114 @@
1
- """routes_oauth.py — the OAuth connector surface (wave 22, contract C5 + A2/A3 / R12).
2
-
3
- Thin over `oauth_connect`, the way `routes_automation` is thin over the engine: sessions,
4
- shapes and status codes here; every decision that could be wrong lives in the module a gate
5
- can drive without a server. GENERIC over `{provider}` (C5-A2): the routes read the registry,
6
- so the day a second provider lands here is the day nothing in this file changes.
7
-
8
- MOUNTED FROM `routes_automation` (not `main.py`): this wave's ownership fence gives no session
9
- `main.py`, and `routes_automation` is already included there — so this router rides inside it
10
- (`/api/v1` + `/oauth/...`). Lifting the include into `main.py` later is a two-line change that
11
- alters no path.
12
-
13
- ⚠ THE TWO REDIRECT LAWS (A3): `/{provider}/start` answers **302 to the provider's consent
14
- screen** — it is a top-level navigation the client reaches by `<a href>`, never JSON. The
15
- callback 302s BACK to the return path the `state` carried (relative-only, sanitised by
16
- `oauth_connect.safe_next`), so the user lands where they left — connected or not, whatever
17
- went wrong rides in the query string; a dead-end error page where the app used to be reads as
18
- "the product broke", not "the connect failed".
19
- """
20
- import os
21
-
22
- from fastapi import APIRouter, Depends, Request
23
- from fastapi.responses import RedirectResponse
24
-
25
- import oauth_connect
26
- from deps import Session, err, require_session
27
-
28
- router = APIRouter(prefix="/oauth")
29
-
30
-
31
- def _redirect_uri(request: Request, provider: str) -> str:
32
- """The redirect URI this deployment registers at the provider — env-pinned when the
33
- container sits behind a proxy that rewrites the scheme (the HF Space), else derived from
34
- the request. MUST match a console-registered URI verbatim, so it is computed in exactly
35
- one place.
36
-
37
- ⭐ WAVE 29 (R4): `deploy_web.py` now PUSHES `AIOS_PUBLIC_BASE` on every deploy, defaulted to
38
- the same URL as `APP_BASE_URL`, so the pinned branch is the one that runs in production and
39
- the request-derived fallback below is effectively dev-only.
40
- ⛔ THAT MAKES THIS FUNCTION A CUSTOM-DOMAIN COUPLING, not merely a scheme fix. Whatever host
41
- this returns is where the provider sends the user BACK, and the session cookie is host-only
42
- (`aios_session.py:114-117`, no `domain=`) — so a callback base that disagrees with the host
43
- the user actually browsed plants the session on the wrong hostname and they return logged
44
- out. Moving the app to a new hostname means moving this value AND re-registering the
45
- resulting URI in the provider console; one without the other fails closed.
46
- Runbook: `.claude/wiki/research/loopable-domain-runbook.md`."""
47
- base = (os.environ.get("AIOS_PUBLIC_BASE") or "").strip().rstrip("/")
48
- if not base:
49
- base = f"{request.url.scheme}://{request.url.netloc}"
50
- return f"{base}/api/v1/oauth/{provider}/callback"
51
-
52
-
53
- @router.get("/status")
54
- def oauth_status(session: Session = Depends(require_session)):
55
- """C5's status shape for the SESSION user, one entry per registry provider:
56
- `{google: {connected, email, reconnect, configured}}` today. The bit the email trigger's
57
- `ready` reads through."""
58
- return oauth_connect.status(session.runtime, session.uname)
59
-
60
-
61
- def _offered_or_404(provider: str):
62
- """⛔⛔ W32-T14 / OWNER ITEM 12 / R6 — A PROVIDER THE PRODUCT DOES NOT OFFER HAS NO DOOR.
63
-
64
- The owner pasted the failure this replaces: clicking Connect on the Google card answered
65
- `503 oauth_unavailable` as raw JSON. R6's fix is not a nicer error — it is that the flow is
66
- not offered at all, so the honest status is the one for a URL that does not exist. `404`
67
- rather than `503`, deliberately: a 503 says *"come back later"* about a door that is not
68
- coming back until somebody pays for CASA verification (D-45, ~$540–1,800/yr).
69
-
70
- ⚠ Every door in this router goes through it, START included, because the JSON the owner saw
71
- came from the start route and a guard on one leg is a guard on one leg.
72
- """
73
- if not oauth_connect.offered(provider):
74
- raise err(404, "unknown_provider", f"{provider!r} is not a connectable provider")
75
-
76
-
77
- @router.get("/{provider}/start")
78
- def oauth_start(provider: str, request: Request, next: str = "",
79
- session: Session = Depends(require_session)):
80
- """302 to the provider's consent screen (A3 — a navigation, never JSON). `?next=` is the
81
- RELATIVE path the callback returns the browser to; it rides inside the single-use state,
82
- sanitised, so the round trip cannot be steered off-origin."""
83
- _offered_or_404(provider)
84
- url, problem = oauth_connect.start(provider, session.uname,
85
- _redirect_uri(request, provider), next_path=next)
86
- if problem:
87
- raise err(503 if "not configured" in problem else 404, "oauth_unavailable", problem)
88
- return RedirectResponse(url, status_code=302)
89
-
90
-
91
- @router.get("/{provider}/callback")
92
- def oauth_callback(provider: str, request: Request,
93
- session: Session = Depends(require_session),
94
- state: str = "", code: str = "", error: str = ""):
95
- """The provider's redirect target. Exchanges the code, stores the per-user slot, and sends
96
- the browser back to the state's return path — connected or not (see module header)."""
97
- _offered_or_404(provider)
98
- if error:
99
- home = "/#/"
100
- return RedirectResponse(f"{home}?oauthError={error[:80]}", status_code=302)
101
- email, home, problem = oauth_connect.callback(session.runtime, session.uname, state, code)
102
- sep = "&" if "?" in home else "?"
103
- if problem:
104
- return RedirectResponse(f"{home}{sep}oauthError=connect_failed", status_code=302)
105
- return RedirectResponse(f"{home}{sep}connected={provider}", status_code=302)
106
-
107
-
108
- @router.post("/{provider}/disconnect")
109
- def oauth_disconnect(provider: str, session: Session = Depends(require_session)):
110
- _offered_or_404(provider)
111
- if oauth_connect.provider_def(provider) is None:
112
- raise err(404, "unknown_provider", f"{provider!r} is not a connectable provider")
113
- oauth_connect.disconnect(session.runtime, session.uname, provider)
114
- return {"disconnected": provider}
 
1
+ """routes_oauth.py — the OAuth connector surface (wave 22, contract C5 + A2/A3 / R12).
2
+
3
+ Thin over `oauth_connect`, the way `routes_automation` is thin over the engine: sessions,
4
+ shapes and status codes here; every decision that could be wrong lives in the module a gate
5
+ can drive without a server. GENERIC over `{provider}` (C5-A2): the routes read the registry,
6
+ so the day a second provider lands here is the day nothing in this file changes.
7
+
8
+ MOUNTED FROM `routes_automation` (not `main.py`): this wave's ownership fence gives no session
9
+ `main.py`, and `routes_automation` is already included there — so this router rides inside it
10
+ (`/api/v1` + `/oauth/...`). Lifting the include into `main.py` later is a two-line change that
11
+ alters no path.
12
+
13
+ ⚠ THE TWO REDIRECT LAWS (A3): `/{provider}/start` answers **302 to the provider's consent
14
+ screen** — it is a top-level navigation the client reaches by `<a href>`, never JSON. The
15
+ callback 302s BACK to the return path the `state` carried (relative-only, sanitised by
16
+ `oauth_connect.safe_next`), so the user lands where they left — connected or not, whatever
17
+ went wrong rides in the query string; a dead-end error page where the app used to be reads as
18
+ "the product broke", not "the connect failed".
19
+ """
20
+ import os
21
+
22
+ from fastapi import APIRouter, Depends, Request
23
+ from fastapi.responses import RedirectResponse
24
+
25
+ import oauth_connect
26
+ from deps import Session, err, require_session
27
+
28
+ router = APIRouter(prefix="/oauth")
29
+
30
+
31
+ def _redirect_uri(request: Request, provider: str) -> str:
32
+ """The redirect URI this deployment registers at the provider — env-pinned when the
33
+ container sits behind a proxy that rewrites the scheme (the HF Space), else derived from
34
+ the request. MUST match a console-registered URI verbatim, so it is computed in exactly
35
+ one place.
36
+
37
+ ⭐ WAVE 29 (R4): `deploy_web.py` now PUSHES `AIOS_PUBLIC_BASE` on every deploy, defaulted to
38
+ the same URL as `APP_BASE_URL`, so the pinned branch is the one that runs in production and
39
+ the request-derived fallback below is effectively dev-only.
40
+ ⛔ THAT MAKES THIS FUNCTION A CUSTOM-DOMAIN COUPLING, not merely a scheme fix. Whatever host
41
+ this returns is where the provider sends the user BACK, and the session cookie is host-only
42
+ (`aios_session.py:114-117`, no `domain=`) — so a callback base that disagrees with the host
43
+ the user actually browsed plants the session on the wrong hostname and they return logged
44
+ out. Moving the app to a new hostname means moving this value AND re-registering the
45
+ resulting URI in the provider console; one without the other fails closed.
46
+ Runbook: `.claude/wiki/research/loopable-domain-runbook.md`."""
47
+ base = (os.environ.get("AIOS_PUBLIC_BASE") or "").strip().rstrip("/")
48
+ if not base:
49
+ base = f"{request.url.scheme}://{request.url.netloc}"
50
+ return f"{base}/api/v1/oauth/{provider}/callback"
51
+
52
+
53
+ @router.get("/status")
54
+ def oauth_status(session: Session = Depends(require_session)):
55
+ """C5's status shape for the SESSION user, one entry per registry provider:
56
+ `{google: {connected, email, reconnect, configured}}` today. The bit the email trigger's
57
+ `ready` reads through."""
58
+ return oauth_connect.status(session.runtime, session.uname)
59
+
60
+
61
+ def _offered_or_404(provider: str):
62
+ """⛔⛔ W32-T14 / OWNER ITEM 12 / R6 — A PROVIDER THE PRODUCT DOES NOT OFFER HAS NO DOOR.
63
+
64
+ The owner pasted the failure this replaces: clicking Connect on the Google card answered
65
+ `503 oauth_unavailable` as raw JSON. R6's fix is not a nicer error — it is that the flow is
66
+ not offered at all, so the honest status is the one for a URL that does not exist. `404`
67
+ rather than `503`, deliberately: a 503 says *"come back later"* about a door that is not
68
+ coming back until somebody pays for CASA verification (D-45, ~$540–1,800/yr).
69
+
70
+ ⚠ Every door in this router goes through it, START included, because the JSON the owner saw
71
+ came from the start route and a guard on one leg is a guard on one leg.
72
+ """
73
+ if not oauth_connect.offered(provider):
74
+ raise err(404, "unknown_provider", f"{provider!r} is not a connectable provider")
75
+
76
+
77
+ @router.get("/{provider}/start")
78
+ def oauth_start(provider: str, request: Request, next: str = "",
79
+ session: Session = Depends(require_session)):
80
+ """302 to the provider's consent screen (A3 — a navigation, never JSON). `?next=` is the
81
+ RELATIVE path the callback returns the browser to; it rides inside the single-use state,
82
+ sanitised, so the round trip cannot be steered off-origin."""
83
+ _offered_or_404(provider)
84
+ url, problem = oauth_connect.start(provider, session.uname,
85
+ _redirect_uri(request, provider), next_path=next)
86
+ if problem:
87
+ raise err(503 if "not configured" in problem else 404, "oauth_unavailable", problem)
88
+ return RedirectResponse(url, status_code=302)
89
+
90
+
91
+ @router.get("/{provider}/callback")
92
+ def oauth_callback(provider: str, request: Request,
93
+ session: Session = Depends(require_session),
94
+ state: str = "", code: str = "", error: str = ""):
95
+ """The provider's redirect target. Exchanges the code, stores the per-user slot, and sends
96
+ the browser back to the state's return path — connected or not (see module header)."""
97
+ _offered_or_404(provider)
98
+ if error:
99
+ home = "/#/"
100
+ return RedirectResponse(f"{home}?oauthError={error[:80]}", status_code=302)
101
+ email, home, problem = oauth_connect.callback(session.runtime, session.uname, state, code)
102
+ sep = "&" if "?" in home else "?"
103
+ if problem:
104
+ return RedirectResponse(f"{home}{sep}oauthError=connect_failed", status_code=302)
105
+ return RedirectResponse(f"{home}{sep}connected={provider}", status_code=302)
106
+
107
+
108
+ @router.post("/{provider}/disconnect")
109
+ def oauth_disconnect(provider: str, session: Session = Depends(require_session)):
110
+ _offered_or_404(provider)
111
+ if oauth_connect.provider_def(provider) is None:
112
+ raise err(404, "unknown_provider", f"{provider!r} is not a connectable provider")
113
+ oauth_connect.disconnect(session.runtime, session.uname, provider)
114
+ return {"disconnected": provider}
api/routes_odoo_tables.py CHANGED
The diff for this file is too large to render. See raw diff
 
api/routes_query.py CHANGED
@@ -1,713 +1,693 @@
1
- """routes_query.py — WAVE 32 item 7 (ruling R1, contract C5): the Query module's doors.
2
-
3
- POST /api/v1/query/build {question, scope} -> {spec, explain, refused} WRITES NOTHING
4
- POST /api/v1/query/save {question, scope, spec} -> the saved view
5
- GET /api/v1/query -> the AI-built views this caller may open
6
- DELETE /api/v1/query/{qid} -> forget one
7
-
8
- **The AI turns a question about ONE granted database into a view SPEC. It may only build what the
9
- backend already supports** (R1), and the wall on which database it may name is the EXISTING one.
10
-
11
- ⛔ THE FOUNDATION IS NOT `harness/semantic.py` — MEASURED, W32-T50
12
- (`.claude/wiki/waves/wave32/proto/query-spike/README.md`). That layer is bound to 8 hand-authored
13
- YAML topics over Odoo mirror tables: **0** are `ut_`-bound, and a `ut_` key answers
14
- `ModelError: topic 'ut_…' has no store binding` from both `store_columns` and `store_query`. Its
15
- own `save_view` writes `platform/data/store/views.json`, a chart workspace the product's grid
16
- cannot read. What it lends this file is its POSTURE, quoted from its system prompt: *the model
17
- never writes SQL and never invents a field name — it only picks registry keys.*
18
-
19
- The real foundation is the view stack that already ships, and this module is `routes_templates.py`
20
- with a model where the curated registry used to be:
21
-
22
- · the wall `routes_templates._target_or_refuse` (imported, never re-implemented)
23
- · the column contract `core.view_templates.columns_named` / `missing_columns`
24
- · the view skeleton `core.view_templates._view` (one definition of what a view is)
25
- · the validators `aios_grid._clean_display` / `clean_filter_tree`
26
- · the writer `core.table_store.make(ws).save_view(..., shared=False)`
27
-
28
- ⛔⛔ THE VALIDATORS DROP; THEY DO NOT REFUSE — and C5 requires a REFUSAL. `_clean_display` returns
29
- None for an unknown mode, and a view with no display block IS A GRID; `clean_filter_tree` drops a
30
- leaf naming an unknown column, and a view whose only condition was dropped SHOWS EVERY ROW under a
31
- name promising a shortlist (`view_templates`' own scar, and `_seed_wave17`'s before it). So
32
- membership is tested HERE, first and explicitly, and the cleaners run second as a belt. They are
33
- the vocabulary ORACLE, never the refusal.
34
-
35
- ⛔ THE REFUSAL PATH IS A MODEL PROPERTY, so this module declares its OWN provider order rather than
36
- inheriting `analyst.available_providers()`' cheap-first one. Measured over 4 arms x 9 questions
37
- (`proto/query-spike/refusal-arms.json`): `cerebras/gpt-oss-120b` refuses 4/4 unanswerable questions
38
- with a usable sentence; `groq/llama-3.3-70b` refuses **0/4** and instead emits a fully valid spec
39
- answering a different question ("Revenue by Creator" against a database holding no revenue).
40
- Changing refusal from an enum value to its own tool changed cerebras not at all and made groq
41
- worse.
42
- ⚠ AND THE LADDER STILL FALLS THROUGH ON A 429, WHICH IS A COMPROMISE, NOT AN OVERSIGHT — stated
43
- here because the first draft of this header claimed the opposite. A door that dies whenever one
44
- free tier rate-limits is worse than one that answers with a weaker model behind the SAME validator
45
- and the SAME human read-back. What "fail closed" means precisely: when the whole ladder is
46
- unreachable the answer is a sentence and never a view. Which provider answered rides back in the
47
- response so a wrong answer stays diagnosable (`_call_model`'s note).
48
-
49
- ⛔ AND A PROVIDER `400` CAN BE THE MODEL'S REFUSAL. Groq validates the model's tool arguments
50
- against the tool schema server-side and answers `400 tool_use_failed` carrying the model's real
51
- output in `failed_generation` — so a refusal that omits a `required` property arrives as an HTTP
52
- error. `analyst._live_chat` calls `raise_for_status()` and discards that body, which is why this
53
- file has its own small ladder: `required` is `["kind"]` ALONE (a refusal must be a legal argument
54
- object), and a 400 body is READ before it is treated as transport.
55
-
56
- ⚠ WHAT NO VALIDATOR CAN SEE, stated so nobody thinks the gate covers it: a spec whose every column
57
- is real, every op legal, kind in the vocabulary and target granted, which answers a DIFFERENT
58
- QUESTION. That class is invisible to server-side validation. The mechanism against it is C5's own
59
- shape — `build` returns `{spec, explain}` and WRITES NOTHING, `explain` is DERIVED from the
60
- validated spec (never asked of the model, which would let it flatter its own output), and saving is
61
- a second, explicit call. The person is the last validator, by design.
62
- """
63
- import datetime as _dt
64
- import hashlib
65
- import json
66
- import os
67
- import re
68
-
69
- from fastapi import APIRouter, Body, Depends
70
-
71
- from deps import Session, err, require_session
72
-
73
- router = APIRouter(prefix="/api/v1")
74
-
75
- #: The store key holding the REGISTRY of AI-built views, per tenant: {qid: {...}}. The view itself
76
- #: is a real saved view in its target's own workspace — this is only the index, so listing the
77
- #: Query module never has to read every granted table's workspace to find them. Same split
78
- #: `core/alerts.py` draws between definitions and the inbox.
79
- QUERY_KEY = 'query_views'
80
-
81
- MAX_QUESTION = 500
82
- MAX_VISIBLE = 24
83
- MAX_REGISTRY = 200 #: a working list, not an archive (alerts' MAX_NOTIFICATIONS rule)
84
-
85
- #: ⭐ THE KINDS THE AI MAY EMIT. Held here as a NAMED SET, and `verify_query.py` asserts it is a
86
- #: subset of BOTH `aios_grid.DISPLAY_MODES` (the stored vocabulary) and the client's
87
- #: `CREATABLE_MODES` (which means "has a mounted renderer" — `verify_icons::mode_parity` enforces
88
- #: that both ways), AND that the excluded remainder is exactly `QUERY_EXCLUDED`. So a mode that
89
- #: lands in `CREATABLE_MODES` later REDS the gate instead of being silently un-offered — the
90
- #: `swipe`/`form` failure ("whole, correct and unreachable") in reverse.
91
- QUERY_KINDS = ('grid', 'chart', 'calendar', 'kanban', 'timeseries', 'map', 'list')
92
-
93
- #: Creatable modes the AI deliberately may NOT emit, each with the reason it cannot be derived
94
- #: from a question about existing records. A reason per member, because "excluded" with no cause
95
- #: is how a hold outlives its own justification.
96
- QUERY_EXCLUDED = {
97
- 'form': "a form is a door records come IN through and shows none — a question about existing "
98
- "records cannot describe one, and its spec carries emails and a public token",
99
- 'catalog': "a catalog is a published artifact needing page authoring and product codes "
100
- "(MAX_CATALOG_CODES is spent in page order), not a shape of the current rows",
101
- 'swipe': "a swipe deck needs an empty single-select plus a chosen option per direction — two "
102
- "product decisions a question does not contain",
103
- }
104
-
105
- #: Mode-required refs, and the field FAMILY each one demands. ⚠ STRICTER THAN `_clean_display` ON
106
- #: PURPOSE. That function accepts any ref naming a real field and leaves the meaning to the client,
107
- #: which "degrades a wrong-typed ref to that surface's default rather than to an error" — correct
108
- #: for a person clicking a picker who SEES the result, wrong for a generated spec nobody has looked
109
- #: at yet. Measured: a free model stacked a kanban on `category` (a text column) for a contract
110
- #: with ZERO select columns, and every validator passed it.
111
- _MODE_REFS = {
112
- 'kanban': [('stackField', ('select',), True)],
113
- 'calendar': [('dateField', ('date',), True)],
114
- 'timeseries': [('dateField', ('date',), True)],
115
- 'map': [('colorField', ('select',), False), ('sizeField', ('int', 'currency', 'pct'), False)],
116
- }
117
-
118
-
119
- def _now_iso():
120
- """UTC WITH the offset — a naive stamp is unsubtractable in a browser (D-18, alerts' rule)."""
121
- return _dt.datetime.now(_dt.timezone.utc).isoformat()
122
-
123
-
124
- def _grid():
125
- import aios_grid
126
- return aios_grid
127
-
128
-
129
- def _templates():
130
- import core.view_templates as view_templates
131
- return view_templates
132
-
133
-
134
- def _target(session, scope):
135
- """`(fields, source)` for a database this session may open — or a refusal.
136
-
137
- ⛔ THE WALL IS `routes_templates._target_or_refuse`, IMPORTED AND NOT COPIED. It is the same
138
- predicate the nav uses, per topic (`ut_*` -> `user_tables.may_open`; a built-in -> the module
139
- grant `session.require`; anything else a 404 that is not a directory of what exists), and T51's
140
- instruction is explicit that the wall is re-used, never re-implemented beside itself. The
141
- private name is deliberate: an alias would mean editing a file no lane's fence covers.
142
-
143
- It returns field KEYS; this door also needs each field's TYPE (for the prompt and for
144
- `_MODE_REFS`), so the types are read from the same source the wall consulted — never from a
145
- list assembled here.
146
- """
147
- import routes_templates
148
- keys, source = routes_templates._target_or_refuse(session, scope)
149
- key = str(scope or '').strip()
150
- defs = []
151
- if key.startswith('ut_'):
152
- import core.user_tables as user_tables
153
- defn = user_tables.get(key, st=session.runtime) or {}
154
- by_key = {str(f.get('key')): f for f in (defn.get('fields') or []) if f.get('key')}
155
- else:
156
- g = _grid()
157
- raw = g.product_fields() if key == 'product_data' else g.FIELDS
158
- by_key = {str(f.get('key')): f for f in raw if f.get('key')}
159
- for k in keys:
160
- f = by_key.get(k) or {}
161
- defs.append({'key': k, 'label': str(f.get('label') or k)[:80],
162
- 'type': str(f.get('type') or 'text')})
163
- return defs, source
164
-
165
-
166
- # ── the model ───────────────────────────────────────────────────────────────────────────────
167
-
168
- #: This module's OWN provider order — cerebras first. See the header: the refusal is a model
169
- #: property, and the ladder's cheap-first order is exactly wrong for a contract that includes one.
170
- #: Override with `AIOS_QUERY_PROVIDER`. The credentials themselves stay in ONE place
171
- #: (`analyst.PROVIDERS`), so a rotated key or a new model reaches this file too.
172
- QUERY_PROVIDER_ORDER = ('cerebras', 'groq', 'openrouter')
173
-
174
-
175
- def _providers():
176
- import harness.analyst as analyst
177
- pin = os.environ.get("AIOS_QUERY_PROVIDER")
178
- by_name = {p["name"]: p for p in analyst.PROVIDERS}
179
- order = [pin] if pin else list(QUERY_PROVIDER_ORDER)
180
- return [by_name[n] for n in order
181
- if n in by_name and os.environ.get(by_name[n]["env"])]
182
-
183
-
184
- def _spec_schema(field_keys):
185
- """The tool schema the model fills. ⛔ `required` IS `["kind"]` ALONE — see the header: a
186
- refusal carries no name, and groq rejects a schema-violating refusal with a 400 that reads
187
- exactly like a transport failure."""
188
- ops = sorted(_grid().FILTER_OPS)
189
- col = {"type": "string", "enum": sorted(field_keys)}
190
- return {
191
- "type": "object",
192
- "properties": {
193
- "kind": {"type": "string", "enum": list(QUERY_KINDS) + ["refused"],
194
- "description": "refused = this database cannot answer the question"},
195
- "name": {"type": "string", "description": "a short title for the view"},
196
- "refusal": {"type": "string",
197
- "description": "when kind=refused: ONE plain sentence naming what is "
198
- "missing"},
199
- "visible": {"type": "array", "items": col},
200
- "filters": {"type": "array", "items": {
201
- "type": "object",
202
- "properties": {"colId": col, "op": {"type": "string", "enum": ops},
203
- "value": {"type": "string"}},
204
- "required": ["colId", "op"]}},
205
- "filterConj": {"type": "string", "enum": ["and", "or"]},
206
- "sorts": {"type": "array", "items": {
207
- "type": "object",
208
- "properties": {"colId": col,
209
- "dir": {"type": "string", "enum": ["asc", "desc"]}},
210
- "required": ["colId", "dir"]}},
211
- "groupBy": col,
212
- "stackField": col,
213
- "dateField": col,
214
- "colorField": col,
215
- "sizeField": col,
216
- },
217
- "required": ["kind"],
218
- }
219
-
220
-
221
- def _system_prompt(fields):
222
- """The standing instructions. The vocabulary is DERIVED (the kinds set, `FILTER_OPS`, the
223
- target's own columns) — the model is never told about a key this server cannot honour."""
224
- cols = json.dumps([{"key": f["key"], "label": f["label"], "type": f["type"]}
225
- for f in fields], separators=(",", ":"))
226
- kinds = ", ".join(QUERY_KINDS)
227
- ops = ", ".join(sorted(_grid().FILTER_OPS))
228
- selects = [f["key"] for f in fields if f["type"] in ('select', 'multiselect')] or ["(none)"]
229
- dates = [f["key"] for f in fields if f["type"] == 'date'] or ["(none)"]
230
- return f"""You turn a question about ONE database into a VIEW SPEC. You never write SQL and you
231
- never invent a column or a kind of view.
232
-
233
- THE DATABASE ({len(fields)} columns) — you may name ONLY these column keys:
234
- {cols}
235
-
236
- VIEW KINDS you may name, and nothing else: {kinds}
237
- FILTER OPS you may name, and nothing else: {ops}
238
- The only select-type columns (a kanban MUST stack on one of these): {", ".join(selects)}
239
- The only date columns (a calendar or timeseries MUST use one of these): {", ".join(dates)}
240
-
241
- RULES:
242
- 1. Emit ONE call to build_view. Every colId in visible/filters/sorts/groupBy and every field ref
243
- MUST be one of the keys above.
244
- 2. Pick the kind by what the question asks for: a list of records -> grid (or list); totals or
245
- counts by a category -> chart with groupBy; cards by a stage -> kanban (stackField REQUIRED,
246
- and it must be a select column); something over time -> calendar or timeseries (dateField
247
- REQUIRED, and it must be a date column); geography -> map.
248
- 3. `visible` is the columns the question is about, most important first — not every column.
249
- 4. Set kind="refused" and put ONE plain sentence in `refusal` when the question needs a column
250
- this database does not have, a number it does not store, or a picture that is not in the kind
251
- list. REFUSING IS A CORRECT ANSWER. A view that answers a DIFFERENT question than the one
252
- asked is the worst possible answer — worse than saying you cannot build it. When in doubt,
253
- refuse.
254
- 5. Every `value` is a string."""
255
-
256
-
257
- def _call_model(question, fields, chat=None):
258
- """`(spec, error_sentence, provider)` — one bounded call down THIS module's ladder.
259
-
260
- `chat` is injectable so `verify_query.py` proves this door end to end with no key and no
261
- tokens (`analyst.ask`'s own posture, and the reason its loop is testable offline).
262
-
263
- ⚠ THE LADDER IS A DELIBERATE COMPROMISE, AND THE PROVIDER RIDES BACK BECAUSE OF IT. Only the
264
- FIRST provider is measured to refuse honestly (README section 4b); the ones below it answer
265
- unanswerable questions with plausible specs. Falling through on a 429 is still the right call —
266
- a Query module that dies whenever one free tier rate-limits is worse than one that answers with
267
- a weaker model behind the same validator and the same human read-back — but *which* model
268
- answered must be knowable, or a wrong answer is undiagnosable after the fact. Failing closed is
269
- what happens when the whole ladder is unreachable, not when the best rung is.
270
- """
271
- tools = [{"type": "function", "function": {
272
- "name": "build_view",
273
- "description": "Emit the view spec, or refuse.",
274
- "parameters": _spec_schema([f["key"] for f in fields])}}]
275
- messages = [{"role": "system", "content": _system_prompt(fields)},
276
- {"role": "user", "content": question}]
277
- if chat is not None:
278
- return chat(messages, tools), None, "injected"
279
-
280
- provs = _providers()
281
- if not provs:
282
- # No key at all: say so. An AI feature that silently does nothing is indistinguishable
283
- # from one that was never built ([[flag-shipped-without-its-writer]]).
284
- return None, ("the assistant is not configured on this deployment, so a view cannot be "
285
- "built from a question yet"), None
286
- import requests
287
- last = None
288
- for p in provs:
289
- try:
290
- r = requests.post(p["url"], timeout=60,
291
- headers={"Authorization": f"Bearer {os.environ[p['env']]}"},
292
- json={"model": p["model"], "messages": messages, "tools": tools,
293
- "tool_choice": "required", "temperature": 0.1,
294
- "max_tokens": 1200})
295
- except Exception as e: # network blip — try the next provider
296
- last = f"{p['name']}: {type(e).__name__}"
297
- continue
298
- if r.status_code == 400:
299
- # ⛔ THE MODEL'S ANSWER CAN BE IN HERE. Groq validates tool arguments against the
300
- # schema and returns the generation it rejected. Reading it turns a "transport
301
- # failure" back into the refusal it actually was.
302
- spec = _spec_from_400(r.text)
303
- if spec is not None:
304
- return spec, None, p["name"]
305
- last = f"{p['name']}: 400"
306
- continue
307
- if r.status_code != 200:
308
- last = f"{p['name']}: HTTP {r.status_code}"
309
- continue # 429 included: the NEXT provider, not a sleep
310
- try:
311
- msg = r.json()["choices"][0]["message"]
312
- calls = msg.get("tool_calls") or []
313
- if not calls:
314
- last = f"{p['name']}: no tool call"
315
- continue
316
- return json.loads(calls[0]["function"].get("arguments") or "{}"), None, p["name"]
317
- except Exception as e:
318
- last = f"{p['name']}: unreadable answer ({type(e).__name__})"
319
- continue
320
- # ⛔ FAIL CLOSED. The first provider in the order is the one measured to refuse honestly; if it
321
- # is rate-limited and the rest are too, the answer is a sentence, never a fabricated view.
322
- return None, ("the assistant could not be reached just now (" + (last or "no provider") +
323
- ") — try the question again in a moment"), None
324
-
325
-
326
- _FAILED_GEN = re.compile(r'<function=build_view>(\{.*?\})\s*</function>', re.S)
327
-
328
-
329
- def _spec_from_400(body):
330
- """The spec out of a `tool_use_failed` body's `failed_generation`, or None."""
331
- try:
332
- err_obj = (json.loads(body) or {}).get("error") or {}
333
- except Exception:
334
- return None
335
- gen = str(err_obj.get("failed_generation") or "")
336
- if not gen:
337
- return None
338
- m = _FAILED_GEN.search(gen)
339
- raw = m.group(1) if m else gen.strip()
340
- try:
341
- spec = json.loads(raw)
342
- except Exception:
343
- return None
344
- return spec if isinstance(spec, dict) else None
345
-
346
-
347
- # ── validation: the REFUSAL this contract promises ───────────────────────────────────────────
348
-
349
- def _validate(spec, fields):
350
- """`(clean, refusal, code)` — exactly one of `clean` / `refusal` is set.
351
-
352
- Membership FIRST and explicitly (the header's whole subject), then the product's own cleaners
353
- as a belt. A refusal is a SENTENCE, because C5's user-facing promise is a plain sentence and
354
- never a broken view.
355
- """
356
- g = _grid()
357
- if not isinstance(spec, dict):
358
- return None, "the assistant did not answer with a view", "no_spec"
359
- by_key = {f['key']: f['type'] for f in fields}
360
- keys = set(by_key)
361
- kind = spec.get('kind')
362
-
363
- if kind == 'refused':
364
- return None, (str(spec.get('refusal') or '').strip()
365
- or "this database cannot answer that question"), "model_refused"
366
- if kind in QUERY_EXCLUDED:
367
- return None, (f"a {kind} view cannot be built from a question — "
368
- + QUERY_EXCLUDED[kind]), "unsupported_kind"
369
- if kind not in QUERY_KINDS:
370
- # ⛔ NOT `_clean_display`'s job: it would DROP this and hand back a grid under a name
371
- # promising something else.
372
- return None, (f"{kind!r} is not a kind of view this product can build"
373
- if kind else "the assistant did not name a kind of view"), "unsupported_kind"
374
-
375
- named = set(spec.get('visible') or ())
376
- for s in spec.get('sorts') or ():
377
- if isinstance(s, dict) and s.get('colId'):
378
- named.add(str(s['colId']))
379
- for f in spec.get('filters') or ():
380
- if isinstance(f, dict) and f.get('colId'):
381
- named.add(str(f['colId']))
382
- if spec.get('groupBy'):
383
- named.add(str(spec['groupBy']))
384
- for ref in ('stackField', 'dateField', 'colorField', 'sizeField'):
385
- if spec.get(ref):
386
- named.add(str(spec[ref]))
387
- missing = sorted(named - keys)
388
- if missing:
389
- # NAMED, not counted (`routes_templates`' rule): the column keys are what tell the reader
390
- # this is the wrong database for this question.
391
- return None, ("this database does not have the columns that answer needs: "
392
- + ", ".join(missing)), "unknown_columns"
393
-
394
- visible = [k for k in (spec.get('visible') or ()) if k in keys][:MAX_VISIBLE]
395
- if not visible:
396
- # An empty projection is not a view. `_default_view_config` would fill one in, which is
397
- # how a question about two columns becomes a 285-column wall.
398
- return None, ("that question did not name anything to show — try naming the columns you "
399
- "want to see"), "no_columns"
400
-
401
- # ⛔ D-137, GUARDED WHERE IT REACHES THIS FEATURE. "The chart cannot key a SET-LIKE or CHECKBOX
402
- # column the way the grid does" — a set row lands in several buckets, so a sum means something
403
- # else, and a checkbox reads "(blank)". The defect lives in the CLIENT's `chartData` bucket
404
- # keying, which is in no lane's fence this wave (booked, with that reason). What IS in this
405
- # lane's power is refusing to GENERATE the broken case: the AI must only build what the backend
406
- # and the renderer already support, which is R1's own sentence. Delete this guard when D-137
407
- # closes — the refusal names the debt so the reader knows it is a limit, not a rule.
408
- if kind == 'chart' and by_key.get(spec.get('groupBy')) in ('multiselect', 'checkbox'):
409
- return None, (f"a chart cannot group by {spec.get('groupBy')!r} yet — a column that holds "
410
- f"several values at once, or a tick box, does not bucket the way a chart "
411
- f"needs. Try a table grouped by it instead"), "chart_key_unsupported"
412
-
413
- for ref, families, required in _MODE_REFS.get(kind, ()):
414
- val = spec.get(ref)
415
- if val and by_key.get(val) not in families:
416
- return None, (f"a {kind} view needs {ref[:-5]} to be a "
417
- f"{' or '.join(families)} column, and {val!r} is a "
418
- f"{by_key.get(val)} column"), "wrong_ref_type"
419
- if required and not val:
420
- return None, (f"a {kind} view needs a {' or '.join(families)} column and this "
421
- f"database has none that fits"), "missing_ref"
422
-
423
- raw_filters = [f for f in (spec.get('filters') or ()) if isinstance(f, dict)]
424
- filters = g.clean_filter_tree(raw_filters, keys)
425
- if len(filters) != len(raw_filters):
426
- # The belt caught something membership did not — an unknown op. Refuse rather than save a
427
- # view whose condition was silently dropped (the widening this file exists to prevent).
428
- return None, ("part of that filter is not something this product can express — try "
429
- "asking it more simply"), "filter_dropped"
430
-
431
- display = {'mode': kind}
432
- for ref, _families, _req in _MODE_REFS.get(kind, ()):
433
- if spec.get(ref):
434
- display[ref] = spec[ref]
435
- cleaned_display = g._clean_display(display, keys) if kind != 'grid' else None
436
- if kind != 'grid' and not cleaned_display:
437
- return None, (f"this product could not build a {kind} view from that question"
438
- ), "display_dropped"
439
-
440
- clean = {
441
- 'kind': kind,
442
- 'name': ' '.join(str(spec.get('name') or 'Query').split())[:60] or 'Query',
443
- 'visible': visible,
444
- 'filters': filters,
445
- 'filterConj': 'or' if spec.get('filterConj') == 'or' else 'and',
446
- 'sorts': [{'colId': s['colId'], 'dir': 'desc' if s.get('dir') == 'desc' else 'asc'}
447
- for s in (spec.get('sorts') or ()) if isinstance(s, dict)
448
- and s.get('colId') in keys][:3],
449
- 'groupBy': spec['groupBy'] if spec.get('groupBy') in keys else None,
450
- 'display': cleaned_display,
451
- }
452
- return clean, None, None
453
-
454
-
455
- def _explain(clean, fields):
456
- """A plain-language read-back of what the view WILL do — DERIVED from the validated spec.
457
-
458
- ⛔ NEVER ASKED OF THE MODEL. A model describing its own output describes what it MEANT, which
459
- is precisely the failure this sentence exists to expose: a spec whose every column is real and
460
- which answers a different question is invisible to validation, so the only thing that catches
461
- it is a person reading an honest description of the spec that was actually accepted.
462
- """
463
- label = {f['key']: f['label'] for f in fields}
464
- ops = {'eq': 'is', 'neq': 'is not', 'gt': 'is over', 'gte': 'is at least',
465
- 'lt': 'is under', 'lte': 'is at most', 'contains': 'contains',
466
- 'doesNotContain': 'does not contain', 'isEmpty': 'is empty',
467
- 'isNotEmpty': 'is not empty', 'between': 'is between', 'within': 'is within',
468
- 'topN': 'is in the top', 'bottomN': 'is in the bottom'}
469
- kinds = {'grid': 'a table', 'list': 'a list', 'chart': 'a chart', 'kanban': 'a board',
470
- 'calendar': 'a calendar', 'timeseries': 'a time series', 'map': 'a map'}
471
- parts = [f"{kinds.get(clean['kind'], clean['kind'])} of "
472
- + ", ".join(label.get(k, k) for k in clean['visible'][:6])
473
- + (f" and {len(clean['visible']) - 6} more columns"
474
- if len(clean['visible']) > 6 else "")]
475
- if clean['filters']:
476
- joiner = " or " if clean['filterConj'] == 'or' else " and "
477
- parts.append("showing only records where " + joiner.join(
478
- f"{label.get(f['colId'], f['colId'])} {ops.get(f['op'], f['op'])}"
479
- + (f" {f['value']}" if f.get('value') else "")
480
- for f in clean['filters'] if isinstance(f, dict) and f.get('colId')))
481
- else:
482
- parts.append("showing every record")
483
- if clean['groupBy']:
484
- parts.append(f"grouped by {label.get(clean['groupBy'], clean['groupBy'])}")
485
- if clean['sorts']:
486
- s = clean['sorts'][0]
487
- parts.append(f"sorted by {label.get(s['colId'], s['colId'])}"
488
- + (" (highest first)" if s['dir'] == 'desc' else " (lowest first)"))
489
- for ref, word in (('stackField', 'in columns by'), ('dateField', 'placed in time by'),
490
- ('colorField', 'coloured by'), ('sizeField', 'sized by')):
491
- val = (clean.get('display') or {}).get(ref)
492
- if val:
493
- parts.append(f"{word} {label.get(val, val)}")
494
- return ", ".join(parts) + "."
495
-
496
-
497
- # ── the registry ────────────────────────────────────────────────────────────────────────────
498
-
499
- def _qid(scope, question):
500
- """PINNED per (database, question), the `view_templates` discipline: asking the same thing
501
- twice UPDATES one view instead of minting "Query 2" — `save_view` de-duplicates NAMES by
502
- appending a number, so an unpinned id fills the store with numbered near-duplicates."""
503
- h = hashlib.sha1(f"{scope}|{' '.join(str(question).split()).lower()}".encode("utf-8"))
504
- return "q_" + h.hexdigest()[:12]
505
-
506
-
507
- def _registry(session):
508
- try:
509
- recs = session.runtime.get(QUERY_KEY) or {}
510
- except Exception:
511
- return {}
512
- return recs if isinstance(recs, dict) else {}
513
-
514
-
515
- def _mine(session, rec):
516
- return session.admin or str(rec.get('createdBy') or '') == session.uname
517
-
518
-
519
- #: The BUILT-IN topics `routes_templates._target_or_refuse` accepts beside any `ut_*` key. ⛔ THE
520
- #: CLIENT'S PICKER MUST MIRROR THIS, and `GET /query` publishes it (`builtins`) rather than leaving
521
- #: the page to re-derive it: the nav hands `QueryPage` every granted entry, which is a WIDER set
522
- #: than this door accepts, so a picker filtered on "not a surface" alone would offer a database the
523
- #: build door 404s. `view_templates`' own rule — "a picker that offered a template the apply door
524
- #: would then refuse is a control that lies" — and it is the same predicate twice, published once.
525
- BUILTIN_SCOPES = ('customer_data', 'product_data')
526
-
527
-
528
- @router.get("/query")
529
- def list_queries(session: Session = Depends(require_session)):
530
- """The AI-built views this caller may open: their own, still-granted, still-existing.
531
-
532
- ⚠ PRUNED ON READ against the CURRENT wall, never merely on write — a grant can be withdrawn
533
- and a database deleted after a view was saved, and the Query module is chrome: it may render
534
- nothing the server would not grant today (`nav.ts`'s chrome law).
535
- """
536
- import routes_templates
537
- out = []
538
- for qid, rec in sorted(_registry(session).items(),
539
- key=lambda kv: str((kv[1] or {}).get('createdAt') or ''),
540
- reverse=True):
541
- if not isinstance(rec, dict) or not _mine(session, rec):
542
- continue
543
- try:
544
- routes_templates._target_or_refuse(session, rec.get('scope'))
545
- except Exception:
546
- continue # no longer granted, or gone
547
- out.append({"id": qid, "scope": rec.get('scope'), "viewId": rec.get('viewId'),
548
- "name": rec.get('name'), "kind": rec.get('kind'),
549
- "question": rec.get('question'), "explain": rec.get('explain'),
550
- "createdAt": rec.get('createdAt')})
551
- return {"views": out, "kinds": list(QUERY_KINDS), "builtins": list(BUILTIN_SCOPES)}
552
-
553
-
554
- @router.post("/query/build")
555
- def build_query(body: dict = Body(default=None),
556
- session: Session = Depends(require_session)):
557
- """A question + a granted database -> `{spec, explain, refused}`. **WRITES NOTHING.**
558
-
559
- A refusal is a 200 carrying a sentence, not a 5xx: "this database cannot answer that" is a
560
- successful answer to the person asking, and an error envelope would paint the error page R1
561
- forbids. The 4xx cases are the ones that ARE the caller's fault — an unnamed database, an
562
- over-long question, a database this session may not open.
563
-
564
- ⛔ THE MODEL SEAM IS **NOT** ON THIS SIGNATURE, AND THAT IS A CORRECTION. The first draft took
565
- `chat=None` as a parameter of the ROUTE — and FastAPI reads an un-annotated defaulted argument
566
- as a **QUERY PARAMETER**, so `chat` appeared in `openapi()["paths"]` (measured) and
567
- `POST /query/build?chat=x` would have reached `_call_model(..., chat="x")` and 500'd on
568
- `"x"(messages, tools)`. The gate never saw it because the gate calls the handler directly —
569
- which is exactly the shape of a test seam that becomes an input. The seam now lives on the
570
- module-level `_build` below: the gate calls that, and the wire cannot.
571
- """
572
- body = body if isinstance(body, dict) else {}
573
- return _build(body.get("question"), body.get("scope"), session)
574
-
575
-
576
- def _build(question, scope, session, chat=None):
577
- """`build_query`'s body, with the model injectable — see that route's note on why the seam is
578
- here and not on the signature FastAPI reads."""
579
- question = ' '.join(str(question or "").split())
580
- if not question:
581
- raise err(400, "bad_request", "no question was asked")
582
- if len(question) > MAX_QUESTION:
583
- raise err(400, "question_too_long",
584
- f"a question must be under {MAX_QUESTION} characters")
585
- fields, _source = _target(session, scope)
586
- if not fields:
587
- raise err(400, "no_columns", "that database has no columns to build a view from")
588
-
589
- spec, transport, provider = _call_model(question, fields, chat=chat)
590
- if transport:
591
- return {"spec": None, "explain": None, "refused": transport, "provider": provider}
592
- clean, refusal, code = _validate(spec, fields)
593
- if refusal:
594
- return {"spec": None, "explain": None, "refused": refusal, "reason": code,
595
- "provider": provider}
596
- return {"spec": clean, "explain": _explain(clean, fields),
597
- "refused": None, "scope": scope,
598
- "id": _qid(scope, question), "provider": provider,
599
- "fallback": _is_fallback(provider)}
600
-
601
-
602
- def _is_fallback(provider):
603
- """True when the answer came from a rung BELOW the top of this module's ladder.
604
-
605
- ⛔ THIS IS THE HALF THAT WAS MISSING, AND IT IS A READER, NOT A WRITER. `_call_model` has
606
- always returned WHICH provider answered, and its own docstring says exactly why that matters:
607
- *"Only the FIRST provider is measured to refuse honestly … the ones below it answer
608
- unanswerable questions with plausible specs."* So on a 429 the ladder falls through — correctly,
609
- a Query module that dies when one free tier rate-limits is worse — and the person then reads a
610
- confident, well-formed view spec built by a model that does not know how to say "this database
611
- cannot answer that". **Nothing on the wire told them.** The value existed and had no consumer.
612
- ⚠ `injected` (the gate's stubbed chat) is NOT a fallback: it is the test seam, and reporting it
613
- as a degraded answer would make every gate assertion about this flag meaningless.
614
- """
615
- if not provider or provider == "injected":
616
- return False
617
- order = [p["name"] for p in _providers()]
618
- return bool(order) and provider != order[0]
619
-
620
-
621
- @router.post("/query/save")
622
- def save_query(body: dict = Body(default=None), session: Session = Depends(require_session)):
623
- """Save an accepted spec as the CALLING USER's own view, and index it under Query.
624
-
625
- ⛔ THE SPEC IS RE-VALIDATED HERE, against the target's contract as it is NOW. The build door
626
- wrote nothing, so this is the first write and it is the only place authorisation and validity
627
- have to hold together — and a client is not a wall. An unsupported kind or an unknown column
628
- is a **400 with a named code** on this door (the caller sent it), where the same fact is a
629
- plain sentence on `build` (the model produced it).
630
- """
631
- body = body if isinstance(body, dict) else {}
632
- scope = str(body.get("scope") or '').strip()
633
- question = ' '.join(str(body.get("question") or "").split())[:MAX_QUESTION]
634
- fields, _source = _target(session, scope)
635
- clean, refusal, code = _validate(body.get("spec"), fields)
636
- if refusal:
637
- raise err(400, code or "bad_spec", refusal)
638
- if not session.runtime.available():
639
- raise err(503, "store_unavailable", "the tenant store is unavailable — nothing was saved")
640
-
641
- vt = _templates()
642
- ws_key = vt.workspace_key(scope)
643
- if not ws_key:
644
- raise err(404, "unknown_table", "that database does not exist")
645
- qid = _qid(scope, question) if question else _qid(scope, clean['name'])
646
-
647
- # ⛔ THE SKELETON IS `view_templates._view`, NOT A SECOND ONE. "A view assembled from a
648
- # DIFFERENT skeleton is a second definition of what a view is — and the two drift on the day
649
- # one of them gains a key." It also puts the mode at `config.display.mode`, which is the key
650
- # the grid actually reads (`config.displayMode` was the first draft, and nothing reads it).
651
- payload = vt._view(qid, clean['name'], f"Built by the assistant from: {question}"[:300],
652
- clean['visible'], mode=clean['kind'],
653
- sorts=clean['sorts'], filters=clean['filters'],
654
- group_by=clean['groupBy'], conj=clean['filterConj'])
655
- if clean.get('display'):
656
- payload['config']['display'] = dict(clean['display'])
657
- payload['createdBy'] = session.uname
658
-
659
- # ⛔ THE CAP REFUSES; IT DOES NOT EVICT — and the first draft evicted. Dropping the oldest entry
660
- # would leave its saved VIEW behind in the table's own workspace: "stranded in the target's view
661
- # list with no way back to the question that made it", which is the thing `delete_query` below
662
- # exists to prevent. And an eviction nobody was told about is W30/R6's second sentence broken —
663
- # a limit that cannot be removed must be REPORTED with its cause. So the 201st is a refusal
664
- # naming the number and the way out, checked BEFORE the view is written.
665
- existing = _registry(session)
666
- if qid not in existing and len(existing) >= MAX_REGISTRY:
667
- raise err(400, "query_limit",
668
- f"you already have {len(existing)} saved questions, which is the limit "
669
- f"({MAX_REGISTRY}). Delete one you no longer need and save this again.")
670
-
671
- import core.table_store as table_store
672
- ops = table_store.make(ws_key, st=session.runtime)
673
- saved = ops.save_view(session.uname, payload, shared=False)
674
-
675
- entry = {"scope": scope, "viewId": qid, "name": saved.get('name') or clean['name'],
676
- "kind": clean['kind'], "question": question,
677
- "explain": _explain(clean, fields), "createdBy": session.uname,
678
- "createdAt": _now_iso()}
679
-
680
- def _up(data):
681
- data = data if isinstance(data, dict) else {}
682
- data[qid] = entry
683
- return data
684
-
685
- session.runtime.update(QUERY_KEY, _up, flush='async')
686
- return {"id": qid, **entry}
687
-
688
-
689
- @router.delete("/query/{qid}")
690
- def delete_query(qid: str, session: Session = Depends(require_session)):
691
- """Forget an AI-built view — the index entry AND the view itself.
692
-
693
- Leaving the view behind would strand it in the target's view list with no way back to the
694
- question that made it ([[a-record-can-outlive-its-subject]]).
695
- """
696
- rec = _registry(session).get(str(qid))
697
- if not isinstance(rec, dict) or not _mine(session, rec):
698
- raise err(404, "unknown_query", "that view does not exist")
699
- _target(session, rec.get('scope')) # the wall, again, on the write path
700
- vt = _templates()
701
- ws_key = vt.workspace_key(rec.get('scope'))
702
- if ws_key and session.runtime.available():
703
- import core.table_store as table_store
704
- table_store.make(ws_key, st=session.runtime).delete_view(
705
- session.uname, rec.get('viewId') or str(qid))
706
-
707
- def _up(data):
708
- data = data if isinstance(data, dict) else {}
709
- data.pop(str(qid), None)
710
- return data
711
-
712
- session.runtime.update(QUERY_KEY, _up, flush='async')
713
- return {"deleted": str(qid)}
 
1
+ """routes_query.py — WAVE 32 item 7 (ruling R1, contract C5): the Query module's doors.
2
+
3
+ POST /api/v1/query/build {question, scope} -> {spec, explain, refused} WRITES NOTHING
4
+ POST /api/v1/query/save {question, scope, spec} -> the saved view
5
+ GET /api/v1/query -> the AI-built views this caller may open
6
+ DELETE /api/v1/query/{qid} -> forget one
7
+
8
+ **The AI turns a question about ONE granted database into a view SPEC. It may only build what the
9
+ backend already supports** (R1), and the wall on which database it may name is the EXISTING one.
10
+
11
+ ⛔ THE FOUNDATION IS NOT `harness/semantic.py` — MEASURED, W32-T50
12
+ (`.claude/wiki/waves/wave32/proto/query-spike/README.md`). That layer is bound to 8 hand-authored
13
+ YAML topics over Odoo mirror tables: **0** are `ut_`-bound, and a `ut_` key answers
14
+ `ModelError: topic 'ut_…' has no store binding` from both `store_columns` and `store_query`. Its
15
+ own `save_view` writes `platform/data/store/views.json`, a chart workspace the product's grid
16
+ cannot read. What it lends this file is its POSTURE, quoted from its system prompt: *the model
17
+ never writes SQL and never invents a field name — it only picks registry keys.*
18
+
19
+ The real foundation is the view stack that already ships, and this module is `routes_templates.py`
20
+ with a model where the curated registry used to be:
21
+
22
+ · the wall `routes_templates._target_or_refuse` (imported, never re-implemented)
23
+ · the column contract `core.view_templates.columns_named` / `missing_columns`
24
+ · the view skeleton `core.view_templates._view` (one definition of what a view is)
25
+ · the validators `aios_grid._clean_display` / `clean_filter_tree`
26
+ · the writer `core.table_store.make(ws).save_view(..., shared=False)`
27
+
28
+ ⛔⛔ THE VALIDATORS DROP; THEY DO NOT REFUSE — and C5 requires a REFUSAL. `_clean_display` returns
29
+ None for an unknown mode, and a view with no display block IS A GRID; `clean_filter_tree` drops a
30
+ leaf naming an unknown column, and a view whose only condition was dropped SHOWS EVERY ROW under a
31
+ name promising a shortlist (`view_templates`' own scar, and `_seed_wave17`'s before it). So
32
+ membership is tested HERE, first and explicitly, and the cleaners run second as a belt. They are
33
+ the vocabulary ORACLE, never the refusal.
34
+
35
+ ⛔ THE REFUSAL PATH IS A MODEL PROPERTY, so this module declares its OWN provider order rather than
36
+ inheriting `analyst.available_providers()`' cheap-first one. Measured over 4 arms x 9 questions
37
+ (`proto/query-spike/refusal-arms.json`): `cerebras/gpt-oss-120b` refuses 4/4 unanswerable questions
38
+ with a usable sentence; `groq/llama-3.3-70b` refuses **0/4** and instead emits a fully valid spec
39
+ answering a different question ("Revenue by Creator" against a database holding no revenue).
40
+ Changing refusal from an enum value to its own tool changed cerebras not at all and made groq
41
+ worse.
42
+ ⚠ AND THE LADDER STILL FALLS THROUGH ON A 429, WHICH IS A COMPROMISE, NOT AN OVERSIGHT — stated
43
+ here because the first draft of this header claimed the opposite. A door that dies whenever one
44
+ free tier rate-limits is worse than one that answers with a weaker model behind the SAME validator
45
+ and the SAME human read-back. What "fail closed" means precisely: when the whole ladder is
46
+ unreachable the answer is a sentence and never a view. Which provider answered rides back in the
47
+ response so a wrong answer stays diagnosable (`_call_model`'s note).
48
+
49
+ ⛔ AND A PROVIDER `400` CAN BE THE MODEL'S REFUSAL. Groq validates the model's tool arguments
50
+ against the tool schema server-side and answers `400 tool_use_failed` carrying the model's real
51
+ output in `failed_generation` — so a refusal that omits a `required` property arrives as an HTTP
52
+ error. `analyst._live_chat` calls `raise_for_status()` and discards that body, which is why this
53
+ file has its own small ladder: `required` is `["kind"]` ALONE (a refusal must be a legal argument
54
+ object), and a 400 body is READ before it is treated as transport.
55
+
56
+ ⚠ WHAT NO VALIDATOR CAN SEE, stated so nobody thinks the gate covers it: a spec whose every column
57
+ is real, every op legal, kind in the vocabulary and target granted, which answers a DIFFERENT
58
+ QUESTION. That class is invisible to server-side validation. The mechanism against it is C5's own
59
+ shape — `build` returns `{spec, explain}` and WRITES NOTHING, `explain` is DERIVED from the
60
+ validated spec (never asked of the model, which would let it flatter its own output), and saving is
61
+ a second, explicit call. The person is the last validator, by design.
62
+ """
63
+ import datetime as _dt
64
+ import hashlib
65
+ import json
66
+ import os
67
+ import re
68
+
69
+ from fastapi import APIRouter, Body, Depends
70
+
71
+ from deps import Session, err, require_session
72
+
73
+ router = APIRouter(prefix="/api/v1")
74
+
75
+ #: The store key holding the REGISTRY of AI-built views, per tenant: {qid: {...}}. The view itself
76
+ #: is a real saved view in its target's own workspace — this is only the index, so listing the
77
+ #: Query module never has to read every granted table's workspace to find them. Same split
78
+ #: `core/alerts.py` draws between definitions and the inbox.
79
+ QUERY_KEY = 'query_views'
80
+
81
+ MAX_QUESTION = 500
82
+ MAX_VISIBLE = 24
83
+ MAX_REGISTRY = 200 #: a working list, not an archive (alerts' MAX_NOTIFICATIONS rule)
84
+
85
+ #: ⭐ THE KINDS THE AI MAY EMIT. Held here as a NAMED SET, and `verify_query.py` asserts it is a
86
+ #: subset of BOTH `aios_grid.DISPLAY_MODES` (the stored vocabulary) and the client's
87
+ #: `CREATABLE_MODES` (which means "has a mounted renderer" — `verify_icons::mode_parity` enforces
88
+ #: that both ways), AND that the excluded remainder is exactly `QUERY_EXCLUDED`. So a mode that
89
+ #: lands in `CREATABLE_MODES` later REDS the gate instead of being silently un-offered — the
90
+ #: `swipe`/`form` failure ("whole, correct and unreachable") in reverse.
91
+ QUERY_KINDS = ('grid', 'chart', 'calendar', 'kanban', 'timeseries', 'map', 'list')
92
+
93
+ #: Creatable modes the AI deliberately may NOT emit, each with the reason it cannot be derived
94
+ #: from a question about existing records. A reason per member, because "excluded" with no cause
95
+ #: is how a hold outlives its own justification.
96
+ QUERY_EXCLUDED = {
97
+ 'form': "a form is a door records come IN through and shows none — a question about existing "
98
+ "records cannot describe one, and its spec carries emails and a public token",
99
+ 'catalog': "a catalog is a published artifact needing page authoring and product codes "
100
+ "(MAX_CATALOG_CODES is spent in page order), not a shape of the current rows",
101
+ 'swipe': "a swipe deck needs an empty single-select plus a chosen option per direction — two "
102
+ "product decisions a question does not contain",
103
+ }
104
+
105
+ #: Mode-required refs, and the field FAMILY each one demands. ⚠ STRICTER THAN `_clean_display` ON
106
+ #: PURPOSE. That function accepts any ref naming a real field and leaves the meaning to the client,
107
+ #: which "degrades a wrong-typed ref to that surface's default rather than to an error" — correct
108
+ #: for a person clicking a picker who SEES the result, wrong for a generated spec nobody has looked
109
+ #: at yet. Measured: a free model stacked a kanban on `category` (a text column) for a contract
110
+ #: with ZERO select columns, and every validator passed it.
111
+ _MODE_REFS = {
112
+ 'kanban': [('stackField', ('select',), True)],
113
+ 'calendar': [('dateField', ('date',), True)],
114
+ 'timeseries': [('dateField', ('date',), True)],
115
+ 'map': [('colorField', ('select',), False), ('sizeField', ('int', 'currency', 'pct'), False)],
116
+ }
117
+
118
+
119
+ def _now_iso():
120
+ """UTC WITH the offset — a naive stamp is unsubtractable in a browser (D-18, alerts' rule)."""
121
+ return _dt.datetime.now(_dt.timezone.utc).isoformat()
122
+
123
+
124
+ def _grid():
125
+ import aios_grid
126
+ return aios_grid
127
+
128
+
129
+ def _templates():
130
+ import core.view_templates as view_templates
131
+ return view_templates
132
+
133
+
134
+ def _target(session, scope):
135
+ """`(fields, source)` for a database this session may open — or a refusal.
136
+
137
+ ⛔ THE WALL IS `routes_templates._target_or_refuse`, IMPORTED AND NOT COPIED. It is the same
138
+ predicate the nav uses, per topic (`ut_*` -> `user_tables.may_open`; a built-in -> the module
139
+ grant `session.require`; anything else a 404 that is not a directory of what exists), and T51's
140
+ instruction is explicit that the wall is re-used, never re-implemented beside itself. The
141
+ private name is deliberate: an alias would mean editing a file no lane's fence covers.
142
+
143
+ It returns field KEYS; this door also needs each field's TYPE (for the prompt and for
144
+ `_MODE_REFS`), so the types are read from the same source the wall consulted — never from a
145
+ list assembled here.
146
+ """
147
+ import routes_templates
148
+ keys, source = routes_templates._target_or_refuse(session, scope)
149
+ key = str(scope or '').strip()
150
+ defs = []
151
+ if key.startswith('ut_'):
152
+ import core.user_tables as user_tables
153
+ defn = user_tables.get(key, st=session.runtime) or {}
154
+ by_key = {str(f.get('key')): f for f in (defn.get('fields') or []) if f.get('key')}
155
+ else:
156
+ g = _grid()
157
+ raw = g.product_fields() if key == 'product_data' else g.FIELDS
158
+ by_key = {str(f.get('key')): f for f in raw if f.get('key')}
159
+ for k in keys:
160
+ f = by_key.get(k) or {}
161
+ defs.append({'key': k, 'label': str(f.get('label') or k)[:80],
162
+ 'type': str(f.get('type') or 'text')})
163
+ return defs, source
164
+
165
+
166
+ # ── the model ───────────────────────────────────────────────────────────────────────────────
167
+
168
+ #: This module's OWN provider order — cerebras first. See the header: the refusal is a model
169
+ #: property, and the ladder's cheap-first order is exactly wrong for a contract that includes one.
170
+ #: Override with `AIOS_QUERY_PROVIDER`. The credentials themselves stay in ONE place
171
+ #: (`analyst.PROVIDERS`), so a rotated key or a new model reaches this file too.
172
+ QUERY_PROVIDER_ORDER = ('cerebras', 'groq', 'openrouter')
173
+
174
+
175
+ def _providers():
176
+ import harness.analyst as analyst
177
+ pin = os.environ.get("AIOS_QUERY_PROVIDER")
178
+ by_name = {p["name"]: p for p in analyst.PROVIDERS}
179
+ order = [pin] if pin else list(QUERY_PROVIDER_ORDER)
180
+ return [by_name[n] for n in order
181
+ if n in by_name and os.environ.get(by_name[n]["env"])]
182
+
183
+
184
+ def _spec_schema(field_keys):
185
+ """The tool schema the model fills. ⛔ `required` IS `["kind"]` ALONE — see the header: a
186
+ refusal carries no name, and groq rejects a schema-violating refusal with a 400 that reads
187
+ exactly like a transport failure."""
188
+ ops = sorted(_grid().FILTER_OPS)
189
+ col = {"type": "string", "enum": sorted(field_keys)}
190
+ return {
191
+ "type": "object",
192
+ "properties": {
193
+ "kind": {"type": "string", "enum": list(QUERY_KINDS) + ["refused"],
194
+ "description": "refused = this database cannot answer the question"},
195
+ "name": {"type": "string", "description": "a short title for the view"},
196
+ "refusal": {"type": "string",
197
+ "description": "when kind=refused: ONE plain sentence naming what is "
198
+ "missing"},
199
+ "visible": {"type": "array", "items": col},
200
+ "filters": {"type": "array", "items": {
201
+ "type": "object",
202
+ "properties": {"colId": col, "op": {"type": "string", "enum": ops},
203
+ "value": {"type": "string"}},
204
+ "required": ["colId", "op"]}},
205
+ "filterConj": {"type": "string", "enum": ["and", "or"]},
206
+ "sorts": {"type": "array", "items": {
207
+ "type": "object",
208
+ "properties": {"colId": col,
209
+ "dir": {"type": "string", "enum": ["asc", "desc"]}},
210
+ "required": ["colId", "dir"]}},
211
+ "groupBy": col,
212
+ "stackField": col,
213
+ "dateField": col,
214
+ "colorField": col,
215
+ "sizeField": col,
216
+ },
217
+ "required": ["kind"],
218
+ }
219
+
220
+
221
+ def _system_prompt(fields):
222
+ """The standing instructions. The vocabulary is DERIVED (the kinds set, `FILTER_OPS`, the
223
+ target's own columns) — the model is never told about a key this server cannot honour."""
224
+ cols = json.dumps([{"key": f["key"], "label": f["label"], "type": f["type"]}
225
+ for f in fields], separators=(",", ":"))
226
+ kinds = ", ".join(QUERY_KINDS)
227
+ ops = ", ".join(sorted(_grid().FILTER_OPS))
228
+ selects = [f["key"] for f in fields if f["type"] in ('select', 'multiselect')] or ["(none)"]
229
+ dates = [f["key"] for f in fields if f["type"] == 'date'] or ["(none)"]
230
+ return f"""You turn a question about ONE database into a VIEW SPEC. You never write SQL and you
231
+ never invent a column or a kind of view.
232
+
233
+ THE DATABASE ({len(fields)} columns) — you may name ONLY these column keys:
234
+ {cols}
235
+
236
+ VIEW KINDS you may name, and nothing else: {kinds}
237
+ FILTER OPS you may name, and nothing else: {ops}
238
+ The only select-type columns (a kanban MUST stack on one of these): {", ".join(selects)}
239
+ The only date columns (a calendar or timeseries MUST use one of these): {", ".join(dates)}
240
+
241
+ RULES:
242
+ 1. Emit ONE call to build_view. Every colId in visible/filters/sorts/groupBy and every field ref
243
+ MUST be one of the keys above.
244
+ 2. Pick the kind by what the question asks for: a list of records -> grid (or list); totals or
245
+ counts by a category -> chart with groupBy; cards by a stage -> kanban (stackField REQUIRED,
246
+ and it must be a select column); something over time -> calendar or timeseries (dateField
247
+ REQUIRED, and it must be a date column); geography -> map.
248
+ 3. `visible` is the columns the question is about, most important first — not every column.
249
+ 4. Set kind="refused" and put ONE plain sentence in `refusal` when the question needs a column
250
+ this database does not have, a number it does not store, or a picture that is not in the kind
251
+ list. REFUSING IS A CORRECT ANSWER. A view that answers a DIFFERENT question than the one
252
+ asked is the worst possible answer — worse than saying you cannot build it. When in doubt,
253
+ refuse.
254
+ 5. Every `value` is a string."""
255
+
256
+
257
+ def _call_model(question, fields, chat=None):
258
+ """`(spec, error_sentence, provider)` — one bounded call down THIS module's ladder.
259
+
260
+ `chat` is injectable so `verify_query.py` proves this door end to end with no key and no
261
+ tokens (`analyst.ask`'s own posture, and the reason its loop is testable offline).
262
+
263
+ ⚠ THE LADDER IS A DELIBERATE COMPROMISE, AND THE PROVIDER RIDES BACK BECAUSE OF IT. Only the
264
+ FIRST provider is measured to refuse honestly (README section 4b); the ones below it answer
265
+ unanswerable questions with plausible specs. Falling through on a 429 is still the right call —
266
+ a Query module that dies whenever one free tier rate-limits is worse than one that answers with
267
+ a weaker model behind the same validator and the same human read-back — but *which* model
268
+ answered must be knowable, or a wrong answer is undiagnosable after the fact. Failing closed is
269
+ what happens when the whole ladder is unreachable, not when the best rung is.
270
+ """
271
+ tools = [{"type": "function", "function": {
272
+ "name": "build_view",
273
+ "description": "Emit the view spec, or refuse.",
274
+ "parameters": _spec_schema([f["key"] for f in fields])}}]
275
+ messages = [{"role": "system", "content": _system_prompt(fields)},
276
+ {"role": "user", "content": question}]
277
+ if chat is not None:
278
+ return chat(messages, tools), None, "injected"
279
+
280
+ provs = _providers()
281
+ if not provs:
282
+ # No key at all: say so. An AI feature that silently does nothing is indistinguishable
283
+ # from one that was never built ([[flag-shipped-without-its-writer]]).
284
+ return None, ("the assistant is not configured on this deployment, so a view cannot be "
285
+ "built from a question yet"), None
286
+ import requests
287
+ last = None
288
+ for p in provs:
289
+ try:
290
+ r = requests.post(p["url"], timeout=60,
291
+ headers={"Authorization": f"Bearer {os.environ[p['env']]}"},
292
+ json={"model": p["model"], "messages": messages, "tools": tools,
293
+ "tool_choice": "required", "temperature": 0.1,
294
+ "max_tokens": 1200})
295
+ except Exception as e: # network blip — try the next provider
296
+ last = f"{p['name']}: {type(e).__name__}"
297
+ continue
298
+ if r.status_code == 400:
299
+ # ⛔ THE MODEL'S ANSWER CAN BE IN HERE. Groq validates tool arguments against the
300
+ # schema and returns the generation it rejected. Reading it turns a "transport
301
+ # failure" back into the refusal it actually was.
302
+ spec = _spec_from_400(r.text)
303
+ if spec is not None:
304
+ return spec, None, p["name"]
305
+ last = f"{p['name']}: 400"
306
+ continue
307
+ if r.status_code != 200:
308
+ last = f"{p['name']}: HTTP {r.status_code}"
309
+ continue # 429 included: the NEXT provider, not a sleep
310
+ try:
311
+ msg = r.json()["choices"][0]["message"]
312
+ calls = msg.get("tool_calls") or []
313
+ if not calls:
314
+ last = f"{p['name']}: no tool call"
315
+ continue
316
+ return json.loads(calls[0]["function"].get("arguments") or "{}"), None, p["name"]
317
+ except Exception as e:
318
+ last = f"{p['name']}: unreadable answer ({type(e).__name__})"
319
+ continue
320
+ # ⛔ FAIL CLOSED. The first provider in the order is the one measured to refuse honestly; if it
321
+ # is rate-limited and the rest are too, the answer is a sentence, never a fabricated view.
322
+ return None, ("the assistant could not be reached just now (" + (last or "no provider") +
323
+ ") — try the question again in a moment"), None
324
+
325
+
326
+ _FAILED_GEN = re.compile(r'<function=build_view>(\{.*?\})\s*</function>', re.S)
327
+
328
+
329
+ def _spec_from_400(body):
330
+ """The spec out of a `tool_use_failed` body's `failed_generation`, or None."""
331
+ try:
332
+ err_obj = (json.loads(body) or {}).get("error") or {}
333
+ except Exception:
334
+ return None
335
+ gen = str(err_obj.get("failed_generation") or "")
336
+ if not gen:
337
+ return None
338
+ m = _FAILED_GEN.search(gen)
339
+ raw = m.group(1) if m else gen.strip()
340
+ try:
341
+ spec = json.loads(raw)
342
+ except Exception:
343
+ return None
344
+ return spec if isinstance(spec, dict) else None
345
+
346
+
347
+ # ── validation: the REFUSAL this contract promises ───────────────────────────────────────────
348
+
349
+ def _validate(spec, fields):
350
+ """`(clean, refusal, code)` — exactly one of `clean` / `refusal` is set.
351
+
352
+ Membership FIRST and explicitly (the header's whole subject), then the product's own cleaners
353
+ as a belt. A refusal is a SENTENCE, because C5's user-facing promise is a plain sentence and
354
+ never a broken view.
355
+ """
356
+ g = _grid()
357
+ if not isinstance(spec, dict):
358
+ return None, "the assistant did not answer with a view", "no_spec"
359
+ by_key = {f['key']: f['type'] for f in fields}
360
+ keys = set(by_key)
361
+ kind = spec.get('kind')
362
+
363
+ if kind == 'refused':
364
+ return None, (str(spec.get('refusal') or '').strip()
365
+ or "this database cannot answer that question"), "model_refused"
366
+ if kind in QUERY_EXCLUDED:
367
+ return None, (f"a {kind} view cannot be built from a question — "
368
+ + QUERY_EXCLUDED[kind]), "unsupported_kind"
369
+ if kind not in QUERY_KINDS:
370
+ # ⛔ NOT `_clean_display`'s job: it would DROP this and hand back a grid under a name
371
+ # promising something else.
372
+ return None, (f"{kind!r} is not a kind of view this product can build"
373
+ if kind else "the assistant did not name a kind of view"), "unsupported_kind"
374
+
375
+ named = set(spec.get('visible') or ())
376
+ for s in spec.get('sorts') or ():
377
+ if isinstance(s, dict) and s.get('colId'):
378
+ named.add(str(s['colId']))
379
+ for f in spec.get('filters') or ():
380
+ if isinstance(f, dict) and f.get('colId'):
381
+ named.add(str(f['colId']))
382
+ if spec.get('groupBy'):
383
+ named.add(str(spec['groupBy']))
384
+ for ref in ('stackField', 'dateField', 'colorField', 'sizeField'):
385
+ if spec.get(ref):
386
+ named.add(str(spec[ref]))
387
+ missing = sorted(named - keys)
388
+ if missing:
389
+ # NAMED, not counted (`routes_templates`' rule): the column keys are what tell the reader
390
+ # this is the wrong database for this question.
391
+ return None, ("this database does not have the columns that answer needs: "
392
+ + ", ".join(missing)), "unknown_columns"
393
+
394
+ visible = [k for k in (spec.get('visible') or ()) if k in keys][:MAX_VISIBLE]
395
+ if not visible:
396
+ # An empty projection is not a view. `_default_view_config` would fill one in, which is
397
+ # how a question about two columns becomes a 285-column wall.
398
+ return None, ("that question did not name anything to show — try naming the columns you "
399
+ "want to see"), "no_columns"
400
+
401
+ # ⛔ D-137, GUARDED WHERE IT REACHES THIS FEATURE. "The chart cannot key a SET-LIKE or CHECKBOX
402
+ # column the way the grid does" — a set row lands in several buckets, so a sum means something
403
+ # else, and a checkbox reads "(blank)". The defect lives in the CLIENT's `chartData` bucket
404
+ # keying, which is in no lane's fence this wave (booked, with that reason). What IS in this
405
+ # lane's power is refusing to GENERATE the broken case: the AI must only build what the backend
406
+ # and the renderer already support, which is R1's own sentence. Delete this guard when D-137
407
+ # closes — the refusal names the debt so the reader knows it is a limit, not a rule.
408
+ if kind == 'chart' and by_key.get(spec.get('groupBy')) in ('multiselect', 'checkbox'):
409
+ return None, (f"a chart cannot group by {spec.get('groupBy')!r} yet — a column that holds "
410
+ f"several values at once, or a tick box, does not bucket the way a chart "
411
+ f"needs. Try a table grouped by it instead"), "chart_key_unsupported"
412
+
413
+ for ref, families, required in _MODE_REFS.get(kind, ()):
414
+ val = spec.get(ref)
415
+ if val and by_key.get(val) not in families:
416
+ return None, (f"a {kind} view needs {ref[:-5]} to be a "
417
+ f"{' or '.join(families)} column, and {val!r} is a "
418
+ f"{by_key.get(val)} column"), "wrong_ref_type"
419
+ if required and not val:
420
+ return None, (f"a {kind} view needs a {' or '.join(families)} column and this "
421
+ f"database has none that fits"), "missing_ref"
422
+
423
+ raw_filters = [f for f in (spec.get('filters') or ()) if isinstance(f, dict)]
424
+ filters = g.clean_filter_tree(raw_filters, keys)
425
+ if len(filters) != len(raw_filters):
426
+ # The belt caught something membership did not — an unknown op. Refuse rather than save a
427
+ # view whose condition was silently dropped (the widening this file exists to prevent).
428
+ return None, ("part of that filter is not something this product can express — try "
429
+ "asking it more simply"), "filter_dropped"
430
+
431
+ display = {'mode': kind}
432
+ for ref, _families, _req in _MODE_REFS.get(kind, ()):
433
+ if spec.get(ref):
434
+ display[ref] = spec[ref]
435
+ cleaned_display = g._clean_display(display, keys) if kind != 'grid' else None
436
+ if kind != 'grid' and not cleaned_display:
437
+ return None, (f"this product could not build a {kind} view from that question"
438
+ ), "display_dropped"
439
+
440
+ clean = {
441
+ 'kind': kind,
442
+ 'name': ' '.join(str(spec.get('name') or 'Query').split())[:60] or 'Query',
443
+ 'visible': visible,
444
+ 'filters': filters,
445
+ 'filterConj': 'or' if spec.get('filterConj') == 'or' else 'and',
446
+ 'sorts': [{'colId': s['colId'], 'dir': 'desc' if s.get('dir') == 'desc' else 'asc'}
447
+ for s in (spec.get('sorts') or ()) if isinstance(s, dict)
448
+ and s.get('colId') in keys][:3],
449
+ 'groupBy': spec['groupBy'] if spec.get('groupBy') in keys else None,
450
+ 'display': cleaned_display,
451
+ }
452
+ return clean, None, None
453
+
454
+
455
+ def _explain(clean, fields):
456
+ """A plain-language read-back of what the view WILL do — DERIVED from the validated spec.
457
+
458
+ ⛔ NEVER ASKED OF THE MODEL. A model describing its own output describes what it MEANT, which
459
+ is precisely the failure this sentence exists to expose: a spec whose every column is real and
460
+ which answers a different question is invisible to validation, so the only thing that catches
461
+ it is a person reading an honest description of the spec that was actually accepted.
462
+ """
463
+ label = {f['key']: f['label'] for f in fields}
464
+ ops = {'eq': 'is', 'neq': 'is not', 'gt': 'is over', 'gte': 'is at least',
465
+ 'lt': 'is under', 'lte': 'is at most', 'contains': 'contains',
466
+ 'doesNotContain': 'does not contain', 'isEmpty': 'is empty',
467
+ 'isNotEmpty': 'is not empty', 'between': 'is between', 'within': 'is within',
468
+ 'topN': 'is in the top', 'bottomN': 'is in the bottom'}
469
+ kinds = {'grid': 'a table', 'list': 'a list', 'chart': 'a chart', 'kanban': 'a board',
470
+ 'calendar': 'a calendar', 'timeseries': 'a time series', 'map': 'a map'}
471
+ parts = [f"{kinds.get(clean['kind'], clean['kind'])} of "
472
+ + ", ".join(label.get(k, k) for k in clean['visible'][:6])
473
+ + (f" and {len(clean['visible']) - 6} more columns"
474
+ if len(clean['visible']) > 6 else "")]
475
+ if clean['filters']:
476
+ joiner = " or " if clean['filterConj'] == 'or' else " and "
477
+ parts.append("showing only records where " + joiner.join(
478
+ f"{label.get(f['colId'], f['colId'])} {ops.get(f['op'], f['op'])}"
479
+ + (f" {f['value']}" if f.get('value') else "")
480
+ for f in clean['filters'] if isinstance(f, dict) and f.get('colId')))
481
+ else:
482
+ parts.append("showing every record")
483
+ if clean['groupBy']:
484
+ parts.append(f"grouped by {label.get(clean['groupBy'], clean['groupBy'])}")
485
+ if clean['sorts']:
486
+ s = clean['sorts'][0]
487
+ parts.append(f"sorted by {label.get(s['colId'], s['colId'])}"
488
+ + (" (highest first)" if s['dir'] == 'desc' else " (lowest first)"))
489
+ for ref, word in (('stackField', 'in columns by'), ('dateField', 'placed in time by'),
490
+ ('colorField', 'coloured by'), ('sizeField', 'sized by')):
491
+ val = (clean.get('display') or {}).get(ref)
492
+ if val:
493
+ parts.append(f"{word} {label.get(val, val)}")
494
+ return ", ".join(parts) + "."
495
+
496
+
497
+ # ── the registry ────────────────────────────────────────────────────────────────────────────
498
+
499
+ def _qid(scope, question):
500
+ """PINNED per (database, question), the `view_templates` discipline: asking the same thing
501
+ twice UPDATES one view instead of minting "Query 2" — `save_view` de-duplicates NAMES by
502
+ appending a number, so an unpinned id fills the store with numbered near-duplicates."""
503
+ h = hashlib.sha1(f"{scope}|{' '.join(str(question).split()).lower()}".encode("utf-8"))
504
+ return "q_" + h.hexdigest()[:12]
505
+
506
+
507
+ def _registry(session):
508
+ try:
509
+ recs = session.runtime.get(QUERY_KEY) or {}
510
+ except Exception:
511
+ return {}
512
+ return recs if isinstance(recs, dict) else {}
513
+
514
+
515
+ def _mine(session, rec):
516
+ return session.admin or str(rec.get('createdBy') or '') == session.uname
517
+
518
+
519
+ #: The BUILT-IN topics `routes_templates._target_or_refuse` accepts beside any `ut_*` key. ⛔ THE
520
+ #: CLIENT'S PICKER MUST MIRROR THIS, and `GET /query` publishes it (`builtins`) rather than leaving
521
+ #: the page to re-derive it: the nav hands `QueryPage` every granted entry, which is a WIDER set
522
+ #: than this door accepts, so a picker filtered on "not a surface" alone would offer a database the
523
+ #: build door 404s. `view_templates`' own rule — "a picker that offered a template the apply door
524
+ #: would then refuse is a control that lies" — and it is the same predicate twice, published once.
525
+ BUILTIN_SCOPES = ('customer_data', 'product_data')
526
+
527
+
528
+ @router.get("/query")
529
+ def list_queries(session: Session = Depends(require_session)):
530
+ """The AI-built views this caller may open: their own, still-granted, still-existing.
531
+
532
+ ⚠ PRUNED ON READ against the CURRENT wall, never merely on write — a grant can be withdrawn
533
+ and a database deleted after a view was saved, and the Query module is chrome: it may render
534
+ nothing the server would not grant today (`nav.ts`'s chrome law).
535
+ """
536
+ import routes_templates
537
+ out = []
538
+ for qid, rec in sorted(_registry(session).items(),
539
+ key=lambda kv: str((kv[1] or {}).get('createdAt') or ''),
540
+ reverse=True):
541
+ if not isinstance(rec, dict) or not _mine(session, rec):
542
+ continue
543
+ try:
544
+ routes_templates._target_or_refuse(session, rec.get('scope'))
545
+ except Exception:
546
+ continue # no longer granted, or gone
547
+ out.append({"id": qid, "scope": rec.get('scope'), "viewId": rec.get('viewId'),
548
+ "name": rec.get('name'), "kind": rec.get('kind'),
549
+ "question": rec.get('question'), "explain": rec.get('explain'),
550
+ "createdAt": rec.get('createdAt')})
551
+ return {"views": out, "kinds": list(QUERY_KINDS), "builtins": list(BUILTIN_SCOPES)}
552
+
553
+
554
+ @router.post("/query/build")
555
+ def build_query(body: dict = Body(default=None),
556
+ session: Session = Depends(require_session)):
557
+ """A question + a granted database -> `{spec, explain, refused}`. **WRITES NOTHING.**
558
+
559
+ A refusal is a 200 carrying a sentence, not a 5xx: "this database cannot answer that" is a
560
+ successful answer to the person asking, and an error envelope would paint the error page R1
561
+ forbids. The 4xx cases are the ones that ARE the caller's fault — an unnamed database, an
562
+ over-long question, a database this session may not open.
563
+
564
+ ⛔ THE MODEL SEAM IS **NOT** ON THIS SIGNATURE, AND THAT IS A CORRECTION. The first draft took
565
+ `chat=None` as a parameter of the ROUTE — and FastAPI reads an un-annotated defaulted argument
566
+ as a **QUERY PARAMETER**, so `chat` appeared in `openapi()["paths"]` (measured) and
567
+ `POST /query/build?chat=x` would have reached `_call_model(..., chat="x")` and 500'd on
568
+ `"x"(messages, tools)`. The gate never saw it because the gate calls the handler directly —
569
+ which is exactly the shape of a test seam that becomes an input. The seam now lives on the
570
+ module-level `_build` below: the gate calls that, and the wire cannot.
571
+ """
572
+ body = body if isinstance(body, dict) else {}
573
+ return _build(body.get("question"), body.get("scope"), session)
574
+
575
+
576
+ def _build(question, scope, session, chat=None):
577
+ """`build_query`'s body, with the model injectable — see that route's note on why the seam is
578
+ here and not on the signature FastAPI reads."""
579
+ question = ' '.join(str(question or "").split())
580
+ if not question:
581
+ raise err(400, "bad_request", "no question was asked")
582
+ if len(question) > MAX_QUESTION:
583
+ raise err(400, "question_too_long",
584
+ f"a question must be under {MAX_QUESTION} characters")
585
+ fields, _source = _target(session, scope)
586
+ if not fields:
587
+ raise err(400, "no_columns", "that database has no columns to build a view from")
588
+
589
+ spec, transport, provider = _call_model(question, fields, chat=chat)
590
+ if transport:
591
+ return {"spec": None, "explain": None, "refused": transport, "provider": provider}
592
+ clean, refusal, code = _validate(spec, fields)
593
+ if refusal:
594
+ return {"spec": None, "explain": None, "refused": refusal, "reason": code,
595
+ "provider": provider}
596
+ return {"spec": clean, "explain": _explain(clean, fields),
597
+ "refused": None, "scope": scope,
598
+ "id": _qid(scope, question), "provider": provider}
599
+
600
+
601
+ @router.post("/query/save")
602
+ def save_query(body: dict = Body(default=None), session: Session = Depends(require_session)):
603
+ """Save an accepted spec as the CALLING USER's own view, and index it under Query.
604
+
605
+ ⛔ THE SPEC IS RE-VALIDATED HERE, against the target's contract as it is NOW. The build door
606
+ wrote nothing, so this is the first write and it is the only place authorisation and validity
607
+ have to hold together — and a client is not a wall. An unsupported kind or an unknown column
608
+ is a **400 with a named code** on this door (the caller sent it), where the same fact is a
609
+ plain sentence on `build` (the model produced it).
610
+ """
611
+ body = body if isinstance(body, dict) else {}
612
+ scope = str(body.get("scope") or '').strip()
613
+ question = ' '.join(str(body.get("question") or "").split())[:MAX_QUESTION]
614
+ fields, _source = _target(session, scope)
615
+ clean, refusal, code = _validate(body.get("spec"), fields)
616
+ if refusal:
617
+ raise err(400, code or "bad_spec", refusal)
618
+ if not session.runtime.available():
619
+ raise err(503, "store_unavailable", "the tenant store is unavailable — nothing was saved")
620
+
621
+ vt = _templates()
622
+ ws_key = vt.workspace_key(scope)
623
+ if not ws_key:
624
+ raise err(404, "unknown_table", "that database does not exist")
625
+ qid = _qid(scope, question) if question else _qid(scope, clean['name'])
626
+
627
+ # ⛔ THE SKELETON IS `view_templates._view`, NOT A SECOND ONE. "A view assembled from a
628
+ # DIFFERENT skeleton is a second definition of what a view is — and the two drift on the day
629
+ # one of them gains a key." It also puts the mode at `config.display.mode`, which is the key
630
+ # the grid actually reads (`config.displayMode` was the first draft, and nothing reads it).
631
+ payload = vt._view(qid, clean['name'], f"Built by the assistant from: {question}"[:300],
632
+ clean['visible'], mode=clean['kind'],
633
+ sorts=clean['sorts'], filters=clean['filters'],
634
+ group_by=clean['groupBy'], conj=clean['filterConj'])
635
+ if clean.get('display'):
636
+ payload['config']['display'] = dict(clean['display'])
637
+ payload['createdBy'] = session.uname
638
+
639
+ # ⛔ THE CAP REFUSES; IT DOES NOT EVICT — and the first draft evicted. Dropping the oldest entry
640
+ # would leave its saved VIEW behind in the table's own workspace: "stranded in the target's view
641
+ # list with no way back to the question that made it", which is the thing `delete_query` below
642
+ # exists to prevent. And an eviction nobody was told about is W30/R6's second sentence broken —
643
+ # a limit that cannot be removed must be REPORTED with its cause. So the 201st is a refusal
644
+ # naming the number and the way out, checked BEFORE the view is written.
645
+ existing = _registry(session)
646
+ if qid not in existing and len(existing) >= MAX_REGISTRY:
647
+ raise err(400, "query_limit",
648
+ f"you already have {len(existing)} saved questions, which is the limit "
649
+ f"({MAX_REGISTRY}). Delete one you no longer need and save this again.")
650
+
651
+ import core.table_store as table_store
652
+ ops = table_store.make(ws_key, st=session.runtime)
653
+ saved = ops.save_view(session.uname, payload, shared=False)
654
+
655
+ entry = {"scope": scope, "viewId": qid, "name": saved.get('name') or clean['name'],
656
+ "kind": clean['kind'], "question": question,
657
+ "explain": _explain(clean, fields), "createdBy": session.uname,
658
+ "createdAt": _now_iso()}
659
+
660
+ def _up(data):
661
+ data = data if isinstance(data, dict) else {}
662
+ data[qid] = entry
663
+ return data
664
+
665
+ session.runtime.update(QUERY_KEY, _up, flush='async')
666
+ return {"id": qid, **entry}
667
+
668
+
669
+ @router.delete("/query/{qid}")
670
+ def delete_query(qid: str, session: Session = Depends(require_session)):
671
+ """Forget an AI-built view — the index entry AND the view itself.
672
+
673
+ Leaving the view behind would strand it in the target's view list with no way back to the
674
+ question that made it ([[a-record-can-outlive-its-subject]]).
675
+ """
676
+ rec = _registry(session).get(str(qid))
677
+ if not isinstance(rec, dict) or not _mine(session, rec):
678
+ raise err(404, "unknown_query", "that view does not exist")
679
+ _target(session, rec.get('scope')) # the wall, again, on the write path
680
+ vt = _templates()
681
+ ws_key = vt.workspace_key(rec.get('scope'))
682
+ if ws_key and session.runtime.available():
683
+ import core.table_store as table_store
684
+ table_store.make(ws_key, st=session.runtime).delete_view(
685
+ session.uname, rec.get('viewId') or str(qid))
686
+
687
+ def _up(data):
688
+ data = data if isinstance(data, dict) else {}
689
+ data.pop(str(qid), None)
690
+ return data
691
+
692
+ session.runtime.update(QUERY_KEY, _up, flush='async')
693
+ return {"deleted": str(qid)}
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
api/routes_records.py CHANGED
@@ -1,211 +1,164 @@
1
- """Record detail routes: durable comments, scoped to the caller's book — ON EVERY DATABASE.
2
-
3
- ⭐ WAVE 19 (owner item 12). This file used to be the CUSTOMER record's comment routes with a
4
- customer-shaped wall bolted to the module import line: `_in_book` asked
5
- `routes_customers.allowed_pids` whatever surface the browser was on. Opening a PRODUCT record and
6
- typing a comment therefore asked the customer book about a CRC32 hash of a SKU code, and the
7
- panel answered "that customer is not in your book" — the owner's report. The dangerous half is
8
- the one nobody sees: a hash that collides with a real partner id passes the wall, and the comment
9
- is filed against somebody's customer where the whole team can read it.
10
-
11
- THE SHAPE NOW: `?scope=` names the database (the same vocabulary `/workspace?scope=` and the
12
- events route's `scopeKey` already speak), and `_pool_or_refuse` resolves BOTH halves of the wall
13
- per scope — the GRANT and the ROW SET — by asking that topic's own route, never by re-deriving
14
- one here:
15
-
16
- customer / cohort `routes_customers.allowed_pids` behind the `customer_data` grant
17
- product `routes_products.scoped_pool` behind the `product_data` grant
18
- ut_<slug> `routes_tables.scoped_pids`, whose `_defn_or_refuse` IS the wall
19
- (404 unknown / 403 not yours — a user table has no module grant).
20
- ⭐ W33-T03/D-183: `scoped_pIDs`, not `scoped_pOOL` — the pool builds every
21
- ROW to derive a pid set this module discards, and RAISES 409 on a
22
- read-through grid past one window, which is why the record drawer painted
23
- an error page on `ut_odoo_gl_lines`.
24
-
25
- ⚠ THE PATH KEEPS ITS `/customers/` SEGMENT. It is the shipped URL and `verify_api.py`'s E1a
26
- section pins it; the scope now travels beside it explicitly. A nicer noun is not worth churning
27
- another session's gate mid-wave — the WALL is the query parameter, not the word.
28
-
29
- ⚠ NO DEFAULT BEYOND THE LEGACY ONE. An absent `scope` means `customer`, which is what every
30
- shipped client sent and what keeps the old callers byte-identical; an UNRECOGNISED scope is a
31
- 400, never a silent fallback to the customer book (`routes_grid._scope_or_400`'s rule, and for
32
- the same reason: a typo served as `customer` answers a question nobody asked).
33
- """
34
- from fastapi import APIRouter, Body, Depends, Query
35
-
36
- from deps import Session, err, require_session
37
-
38
- router = APIRouter(prefix="/api/v1")
39
-
40
- #: The customer topic's two names — one book, two surfaces (the Cohort page is the customer table
41
- #: over hand-curated sets). Mirrors `modules.cohort.LEGACY_SCOPES` / `core.record_comments`.
42
- _CUSTOMER_SCOPES = ("", "customer", "cohort")
43
-
44
-
45
- def _scope_or_400(raw):
46
- scope = str(raw or "customer").strip().lower()
47
- if scope in _CUSTOMER_SCOPES or scope == "product" or scope.startswith("ut_"):
48
- return "customer" if scope in _CUSTOMER_SCOPES else scope
49
- raise err(400, "bad_scope",
50
- "scope must be customer, cohort, product or a ut_ database — refusing to guess")
51
-
52
-
53
- def _pool_or_refuse(session: Session, scope: str):
54
- """The pids this session may attach comments to ON THIS DATABASE — grant wall included.
55
-
56
- Returns **`(pids, unbounded)`** — a 2-tuple on EVERY branch. Raises the topic's own 403/404/503,
57
- so a caller who may not open the surface never learns anything about the row they asked about.
58
-
59
- ⛔ `unbounded` is TRUE only when the row set could not be ENUMERATED (a read-through grid past
60
- one window), never when it is merely EMPTY. Those are opposite answers and `scoped_pids` returns
61
- `frozenset()` for both — see the branch below.
62
- ⚠ THE SHAPE IS A CONTRACT EVEN THOUGH THIS FUNCTION IS PRIVATE, and it has two consumers that
63
- do not travel together: `_in_book` here, and a NEGATIVE CONTROL in `aios-web/api/verify_scopes.py`
64
- that REPLACES this function with its own lambda. A gate's test double is a caller
65
- ([[test-double-patched-by-a-name-list]]); when this signature moved, that double kept returning a
66
- bare frozenset and the section died on `ValueError: too many values to unpack` — no tally, no
67
- failing name. Change the shape here and that double changes with it.
68
- """
69
- if scope == "product":
70
- from routes_products import MODULE as PRODUCT_MODULE, scoped_pool
71
-
72
- session.require(PRODUCT_MODULE)
73
- pids, _team, _rows, _fields = scoped_pool(session)
74
- # ⚠ `(pids, unbounded)` on EVERY branch. This one returned a bare frozenset for ten minutes
75
- # after the `ut_` branch grew its second element, and `_in_book`'s unpack would have raised
76
- # a `TypeError` — a 500 on every product comment — while both other branches worked. A
77
- # return shape is a contract even when the function is private.
78
- return (pids, False)
79
- if scope.startswith("ut_"):
80
- # No module grant exists for a user table — `_defn_or_refuse` inside `scoped_pids` IS
81
- # the wall (creator or admin, fail-closed), and it answers 404 before 403 exactly as the
82
- # rows routes do.
83
- #
84
- # ⭐⭐ W33-T03 / D-183 — `scoped_pids`, NOT `scoped_pool`, AND THAT ONE WORD IS THE BUG.
85
- # `scoped_pool` builds every ROW to derive a pid set this function then throws away, and on
86
- # a read-through grid larger than one window it RAISES `409 window_required`. So opening the
87
- # record drawer on `ut_odoo_gl_lines` (975,137 rows) painted an error page — for a panel
88
- # that renders comments about ONE row it already has. `scoped_pids` answers the identical
89
- # question (its docstring: *"the pid set is IDENTICAL, not merely equivalent"*) and takes
90
- # W31-T20's `limits` OUT-PARAMETER instead of raising, which is the same shape `/workspace`
91
- # used to become openable on those two grids.
92
- from routes_tables import scoped_pids
93
-
94
- limits = []
95
- pids, _fields, _defn = scoped_pids(session, scope, limits=limits)
96
- # ⛔ AN EMPTY PID SET AND AN UNRESOLVABLE ONE ARE OPPOSITE ANSWERS, and collapsing them is
97
- # how a fail-closed default becomes a lie. `scoped_pids` returns `frozenset()` BOTH for a
98
- # database with no rows and for a read-through grid too big to enumerate — it distinguishes
99
- # them by APPENDING R6's sentence to `limits`. Without this branch the drawer would move
100
- # from a 409 error page to a 403 "not in your book" on a row the user is looking at, which
101
- # is the same defect wearing a politer message ([[empty-answer-vs-unfinished-answer]]).
102
- # ⚠ ADMITTING HERE IS NOT A WIDENING, and the ruling is wave 27 / D-72: on a `ut_*` database
103
- # THE TENANT IS THE UNIT — `scoped_pool` itself carries "no per-row owner filter; the
104
- # table-level wall is the WHOLE wall". `_defn_or_refuse` has already run inside
105
- # `scoped_pids` and answered 404/403. The pid set was only ever an existence check.
106
- return (pids, bool(limits))
107
- from routes_customers import MODULE as CUSTOMER_MODULE, allowed_pids
108
-
109
- session.require(CUSTOMER_MODULE)
110
- return (frozenset(allowed_pids(session)), False)
111
-
112
-
113
- def _in_book(pid, session, scope):
114
- pids, unbounded = _pool_or_refuse(session, scope)
115
- if unbounded:
116
- # The table wall passed and the row set is larger than this process will enumerate. Said
117
- # out loud rather than silently admitting: R6's second sentence is that a limit which
118
- # cannot be removed gets REPORTED, and this is the one place the report has no envelope to
119
- # ride in.
120
- print(f"[records] {scope}: pid membership unresolved (read-through beyond one window) — "
121
- f"admitting on the table wall alone, per D-72")
122
- return
123
- if pid not in pids:
124
- # 403, not 404: the record may exist, but this session may not inspect it.
125
- raise err(403, "out_of_scope", "that record is not in your book")
126
-
127
-
128
- def _unavailable():
129
- return err(
130
- 503,
131
- "store_unavailable",
132
- "record comments are temporarily unavailable — no change was saved",
133
- )
134
-
135
-
136
- # ⭐ WAVE 21 (D-17): the CANONICAL path is /records/{pid}/comments — comments hang off a RECORD
137
- # in whatever topic `?scope=` names, and the customer-flavoured noun was wave-19 residue (the
138
- # wall was always the query param). The old path stays as an ALIAS because the shipped client
139
- # still calls it; verify_api pins the canonical path AND that the alias answers, so removing
140
- # the alias later is a decision, never an accident.
141
- @router.get("/records/{pid}/comments")
142
- @router.get("/customers/{pid}/comments")
143
- def comments(pid: int, scope: str = Query(default="customer"),
144
- session: Session = Depends(require_session)):
145
- from core import record_comments
146
-
147
- scope = _scope_or_400(scope)
148
- _in_book(pid, session, scope)
149
- try:
150
- rows = record_comments.list_comments(session.runtime, pid, scope=scope)
151
- except record_comments.CommentsUnavailable:
152
- raise _unavailable()
153
- return {"comments": rows}
154
-
155
-
156
- @router.post("/records/{pid}/comments", status_code=201)
157
- @router.post("/customers/{pid}/comments", status_code=201)
158
- def create_comment(
159
- pid: int,
160
- body: dict = Body(default=None),
161
- scope: str = Query(default="customer"),
162
- session: Session = Depends(require_session),
163
- ):
164
- from core import record_comments
165
-
166
- scope = _scope_or_400(scope)
167
- _in_book(pid, session, scope)
168
- try:
169
- comment = record_comments.add_comment(
170
- session.runtime,
171
- pid,
172
- (body or {}).get("body"),
173
- session.uname,
174
- session.user.get("name") or session.uname,
175
- scope=scope,
176
- )
177
- except ValueError as exc:
178
- raise err(400, "bad_comment", str(exc))
179
- except record_comments.CommentsUnavailable:
180
- raise _unavailable()
181
- return {"comment": comment}
182
-
183
-
184
- @router.delete("/records/{pid}/comments/{comment_id}")
185
- @router.delete("/customers/{pid}/comments/{comment_id}")
186
- def remove_comment(
187
- pid: int,
188
- comment_id: str,
189
- scope: str = Query(default="customer"),
190
- session: Session = Depends(require_session),
191
- ):
192
- from core import record_comments
193
-
194
- scope = _scope_or_400(scope)
195
- _in_book(pid, session, scope)
196
- try:
197
- deleted = record_comments.delete_comment(
198
- session.runtime,
199
- pid,
200
- comment_id,
201
- session.uname,
202
- admin=session.admin,
203
- scope=scope,
204
- )
205
- except record_comments.CommentForbidden:
206
- raise err(403, "comment_forbidden", "only the author may delete this comment")
207
- except record_comments.CommentsUnavailable:
208
- raise _unavailable()
209
- if not deleted:
210
- raise err(404, "comment_not_found", "that comment no longer exists")
211
- return {"ok": True, "id": comment_id}
 
1
+ """Record detail routes: durable comments, scoped to the caller's book — ON EVERY DATABASE.
2
+
3
+ ⭐ WAVE 19 (owner item 12). This file used to be the CUSTOMER record's comment routes with a
4
+ customer-shaped wall bolted to the module import line: `_in_book` asked
5
+ `routes_customers.allowed_pids` whatever surface the browser was on. Opening a PRODUCT record and
6
+ typing a comment therefore asked the customer book about a CRC32 hash of a SKU code, and the
7
+ panel answered "that customer is not in your book" — the owner's report. The dangerous half is
8
+ the one nobody sees: a hash that collides with a real partner id passes the wall, and the comment
9
+ is filed against somebody's customer where the whole team can read it.
10
+
11
+ THE SHAPE NOW: `?scope=` names the database (the same vocabulary `/workspace?scope=` and the
12
+ events route's `scopeKey` already speak), and `_pool_or_refuse` resolves BOTH halves of the wall
13
+ per scope — the GRANT and the ROW SET — by asking that topic's own route, never by re-deriving
14
+ one here:
15
+
16
+ customer / cohort `routes_customers.allowed_pids` behind the `customer_data` grant
17
+ product `routes_products.scoped_pool` behind the `product_data` grant
18
+ ut_<slug> `routes_tables.scoped_pool`, whose `_defn_or_refuse` IS the wall
19
+ (404 unknown / 403 not yours — a user table has no module grant)
20
+
21
+ ⚠ THE PATH KEEPS ITS `/customers/` SEGMENT. It is the shipped URL and `verify_api.py`'s E1a
22
+ section pins it; the scope now travels beside it explicitly. A nicer noun is not worth churning
23
+ another session's gate mid-wave — the WALL is the query parameter, not the word.
24
+
25
+ ⚠ NO DEFAULT BEYOND THE LEGACY ONE. An absent `scope` means `customer`, which is what every
26
+ shipped client sent and what keeps the old callers byte-identical; an UNRECOGNISED scope is a
27
+ 400, never a silent fallback to the customer book (`routes_grid._scope_or_400`'s rule, and for
28
+ the same reason: a typo served as `customer` answers a question nobody asked).
29
+ """
30
+ from fastapi import APIRouter, Body, Depends, Query
31
+
32
+ from deps import Session, err, require_session
33
+
34
+ router = APIRouter(prefix="/api/v1")
35
+
36
+ #: The customer topic's two names — one book, two surfaces (the Cohort page is the customer table
37
+ #: over hand-curated sets). Mirrors `modules.cohort.LEGACY_SCOPES` / `core.record_comments`.
38
+ _CUSTOMER_SCOPES = ("", "customer", "cohort")
39
+
40
+
41
+ def _scope_or_400(raw):
42
+ scope = str(raw or "customer").strip().lower()
43
+ if scope in _CUSTOMER_SCOPES or scope == "product" or scope.startswith("ut_"):
44
+ return "customer" if scope in _CUSTOMER_SCOPES else scope
45
+ raise err(400, "bad_scope",
46
+ "scope must be customer, cohort, product or a ut_ database — refusing to guess")
47
+
48
+
49
+ def _pool_or_refuse(session: Session, scope: str):
50
+ """The pids this session may attach comments to ON THIS DATABASE — grant wall included.
51
+
52
+ Returns the frozenset. Raises the topic's own 403/404/503, so a caller who may not open the
53
+ surface never learns anything about the row they asked about.
54
+ """
55
+ if scope == "product":
56
+ from routes_products import MODULE as PRODUCT_MODULE, scoped_pool
57
+
58
+ session.require(PRODUCT_MODULE)
59
+ pids, _team, _rows, _fields = scoped_pool(session)
60
+ return pids
61
+ if scope.startswith("ut_"):
62
+ # No module grant exists for a user table — `_defn_or_refuse` inside `scoped_pool` IS
63
+ # the wall (creator or admin, fail-closed), and it answers 404 before 403 exactly as the
64
+ # rows routes do.
65
+ from routes_tables import scoped_pool
66
+
67
+ pids, _rows, _fields, _defn = scoped_pool(session, scope)
68
+ return pids
69
+ from routes_customers import MODULE as CUSTOMER_MODULE, allowed_pids
70
+
71
+ session.require(CUSTOMER_MODULE)
72
+ return frozenset(allowed_pids(session))
73
+
74
+
75
+ def _in_book(pid, session, scope):
76
+ if pid not in _pool_or_refuse(session, scope):
77
+ # 403, not 404: the record may exist, but this session may not inspect it.
78
+ raise err(403, "out_of_scope", "that record is not in your book")
79
+
80
+
81
+ def _unavailable():
82
+ return err(
83
+ 503,
84
+ "store_unavailable",
85
+ "record comments are temporarily unavailable — no change was saved",
86
+ )
87
+
88
+
89
+ # ⭐ WAVE 21 (D-17): the CANONICAL path is /records/{pid}/comments — comments hang off a RECORD
90
+ # in whatever topic `?scope=` names, and the customer-flavoured noun was wave-19 residue (the
91
+ # wall was always the query param). The old path stays as an ALIAS because the shipped client
92
+ # still calls it; verify_api pins the canonical path AND that the alias answers, so removing
93
+ # the alias later is a decision, never an accident.
94
+ @router.get("/records/{pid}/comments")
95
+ @router.get("/customers/{pid}/comments")
96
+ def comments(pid: int, scope: str = Query(default="customer"),
97
+ session: Session = Depends(require_session)):
98
+ from core import record_comments
99
+
100
+ scope = _scope_or_400(scope)
101
+ _in_book(pid, session, scope)
102
+ try:
103
+ rows = record_comments.list_comments(session.runtime, pid, scope=scope)
104
+ except record_comments.CommentsUnavailable:
105
+ raise _unavailable()
106
+ return {"comments": rows}
107
+
108
+
109
+ @router.post("/records/{pid}/comments", status_code=201)
110
+ @router.post("/customers/{pid}/comments", status_code=201)
111
+ def create_comment(
112
+ pid: int,
113
+ body: dict = Body(default=None),
114
+ scope: str = Query(default="customer"),
115
+ session: Session = Depends(require_session),
116
+ ):
117
+ from core import record_comments
118
+
119
+ scope = _scope_or_400(scope)
120
+ _in_book(pid, session, scope)
121
+ try:
122
+ comment = record_comments.add_comment(
123
+ session.runtime,
124
+ pid,
125
+ (body or {}).get("body"),
126
+ session.uname,
127
+ session.user.get("name") or session.uname,
128
+ scope=scope,
129
+ )
130
+ except ValueError as exc:
131
+ raise err(400, "bad_comment", str(exc))
132
+ except record_comments.CommentsUnavailable:
133
+ raise _unavailable()
134
+ return {"comment": comment}
135
+
136
+
137
+ @router.delete("/records/{pid}/comments/{comment_id}")
138
+ @router.delete("/customers/{pid}/comments/{comment_id}")
139
+ def remove_comment(
140
+ pid: int,
141
+ comment_id: str,
142
+ scope: str = Query(default="customer"),
143
+ session: Session = Depends(require_session),
144
+ ):
145
+ from core import record_comments
146
+
147
+ scope = _scope_or_400(scope)
148
+ _in_book(pid, session, scope)
149
+ try:
150
+ deleted = record_comments.delete_comment(
151
+ session.runtime,
152
+ pid,
153
+ comment_id,
154
+ session.uname,
155
+ admin=session.admin,
156
+ scope=scope,
157
+ )
158
+ except record_comments.CommentForbidden:
159
+ raise err(403, "comment_forbidden", "only the author may delete this comment")
160
+ except record_comments.CommentsUnavailable:
161
+ raise _unavailable()
162
+ if not deleted:
163
+ raise err(404, "comment_not_found", "that comment no longer exists")
164
+ return {"ok": True, "id": comment_id}
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
api/routes_shares.py CHANGED
@@ -1,393 +1,385 @@
1
- """routes_shares.py — the manage-access surface (wave 20, owner ruling R10, contract C-SHARE).
2
-
3
- GET /api/v1/share/{kind}/{oid} -> {owner, entries:[{user,role}], mayAdminister, people}
4
- PUT /api/v1/share/{kind}/{oid} <- {entries:[{user,role}]} (REPLACES the set)
5
- GET /api/v1/share/mine -> {view:[id], folder:[id], database:[id]}
6
-
7
- `kind` ∈ view | folder | database. Roles are `view` | `edit` — the same two words the view rail
8
- already speaks, now extended to folders and databases so there is ONE vocabulary in the UI
9
- (R10: "the same picker views use").
10
-
11
- ⛔ **RE-SHARING IS THE OWNER'S, AND THAT IS ENFORCED HERE, NOT IN THE CLIENT.** `PUT` requires
12
- `shares.may_administer` (owner or admin). A collaborator with `edit` may change an object's
13
- CONTENT and may not change who else can reach it — otherwise anyone you shared a view with could
14
- widen it to everyone, or grant themselves ownership and lock you out. The client greys the editor
15
- for non-administrators; that is a courtesy, and this check is the wall.
16
-
17
- ⚠ **THE GRANT NEVER WIDENS PAST THE MODULE WALL — ON A GOVERNED MODULE.** `*` ("everyone") means
18
- every account that can already open the surface: `require_session` plus the topic's own gate run
19
- first, and for `customer_data` / `product_data` the receiver's own row scope and hidden-field
20
- closure run BEFORE any foreign view is merged. Sharing there can only narrow-or-equal the set that
21
- could already reach the data ([[aios-permissioning]]).
22
-
23
- ⛔⛔ **AND THAT SENTENCE IS FALSE FOR `kind='database'`, WHICH IS WHY IT NOW SAYS "ON A GOVERNED
24
- MODULE" (W32-T26, audit S-8).** `routes_admin._PERM_MODULES` is `("customer_data","product_data")`
25
- and `_clean_perms` **400s** on anything else, so **no row filter and no hidden field can even be
26
- DECLARED for a `ut_*` database** — `routes_tables.py` makes zero `perm_scope` calls and passes
27
- `hidden_keys=frozenset()`. There is no module wall behind a user table for a grant to be bounded
28
- by: **this registry IS the wall.** So a `database` grant is ALL-OR-NOTHING — every row, every
29
- column — and an `*` database grant admits every account in the tenant to all of it.
30
- That is a real capability, deliberately kept; what was wrong was a docstring promising a second
31
- wall that does not exist for this kind. Scoping user tables is booked, not done
32
- (`waves/wave32/sharing-audit.md` S-8).
33
-
34
- ⚠ **TWO SYSTEMS ANSWER "IS THIS SHARED", AND THEY ARE NOT THE SAME ONE (audit S-4).** THIS
35
- registry decides who appears in *"Shared with me"* and who may re-share. **`table_store.is_shared`
36
- — the view's own `permissions` — is what actually decides who may OPEN a view.** A grant here
37
- whose object is invisible under that one is a row in a list that opens a refusal, which is what
38
- made item 18 worth auditing. `_entries_or_400` closes the common cause (a name nobody has), but
39
- the two vocabularies are still two.
40
- """
41
- from fastapi import APIRouter, Body, Depends
42
-
43
- import core.shares as shares
44
- import core.users as users
45
- from deps import Session, err, require_session
46
- # ⭐ W32-T28 (C3) — the SHARE notification's topic word, imported from the module that CLASSIFIES
47
- # it (`routes_alerts.notification_view`) rather than typed again here. The producer and the
48
- # reader agreeing about one string is the whole difference between an Inbox row that opens the
49
- # shared database and one that is quietly unclickable.
50
- from routes_alerts import SHARE_TOPIC as _SHARE_TOPIC
51
-
52
- router = APIRouter(prefix="/api/v1")
53
-
54
-
55
- def _kind_or_400(raw):
56
- try:
57
- return shares._check_kind(raw)
58
- except ValueError as e:
59
- raise err(400, "bad_kind", str(e))
60
-
61
-
62
- # ── ⭐⭐ WAVE 32 · T26 (owner item 18, ruling R12) — THE WALL THIS FILE SAID IT HAD ─────────────
63
- #
64
- # `put_share`'s comment used to justify the first-claim rule with *"reaching this route at all
65
- # means passing the surface's own wall"*. **There was no such wall.** `kind` and `oid` are free
66
- # strings off the URL and the only dependency was `require_session`, so any signed-in account
67
- # could `PUT` a grant on an id it had never seen. Because the 403 sat behind `if rec["owner"]`,
68
- # an object with no grant record skipped the check entirely and the caller was stamped OWNER —
69
- # sticky, so **the real creator was then refused on their own view, permanently.** Driven, not
70
- # argued: `waves/wave32/sharing-audit.md` S-1 carries the four-step transcript.
71
- #
72
- # ⚠ AND IT WAS SILENT ON BOTH SIDES. The claimant does not even see the object in their own
73
- # "Shared with me" (`shared_with` excludes what you own), so nothing appears anywhere until the
74
- # victim next opens the dialog.
75
-
76
- #: The built-in grid topics. A view or folder lives in `{topic}_table_workspace`, and the share
77
- #: route is not told which topic — so resolving one means asking each.
78
- _BUILTIN_TOPICS = ("customer", "product")
79
-
80
-
81
- def _topics(session):
82
- """Every topic whose workspace could hold a view or folder for this tenant.
83
-
84
- ⚠ `all_defs`, never `all_tables` — the latter is the whole 28.6 MB row payload (~703 ms on
85
- tenant #0) to answer a question about KEYS (D-185).
86
- """
87
- try:
88
- import core.user_tables as ut
89
- return (*_BUILTIN_TOPICS, *(ut.all_defs(st=session.runtime) or {}))
90
- except Exception: # noqa: BLE001
91
- return _BUILTIN_TOPICS
92
-
93
-
94
- def _owns_object(session, kind, oid):
95
- """May this caller CLAIM an object that has no grant record yet — i.e. do they own it?
96
-
97
- ⛔ THIS GUARDS THE CLAIM, NOT THE READ, AND THAT IS DELIBERATE. Resolving a view means asking
98
- each topic's workspace in turn, which is N store reads; making every share call pay that
99
- would put a loop on a route the manage-access dialog opens. The dangerous path is the one
100
- where a caller is about to be stamped OWNER of something nobody owns — so the resolution runs
101
- exactly there, and the common path (a record exists, `may_administer` decides) is untouched.
102
- """
103
- if session.admin:
104
- return True
105
- if kind == "database":
106
- # ⚠ `may_open` is THE resolver for a user table (its own docstring says so) and already
107
- # admits creator, admin, or a `database` grantee. Re-implementing "who owns a table"
108
- # here would be the second definition this wave keeps finding.
109
- try:
110
- import core.user_tables as ut
111
- return bool(ut.may_open(oid, session.uname, is_admin=session.admin,
112
- st=session.runtime))
113
- except Exception: # noqa: BLE001
114
- return False
115
- try:
116
- import core.table_store as table_store
117
- except Exception: # noqa: BLE001
118
- return False
119
- for topic in _topics(session):
120
- try:
121
- ops = table_store.make(f"{topic}_table_workspace", st=session.runtime)
122
- hit = ops.find_view(oid) if kind == "view" else ops.find_folder(oid)
123
- except Exception: # noqa: BLE001
124
- continue
125
- if hit:
126
- # `find_view`/`find_folder` answer `(owner_username, …)`. The claim belongs to the
127
- # person whose personal stratum holds it — anybody else reaching this line is
128
- # exactly the case S-1 describes.
129
- return str(hit[0]) == str(session.uname)
130
- return False
131
-
132
-
133
- def _can_see_object(session, kind, oid):
134
- """May this caller READ an object's grant list — i.e. can they reach the object at all?
135
-
136
- ⛔⛔ THIS IS DELIBERATELY WIDER THAN {@link _owns_object}, AND CONFLATING THE TWO IS A
137
- REGRESSION I SHIPPED AND CAUGHT. The first version of T26 guarded BOTH doors with the
138
- ownership test, which reads sensibly and is wrong for the read, because **`find_view` searches
139
- PERSONAL STRATA ONLY** (its own docstring says so). So a view living in alice's stratum with
140
- `permissions.edit = "collaborative"` and no grant record yet — a view bob **can open and edit
141
- in the grid** — answered `404` when bob opened its manage-access dialog. Measured before
142
- fixing: `table_store._may_see(view, "bob") is True` while `GET /share/view/vc` said
143
- `404 no_object`.
144
- ⚠ THAT IS THE AUDIT'S OWN S-4 BITING THE AUDIT'S OWN FIX: two systems answer "is this shared",
145
- and the wall consulted the grant registry (system A) plus stratum ownership, never the view's
146
- `permissions` (system B) — which is the one that actually decides who may OPEN it.
147
- ⚠ And it hides the ANSWER, not just the editor. `ViewSidebar`'s Share row is deliberately not
148
- gated on edit rights because *"hiding the row from everyone else would hide the ANSWER too —
149
- 'who has this?' is a fair question for anyone the view was shared with"*. A 404 there tells a
150
- legitimate collaborator their view does not exist.
151
-
152
- ⛔ THE CLAIM KEEPS THE NARROW TEST. Being able to SEE an object must not let you become its
153
- owner — that is S-1, and widening this predicate onto `put_share` would re-open it.
154
- """
155
- if _owns_object(session, kind, oid):
156
- return True
157
- if kind != "view":
158
- # A folder carries no per-object visibility flag of its own, and a database's `may_open`
159
- # (inside `_owns_object`) already admits grantees. Nothing wider to ask.
160
- return False
161
- try:
162
- import core.table_store as table_store
163
- for topic in _topics(session):
164
- hit = table_store.make(f"{topic}_table_workspace", st=session.runtime).find_view(oid)
165
- if hit:
166
- return bool(table_store._may_see(hit[1] if len(hit) > 1 else {},
167
- session.uname, is_admin=session.admin))
168
- except Exception: # noqa: BLE001
169
- return False
170
- return False
171
-
172
-
173
- def _entries_or_400(session, entries):
174
- """Validate a grant list against the tenant's REAL, ACTIVE accounts — and refuse BY NAME.
175
-
176
- ⛔ `core.shares._clean_entries` silently drops junk, and its docstring argues that correctly:
177
- a UI mid-save must not lose the whole list to one malformed row. **But it validates the SHAPE
178
- of a string and the role word — never that the user EXISTS, is ACTIVE, or is in this tenant**,
179
- so a typo'd name is stored, reported as a successful save, and never reaches anybody. The
180
- sharer believes the person has access. That is item 18's plain reading.
181
- ⚠ The correct population is computed THREE FUNCTIONS BELOW and served to the picker
182
- (`_people`). One route, two populations, and the write door was the permissive one.
183
- ⚠ `*` (everyone) is not a user and is admitted deliberately — it is R10's vocabulary for
184
- "every account that can already open the surface".
185
- """
186
- known = {p["username"].strip().lower() for p in _people(session.tenant)}
187
- unknown = []
188
- for e in entries or ():
189
- if not isinstance(e, dict):
190
- continue
191
- user = str(e.get("user") or "").strip().lower()
192
- if user and user != shares.EVERYONE and user not in known:
193
- unknown.append(user)
194
- if unknown:
195
- raise err(400, "unknown_people",
196
- "no active account in this workspace is named "
197
- + ", ".join(sorted(set(unknown)))
198
- + " — nothing was shared. Pick people from the list rather than typing a name.")
199
-
200
-
201
- @router.get("/share/mine")
202
- def my_shares(session: Session = Depends(require_session)):
203
- """Everything shared WITH me, by kind — the "Shared with me" rail section (R10).
204
-
205
- Registered before `/share/{kind}/{oid}` so the literal path wins the match; FastAPI resolves
206
- in declaration order and `mine` would otherwise be read as a `kind`, answering 400 for a URL
207
- that is not malformed at all.
208
- """
209
- return shares.shared_with(session.uname, st=session.runtime)
210
-
211
-
212
- @router.get("/share/{kind}/{oid}")
213
- def get_share(kind: str, oid: str, session: Session = Depends(require_session)):
214
- kind = _kind_or_400(kind)
215
- rec = shares.grants(kind, oid, st=session.runtime)
216
- role = shares.role_for(kind, oid, session.uname, is_admin=session.admin, st=session.runtime)
217
- may_admin = shares.may_administer(kind, oid, session.uname, is_admin=session.admin,
218
- st=session.runtime)
219
- # ⭐ W32-T26 (audit S-3) — A STRANGER LEARNS NOTHING. This route used to answer for ANY id:
220
- # who owns it, everyone it is granted to, and the tenant's whole username↔name directory —
221
- # to any signed-in session, about objects it cannot open. Now a caller with no role on an
222
- # object must prove they can reach it, and gets a 404 otherwise: the same answer a
223
- # non-existent id gives, so the route cannot be used to probe which ids are real.
224
- # ⚠ `role is None` is the cheap pre-test, so the N-topic resolution below runs only for a
225
- # caller who has no relationship with the object at all.
226
- if role is None and not _can_see_object(session, kind, oid):
227
- raise err(404, "no_object", "no such item, or it is not shared with this account")
228
- return {
229
- **rec,
230
- "role": role,
231
- "mayAdminister": may_admin,
232
- # ⚠ WAVE 21 (C1 identity fix): grant entries BIND on USERNAMES, so the picker must carry
233
- # them. `assignable_people` serves bare display names because `user`-kind CELLS store
234
- # display names — that list's shape cannot change without migrating cell values — so
235
- # this route serves objects of its own. Existing grants that were written as lowercased
236
- # display names are normalised by the wave-21 cleanup script.
237
- # ⭐ W32-T26 (audit S-3) — the roster is the EDITOR's data, so it rides only for a caller
238
- # who may open the editor. A read-only grantee gets the grant list (their fair question is
239
- # "who else has this?") and not a directory of every account in the workspace.
240
- "people": _people(session.tenant) if may_admin else [],
241
- }
242
-
243
-
244
- def _people(tenant):
245
- """[{username, name}] for this tenant — same population as `assignable_people`, with the
246
- BINDING identity alongside the display one."""
247
- try:
248
- reg = users.registry() or {}
249
- except Exception:
250
- return []
251
- want = str(tenant or '').strip().lower()
252
- out = []
253
- for uname, u in reg.items():
254
- if not isinstance(u, dict) or u.get('active') is False:
255
- continue
256
- if want and str(u.get('tenant') or 'royal-imports').strip().lower() != want:
257
- continue
258
- out.append({"username": str(uname), "name": str(u.get('name') or uname)})
259
- return sorted(out, key=lambda p: p["name"].lower())
260
-
261
-
262
- @router.put("/share/{kind}/{oid}")
263
- def put_share(kind: str, oid: str, body: dict = Body(default=None),
264
- session: Session = Depends(require_session)):
265
- kind = _kind_or_400(kind)
266
- body = body or {}
267
- rec = shares.grants(kind, oid, st=session.runtime)
268
- # An object with NO grant record yet has no owner — the first person to share it claims it.
269
- # That is safe because reaching this route at all means passing the surface's own wall, and
270
- # the alternative (refusing until somebody seeds an owner) would make a brand-new folder
271
- # unshareable by the person who just made it.
272
- if rec["owner"]:
273
- if not shares.may_administer(kind, oid, session.uname, is_admin=session.admin,
274
- st=session.runtime):
275
- raise err(403, "not_owner",
276
- "only the owner of this item (or an administrator) can change who it is "
277
- "shared with")
278
- # ⛔⛔ W32-T26 (audit S-1) — THE CLAIM NOW HAS A PRECONDITION. An object with no grant record
279
- # is still claimed by the first person to share it — that rule is right, and refusing until
280
- # somebody seeds an owner would make a brand-new folder unshareable by the person who just
281
- # made it. What was missing is the half the old comment ASSERTED and the code never did: the
282
- # claimant has to be able to reach the object. Without this, any signed-in account could
283
- # stamp itself owner of an id it had never seen and lock the real creator out for good.
284
- elif not _owns_object(session, kind, oid):
285
- raise err(404, "no_object", "no such item, or it is not shared with this account")
286
- entries = body.get("entries")
287
- if not isinstance(entries, list):
288
- raise err(400, "bad_entries",
289
- "entries must be a list of {user, role} — send [] to un-share, which is how "
290
- "revoking is expressed")
291
- _entries_or_400(session, entries)
292
- out = shares.set_grants(kind, oid, entries, owner=rec["owner"] or session.uname,
293
- st=session.runtime)
294
- _notify_new_grantees(session, kind, oid, before=rec["entries"], after=out.get("entries") or [])
295
- return out
296
-
297
-
298
- def _notify_new_grantees(session, kind, oid, before, after):
299
- """⭐⭐ W32-T28 (owner item 18's last clause, contract C3) — tell the RECEIVER, in their Inbox.
300
-
301
- Owner item 18 ends *"being shared a database notifies the receiver"*. Until now sharing was
302
- silent: the grant landed in a rail section the receiver had to notice on their own, which is
303
- why "I shared it with you" and "I never saw it" were both true.
304
-
305
- ⛔ WRITTEN ON THE SHARE, NEVER POLLED. `/notifications` re-evaluates view-ALERTS on read
306
- because an alert is a live question about rows; a share is an EVENT that happened once, and
307
- polling for it would mean re-deriving "was this new?" on every inbox open — the diff below
308
- only exists here, at the moment the set changes.
309
-
310
- ⚠ ONLY THE NEWLY ADDED. `PUT` REPLACES the whole entry set (revoking is expressed by absence),
311
- so every save re-sends everyone who was already there. Diffing against `before` is what stops
312
- a rename or a role change from ringing the bell for people whose access did not change.
313
- ⚠ `*` IS NOT NOTIFIED: there is no user to name, and minting one notification per account in
314
- the tenant on a single click is a broadcast nobody asked for. The rail still shows it.
315
- ⚠ IT NEVER RAISES. A notification that fails must not fail the share that triggered it — the
316
- grant is the user's actual intent, and `core.alerts.notify` writes with `flush='async'`.
317
- """
318
- try:
319
- was = {e.get("user") for e in (before or ()) if isinstance(e, dict)}
320
- fresh = [str(e.get("user")) for e in (after or ())
321
- if isinstance(e, dict) and e.get("user") not in was
322
- and e.get("user") != shares.EVERYONE]
323
- if not fresh:
324
- return
325
- import core.alerts as alerts
326
-
327
- label, route, view_id = _object_ref(session, kind, oid)
328
- if not route:
329
- # ⛔ NO ROUTE, NO NOTIFICATION — the receiver would get a row that opens nothing, and
330
- # `notification_view` would have to invent a target. Silence is the honest answer
331
- # here; the rail still shows the grant under "Shared with me".
332
- return
333
- sharer = str(session.user.get("name") or session.uname)
334
- for user in fresh:
335
- # ⚠ THE SHAPE IS `routes_alerts.notification_view`'s SHARE BRANCH, and the two must
336
- # agree or the Inbox row is unclickable: `topic` selects the branch and `key` becomes
337
- # `alertId`, which that branch reads as the id to open. Both constants are IMPORTED
338
- # from there rather than typed again — one vocabulary, one owner.
339
- # ⭐⭐ W33-T28 (`ASK C-14`, answered) — `actor` IS THE SENDER, AND IT IS THE ONLY WAY
340
- # THE INBOX CAN NAME ONE. An alert and an automation have no person behind them and
341
- # are honestly named by their machine; a SHARE has a real person, and only this call
342
- # site knows who. ⛔ It is passed as its OWN field rather than recovered from the
343
- # `detail` prose below: a sender parsed out of "<name> shared this with you" breaks
344
- # the first time the sentence is reworded, silently, in the header
345
- # [[grep-output-is-not-source]]. The prose stays as the body; this is the From.
346
- alerts.notify(user, label, topic=_SHARE_TOPIC, key=route, row_id=view_id,
347
- detail=f"{sharer} shared this with you", actor=sharer,
348
- st=session.runtime)
349
- except Exception: # noqa: BLE001
350
- return
351
-
352
-
353
- def _object_ref(session, kind, oid):
354
- """`(label, route, view_id)` — what to CALL the shared thing, and where it OPENS.
355
-
356
- ⛔ THE ROUTE IS RESOLVED HERE, NOT SHAPED IN THE CONSUMER, AND THE FIRST VERSION GOT IT
357
- WRONG: it put the raw `oid` in the notification's key, so a shared VIEW produced
358
- `target: {module: "database", id: "view_42"}` — an instruction to open a database named
359
- `view_42`. It read perfectly in the payload and would have opened nothing. **A view is not
360
- addressable on its own; it is a SELECTION inside a topic's grid**, so the pair is what has to
361
- travel. Caught by looking at the notification the driver actually produced, not by reading
362
- the code back.
363
-
364
- ⚠ `label` never falls back to a raw id. A notification headed `ut_leads_3f2a` tells the
365
- receiver nothing they can act on, and the id is already in the target.
366
- ⚠ An unresolvable object answers `route=None`, and the caller then sends NOTHING rather than
367
- a row that opens nowhere.
368
- """
369
- try:
370
- if kind == "database":
371
- import core.user_tables as ut
372
- defn = (ut.all_defs(st=session.runtime) or {}).get(str(oid)) or {}
373
- # A user table IS its own route key in both vocabularies (`route_for_topic`).
374
- return (str(defn.get("label") or "").strip() or "A database", str(oid), "")
375
- import core.table_store as table_store
376
- from routes_alerts import route_for_topic
377
- for topic in _topics(session):
378
- ops = table_store.make(f"{topic}_table_workspace", st=session.runtime)
379
- hit = ops.find_view(oid) if kind == "view" else ops.find_folder(oid)
380
- if not hit:
381
- continue
382
- route = route_for_topic(topic)
383
- if not route:
384
- break
385
- row = hit[1] if len(hit) > 1 else {}
386
- name = str((row or {}).get("name") or "").strip()
387
- # ⚠ Only a VIEW carries a selection. A folder is a rail grouping, so the target opens
388
- # the grid and stops there rather than naming a view the receiver did not get.
389
- return (name or ("A view" if kind == "view" else "A folder"),
390
- route, str(oid) if kind == "view" else "")
391
- except Exception: # noqa: BLE001
392
- pass
393
- return ({"view": "A view", "folder": "A folder"}.get(kind, "An item"), None, "")
 
1
+ """routes_shares.py — the manage-access surface (wave 20, owner ruling R10, contract C-SHARE).
2
+
3
+ GET /api/v1/share/{kind}/{oid} -> {owner, entries:[{user,role}], mayAdminister, people}
4
+ PUT /api/v1/share/{kind}/{oid} <- {entries:[{user,role}]} (REPLACES the set)
5
+ GET /api/v1/share/mine -> {view:[id], folder:[id], database:[id]}
6
+
7
+ `kind` ∈ view | folder | database. Roles are `view` | `edit` — the same two words the view rail
8
+ already speaks, now extended to folders and databases so there is ONE vocabulary in the UI
9
+ (R10: "the same picker views use").
10
+
11
+ ⛔ **RE-SHARING IS THE OWNER'S, AND THAT IS ENFORCED HERE, NOT IN THE CLIENT.** `PUT` requires
12
+ `shares.may_administer` (owner or admin). A collaborator with `edit` may change an object's
13
+ CONTENT and may not change who else can reach it — otherwise anyone you shared a view with could
14
+ widen it to everyone, or grant themselves ownership and lock you out. The client greys the editor
15
+ for non-administrators; that is a courtesy, and this check is the wall.
16
+
17
+ ⚠ **THE GRANT NEVER WIDENS PAST THE MODULE WALL — ON A GOVERNED MODULE.** `*` ("everyone") means
18
+ every account that can already open the surface: `require_session` plus the topic's own gate run
19
+ first, and for `customer_data` / `product_data` the receiver's own row scope and hidden-field
20
+ closure run BEFORE any foreign view is merged. Sharing there can only narrow-or-equal the set that
21
+ could already reach the data ([[aios-permissioning]]).
22
+
23
+ ⛔⛔ **AND THAT SENTENCE IS FALSE FOR `kind='database'`, WHICH IS WHY IT NOW SAYS "ON A GOVERNED
24
+ MODULE" (W32-T26, audit S-8).** `routes_admin._PERM_MODULES` is `("customer_data","product_data")`
25
+ and `_clean_perms` **400s** on anything else, so **no row filter and no hidden field can even be
26
+ DECLARED for a `ut_*` database** — `routes_tables.py` makes zero `perm_scope` calls and passes
27
+ `hidden_keys=frozenset()`. There is no module wall behind a user table for a grant to be bounded
28
+ by: **this registry IS the wall.** So a `database` grant is ALL-OR-NOTHING — every row, every
29
+ column — and an `*` database grant admits every account in the tenant to all of it.
30
+ That is a real capability, deliberately kept; what was wrong was a docstring promising a second
31
+ wall that does not exist for this kind. Scoping user tables is booked, not done
32
+ (`waves/wave32/sharing-audit.md` S-8).
33
+
34
+ ⚠ **TWO SYSTEMS ANSWER "IS THIS SHARED", AND THEY ARE NOT THE SAME ONE (audit S-4).** THIS
35
+ registry decides who appears in *"Shared with me"* and who may re-share. **`table_store.is_shared`
36
+ — the view's own `permissions` — is what actually decides who may OPEN a view.** A grant here
37
+ whose object is invisible under that one is a row in a list that opens a refusal, which is what
38
+ made item 18 worth auditing. `_entries_or_400` closes the common cause (a name nobody has), but
39
+ the two vocabularies are still two.
40
+ """
41
+ from fastapi import APIRouter, Body, Depends
42
+
43
+ import core.shares as shares
44
+ import core.users as users
45
+ from deps import Session, err, require_session
46
+ # ⭐ W32-T28 (C3) — the SHARE notification's topic word, imported from the module that CLASSIFIES
47
+ # it (`routes_alerts.notification_view`) rather than typed again here. The producer and the
48
+ # reader agreeing about one string is the whole difference between an Inbox row that opens the
49
+ # shared database and one that is quietly unclickable.
50
+ from routes_alerts import SHARE_TOPIC as _SHARE_TOPIC
51
+
52
+ router = APIRouter(prefix="/api/v1")
53
+
54
+
55
+ def _kind_or_400(raw):
56
+ try:
57
+ return shares._check_kind(raw)
58
+ except ValueError as e:
59
+ raise err(400, "bad_kind", str(e))
60
+
61
+
62
+ # ── ⭐⭐ WAVE 32 · T26 (owner item 18, ruling R12) — THE WALL THIS FILE SAID IT HAD ─────────────
63
+ #
64
+ # `put_share`'s comment used to justify the first-claim rule with *"reaching this route at all
65
+ # means passing the surface's own wall"*. **There was no such wall.** `kind` and `oid` are free
66
+ # strings off the URL and the only dependency was `require_session`, so any signed-in account
67
+ # could `PUT` a grant on an id it had never seen. Because the 403 sat behind `if rec["owner"]`,
68
+ # an object with no grant record skipped the check entirely and the caller was stamped OWNER —
69
+ # sticky, so **the real creator was then refused on their own view, permanently.** Driven, not
70
+ # argued: `waves/wave32/sharing-audit.md` S-1 carries the four-step transcript.
71
+ #
72
+ # ⚠ AND IT WAS SILENT ON BOTH SIDES. The claimant does not even see the object in their own
73
+ # "Shared with me" (`shared_with` excludes what you own), so nothing appears anywhere until the
74
+ # victim next opens the dialog.
75
+
76
+ #: The built-in grid topics. A view or folder lives in `{topic}_table_workspace`, and the share
77
+ #: route is not told which topic — so resolving one means asking each.
78
+ _BUILTIN_TOPICS = ("customer", "product")
79
+
80
+
81
+ def _topics(session):
82
+ """Every topic whose workspace could hold a view or folder for this tenant.
83
+
84
+ ⚠ `all_defs`, never `all_tables` — the latter is the whole 28.6 MB row payload (~703 ms on
85
+ tenant #0) to answer a question about KEYS (D-185).
86
+ """
87
+ try:
88
+ import core.user_tables as ut
89
+ return (*_BUILTIN_TOPICS, *(ut.all_defs(st=session.runtime) or {}))
90
+ except Exception: # noqa: BLE001
91
+ return _BUILTIN_TOPICS
92
+
93
+
94
+ def _owns_object(session, kind, oid):
95
+ """May this caller CLAIM an object that has no grant record yet — i.e. do they own it?
96
+
97
+ ⛔ THIS GUARDS THE CLAIM, NOT THE READ, AND THAT IS DELIBERATE. Resolving a view means asking
98
+ each topic's workspace in turn, which is N store reads; making every share call pay that
99
+ would put a loop on a route the manage-access dialog opens. The dangerous path is the one
100
+ where a caller is about to be stamped OWNER of something nobody owns — so the resolution runs
101
+ exactly there, and the common path (a record exists, `may_administer` decides) is untouched.
102
+ """
103
+ if session.admin:
104
+ return True
105
+ if kind == "database":
106
+ # ⚠ `may_open` is THE resolver for a user table (its own docstring says so) and already
107
+ # admits creator, admin, or a `database` grantee. Re-implementing "who owns a table"
108
+ # here would be the second definition this wave keeps finding.
109
+ try:
110
+ import core.user_tables as ut
111
+ return bool(ut.may_open(oid, session.uname, is_admin=session.admin,
112
+ st=session.runtime))
113
+ except Exception: # noqa: BLE001
114
+ return False
115
+ try:
116
+ import core.table_store as table_store
117
+ except Exception: # noqa: BLE001
118
+ return False
119
+ for topic in _topics(session):
120
+ try:
121
+ ops = table_store.make(f"{topic}_table_workspace", st=session.runtime)
122
+ hit = ops.find_view(oid) if kind == "view" else ops.find_folder(oid)
123
+ except Exception: # noqa: BLE001
124
+ continue
125
+ if hit:
126
+ # `find_view`/`find_folder` answer `(owner_username, …)`. The claim belongs to the
127
+ # person whose personal stratum holds it — anybody else reaching this line is
128
+ # exactly the case S-1 describes.
129
+ return str(hit[0]) == str(session.uname)
130
+ return False
131
+
132
+
133
+ def _can_see_object(session, kind, oid):
134
+ """May this caller READ an object's grant list — i.e. can they reach the object at all?
135
+
136
+ ⛔⛔ THIS IS DELIBERATELY WIDER THAN {@link _owns_object}, AND CONFLATING THE TWO IS A
137
+ REGRESSION I SHIPPED AND CAUGHT. The first version of T26 guarded BOTH doors with the
138
+ ownership test, which reads sensibly and is wrong for the read, because **`find_view` searches
139
+ PERSONAL STRATA ONLY** (its own docstring says so). So a view living in alice's stratum with
140
+ `permissions.edit = "collaborative"` and no grant record yet — a view bob **can open and edit
141
+ in the grid** — answered `404` when bob opened its manage-access dialog. Measured before
142
+ fixing: `table_store._may_see(view, "bob") is True` while `GET /share/view/vc` said
143
+ `404 no_object`.
144
+ ⚠ THAT IS THE AUDIT'S OWN S-4 BITING THE AUDIT'S OWN FIX: two systems answer "is this shared",
145
+ and the wall consulted the grant registry (system A) plus stratum ownership, never the view's
146
+ `permissions` (system B) — which is the one that actually decides who may OPEN it.
147
+ ⚠ And it hides the ANSWER, not just the editor. `ViewSidebar`'s Share row is deliberately not
148
+ gated on edit rights because *"hiding the row from everyone else would hide the ANSWER too —
149
+ 'who has this?' is a fair question for anyone the view was shared with"*. A 404 there tells a
150
+ legitimate collaborator their view does not exist.
151
+
152
+ ⛔ THE CLAIM KEEPS THE NARROW TEST. Being able to SEE an object must not let you become its
153
+ owner — that is S-1, and widening this predicate onto `put_share` would re-open it.
154
+ """
155
+ if _owns_object(session, kind, oid):
156
+ return True
157
+ if kind != "view":
158
+ # A folder carries no per-object visibility flag of its own, and a database's `may_open`
159
+ # (inside `_owns_object`) already admits grantees. Nothing wider to ask.
160
+ return False
161
+ try:
162
+ import core.table_store as table_store
163
+ for topic in _topics(session):
164
+ hit = table_store.make(f"{topic}_table_workspace", st=session.runtime).find_view(oid)
165
+ if hit:
166
+ return bool(table_store._may_see(hit[1] if len(hit) > 1 else {},
167
+ session.uname, is_admin=session.admin))
168
+ except Exception: # noqa: BLE001
169
+ return False
170
+ return False
171
+
172
+
173
+ def _entries_or_400(session, entries):
174
+ """Validate a grant list against the tenant's REAL, ACTIVE accounts — and refuse BY NAME.
175
+
176
+ ⛔ `core.shares._clean_entries` silently drops junk, and its docstring argues that correctly:
177
+ a UI mid-save must not lose the whole list to one malformed row. **But it validates the SHAPE
178
+ of a string and the role word — never that the user EXISTS, is ACTIVE, or is in this tenant**,
179
+ so a typo'd name is stored, reported as a successful save, and never reaches anybody. The
180
+ sharer believes the person has access. That is item 18's plain reading.
181
+ ⚠ The correct population is computed THREE FUNCTIONS BELOW and served to the picker
182
+ (`_people`). One route, two populations, and the write door was the permissive one.
183
+ ⚠ `*` (everyone) is not a user and is admitted deliberately — it is R10's vocabulary for
184
+ "every account that can already open the surface".
185
+ """
186
+ known = {p["username"].strip().lower() for p in _people(session.tenant)}
187
+ unknown = []
188
+ for e in entries or ():
189
+ if not isinstance(e, dict):
190
+ continue
191
+ user = str(e.get("user") or "").strip().lower()
192
+ if user and user != shares.EVERYONE and user not in known:
193
+ unknown.append(user)
194
+ if unknown:
195
+ raise err(400, "unknown_people",
196
+ "no active account in this workspace is named "
197
+ + ", ".join(sorted(set(unknown)))
198
+ + " — nothing was shared. Pick people from the list rather than typing a name.")
199
+
200
+
201
+ @router.get("/share/mine")
202
+ def my_shares(session: Session = Depends(require_session)):
203
+ """Everything shared WITH me, by kind — the "Shared with me" rail section (R10).
204
+
205
+ Registered before `/share/{kind}/{oid}` so the literal path wins the match; FastAPI resolves
206
+ in declaration order and `mine` would otherwise be read as a `kind`, answering 400 for a URL
207
+ that is not malformed at all.
208
+ """
209
+ return shares.shared_with(session.uname, st=session.runtime)
210
+
211
+
212
+ @router.get("/share/{kind}/{oid}")
213
+ def get_share(kind: str, oid: str, session: Session = Depends(require_session)):
214
+ kind = _kind_or_400(kind)
215
+ rec = shares.grants(kind, oid, st=session.runtime)
216
+ role = shares.role_for(kind, oid, session.uname, is_admin=session.admin, st=session.runtime)
217
+ may_admin = shares.may_administer(kind, oid, session.uname, is_admin=session.admin,
218
+ st=session.runtime)
219
+ # ⭐ W32-T26 (audit S-3) — A STRANGER LEARNS NOTHING. This route used to answer for ANY id:
220
+ # who owns it, everyone it is granted to, and the tenant's whole username↔name directory —
221
+ # to any signed-in session, about objects it cannot open. Now a caller with no role on an
222
+ # object must prove they can reach it, and gets a 404 otherwise: the same answer a
223
+ # non-existent id gives, so the route cannot be used to probe which ids are real.
224
+ # ⚠ `role is None` is the cheap pre-test, so the N-topic resolution below runs only for a
225
+ # caller who has no relationship with the object at all.
226
+ if role is None and not _can_see_object(session, kind, oid):
227
+ raise err(404, "no_object", "no such item, or it is not shared with this account")
228
+ return {
229
+ **rec,
230
+ "role": role,
231
+ "mayAdminister": may_admin,
232
+ # ⚠ WAVE 21 (C1 identity fix): grant entries BIND on USERNAMES, so the picker must carry
233
+ # them. `assignable_people` serves bare display names because `user`-kind CELLS store
234
+ # display names — that list's shape cannot change without migrating cell values — so
235
+ # this route serves objects of its own. Existing grants that were written as lowercased
236
+ # display names are normalised by the wave-21 cleanup script.
237
+ # ⭐ W32-T26 (audit S-3) — the roster is the EDITOR's data, so it rides only for a caller
238
+ # who may open the editor. A read-only grantee gets the grant list (their fair question is
239
+ # "who else has this?") and not a directory of every account in the workspace.
240
+ "people": _people(session.tenant) if may_admin else [],
241
+ }
242
+
243
+
244
+ def _people(tenant):
245
+ """[{username, name}] for this tenant — same population as `assignable_people`, with the
246
+ BINDING identity alongside the display one."""
247
+ try:
248
+ reg = users.registry() or {}
249
+ except Exception:
250
+ return []
251
+ want = str(tenant or '').strip().lower()
252
+ out = []
253
+ for uname, u in reg.items():
254
+ if not isinstance(u, dict) or u.get('active') is False:
255
+ continue
256
+ if want and str(u.get('tenant') or 'royal-imports').strip().lower() != want:
257
+ continue
258
+ out.append({"username": str(uname), "name": str(u.get('name') or uname)})
259
+ return sorted(out, key=lambda p: p["name"].lower())
260
+
261
+
262
+ @router.put("/share/{kind}/{oid}")
263
+ def put_share(kind: str, oid: str, body: dict = Body(default=None),
264
+ session: Session = Depends(require_session)):
265
+ kind = _kind_or_400(kind)
266
+ body = body or {}
267
+ rec = shares.grants(kind, oid, st=session.runtime)
268
+ # An object with NO grant record yet has no owner — the first person to share it claims it.
269
+ # That is safe because reaching this route at all means passing the surface's own wall, and
270
+ # the alternative (refusing until somebody seeds an owner) would make a brand-new folder
271
+ # unshareable by the person who just made it.
272
+ if rec["owner"]:
273
+ if not shares.may_administer(kind, oid, session.uname, is_admin=session.admin,
274
+ st=session.runtime):
275
+ raise err(403, "not_owner",
276
+ "only the owner of this item (or an administrator) can change who it is "
277
+ "shared with")
278
+ # ⛔⛔ W32-T26 (audit S-1) — THE CLAIM NOW HAS A PRECONDITION. An object with no grant record
279
+ # is still claimed by the first person to share it — that rule is right, and refusing until
280
+ # somebody seeds an owner would make a brand-new folder unshareable by the person who just
281
+ # made it. What was missing is the half the old comment ASSERTED and the code never did: the
282
+ # claimant has to be able to reach the object. Without this, any signed-in account could
283
+ # stamp itself owner of an id it had never seen and lock the real creator out for good.
284
+ elif not _owns_object(session, kind, oid):
285
+ raise err(404, "no_object", "no such item, or it is not shared with this account")
286
+ entries = body.get("entries")
287
+ if not isinstance(entries, list):
288
+ raise err(400, "bad_entries",
289
+ "entries must be a list of {user, role} — send [] to un-share, which is how "
290
+ "revoking is expressed")
291
+ _entries_or_400(session, entries)
292
+ out = shares.set_grants(kind, oid, entries, owner=rec["owner"] or session.uname,
293
+ st=session.runtime)
294
+ _notify_new_grantees(session, kind, oid, before=rec["entries"], after=out.get("entries") or [])
295
+ return out
296
+
297
+
298
+ def _notify_new_grantees(session, kind, oid, before, after):
299
+ """⭐⭐ W32-T28 (owner item 18's last clause, contract C3) �� tell the RECEIVER, in their Inbox.
300
+
301
+ Owner item 18 ends *"being shared a database notifies the receiver"*. Until now sharing was
302
+ silent: the grant landed in a rail section the receiver had to notice on their own, which is
303
+ why "I shared it with you" and "I never saw it" were both true.
304
+
305
+ ⛔ WRITTEN ON THE SHARE, NEVER POLLED. `/notifications` re-evaluates view-ALERTS on read
306
+ because an alert is a live question about rows; a share is an EVENT that happened once, and
307
+ polling for it would mean re-deriving "was this new?" on every inbox open — the diff below
308
+ only exists here, at the moment the set changes.
309
+
310
+ ⚠ ONLY THE NEWLY ADDED. `PUT` REPLACES the whole entry set (revoking is expressed by absence),
311
+ so every save re-sends everyone who was already there. Diffing against `before` is what stops
312
+ a rename or a role change from ringing the bell for people whose access did not change.
313
+ ⚠ `*` IS NOT NOTIFIED: there is no user to name, and minting one notification per account in
314
+ the tenant on a single click is a broadcast nobody asked for. The rail still shows it.
315
+ ⚠ IT NEVER RAISES. A notification that fails must not fail the share that triggered it — the
316
+ grant is the user's actual intent, and `core.alerts.notify` writes with `flush='async'`.
317
+ """
318
+ try:
319
+ was = {e.get("user") for e in (before or ()) if isinstance(e, dict)}
320
+ fresh = [str(e.get("user")) for e in (after or ())
321
+ if isinstance(e, dict) and e.get("user") not in was
322
+ and e.get("user") != shares.EVERYONE]
323
+ if not fresh:
324
+ return
325
+ import core.alerts as alerts
326
+
327
+ label, route, view_id = _object_ref(session, kind, oid)
328
+ if not route:
329
+ # ⛔ NO ROUTE, NO NOTIFICATION — the receiver would get a row that opens nothing, and
330
+ # `notification_view` would have to invent a target. Silence is the honest answer
331
+ # here; the rail still shows the grant under "Shared with me".
332
+ return
333
+ sharer = str(session.user.get("name") or session.uname)
334
+ for user in fresh:
335
+ # ⚠ THE SHAPE IS `routes_alerts.notification_view`'s SHARE BRANCH, and the two must
336
+ # agree or the Inbox row is unclickable: `topic` selects the branch and `key` becomes
337
+ # `alertId`, which that branch reads as the id to open. Both constants are IMPORTED
338
+ # from there rather than typed again — one vocabulary, one owner.
339
+ alerts.notify(user, label, topic=_SHARE_TOPIC, key=route, row_id=view_id,
340
+ detail=f"{sharer} shared this with you", st=session.runtime)
341
+ except Exception: # noqa: BLE001
342
+ return
343
+
344
+
345
+ def _object_ref(session, kind, oid):
346
+ """`(label, route, view_id)` — what to CALL the shared thing, and where it OPENS.
347
+
348
+ ⛔ THE ROUTE IS RESOLVED HERE, NOT SHAPED IN THE CONSUMER, AND THE FIRST VERSION GOT IT
349
+ WRONG: it put the raw `oid` in the notification's key, so a shared VIEW produced
350
+ `target: {module: "database", id: "view_42"}` — an instruction to open a database named
351
+ `view_42`. It read perfectly in the payload and would have opened nothing. **A view is not
352
+ addressable on its own; it is a SELECTION inside a topic's grid**, so the pair is what has to
353
+ travel. Caught by looking at the notification the driver actually produced, not by reading
354
+ the code back.
355
+
356
+ ⚠ `label` never falls back to a raw id. A notification headed `ut_leads_3f2a` tells the
357
+ receiver nothing they can act on, and the id is already in the target.
358
+ ⚠ An unresolvable object answers `route=None`, and the caller then sends NOTHING rather than
359
+ a row that opens nowhere.
360
+ """
361
+ try:
362
+ if kind == "database":
363
+ import core.user_tables as ut
364
+ defn = (ut.all_defs(st=session.runtime) or {}).get(str(oid)) or {}
365
+ # A user table IS its own route key in both vocabularies (`route_for_topic`).
366
+ return (str(defn.get("label") or "").strip() or "A database", str(oid), "")
367
+ import core.table_store as table_store
368
+ from routes_alerts import route_for_topic
369
+ for topic in _topics(session):
370
+ ops = table_store.make(f"{topic}_table_workspace", st=session.runtime)
371
+ hit = ops.find_view(oid) if kind == "view" else ops.find_folder(oid)
372
+ if not hit:
373
+ continue
374
+ route = route_for_topic(topic)
375
+ if not route:
376
+ break
377
+ row = hit[1] if len(hit) > 1 else {}
378
+ name = str((row or {}).get("name") or "").strip()
379
+ # ⚠ Only a VIEW carries a selection. A folder is a rail grouping, so the target opens
380
+ # the grid and stops there rather than naming a view the receiver did not get.
381
+ return (name or ("A view" if kind == "view" else "A folder"),
382
+ route, str(oid) if kind == "view" else "")
383
+ except Exception: # noqa: BLE001
384
+ pass
385
+ return ({"view": "A view", "folder": "A folder"}.get(kind, "An item"), None, "")
 
 
 
 
 
 
 
 
api/routes_tables.py CHANGED
The diff for this file is too large to render. See raw diff
 
jobs/web_agent_job.py CHANGED
@@ -113,21 +113,6 @@ INSTALL_DEPS = os.environ.get("WEB_AGENT_INSTALL_DEPS", "1") != "0"
113
  INSTALL_BROWSER = os.environ.get("WEB_AGENT_INSTALL_BROWSER", "1") != "0"
114
  MAX_VALUE_CHARS = 8000 # one cell's worth; a page is not a payload
115
  MAX_STEPS = 20 # matches the engine's MAX_ACTIONS — one flow, one job
116
- #: ⛔ D-193 — HOW OLD A CHECKPOINT MAY BE AND STILL BE SERVED, in hours. **Deliberately SHORT.**
117
- #: The key is content-addressed (`runId` + a hash of the steps), so a scheduled automation with a
118
- #: stable run id hits the same key every time it runs; with no ceiling it would be served the same
119
- #: stored page for as long as the file existed, which is a measurement of a web page presented with
120
- #: today's date on it. Six hours makes a same-morning retry cheap and a next-day run honest.
121
- #: ⚠ A CEILING, NOT A CACHE POLICY. The point of the checkpoint is that a RETRY of a job that
122
- #: already paid for a fetch does not pay again; that window is minutes, not days.
123
- try:
124
- CHECKPOINT_MAX_AGE_H = float(os.environ.get("WEB_AGENT_CHECKPOINT_MAX_AGE_H", "6") or 6)
125
- except (TypeError, ValueError):
126
- CHECKPOINT_MAX_AGE_H = 6.0
127
- #: How many expired `runs/*.json` one job may delete, in ONE commit. ⛔ Bounded because the repo
128
- #: already spends a commit per run against HF's 256-commits/hr budget, and a sweep that spent one
129
- #: commit per FILE would be a denial of service on the store it is tidying.
130
- CHECKPOINT_SWEEP_MAX = 200
131
 
132
 
133
  def _fail(sentence: str, *, kind: str = "refused", steps=None) -> None:
@@ -177,8 +162,6 @@ def _checkpoint(blob: str, payload: dict) -> None:
177
  repo_id=CHECKPOINT_REPO, repo_type="dataset",
178
  commit_message=f"web-agent run {run_id}")
179
  say(f"CHECKPOINT wrote runs/{run_id}.json to {CHECKPOINT_REPO}")
180
- # ⚠ AFTER the write, never before: the sweep is housekeeping and the result is the job.
181
- _sweep_expired()
182
  except Exception as exc: # noqa: BLE001 — a boundary
183
  # ⛔ CATCH EVERYTHING. This is a report-don't-raise boundary and the ways an upload can
184
  # fail are not enumerable (429 commit budget, 403 scope, DNS, a repo that does not
@@ -188,94 +171,8 @@ def _checkpoint(blob: str, payload: dict) -> None:
188
  f"the stdout frame still carries the result")
189
 
190
 
191
- def _too_old(payload: dict) -> str:
192
- """"" if this checkpoint may still be served, else the SENTENCE saying why it may not.
193
-
194
- ⛔ D-193 — A RESUME THAT IS A YEAR OLD IS NOT A RESUME, IT IS A STALE ANSWER SERVED AS FRESH.
195
- The key is deliberately content-addressed (`runId` + a hash of the steps), so a scheduled
196
- automation reusing a stable run id resumed the SAME STORED PAGE for as long as the file
197
- existed. `finishedAt` was written by `_emit` from the first day and read by nothing.
198
- ⚠ AN UNREADABLE OR ABSENT TIMESTAMP IS TREATED AS TOO OLD, not as fresh. A checkpoint written
199
- before this field existed carries no date, and "no date" must not be the one input that gets
200
- served forever — fail toward re-fetching, which costs a job, rather than toward a wrong answer,
201
- which costs a number nobody can explain.
202
- """
203
- stamp = str(payload.get("finishedAt") or "")
204
- if not stamp:
205
- return "it carries no finishedAt, so its age cannot be established"
206
- try:
207
- when = datetime.fromisoformat(stamp)
208
- if when.tzinfo is None:
209
- when = when.replace(tzinfo=timezone.utc)
210
- except Exception: # noqa: BLE001
211
- return f"its finishedAt ({stamp[:40]}) is not a date this job can read"
212
- hours = (datetime.now(timezone.utc) - when).total_seconds() / 3600.0
213
- if hours > CHECKPOINT_MAX_AGE_H:
214
- return (f"it finished {hours:.1f} h ago and the ceiling is {CHECKPOINT_MAX_AGE_H} h "
215
- f"(WEB_AGENT_CHECKPOINT_MAX_AGE_H)")
216
- return ""
217
-
218
-
219
- def _sweep_expired() -> None:
220
- """Delete checkpoints past the ceiling, ONE COMMIT, at most `CHECKPOINT_SWEEP_MAX` of them.
221
-
222
- ⛔ D-193's SECOND CLAUSE, and it is a different problem from the first. Ageing out the READ
223
- stops a stale answer being served; it does not stop the repo growing one file per run forever.
224
- ⛔ ONE COMMIT FOR THE WHOLE SWEEP. `upload_file`/`delete_file` each spend a commit, and this
225
- repo already spends one per run against HF's 256-per-hour budget — a per-file sweep would
226
- exhaust the budget it exists to protect, and the failure would land on the next run's WRITE.
227
- `create_commit` with a list of delete operations is one commit whatever the count.
228
- ⚠ AGE IS READ FROM THE LISTING, not by downloading each file. `last_commit` is what the API
229
- gives cheaply; downloading N files to read a timestamp inside them would make a tidy-up cost
230
- more than the runs it is tidying.
231
- ⚠ REPORT-DON'T-RAISE, like `_checkpoint`: this runs AFTER the result is in hand, and a
232
- housekeeping failure must never destroy a result that has already been paid for.
233
- """
234
- if not (CHECKPOINT_REPO and HF_TOKEN):
235
- return
236
- try:
237
- from huggingface_hub import CommitOperationDelete, HfApi
238
- api = HfApi(token=HF_TOKEN)
239
- cutoff = datetime.now(timezone.utc).timestamp() - CHECKPOINT_MAX_AGE_H * 3600.0
240
- doomed = []
241
- for info in api.list_repo_tree(CHECKPOINT_REPO, repo_type="dataset", path_in_repo="runs",
242
- expand=True, recursive=False):
243
- path = getattr(info, "path", "")
244
- when = getattr(getattr(info, "last_commit", None), "date", None)
245
- if not path.endswith(".json") or when is None:
246
- continue
247
- if when.timestamp() < cutoff:
248
- doomed.append(path)
249
- if len(doomed) >= CHECKPOINT_SWEEP_MAX:
250
- break
251
- if not doomed:
252
- return
253
- api.create_commit(repo_id=CHECKPOINT_REPO, repo_type="dataset",
254
- operations=[CommitOperationDelete(path_in_repo=p) for p in doomed],
255
- commit_message=f"sweep {len(doomed)} checkpoints past "
256
- f"{CHECKPOINT_MAX_AGE_H} h")
257
- say(f"CHECKPOINT swept {len(doomed)} expired run(s) in one commit")
258
- except Exception as exc: # noqa: BLE001 — a boundary
259
- say(f"CHECKPOINT sweep skipped ({type(exc).__name__}: {str(exc)[:160]}) — "
260
- f"the result is unaffected")
261
-
262
-
263
  def _resume(run_id: str):
264
- """A prior run's SUCCESSFUL and RECENT result, if one exists. None otherwise.
265
-
266
- ⛔ D-199 — A FAILED RUN'S CHECKPOINT USED TO REPLAY FOREVER. `_fail` checkpoints too
267
- (`_fail` → `_emit` → `_checkpoint`), so the stored envelope for a failure is exactly as
268
- truthy as one for a success; the caller tested `if prior:` and nothing ever looked at `ok`.
269
- Under a stable `runId` a failed fetch could then never be re-attempted — it replayed the
270
- stored failure unchanged, with `resumedFromCheckpoint: True` on it. The defect was LATENT
271
- only because no deployment set `WEB_AGENT_REPO`, i.e. it armed itself on the day somebody
272
- switched checkpointing on, which is the day nobody would be reading this function.
273
- ⚠ THE TEST BELONGS HERE, NOT AT THE CALL SITE. `_resume` is the one place that decides what a
274
- resumable checkpoint IS; a caller-side `if prior and prior.get("ok")` would be a second
275
- definition, and the next caller would be written from the first one.
276
- ⚠ A REFUSED CHECKPOINT SAYS SO IN THE LOG. Silently re-running looks identical to never
277
- having had a checkpoint, and those two need to be distinguishable at 3am.
278
- """
279
  if not (CHECKPOINT_REPO and HF_TOKEN and run_id):
280
  return None
281
  try:
@@ -284,21 +181,9 @@ def _resume(run_id: str):
284
  filename=f"runs/{run_id}.json", token=HF_TOKEN,
285
  force_download=True)
286
  with open(path, encoding="utf-8") as fh:
287
- prior = json.load(fh)
288
  except Exception: # noqa: BLE001
289
  return None # absent is the normal case
290
- if not isinstance(prior, dict):
291
- say(f"RESUME refused — runs/{run_id}.json is not an object; re-running")
292
- return None
293
- if not prior.get("ok"):
294
- say(f"RESUME refused — the stored run FAILED "
295
- f"({str(prior.get('error') or 'no reason recorded')[:120]}); re-running")
296
- return None
297
- stale = _too_old(prior)
298
- if stale:
299
- say(f"RESUME refused — {stale}; re-running")
300
- return None
301
- return prior
302
 
303
 
304
  # ── the plan ────────────────────────────────────────────────────────────────────────────────
 
113
  INSTALL_BROWSER = os.environ.get("WEB_AGENT_INSTALL_BROWSER", "1") != "0"
114
  MAX_VALUE_CHARS = 8000 # one cell's worth; a page is not a payload
115
  MAX_STEPS = 20 # matches the engine's MAX_ACTIONS — one flow, one job
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
116
 
117
 
118
  def _fail(sentence: str, *, kind: str = "refused", steps=None) -> None:
 
162
  repo_id=CHECKPOINT_REPO, repo_type="dataset",
163
  commit_message=f"web-agent run {run_id}")
164
  say(f"CHECKPOINT wrote runs/{run_id}.json to {CHECKPOINT_REPO}")
 
 
165
  except Exception as exc: # noqa: BLE001 — a boundary
166
  # ⛔ CATCH EVERYTHING. This is a report-don't-raise boundary and the ways an upload can
167
  # fail are not enumerable (429 commit budget, 403 scope, DNS, a repo that does not
 
171
  f"the stdout frame still carries the result")
172
 
173
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
174
  def _resume(run_id: str):
175
+ """A prior run's result, if this exact run already finished. None otherwise."""
 
 
 
 
 
 
 
 
 
 
 
 
 
 
176
  if not (CHECKPOINT_REPO and HF_TOKEN and run_id):
177
  return None
178
  try:
 
181
  filename=f"runs/{run_id}.json", token=HF_TOKEN,
182
  force_download=True)
183
  with open(path, encoding="utf-8") as fh:
184
+ return json.load(fh)
185
  except Exception: # noqa: BLE001
186
  return None # absent is the normal case
 
 
 
 
 
 
 
 
 
 
 
 
187
 
188
 
189
  # ── the plan ────────────────────────────────────────────────────────────────────────────────
platform/aios_grid.py CHANGED
The diff for this file is too large to render. See raw diff
 
platform/aios_grid_fields.json CHANGED
@@ -1,474 +1,464 @@
1
- {
2
- "_comment": "CANONICAL field contract for the AIOS Airtable-style grid — the SINGLE source of truth. Consumed by platform/aios_grid.py (embed/Space host) and aios-web/api/main.py (standalone API), and regenerated into aios-web/web/public/sample_customers.json. Edit HERE only, then run aios-web/verify_fields_contract.py. source=odoo is READ-ONLY; source=overlay is the editable stratum (notes/tags) outside Odoo. type in {text,status,select,currency,int,date,pct} (select = a fixed-choice READ-ONLY brand attribute; dba is the first, wave 2026-08-02). `description` (wave 5) is the CANONICAL per-field description — every field must carry one, and since wave 7 (owner W8, 2026-07-28) every description is ONE SHORT PLAIN sentence (two only when a fact would otherwise mislead): what the field IS, nothing else — no filter tips, no '(none)' coaching, no rationale; the user's workspace NOTE overrides it in the (i) hover, never in this file. BUILDER FACT (documented here, deliberately NOT in user-facing text): blank text attributes display as '(none)', so `is '(none)'` — not `is empty` — finds the blanks on agent/city/state/country/zip/payment_terms/pricelist/tags. filterable:false = the CONDITION BUILDER does not offer it (still displayed, still sortable); every such field must have a replacement declared in aios-web/verify_fields_contract.py. 2026-07-27 partner attributes: country/zip/payment_terms/pricelist/tags/customer_since all ship default:false. zip is TEXT because a postal code has leading zeros. Odoo's credit_limit (1% populated) and user_id salesperson (2%) are deliberately ABSENT; agent_ids is the salesperson field and AR is where credit exposure comes from. Wave-5 item 8 (2026-07-27): ltm_rev and at_risk are DELETED — LTM's replacement is a creatable Sales measure column (the demo column IS Sales · the last 12 months), at_risk's replacement is a formula field, e.g. MAX(0, {revenue_ly} - {revenue_ytd}). Wave-6 item 8 (2026-07-27, the no-buildable-presets rule): revenue_ytd, revenue_ly, orders_24m, aov and yoy_pct are DELETED — every one is self-buildable, so a frozen pre-set beside the builder was two ways to ask one question. Replacements (recorded in verify_fields_contract.py): creatable measure columns for Sales / Orders / Avg order $ over any period (harness/measure_filter.py ADMITTED carries revenue, orders and the composite aov), and a formula over two measure columns for YoY, e.g. ({sales_ytd} - {sales_ly}) / {sales_ly}. Stale view colIds naming the five self-heal on the next autosave (the established rule).",
3
- "fields": [
4
- {
5
- "key": "customer",
6
- "label": "Customer",
7
- "type": "text",
8
- "source": "odoo",
9
- "pinned": true,
10
- "default": true,
11
- "description": "The customer's name in Odoo. One row per customer who ordered in the last 24 months."
12
- },
13
- {
14
- "key": "partner_id",
15
- "label": "Odoo ID",
16
- "type": "int",
17
- "source": "odoo",
18
- "derived": true,
19
- "default": false,
20
- "description": "The Odoo res.partner id — the key every Odoo document joins on. DERIVED: this row's pid IS the partner id, so a stored copy would be a second source."
21
- },
22
- {
23
- "key": "odoo_status",
24
- "label": "Odoo record",
25
- "type": "status",
26
- "source": "odoo",
27
- "default": false,
28
- "options": ["Active", "Archived"],
29
- "description": "Whether this customer still exists in Odoo. Archived means deleted there."
30
- },
31
- {
32
- "key": "agent",
33
- "label": "Agent",
34
- "type": "text",
35
- "source": "odoo",
36
- "default": true,
37
- "description": "The sales agent who owns this account."
38
- },
39
- {
40
- "key": "dba",
41
- "label": "DBA",
42
- "type": "select",
43
- "source": "odoo",
44
- "default": false,
45
- "options": [
46
- "Fisch",
47
- "Royal",
48
- "Both"
49
- ],
50
- "description": "The brand this customer buys from - Fisch, Royal, or both. Amazon-channel orders are not a DBA."
51
- },
52
- {
53
- "key": "salesperson",
54
- "label": "Salesperson",
55
- "type": "text",
56
- "source": "odoo",
57
- "default": false,
58
- "description": "Who keyed in most of this customer's orders — not the Agent, who owns the account."
59
- },
60
- {
61
- "key": "street",
62
- "label": "Street",
63
- "type": "text",
64
- "source": "odoo",
65
- "default": false,
66
- "description": "First address line, from res.partner directly - not the geocoder, so a customer the map cannot place still shows its address."
67
- },
68
- {
69
- "key": "street2",
70
- "label": "Street 2",
71
- "type": "text",
72
- "source": "odoo",
73
- "default": false,
74
- "description": "Second address line (suite, unit, floor) on the customer's Odoo address."
75
- },
76
- {
77
- "key": "city",
78
- "label": "City",
79
- "type": "text",
80
- "source": "odoo",
81
- "default": true,
82
- "description": "City on the customer's Odoo address."
83
- },
84
- {
85
- "key": "state",
86
- "label": "State",
87
- "type": "text",
88
- "source": "odoo",
89
- "default": true,
90
- "description": "State or province on the customer's Odoo address."
91
- },
92
- {
93
- "key": "country",
94
- "label": "Country",
95
- "type": "text",
96
- "source": "odoo",
97
- "default": false,
98
- "description": "Country on the customer's Odoo address."
99
- },
100
- {
101
- "key": "zip",
102
- "label": "ZIP",
103
- "type": "text",
104
- "source": "odoo",
105
- "default": false,
106
- "description": "Postal code on the customer's Odoo address."
107
- },
108
- {
109
- "key": "customer_since",
110
- "label": "Customer since",
111
- "type": "date",
112
- "source": "odoo",
113
- "default": false,
114
- "description": "When this customer was first set up in Odoo."
115
- },
116
- {
117
- "key": "tags",
118
- "label": "Tags",
119
- "type": "text",
120
- "source": "odoo",
121
- "default": false,
122
- "description": "Odoo labels on this customer, comma-separated."
123
- },
124
- {
125
- "key": "pricelist",
126
- "label": "Price list",
127
- "type": "text",
128
- "source": "odoo",
129
- "default": false,
130
- "description": "The price list this customer buys on."
131
- },
132
- {
133
- "key": "payment_terms",
134
- "label": "Payment terms",
135
- "type": "text",
136
- "source": "odoo",
137
- "default": false,
138
- "description": "Payment terms on this customer's account — Net 30, for example."
139
- },
140
- {
141
- "key": "last_order",
142
- "label": "Last order",
143
- "type": "date",
144
- "source": "odoo",
145
- "default": true,
146
- "description": "Date of the most recent confirmed order."
147
- },
148
- {
149
- "key": "overdue_days",
150
- "label": "Overdue days",
151
- "type": "int",
152
- "source": "odoo",
153
- "default": true,
154
- "description": "How many days late this customer is running against their own usual ordering rhythm."
155
- },
156
- {
157
- "_note": "filterable:false — DERIVED ANALYTIC: est_missed is min(cycles missed, 3) x AOV, a score we compute rather than an object the business has, so a condition on it would read as a fact about the customer when it is a fact about our arithmetic. It still displays and still sorts. Until wave 6 this flag also covered the frozen-window presets (revenue_ytd / revenue_ly / orders_24m / aov / yoy_pct); those are now DELETED outright under the owner's no-buildable-presets rule — see _comment. est_missed itself STAYS: no creatable measure or formula reproduces the cadence model behind it.",
158
- "key": "est_missed",
159
- "label": "Est. missed $",
160
- "type": "currency",
161
- "source": "odoo",
162
- "default": true,
163
- "agg": "sum",
164
- "filterable": false,
165
- "description": "Estimated sales missed while quiet: missed orders (capped at 3) times average order value. An estimate, not money owed."
166
- },
167
- {
168
- "_note": "wave 21 R1 — KEY UNCHANGED, LABEL RENAMED. The computation is a DISJOINT split (ar.py credit_exposure): this column is only the not-yet-due residual, its sibling is the past-grace residual, and the two sum to the total. Under the label 'AR open $' the majority-late book read as 'Overdue > Open', which is nonsense in AR vocabulary — 'open' universally means the total. The label now says what the number is; the key stays so saved views and filters keep working.",
169
- "key": "ar_open",
170
- "label": "AR current $",
171
- "type": "currency",
172
- "source": "odoo",
173
- "default": false,
174
- "description": "Invoiced money owed but not yet due (a 5-day grace applies before it counts as overdue)."
175
- },
176
- {
177
- "key": "ar_overdue",
178
- "label": "AR overdue $",
179
- "type": "currency",
180
- "source": "odoo",
181
- "default": false,
182
- "description": "Invoiced money past due — same basis as the Collections page."
183
- },
184
- {
185
- "_note": "wave 21 R1 — the TOTAL, added beside the rename above. AR current $ + AR overdue $, i.e. what most people mean by 'open AR'. Composed from the same ar.credit_exposure rows the siblings use, so it is transitively reconciled by ar.validate()'s residual read_group tie — no second oracle.",
186
- "key": "ar_outstanding",
187
- "label": "AR outstanding $",
188
- "type": "currency",
189
- "source": "odoo",
190
- "default": false,
191
- "description": "Total invoiced money owed right now: AR current $ plus AR overdue $."
192
- },
193
- {
194
- "key": "ar_exposure",
195
- "label": "Credit exposure $",
196
- "type": "currency",
197
- "source": "odoo",
198
- "default": false,
199
- "description": "The most you could be out if they stopped paying today: open, overdue, draft and not-yet-invoiced."
200
- },
201
- {
202
- "key": "ar_aged_1_30",
203
- "label": "1-30 days $",
204
- "type": "currency",
205
- "source": "odoo",
206
- "default": false,
207
- "description": "Overdue between 1 and 30 days. The four aging buckets sum to AR overdue $."
208
- },
209
- {
210
- "key": "ar_aged_31_60",
211
- "label": "31-60 days $",
212
- "type": "currency",
213
- "source": "odoo",
214
- "default": false,
215
- "description": "Overdue between 31 and 60 days. The four aging buckets sum to AR overdue $."
216
- },
217
- {
218
- "key": "ar_aged_61_90",
219
- "label": "61-90 days $",
220
- "type": "currency",
221
- "source": "odoo",
222
- "default": false,
223
- "description": "Overdue between 61 and 90 days. The four aging buckets sum to AR overdue $."
224
- },
225
- {
226
- "key": "ar_aged_90_plus",
227
- "label": "90+ days $",
228
- "type": "currency",
229
- "source": "odoo",
230
- "default": false,
231
- "description": "Overdue by more than 90 days. The four aging buckets sum to AR overdue $."
232
- },
233
- {
234
- "key": "days_to_pay",
235
- "label": "Days to pay",
236
- "type": "int",
237
- "source": "odoo",
238
- "default": false,
239
- "description": "Average days to pay an invoice in full. Blank means no fully paid invoice yet."
240
- },
241
- {
242
- "key": "top_category",
243
- "label": "Top category",
244
- "type": "text",
245
- "source": "odoo",
246
- "default": false,
247
- "description": "The category this customer spent the most on in the last 12 months."
248
- },
249
- {
250
- "key": "top_category_pct",
251
- "label": "Top category %",
252
- "type": "pct",
253
- "source": "odoo",
254
- "default": false,
255
- "description": "Share of last-12-months spend that went to the top category."
256
- },
257
- {
258
- "key": "sku_count",
259
- "label": "SKUs bought",
260
- "type": "int",
261
- "source": "odoo",
262
- "default": false,
263
- "description": "Distinct products bought in the last 12 months."
264
- },
265
- {
266
- "key": "top_sku",
267
- "label": "Top SKU",
268
- "type": "text",
269
- "source": "odoo",
270
- "default": false,
271
- "description": "The product this customer spent the most on in the last 12 months."
272
- },
273
- {
274
- "key": "days_since",
275
- "label": "Days since order",
276
- "type": "int",
277
- "source": "odoo",
278
- "default": false,
279
- "description": "Days since the last confirmed order."
280
- },
281
- {
282
- "key": "typical_gap_days",
283
- "label": "Typical gap days",
284
- "type": "int",
285
- "source": "odoo",
286
- "default": false,
287
- "description": "Days this customer usually goes between orders, from their own history."
288
- },
289
- {
290
- "key": "notes",
291
- "label": "Notes",
292
- "type": "text",
293
- "source": "overlay",
294
- "default": false,
295
- "description": "Your notes on this customer. Saved in this app only, visible only to you."
296
- }
297
- ],
298
- "_product_comment": "ADDITIVE, wave 15 C-TOPIC. The PRODUCT table's field contract. Kept as a SEPARATE top-level key rather than restructuring `fields` into {customer_data, product_data}: both existing readers (aios_grid._load_fields, aios-web/api/main.py) index doc['fields'] directly, and reshaping that mid-wave would break the embed for a cosmetic gain. The keyed shape can arrive when both readers move in ONE commit; until then this is the product half and `fields` is the customer half.",
299
- "_product_removed_buy_now": "OWNER, 2026-08-03: 'Buy signal' (key buy_now, a select of Buy now / OK) is NO LONGER A PRESET FIELD. It never earned one: it is a formula over two columns that are both still right here, and the platform has a formula field type for exactly that. THE FORMULA, which reproduces the retired column row for row (modules/product_data.validate proves the equivalence, and goes red if it ever stops holding): IF({lead_days} > 0, IF({dos} < {lead_days}, \"Buy now\", \"OK\"), \"\") . Every branch matches the old server rule, including the blanks - the formula engine refuses a comparison against a blank rather than coercing it to 0, so a SKU with no days-of-supply or no lead time comes out empty, which is 'we do not know' and not 'you are fine'. NOTE the column is still COMPUTED in product_data.pool(): it ships nowhere (rows_from_pool projects strictly through this contract, so no Field means no cell on the wire) and exists only as validate()'s oracle. A formula field is PER-USER, so nothing shared may filter on it - the Buy list view filters on dos/lead_days directly (_seed_wave17).",
300
- "product_data": {
301
- "identity": "pid",
302
- "business_key": "code",
303
- "fields": [
304
- {
305
- "key": "code",
306
- "label": "SKU",
307
- "type": "text",
308
- "source": "odoo",
309
- "pinned": true,
310
- "default": true,
311
- "description": "The SKU code — the product's real business key. `pid` is a stable CRC32 of it because the grid keys on an integer."
312
- },
313
- {
314
- "key": "product",
315
- "label": "Product",
316
- "type": "text",
317
- "source": "odoo",
318
- "default": true,
319
- "description": "Product name as it appears in Odoo."
320
- },
321
- {
322
- "key": "category",
323
- "label": "Category",
324
- "type": "select",
325
- "source": "odoo",
326
- "default": true,
327
- "description": "Product category; '(uncategorized)' when Odoo carries none."
328
- },
329
- {
330
- "key": "supplier",
331
- "label": "Supplier",
332
- "type": "text",
333
- "source": "overlay",
334
- "default": true,
335
- "description": "Who makes it. Editable here and shared with everyone in the workspace; seeded from the inventory mastersheet.",
336
- "shared": true
337
- },
338
- {
339
- "key": "origin_country",
340
- "label": "Country",
341
- "type": "text",
342
- "source": "overlay",
343
- "default": false,
344
- "description": "Country of origin. Editable here and shared with everyone; seeded from the inventory mastersheet.",
345
- "shared": true
346
- },
347
- {
348
- "key": "lead_days",
349
- "label": "Lead time (days)",
350
- "type": "int",
351
- "source": "overlay",
352
- "default": true,
353
- "description": "Order-to-arrival days for this supplier. Drives the buy signal. Editable and shared with everyone.",
354
- "shared": true
355
- },
356
- {
357
- "key": "first_cost",
358
- "label": "First cost",
359
- "type": "currency",
360
- "source": "overlay",
361
- "default": false,
362
- "description": "Quoted unit cost at origin, before freight and duty. Editable and shared with everyone.",
363
- "shared": true
364
- },
365
- {
366
- "key": "price_fisch",
367
- "label": "Fisch price",
368
- "type": "currency",
369
- "source": "odoo",
370
- "description": "Fisch pricelist price for this SKU. Blank when that list prices it nowhere."
371
- },
372
- {
373
- "key": "price_royal_1",
374
- "label": "Royal 1 price",
375
- "type": "currency",
376
- "source": "odoo",
377
- "description": "Royal 1 pricelist price for this SKU. Blank when that list prices it nowhere."
378
- },
379
- {
380
- "key": "price_royal_2",
381
- "label": "Royal 2 price",
382
- "type": "currency",
383
- "source": "odoo",
384
- "description": "Royal 2 pricelist price for this SKU. Blank when that list prices it nowhere."
385
- },
386
- {
387
- "key": "rev_ytd",
388
- "label": "Revenue YTD",
389
- "type": "currency",
390
- "source": "odoo",
391
- "default": true,
392
- "description": "Year-to-date revenue for this SKU, BU-scoped when the caller is."
393
- },
394
- {
395
- "key": "rev_ly",
396
- "label": "Revenue LY",
397
- "type": "currency",
398
- "source": "odoo",
399
- "description": "Same period last year — seasonal wholesale compares like for like."
400
- },
401
- {
402
- "key": "yoy_pct",
403
- "label": "YoY %",
404
- "type": "pct",
405
- "source": "odoo",
406
- "description": "Year-over-year change; null when last year was zero (a ratio to zero is not a number)."
407
- },
408
- {
409
- "key": "qty_ytd",
410
- "label": "Units YTD",
411
- "type": "int",
412
- "source": "odoo",
413
- "description": "Units sold year to date."
414
- },
415
- {
416
- "key": "orders_ytd",
417
- "label": "Orders YTD",
418
- "type": "int",
419
- "source": "odoo",
420
- "description": "Distinct orders containing this SKU, year to date."
421
- },
422
- {
423
- "key": "on_hand",
424
- "label": "On hand",
425
- "type": "int",
426
- "source": "odoo",
427
- "description": "Units in stock. CONSOLIDATED — one physical warehouse, not brand-tagged, so this column is ABSENT for a BU-scoped caller rather than silently company-wide."
428
- },
429
- {
430
- "key": "unit_cost",
431
- "label": "Unit cost",
432
- "type": "currency",
433
- "source": "odoo",
434
- "description": "Inventory unit cost. Consolidated; absent for a BU-scoped caller."
435
- },
436
- {
437
- "key": "inv_value",
438
- "label": "Stock value",
439
- "type": "currency",
440
- "source": "odoo",
441
- "description": "On-hand value at cost. Consolidated; absent for a BU-scoped caller."
442
- },
443
- {
444
- "key": "qty_ltm",
445
- "label": "Units LTM",
446
- "type": "int",
447
- "source": "odoo",
448
- "description": "Units sold in the last twelve months. Consolidated; absent for a BU-scoped caller."
449
- },
450
- {
451
- "key": "dos",
452
- "label": "Days of supply",
453
- "type": "int",
454
- "source": "odoo",
455
- "description": "Days of supply at the LTM rate; null means it never sells through. Consolidated; absent for a BU-scoped caller."
456
- },
457
- {
458
- "key": "cover_gap_d",
459
- "label": "Cover gap (days)",
460
- "type": "int",
461
- "source": "odoo",
462
- "default": false,
463
- "description": "Days of supply minus lead time. Negative means it runs out before a reorder lands."
464
- },
465
- {
466
- "key": "stock_bucket",
467
- "label": "Stock status",
468
- "type": "select",
469
- "source": "odoo",
470
- "description": "Dead / excess / healthy bucket from the inventory module. Consolidated; absent for a BU-scoped caller."
471
- }
472
- ]
473
- }
474
- }
 
1
+ {
2
+ "_comment": "CANONICAL field contract for the AIOS Airtable-style grid — the SINGLE source of truth. Consumed by platform/aios_grid.py (embed/Space host) and aios-web/api/main.py (standalone API), and regenerated into aios-web/web/public/sample_customers.json. Edit HERE only, then run aios-web/verify_fields_contract.py. source=odoo is READ-ONLY; source=overlay is the editable stratum (notes/tags) outside Odoo. type in {text,status,select,currency,int,date,pct} (select = a fixed-choice READ-ONLY brand attribute; dba is the first, wave 2026-08-02). `description` (wave 5) is the CANONICAL per-field description — every field must carry one, and since wave 7 (owner W8, 2026-07-28) every description is ONE SHORT PLAIN sentence (two only when a fact would otherwise mislead): what the field IS, nothing else — no filter tips, no '(none)' coaching, no rationale; the user's workspace NOTE overrides it in the (i) hover, never in this file. BUILDER FACT (documented here, deliberately NOT in user-facing text): blank text attributes display as '(none)', so `is '(none)'` — not `is empty` — finds the blanks on agent/city/state/country/zip/payment_terms/pricelist/tags. filterable:false = the CONDITION BUILDER does not offer it (still displayed, still sortable); every such field must have a replacement declared in aios-web/verify_fields_contract.py. 2026-07-27 partner attributes: country/zip/payment_terms/pricelist/tags/customer_since all ship default:false. zip is TEXT because a postal code has leading zeros. Odoo's credit_limit (1% populated) and user_id salesperson (2%) are deliberately ABSENT; agent_ids is the salesperson field and AR is where credit exposure comes from. Wave-5 item 8 (2026-07-27): ltm_rev and at_risk are DELETED — LTM's replacement is a creatable Sales measure column (the demo column IS Sales · the last 12 months), at_risk's replacement is a formula field, e.g. MAX(0, {revenue_ly} - {revenue_ytd}). Wave-6 item 8 (2026-07-27, the no-buildable-presets rule): revenue_ytd, revenue_ly, orders_24m, aov and yoy_pct are DELETED — every one is self-buildable, so a frozen pre-set beside the builder was two ways to ask one question. Replacements (recorded in verify_fields_contract.py): creatable measure columns for Sales / Orders / Avg order $ over any period (harness/measure_filter.py ADMITTED carries revenue, orders and the composite aov), and a formula over two measure columns for YoY, e.g. ({sales_ytd} - {sales_ly}) / {sales_ly}. Stale view colIds naming the five self-heal on the next autosave (the established rule).",
3
+ "fields": [
4
+ {
5
+ "key": "customer",
6
+ "label": "Customer",
7
+ "type": "text",
8
+ "source": "odoo",
9
+ "pinned": true,
10
+ "default": true,
11
+ "description": "The customer's name in Odoo. One row per customer who ordered in the last 24 months."
12
+ },
13
+ {
14
+ "key": "odoo_status",
15
+ "label": "Odoo record",
16
+ "type": "status",
17
+ "source": "odoo",
18
+ "default": false,
19
+ "description": "Whether this customer still exists in Odoo. Archived means deleted there."
20
+ },
21
+ {
22
+ "key": "agent",
23
+ "label": "Agent",
24
+ "type": "text",
25
+ "source": "odoo",
26
+ "default": true,
27
+ "description": "The sales agent who owns this account."
28
+ },
29
+ {
30
+ "key": "dba",
31
+ "label": "DBA",
32
+ "type": "select",
33
+ "source": "odoo",
34
+ "default": false,
35
+ "options": [
36
+ "Fisch",
37
+ "Royal",
38
+ "Both"
39
+ ],
40
+ "description": "The brand this customer buys from - Fisch, Royal, or both. Amazon-channel orders are not a DBA."
41
+ },
42
+ {
43
+ "key": "salesperson",
44
+ "label": "Salesperson",
45
+ "type": "text",
46
+ "source": "odoo",
47
+ "default": false,
48
+ "description": "Who keyed in most of this customer's orders — not the Agent, who owns the account."
49
+ },
50
+ {
51
+ "key": "street",
52
+ "label": "Street",
53
+ "type": "text",
54
+ "source": "odoo",
55
+ "default": false,
56
+ "description": "First address line, from res.partner directly - not the geocoder, so a customer the map cannot place still shows its address."
57
+ },
58
+ {
59
+ "key": "street2",
60
+ "label": "Street 2",
61
+ "type": "text",
62
+ "source": "odoo",
63
+ "default": false,
64
+ "description": "Second address line (suite, unit, floor) on the customer's Odoo address."
65
+ },
66
+ {
67
+ "key": "city",
68
+ "label": "City",
69
+ "type": "text",
70
+ "source": "odoo",
71
+ "default": true,
72
+ "description": "City on the customer's Odoo address."
73
+ },
74
+ {
75
+ "key": "state",
76
+ "label": "State",
77
+ "type": "text",
78
+ "source": "odoo",
79
+ "default": true,
80
+ "description": "State or province on the customer's Odoo address."
81
+ },
82
+ {
83
+ "key": "country",
84
+ "label": "Country",
85
+ "type": "text",
86
+ "source": "odoo",
87
+ "default": false,
88
+ "description": "Country on the customer's Odoo address."
89
+ },
90
+ {
91
+ "key": "zip",
92
+ "label": "ZIP",
93
+ "type": "text",
94
+ "source": "odoo",
95
+ "default": false,
96
+ "description": "Postal code on the customer's Odoo address."
97
+ },
98
+ {
99
+ "key": "customer_since",
100
+ "label": "Customer since",
101
+ "type": "date",
102
+ "source": "odoo",
103
+ "default": false,
104
+ "description": "When this customer was first set up in Odoo."
105
+ },
106
+ {
107
+ "key": "tags",
108
+ "label": "Tags",
109
+ "type": "text",
110
+ "source": "odoo",
111
+ "default": false,
112
+ "description": "Odoo labels on this customer, comma-separated."
113
+ },
114
+ {
115
+ "key": "pricelist",
116
+ "label": "Price list",
117
+ "type": "text",
118
+ "source": "odoo",
119
+ "default": false,
120
+ "description": "The price list this customer buys on."
121
+ },
122
+ {
123
+ "key": "payment_terms",
124
+ "label": "Payment terms",
125
+ "type": "text",
126
+ "source": "odoo",
127
+ "default": false,
128
+ "description": "Payment terms on this customer's account — Net 30, for example."
129
+ },
130
+ {
131
+ "key": "last_order",
132
+ "label": "Last order",
133
+ "type": "date",
134
+ "source": "odoo",
135
+ "default": true,
136
+ "description": "Date of the most recent confirmed order."
137
+ },
138
+ {
139
+ "key": "overdue_days",
140
+ "label": "Overdue days",
141
+ "type": "int",
142
+ "source": "odoo",
143
+ "default": true,
144
+ "description": "How many days late this customer is running against their own usual ordering rhythm."
145
+ },
146
+ {
147
+ "_note": "filterable:false — DERIVED ANALYTIC: est_missed is min(cycles missed, 3) x AOV, a score we compute rather than an object the business has, so a condition on it would read as a fact about the customer when it is a fact about our arithmetic. It still displays and still sorts. Until wave 6 this flag also covered the frozen-window presets (revenue_ytd / revenue_ly / orders_24m / aov / yoy_pct); those are now DELETED outright under the owner's no-buildable-presets rule — see _comment. est_missed itself STAYS: no creatable measure or formula reproduces the cadence model behind it.",
148
+ "key": "est_missed",
149
+ "label": "Est. missed $",
150
+ "type": "currency",
151
+ "source": "odoo",
152
+ "default": true,
153
+ "agg": "sum",
154
+ "filterable": false,
155
+ "description": "Estimated sales missed while quiet: missed orders (capped at 3) times average order value. An estimate, not money owed."
156
+ },
157
+ {
158
+ "_note": "wave 21 R1 — KEY UNCHANGED, LABEL RENAMED. The computation is a DISJOINT split (ar.py credit_exposure): this column is only the not-yet-due residual, its sibling is the past-grace residual, and the two sum to the total. Under the label 'AR open $' the majority-late book read as 'Overdue > Open', which is nonsense in AR vocabulary — 'open' universally means the total. The label now says what the number is; the key stays so saved views and filters keep working.",
159
+ "key": "ar_open",
160
+ "label": "AR current $",
161
+ "type": "currency",
162
+ "source": "odoo",
163
+ "default": false,
164
+ "description": "Invoiced money owed but not yet due (a 5-day grace applies before it counts as overdue)."
165
+ },
166
+ {
167
+ "key": "ar_overdue",
168
+ "label": "AR overdue $",
169
+ "type": "currency",
170
+ "source": "odoo",
171
+ "default": false,
172
+ "description": "Invoiced money past due — same basis as the Collections page."
173
+ },
174
+ {
175
+ "_note": "wave 21 R1 — the TOTAL, added beside the rename above. AR current $ + AR overdue $, i.e. what most people mean by 'open AR'. Composed from the same ar.credit_exposure rows the siblings use, so it is transitively reconciled by ar.validate()'s residual read_group tie — no second oracle.",
176
+ "key": "ar_outstanding",
177
+ "label": "AR outstanding $",
178
+ "type": "currency",
179
+ "source": "odoo",
180
+ "default": false,
181
+ "description": "Total invoiced money owed right now: AR current $ plus AR overdue $."
182
+ },
183
+ {
184
+ "key": "ar_exposure",
185
+ "label": "Credit exposure $",
186
+ "type": "currency",
187
+ "source": "odoo",
188
+ "default": false,
189
+ "description": "The most you could be out if they stopped paying today: open, overdue, draft and not-yet-invoiced."
190
+ },
191
+ {
192
+ "key": "ar_aged_1_30",
193
+ "label": "1-30 days $",
194
+ "type": "currency",
195
+ "source": "odoo",
196
+ "default": false,
197
+ "description": "Overdue between 1 and 30 days. The four aging buckets sum to AR overdue $."
198
+ },
199
+ {
200
+ "key": "ar_aged_31_60",
201
+ "label": "31-60 days $",
202
+ "type": "currency",
203
+ "source": "odoo",
204
+ "default": false,
205
+ "description": "Overdue between 31 and 60 days. The four aging buckets sum to AR overdue $."
206
+ },
207
+ {
208
+ "key": "ar_aged_61_90",
209
+ "label": "61-90 days $",
210
+ "type": "currency",
211
+ "source": "odoo",
212
+ "default": false,
213
+ "description": "Overdue between 61 and 90 days. The four aging buckets sum to AR overdue $."
214
+ },
215
+ {
216
+ "key": "ar_aged_90_plus",
217
+ "label": "90+ days $",
218
+ "type": "currency",
219
+ "source": "odoo",
220
+ "default": false,
221
+ "description": "Overdue by more than 90 days. The four aging buckets sum to AR overdue $."
222
+ },
223
+ {
224
+ "key": "days_to_pay",
225
+ "label": "Days to pay",
226
+ "type": "int",
227
+ "source": "odoo",
228
+ "default": false,
229
+ "description": "Average days to pay an invoice in full. Blank means no fully paid invoice yet."
230
+ },
231
+ {
232
+ "key": "top_category",
233
+ "label": "Top category",
234
+ "type": "text",
235
+ "source": "odoo",
236
+ "default": false,
237
+ "description": "The category this customer spent the most on in the last 12 months."
238
+ },
239
+ {
240
+ "key": "top_category_pct",
241
+ "label": "Top category %",
242
+ "type": "pct",
243
+ "source": "odoo",
244
+ "default": false,
245
+ "description": "Share of last-12-months spend that went to the top category."
246
+ },
247
+ {
248
+ "key": "sku_count",
249
+ "label": "SKUs bought",
250
+ "type": "int",
251
+ "source": "odoo",
252
+ "default": false,
253
+ "description": "Distinct products bought in the last 12 months."
254
+ },
255
+ {
256
+ "key": "top_sku",
257
+ "label": "Top SKU",
258
+ "type": "text",
259
+ "source": "odoo",
260
+ "default": false,
261
+ "description": "The product this customer spent the most on in the last 12 months."
262
+ },
263
+ {
264
+ "key": "days_since",
265
+ "label": "Days since order",
266
+ "type": "int",
267
+ "source": "odoo",
268
+ "default": false,
269
+ "description": "Days since the last confirmed order."
270
+ },
271
+ {
272
+ "key": "typical_gap_days",
273
+ "label": "Typical gap days",
274
+ "type": "int",
275
+ "source": "odoo",
276
+ "default": false,
277
+ "description": "Days this customer usually goes between orders, from their own history."
278
+ },
279
+ {
280
+ "key": "notes",
281
+ "label": "Notes",
282
+ "type": "text",
283
+ "source": "overlay",
284
+ "default": false,
285
+ "description": "Your notes on this customer. Saved in this app only, visible only to you."
286
+ }
287
+ ],
288
+ "_product_comment": "ADDITIVE, wave 15 C-TOPIC. The PRODUCT table's field contract. Kept as a SEPARATE top-level key rather than restructuring `fields` into {customer_data, product_data}: both existing readers (aios_grid._load_fields, aios-web/api/main.py) index doc['fields'] directly, and reshaping that mid-wave would break the embed for a cosmetic gain. The keyed shape can arrive when both readers move in ONE commit; until then this is the product half and `fields` is the customer half.",
289
+ "_product_removed_buy_now": "OWNER, 2026-08-03: 'Buy signal' (key buy_now, a select of Buy now / OK) is NO LONGER A PRESET FIELD. It never earned one: it is a formula over two columns that are both still right here, and the platform has a formula field type for exactly that. THE FORMULA, which reproduces the retired column row for row (modules/product_data.validate proves the equivalence, and goes red if it ever stops holding): IF({lead_days} > 0, IF({dos} < {lead_days}, \"Buy now\", \"OK\"), \"\") . Every branch matches the old server rule, including the blanks - the formula engine refuses a comparison against a blank rather than coercing it to 0, so a SKU with no days-of-supply or no lead time comes out empty, which is 'we do not know' and not 'you are fine'. NOTE the column is still COMPUTED in product_data.pool(): it ships nowhere (rows_from_pool projects strictly through this contract, so no Field means no cell on the wire) and exists only as validate()'s oracle. A formula field is PER-USER, so nothing shared may filter on it - the Buy list view filters on dos/lead_days directly (_seed_wave17).",
290
+ "product_data": {
291
+ "identity": "pid",
292
+ "business_key": "code",
293
+ "fields": [
294
+ {
295
+ "key": "code",
296
+ "label": "SKU",
297
+ "type": "text",
298
+ "source": "odoo",
299
+ "pinned": true,
300
+ "default": true,
301
+ "description": "The SKU code — the product's real business key. `pid` is a stable CRC32 of it because the grid keys on an integer."
302
+ },
303
+ {
304
+ "key": "product",
305
+ "label": "Product",
306
+ "type": "text",
307
+ "source": "odoo",
308
+ "default": true,
309
+ "description": "Product name as it appears in Odoo."
310
+ },
311
+ {
312
+ "key": "category",
313
+ "label": "Category",
314
+ "type": "select",
315
+ "source": "odoo",
316
+ "default": true,
317
+ "description": "Product category; '(uncategorized)' when Odoo carries none."
318
+ },
319
+ {
320
+ "key": "supplier",
321
+ "label": "Supplier",
322
+ "type": "text",
323
+ "source": "overlay",
324
+ "default": true,
325
+ "description": "Who makes it. Editable here and shared with everyone in the workspace; seeded from the inventory mastersheet.",
326
+ "shared": true
327
+ },
328
+ {
329
+ "key": "origin_country",
330
+ "label": "Country",
331
+ "type": "text",
332
+ "source": "overlay",
333
+ "default": false,
334
+ "description": "Country of origin. Editable here and shared with everyone; seeded from the inventory mastersheet.",
335
+ "shared": true
336
+ },
337
+ {
338
+ "key": "lead_days",
339
+ "label": "Lead time (days)",
340
+ "type": "int",
341
+ "source": "overlay",
342
+ "default": true,
343
+ "description": "Order-to-arrival days for this supplier. Drives the buy signal. Editable and shared with everyone.",
344
+ "shared": true
345
+ },
346
+ {
347
+ "key": "first_cost",
348
+ "label": "First cost",
349
+ "type": "currency",
350
+ "source": "overlay",
351
+ "default": false,
352
+ "description": "Quoted unit cost at origin, before freight and duty. Editable and shared with everyone.",
353
+ "shared": true
354
+ },
355
+ {
356
+ "key": "price_fisch",
357
+ "label": "Fisch price",
358
+ "type": "currency",
359
+ "source": "odoo",
360
+ "description": "Fisch pricelist price for this SKU. Blank when that list prices it nowhere."
361
+ },
362
+ {
363
+ "key": "price_royal_1",
364
+ "label": "Royal 1 price",
365
+ "type": "currency",
366
+ "source": "odoo",
367
+ "description": "Royal 1 pricelist price for this SKU. Blank when that list prices it nowhere."
368
+ },
369
+ {
370
+ "key": "price_royal_2",
371
+ "label": "Royal 2 price",
372
+ "type": "currency",
373
+ "source": "odoo",
374
+ "description": "Royal 2 pricelist price for this SKU. Blank when that list prices it nowhere."
375
+ },
376
+ {
377
+ "key": "rev_ytd",
378
+ "label": "Revenue YTD",
379
+ "type": "currency",
380
+ "source": "odoo",
381
+ "default": true,
382
+ "description": "Year-to-date revenue for this SKU, BU-scoped when the caller is."
383
+ },
384
+ {
385
+ "key": "rev_ly",
386
+ "label": "Revenue LY",
387
+ "type": "currency",
388
+ "source": "odoo",
389
+ "description": "Same period last year — seasonal wholesale compares like for like."
390
+ },
391
+ {
392
+ "key": "yoy_pct",
393
+ "label": "YoY %",
394
+ "type": "pct",
395
+ "source": "odoo",
396
+ "description": "Year-over-year change; null when last year was zero (a ratio to zero is not a number)."
397
+ },
398
+ {
399
+ "key": "qty_ytd",
400
+ "label": "Units YTD",
401
+ "type": "int",
402
+ "source": "odoo",
403
+ "description": "Units sold year to date."
404
+ },
405
+ {
406
+ "key": "orders_ytd",
407
+ "label": "Orders YTD",
408
+ "type": "int",
409
+ "source": "odoo",
410
+ "description": "Distinct orders containing this SKU, year to date."
411
+ },
412
+ {
413
+ "key": "on_hand",
414
+ "label": "On hand",
415
+ "type": "int",
416
+ "source": "odoo",
417
+ "description": "Units in stock. CONSOLIDATED — one physical warehouse, not brand-tagged, so this column is ABSENT for a BU-scoped caller rather than silently company-wide."
418
+ },
419
+ {
420
+ "key": "unit_cost",
421
+ "label": "Unit cost",
422
+ "type": "currency",
423
+ "source": "odoo",
424
+ "description": "Inventory unit cost. Consolidated; absent for a BU-scoped caller."
425
+ },
426
+ {
427
+ "key": "inv_value",
428
+ "label": "Stock value",
429
+ "type": "currency",
430
+ "source": "odoo",
431
+ "description": "On-hand value at cost. Consolidated; absent for a BU-scoped caller."
432
+ },
433
+ {
434
+ "key": "qty_ltm",
435
+ "label": "Units LTM",
436
+ "type": "int",
437
+ "source": "odoo",
438
+ "description": "Units sold in the last twelve months. Consolidated; absent for a BU-scoped caller."
439
+ },
440
+ {
441
+ "key": "dos",
442
+ "label": "Days of supply",
443
+ "type": "int",
444
+ "source": "odoo",
445
+ "description": "Days of supply at the LTM rate; null means it never sells through. Consolidated; absent for a BU-scoped caller."
446
+ },
447
+ {
448
+ "key": "cover_gap_d",
449
+ "label": "Cover gap (days)",
450
+ "type": "int",
451
+ "source": "odoo",
452
+ "default": false,
453
+ "description": "Days of supply minus lead time. Negative means it runs out before a reorder lands."
454
+ },
455
+ {
456
+ "key": "stock_bucket",
457
+ "label": "Stock status",
458
+ "type": "select",
459
+ "source": "odoo",
460
+ "description": "Dead / excess / healthy bucket from the inventory module. Consolidated; absent for a BU-scoped caller."
461
+ }
462
+ ]
463
+ }
464
+ }
 
 
 
 
 
 
 
 
 
 
platform/core/alerts.py CHANGED
@@ -209,7 +209,7 @@ def _queue(rec, pids, labels, st=None):
209
  _st(st).update(NOTIFICATIONS_KEY, _apply, flush='async')
210
 
211
 
212
- def notify(owner, label, *, topic='automation', key='', row_id='', detail='', actor='', st=None):
213
  """⭐ WAVE 23 (C6) — put ONE notification in a named user's inbox, for a producer that is not
214
  a view-alert evaluation.
215
 
@@ -222,24 +222,6 @@ def notify(owner, label, *, topic='automation', key='', row_id='', detail='', ac
222
  The inbox shape is UNCHANGED, deliberately: `AlertsPane` reads one list, and a second row
223
  shape would mean a client that must branch on where a notification came from. `alertId`
224
  carries the producer's key so a click-through can route (W23-W5).
225
-
226
- ⭐⭐ W33-T28 (`ASK C-14`) — `actor` IS THE ONLY THING THAT KNOWS WHO. Owner item: *"the Inbox
227
- reads like email"*, and a verifier reading the carried wave-32 surface found the SENDER position
228
- occupied by `kindLabel(n.kind)` — the literals "Alert" / "Automation" / "Shared with you", i.e. a
229
- CATEGORY where mail puts a who — with no sender field anywhere on the wire, in the model or in
230
- the markup. An alert and an automation have no person behind them and are honestly named by
231
- their machine; **a SHARE has a real person, and only the PRODUCER knows their name.** By the time
232
- the read path sees the row that fact is gone, so it has to be carried at write time.
233
-
234
- ⛔ ADDITIVE, AND THAT IS THE WHOLE DESIGN. No existing caller changes, no stored row changes
235
- shape, and every reader already tolerates the key's absence — so an inbox written before this
236
- line reads exactly as it did. The alternative C rejected is worth recording because it is the
237
- tempting one: recovering the name by REGEXING the body prose (`"<name> shared this with you"`).
238
- A sender parsed out of a sentence breaks the first time the sentence is reworded, and it breaks
239
- SILENTLY, in the header — [[one-question-two-normalizers]] with the second normaliser being
240
- English.
241
- ⚠ Blank is a legitimate value and means *"no person did this"*, NOT "unknown". The client falls
242
- back BY KIND rather than to an empty From column, so an automation never borrows a human's name.
243
  """
244
  owner = str(owner or '').strip()
245
  if not owner or not label:
@@ -248,8 +230,7 @@ def notify(owner, label, *, topic='automation', key='', row_id='', detail='', ac
248
  item = {'id': f"{key or topic}:{row_id or 'n'}:{at}",
249
  'alertId': str(key or ''), 'viewId': '', 'topic': str(topic),
250
  'rowId': str(row_id or ''), 'label': str(detail or label),
251
- 'alertLabel': str(label), 'at': at, 'read': False,
252
- 'actor': str(actor or '')}
253
 
254
  def _apply(data):
255
  box = list(data.get(owner) or [])
 
209
  _st(st).update(NOTIFICATIONS_KEY, _apply, flush='async')
210
 
211
 
212
+ def notify(owner, label, *, topic='automation', key='', row_id='', detail='', st=None):
213
  """⭐ WAVE 23 (C6) — put ONE notification in a named user's inbox, for a producer that is not
214
  a view-alert evaluation.
215
 
 
222
  The inbox shape is UNCHANGED, deliberately: `AlertsPane` reads one list, and a second row
223
  shape would mean a client that must branch on where a notification came from. `alertId`
224
  carries the producer's key so a click-through can route (W23-W5).
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
225
  """
226
  owner = str(owner or '').strip()
227
  if not owner or not label:
 
230
  item = {'id': f"{key or topic}:{row_id or 'n'}:{at}",
231
  'alertId': str(key or ''), 'viewId': '', 'topic': str(topic),
232
  'rowId': str(row_id or ''), 'label': str(detail or label),
233
+ 'alertLabel': str(label), 'at': at, 'read': False}
 
234
 
235
  def _apply(data):
236
  box = list(data.get(owner) or [])
platform/core/grid_events.py CHANGED
The diff for this file is too large to render. See raw diff
 
platform/core/perms.py CHANGED
@@ -199,116 +199,6 @@ def nav_pages(user):
199
  return out
200
 
201
 
202
- # ------------------------------------------------------------------ governable modules (C3)
203
- #: ⭐⭐ WAVE 33, OWNER ITEM 11 — *"its an error that ALL fucking tenant can see both Customer and
204
- #: Product database under Manage User, this is only for Royal Imports tenant. Make sure per tenant
205
- #: configuration always work, this is my fourth time I flag it."*
206
- #:
207
- #: The cause was two lines in `routes_admin.py`: `_PERM_MODULES = ("customer_data","product_data")`
208
- #: and a `_MODULE_LABELS` dict beside it. `get_perms` built its answer from the literal and never
209
- #: touched `session.runtime`, so the permission editor was TENANT-BLIND BY CONSTRUCTION — every
210
- #: tenant was handed tenant #0's two grid modules whether or not it is provisioned for either.
211
- #:
212
- #: ⛔ WHY THE DERIVATION LIVES HERE AND NOT AT THE ROUTE. `routes_nav.py::nav` already applies
213
- #: exactly this rule and has since wave 18; the perms route is the ONE surface that skipped it.
214
- #: A second copy of the rule at a second call site is how the two came apart in the first place,
215
- #: so the rule is stated once, as a pure function of (runtime, catalogue), and both the route and
216
- #: `verify_perm_scope` ask THIS.
217
- def tenant_modules(runtime):
218
- """The registry keys this TENANT is provisioned for. None means "every key".
219
-
220
- ⛔ FAIL-CLOSED ON THE EMPTY LIST, and that is not the obvious reading. `routes_admin`'s own
221
- docstring records two shipped fail-OPENs of exactly this shape (`modules: []` and
222
- `bus: [<unknown id>]` both read as UNRESTRICTED), so ONLY the literal `'all'` — or an absent
223
- key, which is how the compiled-in tenant #0 record is written — means every module. A
224
- bucket-provisioned tenant gets `{"modules": rec.get("modules", [])}`
225
- (`harness/runtime.py::_builder_from_record`), so `[]` there means the tenant was provisioned
226
- for NOTHING and must be told so, never handed the catalogue.
227
-
228
- Legacy keys are migrated on read for the same reason `allowed_modules` does it: a tenant
229
- record written before a rename must not silently lose the module it names.
230
- """
231
- cfg = getattr(getattr(runtime, 'tenant', None), 'config', None) or {}
232
- raw = cfg.get('modules', 'all')
233
- if raw is None or raw == 'all':
234
- return None
235
- if isinstance(raw, str):
236
- raw = [raw]
237
- return {_LEGACY_KEYS.get(str(k).strip(), str(k).strip())
238
- for k in raw if str(k or '').strip()}
239
-
240
-
241
- def tenant_has_module(runtime, key):
242
- """Is `key` in this tenant's provisioned set? PARENT-AWARE, like `may_open`.
243
-
244
- `routes_nav` keeps a page whose `parent` is granted even when the child key is not, and a
245
- governable list that disagreed with the nav would offer an admin a wall for a database the
246
- tenant cannot open — or hide one it can.
247
- """
248
- allowed = tenant_modules(runtime)
249
- if allowed is None or key in allowed:
250
- return True
251
- parent = registry.BY_KEY.get(key, {}).get('parent')
252
- return parent in allowed if parent else False
253
-
254
-
255
- def tenant_governable_modules(runtime, topics, ut_entries=()):
256
- """The databases the permission editor may render for THIS tenant, in stable order.
257
-
258
- ⚠ NAMED `tenant_*` TO STAY DISTINCT FROM `registry.governable_modules()`, which lane E
259
- published under contract C3 and which this consumes. They answer different questions —
260
- registry's is "which topics are governable at all, platform-wide"; this one is "which
261
- databases does THIS tenant govern" — and two same-named functions one import apart is the
262
- [[one-question-two-normalizers]] shape waiting to happen.
263
-
264
- `topics` — the registry-backed grid modules the editor can build pickers for, as
265
- `[{'key': ..., 'label'?: ...}]`, i.e. `registry.governable_modules()` narrowed
266
- to the topics the caller has a field-schema arm for. The CALLER owns that
267
- narrowing because the schema behind each arm is code
268
- (`routes_admin::_module_fields`), not a policy list. Tenant-blind on the way
269
- in; this function is the only place the tenant is applied.
270
- `ut_entries` — this tenant's own `ut_*` databases, in `user_tables.nav_entries` shape.
271
-
272
- Returns `[{'key', 'label', 'enforced'}]`.
273
-
274
- ⛔⛔ `enforced` IS THE HONEST HALF AND IT IS NOT DECORATION. `perm_scope` walls the REGISTRY
275
- topics — `grid_assembly` calls `may_access`, `visible_fields` and `apply_row_scope` on every
276
- read. It is never consulted for a `ut_*` database: `routes_tables`' visibility is
277
- `user_tables.may_open` (creator, admin, or a `core.shares` grant) and there is no module
278
- grant in that path at all. So a stored `ut_*` wall would be INERT — the editor would say
279
- DENY, the table routes would keep serving, and nothing anywhere would say so. That is the
280
- same class as the empty-dropdown defect (wave 26 item 24): a control that answers when it
281
- cannot. The database is LISTED, because the owner asked to see every database; the wall is
282
- declared unenforced, and `routes_admin::_clean_perms` REFUSES to store one against it rather
283
- than accepting a rule nobody applies. Arming `perm_scope` over `ut_*` is a change to
284
- `routes_tables`/`user_tables` (lane A's fence) — booked, not faked.
285
- """
286
- out = []
287
- for t in topics or ():
288
- key = str((t or {}).get('key') or '').strip()
289
- if not key or not tenant_has_module(runtime, key):
290
- continue
291
- # The LABEL comes from the registry, never from a dict beside the list. `_MODULE_LABELS`
292
- # was a hardcoded copy of two registry labels, so wave 33's R2 relabel ("Odoo customers" /
293
- # "Odoo products") would have moved one of them and left the permission editor showing the
294
- # old name with nothing to notice.
295
- label = (t.get('label') or registry.BY_KEY.get(key, {}).get('label') or key)
296
- out.append({'key': key, 'label': label, 'enforced': True})
297
- seen = {r['key'] for r in out}
298
- for e in ut_entries or ():
299
- key = str((e or {}).get('key') or '').strip()
300
- if not key or key in seen:
301
- continue
302
- seen.add(key)
303
- out.append({'key': key, 'label': (e.get('label') or key), 'enforced': False})
304
- return out
305
-
306
-
307
- def enforced_module_keys(modules):
308
- """The subset of `governable_modules`' answer a permission block may actually be stored for."""
309
- return {m['key'] for m in (modules or ()) if m.get('enforced')}
310
-
311
-
312
  def landing_page(user):
313
  """Where a restricted user lands: the first MAIN-chrome page in their grant. None for
314
  unrestricted users (the client lands on the Analyst, as the host does).
 
199
  return out
200
 
201
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
202
  def landing_page(user):
203
  """Where a restricted user lands: the first MAIN-chrome page in their grant. None for
204
  unrestricted users (the client lands on the Analyst, as the host does).
platform/core/registry.py CHANGED
@@ -22,22 +22,6 @@ Fields:
22
  2026-07-27 pass (keep = Sales + Customers family + Agents + Procurement +
23
  Collections + Analyst/plumbing): products, assortment, financial, management,
24
  pricing, inventory, vendor, warehouse, returns, outreach, complexity, health.
25
- subject set -> the REAL-WORLD POPULATION this row presents, as a source-qualified id
26
- ('odoo:res.partner'). ⭐ W33-T41 / item 12a: it is what makes "two databases present
27
- the same thing" a statement code can refuse, and it exists because NO OTHER FIELD
28
- CAN CARRY THAT TEST. Not the key (`product_data` and `ut_odoo_products` are different
29
- keys by construction — that is the bug, not the check), and NOT THE LABEL: the owner's
30
- own complaint is "why do we have 'Odoo products' ALREADY with the current Products
31
- database?", i.e. the twin was already carrying the label the legacy row is about to
32
- take (W33-T42). Between T42 and T44 the two rows legitimately share a label, so a
33
- label-keyed check would refuse the rename that fixes the bug.
34
- ⚠ A row without `subject` claims nothing and conflicts with nothing — silence is not
35
- a claim, so adding this field cannot retro-refuse anything already shipped.
36
- topic set -> this row is a GOVERNABLE DATABASE and this is the word the rest of the codebase
37
- already calls it by (`routes_grid._SCOPES`, `routes_alerts._TOPICS`, `scope=` on the
38
- grid routes). ⭐ W33-T46 / contract C3: it is what `governable_modules()` keys on, and
39
- it exists so `routes_admin`'s permission list can be DERIVED from the registry instead
40
- of being the hardcoded two-element tuple that made every tenant see Customer + Product.
41
  note one-line scope note shown in the UI / docs
42
  """
43
  # WAVE-9 I8 — `'source'`: which connected system a module's data comes from.
@@ -81,14 +65,7 @@ REGISTRY = [
81
  # myday->customer_list was done. The STORE keys are deliberately NOT renamed:
82
  # 'customer_table_workspace' and 'customer_lists' hold real user data, and renaming them
83
  # would orphan every saved view. Label-only changes need none of that machinery.
84
- # LABEL renamed 'Customer' -> 'Odoo customers' (owner item 12, 2026-08-14 / W33-T42): the
85
- # naming convention of every other database from this connector ("Odoo orders", "Odoo vendor
86
- # bills"). ⭐ STILL LABEL-ONLY, and the note above is the precedent — the KEY does not move, so
87
- # no grant, saved view, cohort, formula, shared column or `?page=` deep link is touched.
88
- # ⛔ `nav_meta` could not have done this: `routes_nav.py` raises `400 name_not_allowed` for any
89
- # key that does not start with `ut_`, so a tenant rename cannot reach a registry module at all.
90
- {'key': 'customer_data', 'label': 'Odoo customers', 'brand': True, 'hq': False, 'validate': False, 'source': 'Odoo', 'parent': 'customers',
91
- 'subject': 'odoo:res.partner', 'topic': 'customer',
92
  'note': 'Build, filter and save customer lists: the whole scoped book with per-customer '
93
  'metrics; saved lists are visible, editable formulas (Call list and Win-back ship '
94
  'as templates) plus hand-picked members. Sub-module of Customers (owner nav redesign 2026-07-23; replaced My Day).'},
@@ -124,10 +101,7 @@ REGISTRY = [
124
  'group_only': True,
125
  'note': 'Folder for the product surfaces (Product table). The archived SKU dashboard is '
126
  'not part of this family.'},
127
- # LABEL renamed 'Product' -> 'Odoo products' (owner item 12, 2026-08-14 / W33-T42) — see the
128
- # customer_data note above; label-only, the key stays `product_data`.
129
- {'key': 'product_data', 'label': 'Odoo products', 'brand': True, 'hq': False, 'validate': True, 'source': 'Odoo', 'parent': 'products_family',
130
- 'subject': 'odoo:product.product', 'topic': 'product',
131
  'note': 'The SKU catalogue as a grid: per-product revenue, units and (consolidated) '
132
  'stock columns, with saved views and custom fields. Identity = the SKU code.'},
133
  {'key': 'products', 'label': 'SKU', 'brand': True, 'hq': False, 'validate': True, 'archived': True,
@@ -223,90 +197,3 @@ BY_KEY = {m['key']: m for m in REGISTRY}
223
 
224
  def brand_filterable(key) -> bool:
225
  return bool(BY_KEY.get(key, {}).get('brand'))
226
-
227
-
228
- # ═════════════════════════════════════════════════════════════════════════════════════════════
229
- # ⭐⭐ W33-T41 / item 12a — ONE SUBJECT, ONE DATABASE
230
- # ════════════════════════��════════════════════════════════════════════════════════════════════
231
- #
232
- # The owner: *"why do we have 'Odoo products' already with the current Products database? The
233
- # Unique ID is redundant, make sure this never happens. One unique ID per database always."*
234
- #
235
- # ⛔ THE CHECK HAD TO GO WHERE THE BUG WAS MADE, AND THAT IS NOT WHERE ANYONE WOULD LOOK.
236
- # `core.user_tables.create` — the UI door — HAS had a collision loop all along: it slugs the
237
- # label to a key and appends `_2 … _49` until the key is free. It never fired here for two
238
- # reasons, and both matter. (1) It compares KEYS inside ONE tenant's `user_tables` document,
239
- # and `customer_data` / `product_data` are compiled REGISTRY rows that never enter that
240
- # document at all — the two halves of the duplicate live in different namespaces and only meet
241
- # in `routes_nav.py::nav`, where the registry pages and `user_tables.nav_entries` are merged
242
- # for display. (2) The connector path never calls `create()`: `odoo_relational.
243
- # _ensure_table_inplace` writes `cur[key] = {...}` straight into the document. So a check bolted
244
- # onto `create()` would have been green through the entire life of this defect.
245
- #
246
- # ⚠ THE SUFFIX LOOP IS ALSO WHY A RELABEL ALONE WOULD HAVE MADE THIS WORSE, MECHANICALLY: a
247
- # naive "just rename Product to Odoo products" mints `ut_odoo_products_2` — the owner's
248
- # redundant id again, from the fix for it.
249
- #
250
- #: subject -> the ONE registry key entitled to present it. Derived, never typed twice.
251
- SUBJECT_CLAIMS = {m['subject']: m['key'] for m in REGISTRY if m.get('subject')}
252
-
253
-
254
- # ═════════════════════════════════════════════════════════════════════════════════════════════
255
- # ⭐⭐ W33-T46 / CONTRACT C3 — THE GOVERNABLE MODULE LIST IS DERIVED, NOT A LITERAL
256
- # ═════════════════════════════════════════════════════════════════════════════════════════════
257
- #
258
- # The owner, on his FOURTH flag of the same thing: *"its an error that ALL fucking tenant can see
259
- # both Customer and Product database under Manage User, this is only for Royal Imports tenant.
260
- # Make sure per tenant configuration always work."*
261
- #
262
- # ⛔ THE CAUSE IS A TWO-ELEMENT TUPLE. `routes_admin._PERM_MODULES = ("customer_data",
263
- # "product_data")` — a compiled literal that `get_perms` builds its whole list from without ever
264
- # touching `session.runtime`. It is tenant-blind BY CONSTRUCTION, which is why no amount of
265
- # per-tenant work elsewhere ever fixed it, and why the answer is not "filter the tuple" but
266
- # "delete the tuple": a second hand-typed copy of a fact the registry already holds is a thing
267
- # that CAN go stale, and this one did, four times.
268
- #
269
- # ⚠ THE LINE BETWEEN THE TWO HALVES, stated because both sides need it: this map is
270
- # REGISTRY-SIDE and TENANT-BLIND — it answers *"which compiled modules are governable, and what
271
- # are they called"*, nothing more. Applying the tenant's own `config["modules"]` filter and
272
- # merging that tenant's `ut_*` databases is the CALLER's half (`routes_admin`, D's fence), which
273
- # is exactly what `routes_nav.py` already does correctly and `routes_admin` never did.
274
- def governable_modules():
275
- """`{topic: {'key','label','subject'}}` — every registry row that is a governable DATABASE.
276
-
277
- A fresh dict per call, deliberately: a module-level mapping handed to a route is one
278
- `.pop()` away from being a different list for every request after the first.
279
- """
280
- return {m['topic']: {'key': m['key'], 'label': m['label'], 'subject': m.get('subject')}
281
- for m in REGISTRY
282
- if m.get('topic') and not m.get('archived') and not m.get('group_only')}
283
-
284
-
285
- def subject_of(key):
286
- """The subject a REGISTRY row claims, or None. (`ut_*` databases carry theirs on the stored
287
- definition instead — they are per-tenant data, not compiled rows.)"""
288
- return (BY_KEY.get(key) or {}).get('subject')
289
-
290
-
291
- def subject_conflict(subject, key, claimed=None):
292
- """The OTHER key already presenting `subject`, or None.
293
-
294
- `claimed` is an optional {subject: key} of claims outside the registry — in practice the
295
- caller's own tenant document, whose stored definitions carry a `subject` stamp. It is a
296
- PARAMETER rather than a lookup on purpose: the store is per-tenant, two tenants both holding
297
- an "Odoo customers" is entirely legitimate, and a module-level cache of somebody's tables is
298
- how D-169 happened. This function knows nothing about which tenant it is serving.
299
-
300
- ⭐ THE TEST IS "A DIFFERENT KEY", NOT "IS THIS SUBJECT TAKEN". `_ensure_table_inplace` and
301
- `ut_ensure` re-run on every boot and every 1800 s resync, adopting their table BY KEY — so a
302
- check phrased "is this subject already claimed?" goes red on the second boot against the very
303
- table it created on the first, and the container fails to start. Idempotency is not an
304
- exception to the rule here; it is the common case.
305
- """
306
- if not subject:
307
- return None # silence is not a claim
308
- for holder in (SUBJECT_CLAIMS, claimed or {}):
309
- other = holder.get(subject)
310
- if other and other != key:
311
- return other
312
- return None
 
22
  2026-07-27 pass (keep = Sales + Customers family + Agents + Procurement +
23
  Collections + Analyst/plumbing): products, assortment, financial, management,
24
  pricing, inventory, vendor, warehouse, returns, outreach, complexity, health.
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
25
  note one-line scope note shown in the UI / docs
26
  """
27
  # WAVE-9 I8 — `'source'`: which connected system a module's data comes from.
 
65
  # myday->customer_list was done. The STORE keys are deliberately NOT renamed:
66
  # 'customer_table_workspace' and 'customer_lists' hold real user data, and renaming them
67
  # would orphan every saved view. Label-only changes need none of that machinery.
68
+ {'key': 'customer_data', 'label': 'Customer', 'brand': True, 'hq': False, 'validate': False, 'source': 'Odoo', 'parent': 'customers',
 
 
 
 
 
 
 
69
  'note': 'Build, filter and save customer lists: the whole scoped book with per-customer '
70
  'metrics; saved lists are visible, editable formulas (Call list and Win-back ship '
71
  'as templates) plus hand-picked members. Sub-module of Customers (owner nav redesign 2026-07-23; replaced My Day).'},
 
101
  'group_only': True,
102
  'note': 'Folder for the product surfaces (Product table). The archived SKU dashboard is '
103
  'not part of this family.'},
104
+ {'key': 'product_data', 'label': 'Product', 'brand': True, 'hq': False, 'validate': True, 'source': 'Odoo', 'parent': 'products_family',
 
 
 
105
  'note': 'The SKU catalogue as a grid: per-product revenue, units and (consolidated) '
106
  'stock columns, with saved views and custom fields. Identity = the SKU code.'},
107
  {'key': 'products', 'label': 'SKU', 'brand': True, 'hq': False, 'validate': True, 'archived': True,
 
197
 
198
  def brand_filterable(key) -> bool:
199
  return bool(BY_KEY.get(key, {}).get('brand'))
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
platform/core/shares.py CHANGED
@@ -43,32 +43,10 @@ SHARES_KEY = 'object_shares'
43
  #: namespace by typo, which would silently grant nothing to nobody and read as "sharing is broken".
44
  KINDS = ('view', 'folder', 'database')
45
 
46
- #: `*` is "everyone who can already open the surface". It is NOT "every account on the platform".
47
- #: Spelled as a single character so it can never collide with a username (usernames are lower-case
48
- #: and non-empty by `core/users.py`, and are checked against this explicitly below).
49
- #:
50
- #: ⛔⛔ **AND WHAT THAT MEANS DEPENDS ON THE KIND — THE LINE THIS NOTE USED TO CARRY WAS FALSE FOR
51
- #: ONE OF THE THREE** (W33-T30, `waves/wave32/sharing-audit.md` S-8). It read *"the module/table
52
- #: wall runs FIRST and this never widens past it"*, flatly, and the audit's own words for that are
53
- #: *"the third docstring in this audit describing a check that is not on the path"*. Corrected
54
- #: here rather than deleted, because the sentence is TRUE of two kinds and the difference is the
55
- #: whole point:
56
- #: * `kind='view'` / `kind='folder'` on a GOVERNED module (`customer_data`, `product_data`) —
57
- #: the sentence holds. `require_session` plus the topic's own gate run first, and the
58
- #: receiver's row scope and hidden-field closure are applied BEFORE any foreign view is
59
- #: merged, so a grant can only narrow-or-equal what that account could already reach.
60
- #: * `kind='database'` on a `ut_*` table — **THE SENTENCE IS FALSE AND THIS REGISTRY IS THE
61
- #: ONLY WALL.** `routes_admin._PERM_MODULES` is `("customer_data", "product_data")` and
62
- #: `_clean_perms` 400s anything else, so no row filter and no hidden field can even be
63
- #: DECLARED for a user table; `routes_tables.py` makes zero `perm_scope` calls and passes
64
- #: `hidden_keys=frozenset()`. A `database` grant is therefore ALL-OR-NOTHING — every row,
65
- #: every column — and an `*` database grant admits every account in the tenant to all of it.
66
- #:
67
- #: ⚠ THAT IS A REAL CAPABILITY, DELIBERATELY KEPT, not a hole to plug in passing. What was wrong
68
- #: was a docstring promising a second wall that does not exist for this kind; scoping user tables
69
- #: is booked (S-8), not done. `routes_shares.py`'s module docstring says the same thing at the
70
- #: OTHER door — the audit's fix is "say so at both", and one door saying it is how the next reader
71
- #: gets the confident half [[one-question-two-normalizers]].
72
  EVERYONE = '*'
73
 
74
  ROLES = ('view', 'edit')
 
43
  #: namespace by typo, which would silently grant nothing to nobody and read as "sharing is broken".
44
  KINDS = ('view', 'folder', 'database')
45
 
46
+ #: `*` is "everyone who can already open the surface". It is NOT "every account on the platform" —
47
+ #: the module/table wall runs FIRST and this never widens past it. Spelled as a single character
48
+ #: so it can never collide with a username (usernames are lower-case and non-empty by
49
+ #: `core/users.py`, and are checked against this explicitly below).
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
50
  EVERYONE = '*'
51
 
52
  ROLES = ('view', 'edit')
platform/core/store.py CHANGED
@@ -1,838 +1,817 @@
1
- """Tiny JSON persistence on a PRIVATE Hugging Face Dataset repo — the platform's ONLY writable store
2
- (Odoo stays strictly read-only). Holds the user registry, customer notes and per-user tags.
3
-
4
- Auth uses HF_TOKEN (a Space secret / .env value with write access to the royal-imports org). The
5
- repo is created private on first write. Reads use a short in-process cache that writes refresh, so
6
- within a running instance reads are consistent with writes; a fresh instance re-fetches the latest.
7
-
8
- WAVE 18 (C1-TENANT, R2): the store is now a CLASS, one instance per repo. The module-level
9
- functions below delegate to the default instance (`OS_DATA_REPO`, tenant #0) — every existing
10
- `import core.store as store` caller is byte-for-byte unaffected, which the gate battery proves.
11
- `for_repo(repo_id)` hands back the bound instance for a tenant that owns its OWN dataset repo
12
- (nurilab, gtmlab, …); `harness.runtime.TenantRuntime` binds it from the tenant record. Each
13
- instance carries its own cache / dirty-marks / flush workers / commit-gap clock, so one tenant's
14
- write burst can never starve another's flush window.
15
- """
16
- import io
17
- import os
18
- import json
19
- import threading
20
- import time as _time
21
- from pathlib import Path
22
-
23
- try:
24
- from dotenv import load_dotenv
25
- load_dotenv(Path(__file__).resolve().parents[1] / '.env')
26
- except Exception:
27
- pass
28
-
29
- from huggingface_hub import HfApi, hf_hub_download
30
- from huggingface_hub.utils import EntryNotFoundError, RepositoryNotFoundError
31
-
32
- REPO = os.environ.get('OS_DATA_REPO', 'royal-imports/cfo-os-data')
33
- _FLUSH_DELAY = 2.0 # coalesce a burst of edits into one commit
34
- _FLUSH_MIN_GAP = 20.0 # floor between commits of one key (the 256-commits/hr budget)
35
-
36
- #: WAVE 29 (item 20 / ruling R11, contract C6) — THE CHANGE TOKEN'S PROCESS STAMP.
37
- #:
38
- #: `Store`'s revision counter is IN-MEMORY, so a restarted Space would begin again at 0 and a
39
- #: browser holding `…:4` from the previous process could read `…:4` from the new one and conclude
40
- #: nothing had changed. Prefixing every token with a per-process stamp makes a restart read as a
41
- #: change, which is the truthful answer: the client's copy predates this process entirely.
42
- #:
43
- #: ⚠ NOT a clock and never compared for order — a token is opaque and compared ONLY for equality.
44
- REV_EPOCH = f'{int(_time.time()):x}{os.getpid():x}'
45
-
46
-
47
- def _token():
48
- return os.environ.get('HF_TOKEN') or None
49
-
50
-
51
- class _Projected(dict):
52
- """One top-level value of a PROJECTED read — a dict that RAISES for the keys that were dropped.
53
-
54
- ⛔⛔ W32-T01: THE ALTERNATIVE IS A SILENT WRONG ANSWER, WHICH IS WHY THIS CLASS EXISTS AT ALL.
55
- A projection built as a plain dict answers `defn.get('rows') or {}` with `{}` — indistinguishable
56
- from a database that genuinely has no rows. The caller then paints an empty grid, exports an
57
- empty CSV, or (worse) writes that emptiness back. This repo has shipped that exact shape before
58
- ([[empty-answer-vs-unfinished-answer]], and a grouped view that painted zero rows).
59
-
60
- So a dropped key is not absent — it is UNREAD, and asking for it raises with the reason and the
61
- fix in the message. `KeyError` is the type on purpose: `dict.__getitem__`'s own failure mode, so
62
- nothing has to learn a new exception; and the message names `get_projection` so the traceback
63
- points at the caller's mistake rather than at this file.
64
-
65
- ⚠ Still a real `dict` for every OTHER purpose — `json.dumps`, `dict(x)`, `**x`, iteration and
66
- `in` all behave normally, because a projection has to serialise onto the wire like any
67
- definition. Only the named keys are booby-trapped.
68
- """
69
-
70
- __slots__ = ('_dropped',)
71
-
72
- def __init__(self, data=None, dropped=()):
73
- super().__init__(data or {})
74
- object.__setattr__(self, '_dropped', frozenset(dropped))
75
-
76
- def _refuse(self, key):
77
- raise KeyError(
78
- f'{key!r} was DROPPED by a projected store read and is not absent — this document came '
79
- f'from Store.get_projection(drop={sorted(self._dropped)!r}). Reading it as empty would '
80
- f'be a silent wrong answer. Use the whole read (Store.get / user_tables.all_tables) if '
81
- f'you need {key!r}.')
82
-
83
- def __getitem__(self, key):
84
- if key in self._dropped:
85
- self._refuse(key)
86
- return super().__getitem__(key)
87
-
88
- def get(self, key, default=None):
89
- if key in self._dropped:
90
- self._refuse(key)
91
- return super().get(key, default)
92
-
93
- def pop(self, key, *a):
94
- if key in self._dropped:
95
- self._refuse(key)
96
- return super().pop(key, *a)
97
-
98
-
99
- def _project(doc, drop):
100
- """A deep COPY of `doc` with `drop`'s keys removed from every top-level value.
101
-
102
- ⭐ The order is the optimisation: build the retained structure FIRST (a shallow comprehension
103
- that never touches the dropped values), then deep-copy THAT. One `json.dumps` over 0.1% of the
104
- bytes, instead of one over all of them.
105
- """
106
- if not isinstance(doc, dict):
107
- return json.loads(json.dumps(doc))
108
- kept = {k: ({kk: vv for kk, vv in v.items() if kk not in drop} if isinstance(v, dict) else v)
109
- for k, v in doc.items()}
110
- copied = json.loads(json.dumps(kept))
111
- return {k: (_Projected(v, drop) if isinstance(v, dict) else v) for k, v in copied.items()}
112
-
113
-
114
- def is_projected(value):
115
- """Did this value come back from a PROJECTED read? — the TAKEN-PATH predicate (W33-T01/D-213).
116
-
117
- ⭐⭐ WHY A PREDICATE AND NOT A COUNTER, and it is the whole reason this exists. `all_defs`
118
- FALLS BACK to the whole read on any failure, silently and by design — so a caller that gets the
119
- right answer has learned nothing about which read produced it. A gate asserting only "the
120
- definitions came back" passes identically when the projection worked and when it quietly did
121
- not, which is [[gate-can-report-green-on-nothing]] wearing a green tick.
122
-
123
- ⭐ This is INTRINSIC rather than instrumented: `_project` stamps every top-level value as a
124
- `_Projected`, so the evidence rides the document itself. Nothing has to be wrapped, no counter
125
- can be installed on the wrong object ([[measure-the-real-call]]), and the answer is the same in
126
- production as it is under a gate — a monkey-patched tally is true only while the patch is on.
127
-
128
- ⚠ It answers about ONE VALUE (a table's definition), not about a document: `_project` returns a
129
- plain dict OF `_Projected` values, so `is_projected(doc)` is False and `is_projected(doc[key])`
130
- is True. That is the right grain — the thing a route holds and reads is the definition.
131
- """
132
- return isinstance(value, _Projected)
133
-
134
-
135
- class Store:
136
- """One repo's persistence: JSON-per-key + raw byte paths, with the wave-7 async flush.
137
-
138
- Every method body is the pre-wave-18 module function, verbatim except globals→fields.
139
- The behaviour contracts that matter are stated on each method and are UNCHANGED:
140
- strict-read-guards-the-write, exists()-is-True-on-uncertainty, lenient display reads.
141
- """
142
-
143
- def __init__(self, repo):
144
- self.repo = str(repo)
145
- self._lock = threading.RLock()
146
- self._cache = {}
147
- # --- async flush state (wave-7 W3, 2026-07-28) ------------------------------------
148
- # One typed character in a Notes cell used to cost THREE synchronous network calls
149
- # inside the component round-trip. update(..., flush='async') applies the mutation to
150
- # the in-process cache synchronously (read-your-writes: every reader consults the
151
- # cache first) and a background worker coalesces uploads. The restart-wipe protection
152
- # is UNCHANGED where it matters: the FIRST update of a key in a fresh process still
153
- # strict-reads. Known, accepted: with a second concurrent writer the stale window
154
- # widens from ~one round-trip to ~the flush gap.
155
- self._owned = {} # name -> True once cache reflects our own last accepted write
156
- self._dirty = {} # name -> cache is ahead of the hub
157
- self._flushers = {} # name -> live worker thread
158
- self._repo_ok = False # create_repo(exist_ok=True) confirmed once per instance
159
- self._last_flush = {} # name -> monotonic time of the last completed upload
160
- # --- the CHANGE TOKEN (wave 29, item 20 / C6) ------------------------------------
161
- # name -> how many accepted writes this process has applied to that bucket, and when.
162
- # Read by `revision()` in O(1) with NO download and NO deep copy; see its docstring for
163
- # why that constraint is the whole feature rather than an optimisation.
164
- self._revs = {}
165
- # --- ⭐⭐ W31-T13 (D-177(d)) — THE PER-KEY WRITE LOCK ------------------------------
166
- # `self._lock` guards the CACHE. It used to be held across a sync `update`'s whole
167
- # read-modify-write — a hub download, the caller's `fn`, and a blocking hub upload — so
168
- # one write froze every reader in the process for two network round trips. `get()` takes
169
- # the same lock, and `/nav`, `/workspace` and every permission check are `get()`.
170
- # ⛔ IT IS NOT SAFE TO SIMPLY DROP THE LOCK. Two sync writers of one key would interleave
171
- # read-modify-write and one update would vanish — corruption, not slowness. So the two
172
- # jobs are separated: this lock serialises WRITERS OF ONE KEY (held across the network,
173
- # never touching readers), and `self._lock` is taken only for the microseconds a cache
174
- # read or write needs. Readers never wait on a network call again.
175
- # ⚠ PER KEY, not one write lock: two tenants' buckets, or `user_tables` and `nav_meta`,
176
- # have no reason to serialise against each other.
177
- self._key_locks = {}
178
- self._key_locks_guard = threading.Lock()
179
- self._rev_at = {}
180
-
181
- def available(self):
182
- """True when a token is configured — callers degrade gracefully without one."""
183
- return bool(_token())
184
-
185
- def _download(self, name):
186
- """Parse {name}.json into a dict. Returns {} ONLY when the file is genuinely absent.
187
- Any other failure (network, auth, rate-limit, corrupt JSON) RAISES — a transient read
188
- error must never be mistaken for an empty store, or a caller could overwrite real data
189
- with an empty dict (this is what wiped the user registry on restart)."""
190
- try:
191
- path = hf_hub_download(self.repo, f'{name}.json', repo_type='dataset',
192
- token=_token(), force_download=True)
193
- with open(path, encoding='utf-8') as f:
194
- return json.load(f)
195
- except (EntryNotFoundError, RepositoryNotFoundError, FileNotFoundError):
196
- return {}
197
- except ValueError as e:
198
- # Wave 18 (R2, found by SESSION D): current huggingface_hub wraps a missing REPO in
199
- # ValueError("Force download failed…") with the 404 as __cause__ — so a brand-new
200
- # tenant repo's very first strict read raised here and aborted its very first write.
201
- # A genuinely-absent repo is the same fact as a genuinely-absent file: {}.
202
- cause = getattr(e, '__cause__', None)
203
- if isinstance(cause, (EntryNotFoundError, RepositoryNotFoundError)):
204
- return {}
205
- raise
206
-
207
- def get(self, name, fresh=False):
208
- """Load {name}.json as a dict. LENIENT (for display reads): on a transient fetch error,
209
- hand back the last known-good cached value if we have one, else {}. Mutating callers go
210
- through update()/_read_strict, which RAISE instead."""
211
- with self._lock:
212
- if not fresh and name in self._cache:
213
- return json.loads(json.dumps(self._cache[name])) # hand back a copy
214
- try:
215
- data = self._download(name)
216
- except Exception as e:
217
- try: # lazy import — core stays layer-clean
218
- import harness.telemetry as _tel # noqa: PLC0415
219
- _tel.error(f'store:get:{name}', e,
220
- fallback='cached' if name in self._cache else 'empty')
221
- except Exception:
222
- pass
223
- if name in self._cache:
224
- return json.loads(json.dumps(self._cache[name]))
225
- return {}
226
- self._cache[name] = data
227
- return json.loads(json.dumps(data))
228
-
229
- def get_projection(self, name, drop=()):
230
- """Like `get`, but each top-level VALUE comes back WITHOUT the keys named in `drop`.
231
-
232
- ⭐⭐ W32-T01 (D-185) — THE POINT IS THE COPY, NOT THE PAYLOAD SIZE ON THE WIRE.
233
-
234
- `get()` is honest and expensive: it hands back `json.loads(json.dumps(...))` so a caller
235
- cannot mutate the cache, and on tenant #0's `user_tables` that copy is **28.6 MB / 81,330
236
- rows / ~703 ms warm**, of which **99.9% is `rows`** — which no permission check, no nav
237
- render and no workspace envelope ever reads. `/nav` used to pay ~13 of those copies per
238
- request; W31-T10 made it ONE, and that one is still 703 ms of pure copying. This is the
239
- floor D-185 named: copy only what was asked for.
240
-
241
- ⛔⛔ **THE PROJECTION IS NEVER CACHED, AND THAT LINE IS THE WHOLE SAFETY ARGUMENT.** A
242
- rows-less document in `self._cache` would be served to the next WHOLE reader, and the next
243
- `update()` would upload it — every row in the tenant deleted, by a read. So the cold path
244
- downloads whole, caches WHOLE (exactly as `get` does), and projects the RETURN value only.
245
- `verify_store_projection.py` asserts the cache still contains `rows` after a projected read,
246
- with an NC that reds when the projection is cached instead.
247
-
248
- ⚠ **`self._lock` is held across the copy, same as `get`** — the retained structure is
249
- ~0.1% of the bytes, so this is microseconds where `get` is most of a second. Narrowing it
250
- would let a concurrent writer mutate a value mid-serialise for no measurable gain.
251
-
252
- ⚠ **A dropped key RAISES on read rather than reading as absent** (`_Projected`). A
253
- projection that answered `{}` for `rows` would paint an empty grid — the exact
254
- [[empty-answer-vs-unfinished-answer]] shape this repo has shipped before — and the caller
255
- that most wants rows is the one least able to tell. See `_Projected`.
256
-
257
- ⚠ Lenient, like `get`: a transient fetch error falls back to the cached document (projected)
258
- and then to `{}`. Mutating callers still go through `update()`/`_read_strict`, which never
259
- project — a write must always read whole.
260
- """
261
- drop = frozenset(str(d) for d in drop)
262
- with self._lock:
263
- if name in self._cache:
264
- return _project(self._cache[name], drop)
265
- try:
266
- data = self._download(name)
267
- except Exception as e:
268
- try: # lazy import — core stays layer-clean
269
- import harness.telemetry as _tel # noqa: PLC0415
270
- _tel.error(f'store:get_projection:{name}', e,
271
- fallback='cached' if name in self._cache else 'empty')
272
- except Exception:
273
- pass
274
- with self._lock:
275
- if name in self._cache:
276
- return _project(self._cache[name], drop)
277
- return {}
278
- with self._lock:
279
- self._cache[name] = data # ⛔ the WHOLE document, never the projection
280
- return _project(data, drop)
281
-
282
- def _key_lock(self, name):
283
- """The write lock for ONE key (W31-T13). Created on demand; never removed."""
284
- n = str(name)
285
- with self._key_locks_guard:
286
- lk = self._key_locks.get(n)
287
- if lk is None:
288
- lk = self._key_locks[n] = threading.RLock()
289
- return lk
290
-
291
- def _read_strict(self, name):
292
- """Like get(fresh=True) but RAISES on any transient read error ({} only for a genuinely
293
- absent file), so a failed read ABORTS a read-modify-write instead of merging into {}.
294
-
295
- ⭐ W31-T13 — THE DOWNLOAD IS OUTSIDE `self._lock` NOW, and the re-check after it is the
296
- whole of what makes that safe. A hub read takes a round trip; holding the cache lock
297
- across it is what froze every reader. But moving it out opens a window in which an ASYNC
298
- writer can accept a row into the cache while our download is in flight — and assigning our
299
- (older) hub copy afterwards would erase it. That is D-177's original symptom exactly
300
- (`POST /rows` → 201, row gone), re-created by the lock split rather than by a call site.
301
- So the dirty mark is re-read AFTER the download, under the short lock, and a key that
302
- became dirty keeps its cache.
303
- """
304
- data = self._download(name) # network, no lock held
305
- with self._lock:
306
- if self._dirty.get(name) and self._owned.get(name) and name in self._cache:
307
- # Our copy is stale by construction: a pending upload is going to overwrite the
308
- # hub anyway, so the fresh read cannot preserve a remote write — it can only
309
- # destroy a local one we have already acknowledged. Same argument, and the same
310
- # three conditions, as `update`'s pending-async branch below.
311
- return json.loads(json.dumps(self._cache[name]))
312
- self._cache[name] = data
313
- return json.loads(json.dumps(data))
314
-
315
- def exists(self, name):
316
- """True if {name}.json is confirmed present, False if confirmed absent, and True on ANY
317
- uncertainty — a seeder must never clobber a registry we merely failed to reach."""
318
- try:
319
- files = HfApi(token=_token()).list_repo_files(self.repo, repo_type='dataset')
320
- return f'{name}.json' in files
321
- except RepositoryNotFoundError:
322
- return False
323
- except Exception:
324
- return True
325
-
326
- # ------------------------------------------------------- the CHANGE TOKEN (wave 29, C6)
327
- def _bump(self, name):
328
- """Advance one bucket's write counter. THE CALLER HOLDS THE LOCK.
329
-
330
- Called from the two places an accepted write becomes visible to readers of this process
331
- — `put()` (which `update(flush='sync')` goes through) and `update()`'s async branch. NOT
332
- from `_flush_worker`: that uploads data a bump already accounted for, and counting it
333
- again would report a change to every browser each time the coalescing worker woke.
334
- """
335
- n = str(name)
336
- self._revs[n] = self._revs.get(n, 0) + 1
337
- self._rev_at[n] = _time.time()
338
-
339
- def revision(self, name):
340
- """`{'rev', 'updated_at', 'token'}` for ONE bucket — a dict read, nothing else.
341
-
342
- ⛔ THE ZERO-COST PROPERTY IS THE FEATURE, NOT AN OPTIMISATION, and the arithmetic is why
343
- this method exists at all. A browser asking "has anything changed" by re-fetching rows
344
- costs THREE full-tenant deep copies under ONE lock (`get()` json-round-trips on the way
345
- in and on the way out) on ONE uvicorn process, at a documented ceiling of 35.8 MB / ~1.4 s
346
- per bucket — so two tabs polling saturate the server and every unrelated write queues
347
- behind them. This performs no download, no `get()`, and no copy of the value.
348
- ⇒ Anything added here that reads a bucket destroys the whole point (`verify_live_workspace`
349
- counts `get`/`_read_strict`/`_download` on the serving path and goes red on one).
350
-
351
- `rev` and `updated_at` are `core/store_pg.py`'s own column names, on purpose: that backend
352
- ALREADY increments `rev` and sets `updated_at = now()` on every write, so the token stops
353
- being inert the moment D-4 flips, with the wire shape unchanged.
354
-
355
- ⚠ WHAT IT CANNOT SEE, stated rather than implied: this counter is per PROCESS, so a write
356
- by a SECOND container against the same HF dataset is invisible here. That is not a limit
357
- this introduces — `get()` caches a bucket for the life of the process and never re-reads
358
- it, so the other container's write is invisible to every reader on this backend already.
359
- The token is exactly as fresh as the store it reports on. Postgres has no such gap.
360
- ⚠ Bytes (`upload_bytes`/`delete_path`) are a different address space and are NOT counted:
361
- an attachment is fetched on explicit user action, never polled.
362
- """
363
- n = str(name)
364
- with self._lock:
365
- rev = int(self._revs.get(n, 0))
366
- at = self._rev_at.get(n)
367
- return {'rev': rev, 'updated_at': at, 'token': f'{REV_EPOCH}:{rev}'}
368
-
369
- def _upload(self, name, data):
370
- """The raw hub write. create_repo is confirmed ONCE per instance — it was a full extra
371
- API round-trip on every put(), pure latency after the first success."""
372
- api = HfApi(token=_token())
373
- if not self._repo_ok:
374
- api.create_repo(self.repo, repo_type='dataset', private=True, exist_ok=True)
375
- self._repo_ok = True
376
- buf = io.BytesIO(json.dumps(data, ensure_ascii=False, indent=2).encode('utf-8'))
377
- api.upload_file(path_or_fileobj=buf, path_in_repo=f'{name}.json', repo_id=self.repo,
378
- repo_type='dataset', commit_message=f'update {name}')
379
-
380
- def upload_bytes(self, path_in_repo, data, message=None):
381
- """Persist RAW BYTES at an arbitrary path in the tenant dataset (wave-8 C5, documents).
382
- Bytes are never cached in-process: large, cold, fetched on explicit user action."""
383
- if not self.available():
384
- raise RuntimeError('No HF_TOKEN configured — persistence is unavailable.')
385
- with self._lock:
386
- api = HfApi(token=_token())
387
- if not self._repo_ok:
388
- api.create_repo(self.repo, repo_type='dataset', private=True, exist_ok=True)
389
- self._repo_ok = True
390
- api.upload_file(path_or_fileobj=io.BytesIO(data), path_in_repo=path_in_repo,
391
- repo_id=self.repo, repo_type='dataset',
392
- commit_message=message or f'add {path_in_repo}')
393
-
394
- def download_bytes(self, path_in_repo):
395
- """Raw bytes back, or None when absent/unreachable — a missing attachment degrades to
396
- "that file is gone", never to a broken page."""
397
- if not self.available():
398
- return None
399
- try:
400
- local = hf_hub_download(self.repo, path_in_repo, repo_type='dataset',
401
- token=_token())
402
- return Path(local).read_bytes()
403
- except (EntryNotFoundError, RepositoryNotFoundError):
404
- return None
405
- except Exception:
406
- return None
407
-
408
- def delete_path(self, path_in_repo):
409
- """Remove a stored file. Absent is SUCCESS — deleting what is already gone is the goal."""
410
- if not self.available():
411
- return False
412
- try:
413
- HfApi(token=_token()).delete_file(path_in_repo=path_in_repo, repo_id=self.repo,
414
- repo_type='dataset',
415
- commit_message=f'delete {path_in_repo}')
416
- return True
417
- except (EntryNotFoundError, RepositoryNotFoundError):
418
- return True
419
- except Exception:
420
- return False
421
-
422
- def put(self, name, data):
423
- """Persist {name}.json (creates the private repo on first write). Updates the cache."""
424
- if not self.available():
425
- raise RuntimeError('No HF_TOKEN configured — persistence is unavailable.')
426
- with self._lock:
427
- self._upload(name, data)
428
- self._cache[name] = json.loads(json.dumps(data))
429
- self._owned[name] = True
430
- self._last_flush[name] = _time.monotonic()
431
- self._bump(name) # C6 — AFTER the upload: a raise here is not a write
432
- return self._cache[name]
433
-
434
- def _flush_worker(self, name):
435
- """Coalescing uploader for one key. Runs while dirty; retries with backoff on failure —
436
- the cache stays the truth and the dirty mark survives, so nothing is lost silently.
437
-
438
- ⭐⭐ W31-T13 — THIS WORKER IS THE SECOND WRITER, AND THE PER-KEY LOCK MUST COVER IT.
439
- ⛔ THE RACE THE LOCK SPLIT OPENED, spelled out because it is silent and survives a restart
440
- as real data loss. `update`'s big lock used to span `_read_strict` + `fn` + the upload, so
441
- this worker could never snapshot mid-update. With the cache lock narrowed it can:
442
- async write -> `_dirty=True`, this worker scheduled, sleeps `_FLUSH_DELAY`
443
- sync update t+0.5 -> takes the dirty branch, runs `fn`, enters `_upload` (no cache
444
- lock held — that is the whole point of the ticket)
445
- worker t+2.0 -> `self._lock` is FREE and `_dirty` is still True (the sync commit
446
- has not happened yet), so it snapshots the PRE-UPDATE cache,
447
- clears the mark, and uploads it AFTER the sync upload lands
448
- result -> cache has the new value, the hub has the old one, `_dirty` is
449
- False, and nothing will ever re-upload. Divergence with no
450
- symptom until the next process boots and reads the hub.
451
- ⚠ `user_tables` has ~40 sync writers against 2 async ones, so that window is the COMMON
452
- case, not an exotic one — it is D-177's own shape, re-created one layer down.
453
- ⛔ LOCK ORDER IS KEY-THEN-CACHE, EVERYWHERE. `_update_locked` takes them in that order, so
454
- this must too; taking `self._lock` first on this path is the deadlock. Both sleeps stay
455
- OUTSIDE the key lock — a worker that dozed while holding it would block every writer of
456
- that key for `_FLUSH_MIN_GAP`, which is the freeze this ticket exists to remove.
457
- """
458
- backoff = 2.0
459
- while True:
460
- _time.sleep(_FLUSH_DELAY)
461
- wait, err, snapshot = None, None, None
462
- with self._key_lock(name):
463
- with self._lock:
464
- gap = _time.monotonic() - self._last_flush.get(name, 0.0)
465
- if gap < _FLUSH_MIN_GAP:
466
- wait = _FLUSH_MIN_GAP - gap
467
- elif not self._dirty.get(name):
468
- self._flushers.pop(name, None)
469
- return
470
- else:
471
- self._dirty[name] = False
472
- snapshot = json.loads(json.dumps(self._cache.get(name, {})))
473
- if wait is None:
474
- try:
475
- self._upload(name, snapshot)
476
- with self._lock:
477
- self._last_flush[name] = _time.monotonic()
478
- backoff = 2.0
479
- except Exception as e: # noqa: BLE001
480
- with self._lock:
481
- self._dirty[name] = True # keep it queued; cache is still the truth
482
- err = e
483
- if wait is not None:
484
- _time.sleep(wait)
485
- continue
486
- if err is not None:
487
- try:
488
- import harness.telemetry as _tel # noqa: PLC0415
489
- _tel.error(f'store:flush:{name}', err)
490
- except Exception:
491
- pass
492
- _time.sleep(backoff)
493
- backoff = min(backoff * 2, 60.0)
494
-
495
- def _schedule_flush(self, name):
496
- """Mark dirty + ensure exactly one live worker for the key. Caller holds the lock."""
497
- self._dirty[name] = True
498
- t = self._flushers.get(name)
499
- if t is not None and t.is_alive():
500
- return
501
- t = threading.Thread(target=self._flush_worker, args=(name,), daemon=True,
502
- name=f'store-flush-{self.repo}:{name}')
503
- self._flushers[name] = t
504
- t.start()
505
-
506
- def flush(self, name=None, timeout=30.0):
507
- """Block until pending async writes for `name` (or ALL keys) reach the hub — QA gates
508
- and shutdown hooks use this; the app itself never needs to."""
509
- deadline = _time.monotonic() + timeout
510
- names = [name] if name else None
511
- while _time.monotonic() < deadline:
512
- with self._lock:
513
- pending = {k for k, v in self._dirty.items() if v} | {
514
- k for k, t in self._flushers.items() if t.is_alive()}
515
- if names is not None:
516
- pending &= set(names)
517
- if not pending:
518
- return True
519
- _time.sleep(0.2)
520
- return False
521
-
522
- def _flush_now_at_exit(self):
523
- """Process-exit safety net: push any dirty keys synchronously, bypassing the coalescing
524
- gap — a Space restart must not eat the last seconds of the async window.
525
-
526
- ⚠ W31-T13 — DELIBERATELY NOT KEY-LOCKED, stated so the omission is not read as one. This
527
- runs at interpreter exit with the process going away: a writer mid-`update` will never
528
- finish, so waiting for its key lock could only turn a partial save into no save at all.
529
- Best-effort is the correct contract here, and it is the one case where taking the lock
530
- would be worse than skipping it.
531
- """
532
- with self._lock:
533
- dirty = [k for k, v in self._dirty.items() if v]
534
- snaps = {k: json.loads(json.dumps(self._cache.get(k, {}))) for k in dirty}
535
- for k in dirty:
536
- self._dirty[k] = False
537
- for k, d in snaps.items():
538
- try:
539
- self._upload(k, d)
540
- except Exception:
541
- pass # exiting anyway — nothing left to reschedule
542
-
543
- def update(self, name, fn, flush='sync'):
544
- """Read-modify-write. STRICT first read (raises on a transient error rather than
545
- starting from an empty dict), apply fn(data)->data, persist.
546
-
547
- flush='sync' (default) = fresh strict read + blocking upload. flush='async' (the
548
- table-workspace hot path, wave-7 W3) = the mutation applies to the in-process cache and
549
- returns immediately; a coalescing worker commits. The strict read still guards the
550
- FIRST async update of a key per process.
551
-
552
- ⭐⭐ W31-T13 (D-177(d)) — TWO LOCKS, AND THE SPLIT IS THE WHOLE TICKET.
553
- `self._key_lock(name)` is held for the entire read-modify-write, so two writers of one key
554
- still cannot interleave (last-write-wins is the contract; a lost update is not).
555
- `self._lock` — the one every `get()` takes — is now held only for the microseconds a cache
556
- read or write needs, never across a download or an upload. Before this, one sync write
557
- froze every reader in the process for two network round trips, on a store whose warm
558
- `get()` already costs 703 ms of deep copy on tenant #0.
559
- ⚠ Uploading outside `self._lock` is not a new posture: `_flush_worker` has done exactly
560
- that since wave 7, and `_repo_ok` racing is benign because `create_repo` is `exist_ok=True`.
561
- """
562
- with self._key_lock(name):
563
- return self._update_locked(name, fn, flush)
564
-
565
- def _update_locked(self, name, fn, flush):
566
- """`update`'s body, with this key's write lock already held. See its docstring.
567
-
568
- ⚠ THE CACHE LOCK IS TAKEN THREE TIMES AND RELEASED BETWEEN THEM, which is only safe
569
- because THIS key's write lock is held throughout: nothing else can be mid-write on it, so
570
- the state read in the first section is still ours in the third. A caller reaching this
571
- without the key lock would have the old interleaving bug back.
572
- """
573
- # ⚠ A FLAG, NOT A `None` SENTINEL. `None` can be a real cached value, and a sentinel that
574
- # can equal real data is not a sentinel — this repo has paid for that exact shape before
575
- # (`routes_nav`'s locked-mode default). The flag says "the cache answered", full stop.
576
- data, from_cache = None, False
577
- with self._lock:
578
- if flush == 'async' and self._owned.get(name) and name in self._cache:
579
- data, from_cache = json.loads(json.dumps(self._cache[name])), True
580
- elif self._dirty.get(name) and self._owned.get(name) and name in self._cache:
581
- # ⛔ A SYNC READ-MODIFY-WRITE MUST NOT DISCARD A PENDING ASYNC WRITE.
582
- #
583
- # THE BUG THIS LINE EXISTS FOR (owner report 2026-08-09, MEASURED on staging):
584
- # `POST /rows` answered `201 {"rid":"100"}` and the row never existed.
585
- # `user_tables.add_row` commits `flush='async'` — the row lives ONLY in
586
- # `self._cache` until the coalescing worker uploads it, which is `_FLUSH_DELAY`
587
- # (2s) away and up to `_FLUSH_MIN_GAP` (20s) if that key committed recently.
588
- # Inside that window ANY sync writer of the SAME key took the `else` branch
589
- # below, and `_read_strict` does `self._cache[name] = self._download(name)` —
590
- # it REPLACES the cache with the hub's copy, erasing the only place the new row
591
- # existed, and then `put()` uploaded that. Nothing raised. The 201 was truthful
592
- # about `add_row`'s return value and false about the outcome.
593
- #
594
- # ⚠ `user_tables` has ~40 sync writers against 2 async ones, so this was never
595
- # one bad call site — it is a property of the KEY. Wave 27 only made it fire
596
- # every time, by putting the relation refresh on the add path.
597
- #
598
- # ⛔ AND RE-DOWNLOADING WAS PROTECTING NOTHING. Once a key is dirty our cache
599
- # has already diverged and the pending upload was going to overwrite the hub
600
- # anyway — so the fresh read cannot preserve a remote write, it can only destroy
601
- # a local one we already acknowledged. This is the class's OWN documented
602
- # contract, restored: see `_owned` above — *"read-your-writes: every reader
603
- # consults the cache first"* — and `_flush_worker`'s *"the cache stays the truth
604
- # and the dirty mark survives, so nothing is lost silently."*
605
- #
606
- # ⚠ NOT a weakening of `_read_strict`'s purpose. That exists so a FAILED read
607
- # aborts the RMW instead of merging into `{}`; the cache is populated
608
- # known-good local state, and this branch is only reachable when it is.
609
- data, from_cache = json.loads(json.dumps(self._cache[name])), True
610
- # else: the strict hub read happens BELOW, outside this lock (W31-T13)
611
-
612
- # ⭐ W31-T13 — THE HUB READ AND THE CALLER'S `fn` RUN WITH NO CACHE LOCK HELD. `fn` is
613
- # arbitrary caller code — `automation_engine`'s relation pass has run whole-tenant work
614
- # inside one — and it used to execute with every reader in the process blocked behind it.
615
- if not from_cache:
616
- data = self._read_strict(name) # network; re-checks the dirty mark on the way back
617
- result = fn(data)
618
- data = result if result is not None else data
619
-
620
- if flush == 'async':
621
- if not self.available():
622
- raise RuntimeError('No HF_TOKEN configured — persistence is unavailable.')
623
- with self._lock:
624
- self._cache[name] = json.loads(json.dumps(data))
625
- self._owned[name] = True
626
- self._bump(name) # C6 — the async write is ACCEPTED here, not at flush time
627
- self._schedule_flush(name)
628
- return data
629
-
630
- # ⛔ THE SYNC COMMIT, SPELLED OUT HERE RATHER THAN THROUGH `put()`, AND THAT IS A CHOICE.
631
- # `put()` holds `self._lock` across its own `_upload` — which is exactly the freeze this
632
- # ticket removes — but it is PUBLIC with seven callers outside this file (`users.py`,
633
- # `modules/ar.py`, `modules/map.py`, `runtime.py`, `store_backend.py`,
634
- # `ops/seed_pg_from_hf.py`), and changing its locking changes all of their concurrency
635
- # semantics at once. Duplicating four assignments with this comment beside them is the
636
- # smaller blast radius; if `put` is ever split, this collapses back into a call.
637
- if not self.available():
638
- raise RuntimeError('No HF_TOKEN configured — persistence is unavailable.')
639
- self._upload(name, data) # network, no cache lock held
640
- with self._lock:
641
- self._cache[name] = json.loads(json.dumps(data))
642
- self._owned[name] = True
643
- self._last_flush[name] = _time.monotonic()
644
- # ⛔ AFTER THE UPLOAD, NEVER BEFORE — `put()`'s own law. `_upload` raises on failure,
645
- # and a bump on that path would have `revision()` report a change that never landed.
646
- self._bump(name)
647
- # The upload above CARRIED whatever was pending, so the key is no longer ahead of
648
- # the hub. Without this the flush worker wakes on a still-dirty mark and re-uploads
649
- # identical content — a second commit against the 256-commits/hr budget for nothing.
650
- # ⚠ AFTER the upload, for the same reason: on a raise the mark MUST survive (the cache
651
- # is still the truth and the row must still be committed by the worker). Clearing it
652
- # first would turn a transient upload error into the very silent loss the branch above
653
- # exists to stop — which is why it sits here and not before `_upload`.
654
- self._dirty[name] = False
655
- return data
656
-
657
-
658
- # ------------------------------------------------------------------- instances + module API
659
- _INSTANCES = {}
660
- _INSTANCES_LOCK = threading.RLock()
661
-
662
-
663
- def for_repo(repo_id):
664
- """The HF Store bound to `repo_id` — one instance per repo per process (each with its own
665
- cache and flush state).
666
-
667
- ⚠ WAVE 20: this is now the HF-SPECIFIC factory. Callers that want "the right store for this
668
- tenant, whatever the backend is" call `handle()` below. `for_repo` keeps its exact old
669
- behaviour because the seed/migration tooling has to be able to name the FILE store
670
- explicitly while the app runs on Postgres — a migration that could only reach the active
671
- backend could not copy between them.
672
- """
673
- rid = str(repo_id or '').strip() or REPO
674
- with _INSTANCES_LOCK:
675
- inst = _INSTANCES.get(rid)
676
- if inst is None:
677
- inst = Store(rid)
678
- _INSTANCES[rid] = inst
679
- return inst
680
-
681
-
682
- # =============================================================================================
683
- # WAVE 20 (owner ruling R1, closes DEBT D-4) — THE BACKEND SEAM.
684
- #
685
- # `core/store_backend.py` has carried this warning since EXIT-2b, and it was correct at the time:
686
- #
687
- # "Flipping STORE_BACKEND=pg does NOT redirect the ~40 existing callers that say
688
- # `import core.store as store` — they are bound to the HF module directly. Rewiring them is
689
- # task C-4 … THE DEFAULT IS `hf` AND STAYS `hf` until C-4 says otherwise."
690
- #
691
- # **R1 IS C-4.** The owner ruled the cutover on 2026-08-05 (the D-4 trigger that fired: a SECOND
692
- # server process — `royal-imports/cfo-os` came back as a pinned LIVE environment beside staging,
693
- # and two containers on one last-write-wins file store is the race B-3 was always about).
694
- #
695
- # THE REWIRE IS HERE RATHER THAN IN 28 FILES, and that is a deliberate choice over the obvious
696
- # alternative of `sed`-ing every `import core.store as store` to `import core.store_backend`:
697
- # * every one of those callers means "the store for the tenant I am serving", which is exactly
698
- # what this module has always meant. The BACKEND is not their concern and making it their
699
- # concern is how one of them gets missed;
700
- # * a missed caller under a search-and-replace does not fail — it silently keeps writing to the
701
- # file store while everything else writes to Postgres. That is the split-brain this whole
702
- # ruling exists to end, reintroduced by the fix for it;
703
- # * `store_backend.py`'s stated reason for refusing to do it this way — "no dual-read window and
704
- # no way to compare the two stores' contents first" — is satisfied: `ops/seed_pg_from_hf.py`
705
- # copies, then `--verify` diffs the two stores key-by-key before anything flips.
706
- #
707
- # ⛔ THE IMPORT IS LAZY AND MUST STAY LAZY. `core/store_pg.py` imports psycopg only inside
708
- # `_pool()`, so an `hf` deployment installs no driver; importing it at module scope here would
709
- # undo that and make the default backend depend on the optional dependency.
710
- # =============================================================================================
711
-
712
- def backend():
713
- """`'hf'` | `'pg'` — validated, resolved per call so a test can flip the env var.
714
-
715
- Mirrors `core.store_backend.name()` deliberately rather than importing it: that module
716
- imports THIS one, so reaching back would be a cycle. `verify_store_pg` asserts the two agree
717
- on every value, which is the guard against them drifting apart.
718
- """
719
- raw = (os.environ.get('STORE_BACKEND') or 'hf').strip().lower()
720
- if raw not in ('hf', 'pg'):
721
- raise RuntimeError(
722
- f"STORE_BACKEND={raw!r} is not a backend. Use 'hf' (the HF Dataset store) or 'pg' "
723
- f"(Postgres, needs DATABASE_URL). Refusing to guess: a typo that silently served the "
724
- f"other store is how data ends up in two places.")
725
- return raw
726
-
727
-
728
- def handle(repo_id=None, slug=None):
729
- """THE tenant-bound store handle for the ACTIVE backend — the one seam every caller crosses.
730
-
731
- `repo_id` addresses the HF backend (a dataset repo); `slug` addresses Postgres (a schema).
732
- Both are passed by `harness.runtime.get_runtime`, which knows both facts, so flipping the
733
- backend never needs a lookup table between them — and neither identifier has to be invented
734
- for the backend that does not use it.
735
- """
736
- if backend() == 'pg':
737
- import core.store_pg as _pg # noqa: PLC0415 — lazy: see the banner above
738
- return _pg.PgStore(slug or os.environ.get('AIOS_TENANT') or 'royal-imports')
739
- return for_repo(repo_id)
740
-
741
-
742
- #: The HF default instance. Kept as a module global (not a property) because the seed/migration
743
- #: tooling and `_flush_all_at_exit` both need the FILE store by name even when pg is active.
744
- _DEFAULT = for_repo(REPO)
745
-
746
-
747
- def _d():
748
- """The default tenant's store on the active backend — what every module function below uses.
749
-
750
- ⚠ Resolved PER CALL, never cached. A cached default would freeze the backend at import time,
751
- and import order is exactly what nobody controls: `api/main.py` imports half the platform
752
- before it has read a single environment variable it did not inherit.
753
- """
754
- return handle() if backend() == 'pg' else _DEFAULT
755
-
756
-
757
- def available():
758
- return _d().available()
759
-
760
-
761
- def get(name, fresh=False):
762
- return _d().get(name, fresh=fresh)
763
-
764
-
765
- def _read_strict(name):
766
- return _d()._read_strict(name)
767
-
768
-
769
- def exists(name):
770
- return _d().exists(name)
771
-
772
-
773
- def upload_bytes(path_in_repo, data, message=None):
774
- return _d().upload_bytes(path_in_repo, data, message=message)
775
-
776
-
777
- def download_bytes(path_in_repo):
778
- return _d().download_bytes(path_in_repo)
779
-
780
-
781
- def delete_path(path_in_repo):
782
- return _d().delete_path(path_in_repo)
783
-
784
-
785
- def put(name, data):
786
- return _d().put(name, data)
787
-
788
-
789
- def flush(name=None, timeout=30.0):
790
- return _d().flush(name=name, timeout=timeout)
791
-
792
-
793
- def update(name, fn, flush='sync'):
794
- return _d().update(name, fn, flush=flush)
795
-
796
-
797
- def get_projection(name, drop=()):
798
- """A projected read — see `Store.get_projection`. Wave 32, W32-T01 (D-185).
799
-
800
- ⛔ AN ELEVENTH STORE OPERATION, AND IT HAD TO BECOME ONE — the method alone was NOT enough,
801
- which is the opposite of what it looks like from `revision()`'s precedent. `revision` could
802
- ship method-only for a few hours because its caller held a Store. This one's caller is
803
- `harness.runtime.TenantRuntime`, and `TenantRuntime._store()` returns **the MODULE** whenever
804
- `store_handle is None` — which is precisely tenant #0, the tenant whose 703 ms `/nav` copy this
805
- exists to remove. A method-only projection would have been unreachable from the only place that
806
- needed it, while every gate stayed green.
807
-
808
- ⛔ SO `verify_store_pg.IFACE` GAINS IT IN THE SAME CHANGE, and `core/store_pg.py` gains the
809
- twin below it. That gate's `extra:` check would red on this function otherwise — correctly:
810
- this addresses a key and reads it, so it is a store operation and both backends must answer.
811
- `backend`/`handle` are exempt because they SELECT a store; this one reads one.
812
- """
813
- return _d().get_projection(name, drop=drop)
814
-
815
-
816
- def revision(name):
817
- """The bucket's change token — wave 29, C6. See `Store.revision` for why it copies nothing.
818
-
819
- ⛔ THIS IS A TENTH STORE OPERATION, AND IT WAS ONLY ALLOWED TO BECOME ONE ONCE BOTH BACKENDS
820
- HAD IT. `verify_store_pg.py` pins this module's public surface at a contracted list precisely
821
- so a capability one backend lacks cannot be reached through the shared door and die at the
822
- cutover — so this function, `store_pg.revision`, and that gate's `IFACE` entry are ONE change
823
- and must stay one. It shipped for a few hours as a METHOD ONLY, deliberately, while the pg half
824
- had no owner: a method is a capability the HF store has, a module function is a promise both
825
- backends keep.
826
- """
827
- return _d().revision(name)
828
-
829
-
830
- def _flush_all_at_exit():
831
- with _INSTANCES_LOCK:
832
- instances = list(_INSTANCES.values())
833
- for inst in instances:
834
- inst._flush_now_at_exit()
835
-
836
-
837
- import atexit # noqa: E402 (registered after the class it needs)
838
- atexit.register(_flush_all_at_exit)
 
1
+ """Tiny JSON persistence on a PRIVATE Hugging Face Dataset repo — the platform's ONLY writable store
2
+ (Odoo stays strictly read-only). Holds the user registry, customer notes and per-user tags.
3
+
4
+ Auth uses HF_TOKEN (a Space secret / .env value with write access to the royal-imports org). The
5
+ repo is created private on first write. Reads use a short in-process cache that writes refresh, so
6
+ within a running instance reads are consistent with writes; a fresh instance re-fetches the latest.
7
+
8
+ WAVE 18 (C1-TENANT, R2): the store is now a CLASS, one instance per repo. The module-level
9
+ functions below delegate to the default instance (`OS_DATA_REPO`, tenant #0) — every existing
10
+ `import core.store as store` caller is byte-for-byte unaffected, which the gate battery proves.
11
+ `for_repo(repo_id)` hands back the bound instance for a tenant that owns its OWN dataset repo
12
+ (nurilab, gtmlab, …); `harness.runtime.TenantRuntime` binds it from the tenant record. Each
13
+ instance carries its own cache / dirty-marks / flush workers / commit-gap clock, so one tenant's
14
+ write burst can never starve another's flush window.
15
+ """
16
+ import io
17
+ import os
18
+ import json
19
+ import threading
20
+ import time as _time
21
+ from pathlib import Path
22
+
23
+ try:
24
+ from dotenv import load_dotenv
25
+ load_dotenv(Path(__file__).resolve().parents[1] / '.env')
26
+ except Exception:
27
+ pass
28
+
29
+ from huggingface_hub import HfApi, hf_hub_download
30
+ from huggingface_hub.utils import EntryNotFoundError, RepositoryNotFoundError
31
+
32
+ REPO = os.environ.get('OS_DATA_REPO', 'royal-imports/cfo-os-data')
33
+ _FLUSH_DELAY = 2.0 # coalesce a burst of edits into one commit
34
+ _FLUSH_MIN_GAP = 20.0 # floor between commits of one key (the 256-commits/hr budget)
35
+
36
+ #: WAVE 29 (item 20 / ruling R11, contract C6) — THE CHANGE TOKEN'S PROCESS STAMP.
37
+ #:
38
+ #: `Store`'s revision counter is IN-MEMORY, so a restarted Space would begin again at 0 and a
39
+ #: browser holding `…:4` from the previous process could read `…:4` from the new one and conclude
40
+ #: nothing had changed. Prefixing every token with a per-process stamp makes a restart read as a
41
+ #: change, which is the truthful answer: the client's copy predates this process entirely.
42
+ #:
43
+ #: ⚠ NOT a clock and never compared for order — a token is opaque and compared ONLY for equality.
44
+ REV_EPOCH = f'{int(_time.time()):x}{os.getpid():x}'
45
+
46
+
47
+ def _token():
48
+ return os.environ.get('HF_TOKEN') or None
49
+
50
+
51
+ class _Projected(dict):
52
+ """One top-level value of a PROJECTED read — a dict that RAISES for the keys that were dropped.
53
+
54
+ ⛔⛔ W32-T01: THE ALTERNATIVE IS A SILENT WRONG ANSWER, WHICH IS WHY THIS CLASS EXISTS AT ALL.
55
+ A projection built as a plain dict answers `defn.get('rows') or {}` with `{}` — indistinguishable
56
+ from a database that genuinely has no rows. The caller then paints an empty grid, exports an
57
+ empty CSV, or (worse) writes that emptiness back. This repo has shipped that exact shape before
58
+ ([[empty-answer-vs-unfinished-answer]], and a grouped view that painted zero rows).
59
+
60
+ So a dropped key is not absent — it is UNREAD, and asking for it raises with the reason and the
61
+ fix in the message. `KeyError` is the type on purpose: `dict.__getitem__`'s own failure mode, so
62
+ nothing has to learn a new exception; and the message names `get_projection` so the traceback
63
+ points at the caller's mistake rather than at this file.
64
+
65
+ ⚠ Still a real `dict` for every OTHER purpose — `json.dumps`, `dict(x)`, `**x`, iteration and
66
+ `in` all behave normally, because a projection has to serialise onto the wire like any
67
+ definition. Only the named keys are booby-trapped.
68
+ """
69
+
70
+ __slots__ = ('_dropped',)
71
+
72
+ def __init__(self, data=None, dropped=()):
73
+ super().__init__(data or {})
74
+ object.__setattr__(self, '_dropped', frozenset(dropped))
75
+
76
+ def _refuse(self, key):
77
+ raise KeyError(
78
+ f'{key!r} was DROPPED by a projected store read and is not absent — this document came '
79
+ f'from Store.get_projection(drop={sorted(self._dropped)!r}). Reading it as empty would '
80
+ f'be a silent wrong answer. Use the whole read (Store.get / user_tables.all_tables) if '
81
+ f'you need {key!r}.')
82
+
83
+ def __getitem__(self, key):
84
+ if key in self._dropped:
85
+ self._refuse(key)
86
+ return super().__getitem__(key)
87
+
88
+ def get(self, key, default=None):
89
+ if key in self._dropped:
90
+ self._refuse(key)
91
+ return super().get(key, default)
92
+
93
+ def pop(self, key, *a):
94
+ if key in self._dropped:
95
+ self._refuse(key)
96
+ return super().pop(key, *a)
97
+
98
+
99
+ def _project(doc, drop):
100
+ """A deep COPY of `doc` with `drop`'s keys removed from every top-level value.
101
+
102
+ ⭐ The order is the optimisation: build the retained structure FIRST (a shallow comprehension
103
+ that never touches the dropped values), then deep-copy THAT. One `json.dumps` over 0.1% of the
104
+ bytes, instead of one over all of them.
105
+ """
106
+ if not isinstance(doc, dict):
107
+ return json.loads(json.dumps(doc))
108
+ kept = {k: ({kk: vv for kk, vv in v.items() if kk not in drop} if isinstance(v, dict) else v)
109
+ for k, v in doc.items()}
110
+ copied = json.loads(json.dumps(kept))
111
+ return {k: (_Projected(v, drop) if isinstance(v, dict) else v) for k, v in copied.items()}
112
+
113
+
114
+ class Store:
115
+ """One repo's persistence: JSON-per-key + raw byte paths, with the wave-7 async flush.
116
+
117
+ Every method body is the pre-wave-18 module function, verbatim except globals→fields.
118
+ The behaviour contracts that matter are stated on each method and are UNCHANGED:
119
+ strict-read-guards-the-write, exists()-is-True-on-uncertainty, lenient display reads.
120
+ """
121
+
122
+ def __init__(self, repo):
123
+ self.repo = str(repo)
124
+ self._lock = threading.RLock()
125
+ self._cache = {}
126
+ # --- async flush state (wave-7 W3, 2026-07-28) ------------------------------------
127
+ # One typed character in a Notes cell used to cost THREE synchronous network calls
128
+ # inside the component round-trip. update(..., flush='async') applies the mutation to
129
+ # the in-process cache synchronously (read-your-writes: every reader consults the
130
+ # cache first) and a background worker coalesces uploads. The restart-wipe protection
131
+ # is UNCHANGED where it matters: the FIRST update of a key in a fresh process still
132
+ # strict-reads. Known, accepted: with a second concurrent writer the stale window
133
+ # widens from ~one round-trip to ~the flush gap.
134
+ self._owned = {} # name -> True once cache reflects our own last accepted write
135
+ self._dirty = {} # name -> cache is ahead of the hub
136
+ self._flushers = {} # name -> live worker thread
137
+ self._repo_ok = False # create_repo(exist_ok=True) confirmed once per instance
138
+ self._last_flush = {} # name -> monotonic time of the last completed upload
139
+ # --- the CHANGE TOKEN (wave 29, item 20 / C6) ------------------------------------
140
+ # name -> how many accepted writes this process has applied to that bucket, and when.
141
+ # Read by `revision()` in O(1) with NO download and NO deep copy; see its docstring for
142
+ # why that constraint is the whole feature rather than an optimisation.
143
+ self._revs = {}
144
+ # --- ⭐⭐ W31-T13 (D-177(d)) — THE PER-KEY WRITE LOCK ------------------------------
145
+ # `self._lock` guards the CACHE. It used to be held across a sync `update`'s whole
146
+ # read-modify-write — a hub download, the caller's `fn`, and a blocking hub upload — so
147
+ # one write froze every reader in the process for two network round trips. `get()` takes
148
+ # the same lock, and `/nav`, `/workspace` and every permission check are `get()`.
149
+ # ⛔ IT IS NOT SAFE TO SIMPLY DROP THE LOCK. Two sync writers of one key would interleave
150
+ # read-modify-write and one update would vanish — corruption, not slowness. So the two
151
+ # jobs are separated: this lock serialises WRITERS OF ONE KEY (held across the network,
152
+ # never touching readers), and `self._lock` is taken only for the microseconds a cache
153
+ # read or write needs. Readers never wait on a network call again.
154
+ # ⚠ PER KEY, not one write lock: two tenants' buckets, or `user_tables` and `nav_meta`,
155
+ # have no reason to serialise against each other.
156
+ self._key_locks = {}
157
+ self._key_locks_guard = threading.Lock()
158
+ self._rev_at = {}
159
+
160
+ def available(self):
161
+ """True when a token is configured — callers degrade gracefully without one."""
162
+ return bool(_token())
163
+
164
+ def _download(self, name):
165
+ """Parse {name}.json into a dict. Returns {} ONLY when the file is genuinely absent.
166
+ Any other failure (network, auth, rate-limit, corrupt JSON) RAISES — a transient read
167
+ error must never be mistaken for an empty store, or a caller could overwrite real data
168
+ with an empty dict (this is what wiped the user registry on restart)."""
169
+ try:
170
+ path = hf_hub_download(self.repo, f'{name}.json', repo_type='dataset',
171
+ token=_token(), force_download=True)
172
+ with open(path, encoding='utf-8') as f:
173
+ return json.load(f)
174
+ except (EntryNotFoundError, RepositoryNotFoundError, FileNotFoundError):
175
+ return {}
176
+ except ValueError as e:
177
+ # Wave 18 (R2, found by SESSION D): current huggingface_hub wraps a missing REPO in
178
+ # ValueError("Force download failed…") with the 404 as __cause__ — so a brand-new
179
+ # tenant repo's very first strict read raised here and aborted its very first write.
180
+ # A genuinely-absent repo is the same fact as a genuinely-absent file: {}.
181
+ cause = getattr(e, '__cause__', None)
182
+ if isinstance(cause, (EntryNotFoundError, RepositoryNotFoundError)):
183
+ return {}
184
+ raise
185
+
186
+ def get(self, name, fresh=False):
187
+ """Load {name}.json as a dict. LENIENT (for display reads): on a transient fetch error,
188
+ hand back the last known-good cached value if we have one, else {}. Mutating callers go
189
+ through update()/_read_strict, which RAISE instead."""
190
+ with self._lock:
191
+ if not fresh and name in self._cache:
192
+ return json.loads(json.dumps(self._cache[name])) # hand back a copy
193
+ try:
194
+ data = self._download(name)
195
+ except Exception as e:
196
+ try: # lazy import — core stays layer-clean
197
+ import harness.telemetry as _tel # noqa: PLC0415
198
+ _tel.error(f'store:get:{name}', e,
199
+ fallback='cached' if name in self._cache else 'empty')
200
+ except Exception:
201
+ pass
202
+ if name in self._cache:
203
+ return json.loads(json.dumps(self._cache[name]))
204
+ return {}
205
+ self._cache[name] = data
206
+ return json.loads(json.dumps(data))
207
+
208
+ def get_projection(self, name, drop=()):
209
+ """Like `get`, but each top-level VALUE comes back WITHOUT the keys named in `drop`.
210
+
211
+ ⭐⭐ W32-T01 (D-185) — THE POINT IS THE COPY, NOT THE PAYLOAD SIZE ON THE WIRE.
212
+
213
+ `get()` is honest and expensive: it hands back `json.loads(json.dumps(...))` so a caller
214
+ cannot mutate the cache, and on tenant #0's `user_tables` that copy is **28.6 MB / 81,330
215
+ rows / ~703 ms warm**, of which **99.9% is `rows`** — which no permission check, no nav
216
+ render and no workspace envelope ever reads. `/nav` used to pay ~13 of those copies per
217
+ request; W31-T10 made it ONE, and that one is still 703 ms of pure copying. This is the
218
+ floor D-185 named: copy only what was asked for.
219
+
220
+ ⛔⛔ **THE PROJECTION IS NEVER CACHED, AND THAT LINE IS THE WHOLE SAFETY ARGUMENT.** A
221
+ rows-less document in `self._cache` would be served to the next WHOLE reader, and the next
222
+ `update()` would upload it — every row in the tenant deleted, by a read. So the cold path
223
+ downloads whole, caches WHOLE (exactly as `get` does), and projects the RETURN value only.
224
+ `verify_store_projection.py` asserts the cache still contains `rows` after a projected read,
225
+ with an NC that reds when the projection is cached instead.
226
+
227
+ ⚠ **`self._lock` is held across the copy, same as `get`** — the retained structure is
228
+ ~0.1% of the bytes, so this is microseconds where `get` is most of a second. Narrowing it
229
+ would let a concurrent writer mutate a value mid-serialise for no measurable gain.
230
+
231
+ ⚠ **A dropped key RAISES on read rather than reading as absent** (`_Projected`). A
232
+ projection that answered `{}` for `rows` would paint an empty grid — the exact
233
+ [[empty-answer-vs-unfinished-answer]] shape this repo has shipped before — and the caller
234
+ that most wants rows is the one least able to tell. See `_Projected`.
235
+
236
+ ⚠ Lenient, like `get`: a transient fetch error falls back to the cached document (projected)
237
+ and then to `{}`. Mutating callers still go through `update()`/`_read_strict`, which never
238
+ project — a write must always read whole.
239
+ """
240
+ drop = frozenset(str(d) for d in drop)
241
+ with self._lock:
242
+ if name in self._cache:
243
+ return _project(self._cache[name], drop)
244
+ try:
245
+ data = self._download(name)
246
+ except Exception as e:
247
+ try: # lazy import — core stays layer-clean
248
+ import harness.telemetry as _tel # noqa: PLC0415
249
+ _tel.error(f'store:get_projection:{name}', e,
250
+ fallback='cached' if name in self._cache else 'empty')
251
+ except Exception:
252
+ pass
253
+ with self._lock:
254
+ if name in self._cache:
255
+ return _project(self._cache[name], drop)
256
+ return {}
257
+ with self._lock:
258
+ self._cache[name] = data # ⛔ the WHOLE document, never the projection
259
+ return _project(data, drop)
260
+
261
+ def _key_lock(self, name):
262
+ """The write lock for ONE key (W31-T13). Created on demand; never removed."""
263
+ n = str(name)
264
+ with self._key_locks_guard:
265
+ lk = self._key_locks.get(n)
266
+ if lk is None:
267
+ lk = self._key_locks[n] = threading.RLock()
268
+ return lk
269
+
270
+ def _read_strict(self, name):
271
+ """Like get(fresh=True) but RAISES on any transient read error ({} only for a genuinely
272
+ absent file), so a failed read ABORTS a read-modify-write instead of merging into {}.
273
+
274
+ ⭐ W31-T13 — THE DOWNLOAD IS OUTSIDE `self._lock` NOW, and the re-check after it is the
275
+ whole of what makes that safe. A hub read takes a round trip; holding the cache lock
276
+ across it is what froze every reader. But moving it out opens a window in which an ASYNC
277
+ writer can accept a row into the cache while our download is in flight — and assigning our
278
+ (older) hub copy afterwards would erase it. That is D-177's original symptom exactly
279
+ (`POST /rows` → 201, row gone), re-created by the lock split rather than by a call site.
280
+ So the dirty mark is re-read AFTER the download, under the short lock, and a key that
281
+ became dirty keeps its cache.
282
+ """
283
+ data = self._download(name) # network, no lock held
284
+ with self._lock:
285
+ if self._dirty.get(name) and self._owned.get(name) and name in self._cache:
286
+ # Our copy is stale by construction: a pending upload is going to overwrite the
287
+ # hub anyway, so the fresh read cannot preserve a remote write — it can only
288
+ # destroy a local one we have already acknowledged. Same argument, and the same
289
+ # three conditions, as `update`'s pending-async branch below.
290
+ return json.loads(json.dumps(self._cache[name]))
291
+ self._cache[name] = data
292
+ return json.loads(json.dumps(data))
293
+
294
+ def exists(self, name):
295
+ """True if {name}.json is confirmed present, False if confirmed absent, and True on ANY
296
+ uncertainty — a seeder must never clobber a registry we merely failed to reach."""
297
+ try:
298
+ files = HfApi(token=_token()).list_repo_files(self.repo, repo_type='dataset')
299
+ return f'{name}.json' in files
300
+ except RepositoryNotFoundError:
301
+ return False
302
+ except Exception:
303
+ return True
304
+
305
+ # ------------------------------------------------------- the CHANGE TOKEN (wave 29, C6)
306
+ def _bump(self, name):
307
+ """Advance one bucket's write counter. THE CALLER HOLDS THE LOCK.
308
+
309
+ Called from the two places an accepted write becomes visible to readers of this process
310
+ — `put()` (which `update(flush='sync')` goes through) and `update()`'s async branch. NOT
311
+ from `_flush_worker`: that uploads data a bump already accounted for, and counting it
312
+ again would report a change to every browser each time the coalescing worker woke.
313
+ """
314
+ n = str(name)
315
+ self._revs[n] = self._revs.get(n, 0) + 1
316
+ self._rev_at[n] = _time.time()
317
+
318
+ def revision(self, name):
319
+ """`{'rev', 'updated_at', 'token'}` for ONE bucket — a dict read, nothing else.
320
+
321
+ ⛔ THE ZERO-COST PROPERTY IS THE FEATURE, NOT AN OPTIMISATION, and the arithmetic is why
322
+ this method exists at all. A browser asking "has anything changed" by re-fetching rows
323
+ costs THREE full-tenant deep copies under ONE lock (`get()` json-round-trips on the way
324
+ in and on the way out) on ONE uvicorn process, at a documented ceiling of 35.8 MB / ~1.4 s
325
+ per bucket — so two tabs polling saturate the server and every unrelated write queues
326
+ behind them. This performs no download, no `get()`, and no copy of the value.
327
+ ⇒ Anything added here that reads a bucket destroys the whole point (`verify_live_workspace`
328
+ counts `get`/`_read_strict`/`_download` on the serving path and goes red on one).
329
+
330
+ `rev` and `updated_at` are `core/store_pg.py`'s own column names, on purpose: that backend
331
+ ALREADY increments `rev` and sets `updated_at = now()` on every write, so the token stops
332
+ being inert the moment D-4 flips, with the wire shape unchanged.
333
+
334
+ ⚠ WHAT IT CANNOT SEE, stated rather than implied: this counter is per PROCESS, so a write
335
+ by a SECOND container against the same HF dataset is invisible here. That is not a limit
336
+ this introduces — `get()` caches a bucket for the life of the process and never re-reads
337
+ it, so the other container's write is invisible to every reader on this backend already.
338
+ The token is exactly as fresh as the store it reports on. Postgres has no such gap.
339
+ ⚠ Bytes (`upload_bytes`/`delete_path`) are a different address space and are NOT counted:
340
+ an attachment is fetched on explicit user action, never polled.
341
+ """
342
+ n = str(name)
343
+ with self._lock:
344
+ rev = int(self._revs.get(n, 0))
345
+ at = self._rev_at.get(n)
346
+ return {'rev': rev, 'updated_at': at, 'token': f'{REV_EPOCH}:{rev}'}
347
+
348
+ def _upload(self, name, data):
349
+ """The raw hub write. create_repo is confirmed ONCE per instance — it was a full extra
350
+ API round-trip on every put(), pure latency after the first success."""
351
+ api = HfApi(token=_token())
352
+ if not self._repo_ok:
353
+ api.create_repo(self.repo, repo_type='dataset', private=True, exist_ok=True)
354
+ self._repo_ok = True
355
+ buf = io.BytesIO(json.dumps(data, ensure_ascii=False, indent=2).encode('utf-8'))
356
+ api.upload_file(path_or_fileobj=buf, path_in_repo=f'{name}.json', repo_id=self.repo,
357
+ repo_type='dataset', commit_message=f'update {name}')
358
+
359
+ def upload_bytes(self, path_in_repo, data, message=None):
360
+ """Persist RAW BYTES at an arbitrary path in the tenant dataset (wave-8 C5, documents).
361
+ Bytes are never cached in-process: large, cold, fetched on explicit user action."""
362
+ if not self.available():
363
+ raise RuntimeError('No HF_TOKEN configured — persistence is unavailable.')
364
+ with self._lock:
365
+ api = HfApi(token=_token())
366
+ if not self._repo_ok:
367
+ api.create_repo(self.repo, repo_type='dataset', private=True, exist_ok=True)
368
+ self._repo_ok = True
369
+ api.upload_file(path_or_fileobj=io.BytesIO(data), path_in_repo=path_in_repo,
370
+ repo_id=self.repo, repo_type='dataset',
371
+ commit_message=message or f'add {path_in_repo}')
372
+
373
+ def download_bytes(self, path_in_repo):
374
+ """Raw bytes back, or None when absent/unreachable — a missing attachment degrades to
375
+ "that file is gone", never to a broken page."""
376
+ if not self.available():
377
+ return None
378
+ try:
379
+ local = hf_hub_download(self.repo, path_in_repo, repo_type='dataset',
380
+ token=_token())
381
+ return Path(local).read_bytes()
382
+ except (EntryNotFoundError, RepositoryNotFoundError):
383
+ return None
384
+ except Exception:
385
+ return None
386
+
387
+ def delete_path(self, path_in_repo):
388
+ """Remove a stored file. Absent is SUCCESS — deleting what is already gone is the goal."""
389
+ if not self.available():
390
+ return False
391
+ try:
392
+ HfApi(token=_token()).delete_file(path_in_repo=path_in_repo, repo_id=self.repo,
393
+ repo_type='dataset',
394
+ commit_message=f'delete {path_in_repo}')
395
+ return True
396
+ except (EntryNotFoundError, RepositoryNotFoundError):
397
+ return True
398
+ except Exception:
399
+ return False
400
+
401
+ def put(self, name, data):
402
+ """Persist {name}.json (creates the private repo on first write). Updates the cache."""
403
+ if not self.available():
404
+ raise RuntimeError('No HF_TOKEN configured — persistence is unavailable.')
405
+ with self._lock:
406
+ self._upload(name, data)
407
+ self._cache[name] = json.loads(json.dumps(data))
408
+ self._owned[name] = True
409
+ self._last_flush[name] = _time.monotonic()
410
+ self._bump(name) # C6 — AFTER the upload: a raise here is not a write
411
+ return self._cache[name]
412
+
413
+ def _flush_worker(self, name):
414
+ """Coalescing uploader for one key. Runs while dirty; retries with backoff on failure —
415
+ the cache stays the truth and the dirty mark survives, so nothing is lost silently.
416
+
417
+ ⭐⭐ W31-T13 — THIS WORKER IS THE SECOND WRITER, AND THE PER-KEY LOCK MUST COVER IT.
418
+ ⛔ THE RACE THE LOCK SPLIT OPENED, spelled out because it is silent and survives a restart
419
+ as real data loss. `update`'s big lock used to span `_read_strict` + `fn` + the upload, so
420
+ this worker could never snapshot mid-update. With the cache lock narrowed it can:
421
+ async write -> `_dirty=True`, this worker scheduled, sleeps `_FLUSH_DELAY`
422
+ sync update t+0.5 -> takes the dirty branch, runs `fn`, enters `_upload` (no cache
423
+ lock held — that is the whole point of the ticket)
424
+ worker t+2.0 -> `self._lock` is FREE and `_dirty` is still True (the sync commit
425
+ has not happened yet), so it snapshots the PRE-UPDATE cache,
426
+ clears the mark, and uploads it AFTER the sync upload lands
427
+ result -> cache has the new value, the hub has the old one, `_dirty` is
428
+ False, and nothing will ever re-upload. Divergence with no
429
+ symptom until the next process boots and reads the hub.
430
+ ⚠ `user_tables` has ~40 sync writers against 2 async ones, so that window is the COMMON
431
+ case, not an exotic one — it is D-177's own shape, re-created one layer down.
432
+ ⛔ LOCK ORDER IS KEY-THEN-CACHE, EVERYWHERE. `_update_locked` takes them in that order, so
433
+ this must too; taking `self._lock` first on this path is the deadlock. Both sleeps stay
434
+ OUTSIDE the key lock — a worker that dozed while holding it would block every writer of
435
+ that key for `_FLUSH_MIN_GAP`, which is the freeze this ticket exists to remove.
436
+ """
437
+ backoff = 2.0
438
+ while True:
439
+ _time.sleep(_FLUSH_DELAY)
440
+ wait, err, snapshot = None, None, None
441
+ with self._key_lock(name):
442
+ with self._lock:
443
+ gap = _time.monotonic() - self._last_flush.get(name, 0.0)
444
+ if gap < _FLUSH_MIN_GAP:
445
+ wait = _FLUSH_MIN_GAP - gap
446
+ elif not self._dirty.get(name):
447
+ self._flushers.pop(name, None)
448
+ return
449
+ else:
450
+ self._dirty[name] = False
451
+ snapshot = json.loads(json.dumps(self._cache.get(name, {})))
452
+ if wait is None:
453
+ try:
454
+ self._upload(name, snapshot)
455
+ with self._lock:
456
+ self._last_flush[name] = _time.monotonic()
457
+ backoff = 2.0
458
+ except Exception as e: # noqa: BLE001
459
+ with self._lock:
460
+ self._dirty[name] = True # keep it queued; cache is still the truth
461
+ err = e
462
+ if wait is not None:
463
+ _time.sleep(wait)
464
+ continue
465
+ if err is not None:
466
+ try:
467
+ import harness.telemetry as _tel # noqa: PLC0415
468
+ _tel.error(f'store:flush:{name}', err)
469
+ except Exception:
470
+ pass
471
+ _time.sleep(backoff)
472
+ backoff = min(backoff * 2, 60.0)
473
+
474
+ def _schedule_flush(self, name):
475
+ """Mark dirty + ensure exactly one live worker for the key. Caller holds the lock."""
476
+ self._dirty[name] = True
477
+ t = self._flushers.get(name)
478
+ if t is not None and t.is_alive():
479
+ return
480
+ t = threading.Thread(target=self._flush_worker, args=(name,), daemon=True,
481
+ name=f'store-flush-{self.repo}:{name}')
482
+ self._flushers[name] = t
483
+ t.start()
484
+
485
+ def flush(self, name=None, timeout=30.0):
486
+ """Block until pending async writes for `name` (or ALL keys) reach the hub — QA gates
487
+ and shutdown hooks use this; the app itself never needs to."""
488
+ deadline = _time.monotonic() + timeout
489
+ names = [name] if name else None
490
+ while _time.monotonic() < deadline:
491
+ with self._lock:
492
+ pending = {k for k, v in self._dirty.items() if v} | {
493
+ k for k, t in self._flushers.items() if t.is_alive()}
494
+ if names is not None:
495
+ pending &= set(names)
496
+ if not pending:
497
+ return True
498
+ _time.sleep(0.2)
499
+ return False
500
+
501
+ def _flush_now_at_exit(self):
502
+ """Process-exit safety net: push any dirty keys synchronously, bypassing the coalescing
503
+ gap — a Space restart must not eat the last seconds of the async window.
504
+
505
+ ⚠ W31-T13 — DELIBERATELY NOT KEY-LOCKED, stated so the omission is not read as one. This
506
+ runs at interpreter exit with the process going away: a writer mid-`update` will never
507
+ finish, so waiting for its key lock could only turn a partial save into no save at all.
508
+ Best-effort is the correct contract here, and it is the one case where taking the lock
509
+ would be worse than skipping it.
510
+ """
511
+ with self._lock:
512
+ dirty = [k for k, v in self._dirty.items() if v]
513
+ snaps = {k: json.loads(json.dumps(self._cache.get(k, {}))) for k in dirty}
514
+ for k in dirty:
515
+ self._dirty[k] = False
516
+ for k, d in snaps.items():
517
+ try:
518
+ self._upload(k, d)
519
+ except Exception:
520
+ pass # exiting anyway — nothing left to reschedule
521
+
522
+ def update(self, name, fn, flush='sync'):
523
+ """Read-modify-write. STRICT first read (raises on a transient error rather than
524
+ starting from an empty dict), apply fn(data)->data, persist.
525
+
526
+ flush='sync' (default) = fresh strict read + blocking upload. flush='async' (the
527
+ table-workspace hot path, wave-7 W3) = the mutation applies to the in-process cache and
528
+ returns immediately; a coalescing worker commits. The strict read still guards the
529
+ FIRST async update of a key per process.
530
+
531
+ ⭐⭐ W31-T13 (D-177(d)) — TWO LOCKS, AND THE SPLIT IS THE WHOLE TICKET.
532
+ `self._key_lock(name)` is held for the entire read-modify-write, so two writers of one key
533
+ still cannot interleave (last-write-wins is the contract; a lost update is not).
534
+ `self._lock` — the one every `get()` takes — is now held only for the microseconds a cache
535
+ read or write needs, never across a download or an upload. Before this, one sync write
536
+ froze every reader in the process for two network round trips, on a store whose warm
537
+ `get()` already costs 703 ms of deep copy on tenant #0.
538
+ ⚠ Uploading outside `self._lock` is not a new posture: `_flush_worker` has done exactly
539
+ that since wave 7, and `_repo_ok` racing is benign because `create_repo` is `exist_ok=True`.
540
+ """
541
+ with self._key_lock(name):
542
+ return self._update_locked(name, fn, flush)
543
+
544
+ def _update_locked(self, name, fn, flush):
545
+ """`update`'s body, with this key's write lock already held. See its docstring.
546
+
547
+ ⚠ THE CACHE LOCK IS TAKEN THREE TIMES AND RELEASED BETWEEN THEM, which is only safe
548
+ because THIS key's write lock is held throughout: nothing else can be mid-write on it, so
549
+ the state read in the first section is still ours in the third. A caller reaching this
550
+ without the key lock would have the old interleaving bug back.
551
+ """
552
+ # ⚠ A FLAG, NOT A `None` SENTINEL. `None` can be a real cached value, and a sentinel that
553
+ # can equal real data is not a sentinel — this repo has paid for that exact shape before
554
+ # (`routes_nav`'s locked-mode default). The flag says "the cache answered", full stop.
555
+ data, from_cache = None, False
556
+ with self._lock:
557
+ if flush == 'async' and self._owned.get(name) and name in self._cache:
558
+ data, from_cache = json.loads(json.dumps(self._cache[name])), True
559
+ elif self._dirty.get(name) and self._owned.get(name) and name in self._cache:
560
+ # ⛔ A SYNC READ-MODIFY-WRITE MUST NOT DISCARD A PENDING ASYNC WRITE.
561
+ #
562
+ # THE BUG THIS LINE EXISTS FOR (owner report 2026-08-09, MEASURED on staging):
563
+ # `POST /rows` answered `201 {"rid":"100"}` and the row never existed.
564
+ # `user_tables.add_row` commits `flush='async'` — the row lives ONLY in
565
+ # `self._cache` until the coalescing worker uploads it, which is `_FLUSH_DELAY`
566
+ # (2s) away and up to `_FLUSH_MIN_GAP` (20s) if that key committed recently.
567
+ # Inside that window ANY sync writer of the SAME key took the `else` branch
568
+ # below, and `_read_strict` does `self._cache[name] = self._download(name)` —
569
+ # it REPLACES the cache with the hub's copy, erasing the only place the new row
570
+ # existed, and then `put()` uploaded that. Nothing raised. The 201 was truthful
571
+ # about `add_row`'s return value and false about the outcome.
572
+ #
573
+ # ⚠ `user_tables` has ~40 sync writers against 2 async ones, so this was never
574
+ # one bad call site — it is a property of the KEY. Wave 27 only made it fire
575
+ # every time, by putting the relation refresh on the add path.
576
+ #
577
+ # ⛔ AND RE-DOWNLOADING WAS PROTECTING NOTHING. Once a key is dirty our cache
578
+ # has already diverged and the pending upload was going to overwrite the hub
579
+ # anyway — so the fresh read cannot preserve a remote write, it can only destroy
580
+ # a local one we already acknowledged. This is the class's OWN documented
581
+ # contract, restored: see `_owned` above — *"read-your-writes: every reader
582
+ # consults the cache first"* — and `_flush_worker`'s *"the cache stays the truth
583
+ # and the dirty mark survives, so nothing is lost silently."*
584
+ #
585
+ # ⚠ NOT a weakening of `_read_strict`'s purpose. That exists so a FAILED read
586
+ # aborts the RMW instead of merging into `{}`; the cache is populated
587
+ # known-good local state, and this branch is only reachable when it is.
588
+ data, from_cache = json.loads(json.dumps(self._cache[name])), True
589
+ # else: the strict hub read happens BELOW, outside this lock (W31-T13)
590
+
591
+ # ⭐ W31-T13 — THE HUB READ AND THE CALLER'S `fn` RUN WITH NO CACHE LOCK HELD. `fn` is
592
+ # arbitrary caller code — `automation_engine`'s relation pass has run whole-tenant work
593
+ # inside one — and it used to execute with every reader in the process blocked behind it.
594
+ if not from_cache:
595
+ data = self._read_strict(name) # network; re-checks the dirty mark on the way back
596
+ result = fn(data)
597
+ data = result if result is not None else data
598
+
599
+ if flush == 'async':
600
+ if not self.available():
601
+ raise RuntimeError('No HF_TOKEN configured — persistence is unavailable.')
602
+ with self._lock:
603
+ self._cache[name] = json.loads(json.dumps(data))
604
+ self._owned[name] = True
605
+ self._bump(name) # C6 — the async write is ACCEPTED here, not at flush time
606
+ self._schedule_flush(name)
607
+ return data
608
+
609
+ # ⛔ THE SYNC COMMIT, SPELLED OUT HERE RATHER THAN THROUGH `put()`, AND THAT IS A CHOICE.
610
+ # `put()` holds `self._lock` across its own `_upload` — which is exactly the freeze this
611
+ # ticket removes — but it is PUBLIC with seven callers outside this file (`users.py`,
612
+ # `modules/ar.py`, `modules/map.py`, `runtime.py`, `store_backend.py`,
613
+ # `ops/seed_pg_from_hf.py`), and changing its locking changes all of their concurrency
614
+ # semantics at once. Duplicating four assignments with this comment beside them is the
615
+ # smaller blast radius; if `put` is ever split, this collapses back into a call.
616
+ if not self.available():
617
+ raise RuntimeError('No HF_TOKEN configured — persistence is unavailable.')
618
+ self._upload(name, data) # network, no cache lock held
619
+ with self._lock:
620
+ self._cache[name] = json.loads(json.dumps(data))
621
+ self._owned[name] = True
622
+ self._last_flush[name] = _time.monotonic()
623
+ # ⛔ AFTER THE UPLOAD, NEVER BEFORE — `put()`'s own law. `_upload` raises on failure,
624
+ # and a bump on that path would have `revision()` report a change that never landed.
625
+ self._bump(name)
626
+ # The upload above CARRIED whatever was pending, so the key is no longer ahead of
627
+ # the hub. Without this the flush worker wakes on a still-dirty mark and re-uploads
628
+ # identical content — a second commit against the 256-commits/hr budget for nothing.
629
+ # ⚠ AFTER the upload, for the same reason: on a raise the mark MUST survive (the cache
630
+ # is still the truth and the row must still be committed by the worker). Clearing it
631
+ # first would turn a transient upload error into the very silent loss the branch above
632
+ # exists to stop — which is why it sits here and not before `_upload`.
633
+ self._dirty[name] = False
634
+ return data
635
+
636
+
637
+ # ------------------------------------------------------------------- instances + module API
638
+ _INSTANCES = {}
639
+ _INSTANCES_LOCK = threading.RLock()
640
+
641
+
642
+ def for_repo(repo_id):
643
+ """The HF Store bound to `repo_id` — one instance per repo per process (each with its own
644
+ cache and flush state).
645
+
646
+ ⚠ WAVE 20: this is now the HF-SPECIFIC factory. Callers that want "the right store for this
647
+ tenant, whatever the backend is" call `handle()` below. `for_repo` keeps its exact old
648
+ behaviour because the seed/migration tooling has to be able to name the FILE store
649
+ explicitly while the app runs on Postgres — a migration that could only reach the active
650
+ backend could not copy between them.
651
+ """
652
+ rid = str(repo_id or '').strip() or REPO
653
+ with _INSTANCES_LOCK:
654
+ inst = _INSTANCES.get(rid)
655
+ if inst is None:
656
+ inst = Store(rid)
657
+ _INSTANCES[rid] = inst
658
+ return inst
659
+
660
+
661
+ # =============================================================================================
662
+ # WAVE 20 (owner ruling R1, closes DEBT D-4) — THE BACKEND SEAM.
663
+ #
664
+ # `core/store_backend.py` has carried this warning since EXIT-2b, and it was correct at the time:
665
+ #
666
+ # "Flipping STORE_BACKEND=pg does NOT redirect the ~40 existing callers that say
667
+ # `import core.store as store` — they are bound to the HF module directly. Rewiring them is
668
+ # task C-4 … THE DEFAULT IS `hf` AND STAYS `hf` until C-4 says otherwise."
669
+ #
670
+ # **R1 IS C-4.** The owner ruled the cutover on 2026-08-05 (the D-4 trigger that fired: a SECOND
671
+ # server process — `royal-imports/cfo-os` came back as a pinned LIVE environment beside staging,
672
+ # and two containers on one last-write-wins file store is the race B-3 was always about).
673
+ #
674
+ # THE REWIRE IS HERE RATHER THAN IN 28 FILES, and that is a deliberate choice over the obvious
675
+ # alternative of `sed`-ing every `import core.store as store` to `import core.store_backend`:
676
+ # * every one of those callers means "the store for the tenant I am serving", which is exactly
677
+ # what this module has always meant. The BACKEND is not their concern and making it their
678
+ # concern is how one of them gets missed;
679
+ # * a missed caller under a search-and-replace does not fail — it silently keeps writing to the
680
+ # file store while everything else writes to Postgres. That is the split-brain this whole
681
+ # ruling exists to end, reintroduced by the fix for it;
682
+ # * `store_backend.py`'s stated reason for refusing to do it this way — "no dual-read window and
683
+ # no way to compare the two stores' contents first" — is satisfied: `ops/seed_pg_from_hf.py`
684
+ # copies, then `--verify` diffs the two stores key-by-key before anything flips.
685
+ #
686
+ # ⛔ THE IMPORT IS LAZY AND MUST STAY LAZY. `core/store_pg.py` imports psycopg only inside
687
+ # `_pool()`, so an `hf` deployment installs no driver; importing it at module scope here would
688
+ # undo that and make the default backend depend on the optional dependency.
689
+ # =============================================================================================
690
+
691
+ def backend():
692
+ """`'hf'` | `'pg'` — validated, resolved per call so a test can flip the env var.
693
+
694
+ Mirrors `core.store_backend.name()` deliberately rather than importing it: that module
695
+ imports THIS one, so reaching back would be a cycle. `verify_store_pg` asserts the two agree
696
+ on every value, which is the guard against them drifting apart.
697
+ """
698
+ raw = (os.environ.get('STORE_BACKEND') or 'hf').strip().lower()
699
+ if raw not in ('hf', 'pg'):
700
+ raise RuntimeError(
701
+ f"STORE_BACKEND={raw!r} is not a backend. Use 'hf' (the HF Dataset store) or 'pg' "
702
+ f"(Postgres, needs DATABASE_URL). Refusing to guess: a typo that silently served the "
703
+ f"other store is how data ends up in two places.")
704
+ return raw
705
+
706
+
707
+ def handle(repo_id=None, slug=None):
708
+ """THE tenant-bound store handle for the ACTIVE backend — the one seam every caller crosses.
709
+
710
+ `repo_id` addresses the HF backend (a dataset repo); `slug` addresses Postgres (a schema).
711
+ Both are passed by `harness.runtime.get_runtime`, which knows both facts, so flipping the
712
+ backend never needs a lookup table between them — and neither identifier has to be invented
713
+ for the backend that does not use it.
714
+ """
715
+ if backend() == 'pg':
716
+ import core.store_pg as _pg # noqa: PLC0415 — lazy: see the banner above
717
+ return _pg.PgStore(slug or os.environ.get('AIOS_TENANT') or 'royal-imports')
718
+ return for_repo(repo_id)
719
+
720
+
721
+ #: The HF default instance. Kept as a module global (not a property) because the seed/migration
722
+ #: tooling and `_flush_all_at_exit` both need the FILE store by name even when pg is active.
723
+ _DEFAULT = for_repo(REPO)
724
+
725
+
726
+ def _d():
727
+ """The default tenant's store on the active backend — what every module function below uses.
728
+
729
+ ⚠ Resolved PER CALL, never cached. A cached default would freeze the backend at import time,
730
+ and import order is exactly what nobody controls: `api/main.py` imports half the platform
731
+ before it has read a single environment variable it did not inherit.
732
+ """
733
+ return handle() if backend() == 'pg' else _DEFAULT
734
+
735
+
736
+ def available():
737
+ return _d().available()
738
+
739
+
740
+ def get(name, fresh=False):
741
+ return _d().get(name, fresh=fresh)
742
+
743
+
744
+ def _read_strict(name):
745
+ return _d()._read_strict(name)
746
+
747
+
748
+ def exists(name):
749
+ return _d().exists(name)
750
+
751
+
752
+ def upload_bytes(path_in_repo, data, message=None):
753
+ return _d().upload_bytes(path_in_repo, data, message=message)
754
+
755
+
756
+ def download_bytes(path_in_repo):
757
+ return _d().download_bytes(path_in_repo)
758
+
759
+
760
+ def delete_path(path_in_repo):
761
+ return _d().delete_path(path_in_repo)
762
+
763
+
764
+ def put(name, data):
765
+ return _d().put(name, data)
766
+
767
+
768
+ def flush(name=None, timeout=30.0):
769
+ return _d().flush(name=name, timeout=timeout)
770
+
771
+
772
+ def update(name, fn, flush='sync'):
773
+ return _d().update(name, fn, flush=flush)
774
+
775
+
776
+ def get_projection(name, drop=()):
777
+ """A projected read — see `Store.get_projection`. Wave 32, W32-T01 (D-185).
778
+
779
+ ⛔ AN ELEVENTH STORE OPERATION, AND IT HAD TO BECOME ONE — the method alone was NOT enough,
780
+ which is the opposite of what it looks like from `revision()`'s precedent. `revision` could
781
+ ship method-only for a few hours because its caller held a Store. This one's caller is
782
+ `harness.runtime.TenantRuntime`, and `TenantRuntime._store()` returns **the MODULE** whenever
783
+ `store_handle is None` — which is precisely tenant #0, the tenant whose 703 ms `/nav` copy this
784
+ exists to remove. A method-only projection would have been unreachable from the only place that
785
+ needed it, while every gate stayed green.
786
+
787
+ ⛔ SO `verify_store_pg.IFACE` GAINS IT IN THE SAME CHANGE, and `core/store_pg.py` gains the
788
+ twin below it. That gate's `extra:` check would red on this function otherwise — correctly:
789
+ this addresses a key and reads it, so it is a store operation and both backends must answer.
790
+ `backend`/`handle` are exempt because they SELECT a store; this one reads one.
791
+ """
792
+ return _d().get_projection(name, drop=drop)
793
+
794
+
795
+ def revision(name):
796
+ """The bucket's change token — wave 29, C6. See `Store.revision` for why it copies nothing.
797
+
798
+ ⛔ THIS IS A TENTH STORE OPERATION, AND IT WAS ONLY ALLOWED TO BECOME ONE ONCE BOTH BACKENDS
799
+ HAD IT. `verify_store_pg.py` pins this module's public surface at a contracted list precisely
800
+ so a capability one backend lacks cannot be reached through the shared door and die at the
801
+ cutover — so this function, `store_pg.revision`, and that gate's `IFACE` entry are ONE change
802
+ and must stay one. It shipped for a few hours as a METHOD ONLY, deliberately, while the pg half
803
+ had no owner: a method is a capability the HF store has, a module function is a promise both
804
+ backends keep.
805
+ """
806
+ return _d().revision(name)
807
+
808
+
809
+ def _flush_all_at_exit():
810
+ with _INSTANCES_LOCK:
811
+ instances = list(_INSTANCES.values())
812
+ for inst in instances:
813
+ inst._flush_now_at_exit()
814
+
815
+
816
+ import atexit # noqa: E402 (registered after the class it needs)
817
+ atexit.register(_flush_all_at_exit)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
platform/core/table_store.py CHANGED
@@ -1,618 +1,618 @@
1
- """The generic per-user TABLE WORKSPACE store — the persistence half of the table-page factory.
2
-
3
- One durable store key holds one table OBJECT's per-user Airtable-style state:
4
-
5
- {username: {'views': {view_id: SavedView},
6
- 'fields': {field_key: Field}, # notes + custom_ + measure_ strata
7
- 'overlays': {str(pid): {field_key: value}}}}
8
-
9
- `make(table_key)` returns the six operations a table page's host loop needs, closed over that
10
- key. The Customer table's ops (`modules/customer_data.py`, key 'customer_table_workspace') are
11
- these exact functions — the logic MOVED here 2026-07-27 so that duplicating the Customer table
12
- pattern to a new object is a registry row + a config, not a copy of the store plumbing
13
- (owner directive: the table-page factory).
14
-
15
- A LIST (membership/formula semantics) is deliberately a different store from a VIEW
16
- (presentation/query state) — see modules/customer_data.py's customer_lists key.
17
- """
18
- import core.store as store
19
-
20
- #: Wave-9 I17 — the SHARED bucket. Views whose permissions make them visible to anyone but
21
- #: their creator live here instead of in a personal workspace, under a key that cannot collide
22
- #: with a username (usernames come from core/users.py and are never dunder-wrapped; `is_shared`
23
- #: guards it anyway). ONE HOME PER VIEW, never both: a view moved to personal is REMOVED from
24
- #: here, and a view shared is removed from its creator's workspace. Two homes would mean two
25
- #: divergent copies the moment either was edited.
26
- SHARED_KEY = '__shared__'
27
-
28
-
29
- def _may_see(view, viewer, is_admin=False):
30
- """Visibility for ONE shared view, fail-closed.
31
-
32
- 'collaborative' = everyone who can already open the module (the caller has gated that).
33
- 'users' = the named users, plus the creator, plus admins — an admin who could not
34
- see a view could not administer it either.
35
- Anything unrecognised returns False rather than defaulting open: an unreadable permission
36
- must never widen access ([[aios-permissioning]] — no fail-open defaults).
37
- """
38
- if not isinstance(view, dict):
39
- return False
40
- if view.get('createdBy') == viewer or is_admin:
41
- return True
42
- perms = view.get('permissions') or {}
43
- edit = perms.get('edit')
44
- if edit == 'collaborative':
45
- return True
46
- if edit == 'users':
47
- return viewer in set(perms.get('users') or ())
48
- return False # 'personal', absent, or junk
49
-
50
-
51
- def _may_edit(view, viewer, is_admin=False):
52
- """Who may WRITE a shared view. Same set as visibility today — the owner's item asks 'who
53
- can edit' and lists who 'can have access', i.e. seeing and editing are one grant. Kept as a
54
- separate function so they can diverge (a future read-only share) without hunting callers."""
55
- return _may_see(view, viewer, is_admin)
56
-
57
-
58
- def _may_administer(view, viewer, is_admin=False):
59
- """Who may change a view's PERMISSIONS, or delete it: the creator or an admin ONLY.
60
-
61
- Deliberately narrower than _may_edit. If a collaborator could rewrite `permissions` they
62
- could grant themselves sole ownership of somebody else's view, or quietly widen a
63
- users-scoped view to everyone — the classic privilege-escalation-by-edit hole.
64
- """
65
- if not isinstance(view, dict):
66
- return False
67
- return bool(is_admin) or view.get('createdBy') == viewer
68
-
69
-
70
- def is_shared(view):
71
- """A view belongs in the shared bucket when its permissions reach beyond its creator."""
72
- return ((view or {}).get('permissions') or {}).get('edit') in ('collaborative', 'users')
73
-
74
-
75
- def source_override_is_empty(payload):
76
- """Does this stored definition of a SOURCE (non-custom) column carry any user state?
77
-
78
- A cleared note on an immutable source field returns to the canonical schema instead of
79
- leaving a meaningless override row. Custom fields remain even with an empty note — and so
80
- does a PRESET field carrying a measure-window override (wave-2 item 8), a DISPLAY-format
81
- override (wave-5 item 10), and, since W29-T83, a COLUMN SUMMARY.
82
-
83
- ⛔ EVERY CLAUSE IS A SETTING A USER MADE, and each one omitted is a setting that silently
84
- stops surviving a session. `agg` was missing: choosing Average on Customer's `Overdue days`
85
- built an override whose only content was that summary, so this rule threw the whole row away
86
- on write while the menu went on reading "Summary: Average" from the client's own optimistic
87
- copy until the next login — a discarded WRITE wearing the face of a failed read
88
- ([[lost-write-looks-like-failed-read]]). Measured on `bac40c2`; a `ut_*` table, which stores
89
- its definitions through another door entirely, kept it.
90
-
91
- ⚠ ONE RULE, TWO CALLERS — here and `grid_events`' store-less fallback. Two copies of a
92
- discard rule is how one of them keeps a write the other bins ([[one-evaluator-per-question]]).
93
- ⚠ A CLEARED summary still drops the row, which is the intent: with nothing else set, the
94
- column goes back to whatever the contract declares for it.
95
- """
96
- payload = payload or {}
97
- if payload.get('custom'):
98
- return False
99
- return (not str(payload.get('note') or '').strip()
100
- and not isinstance(payload.get('measure'), dict)
101
- and not isinstance(payload.get('format'), dict)
102
- and not str(payload.get('agg') or '').strip())
103
-
104
-
105
- def _unique_name(wanted, existing, *, fallback='Untitled', max_len=120):
106
- """Allocate one human-facing name inside a store.update transaction.
107
-
108
- Keys/ids remain structural identity. Names compare case-insensitively after collapsing
109
- whitespace, because those variants are indistinguishable in the UI. This helper belongs
110
- in the store layer: allocating from a pre-write snapshot lets two concurrent requests both
111
- choose the same free name before either write lands.
112
- """
113
- limit = max(1, int(max_len))
114
-
115
- def _clean(value):
116
- return ' '.join(str(value or '').split())
117
-
118
- base = (_clean(wanted) or _clean(fallback) or 'Untitled')[:limit].rstrip()
119
- taken = {_clean(value).casefold() for value in existing if _clean(value)}
120
- if base.casefold() not in taken:
121
- return base
122
- index = 2
123
- while True:
124
- suffix = f' {index}'
125
- stem = base[:max(0, limit - len(suffix))].rstrip()
126
- candidate = f'{stem}{suffix}' if stem else str(index)[-limit:]
127
- if candidate.casefold() not in taken:
128
- return candidate
129
- index += 1
130
-
131
-
132
- class TableStore:
133
- """The six store operations for one table object's workspace, closed over its store key.
134
-
135
- `st` (wave 18, C3-UT) is the STORE HANDLE — anything exposing `get(name)` /
136
- `update(name, fn, flush=)`. Default = `core.store` (tenant #0, every existing caller,
137
- zero behaviour change). The API passes the session's `TenantRuntime`, whose accessors
138
- apply the tenant prefix / repo binding — which is what makes a user table created by a
139
- Nurilab admin land in Nurilab's store instead of Royal's.
140
- """
141
-
142
- def __init__(self, table_key, st=None):
143
- self.table_key = table_key
144
- self._st = st if st is not None else store
145
-
146
- @property
147
- def st(self):
148
- """The bound store handle — for SIBLING registries (core/shares) that must read the
149
- same tenant's buckets this workspace lives in (wave 21, C1)."""
150
- return self._st
151
-
152
- def find_view(self, view_id):
153
- """`(owner_username, view)` for a view living in ANY personal stratum, else None.
154
-
155
- ⭐ Wave 21 (item 9, C1): the R10 grant registry names bare ids, so projecting a granted
156
- view means locating the OWNER's record inside this topic's bucket. Personal strata
157
- only — the `__shared__` bucket has its own read path (`shared_views`), and serving one
158
- view from two finders is how two copies drift."""
159
- vid = str(view_id or '').strip()
160
- if not vid:
161
- return None
162
- try:
163
- data = self._st.get(self.table_key) or {}
164
- except Exception:
165
- return None
166
- for username, ws in data.items():
167
- if username == SHARED_KEY or not isinstance(ws, dict):
168
- continue
169
- v = (ws.get('views') or {}).get(vid)
170
- if isinstance(v, dict):
171
- return str(username), dict(v)
172
- return None
173
-
174
- def find_folder(self, folder_id):
175
- """`(owner_username, folder_row, {view_id: view})` for a VIEWS folder living in any
176
- personal stratum, else None. `find_view`'s sibling, and here for the same reason.
177
-
178
- ⭐ D-37 (wave 20's R10 remainder, closed 2026-08-05): the grant registry accepts kind
179
- `folder` and has since wave 20, but only the VIEW kind was ever projected — so "share
180
- this folder with Karen" recorded a row, listed under Shared with me, and put nothing on
181
- Karen's screen. Projecting a folder means two lookups the view path does not need: WHO
182
- owns it, and WHICH views are filed in it. Folder membership lives in the owner's
183
- `itemFolders` map (item id -> folder id), never on the view record, so the views are
184
- found by asking that map rather than by reading a list off the folder.
185
-
186
- Views only (`folders['views']`): the cohort surface has its own store and its own
187
- sharing question, and answering both here would make one function mean two things.
188
- """
189
- fid = str(folder_id or '').strip()
190
- if not fid:
191
- return None
192
- try:
193
- data = self._st.get(self.table_key) or {}
194
- except Exception:
195
- return None
196
- for username, ws in data.items():
197
- if username == SHARED_KEY or not isinstance(ws, dict):
198
- continue
199
- rows = (ws.get('folders') or {}).get('views') or []
200
- hit = next((f for f in rows
201
- if isinstance(f, dict) and str(f.get('id') or '') == fid), None)
202
- if not hit:
203
- continue
204
- # ⚠ `itemFolders` IS KEYED BY SURFACE FIRST (`{'views': {itemId: folderId}, …}`) —
205
- # reading item ids off the top level finds the surface names instead and matches
206
- # nothing, so the projection silently returns an EMPTY folder and the feature looks
207
- # exactly as broken as it was before the fix. Caught by this change's own gate,
208
- # which is the entire argument for writing one.
209
- placed = (ws.get('itemFolders') or {}).get('views') or {}
210
- views = ws.get('views') or {}
211
- inside = {str(vid): dict(v) for vid, v in views.items()
212
- if isinstance(v, dict) and str(placed.get(str(vid)) or '') == fid}
213
- return str(username), dict(hit), inside
214
- return None
215
-
216
- # ---------------------------------------------------------------- read
217
- def workspace(self, username, consume_corrections=True):
218
- """One user's durable workspace: always the full three-strata shape."""
219
- try:
220
- data = self._st.get(self.table_key) or {}
221
- ws = data.get(username, {}) or {}
222
- # A collision acknowledgement is protocol state, not part of a field definition.
223
- # Consume it with the first fresh workspace payload after the correcting write,
224
- # then splice a bounded copy into that payload only. Keeping it out of `fields`
225
- # prevents an old request id surviving forever and overriding a later rename.
226
- corrections = {}
227
- if consume_corrections and ws.get('fieldCorrections'):
228
- def _take(current):
229
- current_ws = current.get(username) or {}
230
- pending = current_ws.get('fieldCorrections') or {}
231
- corrections.update({
232
- str(key)[:80]: dict(value)
233
- for key, value in pending.items()
234
- if isinstance(value, dict)
235
- })
236
- current_ws.pop('fieldCorrections', None)
237
- return current
238
-
239
- data = self._st.update(self.table_key, _take, flush='async')
240
- ws = (data or {}).get(username, {}) or {}
241
- except Exception:
242
- ws = {}
243
- corrections = {}
244
- fields = {
245
- key: dict(value) if isinstance(value, dict) else value
246
- for key, value in (ws.get('fields') or {}).items()
247
- }
248
- for key, ack in corrections.items():
249
- field = fields.get(key)
250
- accepted_label = str(ack.get('label') or '')[:120]
251
- requested_label = str(ack.get('labelCorrectedFrom') or '')[:120]
252
- correction_id = str(ack.get('labelCorrectionId') or '')[:180]
253
- # A newer field write clears/replaces the pending ack in the SAME transaction.
254
- # The label check is an extra belt against ever attaching a stale ack to a newer
255
- # definition if a future store implementation weakens that ordering.
256
- if (isinstance(field, dict) and accepted_label
257
- and str(field.get('label') or '') == accepted_label
258
- and requested_label and correction_id):
259
- field['labelCorrectedFrom'] = requested_label
260
- field['labelCorrectionId'] = correction_id
261
- out = {
262
- 'views': dict(ws.get('views') or {}),
263
- 'fields': fields,
264
- 'overlays': dict(ws.get('overlays') or {}),
265
- # wave-8 I11 (C4): folders over the saved views / cohorts sidebars. A FOURTH
266
- # stratum rather than a key on each item — see aios_grid.clean_folders for why
267
- # (a cohort lives in another store, and filing is an organising act, not part of
268
- # what a view is). Absent for every workspace saved before this wave, which is
269
- # exactly "no folders yet".
270
- 'folders': dict(ws.get('folders') or {}),
271
- 'itemFolders': dict(ws.get('itemFolders') or {}),
272
- }
273
- # 2026-07-31 (owner item 3): WHERE THE USER LEFT OFF survives a new browser. The
274
- # client's localStorage copy wins when present; this is the server's answer for a
275
- # fresh profile, which used to fall all the way to the system default view.
276
- if ws.get('activeViewId'):
277
- out['activeViewId'] = str(ws['activeViewId'])
278
- # Wave 2026-08-02 (C-LAYOUT): the per-user record-detail field order. A fifth
279
- # stratum, absent until the user first reorders — exactly "default order".
280
- if isinstance(ws.get('recordLayout'), dict):
281
- out['recordLayout'] = dict(ws['recordLayout'])
282
- return out
283
-
284
- # ---------------------------------------------------------------- write
285
- def _update(self, username, change):
286
- def _up(data):
287
- ws = data.setdefault(username, {})
288
- ws.setdefault('views', {})
289
- ws.setdefault('fields', {})
290
- ws.setdefault('overlays', {})
291
- ws.setdefault('folders', {})
292
- ws.setdefault('itemFolders', {})
293
- change(ws)
294
- return data
295
- # flush='async' (wave-7 W3): this is THE hot path — every autosaved filter tweak,
296
- # column note and typed overlay cell lands here inside the component round-trip, and
297
- # the historical synchronous hub commit cost seconds per edit. The mutation applies to
298
- # the in-process cache (read-your-writes for every subsequent render); the hub write
299
- # coalesces in the background. Registry/auth writes elsewhere stay flush='sync'.
300
- return self._st.update(self.table_key, _up, flush='async')
301
-
302
- def rename_choice_values(self, username, change):
303
- """Apply an arbitrary workspace rewrite (wave 20, item 15 / C-RENAME).
304
-
305
- ⚠ NAMED FOR ITS ONE CALLER RATHER THAN EXPOSED AS A GENERIC `mutate`, deliberately. A
306
- public "do anything to the workspace" method is an invitation to put write logic in
307
- callers instead of here, and every OTHER method on this class exists precisely because
308
- that logic belongs in one place. Renaming a choice is the one operation that must touch
309
- three strata AT ONCE — the field's `choices`, the cells in `overlays`, and the views that
310
- filter or colour by the old value — inside a SINGLE transaction, because a rename that
311
- updated the cells and not the filters would leave a saved view matching nothing.
312
-
313
- `change(ws)` receives the whole workspace with every stratum pre-created (see `_update`).
314
- """
315
- return self._update(username, change)
316
-
317
- def save_active_view(self, username, view_id):
318
- """Remember which view this user last opened (owner item 3, 2026-07-31).
319
-
320
- Presentation state, not authorisation: the READ side re-validates the id against what
321
- the caller may actually see, so a stale or foreign id degrades to the default view
322
- rather than granting anything. Stored per user like every other stratum.
323
- """
324
- vid = str(view_id or '').strip()[:120]
325
- if not vid or username == SHARED_KEY:
326
- return
327
-
328
- def _set(ws):
329
- ws['activeViewId'] = vid
330
- self._update(username, _set)
331
-
332
- def save_record_layout(self, username, order):
333
- """The per-user RECORD-DETAIL field order (wave 2026-08-02, C-LAYOUT).
334
-
335
- Presentation state for ONE surface — the record modal. Deliberately not view config:
336
- the owner's ask is per-user, not per-view, and it must never reorder grid columns.
337
- The event handler validated keys against the live field set; the wire re-validates at
338
- serve time (aios_grid.workspace_wire), so a deleted field cannot outlive itself here.
339
- An empty order clears the stratum back to "default order".
340
- """
341
- if username == SHARED_KEY:
342
- return
343
- clean, seen = [], set()
344
- for key in (order or [])[:200]:
345
- key = str(key or '').strip()[:80]
346
- if key and key not in seen:
347
- seen.add(key)
348
- clean.append(key)
349
-
350
- def _set(ws):
351
- if clean:
352
- ws['recordLayout'] = {'order': clean}
353
- else:
354
- ws.pop('recordLayout', None)
355
-
356
- self._update(username, _set)
357
-
358
- def save_folders(self, username, folders, item_folders):
359
- """Replace the folder stratum wholesale (wave-8 I11).
360
-
361
- Wholesale rather than per-folder because the caller has ALREADY validated the complete
362
- picture through aios_grid.clean_folders / clean_item_folders, and those two are
363
- interdependent: a placement is only legal while its folder exists, so committing them
364
- separately would leave a window where a reader sees an item filed into a folder that is
365
- not there yet. One write, one consistent state.
366
- """
367
- def _set(ws):
368
- ws['folders'] = dict(folders or {})
369
- ws['itemFolders'] = dict(item_folders or {})
370
- self._update(username, _set)
371
-
372
- def save_view_order(self, username, order):
373
- """⭐ WAVE-27 item 5 (contract C7) — this user's own ORDER for the views rail.
374
-
375
- Wholesale, like `save_folders` above and for the same reason: the client sends the full
376
- list it is looking at, not a delta, because a partial order cannot say where an UNNAMED
377
- view went.
378
-
379
- ⛔ PER USER, and it belongs in this stratum rather than on the view records themselves.
380
- `aios_grid`'s own folder note argues it out for placements and every word applies: an
381
- arrangement is a per-user ORGANISING act, not part of what a view IS — so keeping it out
382
- of the view config means duplicating, sharing or exporting a view does not drag one
383
- person's rail position along with it. It also means a SHARED view can sit in a different
384
- place for each person who can see it, which is the only coherent answer once two people
385
- share one view.
386
-
387
- An empty list CLEARS the arrangement (back to server order) rather than storing `[]`.
388
- """
389
- def _set(ws):
390
- clean = []
391
- seen = set()
392
- for vid in (order or []):
393
- vid = str(vid).strip()[:120]
394
- if vid and vid not in seen:
395
- seen.add(vid)
396
- clean.append(vid)
397
- if clean:
398
- ws['viewOrder'] = clean
399
- else:
400
- ws.pop('viewOrder', None)
401
- self._update(username, _set)
402
-
403
- def shared_views(self, viewer, is_admin=False):
404
- """Every SHARED view this viewer may see, by id (wave-9 I17).
405
-
406
- Read-only and independent of the viewer's own workspace: the caller merges. Returns
407
- only what `_may_see` allows, so a caller cannot accidentally render somebody else's
408
- personal view by forgetting to filter.
409
- """
410
- try:
411
- bucket = ((self._st.get(self.table_key) or {}).get(SHARED_KEY) or {}).get('views') or {}
412
- except Exception:
413
- return {}
414
- return {vid: dict(v) for vid, v in bucket.items()
415
- if _may_see(v, viewer, is_admin)}
416
-
417
- def shared_view(self, view_id):
418
- """One shared view RAW — no visibility filter. For authorisation decisions only: a
419
- caller must know a view exists and who owns it before it can decide whether the actor
420
- may touch it. Never hand the result to a renderer without checking `_may_see`."""
421
- try:
422
- return ((self._st.get(self.table_key) or {}).get(SHARED_KEY) or {}
423
- ).get('views', {}).get(str(view_id))
424
- except Exception:
425
- return None
426
-
427
- def save_view(self, username, view, shared=None, reserved_names=(), is_admin=False):
428
- """Upsert a SavedView into its ONE home — personal workspace or the shared bucket.
429
-
430
- `shared` defaults to reading the view's own permissions (`is_shared`). Whichever home
431
- it lands in, the view is REMOVED from the other, so a view can never exist as two
432
- copies that diverge on the next edit.
433
-
434
- ⚠ AUTHORISATION IS THE CALLER'S JOB and must happen BEFORE this is called — this layer
435
- moves data and does not know who is asking. `_cl_handle_one` is the wall.
436
- """
437
- view_id = str((view or {}).get('id') or '').strip()
438
- if not view_id:
439
- raise ValueError('view id is required')
440
- if username == SHARED_KEY:
441
- raise ValueError('reserved username')
442
- to_shared = is_shared(view) if shared is None else bool(shared)
443
- requested = dict(view)
444
- accepted = {}
445
-
446
- def _up(data):
447
- # View names are tenant-global: every personal workspace plus the shared bucket.
448
- # This deliberately includes views the actor cannot see. The only disclosed fact
449
- # is that a display name is already taken, while the categorical "no duplicate
450
- # view names" contract remains true when a personal view is later shared.
451
- names = list(reserved_names or ())
452
- for workspace in data.values():
453
- if not isinstance(workspace, dict):
454
- continue
455
- names.extend(
456
- value.get('name')
457
- for candidate_id, value in (workspace.get('views') or {}).items()
458
- if candidate_id != view_id and isinstance(value, dict)
459
- )
460
- payload = dict(requested)
461
- payload['name'] = _unique_name(payload.get('name'), names)
462
- accepted.clear()
463
- accepted.update(payload)
464
- if to_shared:
465
- bucket = data.setdefault(SHARED_KEY, {})
466
- bucket.setdefault('views', {})[view_id] = payload
467
- # it may have lived in the creator's workspace before being shared
468
- owner = data.get(payload.get('createdBy') or username) or {}
469
- (owner.get('views') or {}).pop(view_id, None)
470
- else:
471
- ws = data.setdefault(username, {})
472
- ws.setdefault('views', {})[view_id] = payload
473
- (data.get(SHARED_KEY, {}).get('views') or {}).pop(view_id, None)
474
- return data
475
-
476
- self._st.update(self.table_key, _up, flush='async')
477
- return dict(accepted)
478
-
479
- def delete_view(self, username, view_id):
480
- """Delete a custom/list view override. The system all-rows view is guarded by caller.
481
-
482
- Removes from BOTH homes: the caller has already authorised the delete, and leaving a
483
- stale copy in the other bucket would resurrect the view on the next read.
484
- """
485
- vid = str(view_id)
486
-
487
- def _up(data):
488
- (data.get(username, {}).get('views') or {}).pop(vid, None)
489
- (data.get(SHARED_KEY, {}).get('views') or {}).pop(vid, None)
490
- return data
491
-
492
- self._st.update(self.table_key, _up, flush='async')
493
-
494
- def save_field(self, username, field, reserved_names=(), correction_id=None):
495
- """Persist a column note or a user-created (custom_/measure_) field definition."""
496
- key = str((field or {}).get('key') or '').strip()
497
- if not key:
498
- raise ValueError('field key is required')
499
- requested = dict(field)
500
- accepted = {}
501
-
502
- def _save(ws):
503
- names = list(reserved_names or ())
504
- names.extend(
505
- value.get('label')
506
- for candidate_key, value in (ws.get('fields') or {}).items()
507
- if candidate_key != key and isinstance(value, dict)
508
- )
509
- payload = dict(requested)
510
- payload.pop('labelCorrectedFrom', None)
511
- payload.pop('labelCorrectionId', None)
512
- requested_label = ' '.join(
513
- str(payload.get('label') or 'Untitled').split())[:120].rstrip()
514
- payload['label'] = _unique_name(requested_label, names)
515
- corrections = ws.setdefault('fieldCorrections', {})
516
- corrections.pop(key, None)
517
- if payload['label'] != requested_label and correction_id:
518
- corrections[key] = {
519
- 'label': payload['label'],
520
- 'labelCorrectedFrom': requested_label,
521
- 'labelCorrectionId': str(correction_id)[:180],
522
- }
523
- if not corrections:
524
- ws.pop('fieldCorrections', None)
525
- accepted.clear()
526
- accepted.update(payload)
527
- if source_override_is_empty(payload):
528
- ws['fields'].pop(key, None)
529
- else:
530
- ws['fields'][key] = payload
531
-
532
- self._update(username, _save)
533
- return dict(accepted)
534
-
535
- def duplicate_field(self, username, source_key, new_key, field,
536
- reserved_names=(), correction_id=None):
537
- """Clone a user-created field in ONE store transaction (wave-5 item 1): the new
538
- definition plus — for `custom_` overlay sources only — every stored cell value under
539
- the source key. One transaction, because a def without its values (or values without a
540
- def) is exactly the orphan state delete_field exists to prevent, in reverse.
541
- The caller validated both keys (same created stratum) and stamped the clone's
542
- createdBy; this layer only moves data."""
543
- source_key = str(source_key or '').strip()
544
- new_key = str(new_key or '').strip()
545
- if not source_key or not new_key or source_key == new_key:
546
- raise ValueError('duplicate_field needs two distinct keys')
547
- requested = dict(field)
548
- accepted = {}
549
-
550
- def _dup(ws):
551
- names = list(reserved_names or ())
552
- names.extend(
553
- value.get('label')
554
- for candidate_key, value in (ws.get('fields') or {}).items()
555
- if candidate_key != new_key and isinstance(value, dict)
556
- )
557
- payload = dict(requested)
558
- payload.pop('labelCorrectedFrom', None)
559
- payload.pop('labelCorrectionId', None)
560
- requested_label = ' '.join(
561
- str(payload.get('label') or 'Untitled').split())[:120].rstrip()
562
- payload['label'] = _unique_name(requested_label, names)
563
- corrections = ws.setdefault('fieldCorrections', {})
564
- corrections.pop(new_key, None)
565
- if payload['label'] != requested_label and correction_id:
566
- corrections[new_key] = {
567
- 'label': payload['label'],
568
- 'labelCorrectedFrom': requested_label,
569
- 'labelCorrectionId': str(correction_id)[:180],
570
- }
571
- if not corrections:
572
- ws.pop('fieldCorrections', None)
573
- accepted.clear()
574
- accepted.update(payload)
575
- ws['fields'][new_key] = payload
576
- if source_key.startswith('custom_'):
577
- for row in ws['overlays'].values():
578
- if isinstance(row, dict) and source_key in row:
579
- row[new_key] = row[source_key]
580
-
581
- self._update(username, _dup)
582
- return dict(accepted)
583
-
584
- def delete_field(self, username, key):
585
- """Delete a USER-CREATED field definition outright (owner gap closed 2026-07-27).
586
-
587
- Only the created strata ever reach here (`custom_` overlay fields, `measure_` formula
588
- columns — the caller enforces the prefix). The stored overlay VALUES for the key are
589
- scrubbed with it: a deleted column's cells must not linger as orphan data that would
590
- silently resurface if the key were ever reused. Views referencing the key self-heal on
591
- their next autosave (an unknown colId is dropped) — the rule every stale key rides.
592
- """
593
- key = str(key or '').strip()
594
- if not key:
595
- return
596
-
597
- def _drop(ws):
598
- ws['fields'].pop(key, None)
599
- for row in ws['overlays'].values():
600
- if isinstance(row, dict):
601
- row.pop(key, None)
602
-
603
- self._update(username, _drop)
604
-
605
- def patch_overlay(self, username, pid, updates):
606
- """Patch only the external editable stratum; never writes to the source system."""
607
- clean = dict(updates or {})
608
- if not clean:
609
- return
610
-
611
- def _patch(ws):
612
- ws['overlays'].setdefault(str(int(pid)), {}).update(clean)
613
-
614
- self._update(username, _patch)
615
-
616
-
617
- def make(table_key, st=None):
618
- return TableStore(table_key, st=st)
 
1
+ """The generic per-user TABLE WORKSPACE store — the persistence half of the table-page factory.
2
+
3
+ One durable store key holds one table OBJECT's per-user Airtable-style state:
4
+
5
+ {username: {'views': {view_id: SavedView},
6
+ 'fields': {field_key: Field}, # notes + custom_ + measure_ strata
7
+ 'overlays': {str(pid): {field_key: value}}}}
8
+
9
+ `make(table_key)` returns the six operations a table page's host loop needs, closed over that
10
+ key. The Customer table's ops (`modules/customer_data.py`, key 'customer_table_workspace') are
11
+ these exact functions — the logic MOVED here 2026-07-27 so that duplicating the Customer table
12
+ pattern to a new object is a registry row + a config, not a copy of the store plumbing
13
+ (owner directive: the table-page factory).
14
+
15
+ A LIST (membership/formula semantics) is deliberately a different store from a VIEW
16
+ (presentation/query state) — see modules/customer_data.py's customer_lists key.
17
+ """
18
+ import core.store as store
19
+
20
+ #: Wave-9 I17 — the SHARED bucket. Views whose permissions make them visible to anyone but
21
+ #: their creator live here instead of in a personal workspace, under a key that cannot collide
22
+ #: with a username (usernames come from core/users.py and are never dunder-wrapped; `is_shared`
23
+ #: guards it anyway). ONE HOME PER VIEW, never both: a view moved to personal is REMOVED from
24
+ #: here, and a view shared is removed from its creator's workspace. Two homes would mean two
25
+ #: divergent copies the moment either was edited.
26
+ SHARED_KEY = '__shared__'
27
+
28
+
29
+ def _may_see(view, viewer, is_admin=False):
30
+ """Visibility for ONE shared view, fail-closed.
31
+
32
+ 'collaborative' = everyone who can already open the module (the caller has gated that).
33
+ 'users' = the named users, plus the creator, plus admins — an admin who could not
34
+ see a view could not administer it either.
35
+ Anything unrecognised returns False rather than defaulting open: an unreadable permission
36
+ must never widen access ([[aios-permissioning]] — no fail-open defaults).
37
+ """
38
+ if not isinstance(view, dict):
39
+ return False
40
+ if view.get('createdBy') == viewer or is_admin:
41
+ return True
42
+ perms = view.get('permissions') or {}
43
+ edit = perms.get('edit')
44
+ if edit == 'collaborative':
45
+ return True
46
+ if edit == 'users':
47
+ return viewer in set(perms.get('users') or ())
48
+ return False # 'personal', absent, or junk
49
+
50
+
51
+ def _may_edit(view, viewer, is_admin=False):
52
+ """Who may WRITE a shared view. Same set as visibility today — the owner's item asks 'who
53
+ can edit' and lists who 'can have access', i.e. seeing and editing are one grant. Kept as a
54
+ separate function so they can diverge (a future read-only share) without hunting callers."""
55
+ return _may_see(view, viewer, is_admin)
56
+
57
+
58
+ def _may_administer(view, viewer, is_admin=False):
59
+ """Who may change a view's PERMISSIONS, or delete it: the creator or an admin ONLY.
60
+
61
+ Deliberately narrower than _may_edit. If a collaborator could rewrite `permissions` they
62
+ could grant themselves sole ownership of somebody else's view, or quietly widen a
63
+ users-scoped view to everyone — the classic privilege-escalation-by-edit hole.
64
+ """
65
+ if not isinstance(view, dict):
66
+ return False
67
+ return bool(is_admin) or view.get('createdBy') == viewer
68
+
69
+
70
+ def is_shared(view):
71
+ """A view belongs in the shared bucket when its permissions reach beyond its creator."""
72
+ return ((view or {}).get('permissions') or {}).get('edit') in ('collaborative', 'users')
73
+
74
+
75
+ def source_override_is_empty(payload):
76
+ """Does this stored definition of a SOURCE (non-custom) column carry any user state?
77
+
78
+ A cleared note on an immutable source field returns to the canonical schema instead of
79
+ leaving a meaningless override row. Custom fields remain even with an empty note — and so
80
+ does a PRESET field carrying a measure-window override (wave-2 item 8), a DISPLAY-format
81
+ override (wave-5 item 10), and, since W29-T83, a COLUMN SUMMARY.
82
+
83
+ ⛔ EVERY CLAUSE IS A SETTING A USER MADE, and each one omitted is a setting that silently
84
+ stops surviving a session. `agg` was missing: choosing Average on Customer's `Overdue days`
85
+ built an override whose only content was that summary, so this rule threw the whole row away
86
+ on write while the menu went on reading "Summary: Average" from the client's own optimistic
87
+ copy until the next login — a discarded WRITE wearing the face of a failed read
88
+ ([[lost-write-looks-like-failed-read]]). Measured on `bac40c2`; a `ut_*` table, which stores
89
+ its definitions through another door entirely, kept it.
90
+
91
+ ⚠ ONE RULE, TWO CALLERS — here and `grid_events`' store-less fallback. Two copies of a
92
+ discard rule is how one of them keeps a write the other bins ([[one-evaluator-per-question]]).
93
+ ⚠ A CLEARED summary still drops the row, which is the intent: with nothing else set, the
94
+ column goes back to whatever the contract declares for it.
95
+ """
96
+ payload = payload or {}
97
+ if payload.get('custom'):
98
+ return False
99
+ return (not str(payload.get('note') or '').strip()
100
+ and not isinstance(payload.get('measure'), dict)
101
+ and not isinstance(payload.get('format'), dict)
102
+ and not str(payload.get('agg') or '').strip())
103
+
104
+
105
+ def _unique_name(wanted, existing, *, fallback='Untitled', max_len=120):
106
+ """Allocate one human-facing name inside a store.update transaction.
107
+
108
+ Keys/ids remain structural identity. Names compare case-insensitively after collapsing
109
+ whitespace, because those variants are indistinguishable in the UI. This helper belongs
110
+ in the store layer: allocating from a pre-write snapshot lets two concurrent requests both
111
+ choose the same free name before either write lands.
112
+ """
113
+ limit = max(1, int(max_len))
114
+
115
+ def _clean(value):
116
+ return ' '.join(str(value or '').split())
117
+
118
+ base = (_clean(wanted) or _clean(fallback) or 'Untitled')[:limit].rstrip()
119
+ taken = {_clean(value).casefold() for value in existing if _clean(value)}
120
+ if base.casefold() not in taken:
121
+ return base
122
+ index = 2
123
+ while True:
124
+ suffix = f' {index}'
125
+ stem = base[:max(0, limit - len(suffix))].rstrip()
126
+ candidate = f'{stem}{suffix}' if stem else str(index)[-limit:]
127
+ if candidate.casefold() not in taken:
128
+ return candidate
129
+ index += 1
130
+
131
+
132
+ class TableStore:
133
+ """The six store operations for one table object's workspace, closed over its store key.
134
+
135
+ `st` (wave 18, C3-UT) is the STORE HANDLE — anything exposing `get(name)` /
136
+ `update(name, fn, flush=)`. Default = `core.store` (tenant #0, every existing caller,
137
+ zero behaviour change). The API passes the session's `TenantRuntime`, whose accessors
138
+ apply the tenant prefix / repo binding — which is what makes a user table created by a
139
+ Nurilab admin land in Nurilab's store instead of Royal's.
140
+ """
141
+
142
+ def __init__(self, table_key, st=None):
143
+ self.table_key = table_key
144
+ self._st = st if st is not None else store
145
+
146
+ @property
147
+ def st(self):
148
+ """The bound store handle — for SIBLING registries (core/shares) that must read the
149
+ same tenant's buckets this workspace lives in (wave 21, C1)."""
150
+ return self._st
151
+
152
+ def find_view(self, view_id):
153
+ """`(owner_username, view)` for a view living in ANY personal stratum, else None.
154
+
155
+ ⭐ Wave 21 (item 9, C1): the R10 grant registry names bare ids, so projecting a granted
156
+ view means locating the OWNER's record inside this topic's bucket. Personal strata
157
+ only — the `__shared__` bucket has its own read path (`shared_views`), and serving one
158
+ view from two finders is how two copies drift."""
159
+ vid = str(view_id or '').strip()
160
+ if not vid:
161
+ return None
162
+ try:
163
+ data = self._st.get(self.table_key) or {}
164
+ except Exception:
165
+ return None
166
+ for username, ws in data.items():
167
+ if username == SHARED_KEY or not isinstance(ws, dict):
168
+ continue
169
+ v = (ws.get('views') or {}).get(vid)
170
+ if isinstance(v, dict):
171
+ return str(username), dict(v)
172
+ return None
173
+
174
+ def find_folder(self, folder_id):
175
+ """`(owner_username, folder_row, {view_id: view})` for a VIEWS folder living in any
176
+ personal stratum, else None. `find_view`'s sibling, and here for the same reason.
177
+
178
+ ⭐ D-37 (wave 20's R10 remainder, closed 2026-08-05): the grant registry accepts kind
179
+ `folder` and has since wave 20, but only the VIEW kind was ever projected — so "share
180
+ this folder with Karen" recorded a row, listed under Shared with me, and put nothing on
181
+ Karen's screen. Projecting a folder means two lookups the view path does not need: WHO
182
+ owns it, and WHICH views are filed in it. Folder membership lives in the owner's
183
+ `itemFolders` map (item id -> folder id), never on the view record, so the views are
184
+ found by asking that map rather than by reading a list off the folder.
185
+
186
+ Views only (`folders['views']`): the cohort surface has its own store and its own
187
+ sharing question, and answering both here would make one function mean two things.
188
+ """
189
+ fid = str(folder_id or '').strip()
190
+ if not fid:
191
+ return None
192
+ try:
193
+ data = self._st.get(self.table_key) or {}
194
+ except Exception:
195
+ return None
196
+ for username, ws in data.items():
197
+ if username == SHARED_KEY or not isinstance(ws, dict):
198
+ continue
199
+ rows = (ws.get('folders') or {}).get('views') or []
200
+ hit = next((f for f in rows
201
+ if isinstance(f, dict) and str(f.get('id') or '') == fid), None)
202
+ if not hit:
203
+ continue
204
+ # ⚠ `itemFolders` IS KEYED BY SURFACE FIRST (`{'views': {itemId: folderId}, …}`) —
205
+ # reading item ids off the top level finds the surface names instead and matches
206
+ # nothing, so the projection silently returns an EMPTY folder and the feature looks
207
+ # exactly as broken as it was before the fix. Caught by this change's own gate,
208
+ # which is the entire argument for writing one.
209
+ placed = (ws.get('itemFolders') or {}).get('views') or {}
210
+ views = ws.get('views') or {}
211
+ inside = {str(vid): dict(v) for vid, v in views.items()
212
+ if isinstance(v, dict) and str(placed.get(str(vid)) or '') == fid}
213
+ return str(username), dict(hit), inside
214
+ return None
215
+
216
+ # ---------------------------------------------------------------- read
217
+ def workspace(self, username, consume_corrections=True):
218
+ """One user's durable workspace: always the full three-strata shape."""
219
+ try:
220
+ data = self._st.get(self.table_key) or {}
221
+ ws = data.get(username, {}) or {}
222
+ # A collision acknowledgement is protocol state, not part of a field definition.
223
+ # Consume it with the first fresh workspace payload after the correcting write,
224
+ # then splice a bounded copy into that payload only. Keeping it out of `fields`
225
+ # prevents an old request id surviving forever and overriding a later rename.
226
+ corrections = {}
227
+ if consume_corrections and ws.get('fieldCorrections'):
228
+ def _take(current):
229
+ current_ws = current.get(username) or {}
230
+ pending = current_ws.get('fieldCorrections') or {}
231
+ corrections.update({
232
+ str(key)[:80]: dict(value)
233
+ for key, value in pending.items()
234
+ if isinstance(value, dict)
235
+ })
236
+ current_ws.pop('fieldCorrections', None)
237
+ return current
238
+
239
+ data = self._st.update(self.table_key, _take, flush='async')
240
+ ws = (data or {}).get(username, {}) or {}
241
+ except Exception:
242
+ ws = {}
243
+ corrections = {}
244
+ fields = {
245
+ key: dict(value) if isinstance(value, dict) else value
246
+ for key, value in (ws.get('fields') or {}).items()
247
+ }
248
+ for key, ack in corrections.items():
249
+ field = fields.get(key)
250
+ accepted_label = str(ack.get('label') or '')[:120]
251
+ requested_label = str(ack.get('labelCorrectedFrom') or '')[:120]
252
+ correction_id = str(ack.get('labelCorrectionId') or '')[:180]
253
+ # A newer field write clears/replaces the pending ack in the SAME transaction.
254
+ # The label check is an extra belt against ever attaching a stale ack to a newer
255
+ # definition if a future store implementation weakens that ordering.
256
+ if (isinstance(field, dict) and accepted_label
257
+ and str(field.get('label') or '') == accepted_label
258
+ and requested_label and correction_id):
259
+ field['labelCorrectedFrom'] = requested_label
260
+ field['labelCorrectionId'] = correction_id
261
+ out = {
262
+ 'views': dict(ws.get('views') or {}),
263
+ 'fields': fields,
264
+ 'overlays': dict(ws.get('overlays') or {}),
265
+ # wave-8 I11 (C4): folders over the saved views / cohorts sidebars. A FOURTH
266
+ # stratum rather than a key on each item — see aios_grid.clean_folders for why
267
+ # (a cohort lives in another store, and filing is an organising act, not part of
268
+ # what a view is). Absent for every workspace saved before this wave, which is
269
+ # exactly "no folders yet".
270
+ 'folders': dict(ws.get('folders') or {}),
271
+ 'itemFolders': dict(ws.get('itemFolders') or {}),
272
+ }
273
+ # 2026-07-31 (owner item 3): WHERE THE USER LEFT OFF survives a new browser. The
274
+ # client's localStorage copy wins when present; this is the server's answer for a
275
+ # fresh profile, which used to fall all the way to the system default view.
276
+ if ws.get('activeViewId'):
277
+ out['activeViewId'] = str(ws['activeViewId'])
278
+ # Wave 2026-08-02 (C-LAYOUT): the per-user record-detail field order. A fifth
279
+ # stratum, absent until the user first reorders — exactly "default order".
280
+ if isinstance(ws.get('recordLayout'), dict):
281
+ out['recordLayout'] = dict(ws['recordLayout'])
282
+ return out
283
+
284
+ # ---------------------------------------------------------------- write
285
+ def _update(self, username, change):
286
+ def _up(data):
287
+ ws = data.setdefault(username, {})
288
+ ws.setdefault('views', {})
289
+ ws.setdefault('fields', {})
290
+ ws.setdefault('overlays', {})
291
+ ws.setdefault('folders', {})
292
+ ws.setdefault('itemFolders', {})
293
+ change(ws)
294
+ return data
295
+ # flush='async' (wave-7 W3): this is THE hot path — every autosaved filter tweak,
296
+ # column note and typed overlay cell lands here inside the component round-trip, and
297
+ # the historical synchronous hub commit cost seconds per edit. The mutation applies to
298
+ # the in-process cache (read-your-writes for every subsequent render); the hub write
299
+ # coalesces in the background. Registry/auth writes elsewhere stay flush='sync'.
300
+ return self._st.update(self.table_key, _up, flush='async')
301
+
302
+ def rename_choice_values(self, username, change):
303
+ """Apply an arbitrary workspace rewrite (wave 20, item 15 / C-RENAME).
304
+
305
+ ⚠ NAMED FOR ITS ONE CALLER RATHER THAN EXPOSED AS A GENERIC `mutate`, deliberately. A
306
+ public "do anything to the workspace" method is an invitation to put write logic in
307
+ callers instead of here, and every OTHER method on this class exists precisely because
308
+ that logic belongs in one place. Renaming a choice is the one operation that must touch
309
+ three strata AT ONCE — the field's `choices`, the cells in `overlays`, and the views that
310
+ filter or colour by the old value — inside a SINGLE transaction, because a rename that
311
+ updated the cells and not the filters would leave a saved view matching nothing.
312
+
313
+ `change(ws)` receives the whole workspace with every stratum pre-created (see `_update`).
314
+ """
315
+ return self._update(username, change)
316
+
317
+ def save_active_view(self, username, view_id):
318
+ """Remember which view this user last opened (owner item 3, 2026-07-31).
319
+
320
+ Presentation state, not authorisation: the READ side re-validates the id against what
321
+ the caller may actually see, so a stale or foreign id degrades to the default view
322
+ rather than granting anything. Stored per user like every other stratum.
323
+ """
324
+ vid = str(view_id or '').strip()[:120]
325
+ if not vid or username == SHARED_KEY:
326
+ return
327
+
328
+ def _set(ws):
329
+ ws['activeViewId'] = vid
330
+ self._update(username, _set)
331
+
332
+ def save_record_layout(self, username, order):
333
+ """The per-user RECORD-DETAIL field order (wave 2026-08-02, C-LAYOUT).
334
+
335
+ Presentation state for ONE surface — the record modal. Deliberately not view config:
336
+ the owner's ask is per-user, not per-view, and it must never reorder grid columns.
337
+ The event handler validated keys against the live field set; the wire re-validates at
338
+ serve time (aios_grid.workspace_wire), so a deleted field cannot outlive itself here.
339
+ An empty order clears the stratum back to "default order".
340
+ """
341
+ if username == SHARED_KEY:
342
+ return
343
+ clean, seen = [], set()
344
+ for key in (order or [])[:200]:
345
+ key = str(key or '').strip()[:80]
346
+ if key and key not in seen:
347
+ seen.add(key)
348
+ clean.append(key)
349
+
350
+ def _set(ws):
351
+ if clean:
352
+ ws['recordLayout'] = {'order': clean}
353
+ else:
354
+ ws.pop('recordLayout', None)
355
+
356
+ self._update(username, _set)
357
+
358
+ def save_folders(self, username, folders, item_folders):
359
+ """Replace the folder stratum wholesale (wave-8 I11).
360
+
361
+ Wholesale rather than per-folder because the caller has ALREADY validated the complete
362
+ picture through aios_grid.clean_folders / clean_item_folders, and those two are
363
+ interdependent: a placement is only legal while its folder exists, so committing them
364
+ separately would leave a window where a reader sees an item filed into a folder that is
365
+ not there yet. One write, one consistent state.
366
+ """
367
+ def _set(ws):
368
+ ws['folders'] = dict(folders or {})
369
+ ws['itemFolders'] = dict(item_folders or {})
370
+ self._update(username, _set)
371
+
372
+ def save_view_order(self, username, order):
373
+ """⭐ WAVE-27 item 5 (contract C7) — this user's own ORDER for the views rail.
374
+
375
+ Wholesale, like `save_folders` above and for the same reason: the client sends the full
376
+ list it is looking at, not a delta, because a partial order cannot say where an UNNAMED
377
+ view went.
378
+
379
+ ⛔ PER USER, and it belongs in this stratum rather than on the view records themselves.
380
+ `aios_grid`'s own folder note argues it out for placements and every word applies: an
381
+ arrangement is a per-user ORGANISING act, not part of what a view IS — so keeping it out
382
+ of the view config means duplicating, sharing or exporting a view does not drag one
383
+ person's rail position along with it. It also means a SHARED view can sit in a different
384
+ place for each person who can see it, which is the only coherent answer once two people
385
+ share one view.
386
+
387
+ An empty list CLEARS the arrangement (back to server order) rather than storing `[]`.
388
+ """
389
+ def _set(ws):
390
+ clean = []
391
+ seen = set()
392
+ for vid in (order or []):
393
+ vid = str(vid).strip()[:120]
394
+ if vid and vid not in seen:
395
+ seen.add(vid)
396
+ clean.append(vid)
397
+ if clean:
398
+ ws['viewOrder'] = clean
399
+ else:
400
+ ws.pop('viewOrder', None)
401
+ self._update(username, _set)
402
+
403
+ def shared_views(self, viewer, is_admin=False):
404
+ """Every SHARED view this viewer may see, by id (wave-9 I17).
405
+
406
+ Read-only and independent of the viewer's own workspace: the caller merges. Returns
407
+ only what `_may_see` allows, so a caller cannot accidentally render somebody else's
408
+ personal view by forgetting to filter.
409
+ """
410
+ try:
411
+ bucket = ((self._st.get(self.table_key) or {}).get(SHARED_KEY) or {}).get('views') or {}
412
+ except Exception:
413
+ return {}
414
+ return {vid: dict(v) for vid, v in bucket.items()
415
+ if _may_see(v, viewer, is_admin)}
416
+
417
+ def shared_view(self, view_id):
418
+ """One shared view RAW — no visibility filter. For authorisation decisions only: a
419
+ caller must know a view exists and who owns it before it can decide whether the actor
420
+ may touch it. Never hand the result to a renderer without checking `_may_see`."""
421
+ try:
422
+ return ((self._st.get(self.table_key) or {}).get(SHARED_KEY) or {}
423
+ ).get('views', {}).get(str(view_id))
424
+ except Exception:
425
+ return None
426
+
427
+ def save_view(self, username, view, shared=None, reserved_names=(), is_admin=False):
428
+ """Upsert a SavedView into its ONE home — personal workspace or the shared bucket.
429
+
430
+ `shared` defaults to reading the view's own permissions (`is_shared`). Whichever home
431
+ it lands in, the view is REMOVED from the other, so a view can never exist as two
432
+ copies that diverge on the next edit.
433
+
434
+ ⚠ AUTHORISATION IS THE CALLER'S JOB and must happen BEFORE this is called — this layer
435
+ moves data and does not know who is asking. `_cl_handle_one` is the wall.
436
+ """
437
+ view_id = str((view or {}).get('id') or '').strip()
438
+ if not view_id:
439
+ raise ValueError('view id is required')
440
+ if username == SHARED_KEY:
441
+ raise ValueError('reserved username')
442
+ to_shared = is_shared(view) if shared is None else bool(shared)
443
+ requested = dict(view)
444
+ accepted = {}
445
+
446
+ def _up(data):
447
+ # View names are tenant-global: every personal workspace plus the shared bucket.
448
+ # This deliberately includes views the actor cannot see. The only disclosed fact
449
+ # is that a display name is already taken, while the categorical "no duplicate
450
+ # view names" contract remains true when a personal view is later shared.
451
+ names = list(reserved_names or ())
452
+ for workspace in data.values():
453
+ if not isinstance(workspace, dict):
454
+ continue
455
+ names.extend(
456
+ value.get('name')
457
+ for candidate_id, value in (workspace.get('views') or {}).items()
458
+ if candidate_id != view_id and isinstance(value, dict)
459
+ )
460
+ payload = dict(requested)
461
+ payload['name'] = _unique_name(payload.get('name'), names)
462
+ accepted.clear()
463
+ accepted.update(payload)
464
+ if to_shared:
465
+ bucket = data.setdefault(SHARED_KEY, {})
466
+ bucket.setdefault('views', {})[view_id] = payload
467
+ # it may have lived in the creator's workspace before being shared
468
+ owner = data.get(payload.get('createdBy') or username) or {}
469
+ (owner.get('views') or {}).pop(view_id, None)
470
+ else:
471
+ ws = data.setdefault(username, {})
472
+ ws.setdefault('views', {})[view_id] = payload
473
+ (data.get(SHARED_KEY, {}).get('views') or {}).pop(view_id, None)
474
+ return data
475
+
476
+ self._st.update(self.table_key, _up, flush='async')
477
+ return dict(accepted)
478
+
479
+ def delete_view(self, username, view_id):
480
+ """Delete a custom/list view override. The system all-rows view is guarded by caller.
481
+
482
+ Removes from BOTH homes: the caller has already authorised the delete, and leaving a
483
+ stale copy in the other bucket would resurrect the view on the next read.
484
+ """
485
+ vid = str(view_id)
486
+
487
+ def _up(data):
488
+ (data.get(username, {}).get('views') or {}).pop(vid, None)
489
+ (data.get(SHARED_KEY, {}).get('views') or {}).pop(vid, None)
490
+ return data
491
+
492
+ self._st.update(self.table_key, _up, flush='async')
493
+
494
+ def save_field(self, username, field, reserved_names=(), correction_id=None):
495
+ """Persist a column note or a user-created (custom_/measure_) field definition."""
496
+ key = str((field or {}).get('key') or '').strip()
497
+ if not key:
498
+ raise ValueError('field key is required')
499
+ requested = dict(field)
500
+ accepted = {}
501
+
502
+ def _save(ws):
503
+ names = list(reserved_names or ())
504
+ names.extend(
505
+ value.get('label')
506
+ for candidate_key, value in (ws.get('fields') or {}).items()
507
+ if candidate_key != key and isinstance(value, dict)
508
+ )
509
+ payload = dict(requested)
510
+ payload.pop('labelCorrectedFrom', None)
511
+ payload.pop('labelCorrectionId', None)
512
+ requested_label = ' '.join(
513
+ str(payload.get('label') or 'Untitled').split())[:120].rstrip()
514
+ payload['label'] = _unique_name(requested_label, names)
515
+ corrections = ws.setdefault('fieldCorrections', {})
516
+ corrections.pop(key, None)
517
+ if payload['label'] != requested_label and correction_id:
518
+ corrections[key] = {
519
+ 'label': payload['label'],
520
+ 'labelCorrectedFrom': requested_label,
521
+ 'labelCorrectionId': str(correction_id)[:180],
522
+ }
523
+ if not corrections:
524
+ ws.pop('fieldCorrections', None)
525
+ accepted.clear()
526
+ accepted.update(payload)
527
+ if source_override_is_empty(payload):
528
+ ws['fields'].pop(key, None)
529
+ else:
530
+ ws['fields'][key] = payload
531
+
532
+ self._update(username, _save)
533
+ return dict(accepted)
534
+
535
+ def duplicate_field(self, username, source_key, new_key, field,
536
+ reserved_names=(), correction_id=None):
537
+ """Clone a user-created field in ONE store transaction (wave-5 item 1): the new
538
+ definition plus — for `custom_` overlay sources only — every stored cell value under
539
+ the source key. One transaction, because a def without its values (or values without a
540
+ def) is exactly the orphan state delete_field exists to prevent, in reverse.
541
+ The caller validated both keys (same created stratum) and stamped the clone's
542
+ createdBy; this layer only moves data."""
543
+ source_key = str(source_key or '').strip()
544
+ new_key = str(new_key or '').strip()
545
+ if not source_key or not new_key or source_key == new_key:
546
+ raise ValueError('duplicate_field needs two distinct keys')
547
+ requested = dict(field)
548
+ accepted = {}
549
+
550
+ def _dup(ws):
551
+ names = list(reserved_names or ())
552
+ names.extend(
553
+ value.get('label')
554
+ for candidate_key, value in (ws.get('fields') or {}).items()
555
+ if candidate_key != new_key and isinstance(value, dict)
556
+ )
557
+ payload = dict(requested)
558
+ payload.pop('labelCorrectedFrom', None)
559
+ payload.pop('labelCorrectionId', None)
560
+ requested_label = ' '.join(
561
+ str(payload.get('label') or 'Untitled').split())[:120].rstrip()
562
+ payload['label'] = _unique_name(requested_label, names)
563
+ corrections = ws.setdefault('fieldCorrections', {})
564
+ corrections.pop(new_key, None)
565
+ if payload['label'] != requested_label and correction_id:
566
+ corrections[new_key] = {
567
+ 'label': payload['label'],
568
+ 'labelCorrectedFrom': requested_label,
569
+ 'labelCorrectionId': str(correction_id)[:180],
570
+ }
571
+ if not corrections:
572
+ ws.pop('fieldCorrections', None)
573
+ accepted.clear()
574
+ accepted.update(payload)
575
+ ws['fields'][new_key] = payload
576
+ if source_key.startswith('custom_'):
577
+ for row in ws['overlays'].values():
578
+ if isinstance(row, dict) and source_key in row:
579
+ row[new_key] = row[source_key]
580
+
581
+ self._update(username, _dup)
582
+ return dict(accepted)
583
+
584
+ def delete_field(self, username, key):
585
+ """Delete a USER-CREATED field definition outright (owner gap closed 2026-07-27).
586
+
587
+ Only the created strata ever reach here (`custom_` overlay fields, `measure_` formula
588
+ columns — the caller enforces the prefix). The stored overlay VALUES for the key are
589
+ scrubbed with it: a deleted column's cells must not linger as orphan data that would
590
+ silently resurface if the key were ever reused. Views referencing the key self-heal on
591
+ their next autosave (an unknown colId is dropped) — the rule every stale key rides.
592
+ """
593
+ key = str(key or '').strip()
594
+ if not key:
595
+ return
596
+
597
+ def _drop(ws):
598
+ ws['fields'].pop(key, None)
599
+ for row in ws['overlays'].values():
600
+ if isinstance(row, dict):
601
+ row.pop(key, None)
602
+
603
+ self._update(username, _drop)
604
+
605
+ def patch_overlay(self, username, pid, updates):
606
+ """Patch only the external editable stratum; never writes to the source system."""
607
+ clean = dict(updates or {})
608
+ if not clean:
609
+ return
610
+
611
+ def _patch(ws):
612
+ ws['overlays'].setdefault(str(int(pid)), {}).update(clean)
613
+
614
+ self._update(username, _patch)
615
+
616
+
617
+ def make(table_key, st=None):
618
+ return TableStore(table_key, st=st)
platform/core/user_tables.py CHANGED
The diff for this file is too large to render. See raw diff
 
platform/core/users.py CHANGED
@@ -445,20 +445,21 @@ def assignable_people(tenant=None):
445
 
446
  Wave 18 (C1-TENANT): pass `tenant` to scope the choices to ONE company — the user registry
447
  is a global control-plane bucket, and a Nurilab picker offering Royal's staff is a
448
- cross-tenant name leak.
449
-
450
- ⛔⛔ WAVE 33 (W33-T37) — A BLANK `tenant` NOW RETURNS NOTHING. It used to skip the filter and
451
- return EVERY tenant's staff: `if want and …` is structurally fail-OPEN, and the exemption was
452
- written for "None = unscoped (the Streamlit host, tenant #0's process)" — a host DELETED at
453
- EXIT-6. So the sanctioned caller no longer exists, and what was left is a whole-platform
454
- roster one forgotten kwarg away, with no error to notice
455
- ([[gate-must-go-red-not-crash]]'s sibling: a wall that answers instead of refusing).
456
- ⚠ The direction is the safety: this can only ever NARROW. All six live call sites pass
457
- `session.tenant`, which `aios_session.read` refuses to admit blank, so nothing legitimate
458
- changes — and a future caller that forgets gets an empty picker somebody notices instead of a
459
- leak nobody does.
460
  """
461
- return sorted({str(u.get('name') or n) for n, u in _tenant_accounts(tenant)})
 
 
 
 
 
 
 
 
 
 
 
 
462
 
463
 
464
  def set_avatar(username, data_url):
@@ -485,41 +486,20 @@ def avatar_map(tenant=None):
485
  name, so the display name is the only join a renderer has. Two active accounts sharing a
486
  display name share one option; the first WITH a photo wins the key rather than a coin
487
  flip deciding whether the option has a face. `tenant` scopes it exactly as
488
- `assignable_people(tenant)` does, and for the same leak — including wave 33's fail-closed
489
- blank, which both take from the ONE resolver below rather than each writing `if want and …`.
490
- """
 
 
 
491
  out = {}
492
- for username, u in sorted(_tenant_accounts(tenant), key=lambda kv: kv[0]):
 
 
 
 
493
  av = u.get('avatar')
494
  nm = str(u.get('name') or username)
495
  if av and nm not in out:
496
  out[nm] = str(av)
497
  return out
498
-
499
-
500
- def _tenant_accounts(tenant):
501
- """`[(username, record), …]` — the ACTIVE accounts of exactly ONE tenant. FAIL-CLOSED.
502
-
503
- ⛔ THE ONE PLACE THE TENANT FILTER FOR THE USER REGISTRY IS WRITTEN. It was written twice,
504
- identically, in `assignable_people` and `avatar_map`, and both copies were fail-OPEN on a
505
- blank slug in the same way (`if want and …`). Two copies of a wall is two places for one of
506
- them to be fixed [[one-question-two-normalizers]]; `routes_shares._people` is a THIRD copy of
507
- the same predicate and is lane C's to fold in.
508
-
509
- ⚠ `str(u.get('tenant') or 'royal-imports')` is kept on the RECORD side deliberately: a
510
- pre-wave-18 account genuinely has no `tenant` key and IS tenant #0's, and `users._public`
511
- normalises it the same way. What changed is the WANT side — a blank want is now a refusal
512
- rather than a wildcard.
513
- """
514
- want = str(tenant or '').strip().lower()
515
- if not want:
516
- # Not an error and not everything: nobody. See `assignable_people`'s note — the one
517
- # caller this exemption was written for (the Streamlit host) was deleted at EXIT-6.
518
- return []
519
- try:
520
- reg = registry() or {}
521
- except Exception:
522
- return []
523
- return [(n, u) for n, u in reg.items()
524
- if isinstance(u, dict) and u.get('active') is not False
525
- and str(u.get('tenant') or 'royal-imports').strip().lower() == want]
 
445
 
446
  Wave 18 (C1-TENANT): pass `tenant` to scope the choices to ONE company — the user registry
447
  is a global control-plane bucket, and a Nurilab picker offering Royal's staff is a
448
+ cross-tenant name leak. None = unscoped (the Streamlit host, tenant #0's process).
 
 
 
 
 
 
 
 
 
 
 
449
  """
450
+ try:
451
+ reg = registry() or {}
452
+ except Exception:
453
+ return []
454
+ want = str(tenant or '').strip().lower()
455
+ out = []
456
+ for username, u in reg.items():
457
+ if not isinstance(u, dict) or u.get('active') is False:
458
+ continue
459
+ if want and str(u.get('tenant') or 'royal-imports').strip().lower() != want:
460
+ continue
461
+ out.append(str(u.get('name') or username))
462
+ return sorted(set(out))
463
 
464
 
465
  def set_avatar(username, data_url):
 
486
  name, so the display name is the only join a renderer has. Two active accounts sharing a
487
  display name share one option; the first WITH a photo wins the key rather than a coin
488
  flip deciding whether the option has a face. `tenant` scopes it exactly as
489
+ `assignable_people(tenant)` does, and for the same leak."""
490
+ try:
491
+ reg = registry() or {}
492
+ except Exception:
493
+ return {}
494
+ want = str(tenant or '').strip().lower()
495
  out = {}
496
+ for username, u in sorted(reg.items()):
497
+ if not isinstance(u, dict) or u.get('active') is False:
498
+ continue
499
+ if want and str(u.get('tenant') or 'royal-imports').strip().lower() != want:
500
+ continue
501
  av = u.get('avatar')
502
  nm = str(u.get('name') or username)
503
  if av and nm not in out:
504
  out[nm] = str(av)
505
  return out
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
platform/harness/datastore.py CHANGED
@@ -188,35 +188,14 @@ ENTITIES = {
188
  # backfill below has walked every partner (a partner's `write_date` did not move because
189
  # WE changed the schema). Every projection degrades a missing/NULL column to blank rather
190
  # than raising, so declaring early is safe — it is just not yet true.
191
- # ⭐⭐ WAVE 33 (W33-T48, owner item 13) — THE CONTACT/IDENTITY HALF, MEASURED BEFORE IT WAS
192
- # ADDED. The owner: *"I could have sworn we have more relevant Fields under 'Odoo vendors'
193
- # etc."* He was right, and the census
194
- # (`.claude/wiki/waves/wave33/odoo-field-census.md`, live read-only, 2026-08-14) says by how
195
- # much: `res.partner` declares 198 fields, **77 are populated on all 3,621 customers and 76
196
- # on all 658 vendors**, and this mirror carried THIRTEEN. Every one of the six added here
197
- # was measured non-empty on the live population first — none is added on the strength of
198
- # "Odoo probably has that", which is how a schema grows columns nobody can fill.
199
- # ⚠ THE MIRROR IS THE CEILING, NOT THE GRID. Every reader in `odoo_relational.py` reads
200
- # THIS file, so a column added to a `*_fields()` contract with no entry here renders as a
201
- # blank column forever — which is D-165's shape exactly, and the reason the widening starts
202
- # on this line rather than on the grid.
203
  "fields": ["name", "team_id", "city", "state_id", "country_id", "active",
204
  "agent", "salesman_as_agent", "customer_rank",
205
- "street", "street2", "zip",
206
- "email", "phone", "mobile", "website", "vat", "ref",
207
- "write_date"],
208
  # `backfill_fields` carries it too, so mirrors that already exist pick it up on the next
209
  # sync_all() instead of needing a reseed.
210
- # ⛔ D-165 IS ABOUT THIS LIST, AND THE CENSUS SETTLED WHICH HALF IS WRONG: `street`,
211
- # `street2` and `zip` are `store=True` on `res.partner` and POPULATED in live Odoo — they
212
- # are declared here and still NULL in the mirror purely because a backfill pass has not
213
- # walked every partner (changing OUR schema does not move a partner's `write_date`, so an
214
- # incremental sync never revisits them). Declaring is not filling; the fix is a run.
215
  "backfill_fields": ["agent", "salesman_as_agent", # added 2026-07-28
216
  "customer_rank", # added 2026-08-11 (W29-T54)
217
- "street", "street2", "zip", # added 2026-08-12 (W30-T33)
218
- "email", "phone", "mobile", # added 2026-08-14 (W33-T48)
219
- "website", "vat", "ref"],
220
  # m2m fields land as <base>_id (the FIRST linked id — the Customers-module convention:
221
  # a customer's assigned agent is agent_ids[0], ~one per customer, MECE) + <field> as a
222
  # JSON list of ALL ids. Declared explicitly: a 2-element m2m read would otherwise be
 
188
  # backfill below has walked every partner (a partner's `write_date` did not move because
189
  # WE changed the schema). Every projection degrades a missing/NULL column to blank rather
190
  # than raising, so declaring early is safe — it is just not yet true.
 
 
 
 
 
 
 
 
 
 
 
 
191
  "fields": ["name", "team_id", "city", "state_id", "country_id", "active",
192
  "agent", "salesman_as_agent", "customer_rank",
193
+ "street", "street2", "zip", "write_date"],
 
 
194
  # `backfill_fields` carries it too, so mirrors that already exist pick it up on the next
195
  # sync_all() instead of needing a reseed.
 
 
 
 
 
196
  "backfill_fields": ["agent", "salesman_as_agent", # added 2026-07-28
197
  "customer_rank", # added 2026-08-11 (W29-T54)
198
+ "street", "street2", "zip"], # added 2026-08-12 (W30-T33)
 
 
199
  # m2m fields land as <base>_id (the FIRST linked id — the Customers-module convention:
200
  # a customer's assigned agent is agent_ids[0], ~one per customer, MECE) + <field> as a
201
  # JSON list of ALL ids. Declared explicitly: a 2-element m2m read would otherwise be
platform/harness/meta_store.py CHANGED
@@ -1,521 +1,521 @@
1
- """meta_store.py — the REST loader for a mirror whose only other loader is XML-RPC (W31-T48).
2
-
3
- python platform/harness/meta_store.py --sync pull into tenant #0's mirror
4
- python platform/harness/meta_store.py --sync --tenant gtmlab
5
- python platform/harness/meta_store.py --status what is in the mirror now
6
-
7
- ⛔ WHY THIS IS A SIBLING AND NOT A ROW IN `datastore.ENTITIES`. `datastore.sync_entity` is
8
- hardwired to XML-RPC — it does `import core.odoo as O` and speaks `search_read` — so no amount of
9
- spec data makes it fetch over HTTPS. What IS reusable is everything *below* the fetch: the
10
- per-tenant file (`path_for`), the process-singleton connection (`connect`), and the
11
- delete-then-insert upsert. This module borrows those and brings its own reader. **The mirror is one
12
- store with two loaders, not two stores.**
13
-
14
- ⭐ THE SHAPE OF THE WHOLE THING, because it is the owner's actual request: *"we just need to pull
15
- the data into our template database for Meta"*, working *"just like Odoo"*. Odoo's path is
16
- `XML-RPC -> DuckDB mirror -> odoo_relational -> ut_odoo_* locked grids`. Meta's is
17
- `Graph -> DuckDB mirror -> meta_relational -> ut_meta_* locked grids`. Same middle, same end, one
18
- different first hop.
19
-
20
- ⚠ EVERY COLUMN NAME BELOW WAS MEASURED, NOT READ OFF A DOC (R8). `proto/meta-entity-fields.json`
21
- carries the run: each level's list is what the API ACCEPTED when asked, on a real ad account.
22
- Ad Account 45 · Campaign 35 · Ad Set 54 · Ad 36 · Creative 58 · Insights 57
23
- ⛔ AND `ACCEPTED` IS NOT `RETURNED`. Graph omits null fields from a response, so a 20-field ask
24
- came back with 14 keys. Building a schema from what came back would drop ~30% of the columns with
25
- nothing going red — which is exactly the "do not drop any column" instruction, broken silently.
26
- The lists below are therefore the ASKED-AND-ACCEPTED set, and a column with no value is NULL.
27
-
28
- ⚠ `owner` IS ABSENT FROM THE AD ACCOUNT LIST AND THAT IS A REPORTED LIMIT, NOT AN OMISSION: it
29
- answers `403 (#200) Requires business_management permission`, which this token does not carry. One
30
- column of 285. R6's second sentence — a limit that cannot be removed gets stated with its cause.
31
-
32
- ⛔ IDS ARE TEXT, ALWAYS. A Meta object id is a 17-digit decimal string; `120273975028650555` is
33
- larger than 2^53, so any float or JS-number path silently corrupts it. Same ruling as `ig_id`
34
- (W26/R3), for the same reason, and it is why every `id` column here is VARCHAR.
35
- """
36
- import argparse
37
- import json
38
- import os
39
- import sys
40
- import time
41
- import urllib.error
42
- import urllib.parse
43
- import urllib.request
44
- from pathlib import Path
45
-
46
- _HERE = Path(__file__).resolve().parent
47
- if str(_HERE.parent) not in sys.path:
48
- sys.path.insert(0, str(_HERE.parent))
49
-
50
- from harness import datastore # noqa: E402
51
-
52
- GRAPH_VERSION = os.environ.get("META_GRAPH_VERSION") or "v21.0"
53
- GRAPH = f"https://graph.facebook.com/{GRAPH_VERSION}"
54
-
55
- #: Numeric columns, by name. Everything else is VARCHAR — including ids (see the header) and
56
- #: including anything Graph returns as a nested object, which is stored as compact JSON text.
57
- _INT = {"impressions", "reach", "clicks", "unique_clicks", "inline_link_clicks",
58
- "inline_post_engagement", "full_view_impressions", "estimated_ad_recallers",
59
- "account_status", "age", "timezone_id", "timezone_offset_hours_utc", "io_number",
60
- "min_daily_budget", "min_campaign_group_spend_cap"}
61
- _DBL = {"spend", "social_spend", "ctr", "unique_ctr", "cpc", "cpm", "cpp", "frequency",
62
- "inline_link_click_ctr", "outbound_clicks_ctr", "amount_spent", "balance", "spend_cap",
63
- "daily_budget", "lifetime_budget", "budget_remaining", "bid_amount", "canvas_avg_view_time",
64
- "daily_min_spend_target", "daily_spend_cap", "lifetime_min_spend_target",
65
- "lifetime_spend_cap", "lifetime_imps"}
66
-
67
- #: ⭐ THE MEASURED CATALOG. `edge` is the connection on the ad account; `None` means the account
68
- #: itself. `parent` names the column that ties a row to its parent, used by nothing here and by
69
- #: the grid links in `meta_relational`.
70
- ACCOUNT_FIELDS = (
71
- "account_id account_status age amount_spent balance business_city business_country_code "
72
- "business_name business_state business_street business_street2 business_zip capabilities "
73
- "created_time currency disable_reason end_advertiser end_advertiser_name funding_source "
74
- "has_migrated_permissions id io_number is_attribution_spec_system_default "
75
- "is_direct_deals_enabled is_notifications_enabled is_personal is_prepay_account "
76
- "is_tax_id_required line_numbers media_agency min_campaign_group_spend_cap min_daily_budget "
77
- "name offsite_pixels_tos_accepted partner spend_cap tax_id tax_id_status tax_id_type "
78
- "timezone_id timezone_name timezone_offset_hours_utc tos_accepted user_tasks user_tos_accepted"
79
- ).split()
80
-
81
- CAMPAIGN_FIELDS = (
82
- "account_id bid_strategy boosted_object_id budget_rebalance_flag budget_remaining buying_type "
83
- "campaign_group_active_time can_create_brand_lift_study can_use_spend_cap configured_status "
84
- "created_time daily_budget effective_status id is_skadnetwork_attribution issues_info "
85
- "last_budget_toggling_time lifetime_budget name objective pacing_type primary_attribution "
86
- "promoted_object smart_promotion_type source_campaign source_campaign_id special_ad_categories "
87
- "special_ad_category special_ad_category_country spend_cap start_time status stop_time "
88
- "topline_id updated_time"
89
- ).split()
90
-
91
- ADSET_FIELDS = (
92
- "account_id adlabels adset_schedule asset_feed_id attribution_spec bid_adjustments bid_amount "
93
- "bid_constraints bid_info bid_strategy billing_event budget_remaining campaign "
94
- "campaign_active_time campaign_attribution campaign_id configured_status created_time "
95
- "creative_sequence daily_budget daily_min_spend_target daily_spend_cap destination_type "
96
- "effective_status end_time frequency_control_specs id instagram_actor_id is_dynamic_creative "
97
- "issues_info learning_stage_info lifetime_budget lifetime_imps lifetime_min_spend_target "
98
- "lifetime_spend_cap multi_optimization_goal_weight name optimization_goal "
99
- "optimization_sub_event pacing_type promoted_object recurring_budget_semantics review_feedback "
100
- "rf_prediction_id source_adset source_adset_id start_time status targeting "
101
- "targeting_optimization_types time_based_ad_rotation_id_blocks "
102
- "time_based_ad_rotation_intervals updated_time use_new_app_click"
103
- ).split()
104
-
105
- AD_FIELDS = (
106
- "account_id ad_active_time ad_review_feedback ad_schedule_end_time ad_schedule_start_time "
107
- "adlabels adset adset_id bid_amount bid_info bid_type campaign campaign_id configured_status "
108
- "conversion_domain created_time creative demolink_hash display_sequence effective_status "
109
- "engagement_audience failed_delivery_checks id issues_info last_updated_by_app_id name "
110
- "preview_shareable_link priority recommendations source_ad source_ad_id status targeting "
111
- "tracking_and_conversion_with_defaults tracking_specs updated_time"
112
- ).split()
113
-
114
- CREATIVE_FIELDS = (
115
- "account_id actor_id adlabels applink_treatment asset_feed_spec authorization_category body "
116
- "branded_content_sponsor_page_id bundle_folder_id call_to_action_type categorization_criteria "
117
- "category_media_source collaborative_ads_lsb_image_bank_id degrees_of_freedom_spec "
118
- "destination_set_id dynamic_ad_voice effective_authorization_category "
119
- "effective_instagram_media_id effective_object_story_id enable_direct_install "
120
- "enable_launch_instant_app id image_crops image_hash image_url instagram_actor_id "
121
- "instagram_permalink_url instagram_story_id instagram_user_id interactive_components_spec "
122
- "link_deep_link_url link_destination_display_url link_og_id link_url "
123
- "messenger_sponsored_message name object_id object_store_url object_story_id object_story_spec "
124
- "object_type object_url place_page_set_id platform_customizations playable_asset_id "
125
- "portrait_customizations product_set_id recommender_settings source_instagram_media_id status "
126
- "template_url template_url_spec thumbnail_id thumbnail_url title url_tags "
127
- "use_page_actor_override video_id"
128
- ).split()
129
-
130
- INSIGHT_FIELDS = (
131
- "account_currency account_id account_name action_values actions ad_id ad_name adset_id "
132
- "adset_name attribution_setting buying_type campaign_id campaign_name canvas_avg_view_time "
133
- "clicks conversion_rate_ranking conversion_values conversions cost_per_action_type "
134
- "cost_per_inline_link_click cost_per_thruplay cost_per_unique_click cpc cpm cpp ctr date_start "
135
- "date_stop engagement_rate_ranking estimated_ad_recallers frequency full_view_impressions "
136
- "impressions inline_link_click_ctr inline_link_clicks inline_post_engagement objective "
137
- "optimization_goal outbound_clicks outbound_clicks_ctr purchase_roas quality_ranking reach "
138
- "social_spend spend unique_clicks unique_ctr unique_outbound_clicks "
139
- "video_avg_time_watched_actions video_p100_watched_actions video_p25_watched_actions "
140
- "video_p50_watched_actions video_p75_watched_actions video_p95_watched_actions "
141
- "video_play_actions video_thruplay_watched_actions website_purchase_roas"
142
- ).split()
143
-
144
- #: table -> (graph edge on the account | None, measured fields, parent column)
145
- SPECS = {
146
- "meta_ad_accounts": (None, ACCOUNT_FIELDS, None),
147
- "meta_campaigns": ("campaigns", CAMPAIGN_FIELDS, "account_id"),
148
- "meta_adsets": ("adsets", ADSET_FIELDS, "campaign_id"),
149
- "meta_ads": ("ads", AD_FIELDS, "adset_id"),
150
- "meta_creatives": ("adcreatives", CREATIVE_FIELDS, "account_id"),
151
- }
152
-
153
- #: The daily Insights grain (R2). One row per (ad, day) — the id is synthesised because Insights
154
- #: has no id of its own, and it must be STABLE so a re-run updates instead of appending.
155
- INSIGHTS_TABLE = "meta_insights_daily"
156
- INSIGHTS_LEVEL = os.environ.get("META_INSIGHTS_LEVEL") or "ad"
157
-
158
- #: ⛔ INSIGHTS IS FETCHED IN TIME SLICES, AND THE REASON IS MEASURED, NOT DEFENSIVE. All 57 fields
159
- #: at ad level over `last_90d` with a 100-row page answers **HTTP 500 "An unknown error occurred"**
160
- #: — Graph's way of saying the synchronous query is too heavy (the async report-run API is the
161
- #: other answer, and it costs a poll loop this does not need). The SAME 57 fields over 7 days at
162
- #: page 25 answer 200. So the window is walked in slices with every column intact:
163
- #: 57 fields · ad level · 7d · limit 25 -> 200, 25 rows
164
- #: 57 fields · account level · 7d -> 200, 7 rows
165
- #: ⚠ Narrowing the FIELD list would also have "fixed" it, and that is the wrong fix twice over —
166
- #: it drops columns R2 requires, and it does so invisibly.
167
- INSIGHTS_DAYS = int(os.environ.get("META_INSIGHTS_DAYS") or 90)
168
- INSIGHTS_SLICE = int(os.environ.get("META_INSIGHTS_SLICE_DAYS") or 7)
169
- INSIGHTS_PAGE = int(os.environ.get("META_INSIGHTS_PAGE") or 25)
170
-
171
-
172
- def _slices(days, size):
173
- """[(since, until)] covering the last `days`, oldest first, in `size`-day windows."""
174
- from datetime import date, timedelta
175
- end = date.today()
176
- start = end - timedelta(days=max(1, days) - 1)
177
- out, cur = [], start
178
- while cur <= end:
179
- stop = min(cur + timedelta(days=max(1, size) - 1), end)
180
- out.append((cur.isoformat(), stop.isoformat()))
181
- cur = stop + timedelta(days=1)
182
- return out
183
-
184
- #: ⚠ A REAL CEILING, AND R6's SECOND SENTENCE APPLIES TO IT. Graph pages at 25-100 rows; this is
185
- #: the number of PAGES a single edge may walk before the loader stops and SAYS it stopped. It is
186
- #: not a row cap on a connected source (R6 forbids that) — it is a runaway guard, and reaching it
187
- #: is reported as a problem, never absorbed.
188
- MAX_PAGES = int(os.environ.get("META_MAX_PAGES") or 200)
189
- PAGE = int(os.environ.get("META_PAGE_SIZE") or 100)
190
-
191
-
192
- class MetaError(RuntimeError):
193
- """A Graph refusal carrying Meta's own words. Safe to print: no token ever reaches it."""
194
-
195
-
196
- def token():
197
- """The Meta token from the environment or gitignored `platform/.env`; "" when absent.
198
-
199
- ⚠ Same resolver `aios-web/api/connectors_meta.py` uses. Duplicated deliberately and minimally:
200
- `platform/` must not import from `aios-web/api/`, which is the layering rule this repo keeps
201
- (`core` never imports up). Fifteen lines is the price of that boundary.
202
- """
203
- tok = os.environ.get("META_ADS_ACCESS_TOKEN") or ""
204
- if tok:
205
- return tok.strip()
206
- env = _HERE.parent / ".env"
207
- if env.exists():
208
- for line in env.read_text(encoding="utf-8", errors="replace").splitlines():
209
- if line.strip().startswith("META_ADS_ACCESS_TOKEN"):
210
- _, _, v = line.partition("=")
211
- return v.strip().strip('"').strip("'")
212
- return ""
213
-
214
-
215
- def _get(path, tok, **params):
216
- params["access_token"] = tok
217
- url = f"{GRAPH}/{path.lstrip('/')}?" + urllib.parse.urlencode(params)
218
- req = urllib.request.Request(url, headers={"User-Agent": "aios-meta-store/1"})
219
- try:
220
- with urllib.request.urlopen(req, timeout=120) as r:
221
- return json.loads(r.read().decode("utf-8", "replace"))
222
- except urllib.error.HTTPError as e:
223
- try:
224
- msg = (json.loads(e.read().decode("utf-8", "replace")).get("error") or {}
225
- ).get("message") or ""
226
- except Exception:
227
- msg = ""
228
- raise MetaError(f"HTTP {e.code} on /{path.lstrip('/')}: {msg[:200]}") from None
229
- except Exception as e:
230
- raise MetaError(f"{type(e).__name__} on /{path.lstrip('/')}") from None
231
-
232
-
233
- #: Graph's own words when a page is too heavy. It arrives as an HTTP **500**, not a 4xx, which is
234
- #: why it cannot be treated as "the server is broken, give up".
235
- _TOO_MUCH = "reduce the amount of data"
236
-
237
- #: ⛔ THE ADS-MANAGEMENT RATE LIMIT, WHICH IS PER AD ACCOUNT AND NOT PER TOKEN. Measured: after a
238
- #: heavy backfill Graph answers **HTTP 400 "There have been too many calls to this ad-account.
239
- #: Wait a bit and try again."** It is a 4xx, so nothing about the status code says "retry" — the
240
- #: MESSAGE is the only signal, which is why it is matched here rather than inferred from a code.
241
- #: ⚠ It persists for minutes, so the backoff is measured in minutes and BOUNDED: after
242
- #: `_RATE_TRIES` waits the loader STOPS and reports what it got, rather than sitting in a retry
243
- #: loop nobody can see. A partial mirror that says it is partial beats a hung sync.
244
- _RATE_LIMITED = "too many calls"
245
- _RATE_TRIES = int(os.environ.get("META_RATE_TRIES") or 3)
246
- _RATE_WAIT = int(os.environ.get("META_RATE_WAIT_S") or 90)
247
-
248
-
249
- def _walk(path, tok, log, **params):
250
- """Every page of an edge, paged by CURSOR under our own parameters.
251
-
252
- ⛔ IT DOES NOT FOLLOW GRAPH'S `paging.next` URL, AND THAT IS THE WHOLE POINT. Measured: the
253
- first call to `/adcreatives` at limit=100 answers **HTTP 500 "Please reduce the amount of data
254
- you're asking for"**, the retry at limit=25 succeeds — and then `next` carries the ORIGINAL
255
- limit=100 and fails again on page 2. A backoff that cannot reach every page is not a backoff.
256
- Re-issuing each page ourselves with `after=<cursor>` keeps the reduced limit for the whole walk.
257
-
258
- ⭐ It shrinks the PAGE, never the FIELD LIST. Dropping columns to make a request fit is the
259
- silent omission R2 forbids, and nothing downstream could see it. Fewer rows per call, always
260
- every column per row.
261
-
262
- -> (rows, hit_page_cap)
263
- """
264
- out, pages = [], 0
265
- limit = int(params.pop("limit", None) or PAGE)
266
- after, waited = None, 0
267
- while True:
268
- call = dict(params, limit=limit)
269
- if after:
270
- call["after"] = after
271
- try:
272
- body = _get(path, tok, **call)
273
- except MetaError as e:
274
- if _TOO_MUCH in str(e) and limit > 5:
275
- limit = max(5, limit // 4)
276
- log(f" page too heavy for /{path} - retrying at limit={limit} "
277
- f"(a payload ceiling, not a row cap; every column is still asked for)")
278
- continue
279
- if _RATE_LIMITED in str(e).lower() and waited < _RATE_TRIES:
280
- waited += 1
281
- log(f" rate-limited on /{path} (per AD ACCOUNT, not per token) - waiting "
282
- f"{_RATE_WAIT}s, attempt {waited}/{_RATE_TRIES}")
283
- time.sleep(_RATE_WAIT)
284
- continue
285
- if _RATE_LIMITED in str(e).lower():
286
- log(f" !! GIVING UP on /{path} after {waited} waits: still rate-limited. "
287
- f"{len(out)} row(s) collected so far are kept. Cause: the ads-management "
288
- f"limit is per ad account and persists for minutes. Fix: re-run "
289
- f"`--sync --only <table>` later; the upsert is idempotent.")
290
- return out, True
291
- raise
292
- out.extend(body.get("data") or [])
293
- pages += 1
294
- after = ((body.get("paging") or {}).get("cursors") or {}).get("after")
295
- has_next = bool((body.get("paging") or {}).get("next")) and bool(after)
296
- if not has_next:
297
- return out, False
298
- if pages >= MAX_PAGES:
299
- log(f" !! STOPPED at MAX_PAGES={MAX_PAGES} on /{path} with more pages left. "
300
- f"Cause: a runaway guard, not a row cap. Fix: raise META_MAX_PAGES, or narrow the "
301
- f"window with META_INSIGHTS_DAYS.")
302
- return out, True
303
-
304
-
305
- def _cell(value):
306
- """One Graph value -> one DuckDB cell. Nested structures become compact JSON TEXT rather than
307
- being dropped: R2 says every field the API returns, and `targeting` is a field."""
308
- if value is None or isinstance(value, (str, int, float, bool)):
309
- return value
310
- return json.dumps(value, separators=(",", ":"), ensure_ascii=False)
311
-
312
-
313
- def _coltype(name, rows=None):
314
- """The DuckDB type for one column — decided by the DATA when there is data, by the name only
315
- as a fallback.
316
-
317
- ⛔ THE NAME LIST WAS WRONG AND ONLY THE API COULD SAY SO. `cost_per_unique_click`,
318
- `cost_per_action_type`, `purchase_roas` and friends READ like money and are **arrays of
319
- action-type objects** on the Insights edge:
320
- [{"action_type":"outbound_click","value":"1.459854"}]
321
- Typed DOUBLE from `_DBL`, the insert died with `Conversion Error: Could not convert string
322
- '[{...}]' to DOUBLE` — after five entity tables had already been written, so the sync looked
323
- like it worked and then blew up on the last table.
324
- ⭐ Same principle the field CATALOG is built on, applied one layer down: **ask the response,
325
- do not assert from a list.** A value that ever arrives as a list or dict is JSON TEXT, because
326
- that is what `_cell` stores; anything else falls back to the measured name hints.
327
- """
328
- if rows:
329
- seen, kinds = 0, set()
330
- for r in rows:
331
- v = r.get(name)
332
- if v is None or v == "":
333
- continue
334
- kinds.add("json" if isinstance(v, (list, dict)) else
335
- "bool" if isinstance(v, bool) else
336
- "int" if isinstance(v, int) else
337
- "float" if isinstance(v, float) else "str")
338
- seen += 1
339
- if seen >= 200:
340
- break
341
- if kinds:
342
- if "json" in kinds or "str" in kinds:
343
- return "VARCHAR" # a JSON blob, or a numeric STRING
344
- if kinds <= {"int", "bool"}:
345
- return "BIGINT" if name in _INT else ("VARCHAR" if "bool" in kinds else "BIGINT")
346
- return "DOUBLE"
347
- if name in _INT:
348
- return "BIGINT"
349
- if name in _DBL:
350
- return "DOUBLE"
351
- return "VARCHAR" # ids included — see the header
352
-
353
-
354
- def _ensure(con, table, fields, rows=None):
355
- """Create or widen the table, with every column typed from `rows` when they are available.
356
-
357
- ⚠ AN EXISTING COLUMN WHOSE TYPE IS NOW WRONG IS REBUILT, NOT PATCHED. DuckDB cannot retype a
358
- column in place, and this table is DERIVED data that can be re-pulled in minutes — so a type
359
- disagreement drops and recreates rather than limping on with a column that refuses every
360
- insert. The alternative is a mirror that is permanently unwritable for one bad guess.
361
- """
362
- want = {f: _coltype(f, rows) for f in fields}
363
- have = {r[1]: str(r[2]).upper() for r in con.execute(f"PRAGMA table_info('{table}')").fetchall()}
364
- if have:
365
- clash = [f for f, ty in want.items() if f in have and have[f] != ty
366
- and not (have[f].startswith("VARCHAR") and ty == "VARCHAR")]
367
- if clash:
368
- con.execute(f"DROP TABLE {table}")
369
- have = {}
370
- if not have:
371
- cols = ", ".join(f"{f} {want[f]}" for f in fields)
372
- con.execute(f"CREATE TABLE IF NOT EXISTS {table} (id VARCHAR PRIMARY KEY, {cols})"
373
- if "id" not in fields else
374
- f"CREATE TABLE IF NOT EXISTS {table} ({cols})")
375
- return
376
- for f in fields:
377
- if f not in have:
378
- con.execute(f"ALTER TABLE {table} ADD COLUMN {f} {want[f]}")
379
-
380
-
381
- def _upsert(con, table, fields, rows):
382
- """Delete-then-insert by id — the same idempotence `datastore._upsert` gives the Odoo half, so
383
- a re-sync updates in place and can never append a second copy of the same object."""
384
- if not rows:
385
- return 0
386
- cols = list(fields)
387
- ids = [str(r.get("id") or "") for r in rows]
388
- q = ",".join("?" for _ in ids)
389
- con.execute(f"DELETE FROM {table} WHERE id IN ({q})", ids)
390
- con.executemany(
391
- f"INSERT INTO {table} ({', '.join(cols)}) VALUES ({', '.join('?' for _ in cols)})",
392
- [[_cell(r.get(c)) for c in cols] for r in rows])
393
- return len(rows)
394
-
395
-
396
- def sync(tenant_key="royal-imports", tok=None, log=print, insights=True, insights_days=None):
397
- """Pull every level into the tenant's mirror. -> a report dict; raises only on a bad token.
398
-
399
- Idempotent: re-running updates rows in place. Safe to call at boot and on the resync loop,
400
- exactly as `odoo_relational.refresh` is.
401
- """
402
- tok = tok or token()
403
- # ⛔ THE WINDOW IS A PARAMETER, NOT AN ENVIRONMENT READ AT CALL TIME — and the difference cost a
404
- # live deploy. `INSIGHTS_DAYS` binds at IMPORT (module scope), so `main._pull_meta`'s
405
- # `os.environ.setdefault("META_INSIGHTS_DAYS", "7")` executed AFTER this module was already
406
- # imported and changed nothing: every boot pulled **90 days**, not 7, which is the slow path
407
- # that trips the per-ad-account rate limit and never finishes. The comment beside that call
408
- # claimed a short window the code could not deliver.
409
- # ⭐ The shape: **a knob read at import cannot be turned by a caller at runtime.** Passing it
410
- # makes the caller's intent effective instead of aspirational; the env var stays the DEFAULT.
411
- days = int(insights_days or INSIGHTS_DAYS)
412
- report = {"tenant": tenant_key, "tables": {}, "problems": [], "accounts": []}
413
- if not tok:
414
- report["problems"].append(
415
- "META_ADS_ACCESS_TOKEN is not set in the environment or in platform/.env, so nothing "
416
- "was pulled. This is a missing CREDENTIAL, not a missing capability.")
417
- return report
418
-
419
- path = datastore.path_for(tenant_key)
420
- if Path(datastore.DB_PATH) != Path(path):
421
- datastore.use_path(path)
422
- con = datastore.connect()
423
- log(f" mirror: {Path(path).name}")
424
-
425
- accts, _ = _walk("me/adaccounts", tok, log, fields="id,name", limit=PAGE)
426
- report["accounts"] = [a.get("id") for a in accts]
427
- if not accts:
428
- report["problems"].append("the token reaches no ad accounts")
429
- return report
430
-
431
- # ── the five entity levels ────────────────────────────────────────────────────────────────
432
- for table, (edge, fields, _parent) in SPECS.items():
433
- rows, capped = [], False
434
- for acct in accts:
435
- aid = acct["id"]
436
- if edge is None:
437
- rows.append(_get(aid, tok, fields=",".join(fields)))
438
- else:
439
- got, hit = _walk(f"{aid}/{edge}", tok, log, fields=",".join(fields), limit=PAGE)
440
- rows.extend(got)
441
- capped = capped or hit
442
- _ensure(con, table, fields, rows)
443
- n = _upsert(con, table, fields, rows)
444
- total = con.execute(f"SELECT count(*) FROM {table}").fetchone()[0]
445
- report["tables"][table] = {"pulled": n, "in_mirror": total, "capped": capped}
446
- log(f" {table:<20} pulled {n:>6} mirror total {total:>6}"
447
- + (" !! PAGE CAP HIT" if capped else ""))
448
- if capped:
449
- report["problems"].append(f"{table}: stopped at MAX_PAGES={MAX_PAGES}")
450
-
451
- # ── the daily Insights grain ──────────────────────────────────────────────────────────────
452
- if insights:
453
- rows, capped = [], False
454
- for acct in accts:
455
- for since, until in _slices(days, INSIGHTS_SLICE):
456
- got, hit = _walk(f"{acct['id']}/insights", tok, log,
457
- fields=",".join(INSIGHT_FIELDS), level=INSIGHTS_LEVEL,
458
- time_increment="1", limit=INSIGHTS_PAGE,
459
- time_range=json.dumps({"since": since, "until": until}))
460
- rows.extend(got)
461
- capped = capped or hit
462
- # ⛔ Insights rows carry no id. The key must be STABLE across runs or every re-sync
463
- # appends a second copy of the same day — so it is composed from the grain itself.
464
- for r in rows:
465
- r["id"] = ":".join(str(r.get(k) or "") for k in
466
- (f"{INSIGHTS_LEVEL}_id", "date_start", "date_stop"))
467
- fields = ["id"] + INSIGHT_FIELDS
468
- _ensure(con, INSIGHTS_TABLE, fields, rows)
469
- n = _upsert(con, INSIGHTS_TABLE, fields, rows)
470
- total = con.execute(f"SELECT count(*) FROM {INSIGHTS_TABLE}").fetchone()[0]
471
- report["tables"][INSIGHTS_TABLE] = {"pulled": n, "in_mirror": total, "capped": capped}
472
- log(f" {INSIGHTS_TABLE:<20} pulled {n:>6} mirror total {total:>6}"
473
- + (" !! PAGE CAP HIT" if capped else ""))
474
-
475
- con.execute("INSERT OR REPLACE INTO _sync_state VALUES (?,?,?,?,?,?)",
476
- ["meta", "done", 0, f"{days}d/{INSIGHTS_SLICE}d@{INSIGHTS_LEVEL}",
477
- sum(t["in_mirror"] for t in report["tables"].values()),
478
- time.strftime("%Y-%m-%d %H:%M:%S")])
479
- return report
480
-
481
-
482
- def status(tenant_key="royal-imports"):
483
- """What is in the mirror right now, per table. Never fetches."""
484
- path = datastore.path_for(tenant_key)
485
- if Path(datastore.DB_PATH) != Path(path):
486
- datastore.use_path(path)
487
- out = {}
488
- con = datastore.connect()
489
- for table in list(SPECS) + [INSIGHTS_TABLE]:
490
- try:
491
- out[table] = con.execute(f"SELECT count(*) FROM {table}").fetchone()[0]
492
- except Exception:
493
- out[table] = None # table absent = never synced
494
- return out
495
-
496
-
497
- def main(argv=None):
498
- ap = argparse.ArgumentParser(description="Pull Meta Ads into the tenant's DuckDB mirror.")
499
- ap.add_argument("--sync", action="store_true")
500
- ap.add_argument("--status", action="store_true")
501
- ap.add_argument("--tenant", default="royal-imports")
502
- ap.add_argument("--no-insights", action="store_true")
503
- ap.add_argument("--insights-days", type=int, default=None,
504
- help="override the Insights window for THIS run (default INSIGHTS_DAYS); the env var is the default, this is the caller's say")
505
- a = ap.parse_args(argv)
506
- if a.status:
507
- for k, v in status(a.tenant).items():
508
- print(f" {k:<20} {'(never synced)' if v is None else v}")
509
- return 0
510
- if not a.sync:
511
- ap.print_help()
512
- return 2
513
- rep = sync(a.tenant, insights=not a.no_insights, insights_days=a.insights_days)
514
- for p in rep["problems"]:
515
- print(" PROBLEM:", p)
516
- print(f" accounts: {len(rep['accounts'])} tables: {len(rep['tables'])}")
517
- return 1 if rep["problems"] else 0
518
-
519
-
520
- if __name__ == "__main__":
521
- sys.exit(main())
 
1
+ """meta_store.py — the REST loader for a mirror whose only other loader is XML-RPC (W31-T48).
2
+
3
+ python platform/harness/meta_store.py --sync pull into tenant #0's mirror
4
+ python platform/harness/meta_store.py --sync --tenant gtmlab
5
+ python platform/harness/meta_store.py --status what is in the mirror now
6
+
7
+ ⛔ WHY THIS IS A SIBLING AND NOT A ROW IN `datastore.ENTITIES`. `datastore.sync_entity` is
8
+ hardwired to XML-RPC — it does `import core.odoo as O` and speaks `search_read` — so no amount of
9
+ spec data makes it fetch over HTTPS. What IS reusable is everything *below* the fetch: the
10
+ per-tenant file (`path_for`), the process-singleton connection (`connect`), and the
11
+ delete-then-insert upsert. This module borrows those and brings its own reader. **The mirror is one
12
+ store with two loaders, not two stores.**
13
+
14
+ ⭐ THE SHAPE OF THE WHOLE THING, because it is the owner's actual request: *"we just need to pull
15
+ the data into our template database for Meta"*, working *"just like Odoo"*. Odoo's path is
16
+ `XML-RPC -> DuckDB mirror -> odoo_relational -> ut_odoo_* locked grids`. Meta's is
17
+ `Graph -> DuckDB mirror -> meta_relational -> ut_meta_* locked grids`. Same middle, same end, one
18
+ different first hop.
19
+
20
+ ⚠ EVERY COLUMN NAME BELOW WAS MEASURED, NOT READ OFF A DOC (R8). `proto/meta-entity-fields.json`
21
+ carries the run: each level's list is what the API ACCEPTED when asked, on a real ad account.
22
+ Ad Account 45 · Campaign 35 · Ad Set 54 · Ad 36 · Creative 58 · Insights 57
23
+ ⛔ AND `ACCEPTED` IS NOT `RETURNED`. Graph omits null fields from a response, so a 20-field ask
24
+ came back with 14 keys. Building a schema from what came back would drop ~30% of the columns with
25
+ nothing going red — which is exactly the "do not drop any column" instruction, broken silently.
26
+ The lists below are therefore the ASKED-AND-ACCEPTED set, and a column with no value is NULL.
27
+
28
+ ⚠ `owner` IS ABSENT FROM THE AD ACCOUNT LIST AND THAT IS A REPORTED LIMIT, NOT AN OMISSION: it
29
+ answers `403 (#200) Requires business_management permission`, which this token does not carry. One
30
+ column of 285. R6's second sentence — a limit that cannot be removed gets stated with its cause.
31
+
32
+ ⛔ IDS ARE TEXT, ALWAYS. A Meta object id is a 17-digit decimal string; `120273975028650555` is
33
+ larger than 2^53, so any float or JS-number path silently corrupts it. Same ruling as `ig_id`
34
+ (W26/R3), for the same reason, and it is why every `id` column here is VARCHAR.
35
+ """
36
+ import argparse
37
+ import json
38
+ import os
39
+ import sys
40
+ import time
41
+ import urllib.error
42
+ import urllib.parse
43
+ import urllib.request
44
+ from pathlib import Path
45
+
46
+ _HERE = Path(__file__).resolve().parent
47
+ if str(_HERE.parent) not in sys.path:
48
+ sys.path.insert(0, str(_HERE.parent))
49
+
50
+ from harness import datastore # noqa: E402
51
+
52
+ GRAPH_VERSION = os.environ.get("META_GRAPH_VERSION") or "v21.0"
53
+ GRAPH = f"https://graph.facebook.com/{GRAPH_VERSION}"
54
+
55
+ #: Numeric columns, by name. Everything else is VARCHAR — including ids (see the header) and
56
+ #: including anything Graph returns as a nested object, which is stored as compact JSON text.
57
+ _INT = {"impressions", "reach", "clicks", "unique_clicks", "inline_link_clicks",
58
+ "inline_post_engagement", "full_view_impressions", "estimated_ad_recallers",
59
+ "account_status", "age", "timezone_id", "timezone_offset_hours_utc", "io_number",
60
+ "min_daily_budget", "min_campaign_group_spend_cap"}
61
+ _DBL = {"spend", "social_spend", "ctr", "unique_ctr", "cpc", "cpm", "cpp", "frequency",
62
+ "inline_link_click_ctr", "outbound_clicks_ctr", "amount_spent", "balance", "spend_cap",
63
+ "daily_budget", "lifetime_budget", "budget_remaining", "bid_amount", "canvas_avg_view_time",
64
+ "daily_min_spend_target", "daily_spend_cap", "lifetime_min_spend_target",
65
+ "lifetime_spend_cap", "lifetime_imps"}
66
+
67
+ #: ⭐ THE MEASURED CATALOG. `edge` is the connection on the ad account; `None` means the account
68
+ #: itself. `parent` names the column that ties a row to its parent, used by nothing here and by
69
+ #: the grid links in `meta_relational`.
70
+ ACCOUNT_FIELDS = (
71
+ "account_id account_status age amount_spent balance business_city business_country_code "
72
+ "business_name business_state business_street business_street2 business_zip capabilities "
73
+ "created_time currency disable_reason end_advertiser end_advertiser_name funding_source "
74
+ "has_migrated_permissions id io_number is_attribution_spec_system_default "
75
+ "is_direct_deals_enabled is_notifications_enabled is_personal is_prepay_account "
76
+ "is_tax_id_required line_numbers media_agency min_campaign_group_spend_cap min_daily_budget "
77
+ "name offsite_pixels_tos_accepted partner spend_cap tax_id tax_id_status tax_id_type "
78
+ "timezone_id timezone_name timezone_offset_hours_utc tos_accepted user_tasks user_tos_accepted"
79
+ ).split()
80
+
81
+ CAMPAIGN_FIELDS = (
82
+ "account_id bid_strategy boosted_object_id budget_rebalance_flag budget_remaining buying_type "
83
+ "campaign_group_active_time can_create_brand_lift_study can_use_spend_cap configured_status "
84
+ "created_time daily_budget effective_status id is_skadnetwork_attribution issues_info "
85
+ "last_budget_toggling_time lifetime_budget name objective pacing_type primary_attribution "
86
+ "promoted_object smart_promotion_type source_campaign source_campaign_id special_ad_categories "
87
+ "special_ad_category special_ad_category_country spend_cap start_time status stop_time "
88
+ "topline_id updated_time"
89
+ ).split()
90
+
91
+ ADSET_FIELDS = (
92
+ "account_id adlabels adset_schedule asset_feed_id attribution_spec bid_adjustments bid_amount "
93
+ "bid_constraints bid_info bid_strategy billing_event budget_remaining campaign "
94
+ "campaign_active_time campaign_attribution campaign_id configured_status created_time "
95
+ "creative_sequence daily_budget daily_min_spend_target daily_spend_cap destination_type "
96
+ "effective_status end_time frequency_control_specs id instagram_actor_id is_dynamic_creative "
97
+ "issues_info learning_stage_info lifetime_budget lifetime_imps lifetime_min_spend_target "
98
+ "lifetime_spend_cap multi_optimization_goal_weight name optimization_goal "
99
+ "optimization_sub_event pacing_type promoted_object recurring_budget_semantics review_feedback "
100
+ "rf_prediction_id source_adset source_adset_id start_time status targeting "
101
+ "targeting_optimization_types time_based_ad_rotation_id_blocks "
102
+ "time_based_ad_rotation_intervals updated_time use_new_app_click"
103
+ ).split()
104
+
105
+ AD_FIELDS = (
106
+ "account_id ad_active_time ad_review_feedback ad_schedule_end_time ad_schedule_start_time "
107
+ "adlabels adset adset_id bid_amount bid_info bid_type campaign campaign_id configured_status "
108
+ "conversion_domain created_time creative demolink_hash display_sequence effective_status "
109
+ "engagement_audience failed_delivery_checks id issues_info last_updated_by_app_id name "
110
+ "preview_shareable_link priority recommendations source_ad source_ad_id status targeting "
111
+ "tracking_and_conversion_with_defaults tracking_specs updated_time"
112
+ ).split()
113
+
114
+ CREATIVE_FIELDS = (
115
+ "account_id actor_id adlabels applink_treatment asset_feed_spec authorization_category body "
116
+ "branded_content_sponsor_page_id bundle_folder_id call_to_action_type categorization_criteria "
117
+ "category_media_source collaborative_ads_lsb_image_bank_id degrees_of_freedom_spec "
118
+ "destination_set_id dynamic_ad_voice effective_authorization_category "
119
+ "effective_instagram_media_id effective_object_story_id enable_direct_install "
120
+ "enable_launch_instant_app id image_crops image_hash image_url instagram_actor_id "
121
+ "instagram_permalink_url instagram_story_id instagram_user_id interactive_components_spec "
122
+ "link_deep_link_url link_destination_display_url link_og_id link_url "
123
+ "messenger_sponsored_message name object_id object_store_url object_story_id object_story_spec "
124
+ "object_type object_url place_page_set_id platform_customizations playable_asset_id "
125
+ "portrait_customizations product_set_id recommender_settings source_instagram_media_id status "
126
+ "template_url template_url_spec thumbnail_id thumbnail_url title url_tags "
127
+ "use_page_actor_override video_id"
128
+ ).split()
129
+
130
+ INSIGHT_FIELDS = (
131
+ "account_currency account_id account_name action_values actions ad_id ad_name adset_id "
132
+ "adset_name attribution_setting buying_type campaign_id campaign_name canvas_avg_view_time "
133
+ "clicks conversion_rate_ranking conversion_values conversions cost_per_action_type "
134
+ "cost_per_inline_link_click cost_per_thruplay cost_per_unique_click cpc cpm cpp ctr date_start "
135
+ "date_stop engagement_rate_ranking estimated_ad_recallers frequency full_view_impressions "
136
+ "impressions inline_link_click_ctr inline_link_clicks inline_post_engagement objective "
137
+ "optimization_goal outbound_clicks outbound_clicks_ctr purchase_roas quality_ranking reach "
138
+ "social_spend spend unique_clicks unique_ctr unique_outbound_clicks "
139
+ "video_avg_time_watched_actions video_p100_watched_actions video_p25_watched_actions "
140
+ "video_p50_watched_actions video_p75_watched_actions video_p95_watched_actions "
141
+ "video_play_actions video_thruplay_watched_actions website_purchase_roas"
142
+ ).split()
143
+
144
+ #: table -> (graph edge on the account | None, measured fields, parent column)
145
+ SPECS = {
146
+ "meta_ad_accounts": (None, ACCOUNT_FIELDS, None),
147
+ "meta_campaigns": ("campaigns", CAMPAIGN_FIELDS, "account_id"),
148
+ "meta_adsets": ("adsets", ADSET_FIELDS, "campaign_id"),
149
+ "meta_ads": ("ads", AD_FIELDS, "adset_id"),
150
+ "meta_creatives": ("adcreatives", CREATIVE_FIELDS, "account_id"),
151
+ }
152
+
153
+ #: The daily Insights grain (R2). One row per (ad, day) — the id is synthesised because Insights
154
+ #: has no id of its own, and it must be STABLE so a re-run updates instead of appending.
155
+ INSIGHTS_TABLE = "meta_insights_daily"
156
+ INSIGHTS_LEVEL = os.environ.get("META_INSIGHTS_LEVEL") or "ad"
157
+
158
+ #: ⛔ INSIGHTS IS FETCHED IN TIME SLICES, AND THE REASON IS MEASURED, NOT DEFENSIVE. All 57 fields
159
+ #: at ad level over `last_90d` with a 100-row page answers **HTTP 500 "An unknown error occurred"**
160
+ #: — Graph's way of saying the synchronous query is too heavy (the async report-run API is the
161
+ #: other answer, and it costs a poll loop this does not need). The SAME 57 fields over 7 days at
162
+ #: page 25 answer 200. So the window is walked in slices with every column intact:
163
+ #: 57 fields · ad level · 7d · limit 25 -> 200, 25 rows
164
+ #: 57 fields · account level · 7d -> 200, 7 rows
165
+ #: ⚠ Narrowing the FIELD list would also have "fixed" it, and that is the wrong fix twice over —
166
+ #: it drops columns R2 requires, and it does so invisibly.
167
+ INSIGHTS_DAYS = int(os.environ.get("META_INSIGHTS_DAYS") or 90)
168
+ INSIGHTS_SLICE = int(os.environ.get("META_INSIGHTS_SLICE_DAYS") or 7)
169
+ INSIGHTS_PAGE = int(os.environ.get("META_INSIGHTS_PAGE") or 25)
170
+
171
+
172
+ def _slices(days, size):
173
+ """[(since, until)] covering the last `days`, oldest first, in `size`-day windows."""
174
+ from datetime import date, timedelta
175
+ end = date.today()
176
+ start = end - timedelta(days=max(1, days) - 1)
177
+ out, cur = [], start
178
+ while cur <= end:
179
+ stop = min(cur + timedelta(days=max(1, size) - 1), end)
180
+ out.append((cur.isoformat(), stop.isoformat()))
181
+ cur = stop + timedelta(days=1)
182
+ return out
183
+
184
+ #: ⚠ A REAL CEILING, AND R6's SECOND SENTENCE APPLIES TO IT. Graph pages at 25-100 rows; this is
185
+ #: the number of PAGES a single edge may walk before the loader stops and SAYS it stopped. It is
186
+ #: not a row cap on a connected source (R6 forbids that) — it is a runaway guard, and reaching it
187
+ #: is reported as a problem, never absorbed.
188
+ MAX_PAGES = int(os.environ.get("META_MAX_PAGES") or 200)
189
+ PAGE = int(os.environ.get("META_PAGE_SIZE") or 100)
190
+
191
+
192
+ class MetaError(RuntimeError):
193
+ """A Graph refusal carrying Meta's own words. Safe to print: no token ever reaches it."""
194
+
195
+
196
+ def token():
197
+ """The Meta token from the environment or gitignored `platform/.env`; "" when absent.
198
+
199
+ ⚠ Same resolver `aios-web/api/connectors_meta.py` uses. Duplicated deliberately and minimally:
200
+ `platform/` must not import from `aios-web/api/`, which is the layering rule this repo keeps
201
+ (`core` never imports up). Fifteen lines is the price of that boundary.
202
+ """
203
+ tok = os.environ.get("META_ADS_ACCESS_TOKEN") or ""
204
+ if tok:
205
+ return tok.strip()
206
+ env = _HERE.parent / ".env"
207
+ if env.exists():
208
+ for line in env.read_text(encoding="utf-8", errors="replace").splitlines():
209
+ if line.strip().startswith("META_ADS_ACCESS_TOKEN"):
210
+ _, _, v = line.partition("=")
211
+ return v.strip().strip('"').strip("'")
212
+ return ""
213
+
214
+
215
+ def _get(path, tok, **params):
216
+ params["access_token"] = tok
217
+ url = f"{GRAPH}/{path.lstrip('/')}?" + urllib.parse.urlencode(params)
218
+ req = urllib.request.Request(url, headers={"User-Agent": "aios-meta-store/1"})
219
+ try:
220
+ with urllib.request.urlopen(req, timeout=120) as r:
221
+ return json.loads(r.read().decode("utf-8", "replace"))
222
+ except urllib.error.HTTPError as e:
223
+ try:
224
+ msg = (json.loads(e.read().decode("utf-8", "replace")).get("error") or {}
225
+ ).get("message") or ""
226
+ except Exception:
227
+ msg = ""
228
+ raise MetaError(f"HTTP {e.code} on /{path.lstrip('/')}: {msg[:200]}") from None
229
+ except Exception as e:
230
+ raise MetaError(f"{type(e).__name__} on /{path.lstrip('/')}") from None
231
+
232
+
233
+ #: Graph's own words when a page is too heavy. It arrives as an HTTP **500**, not a 4xx, which is
234
+ #: why it cannot be treated as "the server is broken, give up".
235
+ _TOO_MUCH = "reduce the amount of data"
236
+
237
+ #: ⛔ THE ADS-MANAGEMENT RATE LIMIT, WHICH IS PER AD ACCOUNT AND NOT PER TOKEN. Measured: after a
238
+ #: heavy backfill Graph answers **HTTP 400 "There have been too many calls to this ad-account.
239
+ #: Wait a bit and try again."** It is a 4xx, so nothing about the status code says "retry" — the
240
+ #: MESSAGE is the only signal, which is why it is matched here rather than inferred from a code.
241
+ #: ⚠ It persists for minutes, so the backoff is measured in minutes and BOUNDED: after
242
+ #: `_RATE_TRIES` waits the loader STOPS and reports what it got, rather than sitting in a retry
243
+ #: loop nobody can see. A partial mirror that says it is partial beats a hung sync.
244
+ _RATE_LIMITED = "too many calls"
245
+ _RATE_TRIES = int(os.environ.get("META_RATE_TRIES") or 3)
246
+ _RATE_WAIT = int(os.environ.get("META_RATE_WAIT_S") or 90)
247
+
248
+
249
+ def _walk(path, tok, log, **params):
250
+ """Every page of an edge, paged by CURSOR under our own parameters.
251
+
252
+ ⛔ IT DOES NOT FOLLOW GRAPH'S `paging.next` URL, AND THAT IS THE WHOLE POINT. Measured: the
253
+ first call to `/adcreatives` at limit=100 answers **HTTP 500 "Please reduce the amount of data
254
+ you're asking for"**, the retry at limit=25 succeeds — and then `next` carries the ORIGINAL
255
+ limit=100 and fails again on page 2. A backoff that cannot reach every page is not a backoff.
256
+ Re-issuing each page ourselves with `after=<cursor>` keeps the reduced limit for the whole walk.
257
+
258
+ ⭐ It shrinks the PAGE, never the FIELD LIST. Dropping columns to make a request fit is the
259
+ silent omission R2 forbids, and nothing downstream could see it. Fewer rows per call, always
260
+ every column per row.
261
+
262
+ -> (rows, hit_page_cap)
263
+ """
264
+ out, pages = [], 0
265
+ limit = int(params.pop("limit", None) or PAGE)
266
+ after, waited = None, 0
267
+ while True:
268
+ call = dict(params, limit=limit)
269
+ if after:
270
+ call["after"] = after
271
+ try:
272
+ body = _get(path, tok, **call)
273
+ except MetaError as e:
274
+ if _TOO_MUCH in str(e) and limit > 5:
275
+ limit = max(5, limit // 4)
276
+ log(f" page too heavy for /{path} - retrying at limit={limit} "
277
+ f"(a payload ceiling, not a row cap; every column is still asked for)")
278
+ continue
279
+ if _RATE_LIMITED in str(e).lower() and waited < _RATE_TRIES:
280
+ waited += 1
281
+ log(f" rate-limited on /{path} (per AD ACCOUNT, not per token) - waiting "
282
+ f"{_RATE_WAIT}s, attempt {waited}/{_RATE_TRIES}")
283
+ time.sleep(_RATE_WAIT)
284
+ continue
285
+ if _RATE_LIMITED in str(e).lower():
286
+ log(f" !! GIVING UP on /{path} after {waited} waits: still rate-limited. "
287
+ f"{len(out)} row(s) collected so far are kept. Cause: the ads-management "
288
+ f"limit is per ad account and persists for minutes. Fix: re-run "
289
+ f"`--sync --only <table>` later; the upsert is idempotent.")
290
+ return out, True
291
+ raise
292
+ out.extend(body.get("data") or [])
293
+ pages += 1
294
+ after = ((body.get("paging") or {}).get("cursors") or {}).get("after")
295
+ has_next = bool((body.get("paging") or {}).get("next")) and bool(after)
296
+ if not has_next:
297
+ return out, False
298
+ if pages >= MAX_PAGES:
299
+ log(f" !! STOPPED at MAX_PAGES={MAX_PAGES} on /{path} with more pages left. "
300
+ f"Cause: a runaway guard, not a row cap. Fix: raise META_MAX_PAGES, or narrow the "
301
+ f"window with META_INSIGHTS_DAYS.")
302
+ return out, True
303
+
304
+
305
+ def _cell(value):
306
+ """One Graph value -> one DuckDB cell. Nested structures become compact JSON TEXT rather than
307
+ being dropped: R2 says every field the API returns, and `targeting` is a field."""
308
+ if value is None or isinstance(value, (str, int, float, bool)):
309
+ return value
310
+ return json.dumps(value, separators=(",", ":"), ensure_ascii=False)
311
+
312
+
313
+ def _coltype(name, rows=None):
314
+ """The DuckDB type for one column — decided by the DATA when there is data, by the name only
315
+ as a fallback.
316
+
317
+ ⛔ THE NAME LIST WAS WRONG AND ONLY THE API COULD SAY SO. `cost_per_unique_click`,
318
+ `cost_per_action_type`, `purchase_roas` and friends READ like money and are **arrays of
319
+ action-type objects** on the Insights edge:
320
+ [{"action_type":"outbound_click","value":"1.459854"}]
321
+ Typed DOUBLE from `_DBL`, the insert died with `Conversion Error: Could not convert string
322
+ '[{...}]' to DOUBLE` — after five entity tables had already been written, so the sync looked
323
+ like it worked and then blew up on the last table.
324
+ ⭐ Same principle the field CATALOG is built on, applied one layer down: **ask the response,
325
+ do not assert from a list.** A value that ever arrives as a list or dict is JSON TEXT, because
326
+ that is what `_cell` stores; anything else falls back to the measured name hints.
327
+ """
328
+ if rows:
329
+ seen, kinds = 0, set()
330
+ for r in rows:
331
+ v = r.get(name)
332
+ if v is None or v == "":
333
+ continue
334
+ kinds.add("json" if isinstance(v, (list, dict)) else
335
+ "bool" if isinstance(v, bool) else
336
+ "int" if isinstance(v, int) else
337
+ "float" if isinstance(v, float) else "str")
338
+ seen += 1
339
+ if seen >= 200:
340
+ break
341
+ if kinds:
342
+ if "json" in kinds or "str" in kinds:
343
+ return "VARCHAR" # a JSON blob, or a numeric STRING
344
+ if kinds <= {"int", "bool"}:
345
+ return "BIGINT" if name in _INT else ("VARCHAR" if "bool" in kinds else "BIGINT")
346
+ return "DOUBLE"
347
+ if name in _INT:
348
+ return "BIGINT"
349
+ if name in _DBL:
350
+ return "DOUBLE"
351
+ return "VARCHAR" # ids included — see the header
352
+
353
+
354
+ def _ensure(con, table, fields, rows=None):
355
+ """Create or widen the table, with every column typed from `rows` when they are available.
356
+
357
+ ⚠ AN EXISTING COLUMN WHOSE TYPE IS NOW WRONG IS REBUILT, NOT PATCHED. DuckDB cannot retype a
358
+ column in place, and this table is DERIVED data that can be re-pulled in minutes — so a type
359
+ disagreement drops and recreates rather than limping on with a column that refuses every
360
+ insert. The alternative is a mirror that is permanently unwritable for one bad guess.
361
+ """
362
+ want = {f: _coltype(f, rows) for f in fields}
363
+ have = {r[1]: str(r[2]).upper() for r in con.execute(f"PRAGMA table_info('{table}')").fetchall()}
364
+ if have:
365
+ clash = [f for f, ty in want.items() if f in have and have[f] != ty
366
+ and not (have[f].startswith("VARCHAR") and ty == "VARCHAR")]
367
+ if clash:
368
+ con.execute(f"DROP TABLE {table}")
369
+ have = {}
370
+ if not have:
371
+ cols = ", ".join(f"{f} {want[f]}" for f in fields)
372
+ con.execute(f"CREATE TABLE IF NOT EXISTS {table} (id VARCHAR PRIMARY KEY, {cols})"
373
+ if "id" not in fields else
374
+ f"CREATE TABLE IF NOT EXISTS {table} ({cols})")
375
+ return
376
+ for f in fields:
377
+ if f not in have:
378
+ con.execute(f"ALTER TABLE {table} ADD COLUMN {f} {want[f]}")
379
+
380
+
381
+ def _upsert(con, table, fields, rows):
382
+ """Delete-then-insert by id — the same idempotence `datastore._upsert` gives the Odoo half, so
383
+ a re-sync updates in place and can never append a second copy of the same object."""
384
+ if not rows:
385
+ return 0
386
+ cols = list(fields)
387
+ ids = [str(r.get("id") or "") for r in rows]
388
+ q = ",".join("?" for _ in ids)
389
+ con.execute(f"DELETE FROM {table} WHERE id IN ({q})", ids)
390
+ con.executemany(
391
+ f"INSERT INTO {table} ({', '.join(cols)}) VALUES ({', '.join('?' for _ in cols)})",
392
+ [[_cell(r.get(c)) for c in cols] for r in rows])
393
+ return len(rows)
394
+
395
+
396
+ def sync(tenant_key="royal-imports", tok=None, log=print, insights=True, insights_days=None):
397
+ """Pull every level into the tenant's mirror. -> a report dict; raises only on a bad token.
398
+
399
+ Idempotent: re-running updates rows in place. Safe to call at boot and on the resync loop,
400
+ exactly as `odoo_relational.refresh` is.
401
+ """
402
+ tok = tok or token()
403
+ # ⛔ THE WINDOW IS A PARAMETER, NOT AN ENVIRONMENT READ AT CALL TIME — and the difference cost a
404
+ # live deploy. `INSIGHTS_DAYS` binds at IMPORT (module scope), so `main._pull_meta`'s
405
+ # `os.environ.setdefault("META_INSIGHTS_DAYS", "7")` executed AFTER this module was already
406
+ # imported and changed nothing: every boot pulled **90 days**, not 7, which is the slow path
407
+ # that trips the per-ad-account rate limit and never finishes. The comment beside that call
408
+ # claimed a short window the code could not deliver.
409
+ # ⭐ The shape: **a knob read at import cannot be turned by a caller at runtime.** Passing it
410
+ # makes the caller's intent effective instead of aspirational; the env var stays the DEFAULT.
411
+ days = int(insights_days or INSIGHTS_DAYS)
412
+ report = {"tenant": tenant_key, "tables": {}, "problems": [], "accounts": []}
413
+ if not tok:
414
+ report["problems"].append(
415
+ "META_ADS_ACCESS_TOKEN is not set in the environment or in platform/.env, so nothing "
416
+ "was pulled. This is a missing CREDENTIAL, not a missing capability.")
417
+ return report
418
+
419
+ path = datastore.path_for(tenant_key)
420
+ if Path(datastore.DB_PATH) != Path(path):
421
+ datastore.use_path(path)
422
+ con = datastore.connect()
423
+ log(f" mirror: {Path(path).name}")
424
+
425
+ accts, _ = _walk("me/adaccounts", tok, log, fields="id,name", limit=PAGE)
426
+ report["accounts"] = [a.get("id") for a in accts]
427
+ if not accts:
428
+ report["problems"].append("the token reaches no ad accounts")
429
+ return report
430
+
431
+ # ── the five entity levels ────────────────────────────────────────────────────────────────
432
+ for table, (edge, fields, _parent) in SPECS.items():
433
+ rows, capped = [], False
434
+ for acct in accts:
435
+ aid = acct["id"]
436
+ if edge is None:
437
+ rows.append(_get(aid, tok, fields=",".join(fields)))
438
+ else:
439
+ got, hit = _walk(f"{aid}/{edge}", tok, log, fields=",".join(fields), limit=PAGE)
440
+ rows.extend(got)
441
+ capped = capped or hit
442
+ _ensure(con, table, fields, rows)
443
+ n = _upsert(con, table, fields, rows)
444
+ total = con.execute(f"SELECT count(*) FROM {table}").fetchone()[0]
445
+ report["tables"][table] = {"pulled": n, "in_mirror": total, "capped": capped}
446
+ log(f" {table:<20} pulled {n:>6} mirror total {total:>6}"
447
+ + (" !! PAGE CAP HIT" if capped else ""))
448
+ if capped:
449
+ report["problems"].append(f"{table}: stopped at MAX_PAGES={MAX_PAGES}")
450
+
451
+ # ── the daily Insights grain ──────────────────────────────────────────────────────────────
452
+ if insights:
453
+ rows, capped = [], False
454
+ for acct in accts:
455
+ for since, until in _slices(days, INSIGHTS_SLICE):
456
+ got, hit = _walk(f"{acct['id']}/insights", tok, log,
457
+ fields=",".join(INSIGHT_FIELDS), level=INSIGHTS_LEVEL,
458
+ time_increment="1", limit=INSIGHTS_PAGE,
459
+ time_range=json.dumps({"since": since, "until": until}))
460
+ rows.extend(got)
461
+ capped = capped or hit
462
+ # ⛔ Insights rows carry no id. The key must be STABLE across runs or every re-sync
463
+ # appends a second copy of the same day — so it is composed from the grain itself.
464
+ for r in rows:
465
+ r["id"] = ":".join(str(r.get(k) or "") for k in
466
+ (f"{INSIGHTS_LEVEL}_id", "date_start", "date_stop"))
467
+ fields = ["id"] + INSIGHT_FIELDS
468
+ _ensure(con, INSIGHTS_TABLE, fields, rows)
469
+ n = _upsert(con, INSIGHTS_TABLE, fields, rows)
470
+ total = con.execute(f"SELECT count(*) FROM {INSIGHTS_TABLE}").fetchone()[0]
471
+ report["tables"][INSIGHTS_TABLE] = {"pulled": n, "in_mirror": total, "capped": capped}
472
+ log(f" {INSIGHTS_TABLE:<20} pulled {n:>6} mirror total {total:>6}"
473
+ + (" !! PAGE CAP HIT" if capped else ""))
474
+
475
+ con.execute("INSERT OR REPLACE INTO _sync_state VALUES (?,?,?,?,?,?)",
476
+ ["meta", "done", 0, f"{days}d/{INSIGHTS_SLICE}d@{INSIGHTS_LEVEL}",
477
+ sum(t["in_mirror"] for t in report["tables"].values()),
478
+ time.strftime("%Y-%m-%d %H:%M:%S")])
479
+ return report
480
+
481
+
482
+ def status(tenant_key="royal-imports"):
483
+ """What is in the mirror right now, per table. Never fetches."""
484
+ path = datastore.path_for(tenant_key)
485
+ if Path(datastore.DB_PATH) != Path(path):
486
+ datastore.use_path(path)
487
+ out = {}
488
+ con = datastore.connect()
489
+ for table in list(SPECS) + [INSIGHTS_TABLE]:
490
+ try:
491
+ out[table] = con.execute(f"SELECT count(*) FROM {table}").fetchone()[0]
492
+ except Exception:
493
+ out[table] = None # table absent = never synced
494
+ return out
495
+
496
+
497
+ def main(argv=None):
498
+ ap = argparse.ArgumentParser(description="Pull Meta Ads into the tenant's DuckDB mirror.")
499
+ ap.add_argument("--sync", action="store_true")
500
+ ap.add_argument("--status", action="store_true")
501
+ ap.add_argument("--tenant", default="royal-imports")
502
+ ap.add_argument("--no-insights", action="store_true")
503
+ ap.add_argument("--insights-days", type=int, default=None,
504
+ help="override the Insights window for THIS run (default INSIGHTS_DAYS); the env var is the default, this is the caller's say")
505
+ a = ap.parse_args(argv)
506
+ if a.status:
507
+ for k, v in status(a.tenant).items():
508
+ print(f" {k:<20} {'(never synced)' if v is None else v}")
509
+ return 0
510
+ if not a.sync:
511
+ ap.print_help()
512
+ return 2
513
+ rep = sync(a.tenant, insights=not a.no_insights, insights_days=a.insights_days)
514
+ for p in rep["problems"]:
515
+ print(" PROBLEM:", p)
516
+ print(f" accounts: {len(rep['accounts'])} tables: {len(rep['tables'])}")
517
+ return 1 if rep["problems"] else 0
518
+
519
+
520
+ if __name__ == "__main__":
521
+ sys.exit(main())
platform/harness/tools.py CHANGED
@@ -65,22 +65,13 @@ def _err(msg):
65
  # ------------------------------------------------------------------ schema tools
66
 
67
  def list_topics():
68
- """The 'what data exists' tool.
69
-
70
- W33-T49 adds `grid` — the store key of the database this topic describes, when it describes
71
- one. It is what lets the agent connect "the thing the person is looking at" to "the topic I
72
- can query", which is the whole point of the entity topics: before this, the semantic layer
73
- could see `sale_order_line` and `account_move` and had never heard of `ut_odoo_customers`.
74
- """
75
  out = []
76
  for k, t in SEM.topics().items():
77
- row = {"topic": k, "label": t.get("label"), "entity": t.get("entity"),
78
- "grain": t.get("grain"),
79
- "dims": list((t.get("store") or {}).get("dims") or {}),
80
- "metrics": [m for m, d in SEM.metrics().items() if d["topic"] == k]}
81
- if t.get("grid"):
82
- row["grid"] = t["grid"]
83
- out.append(row)
84
  return out
85
 
86
 
@@ -92,30 +83,10 @@ def describe_topic(topic):
92
  mets = {k: {"label": m.get("label"), "description": m.get("description"),
93
  "format": m.get("format"), "ai_context": m.get("ai_context")}
94
  for k, m in SEM.metrics().items() if m["topic"] == topic}
95
- out = {"topic": topic, "label": t.get("label"), "scope": t.get("scope"),
96
- "grain": t.get("grain"), "ai_context": t.get("ai_context"),
97
- "dims": {k: v.get("label") for k, v in ((t.get("store") or {}).get("dims") or {}).items()},
98
- "metrics": mets}
99
- # ⭐⭐ W33-T49 — THE FIELD SEMANTICS, which this tool did not carry AT ALL.
100
- #
101
- # ⛔ THE GAP WAS NOT THE TOPIC FILES, IT WAS THIS RETURN. Before this, `describe_topic`
102
- # answered with scope, grain, dims, metrics and `ai_context` — and **no vocabulary of
103
- # COLUMNS**. So an agent told to describe a database could name what it MEASURES and never
104
- # what it HOLDS: the owner's item 13 asks for "ALL populated fields … including its semantic
105
- # layer", and adding eight topic files without this would have left `describe_topic` returning
106
- # the same field-free shape it always did, one screen away from the fields.
107
- #
108
- # ⚠ `kind` is the part worth reading, and it is the difference between a column an agent can
109
- # FILTER and one it cannot: `data` is stored on the row; `rollup` is computed from another
110
- # topic (`from_topic` names it, and that is where a windowed or reconciled number must come
111
- # from); `link` points at another database. An agent that treats a rollup as a stored column
112
- # will ask this topic for a number this topic does not hold.
113
- if t.get("fields"):
114
- out["fields"] = [dict(f) for f in t["fields"]]
115
- for k in ("grid", "subject", "entity"):
116
- if t.get(k):
117
- out[k] = t[k]
118
- return out
119
 
120
 
121
  # ------------------------------------------------------------------ query tools
 
65
  # ------------------------------------------------------------------ schema tools
66
 
67
  def list_topics():
68
+ """The 'what data exists' tool."""
 
 
 
 
 
 
69
  out = []
70
  for k, t in SEM.topics().items():
71
+ out.append({"topic": k, "label": t.get("label"), "entity": t.get("entity"),
72
+ "grain": t.get("grain"),
73
+ "dims": list((t.get("store") or {}).get("dims") or {}),
74
+ "metrics": [m for m, d in SEM.metrics().items() if d["topic"] == k]})
 
 
 
75
  return out
76
 
77
 
 
83
  mets = {k: {"label": m.get("label"), "description": m.get("description"),
84
  "format": m.get("format"), "ai_context": m.get("ai_context")}
85
  for k, m in SEM.metrics().items() if m["topic"] == topic}
86
+ return {"topic": topic, "label": t.get("label"), "scope": t.get("scope"),
87
+ "grain": t.get("grain"), "ai_context": t.get("ai_context"),
88
+ "dims": {k: v.get("label") for k, v in ((t.get("store") or {}).get("dims") or {}).items()},
89
+ "metrics": mets}
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
90
 
91
 
92
  # ------------------------------------------------------------------ query tools
platform/modules/product_data.py CHANGED
@@ -1,804 +1,797 @@
1
- """modules/product_data.py — the PRODUCT table's pool (wave 15 item 9/10, contract C-TOPIC).
2
-
3
- The second object on the table-page factory: same grid, same engine, same permission wall — the
4
- only difference is the field schema and the identity. `modules/customer_data.pool()` is the
5
- template and this deliberately mirrors its signature and its row shape.
6
-
7
- ⛔ THREE DECISIONS THIS FILE HAD TO MAKE, EACH RECORDED BECAUSE A LATER READER WILL WONDER.
8
-
9
- 1. **THE IDENTITY IS A SKU CODE, WHICH IS A STRING, AND THE GRID WANTS AN INTEGER `pid`.**
10
- Cohort membership, `allowed_pids`, the measure channel and `rows_from_pool` all key on an
11
- integer. So each row carries BOTH: `pid` (a stable CRC32 of the code, so the same SKU gets the
12
- same id on every pull and across processes — never an enumeration index, which would reshuffle
13
- whenever the catalogue changes) and `code`, the real business key, as a visible column.
14
- `_assert_no_pid_collision` fails the BUILD rather than the read: two SKUs sharing a pid would
15
- silently merge in every downstream set operation, and a loud build failure is the only version
16
- of that anyone would notice.
17
-
18
- 2. **THE SCOPE RULE — inventory columns are CONSOLIDATED and are therefore OMITTED for a
19
- BU-scoped caller.** `products.directory(t, team_id)` is brand-shaped; `inventory.sku_inventory`
20
- is explicitly NOT (its own docstring: "on-hand stock is one physical warehouse, not
21
- brand-tagged"). Joining them for a Fisch-only user would put BU-shaped revenue beside
22
- company-wide stock IN THE SAME ROW — the mixed-scope value defect wave 15's amendment 3 exists
23
- for, arriving through a different door. The honest options were "omit" or "label the columns
24
- company-wide"; omit is the fail-closed one, and a column that is absent asks a question,
25
- whereas a column that is silently company-wide answers one wrongly.
26
-
27
- 3. **WHAT IS NOT HERE, NAMED RATHER THAN QUIETLY MISSING.** R4 lists ~20 fields. Vendor and
28
- COUNTRY are not in Odoo at all — they live in the inventory WORKBOOK
29
- ([[odoo-vendor-country-origin]]) and need a loader this module deliberately does not invent.
30
- Margin % comes from `modules/pricing.table`, which is a heavier build (channel-rate cost
31
- allocation) and is left for the wave that needs it. `validate()` reconciles what SHIPS; it
32
- does not pretend to cover columns that are absent.
33
-
34
- 4. **THE POOL IS CATALOGUE-FIRST, WITH REVENUE LEFT-JOINED** (wave 29, owner item 22 / R12,
35
- 2026-08-11). It was `for r in products.directory(...)` — and `directory()`'s row set IS the
36
- union of two revenue read-groups, so **a SKU that never sold could not exist** and the grid
37
- showed **2,717** of **5,875** active products. Three things that will be re-derived otherwise:
38
-
39
- · ⛔ THE CAUSE IS A JOIN, NOT A LIMIT. No row cap exists on this path. Removing the date
40
- window from `directory()` would ALSO be wrong twice: it breaks four SKU-health metrics
41
- that legitimately want a sales window, and it lands at 3,327 (all-time-sold), because
42
- ~2,550 active SKUs have never sold in wholesale scope at all.
43
- · ⭐ REVENUE ON A NEVER-SOLD ROW IS **BLANK, NOT $0** (R12). A row that appears in the
44
- revenue universe carries measured numbers INCLUDING a real 0.0 — it sold last year and
45
- not this one, and that zero is a measurement. A row that appears in NO revenue read
46
- carries `None`, which the wire keeps (`aios_grid._round` passes None through) and the
47
- client renders as an empty cell. Blank is an admission; zero is a measurement.
48
- · ⚠ A BU-SCOPED CALLER NOW SEES THE WHOLE CATALOGUE, with ITS OWN revenue and blanks where
49
- that BU never sold. That is not decision 2's mixed-scope defect: the catalogue is the ROW
50
- UNIVERSE, not a company-wide VALUE sitting beside a BU-shaped one. `product.product`
51
- carries no team, so a catalogue cannot be BU-shaped at all — which is exactly why the two
52
- sources are JOINED rather than merged.
53
- """
54
- import json
55
- import zlib
56
- from pathlib import Path
57
-
58
- import core.odoo as O
59
- import core.periods as P
60
- import core.shared_overlay as shared_overlay
61
- import core.table_store as table_store
62
- import modules.products as products
63
-
64
- #: POOL-ROW KEYS that exist ONLY on a consolidated pull. See decision 2.
65
- #: `cover_gap_d` / `buy_now` (wave 17) join it because both are computed FROM `dos`, and a buy
66
- #: signal built on a stock number the caller cannot see would be a recommendation nobody could
67
- #: check ([[no-unverifiable-aggregates]]).
68
- #:
69
- #: ⚠ ROW KEYS, NOT FIELDS, and the distinction started mattering on 2026-08-03. Two readers:
70
- #: `verify_perm_scope` asserts the shape of a POOL ROW against this, and `routes_products`
71
- #: narrows the FIELD list with it. Since owner item 5 `buy_now` is a row key with no Field —
72
- #: computed as `validate()`'s oracle, projected onto no wire — so it belongs here for the first
73
- #: reader and is inert for the second. Removing it would stop the scope gate proving that a
74
- #: BU-scoped pull withholds it.
75
- #:
76
- #: ⭐⭐ OWNER RULING 2026-08-11 — THIS SET IS NOW EMPTY, AND THAT IS THE POINT. Verbatim:
77
- #: *"Just scope any inventory with Sales from Fisch, leave the rest."* Withholding the whole
78
- #: inventory block from a BU-scoped reader meant a Fisch salesperson could not see whether
79
- #: anything was IN STOCK, which is the first question they ask. The ruling splits the block by
80
- #: what a business unit can actually shape:
81
- #:
82
- #: * `on_hand` / `unit_cost` / `inv_value` — ONE physical warehouse, no Fisch shelf and no
83
- #: Royal shelf. Served UNSCOPED to everybody, identical in both pulls. ("leave the rest")
84
- #: * `qty_ltm` / `dos` / `stock_bucket` / `cover_gap_d` / `buy_now` — all SALES-derived, so a
85
- #: BU-scoped pull recomputes them from THAT unit's LTM units. ("scope any inventory with
86
- #: Sales from Fisch")
87
- #:
88
- #: This deliberately relaxes decision 2's "never a company-wide value beside a BU-shaped one":
89
- #: stock is not a value a BU can own, and a column that is absent for a Fisch reader asks a
90
- #: question they cannot answer anywhere else in the product. `verify_perm_scope` no longer
91
- #: asserts the columns are WITHHELD — it asserts the split above, which is a stronger claim and
92
- #: cannot pass on an empty scoped payload (the shape the old rule and an outage share).
93
- CONSOLIDATED_ONLY = ()
94
-
95
- #: ⛔ POOL-ROW KEYS THAT DELIBERATELY HAVE NO FIELD — they must never reach a browser.
96
- #:
97
- #: `rows_from_pool` projects strictly through the field contract, so "no Field" already means
98
- #: "no cell on the wire". This tuple makes that a CHECKED fact rather than a consequence nobody
99
- #: is watching: `verify_perm_scope` asserts every member is absent from BOTH product contracts,
100
- #: and that every OTHER `CONSOLIDATED_ONLY` key is present in the consolidated one.
101
- #:
102
- #: Written because the two tuples silently diverged the moment owner item 5 retired `buy_now`'s
103
- #: Field while keeping its computation, and the gate — which was asserting a FIELD property from
104
- #: a ROW-key list — went red with no way to tell a deliberate divergence from a dropped column.
105
- UNSHIPPED_ROW_KEYS = ("buy_now",)
106
-
107
- #: Wave 17 (owner item 13, ruling R3) — the SUPPLIER MASTER, from the curated mastersheet map
108
- #: `procurement_suppliers.json` (3,963 SKUs; supplier on 3,624, lead time on 3,563). NOT Odoo:
109
- #: the owner confirmed this data came from the Inventory System Mastersheet, which is why
110
- #: `modules/product_data`'s header said Vendor/Country were "NOT HERE, named rather than
111
- #: quietly missing" and needed "a loader this module deliberately does not invent". This is
112
- #: that loader.
113
- #:
114
- #: ⛔ THESE ARE CONTRACT COLUMNS, NOT USER-CREATED FIELDS, AND THE REASON IS A PERMISSION FACT.
115
- #: A user-created field and its values live in the PER-USER strata (`table_store.workspace`
116
- #: reads `store.get(key)[username]`); only VIEWS are shared. So a shared "Buy list" view that
117
- #: filtered on a user-created column would, for every OTHER account, name a column that does not
118
- #: exist — and an unknown column is an INACTIVE condition in the tri-state engine, which IGNORES
119
- #: it and therefore WIDENS. The buy list would silently show the whole catalogue to everyone but
120
- #: its author. Contract columns are identical for every reader, so the view means one thing.
121
- #:
122
- #: ⭐⭐ 2026-08-12 (W30-T36) — THE OWNER'S ASK IS NOW DELIVERED, AND THE PARAGRAPH ABOVE IS WHY IT
123
- #: TOOK THREE WAVES. Owner: *"turn this Excel sheet into a User created Field that we can edit."*
124
- #: It was parked because a per-user column silently WIDENS a shared view — not because editing was
125
- #: hard. Wave 29's `core/shared_overlay.py` (whose header quotes this very comment) is the stratum
126
- #: that removes the objection: **one value per (row, column) for the whole tenant**, so the column
127
- #: still means ONE thing to every reader and a shared view still filters honestly.
128
- #: ⇒ The four columns below are now `source: "overlay"` + `shared: true` in the canonical
129
- #: contract, their values live in `<TABLE_KEY>__shared`, and the master map is what SEEDS an
130
- #: unedited cell rather than what freezes it. See `SHARED_KEYS` and `_ProductTableStore`.
131
- _SUPPLIER_MAP_PATH = Path(__file__).resolve().parent.parent / "procurement_suppliers.json"
132
- _SUPPLIER_CACHE = {}
133
-
134
-
135
- def supplier_master():
136
- """`{code: {supplier, lead_days, origin_country, first_cost}}`, read once per process.
137
-
138
- Degrades to `{}` when the file is unreadable, matching `_inventory_by_code`: a product table
139
- that will not render because a master map is missing is a worse failure than one with blank
140
- supplier columns.
141
- """
142
- if _SUPPLIER_CACHE:
143
- return _SUPPLIER_CACHE
144
- try:
145
- raw = json.loads(_SUPPLIER_MAP_PATH.read_text(encoding="utf-8"))
146
- except Exception:
147
- return {}
148
- for code, meta in (raw or {}).items():
149
- if not isinstance(meta, dict):
150
- continue
151
- lead = meta.get("lead")
152
- _SUPPLIER_CACHE[str(code)] = {
153
- "supplier": (meta.get("vendor") or "") or None,
154
- "lead_days": int(lead) if isinstance(lead, (int, float)) else None,
155
- "origin_country": (meta.get("country") or "") or None,
156
- "first_cost": meta.get("first_cost"),
157
- }
158
- return _SUPPLIER_CACHE
159
-
160
- #: Wave 16 C-TOPIC — the PRODUCT table's OWN workspace bucket. ⛔ Never the customer one:
161
- #: product pids are CRC32 hashes of SKU codes and customer pids are Odoo partner ids, so in a
162
- #: SHARED overlay bucket a hash collision would silently write a product note onto somebody's
163
- #: customer (or the reverse). Separate store keys make that structurally impossible, which is
164
- #: the whole reason the table-page factory exists ("a new table object gets its own
165
- #: table_store.make('<its>_table_workspace')").
166
- TABLE_KEY = 'product_table_workspace'
167
-
168
- _GRID_FIELDS_PATH = Path(__file__).resolve().parent.parent / 'aios_grid_fields.json'
169
- _SHARED_KEYS = None
170
-
171
-
172
- def SHARED_KEYS():
173
- """The product columns whose values are TENANT-WIDE — derived from the canonical contract's
174
- own `shared: true`, never typed out a second time.
175
-
176
- ⛔ IT DELIBERATELY DOES NOT SWALLOW A READ FAILURE. Degrading to `()` would send a shared
177
- write into the per-user stratum with nothing going wrong anywhere — the widening defect
178
- reappearing silently, which is the one outcome this whole mechanism exists to prevent. If the
179
- canonical contract is unreadable the product grid cannot render at all (`pd_fields` parses the
180
- same file with no guard), so a raise here costs nothing that was still working.
181
- """
182
- global _SHARED_KEYS
183
- if _SHARED_KEYS is None:
184
- doc = json.loads(_GRID_FIELDS_PATH.read_text(encoding='utf-8'))
185
- _SHARED_KEYS = tuple(f['key'] for f in (doc.get('product_data') or {}).get('fields') or []
186
- if f.get('shared'))
187
- return _SHARED_KEYS
188
-
189
-
190
- class _ProductTableStore(table_store.TableStore):
191
- """The product workspace, with the SHARED columns routed to the tenant-wide stratum.
192
-
193
- ⭐⭐ THIS SUBCLASS IS THE WHOLE OF W30-T36's WRITE PATH, AND THE REASON IT LIVES HERE RATHER
194
- THAN AT A ROUTE IS THAT **THE BROWSER NEVER CALLS `PATCH /products/{pid}`** — measured, zero
195
- call sites in `aios-web/web/src`. A cell edit travels `POST /grid/events` → `grid_events.
196
- handle_one` → `_tops(ctx).patch_overlay(...)`, and `_tops` returns `ctx.table`, which
197
- `routes_grid._ctx` sets to `TABLE_OPS` for the product scope. So this object IS the seam both
198
- doors pass through; intercepting at either route would have left the other one writing a
199
- per-user value that only its author could see.
200
-
201
- ⚠ `st=self.st`, NEVER the module default. The shared stratum must resolve to the SAME store
202
- handle as the per-user one it sits beside — `_tops`' own comment explains that a split, where
203
- one side is tenant-scoped and the other is not, is worse than a stated residency error because
204
- a user's value would vanish the moment they saved it. Reading `self.st` means both strata move
205
- together the day that singleton gains a tenant handle.
206
- """
207
-
208
- def patch_overlay(self, username, pid, updates):
209
- clean = dict(updates or {})
210
- if not clean:
211
- return
212
- keys = set(SHARED_KEYS())
213
- shared = {k: v for k, v in clean.items() if k in keys}
214
- personal = {k: v for k, v in clean.items() if k not in keys}
215
- if shared:
216
- shared_overlay.put_cells(TABLE_KEY, pid, shared, st=self.st)
217
- if personal:
218
- super().patch_overlay(username, pid, personal)
219
-
220
-
221
- TABLE_OPS = _ProductTableStore(TABLE_KEY)
222
-
223
-
224
- def shared_cells(pids, st=None):
225
- """`{"<pid>": {key: value}}` for the SHARED columns of the rows named by `pids`.
226
-
227
- ⚠ `pids` is required and positional all the way down — `shared_overlay.cells` refuses to serve
228
- "everything" by design, and the caller here always holds an already-scoped pool.
229
- """
230
- return shared_overlay.cells(TABLE_KEY, pids, st=st if st is not None else TABLE_OPS.st)
231
-
232
-
233
- def sku_pid(code):
234
- """A stable integer id for a SKU code. CRC32, masked to 31 bits so it is always positive and
235
- always JSON-safe. Stable across processes and pulls, which an enumeration index is not."""
236
- return zlib.crc32(str(code).encode("utf-8")) & 0x7FFFFFFF
237
-
238
-
239
- def _assert_no_pid_collision(rows):
240
- """Two SKUs sharing a pid would MERGE in every set operation downstream — cohort membership,
241
- allowed_pids, the measure channel — and nothing would report it. Fail the build instead."""
242
- seen = {}
243
- for r in rows:
244
- prior = seen.get(r["pid"])
245
- if prior is not None and prior != r["code"]:
246
- raise ValueError(
247
- f"product_data: pid collision — {prior!r} and {r['code']!r} both hash to "
248
- f"{r['pid']}. Downstream set operations would merge them silently; widen the id "
249
- f"before shipping this catalogue.")
250
- seen[r["pid"]] = r["code"]
251
-
252
-
253
- def _inventory_by_code(t):
254
- """`{code: {...}}` from the inventory module, or `{}` if it cannot be read.
255
-
256
- Degrades to empty rather than raising, matching `customer_data._pool_build`'s treatment of
257
- its own slow families: a product table that will not render because inventory is momentarily
258
- unreachable is a worse failure than one with blank stock columns.
259
- """
260
- try:
261
- import modules.inventory as inventory
262
- return inventory.sku_inventory(t=t) or {}
263
- except Exception:
264
- return {}
265
-
266
-
267
- def _bu_ltm_share(t, team_id):
268
- """`{code: 0.0..1.0}` — this unit's SHARE of the SKU's last-twelve-months units.
269
-
270
- ⭐ OWNER 2026-08-11: *"scope any inventory with Sales from Fisch"*. The velocity half of the
271
- inventory block has to be re-shaped by a BU fact, and this is that fact.
272
-
273
- ⛔ A SHARE, NOT THE UNIT COUNT ITSELF — AND THE REASON IS A MEASURED IMPOSSIBILITY. My first
274
- version returned `products._sku_rev(...)['qty']` and used it directly as the scoped `qty_ltm`,
275
- leaving the consolidated column on `inventory.sku_inventory`'s own figure. Two readers of one
276
- question ([[one-question-two-normalizers]]): they disagree, and on 5 SKUs of 5,871 the live
277
- check found **Fisch's LTM units EXCEEDING the company's** — a subset larger than its superset,
278
- which no reader could explain and no reconciliation could survive.
279
-
280
- Both halves of the ratio come from the SAME read here, so the share is in [0, 1] by
281
- construction and the scoped figure can never exceed the consolidated one. It also leaves the
282
- consolidated column exactly as it was — the Inventory page and the Product grid still agree
283
- about company-wide units, which is what "leave the rest" asked for.
284
-
285
- `all_qty == 0` implies `bu_qty == 0` (same reader), so a share of 0 is the honest answer for
286
- "this unit never sold it": `_bucket` turns that into 'No recent sales', not zero cover.
287
-
288
- Degrades to `{}` on any failure, matching `_inventory_by_code` — and a missing code then takes
289
- the `_DEFAULT_SHARE` below rather than a silent 0.
290
- """
291
- try:
292
- f, to = P.ltm(t)
293
- allq = {code: (r.get('qty') or 0.0)
294
- for code, r in (products._sku_rev(f, to, None) or {}).items()}
295
- buq = {code: (r.get('qty') or 0.0)
296
- for code, r in (products._sku_rev(f, to, team_id) or {}).items()}
297
- except Exception:
298
- return {}
299
- out = {}
300
- for code, total in allq.items():
301
- out[code] = min(1.0, max(0.0, (buq.get(code, 0.0) / total))) if total > 0 else 0.0
302
- return out
303
-
304
-
305
- #: What a SKU absent from the LTM sales read is worth to a business unit. ZERO — it did not sell
306
- #: in anybody's book over the window, so no unit can claim its velocity. Named rather than
307
- #: inlined so the choice is visible: the alternative (1.0, "assume it is all ours") would print
308
- #: company-wide cover on a BU grid, which is the mixed-scope defect this whole rule avoids.
309
- _DEFAULT_SHARE = 0.0
310
-
311
-
312
- def _rescope_inventory(e, share):
313
- """`(qty_ltm, dos, bucket)` recomputed for ONE business unit's sales rate.
314
-
315
- The formulas are `inventory.sku_inventory`'s, applied to a BU-shaped numerator — NOT a second
316
- idea of what days-of-supply means. `_bucket` is imported from there for the same reason: two
317
- copies of a threshold table is how the Product grid and the Inventory page start disagreeing
318
- about which SKUs are dead.
319
-
320
- ⚠ `on_hand` is whatever the warehouse holds, unscoped — so a Fisch reader's `dos` answers
321
- "how long does ALL our stock last at Fisch's rate", which is the question a Fisch salesperson
322
- actually has. It is deliberately NOT a pro-rated share of the shelf: there is no such shelf,
323
- and inventing one would put a number on screen that no Odoo query could reproduce.
324
- """
325
- on_hand = e.get("on_hand")
326
- if on_hand is None:
327
- return None, None, None # no inventory row for this SKU: blank, never zero
328
- qty = float(e.get("qty_ltm") or 0.0) * float(share or 0.0)
329
- daily = qty / 365.0
330
- if daily > 0:
331
- dos_raw = on_hand / daily
332
- else:
333
- dos_raw = float('inf') if on_hand > 0 else 0.0
334
- try:
335
- import modules.inventory as inventory
336
- bucket = inventory._bucket(dos_raw, on_hand, qty)
337
- except Exception:
338
- bucket = None
339
- return qty, (None if dos_raw == float('inf') else round(float(dos_raw), 0)), bucket
340
-
341
-
342
- def _catalogue_by_code():
343
- """`{code: {'product', 'category'}}` — the CATALOGUE universe this pool is built from.
344
-
345
- ⛔ UNLIKE `_inventory_by_code`, THIS ONE RAISES, and the asymmetry is the whole point.
346
- Inventory degrades to `{}` because a product table with blank stock columns beats one that
347
- will not render. The CATALOGUE is not a column — it is the ROW SET. A catalogue read that
348
- failed quietly would drop the grid straight back to the sold-only 2,717 with every gate
349
- green and nothing on screen saying so, which is the defect this seam exists to end
350
- ([[gate-can-report-green-on-nothing]]). `routes_products._pool_for` already turns the raise
351
- into a 503 that names the cause, so the loud failure has somewhere honest to land.
352
-
353
- It exists as a `pd`-level function rather than an inline `products.catalogue()` call for the
354
- same reason `_inventory_by_code` does: it is the seam `verify_perm_scope`'s section H stubs
355
- to build a pool without Odoo.
356
- """
357
- return products.catalogue()
358
-
359
-
360
- def _pricelist_by_code():
361
- """`({code: {price_*: price}}, report)` from `products.pricelist_by_code`, or `({}, …)`.
362
-
363
- A SEAM for the same two reasons `_inventory_by_code` is one: it degrades rather than raises
364
- (these are columns, not the row set), and `verify_perm_scope`'s section H stubs it to build a
365
- pool without Odoo. ⛔ Unstubbed there, section H would reach live Odoo through the back door
366
- and the whole file would stop being runnable offline.
367
- """
368
- try:
369
- return products.pricelist_by_code()
370
- except Exception as e:
371
- return {}, {"error": f"{type(e).__name__}: {str(e)[:200]}"}
372
-
373
-
374
- def pool(team_id=None, t=None):
375
- """One row per ACTIVE product — the PRODUCT analogue of `customer_data.pool`.
376
-
377
- CATALOGUE-FIRST, REVENUE LEFT-JOINED (R12 — see decision 4 in the module header). The row set
378
- is `products.catalogue()`; `products.directory()` supplies the revenue columns for the SKUs
379
- that sold in its window and contributes NO rows of its own.
380
-
381
- `team_id` shapes the revenue columns exactly as it does for customers (`products.directory`
382
- passes it into `_sku_rev`), which is why `core.perm_scope.derive_pool_scope` must keep
383
- driving it rather than a post-filter deciding the BU. It does NOT shape the row set: a
384
- catalogue has no team.
385
- """
386
- t = t or P.today()
387
- consolidated = team_id is None
388
- # ⭐ OWNER 2026-08-11: read inventory on EVERY pull, not just a consolidated one. The stock
389
- # itself is company-wide; only the sales-derived half is re-scoped, by `_rescope_inventory`.
390
- inv = _inventory_by_code(t)
391
- bu_share = {} if consolidated else _bu_ltm_share(t, team_id)
392
- sup = supplier_master()
393
- prices, _price_report = _pricelist_by_code()
394
- cat = _catalogue_by_code()
395
- # The LEFT side of the join, indexed by the same code key. ⛔ A code here that the catalogue
396
- # does not carry belongs to a product no ACTIVE record claims — archived, and R12 keeps those
397
- # out. `validate()` asserts that of Odoo rather than assuming it, and reports the revenue
398
- # that therefore sits outside the grid (MEASURED 2026-08-11: 3 codes, $0.00 YTD / $294.50 LY).
399
- rev = {r["code"]: r for r in products.directory(t=t, team_id=team_id)}
400
-
401
- rows = []
402
- for code, meta in cat.items():
403
- r = rev.get(code)
404
- row = {
405
- "pid": sku_pid(code),
406
- "code": code,
407
- # ⭐ ONE source for the name and the category, the CATALOGUE — not the sale line's
408
- # m2o. For a re-SKUed code the line's name can be the ARCHIVED record's; the active
409
- # record's `display_name` is the current truth, and it is the same string for every
410
- # SKU that is not re-SKUed. `directory()` derives the category identically.
411
- "product": meta.get("product") or code,
412
- "category": meta.get("category") or "(uncategorized)",
413
- # ⭐⭐ W33-T43 (R2 / amendment A2) — ODOO'S OWN PRODUCT ID, beside the hashed `pid`.
414
- # R2 retires `ut_odoo_products` onto this key and keeps every data column it had; this
415
- # is that column. ⛔ NOT DERIVABLE DOWNSTREAM: `pid` is `crc32(default_code)` here,
416
- # unlike a customer row whose `pid` IS the partner id, so `aios_grid.py` cannot recover
417
- # it and it has to be carried from `products.catalogue()`.
418
- # ⚠ `None`, never 0, when the catalogue somehow has no id — 0 is a real Odoo id.
419
- "product_id": meta.get("id"),
420
- # ⭐ R12 — BLANK, NEVER $0, on a SKU no revenue read ever saw. A row that IS in `rev`
421
- # keeps its measured numbers including a real 0.0 (it sold last year, not this one).
422
- "rev_ytd": r.get("rev_ytd", 0.0) if r else None,
423
- "rev_ly": r.get("rev_ly", 0.0) if r else None,
424
- "yoy_pct": r.get("yoy_pct") if r else None,
425
- "qty_ytd": r.get("qty_ytd", 0.0) if r else None,
426
- "orders_ytd": r.get("orders_ytd", 0) if r else None,
427
- }
428
- # ⭐ WAVE 30 W30-T34 — the PRICELIST stratum, one column per declared list. Catalogue
429
- # data like the supplier block, so it rides every pull, scoped or not: a price book is
430
- # not a thing a business unit owns a slice of, and a Fisch reader who cannot see the
431
- # Fisch price is exactly who this column is for.
432
- #
433
- # ⛔ `None`, NEVER 0 AND NEVER A FALLBACK, for the 1,883 SKUs no list prices. That is
434
- # W29-T52's own negative control: the single `3_global` rule computes over `list_price`,
435
- # which is 1.00 on 5,817 of 5,871 products, so the fallback is not a cheaper answer —
436
- # it is a wrong one wearing a currency sign.
437
- p = prices.get(code) or {}
438
- for _col, _name in products.PRICELIST_COLUMNS:
439
- row[_col] = p.get(_col)
440
- # Wave 17 R3 — the supplier master, on every pull (it is catalogue data, not stock).
441
- s = sup.get(code) or {}
442
- row.update({
443
- "supplier": s.get("supplier") or "(none)",
444
- "lead_days": s.get("lead_days"),
445
- "origin_country": s.get("origin_country") or "(none)",
446
- "first_cost": s.get("first_cost"),
447
- })
448
- e = inv.get(code) or {}
449
- if consolidated:
450
- qty_ltm, dos, bucket = e.get("qty_ltm"), e.get("dos"), e.get("bucket")
451
- else:
452
- qty_ltm, dos, bucket = _rescope_inventory(e, bu_share.get(code, _DEFAULT_SHARE))
453
- row.update({
454
- # UNSCOPED on purpose (owner 2026-08-11): one warehouse, no per-brand shelf.
455
- "on_hand": e.get("on_hand"),
456
- "unit_cost": e.get("unit_cost"),
457
- "inv_value": e.get("inv_value"),
458
- # SCOPED: these three are functions of how fast THIS unit sells the SKU.
459
- "qty_ltm": qty_ltm,
460
- "dos": dos,
461
- "stock_bucket": bucket,
462
- })
463
- # THE BUY TRIGGER (R3), ported from `modules/procurement`'s lead-time-cover rule:
464
- # buy when the shelf runs out before a reorder could land.
465
- #
466
- # ⚠ BLANK, NOT "OK", WHEN EITHER INPUT IS MISSING. 339 SKUs have no supplier and 400
467
- # no lead time; `dos` is null for anything that never sells through. "We don't know"
468
- # and "you're fine" are different sentences, and only one of them is safe to print
469
- # beside a purchasing decision.
470
- #
471
- # ⛔ `buy_now` SHIPS NOWHERE ANY MORE, and is computed anyway. OWNER 2026-08-03: it
472
- # is not a preset field — it is a formula over the two columns beside it, and the
473
- # platform has a formula field type for exactly that. Its Field is gone from
474
- # `aios_grid_fields.json`, and `rows_from_pool` projects rows STRICTLY through that
475
- # contract, so no Field means no cell on the wire. Nothing renders this key.
476
- #
477
- # It stays computed because deleting it would delete the PROOF. `validate()` below
478
- # reconciles the formula's predicate against this one, row for row, which is the only
479
- # thing that makes "the same exact figures" a claim rather than an assertion — and
480
- # this repo has the scar already (wave 17: archiving `ar` "would have DELETED the
481
- # proof"). Two comparisons per SKU is what that costs.
482
- #
483
- # ⭐ READS THE SCOPED `dos`, NOT `e['dos']`. On a Fisch pull the cover gap must answer
484
- # "does the shelf outlast a reorder AT FISCH'S RATE" — reading the consolidated figure
485
- # here would print a buy signal computed from both units' velocity beside a days-of-supply
486
- # computed from one, and the two columns would disagree on the same row.
487
- lead = s.get("lead_days")
488
- if isinstance(dos, (int, float)) and isinstance(lead, (int, float)) and lead > 0:
489
- row["cover_gap_d"] = int(round(dos - lead))
490
- row["buy_now"] = "Buy now" if dos < lead else "OK"
491
- else:
492
- row["cover_gap_d"] = None
493
- row["buy_now"] = None
494
- rows.append(row)
495
-
496
- # `or 0.0` reads a BLANK as zero FOR SORTING ONLY — never for the cell. A never-sold SKU
497
- # settles among the zero-revenue ones at the bottom, which is where it belongs; the stored
498
- # value stays None so nothing downstream can mistake "we never saw a sale" for "we measured
499
- # nothing sold".
500
- rows.sort(key=lambda r: -(r["rev_ytd"] or 0.0))
501
- _assert_no_pid_collision(rows)
502
- return rows
503
-
504
-
505
- #: ⭐ THE BUY SIGNAL, AS THE FORMULA FIELD EVALUATES IT (owner, 2026-08-03).
506
- #:
507
- #: The owner's ruling was that "Buy signal" is not a preset field — it is a formula over two
508
- #: columns the product table already carries, and the platform has a formula field type for it.
509
- #: The formula, verbatim, is what `_seed_wave17.BUY_SIGNAL_FORMULA` creates and what the JSON
510
- #: contract's `_product_removed_buy_now` note records:
511
- #:
512
- #: IF({lead_days} > 0, IF({dos} < {lead_days}, "Buy now", "OK"), "")
513
- #:
514
- #: This function is a PORT of how `customer-grid/formulaEngine.ts` evaluates that tree, not a
515
- #: restatement of the business rule — that is the whole point, because the two could drift and
516
- #: `validate()` is where the drift must show. Three engine behaviours it reproduces exactly:
517
- #:
518
- #: · a `ref` to a missing/non-numeric cell is None (`case "ref"` returns null for anything
519
- #: that is not a finite number or a string);
520
- #: · `cmp` returns BLANK unless BOTH sides read as numbers — it never coerces a blank to 0,
521
- #: which is the difference between this and a filter engine's `toNum(null) === 0`;
522
- #: · `IF` with a non-boolean condition returns BLANK ("no truthiness"), so a blank comparison
523
- #: propagates out as a blank cell rather than taking the false branch.
524
- BUY_SIGNAL_FORMULA = 'IF({lead_days} > 0, IF({dos} < {lead_days}, "Buy now", "OK"), "")'
525
-
526
-
527
- def _buy_signal_formula(row):
528
- """Evaluate `BUY_SIGNAL_FORMULA` over one pool row -> 'Buy now' | 'OK' | '' (blank)."""
529
- def num(v):
530
- # The engine's `ref` + `asNumber`: booleans are not values here, and a non-finite
531
- # number is null. `isinstance(True, int)` is True in Python, so bool is excluded first.
532
- if isinstance(v, bool) or not isinstance(v, (int, float)):
533
- return None
534
- return v if v == v and v not in (float('inf'), float('-inf')) else None
535
-
536
- lead, dos = num(row.get("lead_days")), num(row.get("dos"))
537
- if lead is None: # `{lead_days} > 0` is blank -> IF(blank, …) is blank
538
- return ""
539
- if not lead > 0:
540
- return "" # the formula's own else-branch
541
- if dos is None: # `{dos} < {lead_days}` is blank -> IF(blank, …) is blank
542
- return ""
543
- return "Buy now" if dos < lead else "OK"
544
-
545
-
546
- def validate(team_id=None, t=None):
547
- """Reconcile what SHIPS to an independent aggregate — the platform's own rule that a number
548
- which does not tie to Odoo does not ship.
549
-
550
- ⛔⛔ THE POPULATION LEG IS NEW, AND IT IS HERE BECAUSE THIS FUNCTION USED TO BE SELF-SEALING
551
- (wave 29, contract C8). It reconciled Σ per-SKU YTD revenue against `products._sku_rev` — the
552
- same function `pool()` built its rows from. BOTH SIDES CAME FROM THE JOIN, so the oracle could
553
- not see a missing row, and a grid holding 2,717 of 5,875 active products passed for a wave.
554
- No gate anywhere asserted a pool row COUNT against an independent Odoo `search_count` either.
555
- That is the leg below, and it is the control that would have caught it
556
- ([[no-unverifiable-aggregates]]).
557
-
558
- ⚠ The revenue leg was NOT deleted with it — it answers a different question ("did the join
559
- lose money?") and is still the right shape for that one. What changed is that it now has to
560
- account for revenue belonging to codes NO ACTIVE PRODUCT CARRIES, because R12 keeps archived
561
- products out of the grid; the total is decomposed rather than compared loosely, so a growing
562
- "outside the grid" figure shows up as a number rather than as slack in a tolerance.
563
-
564
- Still deliberately NOT covered, and said rather than implied: the inventory columns are a
565
- different module's reconciliation, and they are absent on a scoped pull anyway.
566
- """
567
- t = t or P.today()
568
- rows = pool(team_id=team_id, t=t)
569
-
570
- # ⭐⭐ THE POPULATION, AGAINST AN ORACLE THAT CANNOT SEE OUR JOIN. A bare Odoo count of active
571
- # products, asked fresh — not a `len()` over anything this module or `products.catalogue()`
572
- # built. NC: drop one row from `pool()` and this goes red; that is the whole point of it.
573
- n_active = products.catalogue_count()
574
- checks = [{
575
- "check": "the pool holds one row per ACTIVE product (R12) — row count == an INDEPENDENT "
576
- "Odoo search_count('product.product', active=True)",
577
- "ours": len(rows), "theirs": n_active, "ok": len(rows) == n_active,
578
- }]
579
-
580
- yf, yt = P.ytd(t)
581
- sku_rev = products._sku_rev(yf, yt, team_id) or {}
582
- in_pool = {r["code"] for r in rows}
583
- outside = {c: v for c, v in sku_rev.items() if c not in in_pool}
584
- ours = round(sum(r["rev_ytd"] or 0.0 for r in rows), 2)
585
- theirs = round(sum(v.get("rev", 0.0) for v in sku_rev.values()), 2)
586
- dropped = round(sum(v.get("rev", 0.0) for v in outside.values()), 2)
587
- checks.append({
588
- "check": "Σ per-SKU YTD revenue in the grid + Σ YTD revenue of codes no ACTIVE product "
589
- "carries == modules.products._sku_rev over the same window",
590
- "ours": round(ours + dropped, 2), "theirs": theirs,
591
- "ok": abs((ours + dropped) - theirs) < 0.01,
592
- "detail": {"in_the_grid": ours, "outside_the_grid": dropped,
593
- "codes_outside": len(outside)},
594
- })
595
- # ⛔ AND THE EXCLUSION IS ASSERTED, NOT ASSUMED. "Everything I dropped was archived" is
596
- # trivially true when checked against the dict that did the dropping — that is the
597
- # self-sealing shape all over again. So it is asked of ODOO. A code here that a live ACTIVE
598
- # product carries means `products.catalogue()` missed a row that has revenue, which is the
599
- # 2,717 defect returning in a smaller costume.
600
- # ⚠ `pid:N` keys are not `default_code`s and are skipped: an UNCODED active product is in the
601
- # catalogue under its own `pid:N` key and therefore cannot be in `outside` at all.
602
- coded_outside = sorted(c for c in outside if not str(c).startswith("pid:"))
603
- still_active = (O.get_odoo().search_count(
604
- 'product.product', [('active', '=', True), ('default_code', 'in', coded_outside)])
605
- if coded_outside else 0)
606
- checks.append({
607
- "check": "every SKU with revenue but NO grid row is genuinely ARCHIVED (R12: archived "
608
- "stay out) — asked of Odoo, never of the catalogue that did the dropping",
609
- "ours": still_active, "theirs": 0, "ok": still_active == 0,
610
- "detail": {"codes": coded_outside[:10], "n_codes": len(outside)},
611
- })
612
- # ── ⭐⭐ WAVE 30 W30-T34 — THE PRICELIST COLUMNS, AGAINST ORACLES THAT CANNOT SEE OUR JOIN ──
613
- #
614
- # Three legs, because the column can fail in three different ways and only one of them is a
615
- # count. `products.pricelist_by_code` builds a `{code: {col: price}}` map by expanding each
616
- # rule over the products it names, IN PYTHON — that expansion is the fragile part, so every
617
- # leg below re-asks ODOO instead of re-reading the map.
618
- price_report = _pricelist_by_code()[1]
619
- _pl_rows = O.search_read('product.pricelist', [], ['id', 'name'])
620
- _pl_id = {}
621
- for _p in _pl_rows:
622
- _pl_id.setdefault(str(_p.get('name') or '').strip(), _p['id'])
623
- _today = P.today().isoformat()
624
-
625
- _targets_memo = {}
626
-
627
- def _rule_targets(list_name):
628
- """(variant_ids, template_ids) a pricelist prices today — read FRESH from Odoo.
629
-
630
- Memoised for the LIFE OF THIS CALL only: legs 1 and 2 both need all three lists, and
631
- without this `validate()` makes six identical round trips instead of three (measured:
632
- ~48s of the run). Deliberately NOT an `lru_cache` — an oracle that survives the process
633
- is an oracle reading yesterday's Odoo.
634
- """
635
- if list_name in _targets_memo:
636
- return _targets_memo[list_name]
637
- var, tmpl = set(), set()
638
- plid = _pl_id.get(list_name)
639
- if plid is None:
640
- return var, tmpl
641
- for r in O.search_read(
642
- 'product.pricelist.item',
643
- [('pricelist_id', '=', plid), ('compute_price', '=', 'fixed'),
644
- ('applied_on', 'in', ['0_product_variant', '1_product'])],
645
- ['product_id', 'product_tmpl_id', 'applied_on', 'fixed_price',
646
- 'date_start', 'date_end']):
647
- ds, de = str(r.get('date_start') or '')[:10], str(r.get('date_end') or '')[:10]
648
- if (ds and ds > _today) or (de and de < _today):
649
- continue
650
- fp = r.get('fixed_price')
651
- if not isinstance(fp, (int, float)) or fp <= 0:
652
- continue
653
- if r.get('applied_on') == '0_product_variant' and r.get('product_id'):
654
- var.add(O.m2o_id(r['product_id']))
655
- elif r.get('product_tmpl_id'):
656
- tmpl.add(O.m2o_id(r['product_tmpl_id']))
657
- _targets_memo[list_name] = (var, tmpl)
658
- return var, tmpl
659
-
660
- # LEG 1 — COVERAGE PER LIST. Ours: cells we filled. Theirs: a bare Odoo `search_count` of
661
- # ACTIVE products a fresh read of that list's rules reaches. The rule set is shared (it IS
662
- # the data) but the EXPANSION is not, and the expansion is what breaks.
663
- for _col, _name in products.PRICELIST_COLUMNS:
664
- _var, _tmpl = _rule_targets(_name)
665
- _dom = [('active', '=', True), '|', ('id', 'in', sorted(_var)),
666
- ('product_tmpl_id', 'in', sorted(_tmpl))]
667
- theirs = O.get_odoo().search_count('product.product', _dom) if (_var or _tmpl) else 0
668
- ours = sum(1 for r in rows if isinstance(r.get(_col), (int, float)))
669
- checks.append({
670
- "check": f"{_name} pricelist: SKUs priced in the grid == an INDEPENDENT Odoo count "
671
- f"of ACTIVE products its date-valid fixed rules reach",
672
- "ours": ours, "theirs": theirs, "ok": ours == theirs,
673
- "detail": {"column": _col, "variant_rules": len(_var), "template_rules": len(_tmpl)},
674
- })
675
-
676
- # LEG 2 — ⛔ THE TICKET'S OWN NEGATIVE CONTROL, AS A LEG. A SKU with no specific item must
677
- # render BLANK. Asked of ODOO, never of the map that produced the blank — "everything I left
678
- # empty was genuinely unpriced" is trivially true when checked against the dict that emptied
679
- # it, which is the self-sealing shape the population leg above exists to end.
680
- _unpriced = [r for r in rows
681
- if not any(isinstance(r.get(c), (int, float))
682
- for c, _n in products.PRICELIST_COLUMNS)]
683
- _codes = {r["code"] for r in _unpriced}
684
- _stray = 0
685
- if _codes:
686
- # Resolve those codes back to Odoo ids and ask whether ANY declared list prices them.
687
- _ids, _tmpls = set(), set()
688
- for _p in O.search_read('product.product',
689
- [('active', '=', True),
690
- ('default_code', 'in', sorted(c for c in _codes
691
- if not c.startswith("pid:")))],
692
- ['id', 'product_tmpl_id']):
693
- _ids.add(_p['id'])
694
- _tmpls.add(O.m2o_id(_p.get('product_tmpl_id')))
695
- for _col, _name in products.PRICELIST_COLUMNS:
696
- _v, _t = _rule_targets(_name)
697
- _stray += len((_v & _ids) | ({t for t in _tmpls if t in _t}))
698
- checks.append({
699
- "check": "every SKU rendering a BLANK price is genuinely unpriced on all three declared "
700
- "lists (W29-T52's NC: no fallback dressed as a price) — asked of Odoo",
701
- "ours": _stray, "theirs": 0, "ok": _stray == 0,
702
- "detail": {"blank_skus": len(_unpriced), "priced_skus": len(rows) - len(_unpriced)},
703
- })
704
-
705
- # LEG 3 — ⭐ R6's SECOND SENTENCE, AS A NUMBER. *"If there is lag or it can't be done, you
706
- # need to explicitly tell me why and recommend a fix."* Everything the reader cannot see is
707
- # counted here rather than dropped. The ASSERTION is the one thing that must never be true —
708
- # a price cell that is present and not a positive number, i.e. a "free" SKU — while the
709
- # rest rides `detail` so it is reported without reddening an honest day's data.
710
- _bad = [r["code"] for r in rows
711
- for c, _n in products.PRICELIST_COLUMNS
712
- if r.get(c) is not None and not (isinstance(r.get(c), (int, float)) and r[c] > 0)]
713
- # ⛔ MEASURED 2026-08-12 AND NOT FIXABLE FROM THIS FENCE: `aios_grid.rows_from_pool` sends
714
- # every non-text field through `_round(v)` = `round(v)` with no ndigits, i.e. to a whole
715
- # dollar. 52.1% of fixed prices carry cents and the median relative error is 5.26%, so this
716
- # column ties to the cent HERE and ships rounded. Reported, never silently enforced.
717
- _cents = sum(1 for r in rows for c, _n in products.PRICELIST_COLUMNS
718
- if isinstance(r.get(c), (int, float)) and abs(r[c] - round(r[c])) > 1e-9)
719
- _to_zero = sum(1 for r in rows for c, _n in products.PRICELIST_COLUMNS
720
- if isinstance(r.get(c), (int, float)) and round(r[c]) == 0)
721
- checks.append({
722
- "check": "no price cell is present-but-not-a-positive-number (a 0 would read as FREE; "
723
- "an unset rule must be blank), and what the reader cannot see is COUNTED",
724
- "ours": len(_bad), "theirs": 0, "ok": not _bad,
725
- "detail": {"bad_cells": _bad[:10], "reader_report": price_report,
726
- "wire_rounding_loses_cents_on": _cents,
727
- "wire_rounding_to_zero_dollars": _to_zero},
728
- })
729
-
730
- # WAVE 17 R3 — the BUY SIGNAL must be a total, exact partition of the catalogue, and every
731
- # member of it must be re-derivable from the two columns beside it. A signal somebody buys
732
- # stock on cannot be "mostly right": the failure that matters is a row that says OK because
733
- # an input was missing, so the blank leg is checked as hard as the other two.
734
- if team_id is None: # consolidated only — the inputs exist only there
735
- buy = [r for r in rows if r.get("buy_now") == "Buy now"]
736
- ok_rows = [r for r in rows if r.get("buy_now") == "OK"]
737
- blank = [r for r in rows if r.get("buy_now") is None]
738
- mis = sum(1 for r in buy if not (r["dos"] < r["lead_days"]))
739
- mis += sum(1 for r in ok_rows if not (r["dos"] >= r["lead_days"]))
740
- # A blank must be UNKNOWN — never a row we could have answered and quietly did not.
741
- mis += sum(1 for r in blank
742
- if isinstance(r.get("dos"), (int, float))
743
- and isinstance(r.get("lead_days"), (int, float)) and r["lead_days"] > 0)
744
- checks.append({
745
- "check": "Buy signal partitions the catalogue (buy + ok + unknown == rows, none "
746
- "misclassified)",
747
- "ours": len(buy) + len(ok_rows) + len(blank) - mis, "theirs": len(rows),
748
- "ok": mis == 0 and len(buy) + len(ok_rows) + len(blank) == len(rows),
749
- "detail": {"buy_now": len(buy), "ok": len(ok_rows), "unknown": len(blank),
750
- "misclassified": mis},
751
- })
752
- # ⭐ OWNER 2026-08-03 — "use the formula fields to come up to the same EXACT figures".
753
- #
754
- # This is that sentence, as a check. The retired preset column and the formula that
755
- # replaces it must agree on every SKU, in all three states, or the replacement is not a
756
- # replacement. `_buy_signal_formula` is a line-by-line port of the client formula
757
- # engine's evaluation of
758
- #
759
- # IF({lead_days} > 0, IF({dos} < {lead_days}, "Buy now", "OK"), "")
760
- #
761
- # including the part that is easy to get wrong: a comparison against a BLANK is blank,
762
- # never a coerced 0. (The client engine's `cmp` returns null unless both sides read as
763
- # numbers, and `IF` refuses a non-boolean condition — so a missing `dos` yields "" and
764
- # not "Buy now". A filter engine would have said `0 < 30` and swept in every SKU that
765
- # never sells through; the formula engine does not, and this check is what holds it.)
766
- #
767
- # ⚠ It compares SETS OF SKUs, not counts. Two different partitions can share a shape.
768
- disagree = sorted(r["code"] for r in rows
769
- if (r.get("buy_now") or "") != _buy_signal_formula(r))
770
- checks.append({
771
- "check": 'Buy signal as a FORMULA field == the retired preset column, per SKU '
772
- '(IF({lead_days} > 0, IF({dos} < {lead_days}, "Buy now", "OK"), ""))',
773
- "ours": len(rows) - len(disagree), "theirs": len(rows),
774
- "ok": not disagree,
775
- "detail": {"disagreeing_skus": disagree[:10], "n_disagree": len(disagree)},
776
- })
777
- # The other half of "the same figures": the SHARED Buy list view can no longer filter on
778
- # the retired column, and its replacement conditions must select the same SKUs. They are
779
- # `dos isNotEmpty AND lead_days isNotEmpty AND lead_days > 0 AND dos < lead_days`
780
- # (_seed_wave17.views), so this reproduces exactly that conjunction.
781
- #
782
- # ⛔ THE NEGATIVE CONTROL IS WHY THIS IS NOT `cover_gap_d < 0`, which reads like the
783
- # obvious filter and is WRONG: `cover_gap_d` is `int(round(dos - lead))`, so a genuine
784
- # gap of -0.4 days rounds to 0 and that SKU drops off a buy list it belongs on.
785
- view_rows = {r["code"] for r in rows
786
- if isinstance(r.get("dos"), (int, float))
787
- and isinstance(r.get("lead_days"), (int, float))
788
- and r["lead_days"] > 0 and r["dos"] < r["lead_days"]}
789
- signal_rows = {r["code"] for r in buy}
790
- rounding_would_miss = sorted(
791
- c for c in signal_rows
792
- if next((r for r in rows if r["code"] == c), {}).get("cover_gap_d") == 0)
793
- checks.append({
794
- "check": "Buy list view conditions (dos/lead_days, no retired column) select "
795
- "exactly the Buy-now SKUs",
796
- "ours": len(view_rows), "theirs": len(signal_rows),
797
- "ok": view_rows == signal_rows,
798
- "detail": {"only_in_view": sorted(view_rows - signal_rows)[:10],
799
- "only_in_signal": sorted(signal_rows - view_rows)[:10],
800
- # Reported, not asserted: how many SKUs a `cover_gap_d < 0` filter would
801
- # have silently dropped. 0 today does not make that filter correct.
802
- "cover_gap_rounds_to_zero": len(rounding_would_miss)},
803
- })
804
- return checks
 
1
+ """modules/product_data.py — the PRODUCT table's pool (wave 15 item 9/10, contract C-TOPIC).
2
+
3
+ The second object on the table-page factory: same grid, same engine, same permission wall — the
4
+ only difference is the field schema and the identity. `modules/customer_data.pool()` is the
5
+ template and this deliberately mirrors its signature and its row shape.
6
+
7
+ ⛔ THREE DECISIONS THIS FILE HAD TO MAKE, EACH RECORDED BECAUSE A LATER READER WILL WONDER.
8
+
9
+ 1. **THE IDENTITY IS A SKU CODE, WHICH IS A STRING, AND THE GRID WANTS AN INTEGER `pid`.**
10
+ Cohort membership, `allowed_pids`, the measure channel and `rows_from_pool` all key on an
11
+ integer. So each row carries BOTH: `pid` (a stable CRC32 of the code, so the same SKU gets the
12
+ same id on every pull and across processes — never an enumeration index, which would reshuffle
13
+ whenever the catalogue changes) and `code`, the real business key, as a visible column.
14
+ `_assert_no_pid_collision` fails the BUILD rather than the read: two SKUs sharing a pid would
15
+ silently merge in every downstream set operation, and a loud build failure is the only version
16
+ of that anyone would notice.
17
+
18
+ 2. **THE SCOPE RULE — inventory columns are CONSOLIDATED and are therefore OMITTED for a
19
+ BU-scoped caller.** `products.directory(t, team_id)` is brand-shaped; `inventory.sku_inventory`
20
+ is explicitly NOT (its own docstring: "on-hand stock is one physical warehouse, not
21
+ brand-tagged"). Joining them for a Fisch-only user would put BU-shaped revenue beside
22
+ company-wide stock IN THE SAME ROW — the mixed-scope value defect wave 15's amendment 3 exists
23
+ for, arriving through a different door. The honest options were "omit" or "label the columns
24
+ company-wide"; omit is the fail-closed one, and a column that is absent asks a question,
25
+ whereas a column that is silently company-wide answers one wrongly.
26
+
27
+ 3. **WHAT IS NOT HERE, NAMED RATHER THAN QUIETLY MISSING.** R4 lists ~20 fields. Vendor and
28
+ COUNTRY are not in Odoo at all — they live in the inventory WORKBOOK
29
+ ([[odoo-vendor-country-origin]]) and need a loader this module deliberately does not invent.
30
+ Margin % comes from `modules/pricing.table`, which is a heavier build (channel-rate cost
31
+ allocation) and is left for the wave that needs it. `validate()` reconciles what SHIPS; it
32
+ does not pretend to cover columns that are absent.
33
+
34
+ 4. **THE POOL IS CATALOGUE-FIRST, WITH REVENUE LEFT-JOINED** (wave 29, owner item 22 / R12,
35
+ 2026-08-11). It was `for r in products.directory(...)` — and `directory()`'s row set IS the
36
+ union of two revenue read-groups, so **a SKU that never sold could not exist** and the grid
37
+ showed **2,717** of **5,875** active products. Three things that will be re-derived otherwise:
38
+
39
+ · ⛔ THE CAUSE IS A JOIN, NOT A LIMIT. No row cap exists on this path. Removing the date
40
+ window from `directory()` would ALSO be wrong twice: it breaks four SKU-health metrics
41
+ that legitimately want a sales window, and it lands at 3,327 (all-time-sold), because
42
+ ~2,550 active SKUs have never sold in wholesale scope at all.
43
+ · ⭐ REVENUE ON A NEVER-SOLD ROW IS **BLANK, NOT $0** (R12). A row that appears in the
44
+ revenue universe carries measured numbers INCLUDING a real 0.0 — it sold last year and
45
+ not this one, and that zero is a measurement. A row that appears in NO revenue read
46
+ carries `None`, which the wire keeps (`aios_grid._round` passes None through) and the
47
+ client renders as an empty cell. Blank is an admission; zero is a measurement.
48
+ · ⚠ A BU-SCOPED CALLER NOW SEES THE WHOLE CATALOGUE, with ITS OWN revenue and blanks where
49
+ that BU never sold. That is not decision 2's mixed-scope defect: the catalogue is the ROW
50
+ UNIVERSE, not a company-wide VALUE sitting beside a BU-shaped one. `product.product`
51
+ carries no team, so a catalogue cannot be BU-shaped at all — which is exactly why the two
52
+ sources are JOINED rather than merged.
53
+ """
54
+ import json
55
+ import zlib
56
+ from pathlib import Path
57
+
58
+ import core.odoo as O
59
+ import core.periods as P
60
+ import core.shared_overlay as shared_overlay
61
+ import core.table_store as table_store
62
+ import modules.products as products
63
+
64
+ #: POOL-ROW KEYS that exist ONLY on a consolidated pull. See decision 2.
65
+ #: `cover_gap_d` / `buy_now` (wave 17) join it because both are computed FROM `dos`, and a buy
66
+ #: signal built on a stock number the caller cannot see would be a recommendation nobody could
67
+ #: check ([[no-unverifiable-aggregates]]).
68
+ #:
69
+ #: ⚠ ROW KEYS, NOT FIELDS, and the distinction started mattering on 2026-08-03. Two readers:
70
+ #: `verify_perm_scope` asserts the shape of a POOL ROW against this, and `routes_products`
71
+ #: narrows the FIELD list with it. Since owner item 5 `buy_now` is a row key with no Field —
72
+ #: computed as `validate()`'s oracle, projected onto no wire — so it belongs here for the first
73
+ #: reader and is inert for the second. Removing it would stop the scope gate proving that a
74
+ #: BU-scoped pull withholds it.
75
+ #:
76
+ #: ⭐⭐ OWNER RULING 2026-08-11 — THIS SET IS NOW EMPTY, AND THAT IS THE POINT. Verbatim:
77
+ #: *"Just scope any inventory with Sales from Fisch, leave the rest."* Withholding the whole
78
+ #: inventory block from a BU-scoped reader meant a Fisch salesperson could not see whether
79
+ #: anything was IN STOCK, which is the first question they ask. The ruling splits the block by
80
+ #: what a business unit can actually shape:
81
+ #:
82
+ #: * `on_hand` / `unit_cost` / `inv_value` — ONE physical warehouse, no Fisch shelf and no
83
+ #: Royal shelf. Served UNSCOPED to everybody, identical in both pulls. ("leave the rest")
84
+ #: * `qty_ltm` / `dos` / `stock_bucket` / `cover_gap_d` / `buy_now` — all SALES-derived, so a
85
+ #: BU-scoped pull recomputes them from THAT unit's LTM units. ("scope any inventory with
86
+ #: Sales from Fisch")
87
+ #:
88
+ #: This deliberately relaxes decision 2's "never a company-wide value beside a BU-shaped one":
89
+ #: stock is not a value a BU can own, and a column that is absent for a Fisch reader asks a
90
+ #: question they cannot answer anywhere else in the product. `verify_perm_scope` no longer
91
+ #: asserts the columns are WITHHELD — it asserts the split above, which is a stronger claim and
92
+ #: cannot pass on an empty scoped payload (the shape the old rule and an outage share).
93
+ CONSOLIDATED_ONLY = ()
94
+
95
+ #: ⛔ POOL-ROW KEYS THAT DELIBERATELY HAVE NO FIELD — they must never reach a browser.
96
+ #:
97
+ #: `rows_from_pool` projects strictly through the field contract, so "no Field" already means
98
+ #: "no cell on the wire". This tuple makes that a CHECKED fact rather than a consequence nobody
99
+ #: is watching: `verify_perm_scope` asserts every member is absent from BOTH product contracts,
100
+ #: and that every OTHER `CONSOLIDATED_ONLY` key is present in the consolidated one.
101
+ #:
102
+ #: Written because the two tuples silently diverged the moment owner item 5 retired `buy_now`'s
103
+ #: Field while keeping its computation, and the gate — which was asserting a FIELD property from
104
+ #: a ROW-key list — went red with no way to tell a deliberate divergence from a dropped column.
105
+ UNSHIPPED_ROW_KEYS = ("buy_now",)
106
+
107
+ #: Wave 17 (owner item 13, ruling R3) — the SUPPLIER MASTER, from the curated mastersheet map
108
+ #: `procurement_suppliers.json` (3,963 SKUs; supplier on 3,624, lead time on 3,563). NOT Odoo:
109
+ #: the owner confirmed this data came from the Inventory System Mastersheet, which is why
110
+ #: `modules/product_data`'s header said Vendor/Country were "NOT HERE, named rather than
111
+ #: quietly missing" and needed "a loader this module deliberately does not invent". This is
112
+ #: that loader.
113
+ #:
114
+ #: ⛔ THESE ARE CONTRACT COLUMNS, NOT USER-CREATED FIELDS, AND THE REASON IS A PERMISSION FACT.
115
+ #: A user-created field and its values live in the PER-USER strata (`table_store.workspace`
116
+ #: reads `store.get(key)[username]`); only VIEWS are shared. So a shared "Buy list" view that
117
+ #: filtered on a user-created column would, for every OTHER account, name a column that does not
118
+ #: exist — and an unknown column is an INACTIVE condition in the tri-state engine, which IGNORES
119
+ #: it and therefore WIDENS. The buy list would silently show the whole catalogue to everyone but
120
+ #: its author. Contract columns are identical for every reader, so the view means one thing.
121
+ #:
122
+ #: ⭐⭐ 2026-08-12 (W30-T36) — THE OWNER'S ASK IS NOW DELIVERED, AND THE PARAGRAPH ABOVE IS WHY IT
123
+ #: TOOK THREE WAVES. Owner: *"turn this Excel sheet into a User created Field that we can edit."*
124
+ #: It was parked because a per-user column silently WIDENS a shared view — not because editing was
125
+ #: hard. Wave 29's `core/shared_overlay.py` (whose header quotes this very comment) is the stratum
126
+ #: that removes the objection: **one value per (row, column) for the whole tenant**, so the column
127
+ #: still means ONE thing to every reader and a shared view still filters honestly.
128
+ #: ⇒ The four columns below are now `source: "overlay"` + `shared: true` in the canonical
129
+ #: contract, their values live in `<TABLE_KEY>__shared`, and the master map is what SEEDS an
130
+ #: unedited cell rather than what freezes it. See `SHARED_KEYS` and `_ProductTableStore`.
131
+ _SUPPLIER_MAP_PATH = Path(__file__).resolve().parent.parent / "procurement_suppliers.json"
132
+ _SUPPLIER_CACHE = {}
133
+
134
+
135
+ def supplier_master():
136
+ """`{code: {supplier, lead_days, origin_country, first_cost}}`, read once per process.
137
+
138
+ Degrades to `{}` when the file is unreadable, matching `_inventory_by_code`: a product table
139
+ that will not render because a master map is missing is a worse failure than one with blank
140
+ supplier columns.
141
+ """
142
+ if _SUPPLIER_CACHE:
143
+ return _SUPPLIER_CACHE
144
+ try:
145
+ raw = json.loads(_SUPPLIER_MAP_PATH.read_text(encoding="utf-8"))
146
+ except Exception:
147
+ return {}
148
+ for code, meta in (raw or {}).items():
149
+ if not isinstance(meta, dict):
150
+ continue
151
+ lead = meta.get("lead")
152
+ _SUPPLIER_CACHE[str(code)] = {
153
+ "supplier": (meta.get("vendor") or "") or None,
154
+ "lead_days": int(lead) if isinstance(lead, (int, float)) else None,
155
+ "origin_country": (meta.get("country") or "") or None,
156
+ "first_cost": meta.get("first_cost"),
157
+ }
158
+ return _SUPPLIER_CACHE
159
+
160
+ #: Wave 16 C-TOPIC — the PRODUCT table's OWN workspace bucket. ⛔ Never the customer one:
161
+ #: product pids are CRC32 hashes of SKU codes and customer pids are Odoo partner ids, so in a
162
+ #: SHARED overlay bucket a hash collision would silently write a product note onto somebody's
163
+ #: customer (or the reverse). Separate store keys make that structurally impossible, which is
164
+ #: the whole reason the table-page factory exists ("a new table object gets its own
165
+ #: table_store.make('<its>_table_workspace')").
166
+ TABLE_KEY = 'product_table_workspace'
167
+
168
+ _GRID_FIELDS_PATH = Path(__file__).resolve().parent.parent / 'aios_grid_fields.json'
169
+ _SHARED_KEYS = None
170
+
171
+
172
+ def SHARED_KEYS():
173
+ """The product columns whose values are TENANT-WIDE — derived from the canonical contract's
174
+ own `shared: true`, never typed out a second time.
175
+
176
+ ⛔ IT DELIBERATELY DOES NOT SWALLOW A READ FAILURE. Degrading to `()` would send a shared
177
+ write into the per-user stratum with nothing going wrong anywhere — the widening defect
178
+ reappearing silently, which is the one outcome this whole mechanism exists to prevent. If the
179
+ canonical contract is unreadable the product grid cannot render at all (`pd_fields` parses the
180
+ same file with no guard), so a raise here costs nothing that was still working.
181
+ """
182
+ global _SHARED_KEYS
183
+ if _SHARED_KEYS is None:
184
+ doc = json.loads(_GRID_FIELDS_PATH.read_text(encoding='utf-8'))
185
+ _SHARED_KEYS = tuple(f['key'] for f in (doc.get('product_data') or {}).get('fields') or []
186
+ if f.get('shared'))
187
+ return _SHARED_KEYS
188
+
189
+
190
+ class _ProductTableStore(table_store.TableStore):
191
+ """The product workspace, with the SHARED columns routed to the tenant-wide stratum.
192
+
193
+ ⭐⭐ THIS SUBCLASS IS THE WHOLE OF W30-T36's WRITE PATH, AND THE REASON IT LIVES HERE RATHER
194
+ THAN AT A ROUTE IS THAT **THE BROWSER NEVER CALLS `PATCH /products/{pid}`** — measured, zero
195
+ call sites in `aios-web/web/src`. A cell edit travels `POST /grid/events` → `grid_events.
196
+ handle_one` → `_tops(ctx).patch_overlay(...)`, and `_tops` returns `ctx.table`, which
197
+ `routes_grid._ctx` sets to `TABLE_OPS` for the product scope. So this object IS the seam both
198
+ doors pass through; intercepting at either route would have left the other one writing a
199
+ per-user value that only its author could see.
200
+
201
+ ⚠ `st=self.st`, NEVER the module default. The shared stratum must resolve to the SAME store
202
+ handle as the per-user one it sits beside — `_tops`' own comment explains that a split, where
203
+ one side is tenant-scoped and the other is not, is worse than a stated residency error because
204
+ a user's value would vanish the moment they saved it. Reading `self.st` means both strata move
205
+ together the day that singleton gains a tenant handle.
206
+ """
207
+
208
+ def patch_overlay(self, username, pid, updates):
209
+ clean = dict(updates or {})
210
+ if not clean:
211
+ return
212
+ keys = set(SHARED_KEYS())
213
+ shared = {k: v for k, v in clean.items() if k in keys}
214
+ personal = {k: v for k, v in clean.items() if k not in keys}
215
+ if shared:
216
+ shared_overlay.put_cells(TABLE_KEY, pid, shared, st=self.st)
217
+ if personal:
218
+ super().patch_overlay(username, pid, personal)
219
+
220
+
221
+ TABLE_OPS = _ProductTableStore(TABLE_KEY)
222
+
223
+
224
+ def shared_cells(pids, st=None):
225
+ """`{"<pid>": {key: value}}` for the SHARED columns of the rows named by `pids`.
226
+
227
+ ⚠ `pids` is required and positional all the way down — `shared_overlay.cells` refuses to serve
228
+ "everything" by design, and the caller here always holds an already-scoped pool.
229
+ """
230
+ return shared_overlay.cells(TABLE_KEY, pids, st=st if st is not None else TABLE_OPS.st)
231
+
232
+
233
+ def sku_pid(code):
234
+ """A stable integer id for a SKU code. CRC32, masked to 31 bits so it is always positive and
235
+ always JSON-safe. Stable across processes and pulls, which an enumeration index is not."""
236
+ return zlib.crc32(str(code).encode("utf-8")) & 0x7FFFFFFF
237
+
238
+
239
+ def _assert_no_pid_collision(rows):
240
+ """Two SKUs sharing a pid would MERGE in every set operation downstream — cohort membership,
241
+ allowed_pids, the measure channel — and nothing would report it. Fail the build instead."""
242
+ seen = {}
243
+ for r in rows:
244
+ prior = seen.get(r["pid"])
245
+ if prior is not None and prior != r["code"]:
246
+ raise ValueError(
247
+ f"product_data: pid collision — {prior!r} and {r['code']!r} both hash to "
248
+ f"{r['pid']}. Downstream set operations would merge them silently; widen the id "
249
+ f"before shipping this catalogue.")
250
+ seen[r["pid"]] = r["code"]
251
+
252
+
253
+ def _inventory_by_code(t):
254
+ """`{code: {...}}` from the inventory module, or `{}` if it cannot be read.
255
+
256
+ Degrades to empty rather than raising, matching `customer_data._pool_build`'s treatment of
257
+ its own slow families: a product table that will not render because inventory is momentarily
258
+ unreachable is a worse failure than one with blank stock columns.
259
+ """
260
+ try:
261
+ import modules.inventory as inventory
262
+ return inventory.sku_inventory(t=t) or {}
263
+ except Exception:
264
+ return {}
265
+
266
+
267
+ def _bu_ltm_share(t, team_id):
268
+ """`{code: 0.0..1.0}` — this unit's SHARE of the SKU's last-twelve-months units.
269
+
270
+ ⭐ OWNER 2026-08-11: *"scope any inventory with Sales from Fisch"*. The velocity half of the
271
+ inventory block has to be re-shaped by a BU fact, and this is that fact.
272
+
273
+ ⛔ A SHARE, NOT THE UNIT COUNT ITSELF — AND THE REASON IS A MEASURED IMPOSSIBILITY. My first
274
+ version returned `products._sku_rev(...)['qty']` and used it directly as the scoped `qty_ltm`,
275
+ leaving the consolidated column on `inventory.sku_inventory`'s own figure. Two readers of one
276
+ question ([[one-question-two-normalizers]]): they disagree, and on 5 SKUs of 5,871 the live
277
+ check found **Fisch's LTM units EXCEEDING the company's** — a subset larger than its superset,
278
+ which no reader could explain and no reconciliation could survive.
279
+
280
+ Both halves of the ratio come from the SAME read here, so the share is in [0, 1] by
281
+ construction and the scoped figure can never exceed the consolidated one. It also leaves the
282
+ consolidated column exactly as it was — the Inventory page and the Product grid still agree
283
+ about company-wide units, which is what "leave the rest" asked for.
284
+
285
+ `all_qty == 0` implies `bu_qty == 0` (same reader), so a share of 0 is the honest answer for
286
+ "this unit never sold it": `_bucket` turns that into 'No recent sales', not zero cover.
287
+
288
+ Degrades to `{}` on any failure, matching `_inventory_by_code` — and a missing code then takes
289
+ the `_DEFAULT_SHARE` below rather than a silent 0.
290
+ """
291
+ try:
292
+ f, to = P.ltm(t)
293
+ allq = {code: (r.get('qty') or 0.0)
294
+ for code, r in (products._sku_rev(f, to, None) or {}).items()}
295
+ buq = {code: (r.get('qty') or 0.0)
296
+ for code, r in (products._sku_rev(f, to, team_id) or {}).items()}
297
+ except Exception:
298
+ return {}
299
+ out = {}
300
+ for code, total in allq.items():
301
+ out[code] = min(1.0, max(0.0, (buq.get(code, 0.0) / total))) if total > 0 else 0.0
302
+ return out
303
+
304
+
305
+ #: What a SKU absent from the LTM sales read is worth to a business unit. ZERO — it did not sell
306
+ #: in anybody's book over the window, so no unit can claim its velocity. Named rather than
307
+ #: inlined so the choice is visible: the alternative (1.0, "assume it is all ours") would print
308
+ #: company-wide cover on a BU grid, which is the mixed-scope defect this whole rule avoids.
309
+ _DEFAULT_SHARE = 0.0
310
+
311
+
312
+ def _rescope_inventory(e, share):
313
+ """`(qty_ltm, dos, bucket)` recomputed for ONE business unit's sales rate.
314
+
315
+ The formulas are `inventory.sku_inventory`'s, applied to a BU-shaped numerator — NOT a second
316
+ idea of what days-of-supply means. `_bucket` is imported from there for the same reason: two
317
+ copies of a threshold table is how the Product grid and the Inventory page start disagreeing
318
+ about which SKUs are dead.
319
+
320
+ ⚠ `on_hand` is whatever the warehouse holds, unscoped — so a Fisch reader's `dos` answers
321
+ "how long does ALL our stock last at Fisch's rate", which is the question a Fisch salesperson
322
+ actually has. It is deliberately NOT a pro-rated share of the shelf: there is no such shelf,
323
+ and inventing one would put a number on screen that no Odoo query could reproduce.
324
+ """
325
+ on_hand = e.get("on_hand")
326
+ if on_hand is None:
327
+ return None, None, None # no inventory row for this SKU: blank, never zero
328
+ qty = float(e.get("qty_ltm") or 0.0) * float(share or 0.0)
329
+ daily = qty / 365.0
330
+ if daily > 0:
331
+ dos_raw = on_hand / daily
332
+ else:
333
+ dos_raw = float('inf') if on_hand > 0 else 0.0
334
+ try:
335
+ import modules.inventory as inventory
336
+ bucket = inventory._bucket(dos_raw, on_hand, qty)
337
+ except Exception:
338
+ bucket = None
339
+ return qty, (None if dos_raw == float('inf') else round(float(dos_raw), 0)), bucket
340
+
341
+
342
+ def _catalogue_by_code():
343
+ """`{code: {'product', 'category'}}` — the CATALOGUE universe this pool is built from.
344
+
345
+ ⛔ UNLIKE `_inventory_by_code`, THIS ONE RAISES, and the asymmetry is the whole point.
346
+ Inventory degrades to `{}` because a product table with blank stock columns beats one that
347
+ will not render. The CATALOGUE is not a column — it is the ROW SET. A catalogue read that
348
+ failed quietly would drop the grid straight back to the sold-only 2,717 with every gate
349
+ green and nothing on screen saying so, which is the defect this seam exists to end
350
+ ([[gate-can-report-green-on-nothing]]). `routes_products._pool_for` already turns the raise
351
+ into a 503 that names the cause, so the loud failure has somewhere honest to land.
352
+
353
+ It exists as a `pd`-level function rather than an inline `products.catalogue()` call for the
354
+ same reason `_inventory_by_code` does: it is the seam `verify_perm_scope`'s section H stubs
355
+ to build a pool without Odoo.
356
+ """
357
+ return products.catalogue()
358
+
359
+
360
+ def _pricelist_by_code():
361
+ """`({code: {price_*: price}}, report)` from `products.pricelist_by_code`, or `({}, …)`.
362
+
363
+ A SEAM for the same two reasons `_inventory_by_code` is one: it degrades rather than raises
364
+ (these are columns, not the row set), and `verify_perm_scope`'s section H stubs it to build a
365
+ pool without Odoo. ⛔ Unstubbed there, section H would reach live Odoo through the back door
366
+ and the whole file would stop being runnable offline.
367
+ """
368
+ try:
369
+ return products.pricelist_by_code()
370
+ except Exception as e:
371
+ return {}, {"error": f"{type(e).__name__}: {str(e)[:200]}"}
372
+
373
+
374
+ def pool(team_id=None, t=None):
375
+ """One row per ACTIVE product — the PRODUCT analogue of `customer_data.pool`.
376
+
377
+ CATALOGUE-FIRST, REVENUE LEFT-JOINED (R12 — see decision 4 in the module header). The row set
378
+ is `products.catalogue()`; `products.directory()` supplies the revenue columns for the SKUs
379
+ that sold in its window and contributes NO rows of its own.
380
+
381
+ `team_id` shapes the revenue columns exactly as it does for customers (`products.directory`
382
+ passes it into `_sku_rev`), which is why `core.perm_scope.derive_pool_scope` must keep
383
+ driving it rather than a post-filter deciding the BU. It does NOT shape the row set: a
384
+ catalogue has no team.
385
+ """
386
+ t = t or P.today()
387
+ consolidated = team_id is None
388
+ # ⭐ OWNER 2026-08-11: read inventory on EVERY pull, not just a consolidated one. The stock
389
+ # itself is company-wide; only the sales-derived half is re-scoped, by `_rescope_inventory`.
390
+ inv = _inventory_by_code(t)
391
+ bu_share = {} if consolidated else _bu_ltm_share(t, team_id)
392
+ sup = supplier_master()
393
+ prices, _price_report = _pricelist_by_code()
394
+ cat = _catalogue_by_code()
395
+ # The LEFT side of the join, indexed by the same code key. ⛔ A code here that the catalogue
396
+ # does not carry belongs to a product no ACTIVE record claims — archived, and R12 keeps those
397
+ # out. `validate()` asserts that of Odoo rather than assuming it, and reports the revenue
398
+ # that therefore sits outside the grid (MEASURED 2026-08-11: 3 codes, $0.00 YTD / $294.50 LY).
399
+ rev = {r["code"]: r for r in products.directory(t=t, team_id=team_id)}
400
+
401
+ rows = []
402
+ for code, meta in cat.items():
403
+ r = rev.get(code)
404
+ row = {
405
+ "pid": sku_pid(code),
406
+ "code": code,
407
+ # ⭐ ONE source for the name and the category, the CATALOGUE — not the sale line's
408
+ # m2o. For a re-SKUed code the line's name can be the ARCHIVED record's; the active
409
+ # record's `display_name` is the current truth, and it is the same string for every
410
+ # SKU that is not re-SKUed. `directory()` derives the category identically.
411
+ "product": meta.get("product") or code,
412
+ "category": meta.get("category") or "(uncategorized)",
413
+ # ⭐ R12 — BLANK, NEVER $0, on a SKU no revenue read ever saw. A row that IS in `rev`
414
+ # keeps its measured numbers including a real 0.0 (it sold last year, not this one).
415
+ "rev_ytd": r.get("rev_ytd", 0.0) if r else None,
416
+ "rev_ly": r.get("rev_ly", 0.0) if r else None,
417
+ "yoy_pct": r.get("yoy_pct") if r else None,
418
+ "qty_ytd": r.get("qty_ytd", 0.0) if r else None,
419
+ "orders_ytd": r.get("orders_ytd", 0) if r else None,
420
+ }
421
+ # ⭐ WAVE 30 W30-T34 — the PRICELIST stratum, one column per declared list. Catalogue
422
+ # data like the supplier block, so it rides every pull, scoped or not: a price book is
423
+ # not a thing a business unit owns a slice of, and a Fisch reader who cannot see the
424
+ # Fisch price is exactly who this column is for.
425
+ #
426
+ # ⛔ `None`, NEVER 0 AND NEVER A FALLBACK, for the 1,883 SKUs no list prices. That is
427
+ # W29-T52's own negative control: the single `3_global` rule computes over `list_price`,
428
+ # which is 1.00 on 5,817 of 5,871 products, so the fallback is not a cheaper answer —
429
+ # it is a wrong one wearing a currency sign.
430
+ p = prices.get(code) or {}
431
+ for _col, _name in products.PRICELIST_COLUMNS:
432
+ row[_col] = p.get(_col)
433
+ # Wave 17 R3 — the supplier master, on every pull (it is catalogue data, not stock).
434
+ s = sup.get(code) or {}
435
+ row.update({
436
+ "supplier": s.get("supplier") or "(none)",
437
+ "lead_days": s.get("lead_days"),
438
+ "origin_country": s.get("origin_country") or "(none)",
439
+ "first_cost": s.get("first_cost"),
440
+ })
441
+ e = inv.get(code) or {}
442
+ if consolidated:
443
+ qty_ltm, dos, bucket = e.get("qty_ltm"), e.get("dos"), e.get("bucket")
444
+ else:
445
+ qty_ltm, dos, bucket = _rescope_inventory(e, bu_share.get(code, _DEFAULT_SHARE))
446
+ row.update({
447
+ # UNSCOPED on purpose (owner 2026-08-11): one warehouse, no per-brand shelf.
448
+ "on_hand": e.get("on_hand"),
449
+ "unit_cost": e.get("unit_cost"),
450
+ "inv_value": e.get("inv_value"),
451
+ # SCOPED: these three are functions of how fast THIS unit sells the SKU.
452
+ "qty_ltm": qty_ltm,
453
+ "dos": dos,
454
+ "stock_bucket": bucket,
455
+ })
456
+ # THE BUY TRIGGER (R3), ported from `modules/procurement`'s lead-time-cover rule:
457
+ # buy when the shelf runs out before a reorder could land.
458
+ #
459
+ # ⚠ BLANK, NOT "OK", WHEN EITHER INPUT IS MISSING. 339 SKUs have no supplier and 400
460
+ # no lead time; `dos` is null for anything that never sells through. "We don't know"
461
+ # and "you're fine" are different sentences, and only one of them is safe to print
462
+ # beside a purchasing decision.
463
+ #
464
+ # ⛔ `buy_now` SHIPS NOWHERE ANY MORE, and is computed anyway. OWNER 2026-08-03: it
465
+ # is not a preset field — it is a formula over the two columns beside it, and the
466
+ # platform has a formula field type for exactly that. Its Field is gone from
467
+ # `aios_grid_fields.json`, and `rows_from_pool` projects rows STRICTLY through that
468
+ # contract, so no Field means no cell on the wire. Nothing renders this key.
469
+ #
470
+ # It stays computed because deleting it would delete the PROOF. `validate()` below
471
+ # reconciles the formula's predicate against this one, row for row, which is the only
472
+ # thing that makes "the same exact figures" a claim rather than an assertion — and
473
+ # this repo has the scar already (wave 17: archiving `ar` "would have DELETED the
474
+ # proof"). Two comparisons per SKU is what that costs.
475
+ #
476
+ # ⭐ READS THE SCOPED `dos`, NOT `e['dos']`. On a Fisch pull the cover gap must answer
477
+ # "does the shelf outlast a reorder AT FISCH'S RATE" — reading the consolidated figure
478
+ # here would print a buy signal computed from both units' velocity beside a days-of-supply
479
+ # computed from one, and the two columns would disagree on the same row.
480
+ lead = s.get("lead_days")
481
+ if isinstance(dos, (int, float)) and isinstance(lead, (int, float)) and lead > 0:
482
+ row["cover_gap_d"] = int(round(dos - lead))
483
+ row["buy_now"] = "Buy now" if dos < lead else "OK"
484
+ else:
485
+ row["cover_gap_d"] = None
486
+ row["buy_now"] = None
487
+ rows.append(row)
488
+
489
+ # `or 0.0` reads a BLANK as zero FOR SORTING ONLY — never for the cell. A never-sold SKU
490
+ # settles among the zero-revenue ones at the bottom, which is where it belongs; the stored
491
+ # value stays None so nothing downstream can mistake "we never saw a sale" for "we measured
492
+ # nothing sold".
493
+ rows.sort(key=lambda r: -(r["rev_ytd"] or 0.0))
494
+ _assert_no_pid_collision(rows)
495
+ return rows
496
+
497
+
498
+ #: ⭐ THE BUY SIGNAL, AS THE FORMULA FIELD EVALUATES IT (owner, 2026-08-03).
499
+ #:
500
+ #: The owner's ruling was that "Buy signal" is not a preset field — it is a formula over two
501
+ #: columns the product table already carries, and the platform has a formula field type for it.
502
+ #: The formula, verbatim, is what `_seed_wave17.BUY_SIGNAL_FORMULA` creates and what the JSON
503
+ #: contract's `_product_removed_buy_now` note records:
504
+ #:
505
+ #: IF({lead_days} > 0, IF({dos} < {lead_days}, "Buy now", "OK"), "")
506
+ #:
507
+ #: This function is a PORT of how `customer-grid/formulaEngine.ts` evaluates that tree, not a
508
+ #: restatement of the business rule — that is the whole point, because the two could drift and
509
+ #: `validate()` is where the drift must show. Three engine behaviours it reproduces exactly:
510
+ #:
511
+ #: · a `ref` to a missing/non-numeric cell is None (`case "ref"` returns null for anything
512
+ #: that is not a finite number or a string);
513
+ #: · `cmp` returns BLANK unless BOTH sides read as numbers — it never coerces a blank to 0,
514
+ #: which is the difference between this and a filter engine's `toNum(null) === 0`;
515
+ #: · `IF` with a non-boolean condition returns BLANK ("no truthiness"), so a blank comparison
516
+ #: propagates out as a blank cell rather than taking the false branch.
517
+ BUY_SIGNAL_FORMULA = 'IF({lead_days} > 0, IF({dos} < {lead_days}, "Buy now", "OK"), "")'
518
+
519
+
520
+ def _buy_signal_formula(row):
521
+ """Evaluate `BUY_SIGNAL_FORMULA` over one pool row -> 'Buy now' | 'OK' | '' (blank)."""
522
+ def num(v):
523
+ # The engine's `ref` + `asNumber`: booleans are not values here, and a non-finite
524
+ # number is null. `isinstance(True, int)` is True in Python, so bool is excluded first.
525
+ if isinstance(v, bool) or not isinstance(v, (int, float)):
526
+ return None
527
+ return v if v == v and v not in (float('inf'), float('-inf')) else None
528
+
529
+ lead, dos = num(row.get("lead_days")), num(row.get("dos"))
530
+ if lead is None: # `{lead_days} > 0` is blank -> IF(blank, …) is blank
531
+ return ""
532
+ if not lead > 0:
533
+ return "" # the formula's own else-branch
534
+ if dos is None: # `{dos} < {lead_days}` is blank -> IF(blank, …) is blank
535
+ return ""
536
+ return "Buy now" if dos < lead else "OK"
537
+
538
+
539
+ def validate(team_id=None, t=None):
540
+ """Reconcile what SHIPS to an independent aggregate — the platform's own rule that a number
541
+ which does not tie to Odoo does not ship.
542
+
543
+ ⛔⛔ THE POPULATION LEG IS NEW, AND IT IS HERE BECAUSE THIS FUNCTION USED TO BE SELF-SEALING
544
+ (wave 29, contract C8). It reconciled Σ per-SKU YTD revenue against `products._sku_rev` — the
545
+ same function `pool()` built its rows from. BOTH SIDES CAME FROM THE JOIN, so the oracle could
546
+ not see a missing row, and a grid holding 2,717 of 5,875 active products passed for a wave.
547
+ No gate anywhere asserted a pool row COUNT against an independent Odoo `search_count` either.
548
+ That is the leg below, and it is the control that would have caught it
549
+ ([[no-unverifiable-aggregates]]).
550
+
551
+ ⚠ The revenue leg was NOT deleted with it — it answers a different question ("did the join
552
+ lose money?") and is still the right shape for that one. What changed is that it now has to
553
+ account for revenue belonging to codes NO ACTIVE PRODUCT CARRIES, because R12 keeps archived
554
+ products out of the grid; the total is decomposed rather than compared loosely, so a growing
555
+ "outside the grid" figure shows up as a number rather than as slack in a tolerance.
556
+
557
+ Still deliberately NOT covered, and said rather than implied: the inventory columns are a
558
+ different module's reconciliation, and they are absent on a scoped pull anyway.
559
+ """
560
+ t = t or P.today()
561
+ rows = pool(team_id=team_id, t=t)
562
+
563
+ # ⭐⭐ THE POPULATION, AGAINST AN ORACLE THAT CANNOT SEE OUR JOIN. A bare Odoo count of active
564
+ # products, asked fresh — not a `len()` over anything this module or `products.catalogue()`
565
+ # built. NC: drop one row from `pool()` and this goes red; that is the whole point of it.
566
+ n_active = products.catalogue_count()
567
+ checks = [{
568
+ "check": "the pool holds one row per ACTIVE product (R12) — row count == an INDEPENDENT "
569
+ "Odoo search_count('product.product', active=True)",
570
+ "ours": len(rows), "theirs": n_active, "ok": len(rows) == n_active,
571
+ }]
572
+
573
+ yf, yt = P.ytd(t)
574
+ sku_rev = products._sku_rev(yf, yt, team_id) or {}
575
+ in_pool = {r["code"] for r in rows}
576
+ outside = {c: v for c, v in sku_rev.items() if c not in in_pool}
577
+ ours = round(sum(r["rev_ytd"] or 0.0 for r in rows), 2)
578
+ theirs = round(sum(v.get("rev", 0.0) for v in sku_rev.values()), 2)
579
+ dropped = round(sum(v.get("rev", 0.0) for v in outside.values()), 2)
580
+ checks.append({
581
+ "check": "Σ per-SKU YTD revenue in the grid + Σ YTD revenue of codes no ACTIVE product "
582
+ "carries == modules.products._sku_rev over the same window",
583
+ "ours": round(ours + dropped, 2), "theirs": theirs,
584
+ "ok": abs((ours + dropped) - theirs) < 0.01,
585
+ "detail": {"in_the_grid": ours, "outside_the_grid": dropped,
586
+ "codes_outside": len(outside)},
587
+ })
588
+ # ⛔ AND THE EXCLUSION IS ASSERTED, NOT ASSUMED. "Everything I dropped was archived" is
589
+ # trivially true when checked against the dict that did the dropping — that is the
590
+ # self-sealing shape all over again. So it is asked of ODOO. A code here that a live ACTIVE
591
+ # product carries means `products.catalogue()` missed a row that has revenue, which is the
592
+ # 2,717 defect returning in a smaller costume.
593
+ # ⚠ `pid:N` keys are not `default_code`s and are skipped: an UNCODED active product is in the
594
+ # catalogue under its own `pid:N` key and therefore cannot be in `outside` at all.
595
+ coded_outside = sorted(c for c in outside if not str(c).startswith("pid:"))
596
+ still_active = (O.get_odoo().search_count(
597
+ 'product.product', [('active', '=', True), ('default_code', 'in', coded_outside)])
598
+ if coded_outside else 0)
599
+ checks.append({
600
+ "check": "every SKU with revenue but NO grid row is genuinely ARCHIVED (R12: archived "
601
+ "stay out) — asked of Odoo, never of the catalogue that did the dropping",
602
+ "ours": still_active, "theirs": 0, "ok": still_active == 0,
603
+ "detail": {"codes": coded_outside[:10], "n_codes": len(outside)},
604
+ })
605
+ # ── ⭐⭐ WAVE 30 W30-T34 — THE PRICELIST COLUMNS, AGAINST ORACLES THAT CANNOT SEE OUR JOIN ──
606
+ #
607
+ # Three legs, because the column can fail in three different ways and only one of them is a
608
+ # count. `products.pricelist_by_code` builds a `{code: {col: price}}` map by expanding each
609
+ # rule over the products it names, IN PYTHON — that expansion is the fragile part, so every
610
+ # leg below re-asks ODOO instead of re-reading the map.
611
+ price_report = _pricelist_by_code()[1]
612
+ _pl_rows = O.search_read('product.pricelist', [], ['id', 'name'])
613
+ _pl_id = {}
614
+ for _p in _pl_rows:
615
+ _pl_id.setdefault(str(_p.get('name') or '').strip(), _p['id'])
616
+ _today = P.today().isoformat()
617
+
618
+ _targets_memo = {}
619
+
620
+ def _rule_targets(list_name):
621
+ """(variant_ids, template_ids) a pricelist prices today — read FRESH from Odoo.
622
+
623
+ Memoised for the LIFE OF THIS CALL only: legs 1 and 2 both need all three lists, and
624
+ without this `validate()` makes six identical round trips instead of three (measured:
625
+ ~48s of the run). Deliberately NOT an `lru_cache` — an oracle that survives the process
626
+ is an oracle reading yesterday's Odoo.
627
+ """
628
+ if list_name in _targets_memo:
629
+ return _targets_memo[list_name]
630
+ var, tmpl = set(), set()
631
+ plid = _pl_id.get(list_name)
632
+ if plid is None:
633
+ return var, tmpl
634
+ for r in O.search_read(
635
+ 'product.pricelist.item',
636
+ [('pricelist_id', '=', plid), ('compute_price', '=', 'fixed'),
637
+ ('applied_on', 'in', ['0_product_variant', '1_product'])],
638
+ ['product_id', 'product_tmpl_id', 'applied_on', 'fixed_price',
639
+ 'date_start', 'date_end']):
640
+ ds, de = str(r.get('date_start') or '')[:10], str(r.get('date_end') or '')[:10]
641
+ if (ds and ds > _today) or (de and de < _today):
642
+ continue
643
+ fp = r.get('fixed_price')
644
+ if not isinstance(fp, (int, float)) or fp <= 0:
645
+ continue
646
+ if r.get('applied_on') == '0_product_variant' and r.get('product_id'):
647
+ var.add(O.m2o_id(r['product_id']))
648
+ elif r.get('product_tmpl_id'):
649
+ tmpl.add(O.m2o_id(r['product_tmpl_id']))
650
+ _targets_memo[list_name] = (var, tmpl)
651
+ return var, tmpl
652
+
653
+ # LEG 1 — COVERAGE PER LIST. Ours: cells we filled. Theirs: a bare Odoo `search_count` of
654
+ # ACTIVE products a fresh read of that list's rules reaches. The rule set is shared (it IS
655
+ # the data) but the EXPANSION is not, and the expansion is what breaks.
656
+ for _col, _name in products.PRICELIST_COLUMNS:
657
+ _var, _tmpl = _rule_targets(_name)
658
+ _dom = [('active', '=', True), '|', ('id', 'in', sorted(_var)),
659
+ ('product_tmpl_id', 'in', sorted(_tmpl))]
660
+ theirs = O.get_odoo().search_count('product.product', _dom) if (_var or _tmpl) else 0
661
+ ours = sum(1 for r in rows if isinstance(r.get(_col), (int, float)))
662
+ checks.append({
663
+ "check": f"{_name} pricelist: SKUs priced in the grid == an INDEPENDENT Odoo count "
664
+ f"of ACTIVE products its date-valid fixed rules reach",
665
+ "ours": ours, "theirs": theirs, "ok": ours == theirs,
666
+ "detail": {"column": _col, "variant_rules": len(_var), "template_rules": len(_tmpl)},
667
+ })
668
+
669
+ # LEG 2 — ⛔ THE TICKET'S OWN NEGATIVE CONTROL, AS A LEG. A SKU with no specific item must
670
+ # render BLANK. Asked of ODOO, never of the map that produced the blank — "everything I left
671
+ # empty was genuinely unpriced" is trivially true when checked against the dict that emptied
672
+ # it, which is the self-sealing shape the population leg above exists to end.
673
+ _unpriced = [r for r in rows
674
+ if not any(isinstance(r.get(c), (int, float))
675
+ for c, _n in products.PRICELIST_COLUMNS)]
676
+ _codes = {r["code"] for r in _unpriced}
677
+ _stray = 0
678
+ if _codes:
679
+ # Resolve those codes back to Odoo ids and ask whether ANY declared list prices them.
680
+ _ids, _tmpls = set(), set()
681
+ for _p in O.search_read('product.product',
682
+ [('active', '=', True),
683
+ ('default_code', 'in', sorted(c for c in _codes
684
+ if not c.startswith("pid:")))],
685
+ ['id', 'product_tmpl_id']):
686
+ _ids.add(_p['id'])
687
+ _tmpls.add(O.m2o_id(_p.get('product_tmpl_id')))
688
+ for _col, _name in products.PRICELIST_COLUMNS:
689
+ _v, _t = _rule_targets(_name)
690
+ _stray += len((_v & _ids) | ({t for t in _tmpls if t in _t}))
691
+ checks.append({
692
+ "check": "every SKU rendering a BLANK price is genuinely unpriced on all three declared "
693
+ "lists (W29-T52's NC: no fallback dressed as a price) — asked of Odoo",
694
+ "ours": _stray, "theirs": 0, "ok": _stray == 0,
695
+ "detail": {"blank_skus": len(_unpriced), "priced_skus": len(rows) - len(_unpriced)},
696
+ })
697
+
698
+ # LEG 3 — ⭐ R6's SECOND SENTENCE, AS A NUMBER. *"If there is lag or it can't be done, you
699
+ # need to explicitly tell me why and recommend a fix."* Everything the reader cannot see is
700
+ # counted here rather than dropped. The ASSERTION is the one thing that must never be true —
701
+ # a price cell that is present and not a positive number, i.e. a "free" SKU — while the
702
+ # rest rides `detail` so it is reported without reddening an honest day's data.
703
+ _bad = [r["code"] for r in rows
704
+ for c, _n in products.PRICELIST_COLUMNS
705
+ if r.get(c) is not None and not (isinstance(r.get(c), (int, float)) and r[c] > 0)]
706
+ # ⛔ MEASURED 2026-08-12 AND NOT FIXABLE FROM THIS FENCE: `aios_grid.rows_from_pool` sends
707
+ # every non-text field through `_round(v)` = `round(v)` with no ndigits, i.e. to a whole
708
+ # dollar. 52.1% of fixed prices carry cents and the median relative error is 5.26%, so this
709
+ # column ties to the cent HERE and ships rounded. Reported, never silently enforced.
710
+ _cents = sum(1 for r in rows for c, _n in products.PRICELIST_COLUMNS
711
+ if isinstance(r.get(c), (int, float)) and abs(r[c] - round(r[c])) > 1e-9)
712
+ _to_zero = sum(1 for r in rows for c, _n in products.PRICELIST_COLUMNS
713
+ if isinstance(r.get(c), (int, float)) and round(r[c]) == 0)
714
+ checks.append({
715
+ "check": "no price cell is present-but-not-a-positive-number (a 0 would read as FREE; "
716
+ "an unset rule must be blank), and what the reader cannot see is COUNTED",
717
+ "ours": len(_bad), "theirs": 0, "ok": not _bad,
718
+ "detail": {"bad_cells": _bad[:10], "reader_report": price_report,
719
+ "wire_rounding_loses_cents_on": _cents,
720
+ "wire_rounding_to_zero_dollars": _to_zero},
721
+ })
722
+
723
+ # WAVE 17 R3 — the BUY SIGNAL must be a total, exact partition of the catalogue, and every
724
+ # member of it must be re-derivable from the two columns beside it. A signal somebody buys
725
+ # stock on cannot be "mostly right": the failure that matters is a row that says OK because
726
+ # an input was missing, so the blank leg is checked as hard as the other two.
727
+ if team_id is None: # consolidated only — the inputs exist only there
728
+ buy = [r for r in rows if r.get("buy_now") == "Buy now"]
729
+ ok_rows = [r for r in rows if r.get("buy_now") == "OK"]
730
+ blank = [r for r in rows if r.get("buy_now") is None]
731
+ mis = sum(1 for r in buy if not (r["dos"] < r["lead_days"]))
732
+ mis += sum(1 for r in ok_rows if not (r["dos"] >= r["lead_days"]))
733
+ # A blank must be UNKNOWN — never a row we could have answered and quietly did not.
734
+ mis += sum(1 for r in blank
735
+ if isinstance(r.get("dos"), (int, float))
736
+ and isinstance(r.get("lead_days"), (int, float)) and r["lead_days"] > 0)
737
+ checks.append({
738
+ "check": "Buy signal partitions the catalogue (buy + ok + unknown == rows, none "
739
+ "misclassified)",
740
+ "ours": len(buy) + len(ok_rows) + len(blank) - mis, "theirs": len(rows),
741
+ "ok": mis == 0 and len(buy) + len(ok_rows) + len(blank) == len(rows),
742
+ "detail": {"buy_now": len(buy), "ok": len(ok_rows), "unknown": len(blank),
743
+ "misclassified": mis},
744
+ })
745
+ # ⭐ OWNER 2026-08-03 — "use the formula fields to come up to the same EXACT figures".
746
+ #
747
+ # This is that sentence, as a check. The retired preset column and the formula that
748
+ # replaces it must agree on every SKU, in all three states, or the replacement is not a
749
+ # replacement. `_buy_signal_formula` is a line-by-line port of the client formula
750
+ # engine's evaluation of
751
+ #
752
+ # IF({lead_days} > 0, IF({dos} < {lead_days}, "Buy now", "OK"), "")
753
+ #
754
+ # including the part that is easy to get wrong: a comparison against a BLANK is blank,
755
+ # never a coerced 0. (The client engine's `cmp` returns null unless both sides read as
756
+ # numbers, and `IF` refuses a non-boolean condition — so a missing `dos` yields "" and
757
+ # not "Buy now". A filter engine would have said `0 < 30` and swept in every SKU that
758
+ # never sells through; the formula engine does not, and this check is what holds it.)
759
+ #
760
+ # ⚠ It compares SETS OF SKUs, not counts. Two different partitions can share a shape.
761
+ disagree = sorted(r["code"] for r in rows
762
+ if (r.get("buy_now") or "") != _buy_signal_formula(r))
763
+ checks.append({
764
+ "check": 'Buy signal as a FORMULA field == the retired preset column, per SKU '
765
+ '(IF({lead_days} > 0, IF({dos} < {lead_days}, "Buy now", "OK"), ""))',
766
+ "ours": len(rows) - len(disagree), "theirs": len(rows),
767
+ "ok": not disagree,
768
+ "detail": {"disagreeing_skus": disagree[:10], "n_disagree": len(disagree)},
769
+ })
770
+ # The other half of "the same figures": the SHARED Buy list view can no longer filter on
771
+ # the retired column, and its replacement conditions must select the same SKUs. They are
772
+ # `dos isNotEmpty AND lead_days isNotEmpty AND lead_days > 0 AND dos < lead_days`
773
+ # (_seed_wave17.views), so this reproduces exactly that conjunction.
774
+ #
775
+ # ⛔ THE NEGATIVE CONTROL IS WHY THIS IS NOT `cover_gap_d < 0`, which reads like the
776
+ # obvious filter and is WRONG: `cover_gap_d` is `int(round(dos - lead))`, so a genuine
777
+ # gap of -0.4 days rounds to 0 and that SKU drops off a buy list it belongs on.
778
+ view_rows = {r["code"] for r in rows
779
+ if isinstance(r.get("dos"), (int, float))
780
+ and isinstance(r.get("lead_days"), (int, float))
781
+ and r["lead_days"] > 0 and r["dos"] < r["lead_days"]}
782
+ signal_rows = {r["code"] for r in buy}
783
+ rounding_would_miss = sorted(
784
+ c for c in signal_rows
785
+ if next((r for r in rows if r["code"] == c), {}).get("cover_gap_d") == 0)
786
+ checks.append({
787
+ "check": "Buy list view conditions (dos/lead_days, no retired column) select "
788
+ "exactly the Buy-now SKUs",
789
+ "ours": len(view_rows), "theirs": len(signal_rows),
790
+ "ok": view_rows == signal_rows,
791
+ "detail": {"only_in_view": sorted(view_rows - signal_rows)[:10],
792
+ "only_in_signal": sorted(signal_rows - view_rows)[:10],
793
+ # Reported, not asserted: how many SKUs a `cover_gap_d < 0` filter would
794
+ # have silently dropped. 0 today does not make that filter correct.
795
+ "cover_gap_rounds_to_zero": len(rounding_would_miss)},
796
+ })
797
+ return checks
 
 
 
 
 
 
 
platform/modules/products.py CHANGED
@@ -1,737 +1,728 @@
1
- """Products / SKU module — SKU health: YoY movers (risers & decliners), zombie SKUs
2
- (catalog rot — sellable, formerly selling, now dead), new winners, coverage collapse
3
- (SKUs losing customer breadth — the FFS-recovery early-warning signal), and velocity leaders.
4
-
5
- Per-SKU margin already lives in the Financial module (low_margin_skus); not duplicated here.
6
- Basket / co-purchase (MBA) is intentionally deferred — the existing client app computes it
7
- runtime-side, and a local co-occurrence pull over 74k LTM lines is the kind of heavy job the
8
- project guardrails keep off the local PC. (See BACKLOG.)
9
-
10
- All line-level (sale.order.line), RI+FFS scope, excluded accounts removed — reusing sale_line_domain.
11
- Coverage (distinct customers per SKU) uses a 2-level read_group and is a touch slow (~15s/
12
- window), so it's its own function the app calls lazily and caches.
13
- """
14
- import sys
15
- import datetime as dt
16
- from pathlib import Path
17
- sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
18
- from functools import lru_cache
19
- import core.odoo as O
20
- import core.periods as P
21
- import modules.sales as sales_mod
22
-
23
- # SKU-health views are about real products — exclude service/delivery pseudo-SKUs
24
- # (Delivery Charges otherwise dominate movers/winners). Dot-path FILTER works (groupby
25
- # on the dot-path does not). Validation reconciles on this same non-service universe.
26
- _NO_SVC = [('product_id.type', '!=', 'service')]
27
-
28
-
29
- @lru_cache(maxsize=1)
30
- def _code_map():
31
- """product_id → SKU code. Multiple product records can share a default_code (re-SKUing
32
- /duplicates); keying by code merges them so a re-coded item doesn't read as a fake
33
- decliner + fake riser. Products without a code fall back to a per-id key. Archived
34
- products are INCLUDED — re-SKUing typically archives the old record and creates a new
35
- one under the same code, and the old record still carries last-year sales."""
36
- prods = O.search_read('product.product', [('active', 'in', [True, False])], ['id', 'default_code'],
37
- limit=50000)
38
- return {p['id']: (str(p['default_code']).strip() if p.get('default_code') else f"pid:{p['id']}")
39
- for p in prods}
40
-
41
-
42
- def _sku_rev(date_from, date_to, team_id=None):
43
- """{sku_code: {'name','rev','qty','orders'}} over a window (services excluded,
44
- duplicate product records merged by SKU code)."""
45
- g = O.read_group('sale.order.line', O.sale_line_domain(date_from, date_to, team_id, extra=_NO_SVC),
46
- ['price_subtotal:sum', 'product_uom_qty:sum'], ['product_id'], lazy=False)
47
- codes = _code_map()
48
- out = {}
49
- for r in g:
50
- pid = O.m2o_id(r.get('product_id'))
51
- if not pid:
52
- continue
53
- key = codes.get(pid, f"pid:{pid}")
54
- e = out.setdefault(key, {'name': O.m2o_name(r.get('product_id')),
55
- 'rev': 0.0, 'qty': 0.0, 'orders': 0})
56
- e['rev'] += r.get('price_subtotal') or 0.0
57
- e['qty'] += r.get('product_uom_qty') or 0.0
58
- e['orders'] += r.get('__count') or 0
59
- return out
60
-
61
-
62
- def yoy_movers(t=None, limit=20, team_id=None):
63
- """Top SKU risers and decliners by YTD-vs-same-period-LY revenue change."""
64
- t = t or P.today()
65
- yf, yt = P.ytd(t)
66
- lf, lt = P.ytd_last_year(t)
67
- this = _sku_rev(yf, yt, team_id)
68
- last = _sku_rev(lf, lt, team_id)
69
- rows = []
70
- for pid in set(this) | set(last):
71
- tr = this.get(pid, {'rev': 0.0, 'name': last.get(pid, {}).get('name', '')})
72
- lr = last.get(pid, {'rev': 0.0})
73
- name = this.get(pid, {}).get('name') or last.get(pid, {}).get('name') or ''
74
- rows.append({'code': pid, 'product': name, 'rev_ytd': tr['rev'], 'rev_ly': lr['rev'],
75
- 'change': tr['rev'] - lr['rev']})
76
- risers = sorted([r for r in rows if r['change'] > 0], key=lambda x: -x['change'])[:limit]
77
- decliners = sorted([r for r in rows if r['change'] < 0], key=lambda x: x['change'])[:limit]
78
- return {'risers': risers, 'decliners': decliners}
79
-
80
-
81
- def zombie_skus(t=None, limit=30, min_prior=1500.0, team_id=None):
82
- """Catalog rot: SKUs that sold materially last year but are ~dead this year."""
83
- t = t or P.today()
84
- yf, yt = P.ytd(t)
85
- lf, lt = P.ytd_last_year(t)
86
- this = _sku_rev(yf, yt, team_id)
87
- last = _sku_rev(lf, lt, team_id)
88
- rows = []
89
- for pid, lr in last.items():
90
- if lr['rev'] < min_prior:
91
- continue
92
- tr = this.get(pid, {'rev': 0.0})
93
- if tr['rev'] > 0.05 * lr['rev']: # still selling at >5% of prior → not a zombie
94
- continue
95
- rows.append({'code': pid, 'product': lr['name'], 'rev_ly': lr['rev'], 'rev_ytd': tr['rev'],
96
- 'lost': lr['rev'] - tr['rev']})
97
- rows.sort(key=lambda x: -x['lost'])
98
- return rows[:limit]
99
-
100
-
101
- def new_winners(t=None, limit=20, min_this=1500.0, team_id=None):
102
- """SKUs that barely sold last year but are selling well this year."""
103
- t = t or P.today()
104
- yf, yt = P.ytd(t)
105
- lf, lt = P.ytd_last_year(t)
106
- this = _sku_rev(yf, yt, team_id)
107
- last = _sku_rev(lf, lt, team_id)
108
- rows = []
109
- for pid, tr in this.items():
110
- if tr['rev'] < min_this:
111
- continue
112
- lr = last.get(pid, {'rev': 0.0})
113
- if lr['rev'] > 0.05 * tr['rev']:
114
- continue
115
- rows.append({'code': pid, 'product': tr['name'], 'rev_ytd': tr['rev'], 'rev_ly': lr['rev'],
116
- 'gained': tr['rev'] - lr['rev']})
117
- rows.sort(key=lambda x: -x['gained'])
118
- return rows[:limit]
119
-
120
-
121
- def velocity_leaders(t=None, limit=25, team_id=None):
122
- """Top SKUs by LTM unit velocity (units/month) and revenue."""
123
- t = t or P.today()
124
- lf, lt = P.ltm(t)
125
- sku = _sku_rev(lf, lt, team_id)
126
- rows = [{'code': k, 'product': v['name'], 'units_ltm': v['qty'], 'units_per_mo': v['qty'] / 12.0,
127
- 'rev_ltm': v['rev'], 'orders_ltm': v['orders']} for k, v in sku.items()]
128
- rows.sort(key=lambda x: -x['units_ltm'])
129
- return rows[:limit]
130
-
131
-
132
- def _code_category():
133
- """code -> category name. Built from _code_map (pid->code) + the sales category map
134
- (pid->category); the first record carrying a code sets that code's category."""
135
- codes = _code_map()
136
- cats = sales_mod._product_cat()
137
- out = {}
138
- for pid, code in codes.items():
139
- if code not in out and pid in cats:
140
- out[code] = cats[pid]
141
- return out
142
-
143
-
144
- def directory(t=None, team_id=None):
145
- """Every SKU that sold this YTD or the same period last year, with the fields the SKU
146
- drill-down filters/sorts on: category, YTD/LY revenue, YoY %, units and orders. Mirrors the
147
- customer directory — the full list (filtering happens in the UI), sorted by YTD revenue."""
148
- t = t or P.today()
149
- yf, yt = P.ytd(t)
150
- lf, lt = P.ytd_last_year(t)
151
- this = _sku_rev(yf, yt, team_id)
152
- last = _sku_rev(lf, lt, team_id)
153
- code_cat = _code_category()
154
- rows = []
155
- for code in set(this) | set(last):
156
- tr = this.get(code, {})
157
- lr = last.get(code, {})
158
- rev_ytd = tr.get('rev', 0.0)
159
- rev_ly = lr.get('rev', 0.0)
160
- rows.append({'code': code, 'product': tr.get('name') or lr.get('name') or code,
161
- 'category': code_cat.get(code, '(uncategorized)'),
162
- 'rev_ytd': rev_ytd, 'rev_ly': rev_ly, 'change': rev_ytd - rev_ly,
163
- 'yoy_pct': P.yoy_pct(rev_ytd, rev_ly),
164
- 'qty_ytd': tr.get('qty', 0.0), 'orders_ytd': tr.get('orders', 0)})
165
- rows.sort(key=lambda r: -r['rev_ytd'])
166
- return rows
167
-
168
-
169
- def catalogue():
170
- """`{sku_code: {'product', 'category', 'id'}}` — EVERY ACTIVE product, sold or not.
171
-
172
- ⛔ THIS IS THE CATALOGUE UNIVERSE, AND IT IS DELIBERATELY NOT `directory()`. `directory()`'s
173
- row set IS the union of two revenue `read_group`s over `sale.order.line` (`:155`), so a SKU
174
- that never sold cannot exist in it. That is CORRECT for its own callers — `yoy_movers`,
175
- `zombie_skus`, `new_winners` and `categories` all legitimately want a sales-window universe —
176
- and it is wrong for the PRODUCT GRID, which is a catalogue and was therefore showing 2,717 of
177
- 5,875 SKUs (wave 29, owner item 22 / ruling R12). The fix is this function plus a LEFT JOIN in
178
- the consumer, never a window removed from `directory()`: removing the window alone lands at
179
- 3,327 (all-time-sold), because ~2,550 active SKUs have never sold in wholesale scope at all.
180
-
181
- ⚠ NOT BU-SHAPED, and it cannot be: `product.product` carries no team. A catalogue is one
182
- catalogue. `directory()` stays the BU-shaped half, which is why the two are joined rather than
183
- merged — a scoped caller gets every SKU with ITS OWN revenue, blank where that BU never sold.
184
-
185
- Keyed exactly like `_code_map()` — the `default_code`, or a `pid:N` fallback for the 33 active
186
- records that carry none — so the join against `directory()` is code-for-code with no
187
- normaliser. Names come from `display_name` (`[CODE] NAME`), which is what `O.m2o_name` yields
188
- off a sale line, so a never-sold row wears the same format as a sold one.
189
-
190
- ⛔ RAISES on a truncated read rather than returning a short catalogue. A silently short pull
191
- would put the grid back at a plausible wrong number with every gate green — the exact failure
192
- this function exists to end ([[no-unverifiable-aggregates]], and the same truncation guard
193
- `modules/backorders.py:161` already uses).
194
- """
195
- dom = [('active', '=', True)]
196
- prods = O.search_read('product.product', dom, ['id', 'default_code', 'display_name', 'name'],
197
- limit=50000)
198
- n = O.get_odoo().search_count('product.product', dom)
199
- if len(prods) != n:
200
- raise ValueError(
201
- f"products.catalogue: the product pull is TRUNCATED — read {len(prods)} rows against "
202
- f"a search_count of {n}. A short catalogue renders as a plausible smaller grid with "
203
- f"nothing reporting it; raise the limit before shipping this.")
204
- code_cat = _code_category()
205
- out = {}
206
- for p in prods:
207
- code = (str(p['default_code']).strip() if p.get('default_code') else f"pid:{p['id']}")
208
- # First record wins, matching `_code_category`'s own convention. MEASURED 2026-08-11: zero
209
- # active products share a code, so this branch is a guard, not a merge policy.
210
- # ⭐⭐ W33-T43 (R2 / amendment A2) — `id` RIDES THE ROW. Odoo's `product.product` id is
211
- # already in hand (the `pid:{id}` fallback above uses it and then threw it away), and it is
212
- # the ONE column `ut_odoo_products` had that `product_data` lacked. R2 merges onto the
213
- # legacy key and keeps every data column, so retiring the twin without this would lose a
214
- # real identifier rather than a navigational link.
215
- # ⚠ A product's grid pid is `crc32(default_code)`, NOT the Odoo id — unlike a CUSTOMER row,
216
- # whose pid IS the partner id. That asymmetry is exactly why this cannot be derived
217
- # downstream in `aios_grid.py` and has to be carried from the source read.
218
- out.setdefault(code, {'product': p.get('display_name') or p.get('name') or code,
219
- 'category': code_cat.get(code, '(uncategorized)'),
220
- 'id': p['id']})
221
- return out
222
-
223
-
224
- #: ⭐⭐ WAVE 30 / W30-T34 (the carried W29-T52) — THE PRICELIST STRATUM, PER LIST.
225
- #:
226
- #: MEASURED LIVE 2026-08-12, and every one of these numbers shaped the design rather than
227
- #: decorating it:
228
- #:
229
- #: · **Five pricelists, three of them material.** Date-valid fixed rules: Fisch 5,139,
230
- #: Royal 1 2,702, Royal 2 2,605, Public Pricelist 17, Giftware Deals 1. At SKU level that
231
- #: is Fisch 3,971 · Royal 1 1,920 · Royal 2 1,857 · Public 6 · Giftware 0.
232
- #: · **3,988 of 5,871 active products carry a usable fixed rule; 1,883 carry NONE.**
233
- #: · **1,853 SKUs are priced on all three lists AND THE LISTS DISAGREE** — median relative
234
- #: spread 16%, max 82%. That is why this is three columns rather than one "the price" or a
235
- #: low/high pair: neither can tell a Fisch rep what Fisch sells the SKU for, which is the
236
- #: only question the column exists to answer.
237
- #: · ⛔ **`list_price` is 1.00 on 5,817 of the 5,871** — so the single `3_global` rule
238
- #: computed over it is meaningless, exactly as W29-T52 said. **This reader never falls
239
- #: through to it**, and that is the ticket's own negative control: a SKU with no specific
240
- #: item renders BLANK, never a fallback dressed as a price.
241
- #:
242
- #: ⚠ RESOLVED BY NAME, NOT BY ID. A pricelist renamed or deleted in Odoo makes its column go
243
- #: blank and makes `report["lists_missing"]` name it — a hard-coded id would keep pointing at
244
- #: whatever inherited it and mislabel every cell silently.
245
- PRICELIST_COLUMNS = (
246
- ("price_fisch", "Fisch"),
247
- ("price_royal_1", "Royal 1"),
248
- ("price_royal_2", "Royal 2"),
249
- )
250
-
251
-
252
- def pricelist_by_code():
253
- """`({code: {column_key: price}}, report)` — the date-valid FIXED base-tier price for each
254
- declared pricelist, keyed EXACTLY as `catalogue()` keys its rows.
255
-
256
- ⛔ THE KEYING IS LOAD-BEARING, not a detail. `catalogue()` keys on `default_code` with a
257
- `pid:{id}` fallback for the ~33 active records that carry none. Keying this map any other
258
- way would leave those SKUs permanently blank while an oracle counting active products
259
- counted them — a red gate on a working build, or worse, a silent hole nobody counts.
260
-
261
- **The base tier, deliberately.** `pricecomp._tier_for` picks the highest `min_quantity` at
262
- or below an order's quantity, because it is pricing a LINE that has one. A catalogue column
263
- has no quantity in hand, so it takes the LOWEST `min_quantity` — the price at qty 1. Rules
264
- above that break are a bulk price, and `report["qty_break_only"]` counts the SKUs whose only
265
- rule sits on one (MEASURED: 10 rules of 10,464 carry a break at all).
266
-
267
- **Variant rules beat template rules**, matching `pricecomp._tier_for`: a `0_product_variant`
268
- rule is the more specific statement about this exact SKU.
269
-
270
- ⚠ DEGRADES TO `({}, report)` on a read failure, matching `_inventory_by_code` rather than
271
- `catalogue()`: these are COLUMNS, and a product grid that will not render because pricing is
272
- momentarily unreachable is a worse failure than one with blank price columns. The blanks are
273
- not silent — `product_data.validate()`'s coverage leg reconciles each column against a fresh
274
- Odoo count and goes red at zero.
275
-
276
- ⭐ **The report exists because R6's second sentence is law** (*"if it can't be done, you need
277
- to explicitly tell me why and recommend a fix"*). Everything this reader CANNOT see is
278
- counted rather than dropped: rules on pricelists the contract does not declare, non-fixed
279
- (`formula`/`percent`) rules, the `3_global` fallback, and prices that only exist above a
280
- quantity break.
281
- """
282
- # ⚠ THE COUNTERS ARE DISJOINT AND ORDER-DEPENDENT, and saying so is the difference between a
283
- # report and a misleading one. A rule is classified ONCE, by the first reason it is skipped:
284
- # undeclared list → out of date → not fixed → global → zero price. So `rules_not_fixed: 0`
285
- # means "no formula rule on a list we declare", NOT "this Odoo has no formula rules"
286
- # (MEASURED 2026-08-12: it has exactly one, and it sits on Public Pricelist, which the
287
- # contract does not declare — so it lands in `rules_undeclared_list`).
288
- report = {"lists_missing": [], "rules_total": 0, "rules_undeclared_list": 0,
289
- "rules_not_fixed": 0, "rules_global": 0, "rules_out_of_date": 0,
290
- "rules_zero_price": 0, "qty_break_only": 0}
291
- try:
292
- pl_rows = O.search_read('product.pricelist', [], ['id', 'name'])
293
- by_name = {}
294
- for p in pl_rows:
295
- by_name.setdefault(str(p.get('name') or '').strip(), p['id'])
296
- wanted = {}
297
- for col, name in PRICELIST_COLUMNS:
298
- pid = by_name.get(name)
299
- if pid is None:
300
- report["lists_missing"].append(name)
301
- else:
302
- wanted[pid] = col
303
-
304
- # ⭐⭐ FILTERED SERVER-SIDE, AND THE REASON IS A MEASUREMENT, NOT A STYLE PREFERENCE.
305
- # Reading all 10,464 rules with 9 fields and sorting them in Python costs **30.1s** on
306
- # this connection; the same rules under a server-side domain with 6 fields cost
307
- # **13.6s** (measured 2026-08-12, back to back). `pool()` runs this on a scope's
308
- # first-ever build, so that 16.5s is 16.5s of somebody's page load.
309
- #
310
- # ⚠ The domain reproduces the Python predicate EXACTLY, and the null legs are the part
311
- # that is easy to get wrong: an absent `date_start` is `False`, not a past date, so
312
- # `('date_start','<=',today)` ALONE would drop every open-ended rule — which is almost
313
- # all of them.
314
- today = P.today().isoformat()
315
- _declared = sorted(wanted)
316
- _kinds = ['0_product_variant', '1_product']
317
- _live = [('pricelist_id', 'in', _declared), ('compute_price', '=', 'fixed'),
318
- ('applied_on', 'in', _kinds),
319
- '|', ('date_start', '=', False), ('date_start', '<=', today),
320
- '|', ('date_end', '=', False), ('date_end', '>=', today),
321
- ('fixed_price', '>', 0)]
322
- rules = O.search_read(
323
- 'product.pricelist.item', _live,
324
- ['pricelist_id', 'product_id', 'product_tmpl_id', 'applied_on', 'fixed_price',
325
- 'min_quantity']) if _declared else []
326
-
327
- # ⭐ R6's SECOND SENTENCE, PAID FOR WITH `search_count` RATHER THAN A WIDER READ. Six
328
- # counts cost ~1.2s together; the rules they count would cost 16s to read. What this
329
- # reader cannot see is still REPORTED — it is just no longer transferred.
330
- def _n(extra):
331
- try:
332
- return O.get_odoo().search_count('product.pricelist.item', extra)
333
- except Exception:
334
- return -1 # -1 reads as "not measured", never as zero
335
- _dated = ['|', ('date_start', '=', False), ('date_start', '<=', today),
336
- '|', ('date_end', '=', False), ('date_end', '>=', today)]
337
- report["rules_total"] = _n([])
338
- report["rules_undeclared_list"] = _n([('pricelist_id', 'not in', _declared)]) \
339
- if _declared else report["rules_total"]
340
- if _declared:
341
- _on = [('pricelist_id', 'in', _declared)]
342
- # `percent_price` and formula rules are read NOWHERE in this repo. A rule we cannot
343
- # price is one the operator is TOLD about, never one that quietly becomes a blank.
344
- report["rules_not_fixed"] = _n(_on + [('compute_price', '!=', 'fixed')])
345
- report["rules_global"] = _n(_on + [('compute_price', '=', 'fixed'),
346
- ('applied_on', 'not in', _kinds)])
347
- report["rules_out_of_date"] = (
348
- _n(_on + [('compute_price', '=', 'fixed'), ('applied_on', 'in', _kinds)])
349
- - _n(_on + [('compute_price', '=', 'fixed'),
350
- ('applied_on', 'in', _kinds)] + _dated))
351
- # A zero price is not "free" — it is an unset rule. Blank says so; 0 does not.
352
- report["rules_zero_price"] = _n(
353
- _on + [('compute_price', '=', 'fixed'), ('applied_on', 'in', _kinds)]
354
- + _dated + [('fixed_price', '<=', 0)])
355
-
356
- by_var, by_tmpl = {}, {}
357
- for r in rules:
358
- col = wanted[O.m2o_id(r.get('pricelist_id'))]
359
- if r.get('applied_on') == '0_product_variant' and r.get('product_id'):
360
- by_var.setdefault((col, O.m2o_id(r['product_id'])), []).append(r)
361
- elif r.get('product_tmpl_id'):
362
- by_tmpl.setdefault((col, O.m2o_id(r['product_tmpl_id'])), []).append(r)
363
-
364
- dom = [('active', '=', True)]
365
- prods = O.search_read('product.product', dom,
366
- ['id', 'default_code', 'product_tmpl_id'], limit=50000)
367
- n = O.get_odoo().search_count('product.product', dom)
368
- if len(prods) != n:
369
- # Same guard, same reason as `catalogue()`: a short pull renders as a plausible
370
- # smaller set of priced SKUs with nothing reporting it.
371
- raise ValueError(
372
- f"products.pricelist_by_code: the product pull is TRUNCATED — read {len(prods)} "
373
- f"rows against a search_count of {n}.")
374
- except Exception as e:
375
- report["error"] = f"{type(e).__name__}: {str(e)[:200]}"
376
- return {}, report
377
-
378
- out = {}
379
- for p in prods:
380
- code = (str(p['default_code']).strip() if p.get('default_code') else f"pid:{p['id']}")
381
- tmpl = O.m2o_id(p.get('product_tmpl_id'))
382
- cells = {}
383
- for col, _name in PRICELIST_COLUMNS:
384
- cands = by_var.get((col, p['id'])) or by_tmpl.get((col, tmpl))
385
- if not cands:
386
- continue
387
- base = min(cands, key=lambda r: r.get('min_quantity') or 0.0)
388
- if (base.get('min_quantity') or 0.0) > 1.0:
389
- report["qty_break_only"] += 1
390
- cells[col] = base.get('fixed_price')
391
- if cells:
392
- out.setdefault(code, {}).update(cells)
393
- return out, report
394
-
395
-
396
- def catalogue_count():
397
- """The INDEPENDENT population oracle: Odoo's own count of active products.
398
-
399
- Deliberately a bare `search_count` and not a `len()` over anything this module built — the
400
- 2,717 defect shipped silently for a wave because `product_data.validate()` derived BOTH sides
401
- of its reconciliation from `_sku_rev`, so the oracle could never see a missing row.
402
- """
403
- return O.get_odoo().search_count('product.product', [('active', '=', True)])
404
-
405
-
406
- def categories(t=None, team_id=None):
407
- """Sorted distinct category names present in the SKU directory (for the drill-down filter)."""
408
- return sorted({r['category'] for r in directory(t, team_id)})
409
-
410
-
411
- def _coverage(date_from, date_to, team_id=None):
412
- """{sku_code: distinct_customer_count} via 2-level read_group (slow-ish), merged by code.
413
- (A customer buying two records sharing a code can count twice; duplicates are rare, and
414
- code-level avoids the bigger error of a re-SKUed product reading as full coverage loss.)"""
415
- g = O.read_group('sale.order.line', O.sale_line_domain(date_from, date_to, team_id, extra=_NO_SVC),
416
- ['__count'], ['product_id', 'order_partner_id'], lazy=False)
417
- codes = _code_map()
418
- cov = {}
419
- names = {}
420
- for r in g:
421
- pid = O.m2o_id(r.get('product_id'))
422
- if not pid:
423
- continue
424
- key = codes.get(pid, f"pid:{pid}")
425
- cov[key] = cov.get(key, 0) + 1
426
- names.setdefault(key, O.m2o_name(r.get('product_id')))
427
- return cov, names
428
-
429
-
430
- def coverage_collapse(t=None, limit=25, min_prior_custs=8, team_id=None):
431
- """SKUs that lost the most customer breadth YoY — early warning a SKU is dying even if
432
- revenue hasn't fully cratered yet."""
433
- t = t or P.today()
434
- yf, yt = P.ytd(t)
435
- lf, lt = P.ytd_last_year(t)
436
- cov_t, names_t = _coverage(yf, yt, team_id)
437
- cov_l, names_l = _coverage(lf, lt, team_id)
438
- rows = []
439
- for pid, lc in cov_l.items():
440
- if lc < min_prior_custs:
441
- continue
442
- tc = cov_t.get(pid, 0)
443
- drop = lc - tc
444
- if drop <= 0:
445
- continue
446
- rows.append({'code': pid, 'product': names_l.get(pid) or names_t.get(pid) or '',
447
- 'custs_ly': lc, 'custs_ytd': tc, 'lost_custs': drop,
448
- 'pct_drop': drop / lc * 100})
449
- rows.sort(key=lambda x: (-x['lost_custs'], -x['pct_drop']))
450
- return rows[:limit]
451
-
452
-
453
- # ====================================================================== SKU DRAWER (mirror of customer drawer)
454
- def _sku_product_ids(code):
455
- """All product.product ids sharing this SKU code (merged variants / archived records)."""
456
- return [pid for pid, c in _code_map().items() if c == code]
457
-
458
-
459
- def _sku_dom(pids, date_from, date_to, team_id=None):
460
- return O.sale_line_domain(date_from, date_to, team_id, extra=[('product_id', 'in', pids)] + _NO_SVC)
461
-
462
-
463
- def _sku_name_category(pids):
464
- rows = O.search_read('product.product', [('id', 'in', pids)], ['name', 'categ_id'])
465
- name = rows[0]['name'] if rows else '(unknown)'
466
- catmap = sales_mod._product_cat()
467
- main = next((catmap.get(p) for p in pids if catmap.get(p)), '(uncategorized)')
468
- return name, main
469
-
470
-
471
- def _sku_buyers(pids, date_from, date_to, team_id=None):
472
- """{partner_id: {'name','rev','qty'}} for buyers of this SKU over a window."""
473
- g = O.read_group('sale.order.line', _sku_dom(pids, date_from, date_to, team_id),
474
- ['price_subtotal:sum', 'product_uom_qty:sum'], ['order_partner_id'], lazy=False)
475
- out = {}
476
- for r in g:
477
- pid = O.m2o_id(r.get('order_partner_id'))
478
- if pid:
479
- out[pid] = {'name': O.m2o_name(r.get('order_partner_id')),
480
- 'rev': r.get('price_subtotal') or 0.0, 'qty': r.get('product_uom_qty') or 0.0}
481
- return out
482
-
483
-
484
- def sku_detail(code, t=None, team_id=None, n_months=13, allsku=None):
485
- """KPIs (rev/qty/buyers YoY, GM%), monthly trend, rank & % of BU for one SKU code.
486
- Pass `allsku` (a cached _sku_rev YTD map) to skip the ~all-SKU rank read."""
487
- t = t or P.today()
488
- pids = _sku_product_ids(code)
489
- if not pids:
490
- return None
491
- yf, yt = P.ytd(t)
492
- lf, lt = P.ytd_last_year(t)
493
- mf, mt = P.ltm(t)
494
- name, category = _sku_name_category(pids)
495
-
496
- def s(df, dtt, field='price_subtotal'):
497
- return O.sum_field('sale.order.line', _sku_dom(pids, df, dtt, team_id), field)
498
- rev_ytd, rev_ly = s(yf, yt), s(lf, lt)
499
- qty_ytd, qty_ly = s(yf, yt, 'product_uom_qty'), s(lf, lt, 'product_uom_qty')
500
- buyers_ytd = len(_sku_buyers(pids, yf, yt, team_id))
501
- buyers_ly = len(_sku_buyers(pids, lf, lt, team_id))
502
- g = O.read_group('sale.order.line', _sku_dom(pids, mf, mt, team_id),
503
- ['price_subtotal:sum', 'margin:sum'], [], lazy=False)
504
- line_rev = (g[0].get('price_subtotal') if g else 0) or 0.0
505
- margin = (g[0].get('margin') if g else 0) or 0.0
506
-
507
- # Monthly trend: bin lines by their order's month in Python (dot-path month groupby is rejected
508
- # on sale.order.line), in 2 reads instead of 26 point queries.
509
- range_start = dt.date(t.year - 2, t.month, 1).isoformat()
510
- lines = O.search_read('sale.order.line', _sku_dom(pids, range_start, t.isoformat(), team_id),
511
- ['price_subtotal', 'order_id'])
512
- oids = list({O.m2o_id(line['order_id']) for line in lines if line.get('order_id')})
513
- odate = {}
514
- for i in range(0, len(oids), 1000):
515
- for o in O.search_read('sale.order', [('id', 'in', oids[i:i + 1000])], ['date_order']):
516
- if o.get('date_order'):
517
- odate[o['id']] = str(o['date_order'])[:7]
518
- mrev = {}
519
- for line in lines:
520
- ym = odate.get(O.m2o_id(line.get('order_id')))
521
- if ym:
522
- mrev[ym] = mrev.get(ym, 0.0) + (line.get('price_subtotal') or 0.0)
523
- monthly = []
524
- for ym, start, end in P.month_starts(n_months, t):
525
- y, m = int(ym[:4]) - 1, int(ym[5:7])
526
- monthly.append({'month': ym, 'revenue': mrev.get(ym, 0.0),
527
- 'revenue_ly': mrev.get(f'{y:04d}-{m:02d}', 0.0)})
528
-
529
- allsku = allsku if allsku is not None else _sku_rev(yf, yt, team_id)
530
- total = sum(v['rev'] for v in allsku.values()) or 1.0
531
- rank = next((i + 1 for i, (c, _v) in enumerate(sorted(allsku.items(), key=lambda kv: -kv[1]['rev']))
532
- if c == code), None)
533
- return {
534
- 'code': code, 'name': name, 'category': category,
535
- 'rev_ytd': rev_ytd, 'rev_ly': rev_ly, 'rev_yoy_pct': P.yoy_pct(rev_ytd, rev_ly),
536
- 'qty_ytd': qty_ytd, 'qty_ly': qty_ly, 'qty_yoy_pct': P.yoy_pct(qty_ytd, qty_ly),
537
- 'buyers_ytd': buyers_ytd, 'buyers_ly': buyers_ly, 'buyers_delta': buyers_ytd - buyers_ly,
538
- 'gm_pct': (margin / line_rev * 100) if line_rev else 0.0, 'gm_dollars': margin,
539
- 'rank': rank, 'n_skus': len(allsku), 'pct_of_bu': rev_ytd / total * 100,
540
- 'monthly': monthly,
541
- }
542
-
543
-
544
- def sku_buyer_bridge(code, t=None, team_id=None, top=12):
545
- """Who drives the SKU's YoY: retained / new / churned buyers + a $-ranked churned call list."""
546
- t = t or P.today()
547
- pids = _sku_product_ids(code)
548
- yf, yt = P.ytd(t)
549
- lf, lt = P.ytd_last_year(t)
550
- this, last = _sku_buyers(pids, yf, yt, team_id), _sku_buyers(pids, lf, lt, team_id)
551
- tset, lset = set(this), set(last)
552
- retained, new, churned = tset & lset, tset - lset, lset - tset
553
- churned_list = sorted([{'pid': p, 'customer': last[p]['name'], 'ly_rev': last[p]['rev']}
554
- for p in churned], key=lambda x: -x['ly_rev'])[:top]
555
- return {'retained': {'n': len(retained), 'rev': sum(this[p]['rev'] for p in retained)},
556
- 'new': {'n': len(new), 'rev': sum(this[p]['rev'] for p in new)},
557
- 'churned': {'n': len(churned), 'rev': sum(last[p]['rev'] for p in churned)},
558
- 'buyer_retention_pct': (len(retained) / len(lset) * 100) if lset else 0.0,
559
- 'churned_buyers': churned_list, 'buyers_this': len(tset), 'buyers_last': len(lset),
560
- 'this_total': sum(v['rev'] for v in this.values())}
561
-
562
-
563
- def sku_concentration(code, t=None, team_id=None):
564
- """Buyer-concentration risk: top-1/top-3 share, Herfindahl index, effective buyer count (LTM)."""
565
- t = t or P.today()
566
- mf, mt = P.ltm(t)
567
- buyers = _sku_buyers(_sku_product_ids(code), mf, mt, team_id)
568
- revs = sorted([v['rev'] for v in buyers.values()], reverse=True)
569
- total = sum(revs) or 1.0
570
- hhi = sum((r / total) ** 2 for r in revs)
571
- return {'n_buyers': len(revs), 'top1_pct': (revs[0] / total * 100) if revs else 0.0,
572
- 'top3_pct': (sum(revs[:3]) / total * 100) if revs else 0.0,
573
- 'hhi': hhi, 'eff_buyers': (1 / hhi) if hhi else 0.0}
574
-
575
-
576
- def sku_price_dispersion(code, t=None, team_id=None, cap=60):
577
- """Realized $/unit per buyer (LTM) vs the volume-weighted average; recoverable $ on below-VWAP
578
- accounts. list_price is unreliable, so the dispersion among actual buyers is the margin lever."""
579
- t = t or P.today()
580
- mf, mt = P.ltm(t)
581
- g = O.read_group('sale.order.line', _sku_dom(_sku_product_ids(code), mf, mt, team_id),
582
- ['price_subtotal:sum', 'product_uom_qty:sum'], ['order_partner_id'], lazy=False)
583
- rows, tot_rev, tot_qty = [], 0.0, 0.0
584
- for r in g:
585
- pid = O.m2o_id(r.get('order_partner_id'))
586
- rev = r.get('price_subtotal') or 0.0
587
- qty = r.get('product_uom_qty') or 0.0
588
- if not pid or qty <= 0:
589
- continue
590
- rows.append({'pid': pid, 'customer': O.m2o_name(r.get('order_partner_id')),
591
- 'price': rev / qty, 'qty': qty, 'rev': rev})
592
- tot_rev += rev
593
- tot_qty += qty
594
- vwap = (tot_rev / tot_qty) if tot_qty else 0.0
595
- for r in rows:
596
- r['recoverable'] = max(0.0, vwap - r['price']) * r['qty']
597
- rows.sort(key=lambda x: -x['recoverable'])
598
- return {'vwap': vwap, 'n_buyers': len(rows),
599
- 'recoverable_total': sum(r['recoverable'] for r in rows), 'rows': rows[:cap]}
600
-
601
-
602
- def sku_top_buyers(code, t=None, team_id=None, top=15):
603
- """Ranked buyers of this SKU (YTD) with YoY — clickable to open the customer drawer."""
604
- t = t or P.today()
605
- pids = _sku_product_ids(code)
606
- yf, yt = P.ytd(t)
607
- lf, lt = P.ytd_last_year(t)
608
- this, last = _sku_buyers(pids, yf, yt, team_id), _sku_buyers(pids, lf, lt, team_id)
609
- rows = [{'pid': p, 'customer': v['name'], 'rev': v['rev'], 'qty': v['qty'],
610
- 'yoy_pct': P.yoy_pct(v['rev'], last.get(p, {}).get('rev', 0.0))} for p, v in this.items()]
611
- rows.sort(key=lambda x: -x['rev'])
612
- return rows[:top]
613
-
614
-
615
- _BUYER_TIERS = [('Whale (≥$25k)', 25000.0), ('Large ($10–25k)', 10000.0),
616
- ('Mid ($2–10k)', 2000.0), ('Small (<$2k)', 0.0)]
617
-
618
-
619
- def sku_customer_analysis(code, t=None, team_id=None, top=15):
620
- """WHO buys this SKU (LTM), as customers: the value-tier mix of its buyers (by each buyer's TOTAL
621
- spend), and the accounts most DEPENDENT on it (this SKU as a share of their spend — who gets hurt
622
- most if it stocks out / who to protect)."""
623
- t = t or P.today()
624
- lf, lt = P.ltm(t)
625
- pids = _sku_product_ids(code)
626
- empty = {'segments': [], 'dependency': [], 'n_buyers': 0, 'avg_dependency': 0.0}
627
- if not pids:
628
- return empty
629
- buyers = _sku_buyers(pids, lf, lt, team_id) # {pid: {name, rev(this SKU), qty}}
630
- if not buyers:
631
- return empty
632
- bpids = list(buyers)
633
- g = O.read_group('sale.order', sales_mod.order_domain(lf, lt, team_id) + [('partner_id', 'in', bpids)],
634
- ['amount_untaxed:sum'], ['partner_id'], lazy=False)
635
- total = {O.m2o_id(r['partner_id']): (r.get('amount_untaxed') or 0.0) for r in g if r.get('partner_id')}
636
-
637
- def tier(rev):
638
- for nm, lo in _BUYER_TIERS:
639
- if rev >= lo:
640
- return nm
641
- return _BUYER_TIERS[-1][0]
642
- seg = {nm: {'tier': nm, 'buyers': 0, 'sku_rev': 0.0} for nm, _ in _BUYER_TIERS}
643
- dep = []
644
- for p, v in buyers.items():
645
- ct = total.get(p, v['rev']) or v['rev']
646
- s = seg[tier(ct)]
647
- s['buyers'] += 1
648
- s['sku_rev'] += v['rev']
649
- dep.append({'customer': v['name'], 'pid': p, 'sku_rev': v['rev'], 'cust_total': ct,
650
- 'dependency_pct': (v['rev'] / ct * 100) if ct else None})
651
- tot = sum(s['sku_rev'] for s in seg.values()) or 1.0
652
- segments = []
653
- for nm, _ in _BUYER_TIERS:
654
- s = seg[nm]
655
- s['rev_share'] = s['sku_rev'] / tot * 100
656
- s['avg_per_buyer'] = (s['sku_rev'] / s['buyers']) if s['buyers'] else 0.0
657
- segments.append(s)
658
- dep.sort(key=lambda x: -x['sku_rev'])
659
- deps_known = [d['dependency_pct'] for d in dep if d['dependency_pct'] is not None]
660
- return {'segments': segments, 'dependency': dep[:top], 'n_buyers': len(buyers),
661
- 'avg_dependency': (sum(deps_known) / len(deps_known)) if deps_known else 0.0}
662
-
663
-
664
- def sku_whitespace(code, t=None, team_id=None, top=15):
665
- """Customers who buy this SKU's category but NOT this SKU — ranked prospect list."""
666
- import modules.customers as cust_mod
667
- t = t or P.today()
668
- pids = _sku_product_ids(code)
669
- _name, category = _sku_name_category(pids)
670
- cat_buyers = cust_mod.category_buyers(category, team_id=team_id) or set()
671
- mf, mt = P.ltm(t)
672
- prospects = list(cat_buyers - set(_sku_buyers(pids, mf, mt, team_id)))
673
- if not prospects:
674
- return {'category': category, 'rows': []}
675
- g = O.read_group('sale.order', sales_mod.order_domain(mf, mt, team_id) + [('partner_id', 'in', prospects)],
676
- ['amount_untaxed:sum'], ['partner_id'], lazy=False)
677
- rows = [{'pid': O.m2o_id(r['partner_id']), 'customer': O.m2o_name(r['partner_id']),
678
- 'total_spend': r.get('amount_untaxed') or 0.0} for r in g if r.get('partner_id')]
679
- rows.sort(key=lambda x: -x['total_spend'])
680
- return {'category': category, 'rows': rows[:top]}
681
-
682
-
683
- def sku_drawer_bundle(code, t=None, team_id=None, allsku=None):
684
- """The whole SKU drawer's first paint in one cached unit: sku_detail first (the not-found gate),
685
- then the other six pulls CONCURRENTLY (O.parallel). Cuts a ~6-call cold open to ~max(call)."""
686
- detail = sku_detail(code, t=t, team_id=team_id, allsku=allsku)
687
- if detail is None:
688
- return {'detail': None}
689
- bridge, conc, price, buyers, white, ca = O.parallel([
690
- lambda: sku_buyer_bridge(code, t, team_id),
691
- lambda: sku_concentration(code, t, team_id),
692
- lambda: sku_price_dispersion(code, t, team_id),
693
- lambda: sku_top_buyers(code, t, team_id),
694
- lambda: sku_whitespace(code, t, team_id),
695
- lambda: sku_customer_analysis(code, t, team_id),
696
- ])
697
- return {'detail': detail, 'bridge': bridge, 'conc': conc, 'price': price,
698
- 'buyers': buyers, 'white': white, 'ca': ca}
699
-
700
-
701
- def validate(t=None, team_id=None):
702
- """Reconcile SKU metrics to independent Odoo aggregates. When team_id is set (a single BU
703
- selected) every check runs SCOPED to that BU, so the validation panel never reconciles
704
- against — or exposes — the other BU's numbers. All checks here are BU-scopeable (no cross-BU
705
- mirror), so team_id threads straight through."""
706
- t = t or P.today()
707
- yf, yt = P.ytd(t)
708
- checks = []
709
- sku = _sku_rev(yf, yt, team_id)
710
- sku_sum = sum(v['rev'] for v in sku.values())
711
- line_total = O.sum_field('sale.order.line',
712
- O.sale_line_domain(yf, yt, team_id, extra=_NO_SVC), 'price_subtotal')
713
- checks.append({'check': 'SKU rev: Σ(per-SKU) == total line revenue, ex-services (YTD)',
714
- 'a': round(sku_sum, 2), 'b': round(line_total, 2),
715
- 'gap': round(sku_sum - line_total, 2),
716
- 'ok': abs(sku_sum - line_total) <= 1.0})
717
-
718
- m = yoy_movers(t, limit=10**9, team_id=team_id)
719
- movers_sum = sum(r['change'] for r in m['risers']) + sum(r['change'] for r in m['decliners'])
720
- lf, lt = P.ytd_last_year(t)
721
- last_total = O.sum_field('sale.order.line',
722
- O.sale_line_domain(lf, lt, team_id, extra=_NO_SVC), 'price_subtotal')
723
- checks.append({'check': 'SKU movers: Σ(Δ) == (YTD − LY) total',
724
- 'a': round(movers_sum, 2), 'b': round(line_total - last_total, 2),
725
- 'gap': round(movers_sum - (line_total - last_total), 2),
726
- 'ok': abs(movers_sum - (line_total - last_total)) <= 1.0})
727
-
728
- # SKU drawer: buyer-bridge reconciles — retained + new buyer revenue == this-period SKU revenue
729
- top_code = max(sku.items(), key=lambda kv: kv[1]['rev'])[0] if sku else None
730
- if top_code:
731
- bb = sku_buyer_bridge(top_code, t, team_id=team_id)
732
- recon = bb['retained']['rev'] + bb['new']['rev']
733
- checks.append({'check': 'SKU drawer: retained+new buyer rev == SKU YTD revenue',
734
- 'a': round(recon, 2), 'b': round(bb['this_total'], 2),
735
- 'gap': round(recon - bb['this_total'], 2),
736
- 'ok': abs(recon - bb['this_total']) <= 1.0})
737
- return checks
 
1
+ """Products / SKU module — SKU health: YoY movers (risers & decliners), zombie SKUs
2
+ (catalog rot — sellable, formerly selling, now dead), new winners, coverage collapse
3
+ (SKUs losing customer breadth — the FFS-recovery early-warning signal), and velocity leaders.
4
+
5
+ Per-SKU margin already lives in the Financial module (low_margin_skus); not duplicated here.
6
+ Basket / co-purchase (MBA) is intentionally deferred — the existing client app computes it
7
+ runtime-side, and a local co-occurrence pull over 74k LTM lines is the kind of heavy job the
8
+ project guardrails keep off the local PC. (See BACKLOG.)
9
+
10
+ All line-level (sale.order.line), RI+FFS scope, excluded accounts removed — reusing sale_line_domain.
11
+ Coverage (distinct customers per SKU) uses a 2-level read_group and is a touch slow (~15s/
12
+ window), so it's its own function the app calls lazily and caches.
13
+ """
14
+ import sys
15
+ import datetime as dt
16
+ from pathlib import Path
17
+ sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
18
+ from functools import lru_cache
19
+ import core.odoo as O
20
+ import core.periods as P
21
+ import modules.sales as sales_mod
22
+
23
+ # SKU-health views are about real products — exclude service/delivery pseudo-SKUs
24
+ # (Delivery Charges otherwise dominate movers/winners). Dot-path FILTER works (groupby
25
+ # on the dot-path does not). Validation reconciles on this same non-service universe.
26
+ _NO_SVC = [('product_id.type', '!=', 'service')]
27
+
28
+
29
+ @lru_cache(maxsize=1)
30
+ def _code_map():
31
+ """product_id → SKU code. Multiple product records can share a default_code (re-SKUing
32
+ /duplicates); keying by code merges them so a re-coded item doesn't read as a fake
33
+ decliner + fake riser. Products without a code fall back to a per-id key. Archived
34
+ products are INCLUDED — re-SKUing typically archives the old record and creates a new
35
+ one under the same code, and the old record still carries last-year sales."""
36
+ prods = O.search_read('product.product', [('active', 'in', [True, False])], ['id', 'default_code'],
37
+ limit=50000)
38
+ return {p['id']: (str(p['default_code']).strip() if p.get('default_code') else f"pid:{p['id']}")
39
+ for p in prods}
40
+
41
+
42
+ def _sku_rev(date_from, date_to, team_id=None):
43
+ """{sku_code: {'name','rev','qty','orders'}} over a window (services excluded,
44
+ duplicate product records merged by SKU code)."""
45
+ g = O.read_group('sale.order.line', O.sale_line_domain(date_from, date_to, team_id, extra=_NO_SVC),
46
+ ['price_subtotal:sum', 'product_uom_qty:sum'], ['product_id'], lazy=False)
47
+ codes = _code_map()
48
+ out = {}
49
+ for r in g:
50
+ pid = O.m2o_id(r.get('product_id'))
51
+ if not pid:
52
+ continue
53
+ key = codes.get(pid, f"pid:{pid}")
54
+ e = out.setdefault(key, {'name': O.m2o_name(r.get('product_id')),
55
+ 'rev': 0.0, 'qty': 0.0, 'orders': 0})
56
+ e['rev'] += r.get('price_subtotal') or 0.0
57
+ e['qty'] += r.get('product_uom_qty') or 0.0
58
+ e['orders'] += r.get('__count') or 0
59
+ return out
60
+
61
+
62
+ def yoy_movers(t=None, limit=20, team_id=None):
63
+ """Top SKU risers and decliners by YTD-vs-same-period-LY revenue change."""
64
+ t = t or P.today()
65
+ yf, yt = P.ytd(t)
66
+ lf, lt = P.ytd_last_year(t)
67
+ this = _sku_rev(yf, yt, team_id)
68
+ last = _sku_rev(lf, lt, team_id)
69
+ rows = []
70
+ for pid in set(this) | set(last):
71
+ tr = this.get(pid, {'rev': 0.0, 'name': last.get(pid, {}).get('name', '')})
72
+ lr = last.get(pid, {'rev': 0.0})
73
+ name = this.get(pid, {}).get('name') or last.get(pid, {}).get('name') or ''
74
+ rows.append({'code': pid, 'product': name, 'rev_ytd': tr['rev'], 'rev_ly': lr['rev'],
75
+ 'change': tr['rev'] - lr['rev']})
76
+ risers = sorted([r for r in rows if r['change'] > 0], key=lambda x: -x['change'])[:limit]
77
+ decliners = sorted([r for r in rows if r['change'] < 0], key=lambda x: x['change'])[:limit]
78
+ return {'risers': risers, 'decliners': decliners}
79
+
80
+
81
+ def zombie_skus(t=None, limit=30, min_prior=1500.0, team_id=None):
82
+ """Catalog rot: SKUs that sold materially last year but are ~dead this year."""
83
+ t = t or P.today()
84
+ yf, yt = P.ytd(t)
85
+ lf, lt = P.ytd_last_year(t)
86
+ this = _sku_rev(yf, yt, team_id)
87
+ last = _sku_rev(lf, lt, team_id)
88
+ rows = []
89
+ for pid, lr in last.items():
90
+ if lr['rev'] < min_prior:
91
+ continue
92
+ tr = this.get(pid, {'rev': 0.0})
93
+ if tr['rev'] > 0.05 * lr['rev']: # still selling at >5% of prior → not a zombie
94
+ continue
95
+ rows.append({'code': pid, 'product': lr['name'], 'rev_ly': lr['rev'], 'rev_ytd': tr['rev'],
96
+ 'lost': lr['rev'] - tr['rev']})
97
+ rows.sort(key=lambda x: -x['lost'])
98
+ return rows[:limit]
99
+
100
+
101
+ def new_winners(t=None, limit=20, min_this=1500.0, team_id=None):
102
+ """SKUs that barely sold last year but are selling well this year."""
103
+ t = t or P.today()
104
+ yf, yt = P.ytd(t)
105
+ lf, lt = P.ytd_last_year(t)
106
+ this = _sku_rev(yf, yt, team_id)
107
+ last = _sku_rev(lf, lt, team_id)
108
+ rows = []
109
+ for pid, tr in this.items():
110
+ if tr['rev'] < min_this:
111
+ continue
112
+ lr = last.get(pid, {'rev': 0.0})
113
+ if lr['rev'] > 0.05 * tr['rev']:
114
+ continue
115
+ rows.append({'code': pid, 'product': tr['name'], 'rev_ytd': tr['rev'], 'rev_ly': lr['rev'],
116
+ 'gained': tr['rev'] - lr['rev']})
117
+ rows.sort(key=lambda x: -x['gained'])
118
+ return rows[:limit]
119
+
120
+
121
+ def velocity_leaders(t=None, limit=25, team_id=None):
122
+ """Top SKUs by LTM unit velocity (units/month) and revenue."""
123
+ t = t or P.today()
124
+ lf, lt = P.ltm(t)
125
+ sku = _sku_rev(lf, lt, team_id)
126
+ rows = [{'code': k, 'product': v['name'], 'units_ltm': v['qty'], 'units_per_mo': v['qty'] / 12.0,
127
+ 'rev_ltm': v['rev'], 'orders_ltm': v['orders']} for k, v in sku.items()]
128
+ rows.sort(key=lambda x: -x['units_ltm'])
129
+ return rows[:limit]
130
+
131
+
132
+ def _code_category():
133
+ """code -> category name. Built from _code_map (pid->code) + the sales category map
134
+ (pid->category); the first record carrying a code sets that code's category."""
135
+ codes = _code_map()
136
+ cats = sales_mod._product_cat()
137
+ out = {}
138
+ for pid, code in codes.items():
139
+ if code not in out and pid in cats:
140
+ out[code] = cats[pid]
141
+ return out
142
+
143
+
144
+ def directory(t=None, team_id=None):
145
+ """Every SKU that sold this YTD or the same period last year, with the fields the SKU
146
+ drill-down filters/sorts on: category, YTD/LY revenue, YoY %, units and orders. Mirrors the
147
+ customer directory — the full list (filtering happens in the UI), sorted by YTD revenue."""
148
+ t = t or P.today()
149
+ yf, yt = P.ytd(t)
150
+ lf, lt = P.ytd_last_year(t)
151
+ this = _sku_rev(yf, yt, team_id)
152
+ last = _sku_rev(lf, lt, team_id)
153
+ code_cat = _code_category()
154
+ rows = []
155
+ for code in set(this) | set(last):
156
+ tr = this.get(code, {})
157
+ lr = last.get(code, {})
158
+ rev_ytd = tr.get('rev', 0.0)
159
+ rev_ly = lr.get('rev', 0.0)
160
+ rows.append({'code': code, 'product': tr.get('name') or lr.get('name') or code,
161
+ 'category': code_cat.get(code, '(uncategorized)'),
162
+ 'rev_ytd': rev_ytd, 'rev_ly': rev_ly, 'change': rev_ytd - rev_ly,
163
+ 'yoy_pct': P.yoy_pct(rev_ytd, rev_ly),
164
+ 'qty_ytd': tr.get('qty', 0.0), 'orders_ytd': tr.get('orders', 0)})
165
+ rows.sort(key=lambda r: -r['rev_ytd'])
166
+ return rows
167
+
168
+
169
+ def catalogue():
170
+ """`{sku_code: {'product', 'category'}}` — EVERY ACTIVE product, sold or not.
171
+
172
+ ⛔ THIS IS THE CATALOGUE UNIVERSE, AND IT IS DELIBERATELY NOT `directory()`. `directory()`'s
173
+ row set IS the union of two revenue `read_group`s over `sale.order.line` (`:155`), so a SKU
174
+ that never sold cannot exist in it. That is CORRECT for its own callers — `yoy_movers`,
175
+ `zombie_skus`, `new_winners` and `categories` all legitimately want a sales-window universe —
176
+ and it is wrong for the PRODUCT GRID, which is a catalogue and was therefore showing 2,717 of
177
+ 5,875 SKUs (wave 29, owner item 22 / ruling R12). The fix is this function plus a LEFT JOIN in
178
+ the consumer, never a window removed from `directory()`: removing the window alone lands at
179
+ 3,327 (all-time-sold), because ~2,550 active SKUs have never sold in wholesale scope at all.
180
+
181
+ ⚠ NOT BU-SHAPED, and it cannot be: `product.product` carries no team. A catalogue is one
182
+ catalogue. `directory()` stays the BU-shaped half, which is why the two are joined rather than
183
+ merged — a scoped caller gets every SKU with ITS OWN revenue, blank where that BU never sold.
184
+
185
+ Keyed exactly like `_code_map()` — the `default_code`, or a `pid:N` fallback for the 33 active
186
+ records that carry none — so the join against `directory()` is code-for-code with no
187
+ normaliser. Names come from `display_name` (`[CODE] NAME`), which is what `O.m2o_name` yields
188
+ off a sale line, so a never-sold row wears the same format as a sold one.
189
+
190
+ ⛔ RAISES on a truncated read rather than returning a short catalogue. A silently short pull
191
+ would put the grid back at a plausible wrong number with every gate green — the exact failure
192
+ this function exists to end ([[no-unverifiable-aggregates]], and the same truncation guard
193
+ `modules/backorders.py:161` already uses).
194
+ """
195
+ dom = [('active', '=', True)]
196
+ prods = O.search_read('product.product', dom, ['id', 'default_code', 'display_name', 'name'],
197
+ limit=50000)
198
+ n = O.get_odoo().search_count('product.product', dom)
199
+ if len(prods) != n:
200
+ raise ValueError(
201
+ f"products.catalogue: the product pull is TRUNCATED — read {len(prods)} rows against "
202
+ f"a search_count of {n}. A short catalogue renders as a plausible smaller grid with "
203
+ f"nothing reporting it; raise the limit before shipping this.")
204
+ code_cat = _code_category()
205
+ out = {}
206
+ for p in prods:
207
+ code = (str(p['default_code']).strip() if p.get('default_code') else f"pid:{p['id']}")
208
+ # First record wins, matching `_code_category`'s own convention. MEASURED 2026-08-11: zero
209
+ # active products share a code, so this branch is a guard, not a merge policy.
210
+ out.setdefault(code, {'product': p.get('display_name') or p.get('name') or code,
211
+ 'category': code_cat.get(code, '(uncategorized)')})
212
+ return out
213
+
214
+
215
+ #: ⭐⭐ WAVE 30 / W30-T34 (the carried W29-T52) — THE PRICELIST STRATUM, PER LIST.
216
+ #:
217
+ #: MEASURED LIVE 2026-08-12, and every one of these numbers shaped the design rather than
218
+ #: decorating it:
219
+ #:
220
+ #: · **Five pricelists, three of them material.** Date-valid fixed rules: Fisch 5,139,
221
+ #: Royal 1 2,702, Royal 2 2,605, Public Pricelist 17, Giftware Deals 1. At SKU level that
222
+ #: is Fisch 3,971 · Royal 1 1,920 · Royal 2 1,857 · Public 6 · Giftware 0.
223
+ #: · **3,988 of 5,871 active products carry a usable fixed rule; 1,883 carry NONE.**
224
+ #: · **1,853 SKUs are priced on all three lists AND THE LISTS DISAGREE** — median relative
225
+ #: spread 16%, max 82%. That is why this is three columns rather than one "the price" or a
226
+ #: low/high pair: neither can tell a Fisch rep what Fisch sells the SKU for, which is the
227
+ #: only question the column exists to answer.
228
+ #: · ⛔ **`list_price` is 1.00 on 5,817 of the 5,871** — so the single `3_global` rule
229
+ #: computed over it is meaningless, exactly as W29-T52 said. **This reader never falls
230
+ #: through to it**, and that is the ticket's own negative control: a SKU with no specific
231
+ #: item renders BLANK, never a fallback dressed as a price.
232
+ #:
233
+ #: ⚠ RESOLVED BY NAME, NOT BY ID. A pricelist renamed or deleted in Odoo makes its column go
234
+ #: blank and makes `report["lists_missing"]` name it — a hard-coded id would keep pointing at
235
+ #: whatever inherited it and mislabel every cell silently.
236
+ PRICELIST_COLUMNS = (
237
+ ("price_fisch", "Fisch"),
238
+ ("price_royal_1", "Royal 1"),
239
+ ("price_royal_2", "Royal 2"),
240
+ )
241
+
242
+
243
+ def pricelist_by_code():
244
+ """`({code: {column_key: price}}, report)` — the date-valid FIXED base-tier price for each
245
+ declared pricelist, keyed EXACTLY as `catalogue()` keys its rows.
246
+
247
+ ⛔ THE KEYING IS LOAD-BEARING, not a detail. `catalogue()` keys on `default_code` with a
248
+ `pid:{id}` fallback for the ~33 active records that carry none. Keying this map any other
249
+ way would leave those SKUs permanently blank while an oracle counting active products
250
+ counted them — a red gate on a working build, or worse, a silent hole nobody counts.
251
+
252
+ **The base tier, deliberately.** `pricecomp._tier_for` picks the highest `min_quantity` at
253
+ or below an order's quantity, because it is pricing a LINE that has one. A catalogue column
254
+ has no quantity in hand, so it takes the LOWEST `min_quantity` — the price at qty 1. Rules
255
+ above that break are a bulk price, and `report["qty_break_only"]` counts the SKUs whose only
256
+ rule sits on one (MEASURED: 10 rules of 10,464 carry a break at all).
257
+
258
+ **Variant rules beat template rules**, matching `pricecomp._tier_for`: a `0_product_variant`
259
+ rule is the more specific statement about this exact SKU.
260
+
261
+ ⚠ DEGRADES TO `({}, report)` on a read failure, matching `_inventory_by_code` rather than
262
+ `catalogue()`: these are COLUMNS, and a product grid that will not render because pricing is
263
+ momentarily unreachable is a worse failure than one with blank price columns. The blanks are
264
+ not silent — `product_data.validate()`'s coverage leg reconciles each column against a fresh
265
+ Odoo count and goes red at zero.
266
+
267
+ ⭐ **The report exists because R6's second sentence is law** (*"if it can't be done, you need
268
+ to explicitly tell me why and recommend a fix"*). Everything this reader CANNOT see is
269
+ counted rather than dropped: rules on pricelists the contract does not declare, non-fixed
270
+ (`formula`/`percent`) rules, the `3_global` fallback, and prices that only exist above a
271
+ quantity break.
272
+ """
273
+ # ⚠ THE COUNTERS ARE DISJOINT AND ORDER-DEPENDENT, and saying so is the difference between a
274
+ # report and a misleading one. A rule is classified ONCE, by the first reason it is skipped:
275
+ # undeclared list → out of date → not fixed → global → zero price. So `rules_not_fixed: 0`
276
+ # means "no formula rule on a list we declare", NOT "this Odoo has no formula rules"
277
+ # (MEASURED 2026-08-12: it has exactly one, and it sits on Public Pricelist, which the
278
+ # contract does not declare — so it lands in `rules_undeclared_list`).
279
+ report = {"lists_missing": [], "rules_total": 0, "rules_undeclared_list": 0,
280
+ "rules_not_fixed": 0, "rules_global": 0, "rules_out_of_date": 0,
281
+ "rules_zero_price": 0, "qty_break_only": 0}
282
+ try:
283
+ pl_rows = O.search_read('product.pricelist', [], ['id', 'name'])
284
+ by_name = {}
285
+ for p in pl_rows:
286
+ by_name.setdefault(str(p.get('name') or '').strip(), p['id'])
287
+ wanted = {}
288
+ for col, name in PRICELIST_COLUMNS:
289
+ pid = by_name.get(name)
290
+ if pid is None:
291
+ report["lists_missing"].append(name)
292
+ else:
293
+ wanted[pid] = col
294
+
295
+ # ⭐⭐ FILTERED SERVER-SIDE, AND THE REASON IS A MEASUREMENT, NOT A STYLE PREFERENCE.
296
+ # Reading all 10,464 rules with 9 fields and sorting them in Python costs **30.1s** on
297
+ # this connection; the same rules under a server-side domain with 6 fields cost
298
+ # **13.6s** (measured 2026-08-12, back to back). `pool()` runs this on a scope's
299
+ # first-ever build, so that 16.5s is 16.5s of somebody's page load.
300
+ #
301
+ # ⚠ The domain reproduces the Python predicate EXACTLY, and the null legs are the part
302
+ # that is easy to get wrong: an absent `date_start` is `False`, not a past date, so
303
+ # `('date_start','<=',today)` ALONE would drop every open-ended rule — which is almost
304
+ # all of them.
305
+ today = P.today().isoformat()
306
+ _declared = sorted(wanted)
307
+ _kinds = ['0_product_variant', '1_product']
308
+ _live = [('pricelist_id', 'in', _declared), ('compute_price', '=', 'fixed'),
309
+ ('applied_on', 'in', _kinds),
310
+ '|', ('date_start', '=', False), ('date_start', '<=', today),
311
+ '|', ('date_end', '=', False), ('date_end', '>=', today),
312
+ ('fixed_price', '>', 0)]
313
+ rules = O.search_read(
314
+ 'product.pricelist.item', _live,
315
+ ['pricelist_id', 'product_id', 'product_tmpl_id', 'applied_on', 'fixed_price',
316
+ 'min_quantity']) if _declared else []
317
+
318
+ # ⭐ R6's SECOND SENTENCE, PAID FOR WITH `search_count` RATHER THAN A WIDER READ. Six
319
+ # counts cost ~1.2s together; the rules they count would cost 16s to read. What this
320
+ # reader cannot see is still REPORTED — it is just no longer transferred.
321
+ def _n(extra):
322
+ try:
323
+ return O.get_odoo().search_count('product.pricelist.item', extra)
324
+ except Exception:
325
+ return -1 # -1 reads as "not measured", never as zero
326
+ _dated = ['|', ('date_start', '=', False), ('date_start', '<=', today),
327
+ '|', ('date_end', '=', False), ('date_end', '>=', today)]
328
+ report["rules_total"] = _n([])
329
+ report["rules_undeclared_list"] = _n([('pricelist_id', 'not in', _declared)]) \
330
+ if _declared else report["rules_total"]
331
+ if _declared:
332
+ _on = [('pricelist_id', 'in', _declared)]
333
+ # `percent_price` and formula rules are read NOWHERE in this repo. A rule we cannot
334
+ # price is one the operator is TOLD about, never one that quietly becomes a blank.
335
+ report["rules_not_fixed"] = _n(_on + [('compute_price', '!=', 'fixed')])
336
+ report["rules_global"] = _n(_on + [('compute_price', '=', 'fixed'),
337
+ ('applied_on', 'not in', _kinds)])
338
+ report["rules_out_of_date"] = (
339
+ _n(_on + [('compute_price', '=', 'fixed'), ('applied_on', 'in', _kinds)])
340
+ - _n(_on + [('compute_price', '=', 'fixed'),
341
+ ('applied_on', 'in', _kinds)] + _dated))
342
+ # A zero price is not "free" — it is an unset rule. Blank says so; 0 does not.
343
+ report["rules_zero_price"] = _n(
344
+ _on + [('compute_price', '=', 'fixed'), ('applied_on', 'in', _kinds)]
345
+ + _dated + [('fixed_price', '<=', 0)])
346
+
347
+ by_var, by_tmpl = {}, {}
348
+ for r in rules:
349
+ col = wanted[O.m2o_id(r.get('pricelist_id'))]
350
+ if r.get('applied_on') == '0_product_variant' and r.get('product_id'):
351
+ by_var.setdefault((col, O.m2o_id(r['product_id'])), []).append(r)
352
+ elif r.get('product_tmpl_id'):
353
+ by_tmpl.setdefault((col, O.m2o_id(r['product_tmpl_id'])), []).append(r)
354
+
355
+ dom = [('active', '=', True)]
356
+ prods = O.search_read('product.product', dom,
357
+ ['id', 'default_code', 'product_tmpl_id'], limit=50000)
358
+ n = O.get_odoo().search_count('product.product', dom)
359
+ if len(prods) != n:
360
+ # Same guard, same reason as `catalogue()`: a short pull renders as a plausible
361
+ # smaller set of priced SKUs with nothing reporting it.
362
+ raise ValueError(
363
+ f"products.pricelist_by_code: the product pull is TRUNCATED — read {len(prods)} "
364
+ f"rows against a search_count of {n}.")
365
+ except Exception as e:
366
+ report["error"] = f"{type(e).__name__}: {str(e)[:200]}"
367
+ return {}, report
368
+
369
+ out = {}
370
+ for p in prods:
371
+ code = (str(p['default_code']).strip() if p.get('default_code') else f"pid:{p['id']}")
372
+ tmpl = O.m2o_id(p.get('product_tmpl_id'))
373
+ cells = {}
374
+ for col, _name in PRICELIST_COLUMNS:
375
+ cands = by_var.get((col, p['id'])) or by_tmpl.get((col, tmpl))
376
+ if not cands:
377
+ continue
378
+ base = min(cands, key=lambda r: r.get('min_quantity') or 0.0)
379
+ if (base.get('min_quantity') or 0.0) > 1.0:
380
+ report["qty_break_only"] += 1
381
+ cells[col] = base.get('fixed_price')
382
+ if cells:
383
+ out.setdefault(code, {}).update(cells)
384
+ return out, report
385
+
386
+
387
+ def catalogue_count():
388
+ """The INDEPENDENT population oracle: Odoo's own count of active products.
389
+
390
+ Deliberately a bare `search_count` and not a `len()` over anything this module built — the
391
+ 2,717 defect shipped silently for a wave because `product_data.validate()` derived BOTH sides
392
+ of its reconciliation from `_sku_rev`, so the oracle could never see a missing row.
393
+ """
394
+ return O.get_odoo().search_count('product.product', [('active', '=', True)])
395
+
396
+
397
+ def categories(t=None, team_id=None):
398
+ """Sorted distinct category names present in the SKU directory (for the drill-down filter)."""
399
+ return sorted({r['category'] for r in directory(t, team_id)})
400
+
401
+
402
+ def _coverage(date_from, date_to, team_id=None):
403
+ """{sku_code: distinct_customer_count} via 2-level read_group (slow-ish), merged by code.
404
+ (A customer buying two records sharing a code can count twice; duplicates are rare, and
405
+ code-level avoids the bigger error of a re-SKUed product reading as full coverage loss.)"""
406
+ g = O.read_group('sale.order.line', O.sale_line_domain(date_from, date_to, team_id, extra=_NO_SVC),
407
+ ['__count'], ['product_id', 'order_partner_id'], lazy=False)
408
+ codes = _code_map()
409
+ cov = {}
410
+ names = {}
411
+ for r in g:
412
+ pid = O.m2o_id(r.get('product_id'))
413
+ if not pid:
414
+ continue
415
+ key = codes.get(pid, f"pid:{pid}")
416
+ cov[key] = cov.get(key, 0) + 1
417
+ names.setdefault(key, O.m2o_name(r.get('product_id')))
418
+ return cov, names
419
+
420
+
421
+ def coverage_collapse(t=None, limit=25, min_prior_custs=8, team_id=None):
422
+ """SKUs that lost the most customer breadth YoY — early warning a SKU is dying even if
423
+ revenue hasn't fully cratered yet."""
424
+ t = t or P.today()
425
+ yf, yt = P.ytd(t)
426
+ lf, lt = P.ytd_last_year(t)
427
+ cov_t, names_t = _coverage(yf, yt, team_id)
428
+ cov_l, names_l = _coverage(lf, lt, team_id)
429
+ rows = []
430
+ for pid, lc in cov_l.items():
431
+ if lc < min_prior_custs:
432
+ continue
433
+ tc = cov_t.get(pid, 0)
434
+ drop = lc - tc
435
+ if drop <= 0:
436
+ continue
437
+ rows.append({'code': pid, 'product': names_l.get(pid) or names_t.get(pid) or '',
438
+ 'custs_ly': lc, 'custs_ytd': tc, 'lost_custs': drop,
439
+ 'pct_drop': drop / lc * 100})
440
+ rows.sort(key=lambda x: (-x['lost_custs'], -x['pct_drop']))
441
+ return rows[:limit]
442
+
443
+
444
+ # ====================================================================== SKU DRAWER (mirror of customer drawer)
445
+ def _sku_product_ids(code):
446
+ """All product.product ids sharing this SKU code (merged variants / archived records)."""
447
+ return [pid for pid, c in _code_map().items() if c == code]
448
+
449
+
450
+ def _sku_dom(pids, date_from, date_to, team_id=None):
451
+ return O.sale_line_domain(date_from, date_to, team_id, extra=[('product_id', 'in', pids)] + _NO_SVC)
452
+
453
+
454
+ def _sku_name_category(pids):
455
+ rows = O.search_read('product.product', [('id', 'in', pids)], ['name', 'categ_id'])
456
+ name = rows[0]['name'] if rows else '(unknown)'
457
+ catmap = sales_mod._product_cat()
458
+ main = next((catmap.get(p) for p in pids if catmap.get(p)), '(uncategorized)')
459
+ return name, main
460
+
461
+
462
+ def _sku_buyers(pids, date_from, date_to, team_id=None):
463
+ """{partner_id: {'name','rev','qty'}} for buyers of this SKU over a window."""
464
+ g = O.read_group('sale.order.line', _sku_dom(pids, date_from, date_to, team_id),
465
+ ['price_subtotal:sum', 'product_uom_qty:sum'], ['order_partner_id'], lazy=False)
466
+ out = {}
467
+ for r in g:
468
+ pid = O.m2o_id(r.get('order_partner_id'))
469
+ if pid:
470
+ out[pid] = {'name': O.m2o_name(r.get('order_partner_id')),
471
+ 'rev': r.get('price_subtotal') or 0.0, 'qty': r.get('product_uom_qty') or 0.0}
472
+ return out
473
+
474
+
475
+ def sku_detail(code, t=None, team_id=None, n_months=13, allsku=None):
476
+ """KPIs (rev/qty/buyers YoY, GM%), monthly trend, rank & % of BU for one SKU code.
477
+ Pass `allsku` (a cached _sku_rev YTD map) to skip the ~all-SKU rank read."""
478
+ t = t or P.today()
479
+ pids = _sku_product_ids(code)
480
+ if not pids:
481
+ return None
482
+ yf, yt = P.ytd(t)
483
+ lf, lt = P.ytd_last_year(t)
484
+ mf, mt = P.ltm(t)
485
+ name, category = _sku_name_category(pids)
486
+
487
+ def s(df, dtt, field='price_subtotal'):
488
+ return O.sum_field('sale.order.line', _sku_dom(pids, df, dtt, team_id), field)
489
+ rev_ytd, rev_ly = s(yf, yt), s(lf, lt)
490
+ qty_ytd, qty_ly = s(yf, yt, 'product_uom_qty'), s(lf, lt, 'product_uom_qty')
491
+ buyers_ytd = len(_sku_buyers(pids, yf, yt, team_id))
492
+ buyers_ly = len(_sku_buyers(pids, lf, lt, team_id))
493
+ g = O.read_group('sale.order.line', _sku_dom(pids, mf, mt, team_id),
494
+ ['price_subtotal:sum', 'margin:sum'], [], lazy=False)
495
+ line_rev = (g[0].get('price_subtotal') if g else 0) or 0.0
496
+ margin = (g[0].get('margin') if g else 0) or 0.0
497
+
498
+ # Monthly trend: bin lines by their order's month in Python (dot-path month groupby is rejected
499
+ # on sale.order.line), in 2 reads instead of 26 point queries.
500
+ range_start = dt.date(t.year - 2, t.month, 1).isoformat()
501
+ lines = O.search_read('sale.order.line', _sku_dom(pids, range_start, t.isoformat(), team_id),
502
+ ['price_subtotal', 'order_id'])
503
+ oids = list({O.m2o_id(line['order_id']) for line in lines if line.get('order_id')})
504
+ odate = {}
505
+ for i in range(0, len(oids), 1000):
506
+ for o in O.search_read('sale.order', [('id', 'in', oids[i:i + 1000])], ['date_order']):
507
+ if o.get('date_order'):
508
+ odate[o['id']] = str(o['date_order'])[:7]
509
+ mrev = {}
510
+ for line in lines:
511
+ ym = odate.get(O.m2o_id(line.get('order_id')))
512
+ if ym:
513
+ mrev[ym] = mrev.get(ym, 0.0) + (line.get('price_subtotal') or 0.0)
514
+ monthly = []
515
+ for ym, start, end in P.month_starts(n_months, t):
516
+ y, m = int(ym[:4]) - 1, int(ym[5:7])
517
+ monthly.append({'month': ym, 'revenue': mrev.get(ym, 0.0),
518
+ 'revenue_ly': mrev.get(f'{y:04d}-{m:02d}', 0.0)})
519
+
520
+ allsku = allsku if allsku is not None else _sku_rev(yf, yt, team_id)
521
+ total = sum(v['rev'] for v in allsku.values()) or 1.0
522
+ rank = next((i + 1 for i, (c, _v) in enumerate(sorted(allsku.items(), key=lambda kv: -kv[1]['rev']))
523
+ if c == code), None)
524
+ return {
525
+ 'code': code, 'name': name, 'category': category,
526
+ 'rev_ytd': rev_ytd, 'rev_ly': rev_ly, 'rev_yoy_pct': P.yoy_pct(rev_ytd, rev_ly),
527
+ 'qty_ytd': qty_ytd, 'qty_ly': qty_ly, 'qty_yoy_pct': P.yoy_pct(qty_ytd, qty_ly),
528
+ 'buyers_ytd': buyers_ytd, 'buyers_ly': buyers_ly, 'buyers_delta': buyers_ytd - buyers_ly,
529
+ 'gm_pct': (margin / line_rev * 100) if line_rev else 0.0, 'gm_dollars': margin,
530
+ 'rank': rank, 'n_skus': len(allsku), 'pct_of_bu': rev_ytd / total * 100,
531
+ 'monthly': monthly,
532
+ }
533
+
534
+
535
+ def sku_buyer_bridge(code, t=None, team_id=None, top=12):
536
+ """Who drives the SKU's YoY: retained / new / churned buyers + a $-ranked churned call list."""
537
+ t = t or P.today()
538
+ pids = _sku_product_ids(code)
539
+ yf, yt = P.ytd(t)
540
+ lf, lt = P.ytd_last_year(t)
541
+ this, last = _sku_buyers(pids, yf, yt, team_id), _sku_buyers(pids, lf, lt, team_id)
542
+ tset, lset = set(this), set(last)
543
+ retained, new, churned = tset & lset, tset - lset, lset - tset
544
+ churned_list = sorted([{'pid': p, 'customer': last[p]['name'], 'ly_rev': last[p]['rev']}
545
+ for p in churned], key=lambda x: -x['ly_rev'])[:top]
546
+ return {'retained': {'n': len(retained), 'rev': sum(this[p]['rev'] for p in retained)},
547
+ 'new': {'n': len(new), 'rev': sum(this[p]['rev'] for p in new)},
548
+ 'churned': {'n': len(churned), 'rev': sum(last[p]['rev'] for p in churned)},
549
+ 'buyer_retention_pct': (len(retained) / len(lset) * 100) if lset else 0.0,
550
+ 'churned_buyers': churned_list, 'buyers_this': len(tset), 'buyers_last': len(lset),
551
+ 'this_total': sum(v['rev'] for v in this.values())}
552
+
553
+
554
+ def sku_concentration(code, t=None, team_id=None):
555
+ """Buyer-concentration risk: top-1/top-3 share, Herfindahl index, effective buyer count (LTM)."""
556
+ t = t or P.today()
557
+ mf, mt = P.ltm(t)
558
+ buyers = _sku_buyers(_sku_product_ids(code), mf, mt, team_id)
559
+ revs = sorted([v['rev'] for v in buyers.values()], reverse=True)
560
+ total = sum(revs) or 1.0
561
+ hhi = sum((r / total) ** 2 for r in revs)
562
+ return {'n_buyers': len(revs), 'top1_pct': (revs[0] / total * 100) if revs else 0.0,
563
+ 'top3_pct': (sum(revs[:3]) / total * 100) if revs else 0.0,
564
+ 'hhi': hhi, 'eff_buyers': (1 / hhi) if hhi else 0.0}
565
+
566
+
567
+ def sku_price_dispersion(code, t=None, team_id=None, cap=60):
568
+ """Realized $/unit per buyer (LTM) vs the volume-weighted average; recoverable $ on below-VWAP
569
+ accounts. list_price is unreliable, so the dispersion among actual buyers is the margin lever."""
570
+ t = t or P.today()
571
+ mf, mt = P.ltm(t)
572
+ g = O.read_group('sale.order.line', _sku_dom(_sku_product_ids(code), mf, mt, team_id),
573
+ ['price_subtotal:sum', 'product_uom_qty:sum'], ['order_partner_id'], lazy=False)
574
+ rows, tot_rev, tot_qty = [], 0.0, 0.0
575
+ for r in g:
576
+ pid = O.m2o_id(r.get('order_partner_id'))
577
+ rev = r.get('price_subtotal') or 0.0
578
+ qty = r.get('product_uom_qty') or 0.0
579
+ if not pid or qty <= 0:
580
+ continue
581
+ rows.append({'pid': pid, 'customer': O.m2o_name(r.get('order_partner_id')),
582
+ 'price': rev / qty, 'qty': qty, 'rev': rev})
583
+ tot_rev += rev
584
+ tot_qty += qty
585
+ vwap = (tot_rev / tot_qty) if tot_qty else 0.0
586
+ for r in rows:
587
+ r['recoverable'] = max(0.0, vwap - r['price']) * r['qty']
588
+ rows.sort(key=lambda x: -x['recoverable'])
589
+ return {'vwap': vwap, 'n_buyers': len(rows),
590
+ 'recoverable_total': sum(r['recoverable'] for r in rows), 'rows': rows[:cap]}
591
+
592
+
593
+ def sku_top_buyers(code, t=None, team_id=None, top=15):
594
+ """Ranked buyers of this SKU (YTD) with YoY — clickable to open the customer drawer."""
595
+ t = t or P.today()
596
+ pids = _sku_product_ids(code)
597
+ yf, yt = P.ytd(t)
598
+ lf, lt = P.ytd_last_year(t)
599
+ this, last = _sku_buyers(pids, yf, yt, team_id), _sku_buyers(pids, lf, lt, team_id)
600
+ rows = [{'pid': p, 'customer': v['name'], 'rev': v['rev'], 'qty': v['qty'],
601
+ 'yoy_pct': P.yoy_pct(v['rev'], last.get(p, {}).get('rev', 0.0))} for p, v in this.items()]
602
+ rows.sort(key=lambda x: -x['rev'])
603
+ return rows[:top]
604
+
605
+
606
+ _BUYER_TIERS = [('Whale (≥$25k)', 25000.0), ('Large ($10–25k)', 10000.0),
607
+ ('Mid ($2–10k)', 2000.0), ('Small (<$2k)', 0.0)]
608
+
609
+
610
+ def sku_customer_analysis(code, t=None, team_id=None, top=15):
611
+ """WHO buys this SKU (LTM), as customers: the value-tier mix of its buyers (by each buyer's TOTAL
612
+ spend), and the accounts most DEPENDENT on it (this SKU as a share of their spend — who gets hurt
613
+ most if it stocks out / who to protect)."""
614
+ t = t or P.today()
615
+ lf, lt = P.ltm(t)
616
+ pids = _sku_product_ids(code)
617
+ empty = {'segments': [], 'dependency': [], 'n_buyers': 0, 'avg_dependency': 0.0}
618
+ if not pids:
619
+ return empty
620
+ buyers = _sku_buyers(pids, lf, lt, team_id) # {pid: {name, rev(this SKU), qty}}
621
+ if not buyers:
622
+ return empty
623
+ bpids = list(buyers)
624
+ g = O.read_group('sale.order', sales_mod.order_domain(lf, lt, team_id) + [('partner_id', 'in', bpids)],
625
+ ['amount_untaxed:sum'], ['partner_id'], lazy=False)
626
+ total = {O.m2o_id(r['partner_id']): (r.get('amount_untaxed') or 0.0) for r in g if r.get('partner_id')}
627
+
628
+ def tier(rev):
629
+ for nm, lo in _BUYER_TIERS:
630
+ if rev >= lo:
631
+ return nm
632
+ return _BUYER_TIERS[-1][0]
633
+ seg = {nm: {'tier': nm, 'buyers': 0, 'sku_rev': 0.0} for nm, _ in _BUYER_TIERS}
634
+ dep = []
635
+ for p, v in buyers.items():
636
+ ct = total.get(p, v['rev']) or v['rev']
637
+ s = seg[tier(ct)]
638
+ s['buyers'] += 1
639
+ s['sku_rev'] += v['rev']
640
+ dep.append({'customer': v['name'], 'pid': p, 'sku_rev': v['rev'], 'cust_total': ct,
641
+ 'dependency_pct': (v['rev'] / ct * 100) if ct else None})
642
+ tot = sum(s['sku_rev'] for s in seg.values()) or 1.0
643
+ segments = []
644
+ for nm, _ in _BUYER_TIERS:
645
+ s = seg[nm]
646
+ s['rev_share'] = s['sku_rev'] / tot * 100
647
+ s['avg_per_buyer'] = (s['sku_rev'] / s['buyers']) if s['buyers'] else 0.0
648
+ segments.append(s)
649
+ dep.sort(key=lambda x: -x['sku_rev'])
650
+ deps_known = [d['dependency_pct'] for d in dep if d['dependency_pct'] is not None]
651
+ return {'segments': segments, 'dependency': dep[:top], 'n_buyers': len(buyers),
652
+ 'avg_dependency': (sum(deps_known) / len(deps_known)) if deps_known else 0.0}
653
+
654
+
655
+ def sku_whitespace(code, t=None, team_id=None, top=15):
656
+ """Customers who buy this SKU's category but NOT this SKU — ranked prospect list."""
657
+ import modules.customers as cust_mod
658
+ t = t or P.today()
659
+ pids = _sku_product_ids(code)
660
+ _name, category = _sku_name_category(pids)
661
+ cat_buyers = cust_mod.category_buyers(category, team_id=team_id) or set()
662
+ mf, mt = P.ltm(t)
663
+ prospects = list(cat_buyers - set(_sku_buyers(pids, mf, mt, team_id)))
664
+ if not prospects:
665
+ return {'category': category, 'rows': []}
666
+ g = O.read_group('sale.order', sales_mod.order_domain(mf, mt, team_id) + [('partner_id', 'in', prospects)],
667
+ ['amount_untaxed:sum'], ['partner_id'], lazy=False)
668
+ rows = [{'pid': O.m2o_id(r['partner_id']), 'customer': O.m2o_name(r['partner_id']),
669
+ 'total_spend': r.get('amount_untaxed') or 0.0} for r in g if r.get('partner_id')]
670
+ rows.sort(key=lambda x: -x['total_spend'])
671
+ return {'category': category, 'rows': rows[:top]}
672
+
673
+
674
+ def sku_drawer_bundle(code, t=None, team_id=None, allsku=None):
675
+ """The whole SKU drawer's first paint in one cached unit: sku_detail first (the not-found gate),
676
+ then the other six pulls CONCURRENTLY (O.parallel). Cuts a ~6-call cold open to ~max(call)."""
677
+ detail = sku_detail(code, t=t, team_id=team_id, allsku=allsku)
678
+ if detail is None:
679
+ return {'detail': None}
680
+ bridge, conc, price, buyers, white, ca = O.parallel([
681
+ lambda: sku_buyer_bridge(code, t, team_id),
682
+ lambda: sku_concentration(code, t, team_id),
683
+ lambda: sku_price_dispersion(code, t, team_id),
684
+ lambda: sku_top_buyers(code, t, team_id),
685
+ lambda: sku_whitespace(code, t, team_id),
686
+ lambda: sku_customer_analysis(code, t, team_id),
687
+ ])
688
+ return {'detail': detail, 'bridge': bridge, 'conc': conc, 'price': price,
689
+ 'buyers': buyers, 'white': white, 'ca': ca}
690
+
691
+
692
+ def validate(t=None, team_id=None):
693
+ """Reconcile SKU metrics to independent Odoo aggregates. When team_id is set (a single BU
694
+ selected) every check runs SCOPED to that BU, so the validation panel never reconciles
695
+ against — or exposes — the other BU's numbers. All checks here are BU-scopeable (no cross-BU
696
+ mirror), so team_id threads straight through."""
697
+ t = t or P.today()
698
+ yf, yt = P.ytd(t)
699
+ checks = []
700
+ sku = _sku_rev(yf, yt, team_id)
701
+ sku_sum = sum(v['rev'] for v in sku.values())
702
+ line_total = O.sum_field('sale.order.line',
703
+ O.sale_line_domain(yf, yt, team_id, extra=_NO_SVC), 'price_subtotal')
704
+ checks.append({'check': 'SKU rev: Σ(per-SKU) == total line revenue, ex-services (YTD)',
705
+ 'a': round(sku_sum, 2), 'b': round(line_total, 2),
706
+ 'gap': round(sku_sum - line_total, 2),
707
+ 'ok': abs(sku_sum - line_total) <= 1.0})
708
+
709
+ m = yoy_movers(t, limit=10**9, team_id=team_id)
710
+ movers_sum = sum(r['change'] for r in m['risers']) + sum(r['change'] for r in m['decliners'])
711
+ lf, lt = P.ytd_last_year(t)
712
+ last_total = O.sum_field('sale.order.line',
713
+ O.sale_line_domain(lf, lt, team_id, extra=_NO_SVC), 'price_subtotal')
714
+ checks.append({'check': 'SKU movers: Σ(Δ) == (YTD − LY) total',
715
+ 'a': round(movers_sum, 2), 'b': round(line_total - last_total, 2),
716
+ 'gap': round(movers_sum - (line_total - last_total), 2),
717
+ 'ok': abs(movers_sum - (line_total - last_total)) <= 1.0})
718
+
719
+ # SKU drawer: buyer-bridge reconciles — retained + new buyer revenue == this-period SKU revenue
720
+ top_code = max(sku.items(), key=lambda kv: kv[1]['rev'])[0] if sku else None
721
+ if top_code:
722
+ bb = sku_buyer_bridge(top_code, t, team_id=team_id)
723
+ recon = bb['retained']['rev'] + bb['new']['rev']
724
+ checks.append({'check': 'SKU drawer: retained+new buyer rev == SKU YTD revenue',
725
+ 'a': round(recon, 2), 'b': round(bb['this_total'], 2),
726
+ 'gap': round(recon - bb['this_total'], 2),
727
+ 'ok': abs(recon - bb['this_total']) <= 1.0})
728
+ return checks
 
 
 
 
 
 
 
 
 
web/public/sample_customers.json CHANGED
@@ -1,558 +1,545 @@
1
- {
2
- "fields": [
3
- {
4
- "key": "customer",
5
- "label": "Customer",
6
- "type": "text",
7
- "source": "odoo",
8
- "pinned": true,
9
- "default": true,
10
- "description": "The customer's name in Odoo. One row per customer who ordered in the last 24 months."
11
- },
12
- {
13
- "key": "partner_id",
14
- "label": "Odoo ID",
15
- "type": "int",
16
- "source": "odoo",
17
- "derived": true,
18
- "default": false,
19
- "description": "The Odoo res.partner id — the key every Odoo document joins on. DERIVED: this row's pid IS the partner id, so a stored copy would be a second source."
20
- },
21
- {
22
- "key": "odoo_status",
23
- "label": "Odoo record",
24
- "type": "status",
25
- "source": "odoo",
26
- "default": false,
27
- "options": [
28
- "Active",
29
- "Archived"
30
- ],
31
- "description": "Whether this customer still exists in Odoo. Archived means deleted there."
32
- },
33
- {
34
- "key": "agent",
35
- "label": "Agent",
36
- "type": "text",
37
- "source": "odoo",
38
- "default": true,
39
- "description": "The sales agent who owns this account."
40
- },
41
- {
42
- "key": "dba",
43
- "label": "DBA",
44
- "type": "select",
45
- "source": "odoo",
46
- "default": false,
47
- "options": [
48
- "Fisch",
49
- "Royal",
50
- "Both"
51
- ],
52
- "description": "The brand this customer buys from - Fisch, Royal, or both. Amazon-channel orders are not a DBA."
53
- },
54
- {
55
- "key": "salesperson",
56
- "label": "Salesperson",
57
- "type": "text",
58
- "source": "odoo",
59
- "default": false,
60
- "description": "Who keyed in most of this customer's orders — not the Agent, who owns the account."
61
- },
62
- {
63
- "key": "street",
64
- "label": "Street",
65
- "type": "text",
66
- "source": "odoo",
67
- "default": false,
68
- "description": "First address line, from res.partner directly - not the geocoder, so a customer the map cannot place still shows its address."
69
- },
70
- {
71
- "key": "street2",
72
- "label": "Street 2",
73
- "type": "text",
74
- "source": "odoo",
75
- "default": false,
76
- "description": "Second address line (suite, unit, floor) on the customer's Odoo address."
77
- },
78
- {
79
- "key": "city",
80
- "label": "City",
81
- "type": "text",
82
- "source": "odoo",
83
- "default": true,
84
- "description": "City on the customer's Odoo address."
85
- },
86
- {
87
- "key": "state",
88
- "label": "State",
89
- "type": "text",
90
- "source": "odoo",
91
- "default": true,
92
- "description": "State or province on the customer's Odoo address."
93
- },
94
- {
95
- "key": "country",
96
- "label": "Country",
97
- "type": "text",
98
- "source": "odoo",
99
- "default": false,
100
- "description": "Country on the customer's Odoo address."
101
- },
102
- {
103
- "key": "zip",
104
- "label": "ZIP",
105
- "type": "text",
106
- "source": "odoo",
107
- "default": false,
108
- "description": "Postal code on the customer's Odoo address."
109
- },
110
- {
111
- "key": "customer_since",
112
- "label": "Customer since",
113
- "type": "date",
114
- "source": "odoo",
115
- "default": false,
116
- "description": "When this customer was first set up in Odoo."
117
- },
118
- {
119
- "key": "tags",
120
- "label": "Tags",
121
- "type": "text",
122
- "source": "odoo",
123
- "default": false,
124
- "description": "Odoo labels on this customer, comma-separated."
125
- },
126
- {
127
- "key": "pricelist",
128
- "label": "Price list",
129
- "type": "text",
130
- "source": "odoo",
131
- "default": false,
132
- "description": "The price list this customer buys on."
133
- },
134
- {
135
- "key": "payment_terms",
136
- "label": "Payment terms",
137
- "type": "text",
138
- "source": "odoo",
139
- "default": false,
140
- "description": "Payment terms on this customer's account — Net 30, for example."
141
- },
142
- {
143
- "key": "last_order",
144
- "label": "Last order",
145
- "type": "date",
146
- "source": "odoo",
147
- "default": true,
148
- "description": "Date of the most recent confirmed order."
149
- },
150
- {
151
- "key": "overdue_days",
152
- "label": "Overdue days",
153
- "type": "int",
154
- "source": "odoo",
155
- "default": true,
156
- "description": "How many days late this customer is running against their own usual ordering rhythm."
157
- },
158
- {
159
- "_note": "filterable:false — DERIVED ANALYTIC: est_missed is min(cycles missed, 3) x AOV, a score we compute rather than an object the business has, so a condition on it would read as a fact about the customer when it is a fact about our arithmetic. It still displays and still sorts. Until wave 6 this flag also covered the frozen-window presets (revenue_ytd / revenue_ly / orders_24m / aov / yoy_pct); those are now DELETED outright under the owner's no-buildable-presets rule — see _comment. est_missed itself STAYS: no creatable measure or formula reproduces the cadence model behind it.",
160
- "key": "est_missed",
161
- "label": "Est. missed $",
162
- "type": "currency",
163
- "source": "odoo",
164
- "default": true,
165
- "agg": "sum",
166
- "filterable": false,
167
- "description": "Estimated sales missed while quiet: missed orders (capped at 3) times average order value. An estimate, not money owed."
168
- },
169
- {
170
- "_note": "wave 21 R1 — KEY UNCHANGED, LABEL RENAMED. The computation is a DISJOINT split (ar.py credit_exposure): this column is only the not-yet-due residual, its sibling is the past-grace residual, and the two sum to the total. Under the label 'AR open $' the majority-late book read as 'Overdue > Open', which is nonsense in AR vocabulary — 'open' universally means the total. The label now says what the number is; the key stays so saved views and filters keep working.",
171
- "key": "ar_open",
172
- "label": "AR current $",
173
- "type": "currency",
174
- "source": "odoo",
175
- "default": false,
176
- "description": "Invoiced money owed but not yet due (a 5-day grace applies before it counts as overdue)."
177
- },
178
- {
179
- "key": "ar_overdue",
180
- "label": "AR overdue $",
181
- "type": "currency",
182
- "source": "odoo",
183
- "default": false,
184
- "description": "Invoiced money past due — same basis as the Collections page."
185
- },
186
- {
187
- "_note": "wave 21 R1 — the TOTAL, added beside the rename above. AR current $ + AR overdue $, i.e. what most people mean by 'open AR'. Composed from the same ar.credit_exposure rows the siblings use, so it is transitively reconciled by ar.validate()'s residual read_group tie — no second oracle.",
188
- "key": "ar_outstanding",
189
- "label": "AR outstanding $",
190
- "type": "currency",
191
- "source": "odoo",
192
- "default": false,
193
- "description": "Total invoiced money owed right now: AR current $ plus AR overdue $."
194
- },
195
- {
196
- "key": "ar_exposure",
197
- "label": "Credit exposure $",
198
- "type": "currency",
199
- "source": "odoo",
200
- "default": false,
201
- "description": "The most you could be out if they stopped paying today: open, overdue, draft and not-yet-invoiced."
202
- },
203
- {
204
- "key": "ar_aged_1_30",
205
- "label": "1-30 days $",
206
- "type": "currency",
207
- "source": "odoo",
208
- "default": false,
209
- "description": "Overdue between 1 and 30 days. The four aging buckets sum to AR overdue $."
210
- },
211
- {
212
- "key": "ar_aged_31_60",
213
- "label": "31-60 days $",
214
- "type": "currency",
215
- "source": "odoo",
216
- "default": false,
217
- "description": "Overdue between 31 and 60 days. The four aging buckets sum to AR overdue $."
218
- },
219
- {
220
- "key": "ar_aged_61_90",
221
- "label": "61-90 days $",
222
- "type": "currency",
223
- "source": "odoo",
224
- "default": false,
225
- "description": "Overdue between 61 and 90 days. The four aging buckets sum to AR overdue $."
226
- },
227
- {
228
- "key": "ar_aged_90_plus",
229
- "label": "90+ days $",
230
- "type": "currency",
231
- "source": "odoo",
232
- "default": false,
233
- "description": "Overdue by more than 90 days. The four aging buckets sum to AR overdue $."
234
- },
235
- {
236
- "key": "days_to_pay",
237
- "label": "Days to pay",
238
- "type": "int",
239
- "source": "odoo",
240
- "default": false,
241
- "description": "Average days to pay an invoice in full. Blank means no fully paid invoice yet."
242
- },
243
- {
244
- "key": "top_category",
245
- "label": "Top category",
246
- "type": "text",
247
- "source": "odoo",
248
- "default": false,
249
- "description": "The category this customer spent the most on in the last 12 months."
250
- },
251
- {
252
- "key": "top_category_pct",
253
- "label": "Top category %",
254
- "type": "pct",
255
- "source": "odoo",
256
- "default": false,
257
- "description": "Share of last-12-months spend that went to the top category."
258
- },
259
- {
260
- "key": "sku_count",
261
- "label": "SKUs bought",
262
- "type": "int",
263
- "source": "odoo",
264
- "default": false,
265
- "description": "Distinct products bought in the last 12 months."
266
- },
267
- {
268
- "key": "top_sku",
269
- "label": "Top SKU",
270
- "type": "text",
271
- "source": "odoo",
272
- "default": false,
273
- "description": "The product this customer spent the most on in the last 12 months."
274
- },
275
- {
276
- "key": "days_since",
277
- "label": "Days since order",
278
- "type": "int",
279
- "source": "odoo",
280
- "default": false,
281
- "description": "Days since the last confirmed order."
282
- },
283
- {
284
- "key": "typical_gap_days",
285
- "label": "Typical gap days",
286
- "type": "int",
287
- "source": "odoo",
288
- "default": false,
289
- "description": "Days this customer usually goes between orders, from their own history."
290
- },
291
- {
292
- "key": "notes",
293
- "label": "Notes",
294
- "type": "text",
295
- "source": "overlay",
296
- "default": false,
297
- "description": "Your notes on this customer. Saved in this app only, visible only to you."
298
- }
299
- ],
300
- "rows": [
301
- {
302
- "pid": 101,
303
- "customer": "Poppy Flowers",
304
- "status": "New",
305
- "agent": "Naomi Linnell Rivera",
306
- "city": "Charlottesville",
307
- "state": "Virginia (US)",
308
- "last_order": "2026-07-21",
309
- "est_missed": 0,
310
- "notes": "",
311
- "country": "United States",
312
- "zip": "02720",
313
- "payment_terms": "30 Days",
314
- "pricelist": "Fisch 1 (USD)",
315
- "tags": "Royal",
316
- "customer_since": "2023-01-10",
317
- "salesperson": "Jessica",
318
- "ar_open": 0,
319
- "ar_overdue": 0,
320
- "ar_exposure": 0,
321
- "top_category": "Styrofoam",
322
- "top_category_pct": 0.47,
323
- "sku_count": 93,
324
- "top_sku": "AQUAFOAM FLORAL FOAM BRICK | 48-Piece per Pack",
325
- "days_to_pay": 34,
326
- "_created": "2023-01-15 09:10:00",
327
- "lat": 25.7617,
328
- "lon": -80.1918,
329
- "odoo_status": "Archived",
330
- "dba": "Royal",
331
- "ar_outstanding": 0
332
- },
333
- {
334
- "pid": 102,
335
- "customer": "Meadow & Vine Wholesale",
336
- "status": "Growing",
337
- "agent": "Carla Jimenez",
338
- "city": "Portland",
339
- "state": "Oregon (US)",
340
- "last_order": "2026-07-19",
341
- "est_missed": 0,
342
- "notes": "Expanding to a second storefront.",
343
- "country": "United States",
344
- "zip": "77041",
345
- "payment_terms": "Immediate Payment",
346
- "pricelist": "Royal 1 (USD)",
347
- "tags": "Royal, Key account",
348
- "customer_since": "2024-02-11",
349
- "salesperson": "Naomi",
350
- "ar_open": 1240.5,
351
- "ar_overdue": 0,
352
- "ar_exposure": 1740.5,
353
- "top_category": "Ribbon",
354
- "top_category_pct": 0.95,
355
- "sku_count": 4,
356
- "top_sku": "2\" X 24\" X 36\" GREEN STYROFOAM BOARD",
357
- "days_to_pay": null,
358
- "_created": "2023-02-15 09:11:00",
359
- "lat": 27.9506,
360
- "lon": -82.4572,
361
- "odoo_status": "Active",
362
- "dba": "Fisch",
363
- "ar_outstanding": 1240.5
364
- },
365
- {
366
- "pid": 103,
367
- "customer": "Bluestem Floral Supply",
368
- "status": "Growing",
369
- "agent": "Naomi Linnell Rivera",
370
- "city": "Kansas City",
371
- "state": "Missouri (US)",
372
- "last_order": "2026-07-17",
373
- "est_missed": 0,
374
- "notes": "",
375
- "country": "United States",
376
- "zip": "11219",
377
- "payment_terms": "60 Days",
378
- "pricelist": "Royal 1 (USD)",
379
- "tags": "Fisch",
380
- "customer_since": "2025-03-12",
381
- "salesperson": "Karen",
382
- "ar_open": 0,
383
- "ar_overdue": 5120.25,
384
- "ar_exposure": 5120.25,
385
- "top_category": "Foams & Finishes",
386
- "top_category_pct": 0.31,
387
- "sku_count": 27,
388
- "top_sku": "SATIN RIBBON 2IN",
389
- "days_to_pay": 61,
390
- "_created": "2023-03-15 09:12:00",
391
- "lat": 28.5384,
392
- "lon": -81.3789,
393
- "odoo_status": "Active",
394
- "dba": "Both",
395
- "ar_outstanding": 5120.25
396
- },
397
- {
398
- "pid": 104,
399
- "customer": "Camellia Row Florist",
400
- "status": "Declining",
401
- "agent": "Devon Marsh",
402
- "city": "Savannah",
403
- "state": "Georgia (US)",
404
- "last_order": "2026-05-30",
405
- "est_missed": 41200,
406
- "notes": "Switched some volume to a local grower.",
407
- "country": "United States",
408
- "zip": "07649",
409
- "payment_terms": "30 Days",
410
- "pricelist": "Fisch 1 (USD)",
411
- "tags": "(none)",
412
- "customer_since": "2026-04-13",
413
- "salesperson": "(none)",
414
- "ar_open": 8300,
415
- "ar_overdue": 940,
416
- "ar_exposure": 11440,
417
- "top_category": "All",
418
- "top_category_pct": 1.0,
419
- "sku_count": 1,
420
- "top_sku": "(none)",
421
- "days_to_pay": 12,
422
- "_created": "2023-04-15 09:13:00",
423
- "lat": 30.3322,
424
- "lon": -81.6557,
425
- "odoo_status": "Active",
426
- "dba": "Royal",
427
- "ar_outstanding": 9240
428
- },
429
- {
430
- "pid": 105,
431
- "customer": "Harborlight Wholesale Blooms",
432
- "status": "Growing",
433
- "agent": "Carla Jimenez",
434
- "city": "Seattle",
435
- "state": "Washington (US)",
436
- "last_order": "2026-07-22",
437
- "est_missed": 0,
438
- "notes": "Top-10 account.",
439
- "country": "United States",
440
- "zip": "33125",
441
- "payment_terms": "Immediate Payment",
442
- "pricelist": "Royal 1 (USD)",
443
- "tags": "Royal",
444
- "customer_since": "2023-05-14",
445
- "salesperson": "Jessica",
446
- "ar_open": 0,
447
- "ar_overdue": 0,
448
- "ar_exposure": 0,
449
- "top_category": "Styrofoam",
450
- "top_category_pct": 0.47,
451
- "sku_count": 93,
452
- "top_sku": "AQUAFOAM FLORAL FOAM BRICK | 48-Piece per Pack",
453
- "days_to_pay": 34,
454
- "_created": "2023-05-15 09:14:00",
455
- "lat": 26.1224,
456
- "lon": -80.1373,
457
- "odoo_status": "Active",
458
- "dba": "",
459
- "ar_outstanding": 0
460
- },
461
- {
462
- "pid": 106,
463
- "customer": "Dogwood & Fern Co.",
464
- "status": "Dormant",
465
- "agent": "Devon Marsh",
466
- "city": "Asheville",
467
- "state": "North Carolina (US)",
468
- "last_order": "2026-02-11",
469
- "est_missed": 52400,
470
- "notes": "No spring order this year.",
471
- "country": "United States",
472
- "zip": "90210",
473
- "payment_terms": "60 Days",
474
- "pricelist": "Royal 1 (USD)",
475
- "tags": "Royal, Key account",
476
- "customer_since": "2024-06-15",
477
- "salesperson": "Naomi",
478
- "ar_open": 1240.5,
479
- "ar_overdue": 0,
480
- "ar_exposure": 1740.5,
481
- "top_category": "Ribbon",
482
- "top_category_pct": 0.95,
483
- "sku_count": 4,
484
- "top_sku": "2\" X 24\" X 36\" GREEN STYROFOAM BOARD",
485
- "days_to_pay": null,
486
- "_created": "2023-06-15 09:15:00",
487
- "lat": 27.3364,
488
- "lon": -82.5307,
489
- "odoo_status": "Active",
490
- "dba": "Fisch",
491
- "ar_outstanding": 1240.5
492
- },
493
- {
494
- "pid": 107,
495
- "customer": "Verbena Market Florals",
496
- "status": "Lost",
497
- "agent": "Naomi Linnell Rivera",
498
- "city": "Austin",
499
- "state": "Texas (US)",
500
- "last_order": "2025-11-04",
501
- "est_missed": 78300,
502
- "notes": "Went with a competitor on freight terms.",
503
- "country": "United States",
504
- "zip": "08701",
505
- "payment_terms": "30 Days",
506
- "pricelist": "Fisch 1 (USD)",
507
- "tags": "Fisch",
508
- "customer_since": "2025-07-16",
509
- "salesperson": "Karen",
510
- "ar_open": 0,
511
- "ar_overdue": 5120.25,
512
- "ar_exposure": 5120.25,
513
- "top_category": "Foams & Finishes",
514
- "top_category_pct": 0.31,
515
- "sku_count": 27,
516
- "top_sku": "SATIN RIBBON 2IN",
517
- "days_to_pay": 61,
518
- "_created": "2023-07-15 09:16:00",
519
- "lat": null,
520
- "lon": null,
521
- "odoo_status": "Active",
522
- "dba": "Both",
523
- "ar_outstanding": 5120.25
524
- },
525
- {
526
- "pid": 108,
527
- "customer": "Larkspur Lane Supply",
528
- "status": "New",
529
- "agent": "Carla Jimenez",
530
- "city": "Denver",
531
- "state": "Colorado (US)",
532
- "last_order": "2026-07-14",
533
- "est_missed": 0,
534
- "notes": "First order in April.",
535
- "country": "United States",
536
- "zip": "60614",
537
- "payment_terms": "Immediate Payment",
538
- "pricelist": "Royal 1 (USD)",
539
- "tags": "(none)",
540
- "customer_since": "2026-08-17",
541
- "salesperson": "(none)",
542
- "ar_open": 8300,
543
- "ar_overdue": 940,
544
- "ar_exposure": 11440,
545
- "top_category": "All",
546
- "top_category_pct": 1.0,
547
- "sku_count": 1,
548
- "top_sku": "(none)",
549
- "days_to_pay": 12,
550
- "_created": "2023-08-15 09:17:00",
551
- "lat": 33.749,
552
- "lon": -84.388,
553
- "odoo_status": "Active",
554
- "dba": "Royal",
555
- "ar_outstanding": 9240
556
- }
557
- ]
558
- }
 
1
+ {
2
+ "fields": [
3
+ {
4
+ "key": "customer",
5
+ "label": "Customer",
6
+ "type": "text",
7
+ "source": "odoo",
8
+ "pinned": true,
9
+ "default": true,
10
+ "description": "The customer's name in Odoo. One row per customer who ordered in the last 24 months."
11
+ },
12
+ {
13
+ "key": "odoo_status",
14
+ "label": "Odoo record",
15
+ "type": "status",
16
+ "source": "odoo",
17
+ "default": false,
18
+ "description": "Whether this customer still exists in Odoo. Archived means deleted there."
19
+ },
20
+ {
21
+ "key": "agent",
22
+ "label": "Agent",
23
+ "type": "text",
24
+ "source": "odoo",
25
+ "default": true,
26
+ "description": "The sales agent who owns this account."
27
+ },
28
+ {
29
+ "key": "dba",
30
+ "label": "DBA",
31
+ "type": "select",
32
+ "source": "odoo",
33
+ "default": false,
34
+ "options": [
35
+ "Fisch",
36
+ "Royal",
37
+ "Both"
38
+ ],
39
+ "description": "The brand this customer buys from - Fisch, Royal, or both. Amazon-channel orders are not a DBA."
40
+ },
41
+ {
42
+ "key": "salesperson",
43
+ "label": "Salesperson",
44
+ "type": "text",
45
+ "source": "odoo",
46
+ "default": false,
47
+ "description": "Who keyed in most of this customer's orders — not the Agent, who owns the account."
48
+ },
49
+ {
50
+ "key": "street",
51
+ "label": "Street",
52
+ "type": "text",
53
+ "source": "odoo",
54
+ "default": false,
55
+ "description": "First address line, from res.partner directly - not the geocoder, so a customer the map cannot place still shows its address."
56
+ },
57
+ {
58
+ "key": "street2",
59
+ "label": "Street 2",
60
+ "type": "text",
61
+ "source": "odoo",
62
+ "default": false,
63
+ "description": "Second address line (suite, unit, floor) on the customer's Odoo address."
64
+ },
65
+ {
66
+ "key": "city",
67
+ "label": "City",
68
+ "type": "text",
69
+ "source": "odoo",
70
+ "default": true,
71
+ "description": "City on the customer's Odoo address."
72
+ },
73
+ {
74
+ "key": "state",
75
+ "label": "State",
76
+ "type": "text",
77
+ "source": "odoo",
78
+ "default": true,
79
+ "description": "State or province on the customer's Odoo address."
80
+ },
81
+ {
82
+ "key": "country",
83
+ "label": "Country",
84
+ "type": "text",
85
+ "source": "odoo",
86
+ "default": false,
87
+ "description": "Country on the customer's Odoo address."
88
+ },
89
+ {
90
+ "key": "zip",
91
+ "label": "ZIP",
92
+ "type": "text",
93
+ "source": "odoo",
94
+ "default": false,
95
+ "description": "Postal code on the customer's Odoo address."
96
+ },
97
+ {
98
+ "key": "customer_since",
99
+ "label": "Customer since",
100
+ "type": "date",
101
+ "source": "odoo",
102
+ "default": false,
103
+ "description": "When this customer was first set up in Odoo."
104
+ },
105
+ {
106
+ "key": "tags",
107
+ "label": "Tags",
108
+ "type": "text",
109
+ "source": "odoo",
110
+ "default": false,
111
+ "description": "Odoo labels on this customer, comma-separated."
112
+ },
113
+ {
114
+ "key": "pricelist",
115
+ "label": "Price list",
116
+ "type": "text",
117
+ "source": "odoo",
118
+ "default": false,
119
+ "description": "The price list this customer buys on."
120
+ },
121
+ {
122
+ "key": "payment_terms",
123
+ "label": "Payment terms",
124
+ "type": "text",
125
+ "source": "odoo",
126
+ "default": false,
127
+ "description": "Payment terms on this customer's account — Net 30, for example."
128
+ },
129
+ {
130
+ "key": "last_order",
131
+ "label": "Last order",
132
+ "type": "date",
133
+ "source": "odoo",
134
+ "default": true,
135
+ "description": "Date of the most recent confirmed order."
136
+ },
137
+ {
138
+ "key": "overdue_days",
139
+ "label": "Overdue days",
140
+ "type": "int",
141
+ "source": "odoo",
142
+ "default": true,
143
+ "description": "How many days late this customer is running against their own usual ordering rhythm."
144
+ },
145
+ {
146
+ "_note": "filterable:false — DERIVED ANALYTIC: est_missed is min(cycles missed, 3) x AOV, a score we compute rather than an object the business has, so a condition on it would read as a fact about the customer when it is a fact about our arithmetic. It still displays and still sorts. Until wave 6 this flag also covered the frozen-window presets (revenue_ytd / revenue_ly / orders_24m / aov / yoy_pct); those are now DELETED outright under the owner's no-buildable-presets rule — see _comment. est_missed itself STAYS: no creatable measure or formula reproduces the cadence model behind it.",
147
+ "key": "est_missed",
148
+ "label": "Est. missed $",
149
+ "type": "currency",
150
+ "source": "odoo",
151
+ "default": true,
152
+ "agg": "sum",
153
+ "filterable": false,
154
+ "description": "Estimated sales missed while quiet: missed orders (capped at 3) times average order value. An estimate, not money owed."
155
+ },
156
+ {
157
+ "_note": "wave 21 R1 — KEY UNCHANGED, LABEL RENAMED. The computation is a DISJOINT split (ar.py credit_exposure): this column is only the not-yet-due residual, its sibling is the past-grace residual, and the two sum to the total. Under the label 'AR open $' the majority-late book read as 'Overdue > Open', which is nonsense in AR vocabulary — 'open' universally means the total. The label now says what the number is; the key stays so saved views and filters keep working.",
158
+ "key": "ar_open",
159
+ "label": "AR current $",
160
+ "type": "currency",
161
+ "source": "odoo",
162
+ "default": false,
163
+ "description": "Invoiced money owed but not yet due (a 5-day grace applies before it counts as overdue)."
164
+ },
165
+ {
166
+ "key": "ar_overdue",
167
+ "label": "AR overdue $",
168
+ "type": "currency",
169
+ "source": "odoo",
170
+ "default": false,
171
+ "description": "Invoiced money past due — same basis as the Collections page."
172
+ },
173
+ {
174
+ "_note": "wave 21 R1 — the TOTAL, added beside the rename above. AR current $ + AR overdue $, i.e. what most people mean by 'open AR'. Composed from the same ar.credit_exposure rows the siblings use, so it is transitively reconciled by ar.validate()'s residual read_group tie — no second oracle.",
175
+ "key": "ar_outstanding",
176
+ "label": "AR outstanding $",
177
+ "type": "currency",
178
+ "source": "odoo",
179
+ "default": false,
180
+ "description": "Total invoiced money owed right now: AR current $ plus AR overdue $."
181
+ },
182
+ {
183
+ "key": "ar_exposure",
184
+ "label": "Credit exposure $",
185
+ "type": "currency",
186
+ "source": "odoo",
187
+ "default": false,
188
+ "description": "The most you could be out if they stopped paying today: open, overdue, draft and not-yet-invoiced."
189
+ },
190
+ {
191
+ "key": "ar_aged_1_30",
192
+ "label": "1-30 days $",
193
+ "type": "currency",
194
+ "source": "odoo",
195
+ "default": false,
196
+ "description": "Overdue between 1 and 30 days. The four aging buckets sum to AR overdue $."
197
+ },
198
+ {
199
+ "key": "ar_aged_31_60",
200
+ "label": "31-60 days $",
201
+ "type": "currency",
202
+ "source": "odoo",
203
+ "default": false,
204
+ "description": "Overdue between 31 and 60 days. The four aging buckets sum to AR overdue $."
205
+ },
206
+ {
207
+ "key": "ar_aged_61_90",
208
+ "label": "61-90 days $",
209
+ "type": "currency",
210
+ "source": "odoo",
211
+ "default": false,
212
+ "description": "Overdue between 61 and 90 days. The four aging buckets sum to AR overdue $."
213
+ },
214
+ {
215
+ "key": "ar_aged_90_plus",
216
+ "label": "90+ days $",
217
+ "type": "currency",
218
+ "source": "odoo",
219
+ "default": false,
220
+ "description": "Overdue by more than 90 days. The four aging buckets sum to AR overdue $."
221
+ },
222
+ {
223
+ "key": "days_to_pay",
224
+ "label": "Days to pay",
225
+ "type": "int",
226
+ "source": "odoo",
227
+ "default": false,
228
+ "description": "Average days to pay an invoice in full. Blank means no fully paid invoice yet."
229
+ },
230
+ {
231
+ "key": "top_category",
232
+ "label": "Top category",
233
+ "type": "text",
234
+ "source": "odoo",
235
+ "default": false,
236
+ "description": "The category this customer spent the most on in the last 12 months."
237
+ },
238
+ {
239
+ "key": "top_category_pct",
240
+ "label": "Top category %",
241
+ "type": "pct",
242
+ "source": "odoo",
243
+ "default": false,
244
+ "description": "Share of last-12-months spend that went to the top category."
245
+ },
246
+ {
247
+ "key": "sku_count",
248
+ "label": "SKUs bought",
249
+ "type": "int",
250
+ "source": "odoo",
251
+ "default": false,
252
+ "description": "Distinct products bought in the last 12 months."
253
+ },
254
+ {
255
+ "key": "top_sku",
256
+ "label": "Top SKU",
257
+ "type": "text",
258
+ "source": "odoo",
259
+ "default": false,
260
+ "description": "The product this customer spent the most on in the last 12 months."
261
+ },
262
+ {
263
+ "key": "days_since",
264
+ "label": "Days since order",
265
+ "type": "int",
266
+ "source": "odoo",
267
+ "default": false,
268
+ "description": "Days since the last confirmed order."
269
+ },
270
+ {
271
+ "key": "typical_gap_days",
272
+ "label": "Typical gap days",
273
+ "type": "int",
274
+ "source": "odoo",
275
+ "default": false,
276
+ "description": "Days this customer usually goes between orders, from their own history."
277
+ },
278
+ {
279
+ "key": "notes",
280
+ "label": "Notes",
281
+ "type": "text",
282
+ "source": "overlay",
283
+ "default": false,
284
+ "description": "Your notes on this customer. Saved in this app only, visible only to you."
285
+ }
286
+ ],
287
+ "rows": [
288
+ {
289
+ "pid": 101,
290
+ "customer": "Poppy Flowers",
291
+ "status": "New",
292
+ "agent": "Naomi Linnell Rivera",
293
+ "city": "Charlottesville",
294
+ "state": "Virginia (US)",
295
+ "last_order": "2026-07-21",
296
+ "est_missed": 0,
297
+ "notes": "",
298
+ "country": "United States",
299
+ "zip": "02720",
300
+ "payment_terms": "30 Days",
301
+ "pricelist": "Fisch 1 (USD)",
302
+ "tags": "Royal",
303
+ "customer_since": "2023-01-10",
304
+ "salesperson": "Jessica",
305
+ "ar_open": 0,
306
+ "ar_overdue": 0,
307
+ "ar_exposure": 0,
308
+ "top_category": "Styrofoam",
309
+ "top_category_pct": 0.47,
310
+ "sku_count": 93,
311
+ "top_sku": "AQUAFOAM FLORAL FOAM BRICK | 48-Piece per Pack",
312
+ "days_to_pay": 34,
313
+ "_created": "2023-01-15 09:10:00",
314
+ "lat": 25.7617,
315
+ "lon": -80.1918,
316
+ "odoo_status": "Archived",
317
+ "dba": "Royal",
318
+ "ar_outstanding": 0
319
+ },
320
+ {
321
+ "pid": 102,
322
+ "customer": "Meadow & Vine Wholesale",
323
+ "status": "Growing",
324
+ "agent": "Carla Jimenez",
325
+ "city": "Portland",
326
+ "state": "Oregon (US)",
327
+ "last_order": "2026-07-19",
328
+ "est_missed": 0,
329
+ "notes": "Expanding to a second storefront.",
330
+ "country": "United States",
331
+ "zip": "77041",
332
+ "payment_terms": "Immediate Payment",
333
+ "pricelist": "Royal 1 (USD)",
334
+ "tags": "Royal, Key account",
335
+ "customer_since": "2024-02-11",
336
+ "salesperson": "Naomi",
337
+ "ar_open": 1240.5,
338
+ "ar_overdue": 0,
339
+ "ar_exposure": 1740.5,
340
+ "top_category": "Ribbon",
341
+ "top_category_pct": 0.95,
342
+ "sku_count": 4,
343
+ "top_sku": "2\" X 24\" X 36\" GREEN STYROFOAM BOARD",
344
+ "days_to_pay": null,
345
+ "_created": "2023-02-15 09:11:00",
346
+ "lat": 27.9506,
347
+ "lon": -82.4572,
348
+ "odoo_status": "Active",
349
+ "dba": "Fisch",
350
+ "ar_outstanding": 1240.5
351
+ },
352
+ {
353
+ "pid": 103,
354
+ "customer": "Bluestem Floral Supply",
355
+ "status": "Growing",
356
+ "agent": "Naomi Linnell Rivera",
357
+ "city": "Kansas City",
358
+ "state": "Missouri (US)",
359
+ "last_order": "2026-07-17",
360
+ "est_missed": 0,
361
+ "notes": "",
362
+ "country": "United States",
363
+ "zip": "11219",
364
+ "payment_terms": "60 Days",
365
+ "pricelist": "Royal 1 (USD)",
366
+ "tags": "Fisch",
367
+ "customer_since": "2025-03-12",
368
+ "salesperson": "Karen",
369
+ "ar_open": 0,
370
+ "ar_overdue": 5120.25,
371
+ "ar_exposure": 5120.25,
372
+ "top_category": "Foams & Finishes",
373
+ "top_category_pct": 0.31,
374
+ "sku_count": 27,
375
+ "top_sku": "SATIN RIBBON 2IN",
376
+ "days_to_pay": 61,
377
+ "_created": "2023-03-15 09:12:00",
378
+ "lat": 28.5384,
379
+ "lon": -81.3789,
380
+ "odoo_status": "Active",
381
+ "dba": "Both",
382
+ "ar_outstanding": 5120.25
383
+ },
384
+ {
385
+ "pid": 104,
386
+ "customer": "Camellia Row Florist",
387
+ "status": "Declining",
388
+ "agent": "Devon Marsh",
389
+ "city": "Savannah",
390
+ "state": "Georgia (US)",
391
+ "last_order": "2026-05-30",
392
+ "est_missed": 41200,
393
+ "notes": "Switched some volume to a local grower.",
394
+ "country": "United States",
395
+ "zip": "07649",
396
+ "payment_terms": "30 Days",
397
+ "pricelist": "Fisch 1 (USD)",
398
+ "tags": "(none)",
399
+ "customer_since": "2026-04-13",
400
+ "salesperson": "(none)",
401
+ "ar_open": 8300,
402
+ "ar_overdue": 940,
403
+ "ar_exposure": 11440,
404
+ "top_category": "All",
405
+ "top_category_pct": 1.0,
406
+ "sku_count": 1,
407
+ "top_sku": "(none)",
408
+ "days_to_pay": 12,
409
+ "_created": "2023-04-15 09:13:00",
410
+ "lat": 30.3322,
411
+ "lon": -81.6557,
412
+ "odoo_status": "Active",
413
+ "dba": "Royal",
414
+ "ar_outstanding": 9240
415
+ },
416
+ {
417
+ "pid": 105,
418
+ "customer": "Harborlight Wholesale Blooms",
419
+ "status": "Growing",
420
+ "agent": "Carla Jimenez",
421
+ "city": "Seattle",
422
+ "state": "Washington (US)",
423
+ "last_order": "2026-07-22",
424
+ "est_missed": 0,
425
+ "notes": "Top-10 account.",
426
+ "country": "United States",
427
+ "zip": "33125",
428
+ "payment_terms": "Immediate Payment",
429
+ "pricelist": "Royal 1 (USD)",
430
+ "tags": "Royal",
431
+ "customer_since": "2023-05-14",
432
+ "salesperson": "Jessica",
433
+ "ar_open": 0,
434
+ "ar_overdue": 0,
435
+ "ar_exposure": 0,
436
+ "top_category": "Styrofoam",
437
+ "top_category_pct": 0.47,
438
+ "sku_count": 93,
439
+ "top_sku": "AQUAFOAM FLORAL FOAM BRICK | 48-Piece per Pack",
440
+ "days_to_pay": 34,
441
+ "_created": "2023-05-15 09:14:00",
442
+ "lat": 26.1224,
443
+ "lon": -80.1373,
444
+ "odoo_status": "Active",
445
+ "dba": "",
446
+ "ar_outstanding": 0
447
+ },
448
+ {
449
+ "pid": 106,
450
+ "customer": "Dogwood & Fern Co.",
451
+ "status": "Dormant",
452
+ "agent": "Devon Marsh",
453
+ "city": "Asheville",
454
+ "state": "North Carolina (US)",
455
+ "last_order": "2026-02-11",
456
+ "est_missed": 52400,
457
+ "notes": "No spring order this year.",
458
+ "country": "United States",
459
+ "zip": "90210",
460
+ "payment_terms": "60 Days",
461
+ "pricelist": "Royal 1 (USD)",
462
+ "tags": "Royal, Key account",
463
+ "customer_since": "2024-06-15",
464
+ "salesperson": "Naomi",
465
+ "ar_open": 1240.5,
466
+ "ar_overdue": 0,
467
+ "ar_exposure": 1740.5,
468
+ "top_category": "Ribbon",
469
+ "top_category_pct": 0.95,
470
+ "sku_count": 4,
471
+ "top_sku": "2\" X 24\" X 36\" GREEN STYROFOAM BOARD",
472
+ "days_to_pay": null,
473
+ "_created": "2023-06-15 09:15:00",
474
+ "lat": 27.3364,
475
+ "lon": -82.5307,
476
+ "odoo_status": "Active",
477
+ "dba": "Fisch",
478
+ "ar_outstanding": 1240.5
479
+ },
480
+ {
481
+ "pid": 107,
482
+ "customer": "Verbena Market Florals",
483
+ "status": "Lost",
484
+ "agent": "Naomi Linnell Rivera",
485
+ "city": "Austin",
486
+ "state": "Texas (US)",
487
+ "last_order": "2025-11-04",
488
+ "est_missed": 78300,
489
+ "notes": "Went with a competitor on freight terms.",
490
+ "country": "United States",
491
+ "zip": "08701",
492
+ "payment_terms": "30 Days",
493
+ "pricelist": "Fisch 1 (USD)",
494
+ "tags": "Fisch",
495
+ "customer_since": "2025-07-16",
496
+ "salesperson": "Karen",
497
+ "ar_open": 0,
498
+ "ar_overdue": 5120.25,
499
+ "ar_exposure": 5120.25,
500
+ "top_category": "Foams & Finishes",
501
+ "top_category_pct": 0.31,
502
+ "sku_count": 27,
503
+ "top_sku": "SATIN RIBBON 2IN",
504
+ "days_to_pay": 61,
505
+ "_created": "2023-07-15 09:16:00",
506
+ "lat": null,
507
+ "lon": null,
508
+ "odoo_status": "Active",
509
+ "dba": "Both",
510
+ "ar_outstanding": 5120.25
511
+ },
512
+ {
513
+ "pid": 108,
514
+ "customer": "Larkspur Lane Supply",
515
+ "status": "New",
516
+ "agent": "Carla Jimenez",
517
+ "city": "Denver",
518
+ "state": "Colorado (US)",
519
+ "last_order": "2026-07-14",
520
+ "est_missed": 0,
521
+ "notes": "First order in April.",
522
+ "country": "United States",
523
+ "zip": "60614",
524
+ "payment_terms": "Immediate Payment",
525
+ "pricelist": "Royal 1 (USD)",
526
+ "tags": "(none)",
527
+ "customer_since": "2026-08-17",
528
+ "salesperson": "(none)",
529
+ "ar_open": 8300,
530
+ "ar_overdue": 940,
531
+ "ar_exposure": 11440,
532
+ "top_category": "All",
533
+ "top_category_pct": 1.0,
534
+ "sku_count": 1,
535
+ "top_sku": "(none)",
536
+ "days_to_pay": 12,
537
+ "_created": "2023-08-15 09:17:00",
538
+ "lat": 33.749,
539
+ "lon": -84.388,
540
+ "odoo_status": "Active",
541
+ "dba": "Royal",
542
+ "ar_outstanding": 9240
543
+ }
544
+ ]
545
+ }
 
 
 
 
 
 
 
 
 
 
 
 
 
web/src/alerts/alertsModel.ts CHANGED
@@ -1,418 +1,365 @@
1
- // ---------------------------------------------------------------------------
2
- // alerts/alertsModel.ts — WAVE 20 item 25 (contract C-ALERT): the inbox's PURE
3
- // half. React-free and fetch-free, so `verify_alerts.py` runs it under node.
4
- //
5
- // An alert says "tell me when a record ENTERS this view". The client half is
6
- // small, and every part of it fails silently when it is wrong:
7
- //
8
- // · an unread count taken from `items.length` rather than from the server's
9
- // own `unread` disagrees with the badge the moment a page is capped or a
10
- // read lands in another tab — and a badge that says 3 when the list shows 9
11
- // teaches the reader to ignore the badge;
12
- // · a notification whose `viewId` no longer resolves must open NOTHING rather
13
- // than a wrong view — alerts outlive the views they were made from;
14
- // · `topic` is the surface's scope key ("customer", "product", "ut_…"), and
15
- // the ROUTE is a registry key ("customer_data") — mapping one to the other
16
- // by guesswork sends every click to a page that does not exist.
17
- // ---------------------------------------------------------------------------
18
-
19
- /**
20
- * ⭐⭐ WAVE 32 · T20 · CONTRACT C3 — WHERE AN INBOX ITEM OPENS.
21
- *
22
- * `module` is the destination surface (`"database"` or `"automation"`) and `id` is what to open
23
- * in it. `tab` is the SUB-SELECTION inside that module: the literal `"runs"` for an automation's
24
- * run log, or the VIEW ID to select on a database. One optional key, two destinations.
25
- *
26
- * ⛔ EVERY FIELD IS A PLAIN `string`, NEVER A UNION, and that is alertsModel's wave-9 law rather
27
- * than laziness: the vocabulary is the SERVER's, and a client union over it turns "the server grew
28
- * a module" into "the client silently drops the row". An unknown module is refused by the
29
- * FRAME's dispatcher, out loud, which is a different thing from never arriving.
30
- */
31
- export interface NotificationTarget {
32
- module: string;
33
- id: string;
34
- tab?: string;
35
- }
36
-
37
- /** C3's `kind` vocabulary, mirroring `routes_alerts.NOTIF_KIND_*`. Held as constants so the
38
- * Inbox's branch and the gate's fixtures cannot disagree about the word. */
39
- export const NOTIF_KIND_ALERT = "alert";
40
- export const NOTIF_KIND_AUTOMATION = "automation";
41
- export const NOTIF_KIND_SHARE = "share";
42
-
43
- /** C3's `target.module` vocabulary, and the automation sub-selection. */
44
- export const TARGET_MODULE_DATABASE = "database";
45
- export const TARGET_MODULE_AUTOMATION = "automation";
46
- export const TARGET_TAB_RUNS = "runs";
47
-
48
- /** One notification, as `GET /api/v1/notifications` sends it. */
49
- export interface Notification {
50
- id: string;
51
- alertId: string;
52
- viewId: string;
53
- topic: string;
54
- rowId: string;
55
- /** What entered — the record's own label. */
56
- label: string;
57
- /** What the alert is called, so a row reads without opening anything. */
58
- alertLabel: string;
59
- /** UTC WITH OFFSET (D-18). Kept as the server's STRING: re-formatting it here
60
- * would re-introduce the browser-clock drift the offset exists to remove. */
61
- at: string;
62
- read: boolean;
63
- /**
64
- * WAVE 23 (contract C6) — WHAT KIND of notification this is.
65
- *
66
- * Absent (and every notification written before this wave) means the original one: a record
67
- * ENTERED a watched view, routed by `topic` + `viewId`. `"automation_review"` means a card
68
- * arrived at a review stage and routes by `autoId` instead — a different destination reached
69
- * from the same list.
70
- *
71
- * ⛔ A STRING, NEVER A UNION, and that is the wave-9 law rather than laziness: the vocabulary
72
- * is the SERVER's, and a client union over it turns "the server grew a kind" into "the client
73
- * silently drops the row". Unknown kinds fall through to the view route, which is exactly what
74
- * they did before this field existed.
75
- */
76
- kind?: string;
77
- /** `automation_review` only: the automation whose review stage a card reached. Absent on
78
- * every other kind — and an `automation_review` row that arrives WITHOUT one opens nothing
79
- * rather than guessing, the same posture `viewId` gets. */
80
- autoId?: string;
81
- /** Advisory. A surface that does not scroll to a stage simply selects the automation. */
82
- stageId?: string;
83
- /** How many cards arrived in the batch. C6 queues ONE notification per run naming the count,
84
- * never one per record — so this is the number the row's own text is built from. */
85
- count?: number;
86
- /**
87
- * ⭐ WAVE 32 · C3 — THE HEADER LINE, so the Inbox can be laid out like mail.
88
- *
89
- * `subject` is what the item is ABOUT (the alert's name, the automation's name, the database
90
- * that was shared); `label` stays what HAPPENED (the record that entered, the run summary).
91
- * They were one field, which is why the pane could only ever render a sentence with no sender.
92
- * Absent on a server that predates this wave — {@link subjectOf} falls back rather than
93
- * rendering a blank header.
94
- */
95
- subject?: string;
96
- /**
97
- * ⭐⭐ WAVE 33 · W33-T28 — WHO IT IS FROM. Mail has a sender; this list did not.
98
- *
99
- * A `verifier` reading the finished wave-32 Inbox found the row's sender POSITION occupied by
100
- * `kindLabel(n.kind)` — the literals "Alert" / "Automation" / "Shared with you", a CATEGORY
101
- * standing where a who belongs — and no sender anywhere on the wire. Derived by the server
102
- * (`routes_alerts.notification_view`), never here: only it knows that a share has a person
103
- * behind it and an alert has a machine.
104
- * ⚠ Optional, so a payload from a server that predates this wave keeps today's shape;
105
- * {@link senderOf} falls back rather than rendering a blank From column.
106
- */
107
- sender?: string;
108
- /** ⭐ WAVE 32 · C3 — where clicking it goes. ABSENT when this product cannot resolve a
109
- * destination (an alert on a table this account can no longer route to), and that absence is
110
- * load-bearing: the Inbox renders such a row as plainly unclickable rather than as a click
111
- * that silently does nothing. */
112
- target?: NotificationTarget;
113
- }
114
-
115
- /** One alert, as `GET /api/v1/alerts` sends it. */
116
- export interface Alert {
117
- id: string;
118
- viewId: string;
119
- topic: string;
120
- owner: string;
121
- label: string;
122
- createdAt: string;
123
- /** How many records are in its remembered set right now. */
124
- matched: number;
125
- seeded: boolean;
126
- lastRunAt: string;
127
- lastError: string;
128
- }
129
-
130
- export interface Inbox {
131
- unread: number;
132
- items: Notification[];
133
- /**
134
- * ⭐⭐ WAVE 33 · W33-T28 / D-208 — THE SERVER'S CLOCK, so a stamp can be mail-shaped.
135
- *
136
- * ⛔ It exists so {@link stampText} never reads the BROWSER's clock, which is D-208's exit
137
- * condition word for word. `at` is UTC with its offset (D-18) precisely so every reader sees
138
- * the same instant; asking "is this today?" of a local clock would put the drift back.
139
- * ⚠ Optional: absent, `stampText` returns the old absolute string rather than guessing.
140
- */
141
- now?: string;
142
- }
143
-
144
- export const EMPTY_INBOX: Inbox = { unread: 0, items: [] };
145
-
146
- /**
147
- * ⭐⭐ WAVE 31 · T23 (owner item 1) — WHAT THE PANE IS ENTITLED TO SAY, AS A FUNCTION.
148
- *
149
- * Owner, verbatim: *"Alerts shows notification, but when clicked it says nothing, and it doesn't
150
- * remove the notification number."* Both halves are ONE mechanism. `AlertsPane` held its own
151
- * `inbox` seeded to {@link EMPTY_INBOX} and had no pending state, so between opening the panel
152
- * and `GET /notifications` answering — measured at **3,280 ms live** — and for ever after a
153
- * failed fetch:
154
- * · `items` was `[]`, so the pane printed **"Nothing new."** — a claim, not a wait;
155
- * · `unread` was `0`, so **"Mark all read" was `disabled`**, and the badge the frame had
156
- * already loaded could never be cleared.
157
- * A confident sentence about somebody's inbox, and the one control that would fix it, both
158
- * switched off by the same uninitialised state.
159
- *
160
- * ⛔ IT IS A FUNCTION BECAUSE THE PANE IS TSX AND TSX IS NOT UNDER TEST HERE. `verify_alerts`
161
- * compiles and RUNS this module under node; markup it cannot reach is markup no control can
162
- * mutate. Deciding here means the truth table is asserted and each branch is load-bearing.
163
- */
164
- export type PaneView = "pending" | "rows" | "empty" | "error";
165
-
166
- /**
167
- * `pending` while the first read is in flight · `error` when it failed and we have nothing to
168
- * show · `rows` when there is something · `empty` ONLY when a successful read returned nothing.
169
- *
170
- * ⚠ ROWS WIN OVER AN ERROR, and that is deliberate rather than lax: a refresh that fails while
171
- * the pane already holds notifications should not blank them — the reader loses real information
172
- * to a transient. The error still reaches them as the pane's message line.
173
- */
174
- export function paneView(
175
- phase: "pending" | "ready" | "error",
176
- rowCount: number,
177
- seededUnread = 0
178
- ): PaneView {
179
- if (rowCount > 0) return "rows";
180
- if (phase === "pending") return "pending";
181
- if (phase === "error") return "error";
182
- // ⚠ A SUCCESSFUL READ WITH NO ROWS AND A NON-ZERO COUNT IS NOT "nothing new". The badge says
183
- // there is something; the page we were given does not contain it. Saying "Nothing new" there
184
- // is the same false confidence in a different costume.
185
- return seededUnread > 0 ? "error" : "empty";
186
- }
187
-
188
- /**
189
- * May "Mark all read" be pressed?
190
- *
191
- * ⛔ NOT `unread === 0`, WHICH IS THE SHIPPED BUG. That test asked the PANE's own state — zero
192
- * until its fetch lands, zero for ever if the fetch fails — so the control was dead in exactly
193
- * the situations the owner hit. The question is about the ACCOUNT, so it is asked of the count
194
- * the frame already holds, and a failed read does not take the verb away: `POST /notifications/
195
- * read` with `ids: null` clears the account's inbox whether or not we managed to list it.
196
- */
197
- export function canMarkAll(unread: number, busy = false): boolean {
198
- return !busy && Math.max(0, Math.floor(unread)) > 0;
199
- }
200
-
201
- const str = (v: unknown): string => (typeof v === "string" ? v : "");
202
- const num = (v: unknown): number => (typeof v === "number" && isFinite(v) ? v : 0);
203
-
204
- /**
205
- * ⭐ WAVE 32 · C3 — one wire `target` → a {@link NotificationTarget}, or `null`.
206
- *
207
- * ⛔ BOTH `module` AND `id` ARE REQUIRED, and dropping either test is the failure this guard is
208
- * for: a target with a module and no id dispatches an open request naming NOTHING — a click that
209
- * appears to work and silently does not, which is this repo's most-repeated shape. `null` is
210
- * rendered as an unclickable row, which a reader can SEE.
211
- */
212
- export function parseTarget(raw: unknown): NotificationTarget | null {
213
- if (!raw || typeof raw !== "object") return null;
214
- const t = raw as Record<string, unknown>;
215
- const module = str(t.module).trim();
216
- const id = str(t.id).trim();
217
- if (!module || !id) return null;
218
- const tab = str(t.tab).trim();
219
- return { module, id, ...(tab ? { tab } : {}) };
220
- }
221
-
222
- /**
223
- * `GET /notifications` → the inbox, fail-closed.
224
- *
225
- * ⚠ `unread` COMES FROM THE SERVER, and is not recounted from `items`. The two
226
- * can legitimately differ — the list is what this page holds, the count is what
227
- * the account has — and recomputing it here would make the badge a function of
228
- * whatever the last fetch happened to include.
229
- */
230
- export function parseInbox(body: unknown): Inbox {
231
- const b = (body && typeof body === "object" ? body : {}) as Record<string, unknown>;
232
- const raw = Array.isArray(b.items) ? b.items : [];
233
- const items: Notification[] = [];
234
- for (const item of raw) {
235
- if (!item || typeof item !== "object") continue;
236
- const n = item as Record<string, unknown>;
237
- const id = str(n.id);
238
- // An id-less notification cannot be marked read, so it would sit unread for
239
- // ever and hold the badge up. Dropped, not rendered.
240
- if (!id) continue;
241
- items.push({
242
- id,
243
- alertId: str(n.alertId),
244
- viewId: str(n.viewId),
245
- topic: str(n.topic),
246
- rowId: String(n.rowId ?? ""),
247
- label: str(n.label) || String(n.rowId ?? ""),
248
- alertLabel: str(n.alertLabel),
249
- at: str(n.at),
250
- read: n.read === true,
251
- // WAVE 23 C6 — ADDITIVE and spread-conditional, exactly like `NavPage`'s flags: a payload
252
- // that predates this wave keeps today's shape rather than gaining four `undefined` keys,
253
- // and an `automation_review` row missing its `autoId` is left WITHOUT one rather than
254
- // with an empty string that would render as a real destination.
255
- ...(str(n.kind) ? { kind: str(n.kind) } : {}),
256
- ...(str(n.autoId) ? { autoId: str(n.autoId) } : {}),
257
- ...(str(n.stageId) ? { stageId: str(n.stageId) } : {}),
258
- ...(num(n.count) > 0 ? { count: num(n.count) } : {}),
259
- // ⭐ WAVE 32 · C3 — additive and spread-conditional, exactly like the four above.
260
- ...(str(n.subject) ? { subject: str(n.subject) } : {}),
261
- // ⭐ WAVE 33 · W33-T28 — additive and spread-conditional, like every flag above it.
262
- ...(str(n.sender) ? { sender: str(n.sender) } : {}),
263
- ...(parseTarget(n.target) ? { target: parseTarget(n.target)! } : {}),
264
- });
265
- }
266
- // ⭐ W33-T28 / D-208 — the server's clock rides through, spread-conditionally like every other
267
- // additive key here, so a server that predates this wave yields the same object it always did.
268
- return {
269
- unread: Math.max(0, num(b.unread)),
270
- items,
271
- ...(str(b.now) ? { now: str(b.now) } : {}),
272
- };
273
- }
274
-
275
- /** `GET /alerts` → the alert list, fail-closed. */
276
- export function parseAlerts(body: unknown): Alert[] {
277
- const b = (body && typeof body === "object" ? body : {}) as Record<string, unknown>;
278
- const raw = Array.isArray(b.alerts) ? b.alerts : [];
279
- const out: Alert[] = [];
280
- for (const item of raw) {
281
- if (!item || typeof item !== "object") continue;
282
- const a = item as Record<string, unknown>;
283
- const id = str(a.id);
284
- if (!id) continue;
285
- out.push({
286
- id,
287
- viewId: str(a.viewId),
288
- topic: str(a.topic),
289
- owner: str(a.owner),
290
- label: str(a.label) || "Untitled alert",
291
- createdAt: str(a.createdAt),
292
- matched: num(a.matched),
293
- seeded: a.seeded === true,
294
- lastRunAt: str(a.lastRunAt),
295
- lastError: str(a.lastError),
296
- });
297
- }
298
- return out;
299
- }
300
-
301
- /**
302
- * A topic (the grid's scope key) → the hash route that renders it.
303
- *
304
- * The two built-ins are the only pair that differ, and they differ because the
305
- * REGISTRY names the surface while the GRID names the scope; a user table is its
306
- * own key in both. `null` for anything else: a notification for a topic this
307
- * client cannot route to must do nothing, not navigate somewhere plausible.
308
- */
309
- export function routeForTopic(topic: string): string | null {
310
- const t = str(topic).trim();
311
- if (t === "customer") return "customer_data";
312
- if (t === "product") return "product_data";
313
- if (/^ut_[A-Za-z0-9_]+$/.test(t)) return t;
314
- return null;
315
- }
316
-
317
-
318
-
319
-
320
-
321
-
322
- /**
323
- * The stamp, made readable WITHOUT touching a clock.
324
- *
325
- * ⛔ NO `new Date()`, NO `toLocaleString()`, and that is the whole design. The
326
- * server sends UTC WITH ITS OFFSET (D-18) precisely so every reader sees the
327
- * same instant; parsing it into a browser Date and formatting it back would
328
- * re-introduce the drift the offset exists to remove — a tenant a day ahead
329
- * being told an event happened tomorrow ([[date-window-vocabulary]]). This is
330
- * STRING SURGERY: keep the date and the minutes, drop the seconds and the `T`.
331
- * Anything that does not look like an ISO stamp passes through untouched, so a
332
- * format this function has never seen is shown as sent rather than mangled.
333
- */
334
- const MONTHS = ["Jan", "Feb", "Mar", "Apr", "May", "Jun",
335
- "Jul", "Aug", "Sep", "Oct", "Nov", "Dec"];
336
-
337
- export function stampText(at: string, now?: string): string {
338
- const m = /^(\d{4}-\d{2}-\d{2})T(\d{2}:\d{2})/.exec(str(at));
339
- if (!m) return str(at);
340
- const [, day, hhmm] = m;
341
- // ⭐⭐ W33-T28 / D-208 — MAIL-SHAPED, AND STILL STRING SURGERY.
342
- //
343
- // ⛔ `now` COMES FROM THE SERVER (`inbox.now`), NEVER FROM `new Date()`. That is D-208's exit
344
- // condition word for word — "without reading the browser clock" — and the reason is the same
345
- // one this function has always carried: `at` is sent as UTC WITH its offset (D-18) so every
346
- // reader sees the same instant, and deciding "is this today?" against a browser clock would
347
- // re-introduce exactly the drift the offset removes. Both operands are the server's.
348
- //
349
- // ⚠ AND WITHOUT `now` IT DEGRADES TO THE OLD FORMAT RATHER THAN GUESSING. A caller that has no
350
- // server clock gets `2026-08-13 09:41` — the pre-wave string, unambiguous and never wrong —
351
- // instead of a relative stamp computed from something we do not trust.
352
- const today = /^(\d{4}-\d{2}-\d{2})/.exec(str(now));
353
- if (!today) return `${day} ${hhmm}`;
354
- if (today[1] === day) return hhmm;
355
- const [y, mo, d] = day.split("-");
356
- const month = MONTHS[Number(mo) - 1] || mo;
357
- // The year only when it differs — a mail client does not print "2026" on a message from March.
358
- return today[1].slice(0, 4) === y
359
- ? `${month} ${Number(d)}`
360
- : `${month} ${Number(d)}, ${y}`;
361
- }
362
-
363
- /**
364
- * The badge's text. Never the raw number past 99: a nav row is 236px wide and a
365
- * four-digit badge pushes the label out of it.
366
- */
367
- export function badgeText(unread: number): string {
368
- const n = Math.max(0, Math.floor(unread));
369
- if (n <= 0) return "";
370
- return n > 99 ? "99+" : String(n);
371
- }
372
-
373
- /**
374
- * Apply a read/unread change LOCALLY, mirroring what the server just stored, and
375
- * return the new inbox with the count corrected.
376
- *
377
- * `ids === null` is "all of them" (the API's own convention for mark-all). The
378
- * count is derived from the ITEMS here — deliberately, and it is the one place
379
- * that is right to do so: the server's answer is in flight, and the alternative
380
- * is a badge that keeps its old number until the refetch lands.
381
- */
382
- export function applyRead(inbox: Inbox, ids: string[] | null, read: boolean): Inbox {
383
- const wanted = ids === null ? null : new Set(ids);
384
- const items = inbox.items.map((n) =>
385
- wanted === null || wanted.has(n.id) ? { ...n, read } : n
386
- );
387
- const seenUnread = items.filter((n) => !n.read).length;
388
- // A page can hold fewer notifications than the account has, so a partial read
389
- // must SUBTRACT from the server's count rather than replace it with this
390
- // page's tally — except when marking everything, where zero is the answer.
391
- if (wanted === null) return { unread: read ? 0 : items.length, items };
392
- const changed = inbox.items.filter(
393
- (n) => wanted.has(n.id) && n.read !== read
394
- ).length;
395
- const delta = read ? -changed : changed;
396
- return { unread: Math.max(seenUnread, inbox.unread + delta), items };
397
- }
398
-
399
- /**
400
- * The shell↔rail channel for "make an alert out of this view" — the view rail
401
- * raises it, the frame (which knows the current route, and therefore the topic)
402
- * answers. Same reason as C-SHARE's event: the rail is host-neutral and cannot
403
- * import the shell, and it does not know its own scope key.
404
- */
405
- export const ALERT_CREATE_EVENT = "aios:alert-create";
406
-
407
- export interface AlertCreateRequest {
408
- viewId: string;
409
- label: string;
410
- }
411
-
412
- export function parseAlertCreate(detail: unknown): AlertCreateRequest | null {
413
- if (!detail || typeof detail !== "object") return null;
414
- const d = detail as Record<string, unknown>;
415
- const viewId = str(d.viewId).trim();
416
- if (!viewId) return null;
417
- return { viewId, label: str(d.label).trim() || viewId };
418
- }
 
1
+ // ---------------------------------------------------------------------------
2
+ // alerts/alertsModel.ts — WAVE 20 item 25 (contract C-ALERT): the inbox's PURE
3
+ // half. React-free and fetch-free, so `verify_alerts.py` runs it under node.
4
+ //
5
+ // An alert says "tell me when a record ENTERS this view". The client half is
6
+ // small, and every part of it fails silently when it is wrong:
7
+ //
8
+ // · an unread count taken from `items.length` rather than from the server's
9
+ // own `unread` disagrees with the badge the moment a page is capped or a
10
+ // read lands in another tab — and a badge that says 3 when the list shows 9
11
+ // teaches the reader to ignore the badge;
12
+ // · a notification whose `viewId` no longer resolves must open NOTHING rather
13
+ // than a wrong view — alerts outlive the views they were made from;
14
+ // · `topic` is the surface's scope key ("customer", "product", "ut_…"), and
15
+ // the ROUTE is a registry key ("customer_data") — mapping one to the other
16
+ // by guesswork sends every click to a page that does not exist.
17
+ // ---------------------------------------------------------------------------
18
+
19
+ /**
20
+ * ⭐⭐ WAVE 32 · T20 · CONTRACT C3 — WHERE AN INBOX ITEM OPENS.
21
+ *
22
+ * `module` is the destination surface (`"database"` or `"automation"`) and `id` is what to open
23
+ * in it. `tab` is the SUB-SELECTION inside that module: the literal `"runs"` for an automation's
24
+ * run log, or the VIEW ID to select on a database. One optional key, two destinations.
25
+ *
26
+ * ⛔ EVERY FIELD IS A PLAIN `string`, NEVER A UNION, and that is alertsModel's wave-9 law rather
27
+ * than laziness: the vocabulary is the SERVER's, and a client union over it turns "the server grew
28
+ * a module" into "the client silently drops the row". An unknown module is refused by the
29
+ * FRAME's dispatcher, out loud, which is a different thing from never arriving.
30
+ */
31
+ export interface NotificationTarget {
32
+ module: string;
33
+ id: string;
34
+ tab?: string;
35
+ }
36
+
37
+ /** C3's `kind` vocabulary, mirroring `routes_alerts.NOTIF_KIND_*`. Held as constants so the
38
+ * Inbox's branch and the gate's fixtures cannot disagree about the word. */
39
+ export const NOTIF_KIND_ALERT = "alert";
40
+ export const NOTIF_KIND_AUTOMATION = "automation";
41
+ export const NOTIF_KIND_SHARE = "share";
42
+
43
+ /** C3's `target.module` vocabulary, and the automation sub-selection. */
44
+ export const TARGET_MODULE_DATABASE = "database";
45
+ export const TARGET_MODULE_AUTOMATION = "automation";
46
+ export const TARGET_TAB_RUNS = "runs";
47
+
48
+ /** One notification, as `GET /api/v1/notifications` sends it. */
49
+ export interface Notification {
50
+ id: string;
51
+ alertId: string;
52
+ viewId: string;
53
+ topic: string;
54
+ rowId: string;
55
+ /** What entered — the record's own label. */
56
+ label: string;
57
+ /** What the alert is called, so a row reads without opening anything. */
58
+ alertLabel: string;
59
+ /** UTC WITH OFFSET (D-18). Kept as the server's STRING: re-formatting it here
60
+ * would re-introduce the browser-clock drift the offset exists to remove. */
61
+ at: string;
62
+ read: boolean;
63
+ /**
64
+ * WAVE 23 (contract C6) — WHAT KIND of notification this is.
65
+ *
66
+ * Absent (and every notification written before this wave) means the original one: a record
67
+ * ENTERED a watched view, routed by `topic` + `viewId`. `"automation_review"` means a card
68
+ * arrived at a review stage and routes by `autoId` instead — a different destination reached
69
+ * from the same list.
70
+ *
71
+ * ⛔ A STRING, NEVER A UNION, and that is the wave-9 law rather than laziness: the vocabulary
72
+ * is the SERVER's, and a client union over it turns "the server grew a kind" into "the client
73
+ * silently drops the row". Unknown kinds fall through to the view route, which is exactly what
74
+ * they did before this field existed.
75
+ */
76
+ kind?: string;
77
+ /** `automation_review` only: the automation whose review stage a card reached. Absent on
78
+ * every other kind — and an `automation_review` row that arrives WITHOUT one opens nothing
79
+ * rather than guessing, the same posture `viewId` gets. */
80
+ autoId?: string;
81
+ /** Advisory. A surface that does not scroll to a stage simply selects the automation. */
82
+ stageId?: string;
83
+ /** How many cards arrived in the batch. C6 queues ONE notification per run naming the count,
84
+ * never one per record — so this is the number the row's own text is built from. */
85
+ count?: number;
86
+ /**
87
+ * ⭐ WAVE 32 · C3 — THE HEADER LINE, so the Inbox can be laid out like mail.
88
+ *
89
+ * `subject` is what the item is ABOUT (the alert's name, the automation's name, the database
90
+ * that was shared); `label` stays what HAPPENED (the record that entered, the run summary).
91
+ * They were one field, which is why the pane could only ever render a sentence with no sender.
92
+ * Absent on a server that predates this wave — {@link subjectOf} falls back rather than
93
+ * rendering a blank header.
94
+ */
95
+ subject?: string;
96
+ /** ⭐ WAVE 32 · C3 — where clicking it goes. ABSENT when this product cannot resolve a
97
+ * destination (an alert on a table this account can no longer route to), and that absence is
98
+ * load-bearing: the Inbox renders such a row as plainly unclickable rather than as a click
99
+ * that silently does nothing. */
100
+ target?: NotificationTarget;
101
+ }
102
+
103
+ /** One alert, as `GET /api/v1/alerts` sends it. */
104
+ export interface Alert {
105
+ id: string;
106
+ viewId: string;
107
+ topic: string;
108
+ owner: string;
109
+ label: string;
110
+ createdAt: string;
111
+ /** How many records are in its remembered set right now. */
112
+ matched: number;
113
+ seeded: boolean;
114
+ lastRunAt: string;
115
+ lastError: string;
116
+ }
117
+
118
+ export interface Inbox {
119
+ unread: number;
120
+ items: Notification[];
121
+ }
122
+
123
+ export const EMPTY_INBOX: Inbox = { unread: 0, items: [] };
124
+
125
+ /**
126
+ * ⭐⭐ WAVE 31 · T23 (owner item 1) — WHAT THE PANE IS ENTITLED TO SAY, AS A FUNCTION.
127
+ *
128
+ * Owner, verbatim: *"Alerts shows notification, but when clicked it says nothing, and it doesn't
129
+ * remove the notification number."* Both halves are ONE mechanism. `AlertsPane` held its own
130
+ * `inbox` seeded to {@link EMPTY_INBOX} and had no pending state, so between opening the panel
131
+ * and `GET /notifications` answering — measured at **3,280 ms live** — and for ever after a
132
+ * failed fetch:
133
+ * · `items` was `[]`, so the pane printed **"Nothing new."** — a claim, not a wait;
134
+ * · `unread` was `0`, so **"Mark all read" was `disabled`**, and the badge the frame had
135
+ * already loaded could never be cleared.
136
+ * A confident sentence about somebody's inbox, and the one control that would fix it, both
137
+ * switched off by the same uninitialised state.
138
+ *
139
+ * ⛔ IT IS A FUNCTION BECAUSE THE PANE IS TSX AND TSX IS NOT UNDER TEST HERE. `verify_alerts`
140
+ * compiles and RUNS this module under node; markup it cannot reach is markup no control can
141
+ * mutate. Deciding here means the truth table is asserted and each branch is load-bearing.
142
+ */
143
+ export type PaneView = "pending" | "rows" | "empty" | "error";
144
+
145
+ /**
146
+ * `pending` while the first read is in flight · `error` when it failed and we have nothing to
147
+ * show · `rows` when there is something · `empty` ONLY when a successful read returned nothing.
148
+ *
149
+ * ⚠ ROWS WIN OVER AN ERROR, and that is deliberate rather than lax: a refresh that fails while
150
+ * the pane already holds notifications should not blank them — the reader loses real information
151
+ * to a transient. The error still reaches them as the pane's message line.
152
+ */
153
+ export function paneView(
154
+ phase: "pending" | "ready" | "error",
155
+ rowCount: number,
156
+ seededUnread = 0
157
+ ): PaneView {
158
+ if (rowCount > 0) return "rows";
159
+ if (phase === "pending") return "pending";
160
+ if (phase === "error") return "error";
161
+ // ⚠ A SUCCESSFUL READ WITH NO ROWS AND A NON-ZERO COUNT IS NOT "nothing new". The badge says
162
+ // there is something; the page we were given does not contain it. Saying "Nothing new" there
163
+ // is the same false confidence in a different costume.
164
+ return seededUnread > 0 ? "error" : "empty";
165
+ }
166
+
167
+ /**
168
+ * May "Mark all read" be pressed?
169
+ *
170
+ * ⛔ NOT `unread === 0`, WHICH IS THE SHIPPED BUG. That test asked the PANE's own state — zero
171
+ * until its fetch lands, zero for ever if the fetch fails — so the control was dead in exactly
172
+ * the situations the owner hit. The question is about the ACCOUNT, so it is asked of the count
173
+ * the frame already holds, and a failed read does not take the verb away: `POST /notifications/
174
+ * read` with `ids: null` clears the account's inbox whether or not we managed to list it.
175
+ */
176
+ export function canMarkAll(unread: number, busy = false): boolean {
177
+ return !busy && Math.max(0, Math.floor(unread)) > 0;
178
+ }
179
+
180
+ const str = (v: unknown): string => (typeof v === "string" ? v : "");
181
+ const num = (v: unknown): number => (typeof v === "number" && isFinite(v) ? v : 0);
182
+
183
+ /**
184
+ * ⭐ WAVE 32 · C3 — one wire `target` → a {@link NotificationTarget}, or `null`.
185
+ *
186
+ * ⛔ BOTH `module` AND `id` ARE REQUIRED, and dropping either test is the failure this guard is
187
+ * for: a target with a module and no id dispatches an open request naming NOTHING — a click that
188
+ * appears to work and silently does not, which is this repo's most-repeated shape. `null` is
189
+ * rendered as an unclickable row, which a reader can SEE.
190
+ */
191
+ export function parseTarget(raw: unknown): NotificationTarget | null {
192
+ if (!raw || typeof raw !== "object") return null;
193
+ const t = raw as Record<string, unknown>;
194
+ const module = str(t.module).trim();
195
+ const id = str(t.id).trim();
196
+ if (!module || !id) return null;
197
+ const tab = str(t.tab).trim();
198
+ return { module, id, ...(tab ? { tab } : {}) };
199
+ }
200
+
201
+ /**
202
+ * `GET /notifications` → the inbox, fail-closed.
203
+ *
204
+ * ⚠ `unread` COMES FROM THE SERVER, and is not recounted from `items`. The two
205
+ * can legitimately differ — the list is what this page holds, the count is what
206
+ * the account has — and recomputing it here would make the badge a function of
207
+ * whatever the last fetch happened to include.
208
+ */
209
+ export function parseInbox(body: unknown): Inbox {
210
+ const b = (body && typeof body === "object" ? body : {}) as Record<string, unknown>;
211
+ const raw = Array.isArray(b.items) ? b.items : [];
212
+ const items: Notification[] = [];
213
+ for (const item of raw) {
214
+ if (!item || typeof item !== "object") continue;
215
+ const n = item as Record<string, unknown>;
216
+ const id = str(n.id);
217
+ // An id-less notification cannot be marked read, so it would sit unread for
218
+ // ever and hold the badge up. Dropped, not rendered.
219
+ if (!id) continue;
220
+ items.push({
221
+ id,
222
+ alertId: str(n.alertId),
223
+ viewId: str(n.viewId),
224
+ topic: str(n.topic),
225
+ rowId: String(n.rowId ?? ""),
226
+ label: str(n.label) || String(n.rowId ?? ""),
227
+ alertLabel: str(n.alertLabel),
228
+ at: str(n.at),
229
+ read: n.read === true,
230
+ // WAVE 23 C6 — ADDITIVE and spread-conditional, exactly like `NavPage`'s flags: a payload
231
+ // that predates this wave keeps today's shape rather than gaining four `undefined` keys,
232
+ // and an `automation_review` row missing its `autoId` is left WITHOUT one rather than
233
+ // with an empty string that would render as a real destination.
234
+ ...(str(n.kind) ? { kind: str(n.kind) } : {}),
235
+ ...(str(n.autoId) ? { autoId: str(n.autoId) } : {}),
236
+ ...(str(n.stageId) ? { stageId: str(n.stageId) } : {}),
237
+ ...(num(n.count) > 0 ? { count: num(n.count) } : {}),
238
+ // ⭐ WAVE 32 · C3 — additive and spread-conditional, exactly like the four above.
239
+ ...(str(n.subject) ? { subject: str(n.subject) } : {}),
240
+ ...(parseTarget(n.target) ? { target: parseTarget(n.target)! } : {}),
241
+ });
242
+ }
243
+ return { unread: Math.max(0, num(b.unread)), items };
244
+ }
245
+
246
+ /** `GET /alerts` → the alert list, fail-closed. */
247
+ export function parseAlerts(body: unknown): Alert[] {
248
+ const b = (body && typeof body === "object" ? body : {}) as Record<string, unknown>;
249
+ const raw = Array.isArray(b.alerts) ? b.alerts : [];
250
+ const out: Alert[] = [];
251
+ for (const item of raw) {
252
+ if (!item || typeof item !== "object") continue;
253
+ const a = item as Record<string, unknown>;
254
+ const id = str(a.id);
255
+ if (!id) continue;
256
+ out.push({
257
+ id,
258
+ viewId: str(a.viewId),
259
+ topic: str(a.topic),
260
+ owner: str(a.owner),
261
+ label: str(a.label) || "Untitled alert",
262
+ createdAt: str(a.createdAt),
263
+ matched: num(a.matched),
264
+ seeded: a.seeded === true,
265
+ lastRunAt: str(a.lastRunAt),
266
+ lastError: str(a.lastError),
267
+ });
268
+ }
269
+ return out;
270
+ }
271
+
272
+ /**
273
+ * A topic (the grid's scope key) → the hash route that renders it.
274
+ *
275
+ * The two built-ins are the only pair that differ, and they differ because the
276
+ * REGISTRY names the surface while the GRID names the scope; a user table is its
277
+ * own key in both. `null` for anything else: a notification for a topic this
278
+ * client cannot route to must do nothing, not navigate somewhere plausible.
279
+ */
280
+ export function routeForTopic(topic: string): string | null {
281
+ const t = str(topic).trim();
282
+ if (t === "customer") return "customer_data";
283
+ if (t === "product") return "product_data";
284
+ if (/^ut_[A-Za-z0-9_]+$/.test(t)) return t;
285
+ return null;
286
+ }
287
+
288
+
289
+
290
+
291
+
292
+
293
+ /**
294
+ * The stamp, made readable WITHOUT touching a clock.
295
+ *
296
+ * ⛔ NO `new Date()`, NO `toLocaleString()`, and that is the whole design. The
297
+ * server sends UTC WITH ITS OFFSET (D-18) precisely so every reader sees the
298
+ * same instant; parsing it into a browser Date and formatting it back would
299
+ * re-introduce the drift the offset exists to remove — a tenant a day ahead
300
+ * being told an event happened tomorrow ([[date-window-vocabulary]]). This is
301
+ * STRING SURGERY: keep the date and the minutes, drop the seconds and the `T`.
302
+ * Anything that does not look like an ISO stamp passes through untouched, so a
303
+ * format this function has never seen is shown as sent rather than mangled.
304
+ */
305
+ export function stampText(at: string): string {
306
+ const m = /^(\d{4}-\d{2}-\d{2})T(\d{2}:\d{2})/.exec(str(at));
307
+ return m ? `${m[1]} ${m[2]}` : str(at);
308
+ }
309
+
310
+ /**
311
+ * The badge's text. Never the raw number past 99: a nav row is 236px wide and a
312
+ * four-digit badge pushes the label out of it.
313
+ */
314
+ export function badgeText(unread: number): string {
315
+ const n = Math.max(0, Math.floor(unread));
316
+ if (n <= 0) return "";
317
+ return n > 99 ? "99+" : String(n);
318
+ }
319
+
320
+ /**
321
+ * Apply a read/unread change LOCALLY, mirroring what the server just stored, and
322
+ * return the new inbox with the count corrected.
323
+ *
324
+ * `ids === null` is "all of them" (the API's own convention for mark-all). The
325
+ * count is derived from the ITEMS here — deliberately, and it is the one place
326
+ * that is right to do so: the server's answer is in flight, and the alternative
327
+ * is a badge that keeps its old number until the refetch lands.
328
+ */
329
+ export function applyRead(inbox: Inbox, ids: string[] | null, read: boolean): Inbox {
330
+ const wanted = ids === null ? null : new Set(ids);
331
+ const items = inbox.items.map((n) =>
332
+ wanted === null || wanted.has(n.id) ? { ...n, read } : n
333
+ );
334
+ const seenUnread = items.filter((n) => !n.read).length;
335
+ // A page can hold fewer notifications than the account has, so a partial read
336
+ // must SUBTRACT from the server's count rather than replace it with this
337
+ // page's tally — except when marking everything, where zero is the answer.
338
+ if (wanted === null) return { unread: read ? 0 : items.length, items };
339
+ const changed = inbox.items.filter(
340
+ (n) => wanted.has(n.id) && n.read !== read
341
+ ).length;
342
+ const delta = read ? -changed : changed;
343
+ return { unread: Math.max(seenUnread, inbox.unread + delta), items };
344
+ }
345
+
346
+ /**
347
+ * The shell↔rail channel for "make an alert out of this view" — the view rail
348
+ * raises it, the frame (which knows the current route, and therefore the topic)
349
+ * answers. Same reason as C-SHARE's event: the rail is host-neutral and cannot
350
+ * import the shell, and it does not know its own scope key.
351
+ */
352
+ export const ALERT_CREATE_EVENT = "aios:alert-create";
353
+
354
+ export interface AlertCreateRequest {
355
+ viewId: string;
356
+ label: string;
357
+ }
358
+
359
+ export function parseAlertCreate(detail: unknown): AlertCreateRequest | null {
360
+ if (!detail || typeof detail !== "object") return null;
361
+ const d = detail as Record<string, unknown>;
362
+ const viewId = str(d.viewId).trim();
363
+ if (!viewId) return null;
364
+ return { viewId, label: str(d.label).trim() || viewId };
365
+ }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
web/src/automation/AutomationBuilder.tsx CHANGED
The diff for this file is too large to render. See raw diff
 
web/src/automation/AutomationDetail.tsx CHANGED
@@ -112,17 +112,6 @@ interface Props {
112
  * forever.
113
  */
114
  liveStep: string;
115
- /**
116
- * ⭐ W33 (C-18) — WHICH PANE THE INBOX ASKED FOR: `"runs"`, `"properties"`, or `""`.
117
- *
118
- * ⛔ REQUIRED, not optional, and that is deliberate. An optional pane hint defaulting to `""`
119
- * reads exactly like "no hint was sent" — which is the state the field was already in for the
120
- * whole of its life, declared on the wire and consumed by nobody. Making it required means the
121
- * one caller has to decide, and a second caller cannot forget.
122
- * ⚠ Consumed ONCE, as `aside`'s initial value; see its note. A notification that kept forcing
123
- * the pane would take the switch away from the person using it.
124
- */
125
- openTab: "properties" | "runs" | "";
126
  /**
127
  * ⭐ W31-T31 — `fresh` is the definition the WRITE JUST RETURNED, when the caller has one.
128
  * Given it, the surface corrects that one row; without it, it falls back to a full reload. Every
@@ -183,7 +172,6 @@ export default function AutomationDetail({
183
  vocab,
184
  tick,
185
  liveStep,
186
- openTab,
187
  onSaved,
188
  onDeleted,
189
  }: Props) {
@@ -317,20 +305,7 @@ export default function AutomationDetail({
317
  * the rarer errand; opening it to change something is the common one, and the panel that
318
  * appears should be the one a click on a step is about to fill.
319
  */
320
- /**
321
- * ⭐ W33 (C-18) — AN INBOX ITEM MAY ASK FOR THE RUNS PANE, and until now it asked into the void.
322
- *
323
- * ⛔ AN INITIAL VALUE, NOT A CONTROLLED ONE, and the difference is the whole of why this is safe.
324
- * `openTab` is consumed exactly once, when this component mounts for an automation (the parent
325
- * passes `key={active.id}`, so opening a different automation is a fresh mount and a fresh
326
- * read). Making `aside` follow the prop on every render would take the switch away from the
327
- * person the instant after they used it — the notification would keep dragging them back to a
328
- * pane they had just left.
329
- * ⚠ AND THE DEFAULT IS UNCHANGED. With no `openTab` this is `"properties"`, for the reason the
330
- * note above gives: opening an automation to change something is the common errand.
331
- */
332
- const [aside, setAside] = useState<"properties" | "runs">(
333
- openTab === "runs" ? "runs" : "properties");
334
  /**
335
  * The switch, drawn in BOTH panel heads so it does not move when the panel does.
336
  *
 
112
  * forever.
113
  */
114
  liveStep: string;
 
 
 
 
 
 
 
 
 
 
 
115
  /**
116
  * ⭐ W31-T31 — `fresh` is the definition the WRITE JUST RETURNED, when the caller has one.
117
  * Given it, the surface corrects that one row; without it, it falls back to a full reload. Every
 
172
  vocab,
173
  tick,
174
  liveStep,
 
175
  onSaved,
176
  onDeleted,
177
  }: Props) {
 
305
  * the rarer errand; opening it to change something is the common one, and the panel that
306
  * appears should be the one a click on a step is about to fill.
307
  */
308
+ const [aside, setAside] = useState<"properties" | "runs">("properties");
 
 
 
 
 
 
 
 
 
 
 
 
 
309
  /**
310
  * The switch, drawn in BOTH panel heads so it does not move when the panel does.
311
  *
web/src/automation/AutomationSurface.tsx CHANGED
@@ -1,806 +1,782 @@
1
- // ---------------------------------------------------------------------------
2
- // automation/AutomationSurface.tsx — the Automation surface (contract C-AUTONAV).
3
- //
4
- // ⛔ THE CONTRACT THIS FILE GUARANTEES, verbatim: this module path, a DEFAULT
5
- // export, and NO PROPS. The shell mounts `<AutomationSurface />` and hands it
6
- // nothing; it fetches its own data from `/api/v1/automations`. That is the whole
7
- // interface, and it is stated here because the mount line lives in a file this
8
- // session does not own — a named export would cost a cross-session round trip.
9
- //
10
- // "Automation replaces the Views rail" (owner) is implemented the way the wave
11
- // scout established: the secondary rail is NOT a shell concept — each surface
12
- // owns its own. So this renders a sibling of `.cg-views` built from the same
13
- // `--lp-rail-w` tokens and the same fold behaviour, and the Views rail is
14
- // untouched.
15
- // ---------------------------------------------------------------------------
16
- import type { KeyboardEvent as ReactKeyboardEvent } from "react";
17
- import { Suspense, lazy, useCallback, useEffect, useRef, useState } from "react";
18
-
19
- import type { AutomationOpenDetail } from "../apiContract";
20
- import { AUTOMATION_OPEN_EVENT } from "../apiContract";
21
- // ⭐ WAVE 26 · ITEM 18 / R14 + C7 — THE LOOPABLE LOOP MARK, CONSUMED, NEVER REDRAWN.
22
- // The owner's reason is the mark's own geometry: *"these automations are basically loops."*
23
- // C7 says consume the existing icon component and do not inline a new SVG path, and
24
- // `shell/Brand.tsx`'s header says why in more detail than a rule could: the mark's one source of
25
- // truth is generated, it has already been hand-redrawn once, and the copy silently painted LAST
26
- // WAVE'S BRAND while a comment asserted parity. So this imports the component that POINTS at the
27
- // generated artifact. [[loopable-nav-logo-toggle]] — one element paints the mark.
28
- import { Mark } from "../shell/Brand";
29
- import type { Automation, AutomationList } from "./automationApi";
30
- import {
31
- AutomationError,
32
- cachedAutomations,
33
- createAutomation,
34
- listAutomations,
35
- liveStepOf,
36
- patchAutomation,
37
- rememberAutomation,
38
- } from "./automationApi";
39
-
40
- /**
41
- * ⭐⭐ WAVE 30 · T21 (owner items 4 + 5) — THE EDITOR STOPS SHIPPING INSIDE THE LIST.
42
- *
43
- * ⛔ THIS `lazy()` IS THE TICKET. `AutomationDetail` was a STATIC import, and it statically pulls
44
- * `AutomationBuilder` (which in turn pulls `CondBuilder`, `PresetPlan` and `TriggerPicker`),
45
- * `AutomationFind` and `AutomationTrigger` — so the whole editor was linked into the one chunk a
46
- * person must download before the RAIL can paint a list of names. MEASURED at 88,771 B, against
47
- * `HomePage` at 4,814 and `ConnectorsPage` at 3,551. That is the "Automation takes forever to
48
- * load" complaint, and it had survived two waves because nothing asserted a number.
49
- *
50
- * ⛔ AND IT NEEDS ITS OWN `<Suspense>`, NOT `Shell.tsx`'s. `Lazily` already wraps this whole
51
- * surface, but a Suspense boundary catches ANY suspending descendant — so without an inner
52
- * boundary, opening an automation would suspend the boundary that owns the RAIL and blank the
53
- * list you just clicked in. The inner one keeps the rail painted while the editor streams in.
54
- * `fallback={null}` is deliberate and is `Lazily`'s own choice, not a shortcut: the work area is
55
- * simply not there for the moment the chunk is in flight, which is what every other route in this
56
- * product already does. ⚠ A skeleton here would be a NEW loading screen in the same wave that
57
- * exists to remove one (T22).
58
- * ⚠ Default export, checked: `Shell.tsx:53` records that `lazy()` over a NAMED export fails at
59
- * runtime, on click, and only for that one surface — `AutomationDetail` exports default.
60
- */
61
- const AutomationDetail = lazy(() => import("./AutomationDetail"));
62
-
63
- /**
64
- * The run poll's cadence (item 6). Named constants because they are a MEASURED trade-off, not a
65
- * taste: `POLL_MAX_MS` is the longest a finished run can still look live, and it is the only
66
- * cost the backoff has.
67
- */
68
- const POLL_MIN_MS = 2500;
69
- const POLL_MAX_MS = 8000;
70
-
71
- /*
72
- * ⭐ WAVE 26 · ITEM 18 / R14 — `stateTitle` STOOD HERE AND IS DELETED WITH THE DOT IT DESCRIBED.
73
- *
74
- * It composed the dot's tooltip ("Running now — <step>", "Last run failed — <summary>"), so it had
75
- * exactly one reader and no reason to outlive it. Deleting the render and keeping the composer is
76
- * how a file accumulates functions that look live and are not — and `noUnusedLocals` would have
77
- * caught this one, which is not a reason to lean on it: the NEXT such function might still have a
78
- * second caller and compile fine.
79
- *
80
- * ⛔ WHY THE TOOLTIP WAS NOT MOVED ONTO THE MARK INSTEAD. R14 is "same mark on every automation;
81
- * no status colour". Hanging "Last run failed" off a mark that is deliberately status-free just
82
- * moves the status channel into `title`/`aria-label`, where it is worse: invisible to the eye,
83
- * announced to a screen reader, and contradicting the visual. The run state has a home one click
84
- * away — the Builder's own "Last run succeeded / failed" lines, which R13 explicitly keeps.
85
- *
86
- * ⚠ `stateOf` ITSELF IS UNTOUCHED in `automationApi.ts`. `Shell.tsx:64/1095` imports it and
87
- * `liveStepOf` calls it — deleting the export to tidy up this file would break a fence I do not own.
88
- */
89
-
90
- /**
91
- * The rail's second line — ONE fact, not two.
92
- *
93
- * ⚠ It carried "Next 2026-08-04 06:00 · last 2026-08-04 00:20", and the rail is `--lp-rail-w`
94
- * wide, so it rendered as "Next 2026-08-04 06:00 · l…" — a second line whose only complete word
95
- * was "Next". Two facts that both truncate are worth less than one that fits. The forward-looking
96
- * one wins when a schedule exists (it is the question the rail answers: does this run itself?),
97
- * the last run otherwise. The full picture is one click away in the editor's run history, and the
98
- * row's `title` already says both.
99
- *
100
- * Caught by READING the screenshot: the DOM was correct and the CSS ellipsis was doing exactly
101
- * its job. [[ui-invisible-to-assertions]] — judge the pixels even when everything is green.
102
- */
103
- function railSubtitle(a: Automation): string {
104
- /*
105
- * ⭐ WAVE 24 (owner item 6, "Run once now is laggy / looks stuck") — WHILE IT IS RUNNING,
106
- * THE ONE FACT WORTH THE LINE IS WHAT IT IS DOING.
107
- *
108
- * MEASURED, not guessed (`scratchpad/perf_automations.py`, mailbox D-5): the engine has been
109
- * publishing a live step the whole time (`status.step`, `routes_automation.py:50-51`) and NO
110
- * SURFACE HAS EVER RENDERED IT. So a `discover_instagram` run parked in its legitimate
111
- * `BD_FILTER_WAIT = 120 s` vendor wait and a genuinely hung thread were pixel-identical: a
112
- * pulsing blue dot and a subtitle still reciting the schedule. "Looks stuck" was not a
113
- * performance problem — the poll costs 23 ms — it was the product declining to say.
114
- *
115
- * It takes the line rather than joining it, for the reason this function's note already
116
- * gives: the rail is `--lp-rail-w` wide and two facts that both truncate are worth less than
117
- * one that fits. The schedule is still one click away and the dot's `title` carries both.
118
- */
119
- const step = liveStepOf(a);
120
- if (step) return step;
121
- if (a.schedule?.enabled) return a.nextRunAt ? `Next ${a.nextRunAt}` : "Scheduled";
122
- /*
123
- * ⭐ WAVE 26 · ITEM 19 / R13 — `Last run 2026-08-06 13:58` IS DELETED FROM THIS LINE.
124
- *
125
- * ⛔ SCOPE, because R13 draws a line that is easy to over-read: the ruling names THIS secondary
126
- * line under the automation's name. The Builder's "Last run succeeded / failed" RESULT lines
127
- * (`AutomationBuilder.tsx:1676`, `:1679`) are a different surface and are explicitly NOT in
128
- * scope — they are the answer to "Run once now", so deleting them would leave a button with no
129
- * outcome.
130
- *
131
- * "Manual only" STAYS, and the distinction is the ruling's own: a dated run record is a
132
- * changing FACT ABOUT THE PAST, while "this one has no schedule" is a stable property of the
133
- * automation — the same question `Next …`/`Scheduled` answers for its siblings. Dropping it too
134
- * would leave a manual automation with a blank second line and nothing saying why.
135
- */
136
- return "Manual only";
137
- }
138
-
139
- /*
140
- * ⛔ W23-W5's LISTENER IS AT MODULE SCOPE, AND THAT IS THE POINT — not a stylistic choice.
141
- *
142
- * The frame's click-through sets `window.location.hash = "#/automation"` and signals on the
143
- * VERY NEXT LINE (Shell.tsx:1647-1650, and its comment is right that the order matters). But a
144
- * hash write does not mount anything synchronously: `hashchange` is delivered as a task, the
145
- * router state then updates, React renders, and only THEN does a component effect subscribe.
146
- * A listener registered in `useEffect` therefore misses every click that arrives from another
147
- * page — which is every click, since a reader looking at Alerts is by definition not already
148
- * on this surface. The event would dispatch into nothing and every gate would stay green:
149
- * exactly the wave-20 item-25 failure the contract's own note describes, reproduced one layer
150
- * down.
151
- *
152
- * This module is imported statically by the shell, so this listener exists from app start. It
153
- * LATCHES the request; the component consumes the latch when it mounts and hears live events
154
- * while it is mounted. A request nobody claims is dropped on the next one — the latch is a
155
- * one-slot mailbox, never a queue.
156
- */
157
- let pendingOpen: AutomationOpenDetail | null = null;
158
- const openSubscribers = new Set<(detail: AutomationOpenDetail) => void>();
159
-
160
- if (typeof window !== "undefined") {
161
- window.addEventListener(AUTOMATION_OPEN_EVENT, (event) => {
162
- const detail = (event as CustomEvent<AutomationOpenDetail>).detail;
163
- if (!detail?.autoId) return;
164
- if (openSubscribers.size) {
165
- for (const notify of openSubscribers) notify(detail);
166
- return;
167
- }
168
- pendingOpen = detail;
169
- });
170
- }
171
-
172
- /*
173
- * ⛔ THE `AUTOMATION_CREATE_EVENT` LISTENER STOOD HERE AND IS DELETED WITH ITS SIGNALLER
174
- * (wave 25 item 5a, ruling R8) — latch, subscriber set and all.
175
- *
176
- * Wave 24 built it to close the opposite defect: the event was declared, signalled from
177
- * `Shell.tsx`, and consumed NOWHERE, so all three "Automated database" doors navigated to
178
- * `#/automation` and then did nothing, in production, with every gate green. R8 now deletes those
179
- * three doors — creating a database and pointing an automation at it are two acts — and they were
180
- * the event's only signaller.
181
- *
182
- * ⛔ SO THIS SIDE GOES TOO, IN THE SAME CHANGE. A listener with no signaller is the same defect
183
- * read from the other end: it compiles, it costs nothing at runtime, and it reads to the next
184
- * person as a live channel. The rule that catches both is now DERIVED rather than remembered —
185
- * `verify_automation_ui.py` enumerates every `AUTOMATION_*_EVENT` in `apiContract.ts` and demands
186
- * a signal site AND a listener site for each.
187
- *
188
- * ⚠ NOTHING ABOUT CREATING AN AUTOMATION IS LOST. `createAndOpen` is untouched and still has two
189
- * doors, both on this surface: the rail's button and the empty state's. What is gone is the claim
190
- * that a database can be created BY asking for an automation.
191
- */
192
-
193
- /**
194
- * A default name for a brand-new automation, and it has to be one the server ACCEPTS.
195
- *
196
- * MEASURED against the live route rather than assumed (`scratchpad/probe_names.py`): a create
197
- * with no name is a 400 (`name the automation`), and duplicate names are ALLOWED. So the name
198
- * cannot be omitted, and a fixed literal would stack "New automation" three deep in a rail that
199
- * sorts by name with nothing to tell the rows apart. Numbering from the existing list is a
200
- * client-side convenience over a server that does not care: if two tabs race, the loser gets a
201
- * duplicate name, which is legal, visible and renameable — never an error the user has to read.
202
- */
203
- function nextAutomationName(existing: Automation[]): string {
204
- const taken = new Set(existing.map((a) => (a.name || "").trim().toLowerCase()));
205
- for (let n = existing.length + 1; ; n += 1) {
206
- const candidate = `Automation ${n}`;
207
- if (!taken.has(candidate.toLowerCase())) return candidate;
208
- }
209
- }
210
-
211
- export default function AutomationSurface() {
212
- /* ⭐ T22 — SEEDED FROM THE CLIENT MEMO, so a revisit inside the freshness window never passes
213
- through `data === null` and therefore never paints the skeleton. The lazy initialiser runs
214
- once, on mount, BEFORE the first paint — `useState(cachedAutomations())` would call it on
215
- every render instead, which is the same value at needless cost. A cold start still returns
216
- null and still gets the skeleton: it is made rare, not removed (see `automationApi.ts`). */
217
- const [data, setData] = useState<AutomationList | null>(() => cachedAutomations());
218
- const [error, setError] = useState("");
219
- const [activeId, setActiveId] = useState<string>("");
220
- const [railShut, setRailShut] = useState(false);
221
- const [busy, setBusy] = useState("");
222
- /**
223
- * ⭐ WAVE 24 / C-CREATE — a create is now a ROUND TRIP, not a local wizard, so the door has to
224
- * say it is busy. Without this a second click while the POST is in flight makes a second
225
- * automation, and the server allows duplicate names, so the user gets two rows that look
226
- * identical and no error to explain either of them.
227
- */
228
- const [creatingNow, setCreatingNow] = useState(false);
229
- /** An open request waiting for the list to arrive (see the resolver below). */
230
- const [openRequest, setOpenRequest] = useState<AutomationOpenDetail | null>(null);
231
- /* ⛔ `createRequest` LEFT WITH THE EVENT THAT SET IT (wave 25, R8) — see the tombstone above. */
232
- const listRef = useRef<HTMLDivElement | null>(null);
233
-
234
- /**
235
- * ⛔ C14 LEG 2 — THE ID THE USER LAST ASKED FOR, and it is a ref because it has to be
236
- * written SYNCHRONOUSLY, inside the click, before any promise that was already in flight
237
- * can resolve. State would not do: a resolution racing React's commit would read the
238
- * previous value, which is the exact window this guard exists to close.
239
- */
240
- const wantedId = useRef("");
241
-
242
- /**
243
- * ⭐ THE ONLY PLACE THE RAIL'S SELECTION MOVES ON PURPOSE. Every deliberate change of
244
- * automation — a rail click, a create, a delete — goes through here, so "what the user
245
- * asked for" and "what is on screen" are written in one statement and cannot drift apart.
246
- * A resolution that wants to steer the rail compares itself against `wantedId` instead.
247
- */
248
- const select = useCallback((id: string, _stage = "") => {
249
- wantedId.current = id;
250
- setActiveId(id);
251
- }, []);
252
-
253
- /**
254
- * ⭐⭐ W33 (C-18, from lane C) — WHICH PANE AN INBOX ITEM ASKED FOR, and it was DROPPED here.
255
- *
256
- * ⛔ THE PAYLOAD WAS DECLARED, TRANSPORTED, AND IGNORED AT THE LAST HOP.
257
- * `routes_alerts.py` sets `tab: "runs"` on a notification's target, `inboxModel.routeForTarget`
258
- * forwards it and `Shell.tsx` signals `{autoId, tab}` — and this file read `openRequest.tab`
259
- * NOWHERE. So a person clicking *"12 new creators found"* to see the RUN landed on Properties,
260
- * which is [[flag-shipped-without-its-writer]] with the writer present and the READER missing.
261
- * ⚠ NOT FIXED BY DELETING `tab` FROM THE CONTRACT, which was the other option C offered: the
262
- * field is the only thing that carries "you clicked a run, so show runs", and an inbox that
263
- * lands you on the wrong pane is the complaint, not the field.
264
- * ⚠ IT IS A ONE-SHOT REQUEST, NOT A CONTROLLED VALUE. Forcing the pane on every render would
265
- * take the pane control away from the person the moment after they used it — so this is
266
- * consumed by `AutomationDetail` as an INITIAL value keyed on the automation, and cleared here
267
- * with the request that carried it.
268
- */
269
- const [openTab, setOpenTab] = useState<"properties" | "runs" | "">("");
270
-
271
- /**
272
- * ⛔ C14 LEG 3 (the paint half) — A STALE READ MAY NOT REPAINT.
273
- *
274
- * Every call takes the next generation; only the newest one is allowed to `setData`. The
275
- * defect this closes is not cosmetic: a poll issued BEFORE a delete resolves AFTER it, and
276
- * a list that still contains the deleted automation puts a row back in the rail that the
277
- * user has just watched disappear. Same for a toggle, and same for a create.
278
- *
279
- * ⚠ THE RETURN VALUE IS NOT GUARDED, deliberately. The caller asked a question and gets
280
- * its own answer — coupling the two would mean a create whose `load()` was overtaken by a
281
- * poll silently failed to select the automation it had just made.
282
- */
283
- const gen = useRef(0);
284
-
285
- const load = useCallback(async (abort?: AbortSignal) => {
286
- const mine = ++gen.current;
287
- try {
288
- const next = await listAutomations(abort);
289
- if (mine === gen.current) {
290
- setData(next);
291
- setError("");
292
- }
293
- return next;
294
- } catch (e) {
295
- if ((e as Error)?.name === "AbortError") return null;
296
- if (mine === gen.current)
297
- setError(
298
- e instanceof AutomationError
299
- ? e.message
300
- : "The automation service did not answer."
301
- );
302
- return null;
303
- }
304
- }, []);
305
-
306
- /**
307
- * ⭐⭐ WAVE 31 · T31 — one freshly-written automation replaces its own row, with no round trip.
308
- *
309
- * ⚠ IT TAKES THE GENERATION, and that is not decoration. `load`'s guard exists so a stale READ
310
- * cannot repaint; a WRITE response is by definition newer than any read already in flight, so it
311
- * must win the same race rather than sit outside it. Without the bump, a `GET /automations`
312
- * issued before the save could resolve after the merge and paint the pre-save row back — the
313
- * exact ghost `gen` was introduced for, arriving through the new door.
314
- *
315
- * ⚠ Returns FALSE when there is nothing to merge into (a cold surface has no memo), and the
316
- * caller then does a real load. Painting a one-row list would be worse than a round trip.
317
- */
318
- const mergeSaved = useCallback((fresh: Automation) => {
319
- const next = rememberAutomation(fresh);
320
- if (!next) return false;
321
- gen.current += 1;
322
- setData(next);
323
- setError("");
324
- return true;
325
- }, []);
326
-
327
- useEffect(() => {
328
- const ac = new AbortController();
329
- void load(ac.signal);
330
- return () => ac.abort();
331
- }, [load]);
332
-
333
- /**
334
- * ⭐⭐ WAVE 24 / C-CREATE (a) — CREATE AND OPEN. This REPLACED the three-step wizard.
335
- *
336
- * R6 is what makes it possible: `plain` is a real kind with no machine graph nodes, and it is
337
- * what `POST /automations` stores when the body names none. So "new automation" stopped being
338
- * a question ("which of three kinds?" — two of which can no longer be created at all) and
339
- * became what it says: a new automation, open, on its trigger picker. Choosing a SOURCE is now
340
- * picking the `ig_profile_match` trigger, which is where that choice belongs.
341
- *
342
- * ⛔ THE GUARD IS A REF, not the `creatingNow` state, and this file already carries the scar
343
- * that explains why (C14 leg 2): a state read inside a click closure is the value from the
344
- * last render, so two fast clicks both see `false` and both POST. Duplicate names are legal
345
- * server-side, so the user would get two identical rows and no error. The ref is written
346
- * synchronously, inside the click, before anything can await.
347
- */
348
- const creatingRef = useRef(false);
349
- const createAndOpen = useCallback(async () => {
350
- if (creatingRef.current) return;
351
- creatingRef.current = true;
352
- setCreatingNow(true);
353
- try {
354
- const res = await createAutomation({
355
- name: nextAutomationName(data?.automations || []),
356
- });
357
- const id = res?.automation?.id || "";
358
- // The create flow is the one caller allowed to name a different id (C14 leg 2) — the
359
- // automation did not exist when the click happened, so there is nothing to race with.
360
- const next = await load();
361
- if (id && next) select(id);
362
- } catch (e) {
363
- setError(
364
- e instanceof AutomationError ? e.message : "The automation was not created."
365
- );
366
- } finally {
367
- creatingRef.current = false;
368
- setCreatingNow(false);
369
- }
370
- }, [data, load, select]);
371
-
372
- // A run is a background thread on the server, so the surface has to ASK whether it
373
- // finished. Polling only while something is actually running keeps an idle surface
374
- // silent — a fixed interval would be a request every few seconds forever, for a page
375
- // whose contents change a handful of times a day.
376
- //
377
- // ⛔ C14 LEG 3 (the abort half). It used to call `load()` bare — no signal, nothing to
378
- // cancel — so a request the interval had already issued kept going after the effect that
379
- // owned it was gone. One controller per effect run, aborted with the interval, means a poll
380
- // cannot outlive the condition that justified it. The generation guard inside `load` covers
381
- // the rest: a response that survives the abort still cannot repaint over a newer one.
382
- //
383
- // ⭐ WAVE 24 (item 6, C-PERF) — THE INTERVAL BECAME A BACKOFF, and the measurement is why it is
384
- // a SMALL change rather than the big one the contract's hypothesis 2 asked for.
385
- //
386
- // MEASURED (`scratchpad/perf_automations.py`, mailbox D-5): this poll costs ~23 ms and 24 KB
387
- // for a ten-automation tenant, and the engine work the hypothesis blamed — a `graph()` rebuild
388
- // per automation — is **41 µs each**, i.e. under 4% of the request. The payload was never the
389
- // problem, so nothing here gets a cheaper endpoint.
390
- //
391
- // What the numbers DO indict is the aggregate: a discovery run legitimately blocks up to
392
- // `BD_FILTER_WAIT` = 120 s, and two independent 2.5 s polls across this file and the detail
393
- // spend ~96 requests and ~2.3 MB over that window — in the same single process the run itself
394
- // is a thread in. So the delay grows 2.5 → 5 → 8 s and stops there.
395
- //
396
- // ⚠ NOT "to learn nothing", and the distinction is the whole justification. That WAS true when
397
- // this file rendered no live step: the poll's only observable effect was a dot that had already
398
- // been pulsing for two minutes. It stopped being true in this same change — `liveStepOf` now
399
- // paints `status.step`, so a poll carries the one fact worth having. The backoff is therefore
400
- // NOT "stop asking a pointless question"; it is FEWER ROUND TRIPS FOR THE SAME INFORMATION, on
401
- // a step text that changes every few seconds at most, not every 2.5.
402
- // The cap is deliberately low: it bounds how long a FINISHED run can still look live, which is
403
- // the only thing a backoff can make worse, and 8 s of that is worth ~⅔ fewer requests.
404
- //
405
- // ⛔ THE KEY IS THE RUNNING SET, NOT A BOOLEAN, and that is what makes the reset correct: the
406
- // effect re-runs — and the delay drops back to 2.5 s — the moment a run starts or finishes, so
407
- // a user who clicks Run now gets the fast cadence again instead of inheriting the tail of the
408
- // previous run's backoff. A bare `anyRunning` boolean cannot see the second run start.
409
- // A step text changing does NOT re-key it, so the interval never thrashes.
410
- //
411
- // C14 LEG 3 (the abort half) is unchanged and still load-bearing: one controller per effect
412
- // run, aborted with the timer, so a poll cannot outlive the condition that justified it. The
413
- // generation guard inside `load` covers the rest.
414
- const runningKey = (data?.automations || [])
415
- .filter((a) => a.running)
416
- .map((a) => a.id)
417
- .sort()
418
- .join(",");
419
- useEffect(() => {
420
- if (!runningKey) return undefined;
421
- const ac = new AbortController();
422
- let delay = POLL_MIN_MS;
423
- let timer = 0;
424
- const tick = () => {
425
- void load(ac.signal);
426
- delay = Math.min(delay * 2, POLL_MAX_MS);
427
- timer = window.setTimeout(tick, delay);
428
- };
429
- timer = window.setTimeout(tick, delay);
430
- return () => {
431
- window.clearTimeout(timer);
432
- ac.abort();
433
- };
434
- }, [runningKey, load]);
435
-
436
- // ── W23-W5, the surface's half: hear the request, then answer it when we CAN ──────────
437
- useEffect(() => {
438
- const notify = (detail: AutomationOpenDetail) => setOpenRequest(detail);
439
- openSubscribers.add(notify);
440
- if (pendingOpen) {
441
- const latched = pendingOpen;
442
- pendingOpen = null;
443
- setOpenRequest(latched);
444
- }
445
- return () => {
446
- openSubscribers.delete(notify);
447
- };
448
- }, []);
449
-
450
- /* ⛔ THE C-CREATE(b) SUBSCRIBE EFFECT AND ITS RESOLVER STOOD HERE (wave 25, R8). They heard the
451
- create event and, once the list had arrived, called `createAndOpen` — the "wait for the list
452
- or every automation is named Automation 1" note lives on in `nextAutomationName`, which is
453
- still numbered off `existing`. With no signaller there is nothing to hear. */
454
-
455
- /**
456
- * ⚠ THE REQUEST OUTLIVES THE FETCH, and it has to. The reader clicks a notification from
457
- * another page, so this surface is mounting WITH AN EMPTY LIST — "select it if it is in the
458
- * list" would drop every real click and keep only the one case where the reader was already
459
- * here. So the request is held until `data` exists, and only then answered.
460
- *
461
- * An automation the reader can no longer open does NOTHING (the contract's own words): the
462
- * request is cleared either way, so a stale id cannot sit here re-firing against every
463
- * subsequent list.
464
- */
465
- useEffect(() => {
466
- if (!openRequest || !data) return;
467
- const found = (data.automations || []).some((a) => a.id === openRequest.autoId);
468
- if (found) {
469
- select(openRequest.autoId, openRequest.stageId || "");
470
- // C-18: the pane the notification asked for, read at the one moment it means something.
471
- const want = String((openRequest as { tab?: string }).tab || "");
472
- setOpenTab(want === "runs" || want === "properties" ? want : "");
473
- }
474
- setOpenRequest(null);
475
- }, [openRequest, data, select]);
476
-
477
- const items = data?.automations || [];
478
- const active = items.find((a) => a.id === activeId) || null;
479
-
480
- /**
481
- * ⭐⭐ WAVE 32 · T43 (owner item 2) — TURNING AN AUTOMATION ON OR OFF STOPS TAKING FOREVER.
482
- *
483
- * Owner: turning an automation on or off "takes forever". ⛔ THE TICKET'S HYPOTHESIS WAS THE
484
- * SERVER AND IT IS NOT — MEASURED. `engine.patch` for a schedule-only body takes ONE
485
- * `automations` bucket read, ZERO `user_tables` reads and ~1 ms against a 2.8 MB fixture, and
486
- * `GET /automations` is one read too since W30-T12. The cost was never the document shape
487
- * (D-179/D-175's family): it was that this handler paid **two sequential round trips** for one
488
- * click, and painted nothing until BOTH had landed. The second one re-downloaded the entire
489
- * rail payload — 88,771 B measured, see the note at the top of this file — to learn one boolean
490
- * the first response had already returned.
491
- *
492
- * ⛔ AND THE FIX ALREADY EXISTED IN THIS FILE, POINTED AT A DIFFERENT BUTTON. `mergeSaved` is
493
- * W31-T31, built for the detail pane's saves, with its own note about the write response being
494
- * newer than any read in flight. The rail's own switch never got it — a fix applied to one of
495
- * two twins, which is this wave's recurring shape.
496
- *
497
- * Three states, in order, and the middle one is what the owner actually asked for:
498
- * 1. FLIP IT NOW. The switch is the user's instruction, not a question; painting it after a
499
- * network round trip is what "takes forever" describes even when the round trip is fast.
500
- * 2. adopt the server's own answer when it lands, which carries the derived fields an
501
- * optimistic row cannot invent (`nextRunAt`, the status line).
502
- * 3. put the OLD row back if the write failed, beside the error — an optimistic switch that
503
- * stays on after a refusal is a lie, and a worse one than a slow switch.
504
- *
505
- * ⚠ `mergeSaved` TAKES THE GENERATION, which is what stops [[refetch-eats-its-own-write]] here:
506
- * a `GET /automations` issued before the click can still resolve after it, and without the bump
507
- * it would paint the pre-toggle row back over both the optimistic flip and the confirmation.
508
- * ⚠ `busy` is no longer what makes the switch look right — it stays only to keep a second click
509
- * from racing the first.
510
- */
511
- const toggleEnabled = async (a: Automation) => {
512
- const flipped: Automation = {
513
- ...a,
514
- schedule: { ...a.schedule, enabled: !a.schedule.enabled },
515
- };
516
- setBusy(a.id);
517
- const painted = mergeSaved(flipped);
518
- try {
519
- const { automation: fresh } = await patchAutomation(a.id, {
520
- schedule: { cron: a.schedule.cron, enabled: !a.schedule.enabled },
521
- });
522
- // A cold surface has no memo to merge into (`mergeSaved` answers false), and a one-row
523
- // repaint would be worse than a round trip — the same rule `onSaved` follows.
524
- if (!fresh || !mergeSaved(fresh)) await load();
525
- } catch (e) {
526
- if (painted) mergeSaved(a);
527
- setError(e instanceof AutomationError ? e.message : "That change was not saved.");
528
- } finally {
529
- setBusy("");
530
- }
531
- };
532
-
533
- const onRailKey = (event: ReactKeyboardEvent<HTMLDivElement>) => {
534
- if (event.key !== "ArrowDown" && event.key !== "ArrowUp") return;
535
- const rows = Array.from(
536
- listRef.current?.querySelectorAll<HTMLButtonElement>(".auto-row-main") || []
537
- );
538
- if (!rows.length) return;
539
- event.preventDefault();
540
- const at = rows.indexOf(document.activeElement as HTMLButtonElement);
541
- const next = event.key === "ArrowDown" ? (at + 1) % rows.length
542
- : (at - 1 + rows.length) % rows.length;
543
- rows[next < 0 ? 0 : next]?.focus();
544
- };
545
-
546
- return (
547
- <div className="auto-surface">
548
- <aside
549
- className={"auto-rail" + (railShut ? " is-collapsed" : "")}
550
- aria-label="Automations"
551
- >
552
- {/* The same three-bars fold control the Views rail carries — the two rails are
553
- one idea, so they must not fold with two different affordances. */}
554
- <div className="auto-rail-top">
555
- <button
556
- type="button"
557
- className="cg-rail-toggle"
558
- aria-label={railShut ? "Expand automations" : "Minimize automations"}
559
- aria-expanded={!railShut}
560
- title={railShut ? "Expand automations" : "Minimize automations"}
561
- onClick={() => setRailShut((v) => !v)}
562
- >
563
- <svg width="16" height="16" viewBox="0 0 16 16" fill="none" aria-hidden="true">
564
- <path
565
- d="M2.5 4.4h11M2.5 8h11M2.5 11.6h11"
566
- stroke="currentColor"
567
- strokeWidth="1.35"
568
- strokeLinecap="round"
569
- />
570
- </svg>
571
- </button>
572
- </div>
573
-
574
- <div className="auto-create">
575
- <button
576
- type="button"
577
- className="auto-create-btn"
578
- disabled={creatingNow}
579
- onClick={() => void createAndOpen()}
580
- >
581
- <svg width="14" height="14" viewBox="0 0 16 16" fill="none" aria-hidden="true">
582
- <path
583
- d="M8 3.4v9.2M3.4 8h9.2"
584
- stroke="currentColor"
585
- strokeWidth="1.5"
586
- strokeLinecap="round"
587
- />
588
- </svg>
589
- New automation
590
- </button>
591
- </div>
592
-
593
- <div className="auto-list" ref={listRef} onKeyDown={onRailKey}>
594
- {items.map((a) => {
595
- return (
596
- <div
597
- key={a.id}
598
- className={"auto-row" + (a.id === activeId ? " is-active" : "")}
599
- >
600
- <button
601
- type="button"
602
- className="auto-row-main"
603
- onClick={() => select(a.id)}
604
- >
605
- {/* ⭐ ITEM 18 / R14 — THE LOOP MARK, WHERE THE STATUS DOT WAS. Same mark on
606
- every row, no status colour: the owner's point is that an automation IS a
607
- loop, not that this one is green. The `.auto-loopmark` wrapper is what sizes
608
- it — `.lp-mark` is shell-owned CSS and outside this session's region, so the
609
- box is mine and the mark is theirs. */}
610
- <span className="auto-loopmark">
611
- <Mark size={15} />
612
- </span>
613
- <span className="auto-row-text">
614
- <span className="auto-row-name">{a.name}</span>
615
- <span className="auto-row-desc">{railSubtitle(a)}</span>
616
- </span>
617
- </button>
618
- <button
619
- type="button"
620
- className={"auto-row-switch" + (a.schedule?.enabled ? " is-on" : "")}
621
- disabled={busy === a.id}
622
- aria-pressed={!!a.schedule?.enabled}
623
- title={
624
- a.schedule?.enabled
625
- ? `Scheduled: ${a.schedule.cron}. Click to pause.`
626
- : "Not scheduled. Click to enable."
627
- }
628
- onClick={() => void toggleEnabled(a)}
629
- >
630
- <span className="auto-row-switch-knob" />
631
- </button>
632
- </div>
633
- );
634
- })}
635
- {/* ONE LINE (R13). It used to describe the kinds — "one reads a public web page…
636
- the other fills an automation column…" — which was two sentences, wrong by the
637
- time a third kind shipped, and printed inside a `--lp-rail-w` column. What the
638
- kinds are belongs to the create form, which is one click away and lists all of
639
- them from the server. */}
640
- {/* ⭐ WAVE 29 (W29-T01) — A LOADING RAIL IS NOT AN EMPTY ONE. `data` is null until the
641
- first `listAutomations()` resolves, and this branch read only `items.length`, so
642
- every visit to Automation asserted "No automations yet." to a person who has
643
- several — then replaced it with their list. Half of owner item 5's "takes a while
644
- to appear" is that sentence: the wait is real, but being told you own nothing is
645
- what makes it read as broken rather than slow. `data === null` is the one state
646
- that means NOBODY HAS LOOKED, and it gets skeleton rows, not a claim. */}
647
- {data === null && !error ? (
648
- <div className="auto-rail-loading" aria-hidden="true">
649
- <span className="auto-rail-skel" />
650
- <span className="auto-rail-skel" />
651
- <span className="auto-rail-skel" />
652
- </div>
653
- ) : null}
654
- {data !== null && !items.length && !error ? (
655
- <p className="auto-rail-empty">No automations yet.</p>
656
- ) : null}
657
- </div>
658
- </aside>
659
-
660
- <section className="auto-main">
661
- {error ? (
662
- <div className="auto-banner is-error" role="alert">
663
- {error}
664
- </div>
665
- ) : null}
666
- {data && !data.storeAvailable ? (
667
- <div className="auto-banner is-warn" role="status">
668
- The tenant store is unavailable, so nothing can be saved right now.
669
- </div>
670
- ) : null}
671
-
672
- {/*
673
- ⛔ THE `creating` BRANCH IS GONE (wave 24, C-CREATE a / owner items 3 + 4).
674
- `AutomationCreate` used to render here — a three-step wizard whose first step asked
675
- which DATABASE to write into (an existing one, a new one, or one the automation itself
676
- would create) and whose second asked which of three KINDS.
677
- R6 retires that question at the root: two of the three kinds can no longer be created
678
- at all, and the third is now reached by picking a trigger. A new automation is created
679
- the moment it is asked for (`createAndOpen`) and opens on its own Builder, so there is
680
- no intermediate face left to render and no `creating` state to hold.
681
- ⭐ WAVE 25 (R8) retires the LAST of that vocabulary: the third answer was the
682
- "automated database", and it is gone from every door in the product. This button is now
683
- one of exactly two ways to make an automation, and both are on this surface.
684
- The import went with it — it is what would break `npx tsc -b` for the WHOLE client the
685
- moment session B deletes the file, which is why this deletion is sequenced first.
686
- */}
687
- {/* T21: the editor's own Suspense boundary — see the `lazy()` above for why it cannot be
688
- `Shell.tsx`'s. `fallback={null}`, matching `Lazily`. */}
689
- {active ? (
690
- <Suspense fallback={null}>
691
- <AutomationDetail
692
- key={active.id}
693
- automation={active}
694
- /* C-18: which pane the Inbox asked for, or "" for the ordinary open. Consumed ONCE,
695
- as an initial value — see `openTab`'s note on why it is not controlled. */
696
- openTab={openTab}
697
- /* ⛔ `kinds={data?.kinds || []}` LEFT HERE WITH THE PROP IT FED (wave 25, D-57) —
698
- both halves in one change, because either alone is a `tsc` error. */
699
- cronPresets={data?.cronPresets || []}
700
- paidReady={!!data?.paidReady}
701
- discover={data?.discover}
702
- // The trigger vocabulary (C3). Forwarded as-is — absent stays absent, so the
703
- // trigger face can tell "the server offered nothing" from "the server offered
704
- // an empty list" rather than collapsing both into a picker with no options.
705
- triggers={data?.triggers}
706
- // C4's action menu + the builder's ceilings. Forwarded as-is for the same reason
707
- // `triggers` is: absent must stay absent, so the builder can tell "the server
708
- // offered nothing" from "the server offered an empty list".
709
- catalog={data?.actionsCatalog}
710
- vocab={data?.flow}
711
- // `?? null` and never `|| {enabled:false}`: an absent tick bit is "the server did
712
- // not say", which Step 1 prints as its own sentence. Defaulting it here would
713
- // turn a missing field into a claim about production (C6 amendment #1).
714
- tick={data?.tick ?? null}
715
- /*
716
- * ⭐ W24-W1 (item 6) — WHAT THIS RUN IS DOING, on the one surface that survives in
717
- * BOTH views. REQUIRED on the far side deliberately: an optional prop that nobody
718
- * passes degrades to "the feature does not exist", which is indistinguishable from
719
- * "it was never built" — and this whole item exists because a live step text rode
720
- * the wire for three waves with nothing rendering it.
721
- * `liveStepOf` returns the step ONLY while the automation is running; a stale step
722
- * from a finished run is a worse answer than none.
723
- */
724
- liveStep={liveStepOf(active)}
725
- /*
726
- * ⛔ C14 LEG 2 — THE STALE-ID WRITE-BACK, and this is the ghost's second cause.
727
- *
728
- * The detail calls `onSaved(automation.id)` from six places (save, the trigger
729
- * picker, a schedule change, a node switch, a card move, a run). Each closure
730
- * captures the automation it was mounted for, so a save that resolves AFTER the
731
- * user has clicked a different row used to call `setActiveId(the OLD id)` — the
732
- * rail jumped back, the detail remounted, and what the user saw was the previous
733
- * automation reappearing over the one they had just opened. It looked like a
734
- * rendering bug; it was a resolution steering the selection.
735
- *
736
- * A resolution may no longer steer anything. It reloads the list — that part was
737
- * always right — and it re-asserts the selection ONLY when its id is still the one
738
- * the user asked for, which makes the write a no-op in the good case and nothing
739
- * at all in the bad one.
740
- */
741
- onSaved={async (id, fresh) => {
742
- // ⭐⭐ WAVE 31 · T31 — CORRECT THE ONE ROW WE WERE JUST TOLD ABOUT.
743
- // `patchAutomation` and `toggleNode` return the updated definition; re-reading the
744
- // whole list to learn it is the second of the two round trips "Saving…" used to
745
- // span. `mergeSaved` keeps the memo and the component in step (see
746
- // `automationApi.rememberAutomation`) and takes the generation, because a write
747
- // response is newer than any read still in flight — the same rule `load` follows,
748
- // pointed the other way.
749
- if (fresh && mergeSaved(fresh)) {
750
- if (id && id === wantedId.current) select(id);
751
- return;
752
- }
753
- const next = await load();
754
- if (id && next && id === wantedId.current) select(id);
755
- }}
756
- onDeleted={async () => {
757
- select("");
758
- await load();
759
- }}
760
- />
761
- </Suspense>
762
- ) : (
763
- /*
764
- * THE EMPTY STATE IS ONE LINE AND A BUTTON (owner ruling R13 — "never
765
- * over-explain", now a DESIGN.md law).
766
- *
767
- * It was a heading, a three-sentence paragraph and a 130-word bulleted list
768
- * describing all three kinds. Every word of it was true and none of it was READ:
769
- * an empty state is passed through, not studied, and the person looking at it has
770
- * already decided to make an automation. The kinds are described where the choice
771
- * is actually made — the create form lists them FROM THE SERVER, so that copy also
772
- * cannot go stale the way this list had (it described two kinds after a third
773
- * shipped).
774
- *
775
- * The button is here rather than only in the rail because this pane is where the
776
- * eye is; a create affordance the user has to go find is the same defect as the
777
- * paragraph, spent differently.
778
- */
779
- <div className="autob-empty">
780
- {/*
781
- ⛔ NO TITLE HERE ANY MORE (C13, owner item 2). This pane carried an `h1`
782
- reading "Automations" at 20px/600 — a THIRD title treatment on a page that
783
- also had the header's editable 16px/700 input, against every database page's
784
- single 16px/650 `shell-db-name`. The shell now wraps this branch in the same
785
- `shell-db-frame` + `DbHead` a database gets (wiring W23-W1), so the page's name
786
- is drawn once, by the one component that draws every other page's name. A
787
- stand-in restyled to match would have been a second copy of the same fact,
788
- free to drift the day the header moves.
789
- */}
790
- <p className="autob-empty-line">
791
- A job this workspace runs for you, on demand or on a schedule.
792
- </p>
793
- <button
794
- type="button"
795
- className="auto-btn is-primary"
796
- disabled={creatingNow}
797
- onClick={() => void createAndOpen()}
798
- >
799
- New automation
800
- </button>
801
- </div>
802
- )}
803
- </section>
804
- </div>
805
- );
806
- }
 
1
+ // ---------------------------------------------------------------------------
2
+ // automation/AutomationSurface.tsx — the Automation surface (contract C-AUTONAV).
3
+ //
4
+ // ⛔ THE CONTRACT THIS FILE GUARANTEES, verbatim: this module path, a DEFAULT
5
+ // export, and NO PROPS. The shell mounts `<AutomationSurface />` and hands it
6
+ // nothing; it fetches its own data from `/api/v1/automations`. That is the whole
7
+ // interface, and it is stated here because the mount line lives in a file this
8
+ // session does not own — a named export would cost a cross-session round trip.
9
+ //
10
+ // "Automation replaces the Views rail" (owner) is implemented the way the wave
11
+ // scout established: the secondary rail is NOT a shell concept — each surface
12
+ // owns its own. So this renders a sibling of `.cg-views` built from the same
13
+ // `--lp-rail-w` tokens and the same fold behaviour, and the Views rail is
14
+ // untouched.
15
+ // ---------------------------------------------------------------------------
16
+ import type { KeyboardEvent as ReactKeyboardEvent } from "react";
17
+ import { Suspense, lazy, useCallback, useEffect, useRef, useState } from "react";
18
+
19
+ import type { AutomationOpenDetail } from "../apiContract";
20
+ import { AUTOMATION_OPEN_EVENT } from "../apiContract";
21
+ // ⭐ WAVE 26 · ITEM 18 / R14 + C7 — THE LOOPABLE LOOP MARK, CONSUMED, NEVER REDRAWN.
22
+ // The owner's reason is the mark's own geometry: *"these automations are basically loops."*
23
+ // C7 says consume the existing icon component and do not inline a new SVG path, and
24
+ // `shell/Brand.tsx`'s header says why in more detail than a rule could: the mark's one source of
25
+ // truth is generated, it has already been hand-redrawn once, and the copy silently painted LAST
26
+ // WAVE'S BRAND while a comment asserted parity. So this imports the component that POINTS at the
27
+ // generated artifact. [[loopable-nav-logo-toggle]] — one element paints the mark.
28
+ import { Mark } from "../shell/Brand";
29
+ import type { Automation, AutomationList } from "./automationApi";
30
+ import {
31
+ AutomationError,
32
+ cachedAutomations,
33
+ createAutomation,
34
+ listAutomations,
35
+ liveStepOf,
36
+ patchAutomation,
37
+ rememberAutomation,
38
+ } from "./automationApi";
39
+
40
+ /**
41
+ * ⭐⭐ WAVE 30 · T21 (owner items 4 + 5) — THE EDITOR STOPS SHIPPING INSIDE THE LIST.
42
+ *
43
+ * ⛔ THIS `lazy()` IS THE TICKET. `AutomationDetail` was a STATIC import, and it statically pulls
44
+ * `AutomationBuilder` (which in turn pulls `CondBuilder`, `PresetPlan` and `TriggerPicker`),
45
+ * `AutomationFind` and `AutomationTrigger` — so the whole editor was linked into the one chunk a
46
+ * person must download before the RAIL can paint a list of names. MEASURED at 88,771 B, against
47
+ * `HomePage` at 4,814 and `ConnectorsPage` at 3,551. That is the "Automation takes forever to
48
+ * load" complaint, and it had survived two waves because nothing asserted a number.
49
+ *
50
+ * ⛔ AND IT NEEDS ITS OWN `<Suspense>`, NOT `Shell.tsx`'s. `Lazily` already wraps this whole
51
+ * surface, but a Suspense boundary catches ANY suspending descendant — so without an inner
52
+ * boundary, opening an automation would suspend the boundary that owns the RAIL and blank the
53
+ * list you just clicked in. The inner one keeps the rail painted while the editor streams in.
54
+ * `fallback={null}` is deliberate and is `Lazily`'s own choice, not a shortcut: the work area is
55
+ * simply not there for the moment the chunk is in flight, which is what every other route in this
56
+ * product already does. ⚠ A skeleton here would be a NEW loading screen in the same wave that
57
+ * exists to remove one (T22).
58
+ * ⚠ Default export, checked: `Shell.tsx:53` records that `lazy()` over a NAMED export fails at
59
+ * runtime, on click, and only for that one surface — `AutomationDetail` exports default.
60
+ */
61
+ const AutomationDetail = lazy(() => import("./AutomationDetail"));
62
+
63
+ /**
64
+ * The run poll's cadence (item 6). Named constants because they are a MEASURED trade-off, not a
65
+ * taste: `POLL_MAX_MS` is the longest a finished run can still look live, and it is the only
66
+ * cost the backoff has.
67
+ */
68
+ const POLL_MIN_MS = 2500;
69
+ const POLL_MAX_MS = 8000;
70
+
71
+ /*
72
+ * ⭐ WAVE 26 · ITEM 18 / R14 — `stateTitle` STOOD HERE AND IS DELETED WITH THE DOT IT DESCRIBED.
73
+ *
74
+ * It composed the dot's tooltip ("Running now — <step>", "Last run failed — <summary>"), so it had
75
+ * exactly one reader and no reason to outlive it. Deleting the render and keeping the composer is
76
+ * how a file accumulates functions that look live and are not — and `noUnusedLocals` would have
77
+ * caught this one, which is not a reason to lean on it: the NEXT such function might still have a
78
+ * second caller and compile fine.
79
+ *
80
+ * ⛔ WHY THE TOOLTIP WAS NOT MOVED ONTO THE MARK INSTEAD. R14 is "same mark on every automation;
81
+ * no status colour". Hanging "Last run failed" off a mark that is deliberately status-free just
82
+ * moves the status channel into `title`/`aria-label`, where it is worse: invisible to the eye,
83
+ * announced to a screen reader, and contradicting the visual. The run state has a home one click
84
+ * away — the Builder's own "Last run succeeded / failed" lines, which R13 explicitly keeps.
85
+ *
86
+ * ⚠ `stateOf` ITSELF IS UNTOUCHED in `automationApi.ts`. `Shell.tsx:64/1095` imports it and
87
+ * `liveStepOf` calls it — deleting the export to tidy up this file would break a fence I do not own.
88
+ */
89
+
90
+ /**
91
+ * The rail's second line — ONE fact, not two.
92
+ *
93
+ * ⚠ It carried "Next 2026-08-04 06:00 · last 2026-08-04 00:20", and the rail is `--lp-rail-w`
94
+ * wide, so it rendered as "Next 2026-08-04 06:00 · l…" — a second line whose only complete word
95
+ * was "Next". Two facts that both truncate are worth less than one that fits. The forward-looking
96
+ * one wins when a schedule exists (it is the question the rail answers: does this run itself?),
97
+ * the last run otherwise. The full picture is one click away in the editor's run history, and the
98
+ * row's `title` already says both.
99
+ *
100
+ * Caught by READING the screenshot: the DOM was correct and the CSS ellipsis was doing exactly
101
+ * its job. [[ui-invisible-to-assertions]] — judge the pixels even when everything is green.
102
+ */
103
+ function railSubtitle(a: Automation): string {
104
+ /*
105
+ * ⭐ WAVE 24 (owner item 6, "Run once now is laggy / looks stuck") — WHILE IT IS RUNNING,
106
+ * THE ONE FACT WORTH THE LINE IS WHAT IT IS DOING.
107
+ *
108
+ * MEASURED, not guessed (`scratchpad/perf_automations.py`, mailbox D-5): the engine has been
109
+ * publishing a live step the whole time (`status.step`, `routes_automation.py:50-51`) and NO
110
+ * SURFACE HAS EVER RENDERED IT. So a `discover_instagram` run parked in its legitimate
111
+ * `BD_FILTER_WAIT = 120 s` vendor wait and a genuinely hung thread were pixel-identical: a
112
+ * pulsing blue dot and a subtitle still reciting the schedule. "Looks stuck" was not a
113
+ * performance problem — the poll costs 23 ms — it was the product declining to say.
114
+ *
115
+ * It takes the line rather than joining it, for the reason this function's note already
116
+ * gives: the rail is `--lp-rail-w` wide and two facts that both truncate are worth less than
117
+ * one that fits. The schedule is still one click away and the dot's `title` carries both.
118
+ */
119
+ const step = liveStepOf(a);
120
+ if (step) return step;
121
+ if (a.schedule?.enabled) return a.nextRunAt ? `Next ${a.nextRunAt}` : "Scheduled";
122
+ /*
123
+ * ⭐ WAVE 26 · ITEM 19 / R13 — `Last run 2026-08-06 13:58` IS DELETED FROM THIS LINE.
124
+ *
125
+ * ⛔ SCOPE, because R13 draws a line that is easy to over-read: the ruling names THIS secondary
126
+ * line under the automation's name. The Builder's "Last run succeeded / failed" RESULT lines
127
+ * (`AutomationBuilder.tsx:1676`, `:1679`) are a different surface and are explicitly NOT in
128
+ * scope — they are the answer to "Run once now", so deleting them would leave a button with no
129
+ * outcome.
130
+ *
131
+ * "Manual only" STAYS, and the distinction is the ruling's own: a dated run record is a
132
+ * changing FACT ABOUT THE PAST, while "this one has no schedule" is a stable property of the
133
+ * automation — the same question `Next …`/`Scheduled` answers for its siblings. Dropping it too
134
+ * would leave a manual automation with a blank second line and nothing saying why.
135
+ */
136
+ return "Manual only";
137
+ }
138
+
139
+ /*
140
+ * ⛔ W23-W5's LISTENER IS AT MODULE SCOPE, AND THAT IS THE POINT — not a stylistic choice.
141
+ *
142
+ * The frame's click-through sets `window.location.hash = "#/automation"` and signals on the
143
+ * VERY NEXT LINE (Shell.tsx:1647-1650, and its comment is right that the order matters). But a
144
+ * hash write does not mount anything synchronously: `hashchange` is delivered as a task, the
145
+ * router state then updates, React renders, and only THEN does a component effect subscribe.
146
+ * A listener registered in `useEffect` therefore misses every click that arrives from another
147
+ * page — which is every click, since a reader looking at Alerts is by definition not already
148
+ * on this surface. The event would dispatch into nothing and every gate would stay green:
149
+ * exactly the wave-20 item-25 failure the contract's own note describes, reproduced one layer
150
+ * down.
151
+ *
152
+ * This module is imported statically by the shell, so this listener exists from app start. It
153
+ * LATCHES the request; the component consumes the latch when it mounts and hears live events
154
+ * while it is mounted. A request nobody claims is dropped on the next one — the latch is a
155
+ * one-slot mailbox, never a queue.
156
+ */
157
+ let pendingOpen: AutomationOpenDetail | null = null;
158
+ const openSubscribers = new Set<(detail: AutomationOpenDetail) => void>();
159
+
160
+ if (typeof window !== "undefined") {
161
+ window.addEventListener(AUTOMATION_OPEN_EVENT, (event) => {
162
+ const detail = (event as CustomEvent<AutomationOpenDetail>).detail;
163
+ if (!detail?.autoId) return;
164
+ if (openSubscribers.size) {
165
+ for (const notify of openSubscribers) notify(detail);
166
+ return;
167
+ }
168
+ pendingOpen = detail;
169
+ });
170
+ }
171
+
172
+ /*
173
+ * ⛔ THE `AUTOMATION_CREATE_EVENT` LISTENER STOOD HERE AND IS DELETED WITH ITS SIGNALLER
174
+ * (wave 25 item 5a, ruling R8) — latch, subscriber set and all.
175
+ *
176
+ * Wave 24 built it to close the opposite defect: the event was declared, signalled from
177
+ * `Shell.tsx`, and consumed NOWHERE, so all three "Automated database" doors navigated to
178
+ * `#/automation` and then did nothing, in production, with every gate green. R8 now deletes those
179
+ * three doors — creating a database and pointing an automation at it are two acts — and they were
180
+ * the event's only signaller.
181
+ *
182
+ * ⛔ SO THIS SIDE GOES TOO, IN THE SAME CHANGE. A listener with no signaller is the same defect
183
+ * read from the other end: it compiles, it costs nothing at runtime, and it reads to the next
184
+ * person as a live channel. The rule that catches both is now DERIVED rather than remembered —
185
+ * `verify_automation_ui.py` enumerates every `AUTOMATION_*_EVENT` in `apiContract.ts` and demands
186
+ * a signal site AND a listener site for each.
187
+ *
188
+ * ⚠ NOTHING ABOUT CREATING AN AUTOMATION IS LOST. `createAndOpen` is untouched and still has two
189
+ * doors, both on this surface: the rail's button and the empty state's. What is gone is the claim
190
+ * that a database can be created BY asking for an automation.
191
+ */
192
+
193
+ /**
194
+ * A default name for a brand-new automation, and it has to be one the server ACCEPTS.
195
+ *
196
+ * MEASURED against the live route rather than assumed (`scratchpad/probe_names.py`): a create
197
+ * with no name is a 400 (`name the automation`), and duplicate names are ALLOWED. So the name
198
+ * cannot be omitted, and a fixed literal would stack "New automation" three deep in a rail that
199
+ * sorts by name with nothing to tell the rows apart. Numbering from the existing list is a
200
+ * client-side convenience over a server that does not care: if two tabs race, the loser gets a
201
+ * duplicate name, which is legal, visible and renameable — never an error the user has to read.
202
+ */
203
+ function nextAutomationName(existing: Automation[]): string {
204
+ const taken = new Set(existing.map((a) => (a.name || "").trim().toLowerCase()));
205
+ for (let n = existing.length + 1; ; n += 1) {
206
+ const candidate = `Automation ${n}`;
207
+ if (!taken.has(candidate.toLowerCase())) return candidate;
208
+ }
209
+ }
210
+
211
+ export default function AutomationSurface() {
212
+ /* ⭐ T22 — SEEDED FROM THE CLIENT MEMO, so a revisit inside the freshness window never passes
213
+ through `data === null` and therefore never paints the skeleton. The lazy initialiser runs
214
+ once, on mount, BEFORE the first paint — `useState(cachedAutomations())` would call it on
215
+ every render instead, which is the same value at needless cost. A cold start still returns
216
+ null and still gets the skeleton: it is made rare, not removed (see `automationApi.ts`). */
217
+ const [data, setData] = useState<AutomationList | null>(() => cachedAutomations());
218
+ const [error, setError] = useState("");
219
+ const [activeId, setActiveId] = useState<string>("");
220
+ const [railShut, setRailShut] = useState(false);
221
+ const [busy, setBusy] = useState("");
222
+ /**
223
+ * ⭐ WAVE 24 / C-CREATE — a create is now a ROUND TRIP, not a local wizard, so the door has to
224
+ * say it is busy. Without this a second click while the POST is in flight makes a second
225
+ * automation, and the server allows duplicate names, so the user gets two rows that look
226
+ * identical and no error to explain either of them.
227
+ */
228
+ const [creatingNow, setCreatingNow] = useState(false);
229
+ /** An open request waiting for the list to arrive (see the resolver below). */
230
+ const [openRequest, setOpenRequest] = useState<AutomationOpenDetail | null>(null);
231
+ /* ⛔ `createRequest` LEFT WITH THE EVENT THAT SET IT (wave 25, R8) — see the tombstone above. */
232
+ const listRef = useRef<HTMLDivElement | null>(null);
233
+
234
+ /**
235
+ * ⛔ C14 LEG 2 — THE ID THE USER LAST ASKED FOR, and it is a ref because it has to be
236
+ * written SYNCHRONOUSLY, inside the click, before any promise that was already in flight
237
+ * can resolve. State would not do: a resolution racing React's commit would read the
238
+ * previous value, which is the exact window this guard exists to close.
239
+ */
240
+ const wantedId = useRef("");
241
+
242
+ /**
243
+ * ⭐ THE ONLY PLACE THE RAIL'S SELECTION MOVES ON PURPOSE. Every deliberate change of
244
+ * automation — a rail click, a create, a delete — goes through here, so "what the user
245
+ * asked for" and "what is on screen" are written in one statement and cannot drift apart.
246
+ * A resolution that wants to steer the rail compares itself against `wantedId` instead.
247
+ */
248
+ const select = useCallback((id: string, _stage = "") => {
249
+ wantedId.current = id;
250
+ setActiveId(id);
251
+ }, []);
252
+
253
+ /**
254
+ * ⛔ C14 LEG 3 (the paint half) — A STALE READ MAY NOT REPAINT.
255
+ *
256
+ * Every call takes the next generation; only the newest one is allowed to `setData`. The
257
+ * defect this closes is not cosmetic: a poll issued BEFORE a delete resolves AFTER it, and
258
+ * a list that still contains the deleted automation puts a row back in the rail that the
259
+ * user has just watched disappear. Same for a toggle, and same for a create.
260
+ *
261
+ * ⚠ THE RETURN VALUE IS NOT GUARDED, deliberately. The caller asked a question and gets
262
+ * its own answer — coupling the two would mean a create whose `load()` was overtaken by a
263
+ * poll silently failed to select the automation it had just made.
264
+ */
265
+ const gen = useRef(0);
266
+
267
+ const load = useCallback(async (abort?: AbortSignal) => {
268
+ const mine = ++gen.current;
269
+ try {
270
+ const next = await listAutomations(abort);
271
+ if (mine === gen.current) {
272
+ setData(next);
273
+ setError("");
274
+ }
275
+ return next;
276
+ } catch (e) {
277
+ if ((e as Error)?.name === "AbortError") return null;
278
+ if (mine === gen.current)
279
+ setError(
280
+ e instanceof AutomationError
281
+ ? e.message
282
+ : "The automation service did not answer."
283
+ );
284
+ return null;
285
+ }
286
+ }, []);
287
+
288
+ /**
289
+ * ⭐⭐ WAVE 31 · T31 — one freshly-written automation replaces its own row, with no round trip.
290
+ *
291
+ * ⚠ IT TAKES THE GENERATION, and that is not decoration. `load`'s guard exists so a stale READ
292
+ * cannot repaint; a WRITE response is by definition newer than any read already in flight, so it
293
+ * must win the same race rather than sit outside it. Without the bump, a `GET /automations`
294
+ * issued before the save could resolve after the merge and paint the pre-save row back — the
295
+ * exact ghost `gen` was introduced for, arriving through the new door.
296
+ *
297
+ * ⚠ Returns FALSE when there is nothing to merge into (a cold surface has no memo), and the
298
+ * caller then does a real load. Painting a one-row list would be worse than a round trip.
299
+ */
300
+ const mergeSaved = useCallback((fresh: Automation) => {
301
+ const next = rememberAutomation(fresh);
302
+ if (!next) return false;
303
+ gen.current += 1;
304
+ setData(next);
305
+ setError("");
306
+ return true;
307
+ }, []);
308
+
309
+ useEffect(() => {
310
+ const ac = new AbortController();
311
+ void load(ac.signal);
312
+ return () => ac.abort();
313
+ }, [load]);
314
+
315
+ /**
316
+ * ⭐⭐ WAVE 24 / C-CREATE (a) — CREATE AND OPEN. This REPLACED the three-step wizard.
317
+ *
318
+ * R6 is what makes it possible: `plain` is a real kind with no machine graph nodes, and it is
319
+ * what `POST /automations` stores when the body names none. So "new automation" stopped being
320
+ * a question ("which of three kinds?" — two of which can no longer be created at all) and
321
+ * became what it says: a new automation, open, on its trigger picker. Choosing a SOURCE is now
322
+ * picking the `ig_profile_match` trigger, which is where that choice belongs.
323
+ *
324
+ * ⛔ THE GUARD IS A REF, not the `creatingNow` state, and this file already carries the scar
325
+ * that explains why (C14 leg 2): a state read inside a click closure is the value from the
326
+ * last render, so two fast clicks both see `false` and both POST. Duplicate names are legal
327
+ * server-side, so the user would get two identical rows and no error. The ref is written
328
+ * synchronously, inside the click, before anything can await.
329
+ */
330
+ const creatingRef = useRef(false);
331
+ const createAndOpen = useCallback(async () => {
332
+ if (creatingRef.current) return;
333
+ creatingRef.current = true;
334
+ setCreatingNow(true);
335
+ try {
336
+ const res = await createAutomation({
337
+ name: nextAutomationName(data?.automations || []),
338
+ });
339
+ const id = res?.automation?.id || "";
340
+ // The create flow is the one caller allowed to name a different id (C14 leg 2) — the
341
+ // automation did not exist when the click happened, so there is nothing to race with.
342
+ const next = await load();
343
+ if (id && next) select(id);
344
+ } catch (e) {
345
+ setError(
346
+ e instanceof AutomationError ? e.message : "The automation was not created."
347
+ );
348
+ } finally {
349
+ creatingRef.current = false;
350
+ setCreatingNow(false);
351
+ }
352
+ }, [data, load, select]);
353
+
354
+ // A run is a background thread on the server, so the surface has to ASK whether it
355
+ // finished. Polling only while something is actually running keeps an idle surface
356
+ // silent — a fixed interval would be a request every few seconds forever, for a page
357
+ // whose contents change a handful of times a day.
358
+ //
359
+ // ⛔ C14 LEG 3 (the abort half). It used to call `load()` bare — no signal, nothing to
360
+ // cancel — so a request the interval had already issued kept going after the effect that
361
+ // owned it was gone. One controller per effect run, aborted with the interval, means a poll
362
+ // cannot outlive the condition that justified it. The generation guard inside `load` covers
363
+ // the rest: a response that survives the abort still cannot repaint over a newer one.
364
+ //
365
+ // ⭐ WAVE 24 (item 6, C-PERF) — THE INTERVAL BECAME A BACKOFF, and the measurement is why it is
366
+ // a SMALL change rather than the big one the contract's hypothesis 2 asked for.
367
+ //
368
+ // MEASURED (`scratchpad/perf_automations.py`, mailbox D-5): this poll costs ~23 ms and 24 KB
369
+ // for a ten-automation tenant, and the engine work the hypothesis blamed — a `graph()` rebuild
370
+ // per automation — is **41 µs each**, i.e. under 4% of the request. The payload was never the
371
+ // problem, so nothing here gets a cheaper endpoint.
372
+ //
373
+ // What the numbers DO indict is the aggregate: a discovery run legitimately blocks up to
374
+ // `BD_FILTER_WAIT` = 120 s, and two independent 2.5 s polls across this file and the detail
375
+ // spend ~96 requests and ~2.3 MB over that window — in the same single process the run itself
376
+ // is a thread in. So the delay grows 2.5 → 5 → 8 s and stops there.
377
+ //
378
+ // ⚠ NOT "to learn nothing", and the distinction is the whole justification. That WAS true when
379
+ // this file rendered no live step: the poll's only observable effect was a dot that had already
380
+ // been pulsing for two minutes. It stopped being true in this same change — `liveStepOf` now
381
+ // paints `status.step`, so a poll carries the one fact worth having. The backoff is therefore
382
+ // NOT "stop asking a pointless question"; it is FEWER ROUND TRIPS FOR THE SAME INFORMATION, on
383
+ // a step text that changes every few seconds at most, not every 2.5.
384
+ // The cap is deliberately low: it bounds how long a FINISHED run can still look live, which is
385
+ // the only thing a backoff can make worse, and 8 s of that is worth ~⅔ fewer requests.
386
+ //
387
+ // ⛔ THE KEY IS THE RUNNING SET, NOT A BOOLEAN, and that is what makes the reset correct: the
388
+ // effect re-runs — and the delay drops back to 2.5 s — the moment a run starts or finishes, so
389
+ // a user who clicks Run now gets the fast cadence again instead of inheriting the tail of the
390
+ // previous run's backoff. A bare `anyRunning` boolean cannot see the second run start.
391
+ // A step text changing does NOT re-key it, so the interval never thrashes.
392
+ //
393
+ // C14 LEG 3 (the abort half) is unchanged and still load-bearing: one controller per effect
394
+ // run, aborted with the timer, so a poll cannot outlive the condition that justified it. The
395
+ // generation guard inside `load` covers the rest.
396
+ const runningKey = (data?.automations || [])
397
+ .filter((a) => a.running)
398
+ .map((a) => a.id)
399
+ .sort()
400
+ .join(",");
401
+ useEffect(() => {
402
+ if (!runningKey) return undefined;
403
+ const ac = new AbortController();
404
+ let delay = POLL_MIN_MS;
405
+ let timer = 0;
406
+ const tick = () => {
407
+ void load(ac.signal);
408
+ delay = Math.min(delay * 2, POLL_MAX_MS);
409
+ timer = window.setTimeout(tick, delay);
410
+ };
411
+ timer = window.setTimeout(tick, delay);
412
+ return () => {
413
+ window.clearTimeout(timer);
414
+ ac.abort();
415
+ };
416
+ }, [runningKey, load]);
417
+
418
+ // ── W23-W5, the surface's half: hear the request, then answer it when we CAN ──────────
419
+ useEffect(() => {
420
+ const notify = (detail: AutomationOpenDetail) => setOpenRequest(detail);
421
+ openSubscribers.add(notify);
422
+ if (pendingOpen) {
423
+ const latched = pendingOpen;
424
+ pendingOpen = null;
425
+ setOpenRequest(latched);
426
+ }
427
+ return () => {
428
+ openSubscribers.delete(notify);
429
+ };
430
+ }, []);
431
+
432
+ /* ⛔ THE C-CREATE(b) SUBSCRIBE EFFECT AND ITS RESOLVER STOOD HERE (wave 25, R8). They heard the
433
+ create event and, once the list had arrived, called `createAndOpen` — the "wait for the list
434
+ or every automation is named Automation 1" note lives on in `nextAutomationName`, which is
435
+ still numbered off `existing`. With no signaller there is nothing to hear. */
436
+
437
+ /**
438
+ * ⚠ THE REQUEST OUTLIVES THE FETCH, and it has to. The reader clicks a notification from
439
+ * another page, so this surface is mounting WITH AN EMPTY LIST — "select it if it is in the
440
+ * list" would drop every real click and keep only the one case where the reader was already
441
+ * here. So the request is held until `data` exists, and only then answered.
442
+ *
443
+ * An automation the reader can no longer open does NOTHING (the contract's own words): the
444
+ * request is cleared either way, so a stale id cannot sit here re-firing against every
445
+ * subsequent list.
446
+ */
447
+ useEffect(() => {
448
+ if (!openRequest || !data) return;
449
+ const found = (data.automations || []).some((a) => a.id === openRequest.autoId);
450
+ if (found) {
451
+ select(openRequest.autoId, openRequest.stageId || "");
452
+ }
453
+ setOpenRequest(null);
454
+ }, [openRequest, data, select]);
455
+
456
+ const items = data?.automations || [];
457
+ const active = items.find((a) => a.id === activeId) || null;
458
+
459
+ /**
460
+ * ⭐⭐ WAVE 32 · T43 (owner item 2) — TURNING AN AUTOMATION ON OR OFF STOPS TAKING FOREVER.
461
+ *
462
+ * Owner: turning an automation on or off "takes forever". ⛔ THE TICKET'S HYPOTHESIS WAS THE
463
+ * SERVER AND IT IS NOT — MEASURED. `engine.patch` for a schedule-only body takes ONE
464
+ * `automations` bucket read, ZERO `user_tables` reads and ~1 ms against a 2.8 MB fixture, and
465
+ * `GET /automations` is one read too since W30-T12. The cost was never the document shape
466
+ * (D-179/D-175's family): it was that this handler paid **two sequential round trips** for one
467
+ * click, and painted nothing until BOTH had landed. The second one re-downloaded the entire
468
+ * rail payload — 88,771 B measured, see the note at the top of this file — to learn one boolean
469
+ * the first response had already returned.
470
+ *
471
+ * ⛔ AND THE FIX ALREADY EXISTED IN THIS FILE, POINTED AT A DIFFERENT BUTTON. `mergeSaved` is
472
+ * W31-T31, built for the detail pane's saves, with its own note about the write response being
473
+ * newer than any read in flight. The rail's own switch never got it — a fix applied to one of
474
+ * two twins, which is this wave's recurring shape.
475
+ *
476
+ * Three states, in order, and the middle one is what the owner actually asked for:
477
+ * 1. FLIP IT NOW. The switch is the user's instruction, not a question; painting it after a
478
+ * network round trip is what "takes forever" describes even when the round trip is fast.
479
+ * 2. adopt the server's own answer when it lands, which carries the derived fields an
480
+ * optimistic row cannot invent (`nextRunAt`, the status line).
481
+ * 3. put the OLD row back if the write failed, beside the error — an optimistic switch that
482
+ * stays on after a refusal is a lie, and a worse one than a slow switch.
483
+ *
484
+ * ⚠ `mergeSaved` TAKES THE GENERATION, which is what stops [[refetch-eats-its-own-write]] here:
485
+ * a `GET /automations` issued before the click can still resolve after it, and without the bump
486
+ * it would paint the pre-toggle row back over both the optimistic flip and the confirmation.
487
+ * ⚠ `busy` is no longer what makes the switch look right — it stays only to keep a second click
488
+ * from racing the first.
489
+ */
490
+ const toggleEnabled = async (a: Automation) => {
491
+ const flipped: Automation = {
492
+ ...a,
493
+ schedule: { ...a.schedule, enabled: !a.schedule.enabled },
494
+ };
495
+ setBusy(a.id);
496
+ const painted = mergeSaved(flipped);
497
+ try {
498
+ const { automation: fresh } = await patchAutomation(a.id, {
499
+ schedule: { cron: a.schedule.cron, enabled: !a.schedule.enabled },
500
+ });
501
+ // A cold surface has no memo to merge into (`mergeSaved` answers false), and a one-row
502
+ // repaint would be worse than a round trip — the same rule `onSaved` follows.
503
+ if (!fresh || !mergeSaved(fresh)) await load();
504
+ } catch (e) {
505
+ if (painted) mergeSaved(a);
506
+ setError(e instanceof AutomationError ? e.message : "That change was not saved.");
507
+ } finally {
508
+ setBusy("");
509
+ }
510
+ };
511
+
512
+ const onRailKey = (event: ReactKeyboardEvent<HTMLDivElement>) => {
513
+ if (event.key !== "ArrowDown" && event.key !== "ArrowUp") return;
514
+ const rows = Array.from(
515
+ listRef.current?.querySelectorAll<HTMLButtonElement>(".auto-row-main") || []
516
+ );
517
+ if (!rows.length) return;
518
+ event.preventDefault();
519
+ const at = rows.indexOf(document.activeElement as HTMLButtonElement);
520
+ const next = event.key === "ArrowDown" ? (at + 1) % rows.length
521
+ : (at - 1 + rows.length) % rows.length;
522
+ rows[next < 0 ? 0 : next]?.focus();
523
+ };
524
+
525
+ return (
526
+ <div className="auto-surface">
527
+ <aside
528
+ className={"auto-rail" + (railShut ? " is-collapsed" : "")}
529
+ aria-label="Automations"
530
+ >
531
+ {/* The same three-bars fold control the Views rail carries — the two rails are
532
+ one idea, so they must not fold with two different affordances. */}
533
+ <div className="auto-rail-top">
534
+ <button
535
+ type="button"
536
+ className="cg-rail-toggle"
537
+ aria-label={railShut ? "Expand automations" : "Minimize automations"}
538
+ aria-expanded={!railShut}
539
+ title={railShut ? "Expand automations" : "Minimize automations"}
540
+ onClick={() => setRailShut((v) => !v)}
541
+ >
542
+ <svg width="16" height="16" viewBox="0 0 16 16" fill="none" aria-hidden="true">
543
+ <path
544
+ d="M2.5 4.4h11M2.5 8h11M2.5 11.6h11"
545
+ stroke="currentColor"
546
+ strokeWidth="1.35"
547
+ strokeLinecap="round"
548
+ />
549
+ </svg>
550
+ </button>
551
+ </div>
552
+
553
+ <div className="auto-create">
554
+ <button
555
+ type="button"
556
+ className="auto-create-btn"
557
+ disabled={creatingNow}
558
+ onClick={() => void createAndOpen()}
559
+ >
560
+ <svg width="14" height="14" viewBox="0 0 16 16" fill="none" aria-hidden="true">
561
+ <path
562
+ d="M8 3.4v9.2M3.4 8h9.2"
563
+ stroke="currentColor"
564
+ strokeWidth="1.5"
565
+ strokeLinecap="round"
566
+ />
567
+ </svg>
568
+ New automation
569
+ </button>
570
+ </div>
571
+
572
+ <div className="auto-list" ref={listRef} onKeyDown={onRailKey}>
573
+ {items.map((a) => {
574
+ return (
575
+ <div
576
+ key={a.id}
577
+ className={"auto-row" + (a.id === activeId ? " is-active" : "")}
578
+ >
579
+ <button
580
+ type="button"
581
+ className="auto-row-main"
582
+ onClick={() => select(a.id)}
583
+ >
584
+ {/* ⭐ ITEM 18 / R14 — THE LOOP MARK, WHERE THE STATUS DOT WAS. Same mark on
585
+ every row, no status colour: the owner's point is that an automation IS a
586
+ loop, not that this one is green. The `.auto-loopmark` wrapper is what sizes
587
+ it — `.lp-mark` is shell-owned CSS and outside this session's region, so the
588
+ box is mine and the mark is theirs. */}
589
+ <span className="auto-loopmark">
590
+ <Mark size={15} />
591
+ </span>
592
+ <span className="auto-row-text">
593
+ <span className="auto-row-name">{a.name}</span>
594
+ <span className="auto-row-desc">{railSubtitle(a)}</span>
595
+ </span>
596
+ </button>
597
+ <button
598
+ type="button"
599
+ className={"auto-row-switch" + (a.schedule?.enabled ? " is-on" : "")}
600
+ disabled={busy === a.id}
601
+ aria-pressed={!!a.schedule?.enabled}
602
+ title={
603
+ a.schedule?.enabled
604
+ ? `Scheduled: ${a.schedule.cron}. Click to pause.`
605
+ : "Not scheduled. Click to enable."
606
+ }
607
+ onClick={() => void toggleEnabled(a)}
608
+ >
609
+ <span className="auto-row-switch-knob" />
610
+ </button>
611
+ </div>
612
+ );
613
+ })}
614
+ {/* ONE LINE (R13). It used to describe the kinds — "one reads a public web page…
615
+ the other fills an automation column…" — which was two sentences, wrong by the
616
+ time a third kind shipped, and printed inside a `--lp-rail-w` column. What the
617
+ kinds are belongs to the create form, which is one click away and lists all of
618
+ them from the server. */}
619
+ {/* ⭐ WAVE 29 (W29-T01) — A LOADING RAIL IS NOT AN EMPTY ONE. `data` is null until the
620
+ first `listAutomations()` resolves, and this branch read only `items.length`, so
621
+ every visit to Automation asserted "No automations yet." to a person who has
622
+ several — then replaced it with their list. Half of owner item 5's "takes a while
623
+ to appear" is that sentence: the wait is real, but being told you own nothing is
624
+ what makes it read as broken rather than slow. `data === null` is the one state
625
+ that means NOBODY HAS LOOKED, and it gets skeleton rows, not a claim. */}
626
+ {data === null && !error ? (
627
+ <div className="auto-rail-loading" aria-hidden="true">
628
+ <span className="auto-rail-skel" />
629
+ <span className="auto-rail-skel" />
630
+ <span className="auto-rail-skel" />
631
+ </div>
632
+ ) : null}
633
+ {data !== null && !items.length && !error ? (
634
+ <p className="auto-rail-empty">No automations yet.</p>
635
+ ) : null}
636
+ </div>
637
+ </aside>
638
+
639
+ <section className="auto-main">
640
+ {error ? (
641
+ <div className="auto-banner is-error" role="alert">
642
+ {error}
643
+ </div>
644
+ ) : null}
645
+ {data && !data.storeAvailable ? (
646
+ <div className="auto-banner is-warn" role="status">
647
+ The tenant store is unavailable, so nothing can be saved right now.
648
+ </div>
649
+ ) : null}
650
+
651
+ {/*
652
+ ⛔ THE `creating` BRANCH IS GONE (wave 24, C-CREATE a / owner items 3 + 4).
653
+ `AutomationCreate` used to render here — a three-step wizard whose first step asked
654
+ which DATABASE to write into (an existing one, a new one, or one the automation itself
655
+ would create) and whose second asked which of three KINDS.
656
+ R6 retires that question at the root: two of the three kinds can no longer be created
657
+ at all, and the third is now reached by picking a trigger. A new automation is created
658
+ the moment it is asked for (`createAndOpen`) and opens on its own Builder, so there is
659
+ no intermediate face left to render and no `creating` state to hold.
660
+ ⭐ WAVE 25 (R8) retires the LAST of that vocabulary: the third answer was the
661
+ "automated database", and it is gone from every door in the product. This button is now
662
+ one of exactly two ways to make an automation, and both are on this surface.
663
+ The import went with it — it is what would break `npx tsc -b` for the WHOLE client the
664
+ moment session B deletes the file, which is why this deletion is sequenced first.
665
+ */}
666
+ {/* T21: the editor's own Suspense boundary — see the `lazy()` above for why it cannot be
667
+ `Shell.tsx`'s. `fallback={null}`, matching `Lazily`. */}
668
+ {active ? (
669
+ <Suspense fallback={null}>
670
+ <AutomationDetail
671
+ key={active.id}
672
+ automation={active}
673
+ /* ⛔ `kinds={data?.kinds || []}` LEFT HERE WITH THE PROP IT FED (wave 25, D-57) —
674
+ both halves in one change, because either alone is a `tsc` error. */
675
+ cronPresets={data?.cronPresets || []}
676
+ paidReady={!!data?.paidReady}
677
+ discover={data?.discover}
678
+ // The trigger vocabulary (C3). Forwarded as-is — absent stays absent, so the
679
+ // trigger face can tell "the server offered nothing" from "the server offered
680
+ // an empty list" rather than collapsing both into a picker with no options.
681
+ triggers={data?.triggers}
682
+ // C4's action menu + the builder's ceilings. Forwarded as-is for the same reason
683
+ // `triggers` is: absent must stay absent, so the builder can tell "the server
684
+ // offered nothing" from "the server offered an empty list".
685
+ catalog={data?.actionsCatalog}
686
+ vocab={data?.flow}
687
+ // `?? null` and never `|| {enabled:false}`: an absent tick bit is "the server did
688
+ // not say", which Step 1 prints as its own sentence. Defaulting it here would
689
+ // turn a missing field into a claim about production (C6 amendment #1).
690
+ tick={data?.tick ?? null}
691
+ /*
692
+ * ⭐ W24-W1 (item 6) — WHAT THIS RUN IS DOING, on the one surface that survives in
693
+ * BOTH views. REQUIRED on the far side deliberately: an optional prop that nobody
694
+ * passes degrades to "the feature does not exist", which is indistinguishable from
695
+ * "it was never built" — and this whole item exists because a live step text rode
696
+ * the wire for three waves with nothing rendering it.
697
+ * `liveStepOf` returns the step ONLY while the automation is running; a stale step
698
+ * from a finished run is a worse answer than none.
699
+ */
700
+ liveStep={liveStepOf(active)}
701
+ /*
702
+ * ⛔ C14 LEG 2 — THE STALE-ID WRITE-BACK, and this is the ghost's second cause.
703
+ *
704
+ * The detail calls `onSaved(automation.id)` from six places (save, the trigger
705
+ * picker, a schedule change, a node switch, a card move, a run). Each closure
706
+ * captures the automation it was mounted for, so a save that resolves AFTER the
707
+ * user has clicked a different row used to call `setActiveId(the OLD id)` — the
708
+ * rail jumped back, the detail remounted, and what the user saw was the previous
709
+ * automation reappearing over the one they had just opened. It looked like a
710
+ * rendering bug; it was a resolution steering the selection.
711
+ *
712
+ * A resolution may no longer steer anything. It reloads the list — that part was
713
+ * always right — and it re-asserts the selection ONLY when its id is still the one
714
+ * the user asked for, which makes the write a no-op in the good case and nothing
715
+ * at all in the bad one.
716
+ */
717
+ onSaved={async (id, fresh) => {
718
+ // ⭐⭐ WAVE 31 · T31 — CORRECT THE ONE ROW WE WERE JUST TOLD ABOUT.
719
+ // `patchAutomation` and `toggleNode` return the updated definition; re-reading the
720
+ // whole list to learn it is the second of the two round trips "Saving…" used to
721
+ // span. `mergeSaved` keeps the memo and the component in step (see
722
+ // `automationApi.rememberAutomation`) and takes the generation, because a write
723
+ // response is newer than any read still in flight — the same rule `load` follows,
724
+ // pointed the other way.
725
+ if (fresh && mergeSaved(fresh)) {
726
+ if (id && id === wantedId.current) select(id);
727
+ return;
728
+ }
729
+ const next = await load();
730
+ if (id && next && id === wantedId.current) select(id);
731
+ }}
732
+ onDeleted={async () => {
733
+ select("");
734
+ await load();
735
+ }}
736
+ />
737
+ </Suspense>
738
+ ) : (
739
+ /*
740
+ * THE EMPTY STATE IS ONE LINE AND A BUTTON (owner ruling R13 — "never
741
+ * over-explain", now a DESIGN.md law).
742
+ *
743
+ * It was a heading, a three-sentence paragraph and a 130-word bulleted list
744
+ * describing all three kinds. Every word of it was true and none of it was READ:
745
+ * an empty state is passed through, not studied, and the person looking at it has
746
+ * already decided to make an automation. The kinds are described where the choice
747
+ * is actually made — the create form lists them FROM THE SERVER, so that copy also
748
+ * cannot go stale the way this list had (it described two kinds after a third
749
+ * shipped).
750
+ *
751
+ * The button is here rather than only in the rail because this pane is where the
752
+ * eye is; a create affordance the user has to go find is the same defect as the
753
+ * paragraph, spent differently.
754
+ */
755
+ <div className="autob-empty">
756
+ {/*
757
+ ⛔ NO TITLE HERE ANY MORE (C13, owner item 2). This pane carried an `h1`
758
+ reading "Automations" at 20px/600 — a THIRD title treatment on a page that
759
+ also had the header's editable 16px/700 input, against every database page's
760
+ single 16px/650 `shell-db-name`. The shell now wraps this branch in the same
761
+ `shell-db-frame` + `DbHead` a database gets (wiring W23-W1), so the page's name
762
+ is drawn once, by the one component that draws every other page's name. A
763
+ stand-in restyled to match would have been a second copy of the same fact,
764
+ free to drift the day the header moves.
765
+ */}
766
+ <p className="autob-empty-line">
767
+ A job this workspace runs for you, on demand or on a schedule.
768
+ </p>
769
+ <button
770
+ type="button"
771
+ className="auto-btn is-primary"
772
+ disabled={creatingNow}
773
+ onClick={() => void createAndOpen()}
774
+ >
775
+ New automation
776
+ </button>
777
+ </div>
778
+ )}
779
+ </section>
780
+ </div>
781
+ );
782
+ }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
web/src/automation/automationApi.ts CHANGED
@@ -518,17 +518,6 @@ export interface FlowVocab {
518
  /** The ceiling `clean_post_groups` judges a group's limit against (the action's own maxPosts
519
  * is the real bound; this is the absolute one, so a hint can be honest about both). */
520
  maxPostsPerPull?: number;
521
- /**
522
- * ⭐⭐ W33-T52 (owner item 16) — THE CONFIG KEYS EACH ACTION KIND REQUIRES, straight off the
523
- * server's `ACTION_REQUIRED`. `phrase` is the human wording the runner's own refusal sentence is
524
- * built from ("a column to write into"); `key` is the config key it names.
525
- * ⛔ The panel reads THIS to decide which controls carry a required mark. A client-side copy of
526
- * the same table would agree on the day it was written and diverge the first time a kind gained
527
- * a key — leaving the one surface that exists to say "this step is not finished" saying it is.
528
- * ⚠ OPTIONAL, like every other vocabulary here: absent means a server older than this client,
529
- * and the panel marks nothing rather than inventing requirements the save door does not have.
530
- */
531
- actionRequired?: Record<string, { phrase: string; key: string }[]>;
532
  /** Legacy read compatibility only; the server no longer sends Board review vocabulary. */
533
  reviewDeciders?: string[];
534
  /** Legacy read compatibility only; the server no longer sends Board review readiness. */
 
518
  /** The ceiling `clean_post_groups` judges a group's limit against (the action's own maxPosts
519
  * is the real bound; this is the absolute one, so a hint can be honest about both). */
520
  maxPostsPerPull?: number;
 
 
 
 
 
 
 
 
 
 
 
521
  /** Legacy read compatibility only; the server no longer sends Board review vocabulary. */
522
  reviewDeciders?: string[];
523
  /** Legacy read compatibility only; the server no longer sends Board review readiness. */
web/src/connectors/ConnectorsPage.tsx CHANGED
@@ -622,20 +622,6 @@ export function ConnectorCard({
622
  {connected ? "Manage keys" : "Add keys"}
623
  </button>
624
  ) : null}
625
- {/* ⭐⭐ WAVE 33 · D-204 — THE MEMBER'S OWN DOOR. Same room, different promise: this
626
- card connects an account that is YOURS, which is why the word is not "Manage
627
- keys". It appears only for connectors whose credential may be personal — the
628
- server decides that from `TENANT_WIDE_TYPES`, so a member never sees it on Odoo
629
- or Meta Ads, where storing a personal key is refused.
630
- ⚠ `connected` on the row is per-reader (`_entry_types` reads THIS user's
631
- entries), so a member whose colleague connected Stripe business-wide still reads
632
- the card as connected and is offered nothing to fix — which is correct: there is
633
- nothing for them to do. */}
634
- {!planned && row.manage === "keychain_personal" ? (
635
- <button type="button" className="conn-card-go" onClick={onKeychain}>
636
- {connected ? "Your connection" : "Connect your own"}
637
- </button>
638
- ) : null}
639
  {!planned && row.manage === "automation" ? (
640
  <a className="conn-card-go" href="#/automation">
641
  Open automations
 
622
  {connected ? "Manage keys" : "Add keys"}
623
  </button>
624
  ) : null}
 
 
 
 
 
 
 
 
 
 
 
 
 
 
625
  {!planned && row.manage === "automation" ? (
626
  <a className="conn-card-go" href="#/automation">
627
  Open automations
web/src/connectors/brandMarks.tsx CHANGED
@@ -1,411 +1,338 @@
1
- // ---------------------------------------------------------------------------
2
- // connectors/brandMarks.tsx — WAVE 30, owner item 7 (rulings R3 + R4):
3
- // THE ONE PLACE A CONNECTOR'S IDENTITY IS DRAWN.
4
- //
5
- // The owner, verbatim and for the third time: *"their Instagram and Tiktok logo
6
- // to be fucking displayed, same with all logos under the Connectors module."*
7
- //
8
- // ⛔ THIS SUPERSEDES A RULE THAT WAS STILL SHIPPING. Wave-23 R8 said connector
9
- // identity is a two-letter monogram in the chip family, and it was enforced —
10
- // `verify_home.py:section_connectors` asserted a real mark could not appear.
11
- // R4 replaces it FOR CONNECTOR IDENTITY ONLY. Everything else the design
12
- // constitution says is untouched: no emojis anywhere, and no decorative
13
- // third-party imagery outside this file's purpose, which is telling a person
14
- // which company a row is about.
15
- //
16
- // ⛔⛔ WAVE 33 · OWNER ITEM 6 / RULING R4 — THE PARAGRAPH BELOW IS SUPERSEDED FOR
17
- // A COMPANY'S MARK, AND IT IS KEPT BECAUSE IT IS STILL RIGHT ABOUT WHAT IT WAS
18
- // ACTUALLY DEFENDING. It read:
19
- //
20
- // "INLINE SVG, NEVER A FILE AND NEVER A REMOTE URL. There is no `assets/` or
21
- // `icons/` directory in this app and there is not going to be one … A remote
22
- // logo would work today (no CSP header is set) and would be the one thing on
23
- // the page that can be taken away by somebody else's outage."
24
- //
25
- // ⭐ THE RULE IT WAS PROTECTING IS **NO THIRD-PARTY ORIGIN**, and that rule is
26
- // untouched: `/api/v1/brand/<name>` is OUR door, same-origin, session-gated,
27
- // served off bytes that ship inside the image (`web/public/brand/`). Nobody
28
- // else's outage can take it away, and there is still no `assets/` directory —
29
- // there is a brand directory, which is a different thing with a route in front
30
- // of it.
31
- // ⛔ WHAT DID CHANGE IS THE OWNER'S VERDICT ON THE ARTWORK, twice now. Wave 30's
32
- // R4 replaced wave 23's two-letter monogram with these hand-drawn marks; wave
33
- // 33's R4 replaces the hand-drawn marks with each vendor's OFFICIAL artwork,
34
- // because the honest sentence three paragraphs down — *"recognisable
35
- // SIMPLIFICATIONS … not the trademarked artwork"* — is exactly what the owner
36
- // saw: *"Do not generate your own logo for Connectors, they look really uncanny
37
- // /browse them the PNG, cut appropriately for our needs, verify visually."*
38
- // A ruling that supersedes a rule leaves that rule's gates green
39
- // ([[two-gates-can-assert-opposite-things]]); see `mailbox/G.md` G-2/G-3 for the
40
- // four legs in `verify_automation_ui` / `picker_render.test.tsx` this moves.
41
- //
42
- // ⚠ THE HAND-DRAWN SET IS NOT DELETED. Every arm below still exists and is what
43
- // renders when the door does not answer, so a 404, a 401 or a container without
44
- // the files degrades to the wave-30 mark rather than to a broken-image glyph.
45
- // Two of the arms are not fallbacks at all: `webhooks` and `scraper` are OURS,
46
- // there is no vendor artwork to fetch, and they stay inline permanently.
47
- //
48
- // ⛔ ONE MODULE FOR THREE SURFACES, and that is the whole design. The trigger
49
- // picker, the action menu and the Connectors directory all answer "which
50
- // company is this?" — three copies of a logo set is three chances to add a
51
- // platform in two places and forget the third, which is precisely the defect
52
- // wave 30 exists to fix on the server side of the same question.
53
- //
54
- // ⚠ WHAT THESE ARE, HONESTLY: recognisable SIMPLIFICATIONS drawn from each
55
- // brand's published silhouette and brand colour, at 16px, not the trademarked
56
- // artwork. That is the right call at this size — the real Google `G` is a
57
- // 500-byte path that renders as mush in a 15px box — and it is stated here so
58
- // nobody "upgrades" them by pasting a vendor's asset file into a repo that has
59
- // no place to put one.
60
- //
61
- // ⚠ THEME. Marks are brand-coloured and must survive BOTH themes, so no mark
62
- // relies on pure black or pure white for its silhouette: TikTok's note is
63
- // `currentColor` with its two brand offsets behind it, which is also how the
64
- // real mark is constructed.
65
- // ---------------------------------------------------------------------------
66
- import type { ReactElement } from "react";
67
- import { useId, useState } from "react";
68
-
69
- import { API_V1 } from "../apiContract";
70
-
71
- /**
72
- * The canonical brand names this module can draw. Deliberately NOT the same
73
- * strings as the connector keys or the trigger kinds: three vocabularies point
74
- * AT this one, and folding them together would make a rename in any of the
75
- * three a silent loss of an icon.
76
- */
77
- export type BrandName =
78
- | "instagram"
79
- | "tiktok"
80
- | "odoo"
81
- | "google"
82
- | "gmail"
83
- | "stripe"
84
- | "shopify"
85
- | "webhooks"
86
- | "scraper"
87
- | "slack"
88
- | "meta";
89
-
90
- /**
91
- * Trigger and action KIND -> brand.
92
- *
93
- * ⛔ KEYED ON THE KIND AND NOT ON THE CONNECTOR, and the reason is R3 itself:
94
- * after this wave Instagram and TikTok share `connector.key === "scraper"`, so
95
- * the connector can no longer tell them apart. The kind is the only thing that
96
- * still distinguishes the two products, which is exactly why the picker's own
97
- * `TriggerMark` has always been keyed this way.
98
- *
99
- * ⚠ An unmapped kind returns null and the caller draws its existing glyph — a
100
- * trigger the engine adds tomorrow renders plainly rather than not at all.
101
- */
102
- export const BRAND_OF_KIND: Record<string, BrandName> = {
103
- ig_profile_match: "instagram",
104
- tiktok_profile_match: "tiktok",
105
- enrich_instagram: "instagram",
106
- enrich_tiktok: "tiktok",
107
- field_instagram: "instagram",
108
- };
109
-
110
- /**
111
- * Connector/provider KEY -> brand, for the Connectors directory and for the
112
- * collapsed connector nests in both automation menus.
113
- *
114
- * ⚠ `scraper` and `webhooks` are OURS, not brands — they get house glyphs in
115
- * `currentColor` so they sit in the chip family the way they always have. They
116
- * are in this map rather than special-cased at the call site so that "what mark
117
- * does this row wear" has ONE answer, whoever is asking.
118
- */
119
- export const BRAND_OF_CONNECTOR: Record<string, BrandName> = {
120
- odoo: "odoo",
121
- // ⛔⛔ WAVE 32 · OWNER ITEM 12 / R6 — THESE TWO MARKS ARE CURRENTLY UNREACHABLE, AND KEPT.
122
- // `oauth_connect.offered("google")` is False, so the directory serves no `google` row and this
123
- // map is never asked for either of these. They stay for the reason the ruling is worded as
124
- // *"HIDDEN until they are real"* rather than "removed": the blocker is money, not code (D-45 —
125
- // Google's CASA verification, ~$540–1,800/yr, for the `gmail.readonly` scope). The day that is
126
- // paid for, ONE key flips server-side and the card comes back wearing its own logo. Deleting
127
- // the art would turn a one-key restore into a re-draw, and would leave the next reader
128
- // wondering where Gmail's mark went as well as where Gmail went.
129
- google: "google",
130
- gmail: "gmail",
131
- stripe: "stripe",
132
- shopify: "shopify",
133
- scraper: "scraper",
134
- webhooks: "webhooks",
135
- instagram: "instagram",
136
- tiktok: "tiktok",
137
- // ⭐ WAVE 31 · R7 — the Nango breadth play's first two marks. `meta_ads` is the KEYCHAIN
138
- // connector (a token an admin pastes today); `meta` would be the delegated-OAuth door the same
139
- // company owns, so BOTH keys point at one mark rather than the map growing a near-duplicate.
140
- slack: "slack",
141
- meta: "meta",
142
- meta_ads: "meta",
143
- };
144
-
145
- /** The house frame every mark is drawn in. 16×16 user units, sized by the caller. */
146
- function frame(size: number) {
147
- return { width: size, height: size, viewBox: "0 0 16 16", "aria-hidden": true } as const;
148
- }
149
-
150
- /**
151
- * ⚠ TIKTOK'S NOTE, DRAWN THREE TIMES. The real mark is one glyph offset in
152
- * cyan and magenta behind a solid copy; drawing only the solid copy loses the
153
- * brand entirely at this size, and drawing the offsets in black loses it in
154
- * dark mode. The solid copy is `currentColor` so it inherits the row.
155
- */
156
- const TT_NOTE =
157
- "M9.05 1.9h2.02c.13 1.2.87 2.13 2.03 2.3v2.03c-.78 0-1.53-.2-2.18-.58v3.98"
158
- + "a3.62 3.62 0 1 1-3.62-3.62c.16 0 .32.01.48.03v2.06a1.6 1.6 0 1 0 1.27 1.56z";
159
-
160
- /**
161
- * Brands we hold OFFICIAL artwork for, i.e. the ones `/api/v1/brand/<name>`
162
- * answers 200 on. ⛔ Derived from one place — the server's own directory listing
163
- * would be a round trip on every chip, and a hardcoded list here would drift the
164
- * moment a file is added, so `verify_meta::section_brand_door` pins this set
165
- * against the files on disk in BOTH directions. Adding a mark is: drop the PNG
166
- * in `web/public/brand/`, add its SOURCES.md row, add the key here.
167
- *
168
- * ⚠ `webhooks` and `scraper` are ABSENT ON PURPOSE and must stay absent: they are
169
- * house glyphs in `currentColor`, there is no vendor and no artwork to fetch, and
170
- * asking the door for one would be a guaranteed 404 on every render.
171
- */
172
- const OFFICIAL: ReadonlySet<BrandName> = new Set<BrandName>([
173
- "instagram", "tiktok", "odoo", "google", "gmail", "slack", "meta",
174
- "stripe", "shopify",
175
- ]);
176
-
177
- /** The door R4 mandates. Relative, so it is same-origin by construction — the
178
- * one property the superseded "never a remote URL" rule was really protecting. */
179
- export function brandSrc(name: BrandName): string {
180
- return `${API_V1}/brand/${name}`;
181
- }
182
-
183
- export function BrandMark({
184
- name,
185
- size = 15,
186
- }: {
187
- name: BrandName;
188
- size?: number;
189
- }): ReactElement | null {
190
- const f = frame(size);
191
- // ⭐⭐ WAVE 33 · R4 — THE OFFICIAL MARK, FROM OUR OWN DOOR, WITH THE WAVE-30 ART
192
- // BEHIND IT. `useState` rather than a bare `<img>` because a door that cannot
193
- // answer must degrade to a mark and never to a browser's broken-image glyph —
194
- // which is the one outcome worse than the hand-drawn logo the owner complained
195
- // about. `onError` fires for a 404, a 401 after a session expires, and a
196
- // container built without `web/public/brand/`, and each of those falls through
197
- // to the arms below.
198
- // ⚠ Hooks run for EVERY name, including the two house glyphs, because a hook
199
- // cannot sit behind a conditional return — same reason `useId` is unconditional.
200
- const [failed, setFailed] = useState(false);
201
- if (OFFICIAL.has(name) && !failed)
202
- return (
203
- <img
204
- src={brandSrc(name)}
205
- width={size}
206
- height={size}
207
- alt=""
208
- aria-hidden="true"
209
- // ⚠ `block` kills the inline-baseline gap that would make a 15px logo sit
210
- // a pixel lower than the 15px SVG beside it in the same chip row.
211
- style={{ display: "block", width: size, height: size }}
212
- onError={() => setFailed(true)}
213
- />
214
- );
215
- // Per-instance, for the reason spelled out at the `<defs>` below. Hooks run for every
216
- // name, not only Instagram, because a hook cannot sit behind a conditional return.
217
- const gradId = `aios-brand-ig-${useId()}`;
218
- if (name === "instagram")
219
- return (
220
- <svg {...f} fill="none">
221
- {/*
222
- ⛔⛔ THE GRADIENT ID IS PER-INSTANCE, AND THE FIXED ONE WAS A REAL BUG — FOUND BY A
223
- PICTURE, NOT BY A GATE. It shipped for an hour as the constant `aios-brand-ig`, on the
224
- reasoning that duplicate ids resolve to the FIRST definition and every definition is
225
- byte-identical, so it cannot matter. MEASURED in the connectors shot: the Scraper card
226
- holds an Instagram mark inside a COLLAPSED `<details>`, a second card below holds a
227
- visible one — and the visible one painted NOTHING. A closed `<details>` is
228
- `content-visibility: hidden`, its subtree is not rendered, and a paint server defined
229
- inside a skipped subtree does not resolve for a user outside it. "The first definition
230
- wins" IS the failure: whichever copy happens to be hidden decides for all of them.
231
- ⚠ Nothing about `<details>` is special — `display:none` does the same. Any page holding
232
- two of this mark with the first one hidden loses the second.
233
- ⚠ `useId` rather than a module counter, so the markup stays DETERMINISTIC for a given
234
- tree — the render gates could not assert against a random id — and stable across a
235
- server render and its hydration.
236
- ⚠ Every check that looks for this mark matches the PREFIX `url(#aios-brand-ig`.
237
- */}
238
- <defs>
239
- {/* ⚠ `userSpaceOnUse` IS LOAD-BEARING AND ITS ABSENCE LOOKED FINE. The default is
240
- `objectBoundingBox`, where the coordinates are FRACTIONS of the shape's box — so
241
- `1,15 → 15,1` put the whole gradient vector far outside every path and each one
242
- painted a single clamped stop. The mark still rendered, still read as Instagram,
243
- and was simply flat pink: a defect only a picture catches, which is why one was
244
- taken ([[ui-invisible-to-assertions]]). In user space the numbers mean viewBox
245
- units, which is what they were written as. */}
246
- <linearGradient
247
- id={gradId} gradientUnits="userSpaceOnUse"
248
- x1="2" y1="14" x2="14" y2="2"
249
- >
250
- <stop offset="0" stopColor="#FEDA75" />
251
- <stop offset="0.32" stopColor="#FA7E1E" />
252
- <stop offset="0.62" stopColor="#D62976" />
253
- <stop offset="1" stopColor="#962FBF" />
254
- </linearGradient>
255
- </defs>
256
- <rect
257
- x="1.7" y="1.7" width="12.6" height="12.6" rx="3.7"
258
- stroke={`url(#${gradId})`} strokeWidth="1.5"
259
- />
260
- <circle cx="8" cy="8" r="3" stroke={`url(#${gradId})`} strokeWidth="1.5" />
261
- <circle cx="11.6" cy="4.4" r="0.95" fill={`url(#${gradId})`} />
262
- </svg>
263
- );
264
- if (name === "tiktok")
265
- return (
266
- <svg {...f}>
267
- <path d={TT_NOTE} fill="#25F4EE" transform="translate(-0.9 -0.5)" />
268
- <path d={TT_NOTE} fill="#FE2C55" transform="translate(0.9 0.5)" />
269
- <path d={TT_NOTE} fill="currentColor" />
270
- </svg>
271
- );
272
- if (name === "odoo")
273
- return (
274
- <svg {...f}>
275
- {/* Odoo's brand purple, and its wordmark's twin `o`s — the part of that
276
- wordmark that survives being shrunk to 15px. */}
277
- <rect x="1" y="1" width="14" height="14" rx="3.4" fill="#714B67" />
278
- <circle cx="5.7" cy="8.2" r="2.1" fill="none" stroke="#FFFFFF" strokeWidth="1.5" />
279
- <circle cx="10.4" cy="8.2" r="2.1" fill="none" stroke="#FFFFFF" strokeWidth="1.5" />
280
- </svg>
281
- );
282
- if (name === "google")
283
- return (
284
- <svg {...f} fill="none" strokeWidth="2.3">
285
- {/* The four-arc G. Each arc is one brand colour, and the bar into the
286
- centre is what makes it a G rather than a ring. */}
287
- <path d="M4.46 4.46A5 5 0 0 1 11.54 4.46" stroke="#EA4335" strokeLinecap="round" />
288
- <path d="M4.46 4.46A5 5 0 0 0 4.46 11.54" stroke="#FBBC05" strokeLinecap="round" />
289
- <path d="M4.46 11.54A5 5 0 0 0 11.54 11.54" stroke="#34A853" strokeLinecap="round" />
290
- <path d="M11.54 11.54A5 5 0 0 0 13 8" stroke="#4285F4" strokeLinecap="round" />
291
- <path d="M13 8H8.4" stroke="#4285F4" />
292
- </svg>
293
- );
294
- if (name === "gmail")
295
- return (
296
- <svg {...f} fill="none" stroke="#EA4335" strokeWidth="1.5" strokeLinejoin="round">
297
- <rect x="1.6" y="3.6" width="12.8" height="9" rx="1.6" />
298
- <path d="m2 4.4 6 4.6 6-4.6" />
299
- </svg>
300
- );
301
- // ⭐ WAVE 31 · R7. Slack's mark is FOUR bars in four brand colours, drawn as rounded rects
302
- // rather than the true interlocking glyph: at 15px the real lozenges collapse into mud, and a
303
- // mark nobody can identify is worse than a two-letter tile. Colours are Slack's own.
304
- if (name === "slack")
305
- return (
306
- <svg {...f}>
307
- <rect x="2" y="6.6" width="5.4" height="2.8" rx="1.4" fill="#36C5F0" />
308
- <rect x="8.6" y="6.6" width="5.4" height="2.8" rx="1.4" fill="#2EB67D" />
309
- <rect x="6.6" y="2" width="2.8" height="5.4" rx="1.4" fill="#ECB22E" />
310
- <rect x="6.6" y="8.6" width="2.8" height="5.4" rx="1.4" fill="#E01E5A" />
311
- </svg>
312
- );
313
- // Meta's infinity loop, as two overlapping strokes in the brand blue. ⚠ `fill="none"` with a
314
- // round cap: filled, the two lobes merge into a blob at this size.
315
- if (name === "meta")
316
- return (
317
- <svg {...f} fill="none">
318
- <path
319
- d="M2.2 10.4c0-3 1.5-5.2 3.3-5.2 1.2 0 2 .8 3 2.4l1 1.6c.9 1.4 1.5 2.1 2.2 2.1.8 0
320
- 1.3-.8 1.3-2.1 0-1.6-.7-3.4-2-3.4-.9 0-1.7.7-2.6 2"
321
- stroke="#0081FB" strokeWidth="1.7" strokeLinecap="round" strokeLinejoin="round"
322
- />
323
- <path
324
- d="M2.2 10.4c0 1.3.6 2 1.5 2 .8 0 1.4-.5 2.5-2.2l3-4.6"
325
- stroke="#0081FB" strokeWidth="1.7" strokeLinecap="round" strokeLinejoin="round"
326
- />
327
- </svg>
328
- );
329
- if (name === "stripe")
330
- return (
331
- <svg {...f}>
332
- <rect x="1" y="1" width="14" height="14" rx="3.4" fill="#635BFF" />
333
- <path
334
- d="M10.5 5.6a2.3 2.3 0 0 0-3.6 1.2c0 2.1 4.1 1.5 4.1 3.5a2.4 2.4 0 0 1-3.7 1.3"
335
- fill="none" stroke="#FFFFFF" strokeWidth="1.5" strokeLinecap="round"
336
- />
337
- </svg>
338
- );
339
- if (name === "shopify")
340
- return (
341
- <svg {...f}>
342
- {/* The bag, in Shopify's two greens: the darker one is the handle, which
343
- is the part that reads as a bag rather than as a box. */}
344
- <path d="M3.5 5.2h9l-.75 8.4a.9.9 0 0 1-.9.8H5.15a.9.9 0 0 1-.9-.8z" fill="#95BF47" />
345
- <path
346
- d="M6.1 6.4V4.5a1.9 1.9 0 0 1 3.8 0v1.9"
347
- fill="none" stroke="#5E8E3E" strokeWidth="1.4" strokeLinecap="round"
348
- />
349
- </svg>
350
- );
351
- /*
352
- ⚠ THE LAST TWO ARE OURS, NOT BRANDS — `currentColor`, house stroke weight,
353
- so they sit in the chip family beside the coloured logos rather than
354
- pretending to be a company. Keeping them here is what lets every caller ask
355
- one question instead of two.
356
- */
357
- if (name === "webhooks")
358
- return (
359
- <svg {...f} fill="none" stroke="currentColor" strokeWidth="1.4"
360
- strokeLinecap="round" strokeLinejoin="round">
361
- <path d="M6.6 9.4 4.9 11a2.6 2.6 0 1 1-1.5-4.4" />
362
- <path d="M9.4 6.6 11.1 5a2.6 2.6 0 1 1 1.5 4.4" />
363
- <path d="M6.2 8h3.6" />
364
- </svg>
365
- );
366
- if (name === "scraper")
367
- return (
368
- <svg {...f} fill="none" stroke="currentColor" strokeWidth="1.4"
369
- strokeLinecap="round" strokeLinejoin="round">
370
- {/* A magnifier over a grid of rows: it reads a catalogue of profiles and
371
- makes records out of what matches — which is what the bucket does,
372
- and it is deliberately NOT either platform's mark, because the whole
373
- point of R3 is that the bucket is neither one of them. */}
374
- <path d="M2.6 3.4h10.8M2.6 6.2h5.2M2.6 9h3.4" />
375
- <circle cx="10.4" cy="10.2" r="3" />
376
- <path d="m12.6 12.4 1.6 1.6" />
377
- </svg>
378
- );
379
- return null;
380
- }
381
-
382
- /**
383
- * Does this kind wear a LOGO rather than a house glyph?
384
- *
385
- * ⛔ IT EXISTS BECAUSE THE TILE HAS TO CHANGE TOO, and forgetting that would have shipped the
386
- * defect quietly: `.autox-card-mark` paints a blue tint and sets `color: var(--lp-blue-deep)`,
387
- * so TikTok's note — which is `currentColor` by design, so it survives dark mode — would have
388
- * rendered BLUE, and Instagram's gradient would have sat on a coloured chip. The caller adds
389
- * `is-brand`, which neutralises both. One predicate, so the mark and its tile cannot disagree
390
- * about whether a row is branded.
391
- */
392
- export function hasBrandKind(kind: string): boolean {
393
- return Boolean(BRAND_OF_KIND[kind]);
394
- }
395
-
396
- /** The connector-key twin of `hasBrandKind`, for the directory's tiles. */
397
- export function hasBrandConnector(key: string): boolean {
398
- return Boolean(BRAND_OF_CONNECTOR[key]);
399
- }
400
-
401
- /** The mark for a trigger or action KIND, or null when it is not a branded one. */
402
- export function brandForKind(kind: string, size = 15): ReactElement | null {
403
- const name = BRAND_OF_KIND[kind];
404
- return name ? <BrandMark name={name} size={size} /> : null;
405
- }
406
-
407
- /** The mark for a connector/provider KEY, or null when we have no logo for it. */
408
- export function brandForConnector(key: string, size = 15): ReactElement | null {
409
- const name = BRAND_OF_CONNECTOR[key];
410
- return name ? <BrandMark name={name} size={size} /> : null;
411
- }
 
1
+ // ---------------------------------------------------------------------------
2
+ // connectors/brandMarks.tsx — WAVE 30, owner item 7 (rulings R3 + R4):
3
+ // THE ONE PLACE A CONNECTOR'S IDENTITY IS DRAWN.
4
+ //
5
+ // The owner, verbatim and for the third time: *"their Instagram and Tiktok logo
6
+ // to be fucking displayed, same with all logos under the Connectors module."*
7
+ //
8
+ // ⛔ THIS SUPERSEDES A RULE THAT WAS STILL SHIPPING. Wave-23 R8 said connector
9
+ // identity is a two-letter monogram in the chip family, and it was enforced —
10
+ // `verify_home.py:section_connectors` asserted a real mark could not appear.
11
+ // R4 replaces it FOR CONNECTOR IDENTITY ONLY. Everything else the design
12
+ // constitution says is untouched: no emojis anywhere, and no decorative
13
+ // third-party imagery outside this file's purpose, which is telling a person
14
+ // which company a row is about.
15
+ //
16
+ // ⛔ INLINE SVG, NEVER A FILE AND NEVER A REMOTE URL. There is no `assets/` or
17
+ // `icons/` directory in this app and there is not going to be one: every icon
18
+ // in the product is a React component, `index.css` inlines even its font as a
19
+ // `data:` URI, and `customer-grid/types.ts` carries the standing sentence
20
+ // *"Never an external URL: those are neither same-origin nor CSP-safe here."*
21
+ // A remote logo would work today (no CSP header is set) and would be the one
22
+ // thing on the page that can be taken away by somebody else's outage.
23
+ //
24
+ // ⛔ ONE MODULE FOR THREE SURFACES, and that is the whole design. The trigger
25
+ // picker, the action menu and the Connectors directory all answer "which
26
+ // company is this?" — three copies of a logo set is three chances to add a
27
+ // platform in two places and forget the third, which is precisely the defect
28
+ // wave 30 exists to fix on the server side of the same question.
29
+ //
30
+ // ⚠ WHAT THESE ARE, HONESTLY: recognisable SIMPLIFICATIONS drawn from each
31
+ // brand's published silhouette and brand colour, at 16px, not the trademarked
32
+ // artwork. That is the right call at this size — the real Google `G` is a
33
+ // 500-byte path that renders as mush in a 15px box — and it is stated here so
34
+ // nobody "upgrades" them by pasting a vendor's asset file into a repo that has
35
+ // no place to put one.
36
+ //
37
+ // ⚠ THEME. Marks are brand-coloured and must survive BOTH themes, so no mark
38
+ // relies on pure black or pure white for its silhouette: TikTok's note is
39
+ // `currentColor` with its two brand offsets behind it, which is also how the
40
+ // real mark is constructed.
41
+ // ---------------------------------------------------------------------------
42
+ import type { ReactElement } from "react";
43
+ import { useId } from "react";
44
+
45
+ /**
46
+ * The canonical brand names this module can draw. Deliberately NOT the same
47
+ * strings as the connector keys or the trigger kinds: three vocabularies point
48
+ * AT this one, and folding them together would make a rename in any of the
49
+ * three a silent loss of an icon.
50
+ */
51
+ export type BrandName =
52
+ | "instagram"
53
+ | "tiktok"
54
+ | "odoo"
55
+ | "google"
56
+ | "gmail"
57
+ | "stripe"
58
+ | "shopify"
59
+ | "webhooks"
60
+ | "scraper"
61
+ | "slack"
62
+ | "meta";
63
+
64
+ /**
65
+ * Trigger and action KIND -> brand.
66
+ *
67
+ * ⛔ KEYED ON THE KIND AND NOT ON THE CONNECTOR, and the reason is R3 itself:
68
+ * after this wave Instagram and TikTok share `connector.key === "scraper"`, so
69
+ * the connector can no longer tell them apart. The kind is the only thing that
70
+ * still distinguishes the two products, which is exactly why the picker's own
71
+ * `TriggerMark` has always been keyed this way.
72
+ *
73
+ * ⚠ An unmapped kind returns null and the caller draws its existing glyph — a
74
+ * trigger the engine adds tomorrow renders plainly rather than not at all.
75
+ */
76
+ export const BRAND_OF_KIND: Record<string, BrandName> = {
77
+ ig_profile_match: "instagram",
78
+ tiktok_profile_match: "tiktok",
79
+ enrich_instagram: "instagram",
80
+ enrich_tiktok: "tiktok",
81
+ field_instagram: "instagram",
82
+ };
83
+
84
+ /**
85
+ * Connector/provider KEY -> brand, for the Connectors directory and for the
86
+ * collapsed connector nests in both automation menus.
87
+ *
88
+ * ⚠ `scraper` and `webhooks` are OURS, not brands — they get house glyphs in
89
+ * `currentColor` so they sit in the chip family the way they always have. They
90
+ * are in this map rather than special-cased at the call site so that "what mark
91
+ * does this row wear" has ONE answer, whoever is asking.
92
+ */
93
+ export const BRAND_OF_CONNECTOR: Record<string, BrandName> = {
94
+ odoo: "odoo",
95
+ // ⛔⛔ WAVE 32 · OWNER ITEM 12 / R6 — THESE TWO MARKS ARE CURRENTLY UNREACHABLE, AND KEPT.
96
+ // `oauth_connect.offered("google")` is False, so the directory serves no `google` row and this
97
+ // map is never asked for either of these. They stay for the reason the ruling is worded as
98
+ // *"HIDDEN until they are real"* rather than "removed": the blocker is money, not code (D-45 —
99
+ // Google's CASA verification, ~$540–1,800/yr, for the `gmail.readonly` scope). The day that is
100
+ // paid for, ONE key flips server-side and the card comes back wearing its own logo. Deleting
101
+ // the art would turn a one-key restore into a re-draw, and would leave the next reader
102
+ // wondering where Gmail's mark went as well as where Gmail went.
103
+ google: "google",
104
+ gmail: "gmail",
105
+ stripe: "stripe",
106
+ shopify: "shopify",
107
+ scraper: "scraper",
108
+ webhooks: "webhooks",
109
+ instagram: "instagram",
110
+ tiktok: "tiktok",
111
+ // ⭐ WAVE 31 · R7 — the Nango breadth play's first two marks. `meta_ads` is the KEYCHAIN
112
+ // connector (a token an admin pastes today); `meta` would be the delegated-OAuth door the same
113
+ // company owns, so BOTH keys point at one mark rather than the map growing a near-duplicate.
114
+ slack: "slack",
115
+ meta: "meta",
116
+ meta_ads: "meta",
117
+ };
118
+
119
+ /** The house frame every mark is drawn in. 16×16 user units, sized by the caller. */
120
+ function frame(size: number) {
121
+ return { width: size, height: size, viewBox: "0 0 16 16", "aria-hidden": true } as const;
122
+ }
123
+
124
+ /**
125
+ * ⚠ TIKTOK'S NOTE, DRAWN THREE TIMES. The real mark is one glyph offset in
126
+ * cyan and magenta behind a solid copy; drawing only the solid copy loses the
127
+ * brand entirely at this size, and drawing the offsets in black loses it in
128
+ * dark mode. The solid copy is `currentColor` so it inherits the row.
129
+ */
130
+ const TT_NOTE =
131
+ "M9.05 1.9h2.02c.13 1.2.87 2.13 2.03 2.3v2.03c-.78 0-1.53-.2-2.18-.58v3.98"
132
+ + "a3.62 3.62 0 1 1-3.62-3.62c.16 0 .32.01.48.03v2.06a1.6 1.6 0 1 0 1.27 1.56z";
133
+
134
+ export function BrandMark({
135
+ name,
136
+ size = 15,
137
+ }: {
138
+ name: BrandName;
139
+ size?: number;
140
+ }): ReactElement | null {
141
+ const f = frame(size);
142
+ // Per-instance, for the reason spelled out at the `<defs>` below. Hooks run for every
143
+ // name, not only Instagram, because a hook cannot sit behind a conditional return.
144
+ const gradId = `aios-brand-ig-${useId()}`;
145
+ if (name === "instagram")
146
+ return (
147
+ <svg {...f} fill="none">
148
+ {/*
149
+ ⛔⛔ THE GRADIENT ID IS PER-INSTANCE, AND THE FIXED ONE WAS A REAL BUG — FOUND BY A
150
+ PICTURE, NOT BY A GATE. It shipped for an hour as the constant `aios-brand-ig`, on the
151
+ reasoning that duplicate ids resolve to the FIRST definition and every definition is
152
+ byte-identical, so it cannot matter. MEASURED in the connectors shot: the Scraper card
153
+ holds an Instagram mark inside a COLLAPSED `<details>`, a second card below holds a
154
+ visible one — and the visible one painted NOTHING. A closed `<details>` is
155
+ `content-visibility: hidden`, its subtree is not rendered, and a paint server defined
156
+ inside a skipped subtree does not resolve for a user outside it. "The first definition
157
+ wins" IS the failure: whichever copy happens to be hidden decides for all of them.
158
+ ⚠ Nothing about `<details>` is special — `display:none` does the same. Any page holding
159
+ two of this mark with the first one hidden loses the second.
160
+ ⚠ `useId` rather than a module counter, so the markup stays DETERMINISTIC for a given
161
+ tree — the render gates could not assert against a random id — and stable across a
162
+ server render and its hydration.
163
+ ⚠ Every check that looks for this mark matches the PREFIX `url(#aios-brand-ig`.
164
+ */}
165
+ <defs>
166
+ {/* ⚠ `userSpaceOnUse` IS LOAD-BEARING AND ITS ABSENCE LOOKED FINE. The default is
167
+ `objectBoundingBox`, where the coordinates are FRACTIONS of the shape's box — so
168
+ `1,15 → 15,1` put the whole gradient vector far outside every path and each one
169
+ painted a single clamped stop. The mark still rendered, still read as Instagram,
170
+ and was simply flat pink: a defect only a picture catches, which is why one was
171
+ taken ([[ui-invisible-to-assertions]]). In user space the numbers mean viewBox
172
+ units, which is what they were written as. */}
173
+ <linearGradient
174
+ id={gradId} gradientUnits="userSpaceOnUse"
175
+ x1="2" y1="14" x2="14" y2="2"
176
+ >
177
+ <stop offset="0" stopColor="#FEDA75" />
178
+ <stop offset="0.32" stopColor="#FA7E1E" />
179
+ <stop offset="0.62" stopColor="#D62976" />
180
+ <stop offset="1" stopColor="#962FBF" />
181
+ </linearGradient>
182
+ </defs>
183
+ <rect
184
+ x="1.7" y="1.7" width="12.6" height="12.6" rx="3.7"
185
+ stroke={`url(#${gradId})`} strokeWidth="1.5"
186
+ />
187
+ <circle cx="8" cy="8" r="3" stroke={`url(#${gradId})`} strokeWidth="1.5" />
188
+ <circle cx="11.6" cy="4.4" r="0.95" fill={`url(#${gradId})`} />
189
+ </svg>
190
+ );
191
+ if (name === "tiktok")
192
+ return (
193
+ <svg {...f}>
194
+ <path d={TT_NOTE} fill="#25F4EE" transform="translate(-0.9 -0.5)" />
195
+ <path d={TT_NOTE} fill="#FE2C55" transform="translate(0.9 0.5)" />
196
+ <path d={TT_NOTE} fill="currentColor" />
197
+ </svg>
198
+ );
199
+ if (name === "odoo")
200
+ return (
201
+ <svg {...f}>
202
+ {/* Odoo's brand purple, and its wordmark's twin `o`s — the part of that
203
+ wordmark that survives being shrunk to 15px. */}
204
+ <rect x="1" y="1" width="14" height="14" rx="3.4" fill="#714B67" />
205
+ <circle cx="5.7" cy="8.2" r="2.1" fill="none" stroke="#FFFFFF" strokeWidth="1.5" />
206
+ <circle cx="10.4" cy="8.2" r="2.1" fill="none" stroke="#FFFFFF" strokeWidth="1.5" />
207
+ </svg>
208
+ );
209
+ if (name === "google")
210
+ return (
211
+ <svg {...f} fill="none" strokeWidth="2.3">
212
+ {/* The four-arc G. Each arc is one brand colour, and the bar into the
213
+ centre is what makes it a G rather than a ring. */}
214
+ <path d="M4.46 4.46A5 5 0 0 1 11.54 4.46" stroke="#EA4335" strokeLinecap="round" />
215
+ <path d="M4.46 4.46A5 5 0 0 0 4.46 11.54" stroke="#FBBC05" strokeLinecap="round" />
216
+ <path d="M4.46 11.54A5 5 0 0 0 11.54 11.54" stroke="#34A853" strokeLinecap="round" />
217
+ <path d="M11.54 11.54A5 5 0 0 0 13 8" stroke="#4285F4" strokeLinecap="round" />
218
+ <path d="M13 8H8.4" stroke="#4285F4" />
219
+ </svg>
220
+ );
221
+ if (name === "gmail")
222
+ return (
223
+ <svg {...f} fill="none" stroke="#EA4335" strokeWidth="1.5" strokeLinejoin="round">
224
+ <rect x="1.6" y="3.6" width="12.8" height="9" rx="1.6" />
225
+ <path d="m2 4.4 6 4.6 6-4.6" />
226
+ </svg>
227
+ );
228
+ // ⭐ WAVE 31 · R7. Slack's mark is FOUR bars in four brand colours, drawn as rounded rects
229
+ // rather than the true interlocking glyph: at 15px the real lozenges collapse into mud, and a
230
+ // mark nobody can identify is worse than a two-letter tile. Colours are Slack's own.
231
+ if (name === "slack")
232
+ return (
233
+ <svg {...f}>
234
+ <rect x="2" y="6.6" width="5.4" height="2.8" rx="1.4" fill="#36C5F0" />
235
+ <rect x="8.6" y="6.6" width="5.4" height="2.8" rx="1.4" fill="#2EB67D" />
236
+ <rect x="6.6" y="2" width="2.8" height="5.4" rx="1.4" fill="#ECB22E" />
237
+ <rect x="6.6" y="8.6" width="2.8" height="5.4" rx="1.4" fill="#E01E5A" />
238
+ </svg>
239
+ );
240
+ // Meta's infinity loop, as two overlapping strokes in the brand blue. ⚠ `fill="none"` with a
241
+ // round cap: filled, the two lobes merge into a blob at this size.
242
+ if (name === "meta")
243
+ return (
244
+ <svg {...f} fill="none">
245
+ <path
246
+ d="M2.2 10.4c0-3 1.5-5.2 3.3-5.2 1.2 0 2 .8 3 2.4l1 1.6c.9 1.4 1.5 2.1 2.2 2.1.8 0
247
+ 1.3-.8 1.3-2.1 0-1.6-.7-3.4-2-3.4-.9 0-1.7.7-2.6 2"
248
+ stroke="#0081FB" strokeWidth="1.7" strokeLinecap="round" strokeLinejoin="round"
249
+ />
250
+ <path
251
+ d="M2.2 10.4c0 1.3.6 2 1.5 2 .8 0 1.4-.5 2.5-2.2l3-4.6"
252
+ stroke="#0081FB" strokeWidth="1.7" strokeLinecap="round" strokeLinejoin="round"
253
+ />
254
+ </svg>
255
+ );
256
+ if (name === "stripe")
257
+ return (
258
+ <svg {...f}>
259
+ <rect x="1" y="1" width="14" height="14" rx="3.4" fill="#635BFF" />
260
+ <path
261
+ d="M10.5 5.6a2.3 2.3 0 0 0-3.6 1.2c0 2.1 4.1 1.5 4.1 3.5a2.4 2.4 0 0 1-3.7 1.3"
262
+ fill="none" stroke="#FFFFFF" strokeWidth="1.5" strokeLinecap="round"
263
+ />
264
+ </svg>
265
+ );
266
+ if (name === "shopify")
267
+ return (
268
+ <svg {...f}>
269
+ {/* The bag, in Shopify's two greens: the darker one is the handle, which
270
+ is the part that reads as a bag rather than as a box. */}
271
+ <path d="M3.5 5.2h9l-.75 8.4a.9.9 0 0 1-.9.8H5.15a.9.9 0 0 1-.9-.8z" fill="#95BF47" />
272
+ <path
273
+ d="M6.1 6.4V4.5a1.9 1.9 0 0 1 3.8 0v1.9"
274
+ fill="none" stroke="#5E8E3E" strokeWidth="1.4" strokeLinecap="round"
275
+ />
276
+ </svg>
277
+ );
278
+ /*
279
+ ⚠ THE LAST TWO ARE OURS, NOT BRANDS — `currentColor`, house stroke weight,
280
+ so they sit in the chip family beside the coloured logos rather than
281
+ pretending to be a company. Keeping them here is what lets every caller ask
282
+ one question instead of two.
283
+ */
284
+ if (name === "webhooks")
285
+ return (
286
+ <svg {...f} fill="none" stroke="currentColor" strokeWidth="1.4"
287
+ strokeLinecap="round" strokeLinejoin="round">
288
+ <path d="M6.6 9.4 4.9 11a2.6 2.6 0 1 1-1.5-4.4" />
289
+ <path d="M9.4 6.6 11.1 5a2.6 2.6 0 1 1 1.5 4.4" />
290
+ <path d="M6.2 8h3.6" />
291
+ </svg>
292
+ );
293
+ if (name === "scraper")
294
+ return (
295
+ <svg {...f} fill="none" stroke="currentColor" strokeWidth="1.4"
296
+ strokeLinecap="round" strokeLinejoin="round">
297
+ {/* A magnifier over a grid of rows: it reads a catalogue of profiles and
298
+ makes records out of what matches — which is what the bucket does,
299
+ and it is deliberately NOT either platform's mark, because the whole
300
+ point of R3 is that the bucket is neither one of them. */}
301
+ <path d="M2.6 3.4h10.8M2.6 6.2h5.2M2.6 9h3.4" />
302
+ <circle cx="10.4" cy="10.2" r="3" />
303
+ <path d="m12.6 12.4 1.6 1.6" />
304
+ </svg>
305
+ );
306
+ return null;
307
+ }
308
+
309
+ /**
310
+ * Does this kind wear a LOGO rather than a house glyph?
311
+ *
312
+ * ⛔ IT EXISTS BECAUSE THE TILE HAS TO CHANGE TOO, and forgetting that would have shipped the
313
+ * defect quietly: `.autox-card-mark` paints a blue tint and sets `color: var(--lp-blue-deep)`,
314
+ * so TikTok's note — which is `currentColor` by design, so it survives dark mode — would have
315
+ * rendered BLUE, and Instagram's gradient would have sat on a coloured chip. The caller adds
316
+ * `is-brand`, which neutralises both. One predicate, so the mark and its tile cannot disagree
317
+ * about whether a row is branded.
318
+ */
319
+ export function hasBrandKind(kind: string): boolean {
320
+ return Boolean(BRAND_OF_KIND[kind]);
321
+ }
322
+
323
+ /** The connector-key twin of `hasBrandKind`, for the directory's tiles. */
324
+ export function hasBrandConnector(key: string): boolean {
325
+ return Boolean(BRAND_OF_CONNECTOR[key]);
326
+ }
327
+
328
+ /** The mark for a trigger or action KIND, or null when it is not a branded one. */
329
+ export function brandForKind(kind: string, size = 15): ReactElement | null {
330
+ const name = BRAND_OF_KIND[kind];
331
+ return name ? <BrandMark name={name} size={size} /> : null;
332
+ }
333
+
334
+ /** The mark for a connector/provider KEY, or null when we have no logo for it. */
335
+ export function brandForConnector(key: string, size = 15): ReactElement | null {
336
+ const name = BRAND_OF_CONNECTOR[key];
337
+ return name ? <BrandMark name={name} size={size} /> : null;
338
+ }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
web/src/customer-grid/ColumnMenu.tsx CHANGED
The diff for this file is too large to render. See raw diff
 
web/src/customer-grid/CustomerGrid.tsx CHANGED
The diff for this file is too large to render. See raw diff
 
web/src/customer-grid/ImportDialog.css CHANGED
@@ -1,154 +1,154 @@
1
- /* customer-grid / ImportDialog.css — WAVE-29 T25 (owner item 6).
2
- Tokens only (`--lp-*`): this file may not invent a colour, and `index.css` is another
3
- session's fence, so the dialog's own styles live beside its component. */
4
-
5
- .cg-import-scrim {
6
- position: fixed;
7
- inset: 0;
8
- z-index: 60;
9
- display: flex;
10
- align-items: center;
11
- justify-content: center;
12
- background: rgba(32, 36, 51, 0.28);
13
- padding: 24px;
14
- }
15
-
16
- .cg-import {
17
- display: flex;
18
- flex-direction: column;
19
- width: min(560px, 100%);
20
- max-height: min(80vh, 720px);
21
- background: var(--lp-bg);
22
- border: 1px solid var(--lp-line);
23
- border-radius: var(--lp-r-lg);
24
- box-shadow: 0 12px 32px rgba(32, 36, 51, 0.18);
25
- overflow: hidden;
26
- }
27
-
28
- .cg-import-head {
29
- display: flex;
30
- align-items: center;
31
- justify-content: space-between;
32
- gap: 12px;
33
- padding: 14px 16px;
34
- border-bottom: 1px solid var(--lp-line);
35
- }
36
-
37
- .cg-import-head h2 {
38
- margin: 0;
39
- font-size: var(--lp-fs-md);
40
- font-weight: 600;
41
- color: var(--lp-ink);
42
- }
43
-
44
- .cg-import-x {
45
- border: 0;
46
- background: none;
47
- color: var(--lp-muted);
48
- font-size: var(--lp-fs-lg);
49
- line-height: 1;
50
- cursor: pointer;
51
- padding: 2px 6px;
52
- }
53
-
54
- .cg-import-x:hover {
55
- color: var(--lp-ink);
56
- }
57
-
58
- .cg-import-body {
59
- padding: 14px 16px;
60
- overflow: auto;
61
- }
62
-
63
- .cg-import-lede {
64
- margin: 0 0 8px;
65
- font-size: var(--lp-fs-sm);
66
- color: var(--lp-ink);
67
- }
68
-
69
- .cg-import-note {
70
- margin: 0 0 12px;
71
- font-size: var(--lp-fs-xs);
72
- color: var(--lp-muted);
73
- }
74
-
75
- .cg-import-file {
76
- font-size: var(--lp-fs-sm);
77
- }
78
-
79
- /* The mapping list: the spreadsheet's column on the left, this database's field on the right,
80
- so the eye reads it as "this becomes that" rather than as a form. */
81
- .cg-import-map {
82
- display: flex;
83
- flex-direction: column;
84
- gap: 6px;
85
- }
86
-
87
- .cg-import-row {
88
- display: grid;
89
- grid-template-columns: 1fr 200px;
90
- align-items: center;
91
- gap: 10px;
92
- }
93
-
94
- .cg-import-col {
95
- font-size: var(--lp-fs-sm);
96
- color: var(--lp-ink);
97
- overflow: hidden;
98
- text-overflow: ellipsis;
99
- white-space: nowrap;
100
- }
101
-
102
- .cg-import-row select {
103
- font-family: inherit;
104
- font-size: var(--lp-fs-xs);
105
- padding: 4px 6px;
106
- border: 1px solid var(--lp-line);
107
- border-radius: var(--lp-r-sm);
108
- background: var(--lp-bg);
109
- color: var(--lp-ink);
110
- }
111
-
112
- /* A refusal is not an error state to be styled red-alert; it is a list of things to fix, so it
113
- takes the yellow tint the rest of the product uses for "look at this before continuing". */
114
- .cg-import-problems {
115
- margin-top: 12px;
116
- padding: 10px 12px;
117
- border: 1px solid var(--lp-yellow);
118
- border-radius: var(--lp-r-sm);
119
- background: var(--lp-yellow-tint);
120
- color: var(--lp-yellow-deep);
121
- font-size: var(--lp-fs-xs);
122
- }
123
-
124
- .cg-import-problems ul {
125
- margin: 6px 0;
126
- padding-left: 18px;
127
- }
128
-
129
- .cg-import-problems p {
130
- margin: 6px 0 0;
131
- }
132
-
133
- .cg-import-error {
134
- margin: 0;
135
- padding: 10px 16px;
136
- background: var(--lp-red-tint);
137
- color: var(--lp-red-deep);
138
- font-size: var(--lp-fs-xs);
139
- }
140
-
141
- .cg-import-foot {
142
- display: flex;
143
- align-items: center;
144
- justify-content: flex-end;
145
- gap: 8px;
146
- padding: 12px 16px;
147
- border-top: 1px solid var(--lp-line);
148
- }
149
-
150
- .cg-import-count {
151
- margin-right: auto;
152
- font-size: var(--lp-fs-xs);
153
- color: var(--lp-muted);
154
- }
 
1
+ /* customer-grid / ImportDialog.css — WAVE-29 T25 (owner item 6).
2
+ Tokens only (`--lp-*`): this file may not invent a colour, and `index.css` is another
3
+ session's fence, so the dialog's own styles live beside its component. */
4
+
5
+ .cg-import-scrim {
6
+ position: fixed;
7
+ inset: 0;
8
+ z-index: 60;
9
+ display: flex;
10
+ align-items: center;
11
+ justify-content: center;
12
+ background: rgba(32, 36, 51, 0.28);
13
+ padding: 24px;
14
+ }
15
+
16
+ .cg-import {
17
+ display: flex;
18
+ flex-direction: column;
19
+ width: min(560px, 100%);
20
+ max-height: min(80vh, 720px);
21
+ background: var(--lp-bg);
22
+ border: 1px solid var(--lp-line);
23
+ border-radius: var(--lp-r-lg);
24
+ box-shadow: 0 12px 32px rgba(32, 36, 51, 0.18);
25
+ overflow: hidden;
26
+ }
27
+
28
+ .cg-import-head {
29
+ display: flex;
30
+ align-items: center;
31
+ justify-content: space-between;
32
+ gap: 12px;
33
+ padding: 14px 16px;
34
+ border-bottom: 1px solid var(--lp-line);
35
+ }
36
+
37
+ .cg-import-head h2 {
38
+ margin: 0;
39
+ font-size: var(--lp-fs-md);
40
+ font-weight: 600;
41
+ color: var(--lp-ink);
42
+ }
43
+
44
+ .cg-import-x {
45
+ border: 0;
46
+ background: none;
47
+ color: var(--lp-muted);
48
+ font-size: var(--lp-fs-lg);
49
+ line-height: 1;
50
+ cursor: pointer;
51
+ padding: 2px 6px;
52
+ }
53
+
54
+ .cg-import-x:hover {
55
+ color: var(--lp-ink);
56
+ }
57
+
58
+ .cg-import-body {
59
+ padding: 14px 16px;
60
+ overflow: auto;
61
+ }
62
+
63
+ .cg-import-lede {
64
+ margin: 0 0 8px;
65
+ font-size: var(--lp-fs-sm);
66
+ color: var(--lp-ink);
67
+ }
68
+
69
+ .cg-import-note {
70
+ margin: 0 0 12px;
71
+ font-size: var(--lp-fs-xs);
72
+ color: var(--lp-muted);
73
+ }
74
+
75
+ .cg-import-file {
76
+ font-size: var(--lp-fs-sm);
77
+ }
78
+
79
+ /* The mapping list: the spreadsheet's column on the left, this database's field on the right,
80
+ so the eye reads it as "this becomes that" rather than as a form. */
81
+ .cg-import-map {
82
+ display: flex;
83
+ flex-direction: column;
84
+ gap: 6px;
85
+ }
86
+
87
+ .cg-import-row {
88
+ display: grid;
89
+ grid-template-columns: 1fr 200px;
90
+ align-items: center;
91
+ gap: 10px;
92
+ }
93
+
94
+ .cg-import-col {
95
+ font-size: var(--lp-fs-sm);
96
+ color: var(--lp-ink);
97
+ overflow: hidden;
98
+ text-overflow: ellipsis;
99
+ white-space: nowrap;
100
+ }
101
+
102
+ .cg-import-row select {
103
+ font-family: inherit;
104
+ font-size: var(--lp-fs-xs);
105
+ padding: 4px 6px;
106
+ border: 1px solid var(--lp-line);
107
+ border-radius: var(--lp-r-sm);
108
+ background: var(--lp-bg);
109
+ color: var(--lp-ink);
110
+ }
111
+
112
+ /* A refusal is not an error state to be styled red-alert; it is a list of things to fix, so it
113
+ takes the yellow tint the rest of the product uses for "look at this before continuing". */
114
+ .cg-import-problems {
115
+ margin-top: 12px;
116
+ padding: 10px 12px;
117
+ border: 1px solid var(--lp-yellow);
118
+ border-radius: var(--lp-r-sm);
119
+ background: var(--lp-yellow-tint);
120
+ color: var(--lp-yellow-deep);
121
+ font-size: var(--lp-fs-xs);
122
+ }
123
+
124
+ .cg-import-problems ul {
125
+ margin: 6px 0;
126
+ padding-left: 18px;
127
+ }
128
+
129
+ .cg-import-problems p {
130
+ margin: 6px 0 0;
131
+ }
132
+
133
+ .cg-import-error {
134
+ margin: 0;
135
+ padding: 10px 16px;
136
+ background: var(--lp-red-tint);
137
+ color: var(--lp-red-deep);
138
+ font-size: var(--lp-fs-xs);
139
+ }
140
+
141
+ .cg-import-foot {
142
+ display: flex;
143
+ align-items: center;
144
+ justify-content: flex-end;
145
+ gap: 8px;
146
+ padding: 12px 16px;
147
+ border-top: 1px solid var(--lp-line);
148
+ }
149
+
150
+ .cg-import-count {
151
+ margin-right: auto;
152
+ font-size: var(--lp-fs-xs);
153
+ color: var(--lp-muted);
154
+ }
web/src/customer-grid/ImportDialog.tsx CHANGED
@@ -1,299 +1,299 @@
1
- // ---------------------------------------------------------------------------
2
- // customer-grid / ImportDialog.tsx
3
- //
4
- // ⭐⭐ WAVE-29 T25 (owner item 6) — IMPORT A SPREADSHEET INTO A USER DATABASE.
5
- //
6
- // Three steps in one window, because each answers the question the last one raised:
7
- // 1. pick a file -> the server parses it and says which columns it found
8
- // 2. map its columns -> to fields, or to nothing; unmapped is the default
9
- // 3. check + import -> every value coerced FIRST, the problems named BEFORE anything writes
10
- //
11
- // ⛔ APPEND-ONLY IN V1, AND THE DIALOG SAYS SO. Nothing is matched, nothing is overwritten — every
12
- // row becomes a new record. The only bulk-merge primitive in the product is upsert-BY-KEY shaped,
13
- // and choosing a key on the user's behalf is how an import quietly rewrites rows somebody edited.
14
- // The sentence is on screen rather than in a doc, because that is where the decision is made.
15
- //
16
- // ⛔ REUSE, NOT A SECOND COERCER. `clipboard.coerceClipboardValue` is THE client answer to "can
17
- // this text be this field's value" (it is what a paste already goes through), and
18
- // `POST /api/v1/uploads/tabular` is the parser that already exists — 5 MB / 20,000 rows, xlsx AND
19
- // csv, and it never writes a row. A second copy of either is a second set of rules for one
20
- // question ([[one-evaluator-per-question]]).
21
- //
22
- // ⚠ THE SERVER IS THE WALL, not this dialog. It refuses computed columns BY NAME and refuses the
23
- // whole batch if it would pass the row cap. What happens here is the part a person needs before
24
- // they press the button: which column goes where, and what will not survive the trip.
25
- // ---------------------------------------------------------------------------
26
-
27
- import { useMemo, useRef, useState } from "react";
28
- // ⚠ From `apiContract`, which is where these two are DECLARED — not re-exported from
29
- // `apiBridge`, which does not export them and is another session's fence this wave.
30
- import { API_V1, CREDENTIALS } from "../apiContract";
31
- import { isImportTarget } from "./types";
32
- import type { Field } from "./types";
33
- // ⭐ W29-T76 — the plan lives in a pure module so a test can EXECUTE it against the server's real
34
- // payload. This file holds the WINDOW; `importPlan.ts` holds what would be written.
35
- import { buildImportPlan } from "./importPlan";
36
- import type { ImportMapping, ParsedUpload } from "./importPlan";
37
- import "./ImportDialog.css";
38
-
39
- /** The one header a JSON POST needs. `apiBridge` keeps its own module-local copy; this is a
40
- * literal rather than an import because that constant is not exported and that file is not
41
- * this session's to change. */
42
- const JSON_HEADERS = { "Content-Type": "application/json" };
43
-
44
- /** The upload door's answer (`routes_uploads.tabular`) — declared in `importPlan.ts`, beside the
45
- * only code that reads its shape. ⛔ It is COLUMN-MAJOR; the copy that used to live here said
46
- * `string[][]` and that one wrong word is why nothing ever imported. */
47
- type Parsed = ParsedUpload;
48
-
49
- export interface ImportDialogProps {
50
- /** The target database's fields — the mapping targets, before this file filters them. */
51
- fields: Field[];
52
- /** `ut_*` table key; the import door is per-table. */
53
- scope: string;
54
- /** How the database calls itself, for the sentence that says where rows are going. */
55
- tableName: string;
56
- /** Fired after a successful import so the host can re-read its rows. */
57
- onImported: (count: number) => void;
58
- onClose: () => void;
59
- }
60
-
61
- const NONE = "";
62
-
63
- export default function ImportDialog({
64
- fields,
65
- scope,
66
- tableName,
67
- onImported,
68
- onClose,
69
- }: ImportDialogProps): React.ReactElement {
70
- const [parsed, setParsed] = useState<Parsed | null>(null);
71
- const [mapping, setMapping] = useState<ImportMapping>({});
72
- const [busy, setBusy] = useState(false);
73
- const [error, setError] = useState("");
74
- const fileRef = useRef<HTMLInputElement | null>(null);
75
-
76
- const targets = useMemo(() => fields.filter(isImportTarget), [fields]);
77
-
78
- /**
79
- * The parse. `FormData` and NO `Content-Type` header: the browser has to set the multipart
80
- * boundary itself, and spelling the header by hand is the classic way to make a working upload
81
- * 400 with a message about a missing part.
82
- */
83
- async function pick(file: File): Promise<void> {
84
- setBusy(true);
85
- setError("");
86
- try {
87
- const form = new FormData();
88
- form.append("file", file);
89
- const res = await fetch(`${API_V1}/uploads/tabular`, {
90
- method: "POST",
91
- credentials: CREDENTIALS,
92
- body: form,
93
- });
94
- const body = (await res.json().catch(() => null)) as
95
- | (Parsed & { error?: { message?: string } })
96
- | null;
97
- if (!res.ok || !body) {
98
- setError(
99
- body?.error?.message ??
100
- `That file could not be read (the server answered ${res.status}).`
101
- );
102
- return;
103
- }
104
- setParsed(body);
105
- // ⭐ AUTO-MAP BY NAME, case- and space-insensitively, because the overwhelmingly common
106
- // case is a sheet exported from this very grid. Every guess is VISIBLE in a picker the
107
- // user can change — a silent auto-map is only dangerous when it is also invisible.
108
- const byName = new Map(
109
- targets.map((f) => [f.label.trim().toLowerCase(), f.key] as const)
110
- );
111
- const guess: ImportMapping = {};
112
- (body.columns ?? []).forEach((name, i) => {
113
- const hit = byName.get(String(name).trim().toLowerCase());
114
- if (hit) guess[i] = hit;
115
- });
116
- setMapping(guess);
117
- } catch {
118
- setError("That file could not be read.");
119
- } finally {
120
- setBusy(false);
121
- }
122
- }
123
-
124
- /**
125
- * What would be written, and what cannot be. The work is `importPlan.buildImportPlan` — a pure
126
- * module, so the node suite executes it against the server's real payload rather than grepping
127
- * this file for the shape of a loop.
128
- */
129
- const plan = useMemo(
130
- () => buildImportPlan(parsed, mapping, targets),
131
- [parsed, mapping, targets]
132
- );
133
-
134
- async function run(): Promise<void> {
135
- setBusy(true);
136
- setError("");
137
- try {
138
- const res = await fetch(
139
- `${API_V1}/tables/${encodeURIComponent(scope)}/rows/import`,
140
- {
141
- method: "POST",
142
- credentials: CREDENTIALS,
143
- headers: JSON_HEADERS,
144
- body: JSON.stringify({ rows: plan.rows }),
145
- }
146
- );
147
- const body = (await res.json().catch(() => null)) as
148
- | { imported?: number; error?: { message?: string } }
149
- | null;
150
- if (!res.ok) {
151
- // The server's sentence beats anything invented here: it names the column, or the cap.
152
- setError(body?.error?.message ?? `The server answered ${res.status}.`);
153
- return;
154
- }
155
- onImported(body?.imported ?? plan.rows.length);
156
- onClose();
157
- } catch {
158
- setError("Nothing was imported — the request did not complete.");
159
- } finally {
160
- setBusy(false);
161
- }
162
- }
163
-
164
- const mappedCount = Object.values(mapping).filter((k) => k !== NONE).length;
165
-
166
- return (
167
- <div className="cg-import-scrim" role="presentation" onClick={onClose}>
168
- <div
169
- className="cg-import"
170
- role="dialog"
171
- aria-modal="true"
172
- aria-label={`Import records into ${tableName}`}
173
- onClick={(event) => event.stopPropagation()}
174
- >
175
- <div className="cg-import-head">
176
- <h2>Import records</h2>
177
- <button type="button" className="cg-import-x" aria-label="Close" onClick={onClose}>
178
- ×
179
- </button>
180
- </div>
181
-
182
- {!parsed ? (
183
- <div className="cg-import-body">
184
- <p className="cg-import-lede">
185
- Add records to <strong>{tableName}</strong> from a spreadsheet. Excel
186
- (<code>.xlsx</code>) or <code>.csv</code>, up to 5&nbsp;MB.
187
- </p>
188
- <p className="cg-import-note">
189
- Every row becomes a <strong>new record</strong>. Nothing already in this database is
190
- matched or overwritten.
191
- </p>
192
- <input
193
- ref={fileRef}
194
- type="file"
195
- accept=".xlsx,.csv"
196
- className="cg-import-file"
197
- onChange={(event) => {
198
- const file = event.currentTarget.files?.[0];
199
- if (file) void pick(file);
200
- }}
201
- />
202
- </div>
203
- ) : (
204
- <div className="cg-import-body">
205
- <p className="cg-import-lede">
206
- {parsed.rows.toLocaleString()} row{parsed.rows === 1 ? "" : "s"} read
207
- {parsed.truncated ? " (the first 20,000 — the rest were not read)" : ""}. Choose
208
- where each column goes.
209
- </p>
210
- <div className="cg-import-map" role="list">
211
- {parsed.columns.map((name, i) => (
212
- <label className="cg-import-row" role="listitem" key={`${name}:${i}`}>
213
- <span className="cg-import-col" title={name}>
214
- {name || `Column ${i + 1}`}
215
- </span>
216
- <select
217
- value={mapping[i] ?? NONE}
218
- onChange={(event) => {
219
- const next = event.target.value;
220
- setMapping((cur) => ({ ...cur, [i]: next }));
221
- }}
222
- >
223
- <option value={NONE}>Don&rsquo;t import</option>
224
- {targets.map((f) => (
225
- <option key={f.key} value={f.key}>
226
- {f.label}
227
- </option>
228
- ))}
229
- </select>
230
- </label>
231
- ))}
232
- </div>
233
- {plan.problems.length > 0 && (
234
- <div className="cg-import-problems" role="alert">
235
- <strong>
236
- {plan.problems.length.toLocaleString()} value
237
- {plan.problems.length === 1 ? "" : "s"} cannot go in the column
238
- {plan.problems.length === 1 ? "" : "s"} chosen for them
239
- </strong>
240
- <ul>
241
- {plan.problems.slice(0, 5).map((p, i) => (
242
- <li key={i}>
243
- Row {p.row}, {p.column}: {p.value ? `“${p.value}”` : "(blank)"} is not a
244
- valid {p.field}
245
- </li>
246
- ))}
247
- </ul>
248
- {plan.problems.length > 5 && (
249
- <p>…and {(plan.problems.length - 5).toLocaleString()} more.</p>
250
- )}
251
- <p>
252
- Fix them in the file, or point that column somewhere else — nothing has been
253
- imported yet.
254
- </p>
255
- </div>
256
- )}
257
- </div>
258
- )}
259
-
260
- {error && (
261
- <p className="cg-import-error" role="alert">
262
- {error}
263
- </p>
264
- )}
265
-
266
- <div className="cg-import-foot">
267
- {parsed && (
268
- <span className="cg-import-count">
269
- {mappedCount === 0
270
- ? "No columns chosen yet"
271
- : `${plan.rows.length.toLocaleString()} record${
272
- plan.rows.length === 1 ? "" : "s"
273
- } from ${mappedCount} column${mappedCount === 1 ? "" : "s"}`}
274
- </span>
275
- )}
276
- <button type="button" className="cg-btn" onClick={onClose}>
277
- Cancel
278
- </button>
279
- {parsed && (
280
- <button
281
- type="button"
282
- className="cg-btn cg-btn--primary"
283
- /* ⛔ DISABLED WHILE ANY VALUE IS REFUSED. The alternative — import the good rows and
284
- report the rest — leaves somebody reconciling a spreadsheet against a table by
285
- hand, and neither existing write door demonstrates a partial policy. Refusing
286
- whole is recoverable in one edit; a half-import is not. */
287
- disabled={busy || plan.rows.length === 0 || plan.problems.length > 0}
288
- onClick={() => void run()}
289
- >
290
- {busy ? "Importing…" : `Import ${plan.rows.length.toLocaleString()} record${
291
- plan.rows.length === 1 ? "" : "s"
292
- }`}
293
- </button>
294
- )}
295
- </div>
296
- </div>
297
- </div>
298
- );
299
- }
 
1
+ // ---------------------------------------------------------------------------
2
+ // customer-grid / ImportDialog.tsx
3
+ //
4
+ // ⭐⭐ WAVE-29 T25 (owner item 6) — IMPORT A SPREADSHEET INTO A USER DATABASE.
5
+ //
6
+ // Three steps in one window, because each answers the question the last one raised:
7
+ // 1. pick a file -> the server parses it and says which columns it found
8
+ // 2. map its columns -> to fields, or to nothing; unmapped is the default
9
+ // 3. check + import -> every value coerced FIRST, the problems named BEFORE anything writes
10
+ //
11
+ // ⛔ APPEND-ONLY IN V1, AND THE DIALOG SAYS SO. Nothing is matched, nothing is overwritten — every
12
+ // row becomes a new record. The only bulk-merge primitive in the product is upsert-BY-KEY shaped,
13
+ // and choosing a key on the user's behalf is how an import quietly rewrites rows somebody edited.
14
+ // The sentence is on screen rather than in a doc, because that is where the decision is made.
15
+ //
16
+ // ⛔ REUSE, NOT A SECOND COERCER. `clipboard.coerceClipboardValue` is THE client answer to "can
17
+ // this text be this field's value" (it is what a paste already goes through), and
18
+ // `POST /api/v1/uploads/tabular` is the parser that already exists — 5 MB / 20,000 rows, xlsx AND
19
+ // csv, and it never writes a row. A second copy of either is a second set of rules for one
20
+ // question ([[one-evaluator-per-question]]).
21
+ //
22
+ // ⚠ THE SERVER IS THE WALL, not this dialog. It refuses computed columns BY NAME and refuses the
23
+ // whole batch if it would pass the row cap. What happens here is the part a person needs before
24
+ // they press the button: which column goes where, and what will not survive the trip.
25
+ // ---------------------------------------------------------------------------
26
+
27
+ import { useMemo, useRef, useState } from "react";
28
+ // ⚠ From `apiContract`, which is where these two are DECLARED — not re-exported from
29
+ // `apiBridge`, which does not export them and is another session's fence this wave.
30
+ import { API_V1, CREDENTIALS } from "../apiContract";
31
+ import { isImportTarget } from "./types";
32
+ import type { Field } from "./types";
33
+ // ⭐ W29-T76 — the plan lives in a pure module so a test can EXECUTE it against the server's real
34
+ // payload. This file holds the WINDOW; `importPlan.ts` holds what would be written.
35
+ import { buildImportPlan } from "./importPlan";
36
+ import type { ImportMapping, ParsedUpload } from "./importPlan";
37
+ import "./ImportDialog.css";
38
+
39
+ /** The one header a JSON POST needs. `apiBridge` keeps its own module-local copy; this is a
40
+ * literal rather than an import because that constant is not exported and that file is not
41
+ * this session's to change. */
42
+ const JSON_HEADERS = { "Content-Type": "application/json" };
43
+
44
+ /** The upload door's answer (`routes_uploads.tabular`) — declared in `importPlan.ts`, beside the
45
+ * only code that reads its shape. ⛔ It is COLUMN-MAJOR; the copy that used to live here said
46
+ * `string[][]` and that one wrong word is why nothing ever imported. */
47
+ type Parsed = ParsedUpload;
48
+
49
+ export interface ImportDialogProps {
50
+ /** The target database's fields — the mapping targets, before this file filters them. */
51
+ fields: Field[];
52
+ /** `ut_*` table key; the import door is per-table. */
53
+ scope: string;
54
+ /** How the database calls itself, for the sentence that says where rows are going. */
55
+ tableName: string;
56
+ /** Fired after a successful import so the host can re-read its rows. */
57
+ onImported: (count: number) => void;
58
+ onClose: () => void;
59
+ }
60
+
61
+ const NONE = "";
62
+
63
+ export default function ImportDialog({
64
+ fields,
65
+ scope,
66
+ tableName,
67
+ onImported,
68
+ onClose,
69
+ }: ImportDialogProps): React.ReactElement {
70
+ const [parsed, setParsed] = useState<Parsed | null>(null);
71
+ const [mapping, setMapping] = useState<ImportMapping>({});
72
+ const [busy, setBusy] = useState(false);
73
+ const [error, setError] = useState("");
74
+ const fileRef = useRef<HTMLInputElement | null>(null);
75
+
76
+ const targets = useMemo(() => fields.filter(isImportTarget), [fields]);
77
+
78
+ /**
79
+ * The parse. `FormData` and NO `Content-Type` header: the browser has to set the multipart
80
+ * boundary itself, and spelling the header by hand is the classic way to make a working upload
81
+ * 400 with a message about a missing part.
82
+ */
83
+ async function pick(file: File): Promise<void> {
84
+ setBusy(true);
85
+ setError("");
86
+ try {
87
+ const form = new FormData();
88
+ form.append("file", file);
89
+ const res = await fetch(`${API_V1}/uploads/tabular`, {
90
+ method: "POST",
91
+ credentials: CREDENTIALS,
92
+ body: form,
93
+ });
94
+ const body = (await res.json().catch(() => null)) as
95
+ | (Parsed & { error?: { message?: string } })
96
+ | null;
97
+ if (!res.ok || !body) {
98
+ setError(
99
+ body?.error?.message ??
100
+ `That file could not be read (the server answered ${res.status}).`
101
+ );
102
+ return;
103
+ }
104
+ setParsed(body);
105
+ // ⭐ AUTO-MAP BY NAME, case- and space-insensitively, because the overwhelmingly common
106
+ // case is a sheet exported from this very grid. Every guess is VISIBLE in a picker the
107
+ // user can change — a silent auto-map is only dangerous when it is also invisible.
108
+ const byName = new Map(
109
+ targets.map((f) => [f.label.trim().toLowerCase(), f.key] as const)
110
+ );
111
+ const guess: ImportMapping = {};
112
+ (body.columns ?? []).forEach((name, i) => {
113
+ const hit = byName.get(String(name).trim().toLowerCase());
114
+ if (hit) guess[i] = hit;
115
+ });
116
+ setMapping(guess);
117
+ } catch {
118
+ setError("That file could not be read.");
119
+ } finally {
120
+ setBusy(false);
121
+ }
122
+ }
123
+
124
+ /**
125
+ * What would be written, and what cannot be. The work is `importPlan.buildImportPlan` — a pure
126
+ * module, so the node suite executes it against the server's real payload rather than grepping
127
+ * this file for the shape of a loop.
128
+ */
129
+ const plan = useMemo(
130
+ () => buildImportPlan(parsed, mapping, targets),
131
+ [parsed, mapping, targets]
132
+ );
133
+
134
+ async function run(): Promise<void> {
135
+ setBusy(true);
136
+ setError("");
137
+ try {
138
+ const res = await fetch(
139
+ `${API_V1}/tables/${encodeURIComponent(scope)}/rows/import`,
140
+ {
141
+ method: "POST",
142
+ credentials: CREDENTIALS,
143
+ headers: JSON_HEADERS,
144
+ body: JSON.stringify({ rows: plan.rows }),
145
+ }
146
+ );
147
+ const body = (await res.json().catch(() => null)) as
148
+ | { imported?: number; error?: { message?: string } }
149
+ | null;
150
+ if (!res.ok) {
151
+ // The server's sentence beats anything invented here: it names the column, or the cap.
152
+ setError(body?.error?.message ?? `The server answered ${res.status}.`);
153
+ return;
154
+ }
155
+ onImported(body?.imported ?? plan.rows.length);
156
+ onClose();
157
+ } catch {
158
+ setError("Nothing was imported — the request did not complete.");
159
+ } finally {
160
+ setBusy(false);
161
+ }
162
+ }
163
+
164
+ const mappedCount = Object.values(mapping).filter((k) => k !== NONE).length;
165
+
166
+ return (
167
+ <div className="cg-import-scrim" role="presentation" onClick={onClose}>
168
+ <div
169
+ className="cg-import"
170
+ role="dialog"
171
+ aria-modal="true"
172
+ aria-label={`Import records into ${tableName}`}
173
+ onClick={(event) => event.stopPropagation()}
174
+ >
175
+ <div className="cg-import-head">
176
+ <h2>Import records</h2>
177
+ <button type="button" className="cg-import-x" aria-label="Close" onClick={onClose}>
178
+ ×
179
+ </button>
180
+ </div>
181
+
182
+ {!parsed ? (
183
+ <div className="cg-import-body">
184
+ <p className="cg-import-lede">
185
+ Add records to <strong>{tableName}</strong> from a spreadsheet. Excel
186
+ (<code>.xlsx</code>) or <code>.csv</code>, up to 5&nbsp;MB.
187
+ </p>
188
+ <p className="cg-import-note">
189
+ Every row becomes a <strong>new record</strong>. Nothing already in this database is
190
+ matched or overwritten.
191
+ </p>
192
+ <input
193
+ ref={fileRef}
194
+ type="file"
195
+ accept=".xlsx,.csv"
196
+ className="cg-import-file"
197
+ onChange={(event) => {
198
+ const file = event.currentTarget.files?.[0];
199
+ if (file) void pick(file);
200
+ }}
201
+ />
202
+ </div>
203
+ ) : (
204
+ <div className="cg-import-body">
205
+ <p className="cg-import-lede">
206
+ {parsed.rows.toLocaleString()} row{parsed.rows === 1 ? "" : "s"} read
207
+ {parsed.truncated ? " (the first 20,000 — the rest were not read)" : ""}. Choose
208
+ where each column goes.
209
+ </p>
210
+ <div className="cg-import-map" role="list">
211
+ {parsed.columns.map((name, i) => (
212
+ <label className="cg-import-row" role="listitem" key={`${name}:${i}`}>
213
+ <span className="cg-import-col" title={name}>
214
+ {name || `Column ${i + 1}`}
215
+ </span>
216
+ <select
217
+ value={mapping[i] ?? NONE}
218
+ onChange={(event) => {
219
+ const next = event.target.value;
220
+ setMapping((cur) => ({ ...cur, [i]: next }));
221
+ }}
222
+ >
223
+ <option value={NONE}>Don&rsquo;t import</option>
224
+ {targets.map((f) => (
225
+ <option key={f.key} value={f.key}>
226
+ {f.label}
227
+ </option>
228
+ ))}
229
+ </select>
230
+ </label>
231
+ ))}
232
+ </div>
233
+ {plan.problems.length > 0 && (
234
+ <div className="cg-import-problems" role="alert">
235
+ <strong>
236
+ {plan.problems.length.toLocaleString()} value
237
+ {plan.problems.length === 1 ? "" : "s"} cannot go in the column
238
+ {plan.problems.length === 1 ? "" : "s"} chosen for them
239
+ </strong>
240
+ <ul>
241
+ {plan.problems.slice(0, 5).map((p, i) => (
242
+ <li key={i}>
243
+ Row {p.row}, {p.column}: {p.value ? `“${p.value}”` : "(blank)"} is not a
244
+ valid {p.field}
245
+ </li>
246
+ ))}
247
+ </ul>
248
+ {plan.problems.length > 5 && (
249
+ <p>…and {(plan.problems.length - 5).toLocaleString()} more.</p>
250
+ )}
251
+ <p>
252
+ Fix them in the file, or point that column somewhere else — nothing has been
253
+ imported yet.
254
+ </p>
255
+ </div>
256
+ )}
257
+ </div>
258
+ )}
259
+
260
+ {error && (
261
+ <p className="cg-import-error" role="alert">
262
+ {error}
263
+ </p>
264
+ )}
265
+
266
+ <div className="cg-import-foot">
267
+ {parsed && (
268
+ <span className="cg-import-count">
269
+ {mappedCount === 0
270
+ ? "No columns chosen yet"
271
+ : `${plan.rows.length.toLocaleString()} record${
272
+ plan.rows.length === 1 ? "" : "s"
273
+ } from ${mappedCount} column${mappedCount === 1 ? "" : "s"}`}
274
+ </span>
275
+ )}
276
+ <button type="button" className="cg-btn" onClick={onClose}>
277
+ Cancel
278
+ </button>
279
+ {parsed && (
280
+ <button
281
+ type="button"
282
+ className="cg-btn cg-btn--primary"
283
+ /* ⛔ DISABLED WHILE ANY VALUE IS REFUSED. The alternative — import the good rows and
284
+ report the rest — leaves somebody reconciling a spreadsheet against a table by
285
+ hand, and neither existing write door demonstrates a partial policy. Refusing
286
+ whole is recoverable in one edit; a half-import is not. */
287
+ disabled={busy || plan.rows.length === 0 || plan.problems.length > 0}
288
+ onClick={() => void run()}
289
+ >
290
+ {busy ? "Importing…" : `Import ${plan.rows.length.toLocaleString()} record${
291
+ plan.rows.length === 1 ? "" : "s"
292
+ }`}
293
+ </button>
294
+ )}
295
+ </div>
296
+ </div>
297
+ </div>
298
+ );
299
+ }