fsanyoto commited on
Commit
75b4008
Β·
verified Β·
1 Parent(s): e78e601

Deploy AIOS web (React glide grid + FastAPI slice)

Browse files
RELEASES.json CHANGED
@@ -1,6 +1,12 @@
1
  {
2
- "current": "v21 (5fdffce)",
3
  "releases": [
 
 
 
 
 
 
4
  {
5
  "version": "v21",
6
  "sha": "5fdffce",
 
1
  {
2
+ "current": "v22 (f14e199)",
3
  "releases": [
4
+ {
5
+ "version": "v22",
6
+ "sha": "f14e199",
7
+ "date": "2026-08-09",
8
+ "subject": "release v22"
9
+ },
10
  {
11
  "version": "v21",
12
  "sha": "5fdffce",
VERSION CHANGED
@@ -1 +1 @@
1
- v21 (5fdffce)
 
1
+ v22 (f14e199)
api/automation_engine.py CHANGED
@@ -5218,7 +5218,18 @@ def run_plain(rt, defn, username="automation", log=print, step=_no_step, rows=No
5218
  enrich_cfg = enabled_direct[0].get("config") or {}
5219
  profile_key = profile_field_key(ut_get(rt, table), enrich_cfg.get("profileField"))
5220
  if profile_key:
5221
- selected, enrich_scope_note = enrich_selection(rt, table, enrich_cfg, profile_key)
 
 
 
 
 
 
 
 
 
 
 
5222
  if enrich_scope_note.startswith("the enrich step names a view it cannot read"):
5223
  return ("partial", f"nothing walked in {table} β€” {enrich_scope_note}", {}, [], {})
5224
  ids = selected
@@ -5251,6 +5262,14 @@ def run_plain(rt, defn, username="automation", log=print, step=_no_step, rows=No
5251
  else "every record β€” this run was started by hand, so nothing narrowed it"))
5252
  if enrich_scope_note:
5253
  scope += f" ({enrich_scope_note})"
 
 
 
 
 
 
 
 
5254
  return ("ok", f"{n} record{'' if n == 1 else 's'} walked in {label}: {scope}",
5255
  counts, ids, {})
5256
 
@@ -9148,14 +9167,17 @@ def run_now(rt, tenant, auto_id, username="automation", log=print, rows=None):
9148
  # ⚠ BOUND BEFORE THE `try`. The `except` below falls through to the same `_commit_run`,
9149
  # which now reads this name β€” an assignment only on the success path would turn any
9150
  # action failure into a NameError inside the handler that exists to prevent exactly that.
9151
- run_notes = []
 
 
 
9152
  try:
9153
  a_counts = apply_actions(rt, defn, _flow_table(defn), affected or [],
9154
  username=username, log=log)
9155
  # ⭐⭐ D-103 β€” POPPED BEFORE THE MERGE. The per-record reasons ride inside `counts` so
9156
  # the runner contract keeps its shape, and they must leave before the merge or they
9157
  # would be a "count" everywhere downstream.
9158
- run_notes = list(a_counts.pop(RUN_NOTES_KEY, None) or [])
9159
  counts = {**(counts or {}), **{k: v for k, v in a_counts.items() if v}}
9160
  if a_counts.get("enrichMetricBatchesPending"):
9161
  batches = int(a_counts["enrichMetricBatchesPending"])
 
5218
  enrich_cfg = enabled_direct[0].get("config") or {}
5219
  profile_key = profile_field_key(ut_get(rt, table), enrich_cfg.get("profileField"))
5220
  if profile_key:
5221
+ # β›”β›” THE SAME QUESTION MUST GET THE SAME ANSWER IN BOTH PLACES.
5222
+ # `enrich_selection` is called TWICE per run β€” here, to decide which records the
5223
+ # runner walks and announces, and again inside `_walk` to decide which the action
5224
+ # SPENDS on. MEASURED LIVE 2026-08-09: this call omitted the not-found verdicts, so
5225
+ # the run announced *"(1 of the 30 asked for)"* in its summary while its own note
5226
+ # from the second call said *"0 of the 30"* β€” one run, two answers, both printed.
5227
+ # ⚠ `gone` is read from the definition here rather than passed in, because this
5228
+ # function has the definition and `_walk` does the same read; a third source of the
5229
+ # same fact is how the two would drift again ([[one-evaluator-per-question]]).
5230
+ selected, enrich_scope_note = enrich_selection(
5231
+ rt, table, enrich_cfg, profile_key,
5232
+ gone=dict(((defn or {}).get("state") or {}).get("enrichNotFound") or {}))
5233
  if enrich_scope_note.startswith("the enrich step names a view it cannot read"):
5234
  return ("partial", f"nothing walked in {table} β€” {enrich_scope_note}", {}, [], {})
5235
  ids = selected
 
5262
  else "every record β€” this run was started by hand, so nothing narrowed it"))
5263
  if enrich_scope_note:
5264
  scope += f" ({enrich_scope_note})"
5265
+ # ⭐⭐ AND ON THE RUN, not only inside the summary sentence. `summary` is capped at 400
5266
+ # characters by `_commit_run`, and this note is the longest thing a run says β€” it names
5267
+ # every skipped handle and what to do about them. MEASURED: a run that selects NOTHING
5268
+ # (every candidate is a known-dead handle) never enters `_walk`, so the walk's own copy
5269
+ # of this sentence is never produced. Without this line the explanation exists only in a
5270
+ # string that is about to be truncated, on exactly the runs that look like the automation
5271
+ # has stopped working.
5272
+ counts[RUN_NOTES_KEY] = [enrich_scope_note]
5273
  return ("ok", f"{n} record{'' if n == 1 else 's'} walked in {label}: {scope}",
5274
  counts, ids, {})
5275
 
 
9167
  # ⚠ BOUND BEFORE THE `try`. The `except` below falls through to the same `_commit_run`,
9168
  # which now reads this name β€” an assignment only on the success path would turn any
9169
  # action failure into a NameError inside the handler that exists to prevent exactly that.
9170
+ # ⚠ THE RUNNER PRODUCES NOTES TOO, and its are the ones that survive a run which walked
9171
+ # NOTHING β€” the branch where every candidate was a known-dead handle, i.e. exactly the
9172
+ # run a person stares at wondering why the automation stopped doing anything.
9173
+ run_notes = list((counts or {}).pop(RUN_NOTES_KEY, None) or [])
9174
  try:
9175
  a_counts = apply_actions(rt, defn, _flow_table(defn), affected or [],
9176
  username=username, log=log)
9177
  # ⭐⭐ D-103 β€” POPPED BEFORE THE MERGE. The per-record reasons ride inside `counts` so
9178
  # the runner contract keeps its shape, and they must leave before the merge or they
9179
  # would be a "count" everywhere downstream.
9180
+ run_notes += list(a_counts.pop(RUN_NOTES_KEY, None) or [])
9181
  counts = {**(counts or {}), **{k: v for k, v in a_counts.items() if v}}
9182
  if a_counts.get("enrichMetricBatchesPending"):
9183
  batches = int(a_counts["enrichMetricBatchesPending"])
api/odoo_relational.py CHANGED
@@ -1,57 +1,76 @@
1
- """odoo_relational.py β€” Odoo entities as LOCKED relational databases (wave 27 item 17, PHASE 1).
2
 
3
  Owner ruling R1 / contract C8: spawn preset Odoo databases for Royal Imports, give them preset
4
  Link + Rollup fields, and prove each rollup against the `measure_` column it will eventually
5
- replace. `measure_` columns are NOT touched this wave.
6
-
7
- β›” WHAT C8 ASKED FOR AND WHY TWO THIRDS OF IT IS NOT HERE (AMENDMENT E1, measured 2026-08-08).
8
- C8 named three tables β€” orders, order lines, invoices. `core.user_tables.MAX_ROWS` is **5000**, and
9
- the Royal mirror holds **32,700** confirmed orders, **254,189** order lines and **31,418** posted
10
- customer invoices. Order lines are **50.8x** the cap and still **3.3x** over it at a 90-day window,
11
- so no window rescues that table. Every `ut_*` table also lives in ONE JSON blob under one store key
12
- that is read whole per render, so this is not a limit to raise β€” 318k rows there is a different
13
- store, not a bigger number. C8's other half (Link/Rollup fields hosted on the Customer/Product
14
- grids) is dead twice over: `aios_grid.fields_from_workspace` emits no `link`/`rollup` bag for a
15
- `custom_` field, and `compute_relation_cells` reads `ut_all(rt)` exclusively while those grids have
16
- no stored rows at all. Phase 2 therefore needs a READ-THROUGH binding over the DuckDB store the
17
- rows already live in β€” not a copy. That is booked as debt, not attempted here.
18
-
19
- ⭐ SO WHAT SHIPS IS THE POPULATION THAT FITS *AND* IS COMPLETE FOR ITS QUESTION: every OPEN customer
20
- receivable (**1,228 rows / 438 partners / $591,828.54**, measured), two locked databases joined by a
21
- derived link,
22
- carrying the rollups that answer "what does this customer owe". A slice that is complete for a
23
- question is a feature; a truncated copy of a bigger table is a number that is quietly wrong, which
24
- is the one thing this repo refuses everywhere else. **This module REFUSES rather than truncates.**
25
-
26
- THE POPULATION IS THE AR ORACLE'S, NOT ONE OF ITS OWN. `modules/ar._open_docs` selects
27
- `state='posted'` AND `move_type in (out_invoice,out_refund)` AND
28
- `payment_state in (not_paid,partial)`, minus `core.odoo.excluded_partner_ids()` (the GIFTWARE /
29
- Amazon channel, out of wholesale scope). This module uses THAT predicate β€” not an equivalent one.
30
- ⚠ Measured 2026-08-08: `payment_state in (not_paid,partial)` and `residual <> 0` return the
31
- identical 1,238 rows / $2,347,608.49 today. They still are not the same QUESTION, and copying the
32
- oracle's definition is what keeps them from drifting apart later.
33
-
34
- ⚠⚠ AND THE EXCLUSION IS NOT A ROUNDING DETAIL: **GIFTWARE DEALS alone carries $1,755,779.95 of the
35
- $2,347,608.49 raw open balance β€” 75% of it β€” across just 10 invoices.** So wholesale open AR is
36
- $591,828.54, and anyone comparing this table against a raw `account_move` sum will see a 4x gap and
37
- reasonably conclude something is broken. It is the Amazon channel being out of wholesale scope,
38
- exactly as `modules/ar` and the Customer grid already have it.
 
 
 
 
39
  """
40
  import datetime as _dt
41
 
42
  #: Royal Imports only (R1). A tenant slug that is not this one gets a refusal, never a spawn:
43
- #: nurilab has no Odoo mirror behind these tables and would get two empty locked databases.
44
  RI_SLUGS = ("", "royal-imports")
45
 
46
  INVOICES_KEY = "ut_odoo_invoices"
47
  CUSTOMERS_KEY = "ut_odoo_customers"
 
 
48
 
49
- #: The join column both tables carry. The derived link resolves through it (`on`/`from`).
50
  JOIN_KEY = "partner_id"
 
 
51
 
52
- #: The oracle's own predicate β€” see the module docstring.
53
- _AR_WHERE = ("state = 'posted' AND move_type IN ('out_invoice','out_refund') "
54
- "AND payment_state IN ('not_paid','partial')")
 
 
 
 
 
 
 
 
 
 
 
55
 
56
 
57
  def _ut():
@@ -71,15 +90,39 @@ def _preset(field, flow="odoo_relational"):
71
  return field
72
 
73
 
 
 
 
 
 
 
 
 
74
  # ---------------------------------------------------------------------------------------------
75
  # FIELD CONTRACTS
76
  # ---------------------------------------------------------------------------------------------
77
  # ⚠ Every type here must be in `core.user_tables.UT_FIELD_TYPES`, and `_clean_field` returns None
78
  # for an unknown one β€” which DELETES the column silently on the next read rather than erroring.
 
 
 
 
 
 
 
 
 
 
 
 
 
79
  def invoice_fields():
 
80
  return [_preset(f) for f in (
81
  {"key": "invoice_no", "label": "Invoice", "type": "text", "source": "overlay",
82
  "default": True, "pinned": True},
 
 
83
  {"key": "customer", "label": "Customer", "type": "text", "source": "overlay",
84
  "default": True},
85
  {"key": JOIN_KEY, "label": "Odoo partner id", "type": "int", "source": "overlay",
@@ -89,77 +132,203 @@ def invoice_fields():
89
  {"key": "due_date", "label": "Due date", "type": "date", "source": "overlay",
90
  "default": True},
91
  {"key": "residual", "label": "Outstanding $", "type": "currency", "source": "overlay",
92
- "default": True, "agg": "sum"},
 
 
93
  {"key": "amount_untaxed", "label": "Invoiced $", "type": "currency", "source": "overlay",
94
- "default": False, "agg": "sum"},
 
 
 
 
95
  {"key": "payment_state", "label": "Payment state", "type": "select", "source": "overlay",
96
- "default": True, "options": ["not_paid", "partial"]},
 
97
  {"key": "move_type", "label": "Document", "type": "select", "source": "overlay",
98
  "default": False, "options": ["out_invoice", "out_refund"]},
99
- # ⭐ THE RECIPROCAL HALF (owner item 2, 2026-08-09: *"unique ID is still not fully mapped
100
- # into the new locked database"*). The pair was one-directional: customers carried an
101
- # `invoices` link and invoices carried the join VALUE (`partner_id`) with no link back,
102
- # so you could traverse customer -> invoices and never invoice -> customer.
103
- #
104
- # ⚠ DERIVED (`on` declared), exactly like its twin, so the engine owns the cell and no
105
- # human can edit a relation Odoo already decided. It resolves through the SAME join
106
- # column in the other direction β€” one shared row truth, two doors onto it.
107
  {"key": "customer_link", "label": "Customer record", "type": "link", "source": "overlay",
108
  "default": False,
109
  "link": {"table": CUSTOMERS_KEY, "on": JOIN_KEY, "from": JOIN_KEY}},
110
- {"key": "refreshed", "label": "Refreshed", "type": "date", "source": "overlay",
 
 
 
 
 
 
 
 
 
 
 
 
 
111
  "default": False},
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
112
  )]
113
 
114
 
115
  def customer_fields():
116
- """⭐ The LINK is DERIVED (`on` declared), so the engine owns the cell and a human cannot edit
117
- a relation that Odoo already decided. The three rollups are C8's parity targets that this
118
- substrate can actually host; the other two (order count, fully-invoiced sales) have no table to
119
- roll up β€” they are proven in `verify_odoo_relational.py` instead, against the same fold."""
 
 
 
 
 
 
 
120
  return [_preset(f) for f in (
121
  {"key": "customer", "label": "Customer", "type": "text", "source": "overlay",
122
  "default": True, "pinned": True},
123
- {"key": JOIN_KEY, "label": "Odoo partner id", "type": "int", "source": "overlay",
 
 
 
 
124
  "default": False},
125
- {"key": "invoices", "label": "Open invoices", "type": "link", "source": "overlay",
 
 
 
 
 
 
 
126
  "default": True,
127
  "link": {"table": INVOICES_KEY, "on": JOIN_KEY, "from": JOIN_KEY}},
 
 
 
 
 
 
 
128
  {"key": "ar_outstanding", "label": "AR outstanding $", "type": "rollup",
129
  "source": "overlay", "default": True, "agg": "sum",
130
  "rollup": {"link": "invoices", "field": "residual", "fn": "sum"}},
 
 
 
 
 
 
131
  {"key": "open_invoices", "label": "Open invoices #", "type": "rollup",
132
  "source": "overlay", "default": True,
133
- "rollup": {"link": "invoices", "fn": "countall"}},
134
  # β›” NOT `min`. `_rollup_fold`'s min/max are NUMERIC folds (`_lane_num`), so `min` over a
135
  # date column finds no numbers and returns BLANK β€” a column that renders empty forever
136
- # while looking configured. Ranking a DATE is what `latest` + `sortBy` is for: sort the
137
- # linked rows ascending by due date and take the first. Caught by this module's own gate.
138
  {"key": "oldest_due", "label": "Oldest due date", "type": "rollup", "source": "overlay",
139
  "default": True,
140
  "rollup": {"link": "invoices", "field": "due_date", "fn": "latest",
141
- "sortBy": "due_date", "sortDir": "asc"}},
142
- # ⭐ THE READ-THROUGH ROLLUP β€” the column C8 said this substrate "cannot host", now hosted
143
- # WITHOUT hosting the rows (owner 2026-08-09). It names a governed TOPIC and a METRIC KEY;
144
- # `rollup_sql` answers every customer in ONE grouped query against DuckDB.
145
- # MEASURED: 1,748 customers over 256,810 order lines in 232 ms, and PARITY-PROVEN at
146
- # 1,716/1,716 customers TO THE CENT against the topic's own scope.
147
- #
148
- # β›” IT DOES NOT AND CANNOT COME FROM `ut_odoo_invoices`. That table is OPEN AR only
149
- # (`_AR_WHERE` filters `payment_state IN ('not_paid','partial')`), so a rollup over the
150
- # `invoices` link would sum UNPAID invoices and call it sales β€” a wrong number that looks
151
- # right. Different question, different source; stated so nobody "simplifies" it later.
152
- #
153
- # ⚠ `revenue_invoiced` is ORDER-LINE revenue narrowed by the order's fully-invoiced flag β€”
154
- # NOT posted-invoice billing (that is `customer_invoices`, a different topic which says so
155
- # itself). The metric KEY carries that distinction; this field only names the key, which is
156
- # the whole reason it may not carry SQL of its own.
157
  {"key": "sales_ytd", "label": "Sales YTD - invoiced", "type": "rollup",
158
  "source": "overlay", "default": True, "agg": "sum",
159
  "rollup": {"source": {"topic": "sales_lines", "measure": "revenue_invoiced",
160
  "groupBy": "order_partner", "on": JOIN_KEY, "window": "ytd"}}},
161
- {"key": "refreshed", "label": "Refreshed", "type": "date", "source": "overlay",
162
- "default": False},
 
 
 
 
 
 
 
 
 
 
 
163
  )]
164
 
165
 
@@ -180,23 +349,70 @@ def excluded_names():
180
  return set()
181
 
182
 
183
- def read_open_ar(cur, excluded=None):
184
- """[(row dict)] β€” every open customer receivable in the mirror, oracle-scoped.
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
185
 
186
- Takes a CURSOR so the gate can hand it a fixture connection; no global store binding here.
187
  """
188
- excluded = excluded if excluded is not None else excluded_names()
 
189
  sql = ("SELECT id, name, partner_id, partner_name, invoice_date, invoice_date_due, "
190
  " amount_untaxed_signed, amount_residual_signed, payment_state, move_type "
191
- f"FROM account_move WHERE {_AR_WHERE} AND partner_id IS NOT NULL")
192
  out = []
193
  for r in cur.execute(sql).fetchall():
194
  (mid, name, pid, pname, inv_date, due, untaxed, residual, pay_state, mtype) = r
195
- if str(pname or "").strip().lower() in excluded:
 
196
  continue
197
  out.append({
198
  "_id": str(mid),
199
  "invoice_no": str(name or ""),
 
200
  "customer": str(pname or ""),
201
  JOIN_KEY: int(pid),
202
  "invoice_date": _as_date(inv_date),
@@ -205,22 +421,119 @@ def read_open_ar(cur, excluded=None):
205
  "amount_untaxed": float(untaxed or 0.0),
206
  "payment_state": str(pay_state or ""),
207
  "move_type": str(mtype or ""),
 
208
  })
209
  return out
210
 
211
 
212
- def _as_date(value):
213
- """ISO date string, or ''. The grid renders `date` cells itself (W26: `Aug 5, 2026`), so the
214
- STORED value stays ISO β€” a formatted string in the cell is a value the filters cannot sort."""
215
- if not value:
216
- return ""
217
- text = str(value)
218
- return text[:10]
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
219
 
220
 
221
  def customers_from(invoice_rows):
222
- """The partners carrying open AR β€” derived from the SAME rows the invoice table holds, never
223
- from a second query. Two populations built by two queries drift the moment one is edited."""
 
 
 
 
224
  out = {}
225
  for row in invoice_rows:
226
  pid = row[JOIN_KEY]
@@ -241,6 +554,17 @@ class Refused(Exception):
241
  been written wrong."""
242
 
243
 
 
 
 
 
 
 
 
 
 
 
 
244
  def plan(cur, rt=None):
245
  """The rows that WOULD be written, plus the refusals that apply β€” no store WRITE at all.
246
 
@@ -248,57 +572,81 @@ def plan(cur, rt=None):
248
  so **every cap is checked before anything is committed**.
249
 
250
  β›” `rt` IS WHAT MAKES THE TABLE-COUNT CHECK HONEST, and leaving it out was a real half-spawn
251
- bug. This spawn writes TWO tables in two updater passes; a tenant one slot short of
252
- `MAX_TABLES` would create `ut_odoo_invoices`, then refuse `ut_odoo_customers` β€” leaving a
253
- locked invoices database with no rollup host, while the route answered 409 as though nothing
254
- had happened. A partial spawn is worse than a refused one, so the count is checked against the
255
- tables that ALREADY exist, before the first write.
256
  """
257
  ut = _ut()
258
- invoices = read_open_ar(cur)
259
- customers = customers_from(invoices)
 
 
 
 
 
260
  problems = []
 
 
261
  if rt is not None:
262
  try:
263
  existing = dict(rt.get(ut.STORE_KEY) or {})
264
  except Exception: # noqa: BLE001
265
  existing = {}
266
- needed = [k for k in (INVOICES_KEY, CUSTOMERS_KEY) if k not in existing]
267
  if needed and len(existing) + len(needed) > ut.MAX_TABLES:
268
  problems.append(
269
  f"tenant holds {len(existing)} of MAX_TABLES={ut.MAX_TABLES} user tables and "
270
  f"needs {len(needed)} more ({', '.join(needed)}); refusing rather than creating "
271
- f"one half of a linked pair")
272
- # β›” REFUSE, NEVER TRUNCATE. A capped AR table understates what customers owe while looking
273
- # exactly like a complete one -- the failure mode this whole module is arranged against.
274
- for key, rows in ((INVOICES_KEY, invoices), (CUSTOMERS_KEY, customers)):
 
 
275
  if len(rows) > ut.MAX_ROWS:
276
  problems.append(f"{key}: {len(rows)} rows exceeds MAX_ROWS={ut.MAX_ROWS}; refusing "
277
- f"(a truncated receivables table understates the balance)")
278
- return {"invoices": invoices, "customers": customers, "problems": problems}
 
 
 
 
 
 
 
279
 
280
 
281
  def apply_plan(rt, built, username="automation", today=None):
282
- """Create-or-merge both tables and their rows. Idempotent by construction.
283
 
284
  Row ids ARE the Odoo ids, so a re-run updates in place and never appends a second copy of the
285
- same invoice. Rows that left the population (paid since the last run) are REMOVED β€” an AR
286
- worklist that keeps settled invoices is worse than a stale one, because every total it feeds
287
- is then too high.
 
288
  """
289
  if built.get("problems"):
290
  raise Refused("; ".join(built["problems"]))
291
  stamp = today or _iso_today()
292
  written = {}
293
- for key, label, fields, rows in (
294
- (INVOICES_KEY, "Odoo open invoices", invoice_fields(), built["invoices"]),
295
- (CUSTOMERS_KEY, "Odoo AR customers", customer_fields(), built["customers"])):
296
- written[key] = _ensure_table(rt, key, label, fields, rows, username, stamp)
297
  return written
298
 
299
 
300
  def _ensure_table(rt, key, label, fields, rows, username, stamp):
301
- """One table: definition merged, rows reconciled, written only when something changed."""
 
 
 
 
 
 
 
302
  ut = _ut()
303
  wanted = {r["_id"]: {k: v for k, v in r.items() if k != "_id"} for r in rows}
304
  for row in wanted.values():
@@ -321,15 +669,31 @@ def _ensure_table(rt, key, label, fields, rows, username, stamp):
321
  "recordMode": ut.AUTOMATION_RECORD_MODE,
322
  }
323
  table.setdefault("recordMode", ut.AUTOMATION_RECORD_MODE)
324
- have = {str(f.get("key")) for f in (table.get("fields") or [])}
325
  for field in fields:
326
- if str(field.get("key")) not in have:
 
327
  table.setdefault("fields", []).append(dict(field))
328
- stored = table.setdefault("rows", {})
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
329
  for rid, values in wanted.items():
330
- current = stored.get(rid)
331
  if current is None:
332
- stored[rid] = dict(values)
333
  counts["added"] += 1
334
  elif any(str(current.get(k, "")) != str(v) for k, v in values.items()
335
  if k != "refreshed"):
@@ -337,8 +701,8 @@ def _ensure_table(rt, key, label, fields, rows, username, stamp):
337
  counts["updated"] += 1
338
  else:
339
  current["refreshed"] = values["refreshed"]
340
- for rid in [r for r in stored if r not in wanted]:
341
- stored.pop(rid, None)
342
  counts["removed"] += 1
343
  return cur
344
 
@@ -353,17 +717,17 @@ def is_royal(tenant):
353
  def refresh(rt, tenant, username="automation", cur=None, today=None):
354
  """THE entry point β€” the store-resync path and the route both call this.
355
 
356
- ⚠ It must be CALLED on resync by something outside this file (see the ASK to session A). If it
357
- is not wired, every row still carries a `refreshed` stamp, so a stale worklist is at least
358
- LEGIBLE rather than silently authoritative.
359
  """
360
  if not is_royal(tenant):
361
  raise Refused(f"tenant {tenant!r} has no Odoo mirror behind these tables (R1: Royal "
362
- f"Imports only); refusing to spawn two empty locked databases")
363
  if cur is None:
364
  from harness import datastore
365
  cur = datastore.ro_con()
366
  built = plan(cur, rt=rt)
367
  written = apply_plan(rt, built, username=username, today=today)
368
  return {"tables": written,
369
- "invoices": len(built["invoices"]), "customers": len(built["customers"])}
 
1
+ """odoo_relational.py β€” Odoo entities as LOCKED relational databases.
2
 
3
  Owner ruling R1 / contract C8: spawn preset Odoo databases for Royal Imports, give them preset
4
  Link + Rollup fields, and prove each rollup against the `measure_` column it will eventually
5
+ replace.
6
+
7
+ ⭐⭐ 2026-08-09 β€” THE POPULATIONS WIDENED FROM "OPEN AR" TO **EVERY ODOO ID**, which is the
8
+ owner's item: *"make sure we have all the Unique ID in Odoo in Database for the Royal Imports
9
+ tenant."* Before this change there were two tables holding 438 partners and 1,228 invoices β€” the
10
+ partners who owed money β€” so most Odoo ids were simply absent, and the missing rows were the
11
+ reason a Rollup could not answer a sales question. Four tables now, keyed on the Odoo id itself:
12
+
13
+ ut_odoo_customers 2,465 rows 0.43 MB every partner with a confirmed order or a
14
+ posted customer document
15
+ ut_odoo_products 5,829 rows 1.20 MB every active product carrying a SKU code
16
+ ut_odoo_invoices 31,418 rows 8.68 MB EVERY posted customer invoice + refund
17
+ ut_odoo_orders 32,700 rows 7.50 MB every confirmed sale order
18
+
19
+ β›” WHAT MADE THAT LEGAL, AND IT WAS NOT A BIGGER NUMBER. `MAX_ROWS` was 5,000 and this module's
20
+ own `plan()` refused above it β€” but the cap was never a property of the store (see the measured
21
+ banner on `core.user_tables.MAX_ROWS`; `ig_master` has run a 500,000-row bucket the whole time).
22
+ The cap is now 100,000, DERIVED from what a row actually weighs. The four tables together are
23
+ 17.81 MB in one `user_tables` document β€” real, bounded, and booked: the per-table row-key split
24
+ is D-87's next increment. β›” ORDER LINES REMAIN OUT (256,810 rows / 63.9 MB / 2.57 s per copy);
25
+ they are answered by the read-through rollup, which never copies a row.
26
+
27
+ ⭐ THE EXCLUDED CHANNEL IS NOW A COLUMN, NOT A DELETION. `core.odoo.EXCLUDE_PARTNER_NAMES` puts
28
+ GIFTWARE DEALS (partner 6369 β€” the Amazon channel) outside WHOLESALE scope, and the old tables
29
+ dropped its rows entirely. Dropping them contradicts "every Odoo id", so the rows are kept and
30
+ carry **`wholesale_scope`** instead. ⚠⚠ READ THIS BEFORE COMPARING ANY TOTAL: that one partner
31
+ holds **$1,755,779.95 of the $2,347,608.49** raw open balance β€” 75% of it β€” across 25 invoices.
32
+ Wholesale open AR is $591,828.54. So a column total here will not equal the AR page unless you
33
+ filter `wholesale_scope`, and that is the scope difference, not a defect. `read_open_ar` keeps
34
+ excluding, because `modules/ar` is its oracle and an oracle answers ONE question.
35
+
36
+ ⭐ AR SURVIVED THE WIDENING UNCHANGED, AND THAT IS MEASURED, NOT ASSUMED. `sum(residual)` over
37
+ ALL posted customer documents equals `sum(residual)` over `modules/ar._open_docs`' own predicate
38
+ **to the cent** ($2,347,608.49): zero posted rows carry a non-zero residual outside
39
+ `payment_state IN ('not_paid','partial')`, and zero rows inside it carry a residual of 0. So the
40
+ `ar_outstanding` rollup needs no condition. β›” THE COUNT AND THE DATE DO β€” `countall` over the
41
+ wider link would count 31,418 documents and call them open invoices, so those two rollups carry
42
+ the oracle's predicate as an explicit `payment_state` condition pair.
43
  """
44
  import datetime as _dt
45
 
46
  #: Royal Imports only (R1). A tenant slug that is not this one gets a refusal, never a spawn:
47
+ #: nurilab has no Odoo mirror behind these tables and would get empty locked databases.
48
  RI_SLUGS = ("", "royal-imports")
49
 
50
  INVOICES_KEY = "ut_odoo_invoices"
51
  CUSTOMERS_KEY = "ut_odoo_customers"
52
+ ORDERS_KEY = "ut_odoo_orders"
53
+ PRODUCTS_KEY = "ut_odoo_products"
54
 
55
+ #: The join column the partner-grain tables carry. Derived links resolve through it (`on`/`from`).
56
  JOIN_KEY = "partner_id"
57
+ #: The product-grain equivalent.
58
+ PRODUCT_JOIN_KEY = "product_id"
59
 
60
+ #: The oracle's own predicate β€” `modules/ar._open_docs`, copied rather than re-derived so the two
61
+ #: cannot drift. It now selects a SUBSET of the invoices table rather than defining it.
62
+ _AR_OPEN = "payment_state IN ('not_paid','partial')"
63
+ _POSTED_DOCS = "state = 'posted' AND move_type IN ('out_invoice','out_refund')"
64
+ _AR_WHERE = f"{_POSTED_DOCS} AND {_AR_OPEN}"
65
+ _CONFIRMED = "state IN ('sale','done')"
66
+
67
+ #: A refresh that would delete more than this share of a table's stored rows REFUSES instead.
68
+ #: β›” THE GUARD ONLY BECAME NECESSARY WHEN THE TABLES GOT BIG. `_ensure_table` removes rows that
69
+ #: left the population, which is right β€” a reversed invoice must not keep inflating a total. But
70
+ #: the population comes from the DuckDB mirror, and a mirror caught mid-resync (or one seeded
71
+ #: against an empty store) answers with FEWER rows and no error. At 1,228 rows that was a visible
72
+ #: mistake; at 31,418 it is a silent one. Odoo history does not halve, so a halving is a bad read.
73
+ MAX_SHRINK = 0.5
74
 
75
 
76
  def _ut():
 
90
  return field
91
 
92
 
93
+ #: The two conditions that reproduce `modules/ar`'s open-document predicate inside a rollup.
94
+ #: ⚠ Two `eq` legs joined by OR, not one `in` β€” `ROLLUP_CONDITION_OPS` has no `in`, and inventing
95
+ #: one here would be a second condition vocabulary beside `_clean_rollup`'s.
96
+ _OPEN_ONLY = {"conditions": [{"field": "payment_state", "op": "eq", "value": "not_paid"},
97
+ {"field": "payment_state", "op": "eq", "value": "partial"}],
98
+ "conditionConj": "or"}
99
+
100
+
101
  # ---------------------------------------------------------------------------------------------
102
  # FIELD CONTRACTS
103
  # ---------------------------------------------------------------------------------------------
104
  # ⚠ Every type here must be in `core.user_tables.UT_FIELD_TYPES`, and `_clean_field` returns None
105
  # for an unknown one β€” which DELETES the column silently on the next read rather than erroring.
106
+ def _scope_field():
107
+ return {"key": "wholesale_scope", "label": "In wholesale scope", "type": "checkbox",
108
+ "source": "overlay", "default": False,
109
+ "description": "Unticked = the GIFTWARE DEALS / Amazon channel, which every wholesale "
110
+ "metric in this product excludes. The row is kept so no Odoo id is "
111
+ "missing; filter on this column to reconcile against the AR page."}
112
+
113
+
114
+ def _refreshed_field():
115
+ return {"key": "refreshed", "label": "Refreshed", "type": "date", "source": "overlay",
116
+ "default": False, "description": "When this row was last reconciled against Odoo."}
117
+
118
+
119
  def invoice_fields():
120
+ """One row per POSTED customer invoice or refund β€” the full history, not just what is open."""
121
  return [_preset(f) for f in (
122
  {"key": "invoice_no", "label": "Invoice", "type": "text", "source": "overlay",
123
  "default": True, "pinned": True},
124
+ {"key": "odoo_id", "label": "Odoo ID", "type": "int", "source": "overlay",
125
+ "default": False, "description": "The `account.move` id. Also this row's id."},
126
  {"key": "customer", "label": "Customer", "type": "text", "source": "overlay",
127
  "default": True},
128
  {"key": JOIN_KEY, "label": "Odoo partner id", "type": "int", "source": "overlay",
 
132
  {"key": "due_date", "label": "Due date", "type": "date", "source": "overlay",
133
  "default": True},
134
  {"key": "residual", "label": "Outstanding $", "type": "currency", "source": "overlay",
135
+ "default": True, "agg": "sum",
136
+ "description": "Odoo's signed residual. Exactly 0 on every settled document, which is "
137
+ "why AR rollups need no filter."},
138
  {"key": "amount_untaxed", "label": "Invoiced $", "type": "currency", "source": "overlay",
139
+ "default": True, "agg": "sum"},
140
+ # β›” THE OPTION LIST WIDENED WITH THE POPULATION. It read ['not_paid','partial'] while the
141
+ # table held open AR only; the full posted history also carries paid / in_payment /
142
+ # reversed. A select holding a value its options do not declare is the wave-26 item-24
143
+ # defect β€” the filter panel answers with a list that cannot match what is stored.
144
  {"key": "payment_state", "label": "Payment state", "type": "select", "source": "overlay",
145
+ "default": True,
146
+ "options": ["not_paid", "partial", "in_payment", "paid", "reversed"]},
147
  {"key": "move_type", "label": "Document", "type": "select", "source": "overlay",
148
  "default": False, "options": ["out_invoice", "out_refund"]},
149
+ _scope_field(),
150
+ # ⭐ THE RECIPROCAL HALF (owner item 2, 2026-08-09). DERIVED (`on` declared), exactly like
151
+ # its twin, so the engine owns the cell and no human can edit a relation Odoo decided.
 
 
 
 
 
152
  {"key": "customer_link", "label": "Customer record", "type": "link", "source": "overlay",
153
  "default": False,
154
  "link": {"table": CUSTOMERS_KEY, "on": JOIN_KEY, "from": JOIN_KEY}},
155
+ _refreshed_field(),
156
+ )]
157
+
158
+
159
+ def order_fields():
160
+ """One row per CONFIRMED sale order β€” `state in (sale, done)`, the fixed wholesale scope."""
161
+ return [_preset(f) for f in (
162
+ {"key": "order_no", "label": "Order", "type": "text", "source": "overlay",
163
+ "default": True, "pinned": True},
164
+ {"key": "odoo_id", "label": "Odoo ID", "type": "int", "source": "overlay",
165
+ "default": False, "description": "The `sale.order` id. Also this row's id."},
166
+ {"key": "customer", "label": "Customer", "type": "text", "source": "overlay",
167
+ "default": True},
168
+ {"key": JOIN_KEY, "label": "Odoo partner id", "type": "int", "source": "overlay",
169
  "default": False},
170
+ {"key": "order_date", "label": "Order date", "type": "date", "source": "overlay",
171
+ "default": True},
172
+ {"key": "amount_untaxed", "label": "Order $", "type": "currency", "source": "overlay",
173
+ "default": True, "agg": "sum"},
174
+ {"key": "team", "label": "Business unit", "type": "select", "source": "overlay",
175
+ "default": True, "options": ["Fisch", "Royal", "Sales", "Giftware Deals"]},
176
+ {"key": "state", "label": "State", "type": "select", "source": "overlay",
177
+ "default": False, "options": ["sale", "done"]},
178
+ {"key": "invoice_status", "label": "Invoice status", "type": "select", "source": "overlay",
179
+ "default": True, "options": ["invoiced", "to invoice", "upselling", "no"]},
180
+ _scope_field(),
181
+ {"key": "customer_link", "label": "Customer record", "type": "link", "source": "overlay",
182
+ "default": False,
183
+ "link": {"table": CUSTOMERS_KEY, "on": JOIN_KEY, "from": JOIN_KEY}},
184
+ _refreshed_field(),
185
+ )]
186
+
187
+
188
+ def product_fields():
189
+ """One row per `product.product`, keyed on its id β€” EVERY product, archived ones included.
190
+
191
+ ⚠ THE ROW ID IS THE PRODUCT ID, NOT THE SKU CODE, and the difference is measurable: 12 codes
192
+ map to more than one product id (re-SKU / merge history). The code is what a human reads and
193
+ the id is what `sales_lines.product` groups by, so both are columns and only the id is the
194
+ identity.
195
+
196
+ β›” THE PINNED COLUMN IS THE NAME, NOT THE SKU, and that is not a style choice. 62 products
197
+ carry no `default_code` at all (UBER CHARGE, Delivery Charges, PICK UP …) while ZERO carry a
198
+ blank name β€” measured. Pinning `code` would give those rows a blank primary cell, which is
199
+ exactly D-80: a first column nothing populates quietly becoming the row's identity
200
+ ([[fallback-that-became-the-rule]]).
201
+ """
202
+ return [_preset(f) for f in (
203
+ {"key": "product", "label": "Product", "type": "text", "source": "overlay",
204
+ "default": True, "pinned": True},
205
+ {"key": PRODUCT_JOIN_KEY, "label": "Odoo ID", "type": "int", "source": "overlay",
206
+ "default": False, "description": "The `product.product` id. Also this row's id, and what "
207
+ "every product rollup groups by."},
208
+ {"key": "code", "label": "SKU", "type": "text", "source": "overlay",
209
+ "default": True, "description": "Odoo's `default_code`. Blank on the 62 charge/service "
210
+ "products that are not stocked SKUs."},
211
+ {"key": "active", "label": "Active in Odoo", "type": "checkbox", "source": "overlay",
212
+ "default": True,
213
+ "description": "Unticked = archived. Archived products are kept because they still "
214
+ "carry sales history β€” two of them sold this year."},
215
+ # ⚠ TEXT, NOT SELECT. 71 categories exist today and Odoo gains them without telling us; a
216
+ # select whose options go stale answers a filter with a list that cannot match a stored
217
+ # value (wave-26 item 24). Text filters honestly and never goes out of date.
218
+ {"key": "category", "label": "Category", "type": "text", "source": "overlay",
219
+ "default": True},
220
+ {"key": "product_type", "label": "Type", "type": "select", "source": "overlay",
221
+ "default": False, "options": ["product", "consu", "service"]},
222
+ {"key": "standard_price", "label": "Standard cost", "type": "currency",
223
+ "source": "overlay", "default": True},
224
+ # ⭐ SOURCE-BACKED (read-through) β€” the product-grain half of "compute all of the data in
225
+ # Odoo". It names a governed TOPIC + METRIC KEY and one grouped query answers every SKU;
226
+ # `sales_lines` holds 256,810 rows that are never copied into this table.
227
+ {"key": "sales_ytd", "label": "Sales YTD", "type": "rollup", "source": "overlay",
228
+ "default": True, "agg": "sum",
229
+ "rollup": {"source": {"topic": "sales_lines", "measure": "revenue",
230
+ "groupBy": "product", "on": PRODUCT_JOIN_KEY, "window": "ytd"}}},
231
+ {"key": "units_ytd", "label": "Units YTD", "type": "rollup", "source": "overlay",
232
+ "default": True, "agg": "sum",
233
+ "rollup": {"source": {"topic": "sales_lines", "measure": "units",
234
+ "groupBy": "product", "on": PRODUCT_JOIN_KEY, "window": "ytd"}}},
235
+ {"key": "margin_ytd", "label": "Gross margin YTD $", "type": "rollup", "source": "overlay",
236
+ "default": True, "agg": "sum",
237
+ "rollup": {"source": {"topic": "sales_lines", "measure": "margin",
238
+ "groupBy": "product", "on": PRODUCT_JOIN_KEY, "window": "ytd"}}},
239
+ _refreshed_field(),
240
  )]
241
 
242
 
243
  def customer_fields():
244
+ """One row per partner Odoo has transacted with, keyed on the `res.partner` id.
245
+
246
+ ⭐ Two DERIVED links (`on` declared), so the engine owns both cells and a human cannot edit a
247
+ relation Odoo already decided. The rollups come in two kinds on purpose:
248
+ * LINK rollups fold the rows in `ut_odoo_invoices` / `ut_odoo_orders` β€” they can answer
249
+ anything about a document the table holds, including a date rank;
250
+ * SOURCE rollups name a governed topic + metric and are answered by ONE grouped SQL query
251
+ over the whole mirror β€” they can answer a DATE-WINDOWED money question, which a link
252
+ rollup cannot, because a condition can only compare against a literal and a literal year
253
+ start is right until 1 January.
254
+ """
255
  return [_preset(f) for f in (
256
  {"key": "customer", "label": "Customer", "type": "text", "source": "overlay",
257
  "default": True, "pinned": True},
258
+ {"key": JOIN_KEY, "label": "Odoo ID", "type": "int", "source": "overlay",
259
+ "default": False, "description": "The `res.partner` id. Also this row's id."},
260
+ {"key": "city", "label": "City", "type": "text", "source": "overlay", "default": True},
261
+ {"key": "state", "label": "State", "type": "text", "source": "overlay", "default": True},
262
+ {"key": "country", "label": "Country", "type": "text", "source": "overlay",
263
  "default": False},
264
+ {"key": "agent", "label": "Sales agent", "type": "text", "source": "overlay",
265
+ "default": True,
266
+ "description": "The customer's assigned agent (res.partner.agent_ids[0] β€” the "
267
+ "Customers-module convention)."},
268
+ _scope_field(),
269
+
270
+ # --- the relations -------------------------------------------------------------------
271
+ {"key": "invoices", "label": "Invoices", "type": "link", "source": "overlay",
272
  "default": True,
273
  "link": {"table": INVOICES_KEY, "on": JOIN_KEY, "from": JOIN_KEY}},
274
+ {"key": "orders", "label": "Orders", "type": "link", "source": "overlay",
275
+ "default": True,
276
+ "link": {"table": ORDERS_KEY, "on": JOIN_KEY, "from": JOIN_KEY}},
277
+
278
+ # --- link rollups over the invoice history --------------------------------------------
279
+ # ⭐ NO CONDITION, and that is measured rather than assumed: a settled document's residual
280
+ # is exactly 0, so summing the full history gives the open balance to the cent.
281
  {"key": "ar_outstanding", "label": "AR outstanding $", "type": "rollup",
282
  "source": "overlay", "default": True, "agg": "sum",
283
  "rollup": {"link": "invoices", "field": "residual", "fn": "sum"}},
284
+ {"key": "invoiced_all_time", "label": "Invoiced $ - all time", "type": "rollup",
285
+ "source": "overlay", "default": False, "agg": "sum",
286
+ "rollup": {"link": "invoices", "field": "amount_untaxed", "fn": "sum"}},
287
+ # β›” THESE TWO DO NEED THE PREDICATE. Over the widened link a bare `countall` counts every
288
+ # document ever posted and labels it "open invoices" β€” the wrong-number-that-looks-right
289
+ # this module refuses everywhere else.
290
  {"key": "open_invoices", "label": "Open invoices #", "type": "rollup",
291
  "source": "overlay", "default": True,
292
+ "rollup": {"link": "invoices", "fn": "countall", **_OPEN_ONLY}},
293
  # β›” NOT `min`. `_rollup_fold`'s min/max are NUMERIC folds (`_lane_num`), so `min` over a
294
  # date column finds no numbers and returns BLANK β€” a column that renders empty forever
295
+ # while looking configured. Ranking a DATE is what `latest` + `sortBy` is for.
 
296
  {"key": "oldest_due", "label": "Oldest due date", "type": "rollup", "source": "overlay",
297
  "default": True,
298
  "rollup": {"link": "invoices", "field": "due_date", "fn": "latest",
299
+ "sortBy": "due_date", "sortDir": "asc", **_OPEN_ONLY}},
300
+
301
+ # --- link rollups over the order history ----------------------------------------------
302
+ {"key": "order_count", "label": "Orders #", "type": "rollup", "source": "overlay",
303
+ "default": True,
304
+ "rollup": {"link": "orders", "fn": "countall"}},
305
+ {"key": "last_order", "label": "Last order date", "type": "rollup", "source": "overlay",
306
+ "default": True,
307
+ "rollup": {"link": "orders", "field": "order_date", "fn": "latest",
308
+ "sortBy": "order_date", "sortDir": "desc"}},
309
+
310
+ # --- source-backed (read-through) rollups ---------------------------------------------
311
+ # β›” THESE DO NOT AND CANNOT COME FROM THE `invoices` LINK. `ut_odoo_invoices` is posted
312
+ # BILLING; `revenue_invoiced` is ORDER-LINE revenue narrowed by the order's fully-invoiced
313
+ # flag. Different grain, different question β€” the metric KEY carries the distinction,
314
+ # which is the whole reason a rollup may not carry SQL of its own.
315
  {"key": "sales_ytd", "label": "Sales YTD - invoiced", "type": "rollup",
316
  "source": "overlay", "default": True, "agg": "sum",
317
  "rollup": {"source": {"topic": "sales_lines", "measure": "revenue_invoiced",
318
  "groupBy": "order_partner", "on": JOIN_KEY, "window": "ytd"}}},
319
+ {"key": "sales_ltm", "label": "Sales LTM", "type": "rollup", "source": "overlay",
320
+ "default": True, "agg": "sum",
321
+ "rollup": {"source": {"topic": "sales_lines", "measure": "revenue",
322
+ "groupBy": "order_partner", "on": JOIN_KEY, "window": "ltm"}}},
323
+ {"key": "margin_ytd", "label": "Gross margin YTD $", "type": "rollup", "source": "overlay",
324
+ "default": False, "agg": "sum",
325
+ "rollup": {"source": {"topic": "sales_lines", "measure": "margin",
326
+ "groupBy": "order_partner", "on": JOIN_KEY, "window": "ytd"}}},
327
+ {"key": "orders_ytd", "label": "Orders YTD #", "type": "rollup", "source": "overlay",
328
+ "default": False,
329
+ "rollup": {"source": {"topic": "sales_orders", "measure": "orders",
330
+ "groupBy": "partner", "on": JOIN_KEY, "window": "ytd"}}},
331
+ _refreshed_field(),
332
  )]
333
 
334
 
 
349
  return set()
350
 
351
 
352
+ def excluded_ids(cur, names=None):
353
+ """The out-of-scope partner IDS, resolved against the MIRROR.
354
+
355
+ ⭐ IDS, NOT THE DENORMALISED NAME ON THE DOCUMENT, for two reasons that both bite.
356
+ `account_move.partner_name` is a copy taken when the document was written, and this module's
357
+ own `customers_from` says so out loud β€” *"a partner's name can differ across documents
358
+ (renames land on new invoices only)"*. So a rename would put some of one partner's documents
359
+ in scope and the rest out, silently, and the totals would stop reconciling with nothing to
360
+ point at. `modules/ar`, the oracle these numbers answer to, has always excluded by ID.
361
+
362
+ β›” RESOLVED FROM THE MIRROR, NOT `core.odoo.excluded_partner_ids()`. That function issues a
363
+ LIVE `search_read`, so importing it here would make spawning four locked databases fail
364
+ whenever Odoo is unreachable β€” including on this developer machine, where the handshake dies
365
+ on an expired certificate ([[local-odoo-ssl-quirk]]). Same names, same answer, no network.
366
+
367
+ ⚠ MATCHED CASE- AND WHITESPACE-INSENSITIVELY, and a NULL name simply does not match β€” which
368
+ is the correct direction. 44 transacting partners carry no name at all; treating an
369
+ unanswerable name as "excluded" would drop $7,734.83 of real open AR out of scope.
370
+ """
371
+ names = names if names is not None else excluded_names()
372
+ if not names:
373
+ return set()
374
+ rows = cur.execute("SELECT id, name FROM res_partner WHERE name IS NOT NULL").fetchall()
375
+ return {int(pid) for pid, name in rows if str(name).strip().lower() in names}
376
+
377
+
378
+ def _as_date(value):
379
+ """ISO date string, or ''. The grid renders `date` cells itself (W26: `Aug 5, 2026`), so the
380
+ STORED value stays ISO β€” a formatted string in the cell is a value the filters cannot sort."""
381
+ if not value:
382
+ return ""
383
+ return str(value)[:10]
384
+
385
+
386
+ def _in_scope(pid, excluded):
387
+ """`'1'` | `''` β€” the `checkbox` cell convention (`aios_grid`: the overlay stores '1' or '')."""
388
+ return "" if int(pid) in excluded else "1"
389
+
390
+
391
+ def read_invoices(cur, excluded=None, open_only=False):
392
+ """[(row dict)] β€” posted customer invoices and refunds, keyed on the `account.move` id.
393
+
394
+ ONE reader, two projections. `open_only` applies the AR oracle's predicate and drops the
395
+ out-of-scope channel, which is what `read_open_ar` wants; the default keeps every row and
396
+ TAGS the channel instead. Two queries would be two populations, and they drift the moment
397
+ either is edited.
398
 
399
+ Takes a CURSOR so a gate can hand it a fixture connection; no global store binding here.
400
  """
401
+ excluded = excluded if excluded is not None else excluded_ids(cur)
402
+ where = _AR_WHERE if open_only else _POSTED_DOCS
403
  sql = ("SELECT id, name, partner_id, partner_name, invoice_date, invoice_date_due, "
404
  " amount_untaxed_signed, amount_residual_signed, payment_state, move_type "
405
+ f"FROM account_move WHERE {where} AND partner_id IS NOT NULL")
406
  out = []
407
  for r in cur.execute(sql).fetchall():
408
  (mid, name, pid, pname, inv_date, due, untaxed, residual, pay_state, mtype) = r
409
+ scope = _in_scope(pid, excluded)
410
+ if open_only and not scope:
411
  continue
412
  out.append({
413
  "_id": str(mid),
414
  "invoice_no": str(name or ""),
415
+ "odoo_id": int(mid),
416
  "customer": str(pname or ""),
417
  JOIN_KEY: int(pid),
418
  "invoice_date": _as_date(inv_date),
 
421
  "amount_untaxed": float(untaxed or 0.0),
422
  "payment_state": str(pay_state or ""),
423
  "move_type": str(mtype or ""),
424
+ "wholesale_scope": scope,
425
  })
426
  return out
427
 
428
 
429
+ def read_open_ar(cur, excluded=None):
430
+ """The OPEN, wholesale-scoped subset β€” `modules/ar._open_docs`' own population.
431
+
432
+ Kept as its own door because `modules/ar` is this module's oracle for the AR numbers, and an
433
+ oracle answers exactly one question. It is a projection of `read_invoices`, never a second
434
+ query.
435
+ """
436
+ return read_invoices(cur, excluded=excluded, open_only=True)
437
+
438
+
439
+ def read_orders(cur, excluded=None):
440
+ """[(row dict)] β€” confirmed sale orders, keyed on the `sale.order` id."""
441
+ excluded = excluded if excluded is not None else excluded_ids(cur)
442
+ sql = ("SELECT id, name, date_order, partner_id, partner_name, team_name, state, "
443
+ " amount_untaxed, invoice_status "
444
+ f"FROM sale_order WHERE {_CONFIRMED} AND partner_id IS NOT NULL")
445
+ out = []
446
+ for r in cur.execute(sql).fetchall():
447
+ (oid, name, when, pid, pname, team, state, untaxed, inv_status) = r
448
+ out.append({
449
+ "_id": str(oid),
450
+ "order_no": str(name or ""),
451
+ "odoo_id": int(oid),
452
+ "customer": str(pname or ""),
453
+ JOIN_KEY: int(pid),
454
+ "order_date": _as_date(when),
455
+ "amount_untaxed": float(untaxed or 0.0),
456
+ "team": str(team or ""),
457
+ "state": str(state or ""),
458
+ "invoice_status": str(inv_status or ""),
459
+ "wholesale_scope": _in_scope(pid, excluded),
460
+ })
461
+ return out
462
+
463
+
464
+ def read_products(cur):
465
+ """[(row dict)] β€” EVERY `product.product`, keyed on its id.
466
+
467
+ β›” NO `active` AND NO `default_code` FILTER, and both exclusions were measured before they
468
+ were dropped. Filtering to active-and-coded gave 5,829 of 5,948 rows and left FIVE products
469
+ that sold this very year with no row at all: two archived SKUs (`9SAT-FY`, `2GSTY`) and three
470
+ uncoded charge lines (`UBER CHARGE`, `[Delivery_009] Delivery Charges`, `PICK UP`). A product
471
+ grouped by `sales_lines.product` that has no parent row is a rollup value with nowhere to
472
+ land β€” silently. 119 extra rows is the whole cost of the claim being literally true.
473
+ """
474
+ sql = ("SELECT id, default_code, name, categ_name, type, standard_price, active "
475
+ "FROM product_product")
476
+ out = []
477
+ for r in cur.execute(sql).fetchall():
478
+ (prid, code, name, categ, ptype, cost, active) = r
479
+ out.append({
480
+ "_id": str(prid),
481
+ "product": str(name or ""),
482
+ PRODUCT_JOIN_KEY: int(prid),
483
+ "code": str(code or ""),
484
+ "active": "1" if active else "",
485
+ "category": str(categ or ""),
486
+ "product_type": str(ptype or ""),
487
+ "standard_price": float(cost or 0.0),
488
+ })
489
+ return out
490
+
491
+
492
+ def read_customers(cur, excluded=None):
493
+ """[(row dict)] β€” every partner Odoo has TRANSACTED with, keyed on the `res.partner` id.
494
+
495
+ ⭐ THE POPULATION IS THE UNION OF THE TWO DOCUMENT UNIVERSES, and both legs are load-bearing:
496
+ Amazon books as direct invoices with no sale order (the `odoo-api` gotcha), so a sale-order
497
+ leg alone would silently drop a real customer. Partners with no document at all are left out
498
+ on purpose β€” a row that can never appear in any topic has nothing to roll up and would render
499
+ as an all-blank line.
500
+
501
+ β›” NOT DERIVED FROM THE INVOICE ROWS. `customers_from` did that when the table WAS the open-AR
502
+ partners; sourcing a customer registry from its own receivables is what kept most Odoo ids
503
+ out of the store in the first place.
504
+ """
505
+ excluded = excluded if excluded is not None else excluded_ids(cur)
506
+ sql = ("SELECT p.id, p.name, p.city, p.state_name, p.country_name, ag.name "
507
+ "FROM res_partner p "
508
+ "LEFT JOIN res_partner ag ON ag.id = p.agent_id "
509
+ "WHERE p.id IN ("
510
+ f" SELECT partner_id FROM sale_order WHERE {_CONFIRMED} AND partner_id IS NOT NULL "
511
+ " UNION "
512
+ " SELECT partner_id FROM account_move "
513
+ f" WHERE {_POSTED_DOCS} AND partner_id IS NOT NULL)")
514
+ out = []
515
+ for r in cur.execute(sql).fetchall():
516
+ (pid, name, city, state, country, agent) = r
517
+ out.append({
518
+ "_id": str(pid),
519
+ "customer": str(name or ""),
520
+ JOIN_KEY: int(pid),
521
+ "city": str(city or ""),
522
+ "state": str(state or ""),
523
+ "country": str(country or ""),
524
+ "agent": str(agent or ""),
525
+ "wholesale_scope": _in_scope(pid, excluded),
526
+ })
527
+ return out
528
 
529
 
530
  def customers_from(invoice_rows):
531
+ """The partners carrying the given invoice rows β€” the pre-2026-08-09 population builder.
532
+
533
+ ⚠ NO LONGER WHAT SPAWNS `ut_odoo_customers` (that is `read_customers`). Kept because it is a
534
+ pure function over rows and the gate uses it to prove the FOLD against a fixture without a
535
+ mirror; deleting it would cost a test its independence from the SQL.
536
+ """
537
  out = {}
538
  for row in invoice_rows:
539
  pid = row[JOIN_KEY]
 
554
  been written wrong."""
555
 
556
 
557
+ #: `plan()` bucket -> (store key, nav label, field contract). ⭐ ONE ROW PER TABLE is the whole
558
+ #: point: adding an Odoo entity is a spec row plus a reader, not a fifth copy of the spawn code.
559
+ #: ⚠ ORDER MATTERS ONLY FOR THE REFUSAL MESSAGE; `plan` checks every cap before anything commits.
560
+ TABLES = (
561
+ ("customers", CUSTOMERS_KEY, "Odoo customers", customer_fields),
562
+ ("products", PRODUCTS_KEY, "Odoo products", product_fields),
563
+ ("invoices", INVOICES_KEY, "Odoo invoices", invoice_fields),
564
+ ("orders", ORDERS_KEY, "Odoo orders", order_fields),
565
+ )
566
+
567
+
568
  def plan(cur, rt=None):
569
  """The rows that WOULD be written, plus the refusals that apply β€” no store WRITE at all.
570
 
 
572
  so **every cap is checked before anything is committed**.
573
 
574
  β›” `rt` IS WHAT MAKES THE TABLE-COUNT CHECK HONEST, and leaving it out was a real half-spawn
575
+ bug. This spawn writes FOUR tables in four updater passes; a tenant near `MAX_TABLES` would
576
+ create some and refuse the rest β€” leaving a locked invoices database with no rollup host,
577
+ while the route answered as though nothing had happened. A partial spawn is worse than a
578
+ refused one, so the count is checked against the tables that ALREADY exist, before the first
579
+ write. `rt` also carries the stored row counts the shrink guard compares against.
580
  """
581
  ut = _ut()
582
+ excluded = excluded_ids(cur)
583
+ built = {
584
+ "customers": read_customers(cur, excluded=excluded),
585
+ "products": read_products(cur),
586
+ "invoices": read_invoices(cur, excluded=excluded),
587
+ "orders": read_orders(cur, excluded=excluded),
588
+ }
589
  problems = []
590
+
591
+ existing = {}
592
  if rt is not None:
593
  try:
594
  existing = dict(rt.get(ut.STORE_KEY) or {})
595
  except Exception: # noqa: BLE001
596
  existing = {}
597
+ needed = [k for _b, k, _l, _f in TABLES if k not in existing]
598
  if needed and len(existing) + len(needed) > ut.MAX_TABLES:
599
  problems.append(
600
  f"tenant holds {len(existing)} of MAX_TABLES={ut.MAX_TABLES} user tables and "
601
  f"needs {len(needed)} more ({', '.join(needed)}); refusing rather than creating "
602
+ f"part of a linked set")
603
+
604
+ for bucket, key, _label, _fields in TABLES:
605
+ rows = built[bucket]
606
+ # β›” REFUSE, NEVER TRUNCATE. A capped table understates every total it feeds while looking
607
+ # exactly like a complete one β€” the failure mode this whole module is arranged against.
608
  if len(rows) > ut.MAX_ROWS:
609
  problems.append(f"{key}: {len(rows)} rows exceeds MAX_ROWS={ut.MAX_ROWS}; refusing "
610
+ f"(a truncated table understates every rollup it feeds)")
611
+ stored = len(((existing.get(key) or {}).get("rows")) or {})
612
+ if stored and len(rows) < stored * MAX_SHRINK:
613
+ problems.append(
614
+ f"{key}: the mirror answered {len(rows)} rows against {stored} stored β€” a drop of "
615
+ f"more than {int((1 - MAX_SHRINK) * 100)}% is a bad read, not Odoo history "
616
+ f"shrinking; refusing rather than deleting rows that still exist")
617
+ built["problems"] = problems
618
+ return built
619
 
620
 
621
  def apply_plan(rt, built, username="automation", today=None):
622
+ """Create-or-merge every table in `built` and its rows. Idempotent by construction.
623
 
624
  Row ids ARE the Odoo ids, so a re-run updates in place and never appends a second copy of the
625
+ same record β€” which is also what makes "every Odoo unique id is in the database" a checkable
626
+ statement rather than a hopeful one.
627
+
628
+ ⚠ ONLY THE BUCKETS PRESENT ARE WRITTEN, so a caller (or a gate) may hand in a subset.
629
  """
630
  if built.get("problems"):
631
  raise Refused("; ".join(built["problems"]))
632
  stamp = today or _iso_today()
633
  written = {}
634
+ for bucket, key, label, fields in TABLES:
635
+ if bucket not in built:
636
+ continue
637
+ written[key] = _ensure_table(rt, key, label, fields(), built[bucket], username, stamp)
638
  return written
639
 
640
 
641
  def _ensure_table(rt, key, label, fields, rows, username, stamp):
642
+ """One table: definition merged, rows reconciled, written only when something changed.
643
+
644
+ ⚠ ROWS THAT LEFT THE POPULATION ARE REMOVED, and that stayed correct through the widening β€”
645
+ but only because the populations widened to "everything Odoo has". While `ut_odoo_customers`
646
+ was built FROM open invoices, removal meant a customer who paid their bill vanished from the
647
+ registry; now a partner leaves only when their last document does. The shrink guard in
648
+ `plan()` is the backstop for the case this policy cannot distinguish: a partial mirror read.
649
+ """
650
  ut = _ut()
651
  wanted = {r["_id"]: {k: v for k, v in r.items() if k != "_id"} for r in rows}
652
  for row in wanted.values():
 
669
  "recordMode": ut.AUTOMATION_RECORD_MODE,
670
  }
671
  table.setdefault("recordMode", ut.AUTOMATION_RECORD_MODE)
672
+ have = {str(f.get("key")): f for f in (table.get("fields") or [])}
673
  for field in fields:
674
+ fkey = str(field.get("key"))
675
+ if fkey not in have:
676
  table.setdefault("fields", []).append(dict(field))
677
+ continue
678
+ # ⭐ A PRESET FIELD'S CONTRACT IS FORWARD-MIGRATED, not merely created once. The
679
+ # widening moved `payment_state`'s option list and every rollup's conditions; a
680
+ # create-only merge would have left the LIVE table declaring the old contract
681
+ # forever, so the column would render but its filter could not match what is stored.
682
+ # ⚠ Only machine-owned keys are touched β€” `automation.preset` is the wall β€” so a
683
+ # column a user added to a locked database is never rewritten.
684
+ stored = have[fkey]
685
+ if (stored.get("automation") or {}).get("preset"):
686
+ for prop in ("label", "type", "options", "link", "rollup", "description",
687
+ "agg", "pinned", "default"):
688
+ if prop in field:
689
+ stored[prop] = field[prop]
690
+ else:
691
+ stored.pop(prop, None)
692
+ stored_rows = table.setdefault("rows", {})
693
  for rid, values in wanted.items():
694
+ current = stored_rows.get(rid)
695
  if current is None:
696
+ stored_rows[rid] = dict(values)
697
  counts["added"] += 1
698
  elif any(str(current.get(k, "")) != str(v) for k, v in values.items()
699
  if k != "refreshed"):
 
701
  counts["updated"] += 1
702
  else:
703
  current["refreshed"] = values["refreshed"]
704
+ for rid in [r for r in stored_rows if r not in wanted]:
705
+ stored_rows.pop(rid, None)
706
  counts["removed"] += 1
707
  return cur
708
 
 
717
  def refresh(rt, tenant, username="automation", cur=None, today=None):
718
  """THE entry point β€” the store-resync path and the route both call this.
719
 
720
+ ⚠ It must be CALLED on resync by something outside this file. If it is not wired, every row
721
+ still carries a `refreshed` stamp, so a stale worklist is at least LEGIBLE rather than
722
+ silently authoritative.
723
  """
724
  if not is_royal(tenant):
725
  raise Refused(f"tenant {tenant!r} has no Odoo mirror behind these tables (R1: Royal "
726
+ f"Imports only); refusing to spawn empty locked databases")
727
  if cur is None:
728
  from harness import datastore
729
  cur = datastore.ro_con()
730
  built = plan(cur, rt=rt)
731
  written = apply_plan(rt, built, username=username, today=today)
732
  return {"tables": written,
733
+ **{bucket: len(built[bucket]) for bucket, _k, _l, _f in TABLES}}
api/rollup_sql.py CHANGED
@@ -1,11 +1,12 @@
1
  """rollup_sql.py β€” the READ-THROUGH rollup: one grouped SQL query answers every parent row.
2
 
3
- β›” WHY THIS EXISTS. A linked rollup folds rows that live in the store, and `MAX_ROWS = 5000`
4
- bounds them for a measured reason: every `ut_*` table lives inside ONE `user_tables.json` that is
5
- parsed and deep-copied on essentially every request (MEASURED 2026-08-09 on nurilab β€” 5.1 MB /
6
- 1,567 rows = 67 ms per copy, and rendering a 91-row table still pays for all of it). Royal's
7
- 254k+ order lines cannot live there and never will: at that density the blob is ~829 MB and one
8
- deep copy is ~9.8 s.
 
9
 
10
  ⭐ SO THIS KIND NEVER COPIES THE ROWS AT ALL. It names a governed semantic TOPIC and a METRIC KEY,
11
  and `store_query(..., group_by=[dim])` answers EVERY parent in one pass against DuckDB.
@@ -95,10 +96,21 @@ def group_values(bag, today=None):
95
  f"{topic}/{measure} grouped by {dim} exceeded {sem.MAX_GROUPS} groups β€” refusing to "
96
  f"write cells from a truncated result")
97
 
 
 
 
 
 
 
 
 
 
 
98
  id_col = f"{dim}_id"
 
99
  out = {}
100
  for row in res.get("rows") or []:
101
- gid = row.get(id_col)
102
  if gid is None:
103
  continue
104
  out[str(gid)] = row.get(measure)
 
1
  """rollup_sql.py β€” the READ-THROUGH rollup: one grouped SQL query answers every parent row.
2
 
3
+ β›” WHY THIS EXISTS. A linked rollup folds rows that live in the store, and `MAX_ROWS` bounds them
4
+ for a measured reason: every `ut_*` table lives inside ONE `user_tables.json` that is parsed and
5
+ copied on essentially every request. ⚠ THE CAP MOVED TO 100,000 ON 2026-08-09 AND THAT DID NOT
6
+ RETIRE THIS MODULE β€” the arithmetic is why. Measured against tenant #0's own mirror, Royal's
7
+ **256,810 order lines are 63.9 MB and 2.57 s per copy**: 2.6x the new cap and 8x any tolerable
8
+ per-request cost. Four Odoo databases that DO fit (customers, products, invoices, orders) already
9
+ total 17.81 MB / ~320 ms. Order lines never live in that document, at any cap.
10
 
11
  ⭐ SO THIS KIND NEVER COPIES THE ROWS AT ALL. It names a governed semantic TOPIC and a METRIC KEY,
12
  and `store_query(..., group_by=[dim])` answers EVERY parent in one pass against DuckDB.
 
96
  f"{topic}/{measure} grouped by {dim} exceeded {sem.MAX_GROUPS} groups β€” refusing to "
97
  f"write cells from a truncated result")
98
 
99
+ # β›” THE KEY COLUMN IS NOT ALWAYS `<dim>_id`, and reading it unconditionally was a SILENT
100
+ # BLANK. `store_query` emits `<dim>_id` only when the dim declares a `name_col`
101
+ # (`f"{d['col']} AS {g}_id" if d.get("name_col") else f"{d['col']} AS {g}"`) β€” an id plus a
102
+ # display name. A dim whose value IS its own label emits the bare `<dim>` instead, and two
103
+ # shipped dims are that shape: `receivables.payment_state` and `gl_lines.account_type`.
104
+ # Against either, this returned an EMPTY map and every cell was written blank β€” a configured
105
+ # column that computes nothing, with no error anywhere. `payment_state` is not hypothetical:
106
+ # it is a real column on `ut_odoo_invoices` and an obvious thing to roll up against.
107
+ # ⚠ `_id` is tried FIRST so an id-keyed dim can never be keyed by its display name, which
108
+ # would silently merge two customers who share one.
109
  id_col = f"{dim}_id"
110
+ key_col = id_col if any(id_col in (r or {}) for r in (res.get("rows") or [])) else dim
111
  out = {}
112
  for row in res.get("rows") or []:
113
+ gid = row.get(key_col)
114
  if gid is None:
115
  continue
116
  out[str(gid)] = row.get(measure)
api/routes_odoo_tables.py CHANGED
@@ -10,8 +10,8 @@ only prove both files mention the same path SEGMENT). The pairing is a named che
10
  doc, not an assumption -- and `verify_api` now enumerates `main.app.routes`, which is the control
11
  that actually answers it.
12
 
13
- ADMIN-GATED, and not for tidiness: a refresh REWRITES two locked databases for the whole tenant and
14
- deletes the rows that left the population. That is an operator action.
15
  """
16
  from fastapi import Depends
17
  from fastapi import APIRouter
@@ -28,7 +28,8 @@ def _rel():
28
 
29
  @router.post("/odoo-tables/refresh")
30
  def refresh_odoo_tables(session: Session = Depends(require_session)):
31
- """Rebuild `ut_odoo_invoices` + `ut_odoo_customers` from the tenant's Odoo mirror.
 
32
 
33
  Idempotent: row ids are Odoo ids, so a re-run updates in place. Returns what MOVED, because
34
  "refreshed" with no counts is indistinguishable from a no-op over an empty mirror.
@@ -67,7 +68,11 @@ def odoo_tables_status(session: Session = Depends(require_session)):
67
  """
68
  rel, out = _rel(), {}
69
  import core.user_tables as user_tables
70
- for key in (rel.INVOICES_KEY, rel.CUSTOMERS_KEY):
 
 
 
 
71
  table = user_tables.get(key, st=session.runtime) or {}
72
  rows = (table.get("rows") or {})
73
  stamps = [str((r or {}).get("refreshed") or "") for r in rows.values()]
 
10
  doc, not an assumption -- and `verify_api` now enumerates `main.app.routes`, which is the control
11
  that actually answers it.
12
 
13
+ ADMIN-GATED, and not for tidiness: a refresh REWRITES four locked databases for the whole tenant
14
+ and deletes the rows that left the population. That is an operator action.
15
  """
16
  from fastapi import Depends
17
  from fastapi import APIRouter
 
28
 
29
  @router.post("/odoo-tables/refresh")
30
  def refresh_odoo_tables(session: Session = Depends(require_session)):
31
+ """Rebuild all four locked Odoo databases from the tenant's mirror β€” customers, products,
32
+ invoices and orders (it was invoices + customers until 2026-08-09).
33
 
34
  Idempotent: row ids are Odoo ids, so a re-run updates in place. Returns what MOVED, because
35
  "refreshed" with no counts is indistinguishable from a no-op over an empty mirror.
 
68
  """
69
  rel, out = _rel(), {}
70
  import core.user_tables as user_tables
71
+ # ⚠ ITERATES `rel.TABLES`, NEVER A LITERAL PAIR. It read `(INVOICES_KEY, CUSTOMERS_KEY)`
72
+ # while those were the only two; the 2026-08-09 widening added products and orders, and a
73
+ # hard-coded list here would have reported "everything is fine" over two databases it had
74
+ # stopped knowing about ([[gate-answers-the-wrong-question]] β€” the hard-coded count).
75
+ for _bucket, key, _label, _fields in rel.TABLES:
76
  table = user_tables.get(key, st=session.runtime) or {}
77
  rows = (table.get("rows") or {})
78
  stamps = [str((r or {}).get("refreshed") or "") for r in rows.values()]
api/routes_tables.py CHANGED
@@ -263,6 +263,98 @@ def list_tables(session: Session = Depends(require_session)):
263
  return {"tables": out}
264
 
265
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
266
  @router.post("/tables", status_code=201)
267
  def create_table(body: dict = Body(default=None),
268
  session: Session = Depends(require_session)):
 
263
  return {"tables": out}
264
 
265
 
266
+ #: ⭐⭐ THE ROLLUP SOURCE OFFER β€” what makes the read-through rollup a FEATURE rather than a
267
+ #: capability. Owner 2026-08-09: *"Full field editor: pick topic β†’ metric β†’ window."*
268
+ #:
269
+ #: β›” THE ENGINE SHIPPED WITH NO WRITER. `_clean_rollup` has accepted a `source` bag since
270
+ #: 2026-08-09 and `rollup_sql` answers it in one grouped query, but the ONLY thing in the product
271
+ #: that ever produced one was a hard-coded field in `odoo_relational.py`. Nothing in the client
272
+ #: could make one, so the whole read-through path was reachable by editing Python β€” the
273
+ #: [[artifact-with-no-importer]] shape, twice burned in this repo already.
274
+ #:
275
+ #: ⚠ DERIVED FROM THE MODEL FILES, NEVER A HAND-WRITTEN LIST. `model/topics/*.yml` and
276
+ #: `model/metrics/*.yml` already state which measures belong to which topic and which dims that
277
+ #: topic can group by; a second list here would be a second definition of the same fact, and the
278
+ #: two would drift the day somebody adds a metric. Same argument `rollup_sql` makes for binding
279
+ #: to a metric KEY instead of carrying SQL.
280
+ _ROLLUP_CACHE = {}
281
+
282
+
283
+ def _rollup_source_offer():
284
+ """`{topics:[…], windows:[…]}` β€” every (topic, measure, dim) the engine can actually answer.
285
+
286
+ β›” ONLY COMBINATIONS THAT CAN RESOLVE ARE OFFERED, because a rollup that refuses at COMPUTE
287
+ time refuses silently β€” the cells are simply left blank, hours later, on a column that looks
288
+ configured. Two exclusions do real work:
289
+ * a topic with NO dims cannot be grouped at all, so it can never key a parent row;
290
+ * a CROSS-TOPIC metric (`aov`, `margin_pct`, `returns_pct` β€” `agg: ratio`/`derived` whose
291
+ inputs live elsewhere) is refused by `store_query` the moment a `group_by` is present:
292
+ *"cross-topic measures are scalar-only"*. Offering one would mint a column that can only
293
+ ever error.
294
+ Each dim also declares HOW it keys β€” by an Odoo id or by its own value β€” because that is what
295
+ the user is matching their own column against, and `payment_state` (a value) and
296
+ `partner` (an id) are matched to very different columns.
297
+ """
298
+ if _ROLLUP_CACHE.get("offer"):
299
+ return _ROLLUP_CACHE["offer"]
300
+ from harness import semantic as sem
301
+ from harness import windows as W
302
+ ut = _ut()
303
+
304
+ topics, metrics = sem.topics(), sem.metrics()
305
+ by_topic = {}
306
+ for key, m in metrics.items():
307
+ # A ratio/derived metric whose parts sit on another topic cannot be grouped β€” see above.
308
+ if m.get("agg") in ("ratio", "derived"):
309
+ continue
310
+ by_topic.setdefault(m["topic"], []).append(
311
+ {"key": key, "label": m.get("label") or key, "format": m.get("format") or "usd",
312
+ "description": m.get("description") or ""})
313
+
314
+ out = []
315
+ for tkey, t in sorted(topics.items()):
316
+ dims = ((t.get("store") or {}).get("dims") or {})
317
+ measures = by_topic.get(tkey) or []
318
+ if not dims or not measures:
319
+ continue
320
+ out.append({
321
+ "key": tkey,
322
+ "label": t.get("label") or tkey,
323
+ "grain": t.get("grain") or "",
324
+ "dims": [{"key": dkey,
325
+ "label": d.get("label") or dkey,
326
+ # `store_query` emits `<dim>_id` only when the dim carries a display name
327
+ # alongside the key; otherwise the value IS the key. `rollup_sql` handles
328
+ # both, and the editor says which so the user matches the right column.
329
+ "keyedBy": "id" if d.get("name_col") else "value"}
330
+ for dkey, d in dims.items()],
331
+ "measures": sorted(measures, key=lambda m: m["label"].lower()),
332
+ })
333
+
334
+ # ⚠ THE WINDOW LIST IS `core.user_tables`' OWN, not `harness.windows`'. `ROLLUP_SOURCE_WINDOWS`
335
+ # is the validator's closed set and is deliberately NARROWER (it omits the parameterised kinds
336
+ # like `last_n_days`, which have nowhere in the bag to carry their `n`). Offering a kind the
337
+ # validator refuses would let the editor build a field the save door rejects.
338
+ windows = [{"key": k, "label": W.WINDOW_LABELS.get(k, k).format(n="N")}
339
+ for k in ut.ROLLUP_SOURCE_WINDOWS]
340
+ offer = {"topics": out, "windows": windows}
341
+ _ROLLUP_CACHE["offer"] = offer
342
+ return offer
343
+
344
+
345
+ @router.get("/tables/rollup-sources")
346
+ def rollup_sources(session: Session = Depends(require_session)):
347
+ """The topic β†’ metric β†’ dim β†’ window offer the rollup field editor renders.
348
+
349
+ ⚠ DECLARED ABOVE EVERY `/tables/{table_key}/…` ROUTE, and kept there deliberately. FastAPI
350
+ matches in declaration order, so the day somebody adds a bare `GET /tables/{table_key}` below
351
+ this line it still resolves; added ABOVE it, this endpoint would silently start arriving as
352
+ `table_key='rollup-sources'` and 404 from the table wall. There is no such route today β€”
353
+ this is the cheap ordering that keeps it from mattering.
354
+ """
355
+ return _rollup_source_offer()
356
+
357
+
358
  @router.post("/tables", status_code=201)
359
  def create_table(body: dict = Body(default=None),
360
  session: Session = Depends(require_session)):
platform/core/user_tables.py CHANGED
The diff for this file is too large to render. See raw diff
 
web/src/customer-grid/ColumnMenu.tsx CHANGED
The diff for this file is too large to render. See raw diff
 
web/src/customer-grid/CustomerGrid.tsx CHANGED
The diff for this file is too large to render. See raw diff
 
web/src/customer-grid/apiBridge.ts CHANGED
@@ -1,809 +1,852 @@
1
- // ---------------------------------------------------------------------------
2
- // customer-grid / apiBridge.ts β€” the STANDALONE half of the data path (X2).
3
- //
4
- // The branch beside hostBridge.ts. Where `hostBridge` speaks the Streamlit
5
- // Components v1 protocol to a surrounding Python app, this speaks HTTP to the
6
- // API β€” the SAME event objects, the same ids, the same optimism semantics on
7
- // the client. Which one is live is decided once, by mode, and never mixed:
8
- // `installStandaloneBridge()` is called from main.tsx ONLY in standalone, so
9
- // the embed bundle never registers a sink and its behaviour is byte-unchanged
10
- // (pinned by verify_bridge.py, which was written BEFORE this file existed).
11
- //
12
- // ⚠ WHAT THIS WAVE'S WRITE PATH CAN AND CANNOT DO β€” read before extending.
13
- // The client's whole write model is "emit β†’ the host reruns β†’ a fresh payload
14
- // arrives β†’ the UI reflects it" (CustomerGrid's own words: *"the emit itself
15
- // triggers the host rerun, so the fresh name/list arrives with the next payload
16
- // β€” no local mutation to drift from the store"*). Standalone has no rerun and
17
- // X2 fixes `GET /api/v1/customers` at the EXACT current shape β€” `{fields, rows,
18
- // today, pulled_at}`, with no `workspace`, no `cohorts`, no `docs`. So:
19
- //
20
- // VISIBLE + DURABLE view_upsert Β· view_delete Β· field_* Β· overlay_patch
21
- // (the client already holds these in state + localStorage;
22
- // the POST is what makes them durable server-side)
23
- // DURABLE, INVISIBLE cohort_* Β· add_to_list Β· folder_* Β· item_move
24
- // (they reach the store, but nothing re-reads them until
25
- // the payload carries that state β€” EXIT-5). The server's
26
- // `toast` is the only feedback the user gets, which is
27
- // why it is wired through and not dropped.
28
- // NOT REACHABLE doc_add / doc_fetch / doc_delete β€” the Documents panel
29
- // is driven by `payload.docs`, which standalone never
30
- // receives, so no doc event can be emitted at all. X2's
31
- // `doc` field is therefore deliberately NOT handled here:
32
- // wiring a response nobody can trigger is untestable dead
33
- // code. When `/customers` grows `docs`, feed `doc` into
34
- // the same `payload.docPayload` slot the embed uses.
35
- //
36
- // NOTHING HERE FALLS BACK TO `sample_customers.json`. That affordance ("the
37
- // grid still renders with no backend") is unreachable now β€” the frame requires
38
- // a session before CustomerGrid mounts at all β€” and behind a login, sample
39
- // revenue rendered after a server hiccup is fabricated data on a screen the
40
- // user has every reason to trust. An honest failure, always.
41
- // ---------------------------------------------------------------------------
42
-
43
- import {
44
- API_V1,
45
- CREDENTIALS,
46
- DATA_ERROR_EVENT,
47
- DERIVED_CELLS_EVENT,
48
- ROWS_STALE_EVENT,
49
- TOAST_EVENT,
50
- WORKSPACE_STALE_EVENT,
51
- UNAUTHORIZED_EVENT,
52
- checkTenant,
53
- signal,
54
- } from "../apiContract";
55
- import { setStandaloneSink } from "./hostBridge";
56
- import { CUSTOMER_TOPIC } from "./types";
57
- import type { CustomersPayload, Field, GridWorkspace, HostEvent, Row, TopicConfig } from "./types";
58
- import type { TsPayload, TsRequest } from "./timeSeriesData";
59
-
60
- const JSON_HEADERS = { "Content-Type": "application/json" };
61
-
62
- async function readJson(res: Response): Promise<unknown> {
63
- try {
64
- return await res.json();
65
- } catch {
66
- return null;
67
- }
68
- }
69
-
70
- /** 401 is the one status that is never a data problem. Raised once, centrally,
71
- * so every call site cannot forget to. Returns true when it handled it.
72
- * The rows caches die with the session β€” the next signed-in user may not
73
- * be the same person, and a cached book must never cross that boundary. */
74
- function handledUnauthorized(status: number): boolean {
75
- if (status !== 401) return false;
76
- rowsCache.clear();
77
- signal(UNAUTHORIZED_EVENT);
78
- return true;
79
- }
80
-
81
- /**
82
- * The session check every authenticated call makes: the 401 above, PLUS the TENANT swap the
83
- * 401 path structurally cannot see.
84
- *
85
- * β›” WHY `handledUnauthorized` WAS NEVER ENOUGH. Its own note says the rows cache "dies with the
86
- * session β€” the next signed-in user may not be the same person". That is the USER boundary, and
87
- * it is enforced on 401. A TENANT swap produces no 401 at all: `aios_session` is one cookie per
88
- * ORIGIN, so signing into another tenant in a second tab repoints this one, and every request
89
- * here keeps returning a cheerful 200 full of somebody else's data. `checkTenant` reads the
90
- * server's own stamp and reloads the frame; see `apiContract.ts::TENANT_HEADER`.
91
- */
92
- function handledSession(res: Response): boolean {
93
- if (checkTenant(res)) return true;
94
- return handledUnauthorized(res.status);
95
- }
96
-
97
- // --- the READ (CP-B) -------------------------------------------------------
98
-
99
- /**
100
- * The last good `/customers` payload, reused across surface switches. The rows
101
- * are a 15-minute-cached Odoo pull SERVER-side, but every Customer⇄Cohort
102
- * route change remounts the grid and re-downloaded the whole ~1 MB payload β€”
103
- * three full transfers in one short session, measured live. Within this window
104
- * a remount reuses the copy; the workspace (the cheap per-scope call) is always
105
- * re-fetched, so views/cohorts stay live.
106
- *
107
- * β›” THE CACHE IS SESSION-SCOPED OR IT IS A LEAK. It is cleared on ANY 401 and
108
- * on sign-out/sign-in (Shell calls `clearCustomersCache`) β€” one browser, two
109
- * accounts, and a cached book served across the boundary would be the exact
110
- * cross-user leak EXIT-3b exists to prevent, client-side this time.
111
- *
112
- * Own edits stay visible ([[the NO-BLIP law]]): `patchCustomer` writes accepted
113
- * values through into the cached rows, so a scope switch after an edit shows
114
- * the edit, not the pre-edit pull.
115
- */
116
- const CUSTOMERS_FRESH_MS = 5 * 60_000;
117
- // Wave 16 C-TOPIC: ONE cache map keyed by the topic's rows path, so the product pull and the
118
- // customer pull each get the same 5-minute remount window without ever serving each other.
119
- const rowsCache = new Map<string, { at: number; payload: CustomersPayload }>();
120
-
121
- export function clearCustomersCache(): void {
122
- rowsCache.clear();
123
- }
124
-
125
- /**
126
- * X2 `GET /api/v1/<topic.rowsPath>` β†’ `{fields, rows, today, pulled_at}`, rows already
127
- * scoped to the session's BU pool (EXIT-3b). The cookie replaces HTTP Basic.
128
- *
129
- * Returns null on ANY failure, having raised the matching signal. A null is not
130
- * an empty table: the frame renders a failure, not "this tenant has no
131
- * records", because those two look identical and only one of them is true.
132
- */
133
- export async function fetchTopicRows(topic: TopicConfig): Promise<CustomersPayload | null> {
134
- const cached = rowsCache.get(topic.rowsPath);
135
- if (cached && Date.now() - cached.at < CUSTOMERS_FRESH_MS) {
136
- return cached.payload;
137
- }
138
- let res: Response;
139
- try {
140
- res = await fetch(`${API_V1}/${topic.rowsPath}`, { credentials: CREDENTIALS });
141
- } catch {
142
- signal(DATA_ERROR_EVENT, "Cannot reach the server.");
143
- return null;
144
- }
145
- if (handledSession(res)) return null;
146
- if (!res.ok) {
147
- signal(DATA_ERROR_EVENT, `The server answered ${res.status}.`);
148
- return null;
149
- }
150
- const body = (await readJson(res)) as CustomersPayload | null;
151
- if (!body || !Array.isArray(body.rows) || !Array.isArray(body.fields)) {
152
- signal(DATA_ERROR_EVENT, "The server sent an unreadable payload.");
153
- return null;
154
- }
155
- rowsCache.set(topic.rowsPath, { at: Date.now(), payload: body });
156
- return body;
157
- }
158
-
159
- /** The customer topic's fetch β€” kept under its own name because half the write-path notes in
160
- * this repo cite it; it IS `fetchTopicRows(CUSTOMER_TOPIC)`. */
161
- export function fetchCustomers(): Promise<CustomersPayload | null> {
162
- return fetchTopicRows(CUSTOMER_TOPIC);
163
- }
164
-
165
- /**
166
- * `GET /api/v1/workspace` β€” the saved views, custom fields, folders and cohorts
167
- * for the session user (S1's amendment, 2026-07-30). It exists because X2 pins
168
- * `/customers` at the EXACT current shape, which carries no `workspace`, and
169
- * without one the standalone write path is WRITE-ONLY: a saved view reaches the
170
- * store and is gone from the screen on reload.
171
- *
172
- * The body is `{workspace: GridWorkspace}` β€” the client's own built type
173
- * (types.ts `GridWorkspace`), which is what the embed's host already projects.
174
- *
175
- * ⚠ ABSENT IS FINE AND MEANS TODAY'S BEHAVIOUR. A 404 (route not shipped) or a
176
- * body carrying no workspace return null and the grid runs exactly as it does
177
- * now β€” views in memory, no `storageKey`, nothing durable. It degrades to the
178
- * honest gap rather than failing the whole read, because the workspace is an
179
- * enhancement of the table, not a precondition for it. A 401 still ends the
180
- * session: that is the one status that is never about this route.
181
- *
182
- * β›” WAVE 21 item 3 (3c) β€” BUT A REFUSAL IS NOT AN ABSENCE, and conflating the
183
- * two is half of the "RI fields on a new database" report.
184
- *
185
- * This used to answer `null` to every unhappy path alike: route-not-shipped,
186
- * 403, 500, connection refused, garbage body. The caller cannot tell those
187
- * apart, so it did the only thing it could β€” carried on without a workspace β€”
188
- * and `CustomerGrid` then had no `storageKey` and fell back to a bucket shared
189
- * with every other surface (see the note at its `storageKey`). A tenant whose
190
- * `/workspace` 403s for `ut_*` therefore got ANOTHER TABLE'S views and fields,
191
- * silently, with the grid looking entirely healthy.
192
- *
193
- * `announceFailure` is the caller saying "I am the INITIAL load; if this fails
194
- * for a reason that is not 'no workspace here', say so". It raises the same
195
- * `DATA_ERROR_EVENT` a failed rows read raises, so the frame renders its honest
196
- * failure card instead of a grid furnished with somebody else's schema.
197
- *
198
- * ⚠ IT IS OFF BY DEFAULT, and that is load-bearing rather than cautious.
199
- * `reread()` calls this on every workspace-stale signal, and its own rule is
200
- * "absent stays absent β€” never blank a live panel"; a signal from there would
201
- * let one dropped packet replace a working table with an error card. The flag
202
- * is passed explicitly at the one call site that owns the first paint.
203
- */
204
- export async function fetchWorkspace(
205
- scope: SurfaceScope = "customer",
206
- opts: { announceFailure?: boolean } = {},
207
- ): Promise<GridWorkspace | null> {
208
- const loud = opts.announceFailure === true;
209
- let res: Response;
210
- try {
211
- res = await fetch(`${API_V1}/workspace?scope=${encodeURIComponent(scope)}`,
212
- { credentials: CREDENTIALS });
213
- } catch {
214
- if (loud) signal(DATA_ERROR_EVENT, "Cannot reach the server.");
215
- return null;
216
- }
217
- if (handledSession(res)) return null;
218
- if (!res.ok) {
219
- // 404 is the ONE status that means "this host does not serve a workspace",
220
- // which is the enhancement posture above and stays quiet. Everything else
221
- // is a server that had an answer and would not give it.
222
- if (loud && res.status !== 404)
223
- signal(DATA_ERROR_EVENT,
224
- `The server answered ${res.status} for this table's saved views and columns.`);
225
- return null;
226
- }
227
- const ws = (await readJson(res)) as { workspace?: GridWorkspace } | null;
228
- const w = ws?.workspace;
229
- // A 200 carrying no workspace is still an ABSENCE, not a refusal β€” an older
230
- // host, or a scope this one has nothing stored for. Unchanged, and quiet.
231
- return w && typeof w.storageKey === "string" && Array.isArray(w.views) ? w : null;
232
- }
233
-
234
- /**
235
- * X2 `PATCH /api/v1/<topic.rowsPath>/{pid}` β€” the overlay stratum only (Odoo is never
236
- * written). Kept as its own route rather than folded into the events log
237
- * because X2 keeps it: cell edits are the one write with a per-row identity and
238
- * a rollback the caller already implements.
239
- */
240
- export async function patchTopicRow(
241
- topic: TopicConfig,
242
- pid: number,
243
- updates: Partial<Row>,
244
- /**
245
- * ⭐ Wave-25 (C3/R6) β€” what the SERVER says this row now holds, for callers that keep an
246
- * optimistic copy. Two things a cell PATCH can now report that the request cannot predict:
247
- * Β· a CANONICALISED value β€” a profile column stores `nurilab` for `@Nurilab` or a pasted
248
- * profile URL, so the optimistic copy is a value the store never took;
249
- * Β· CLEARED cells β€” blanking a profile handle also clears that row's enriched columns, and
250
- * the client never typed those, so nothing else would ever repaint them.
251
- * An optional callback rather than a richer return type, deliberately: the boolean IS the
252
- * rollback contract every existing caller is written against, and widening it would make the
253
- * two call sites that ignore this the ones most likely to get it wrong.
254
- */
255
- onAccepted?: (accepted: Partial<Row>, cleared: string[]) => void,
256
- ): Promise<boolean> {
257
- try {
258
- const res = await fetch(`${API_V1}/${topic.rowsPath}/${pid}`, {
259
- method: "PATCH",
260
- credentials: CREDENTIALS,
261
- headers: JSON_HEADERS,
262
- body: JSON.stringify(updates),
263
- });
264
- if (handledSession(res)) return false;
265
- const cached = rowsCache.get(topic.rowsPath);
266
- if (res.ok) {
267
- // Write the ACCEPTED values (the server's own report, not the request)
268
- // through into the cached rows, so a remount within the cache window
269
- // shows this edit instead of the pre-edit pull.
270
- const body = (await readJson(res)) as
271
- { updates?: Record<string, unknown>; cleared?: unknown } | null;
272
- const accepted = body?.updates;
273
- const cleared = Array.isArray(body?.cleared)
274
- ? (body.cleared as unknown[]).filter((k): k is string => typeof k === "string")
275
- : [];
276
- const blanks: Partial<Row> = {};
277
- for (const k of cleared) blanks[k] = "";
278
- if (accepted && typeof accepted === "object") {
279
- const row = cached?.payload.rows.find((r) => r.pid === pid);
280
- if (row) Object.assign(row, accepted, blanks);
281
- onAccepted?.({ ...(accepted as Partial<Row>), ...blanks }, cleared);
282
- }
283
- // A user-table edit may change a reciprocal Link or a Rollup in another database. Their
284
- // row payloads share no cache key with this table, so invalidate the user-table family and
285
- // let the active grid re-read the server's materialised relationship cells.
286
- if (topic.rowsPath.startsWith("tables/")) {
287
- for (const key of [...rowsCache.keys()])
288
- if (key.startsWith("tables/")) rowsCache.delete(key);
289
- signal(ROWS_STALE_EVENT);
290
- }
291
- }
292
- return res.ok;
293
- } catch {
294
- return false;
295
- }
296
- }
297
-
298
- /** The customer topic's patch, by its historical name. */
299
- export function patchCustomer(pid: number, updates: Partial<Row>): Promise<boolean> {
300
- return patchTopicRow(CUSTOMER_TOPIC, pid, updates);
301
- }
302
-
303
- /**
304
- * ⭐ Wave-20 owner item 4 / contract C-ADDROW β€” **APPEND A ROW TO A USER DATABASE**, and
305
- * (C-UNDO) restore a deleted one under its old id.
306
- *
307
- * USER TABLES ONLY, and the refusal is structural rather than checked here: no other scope has
308
- * a `/tables/{key}/rows` endpoint at all. A connector's rows are read-synced from its source β€”
309
- * R8 is explicit that a "+" which must refuse is a fake affordance, so the caller never renders
310
- * one there.
311
- *
312
- * ⚠ THE ANSWER IS THE ID THAT WAS STORED, never the one that was asked for. An undo that
313
- * requests `rid` may find that id re-used, and the server's own note says it answers with what
314
- * it actually wrote; the caller re-anchors on the returned value rather than assuming.
315
- *
316
- * ⚠ The rows cache is CLEARED on success. `fetchTopicRows` holds a 5-minute window per topic,
317
- * so a re-read straight after an append would serve the payload from before it β€” the new row
318
- * would appear minutes later, which reads as "the button did nothing". Same reason the shell's
319
- * retired Add-record bar cleared it (wave 20 item 4 moved that door here).
320
- */
321
- export async function addTableRow(
322
- tableKey: string,
323
- values: Record<string, unknown> = {},
324
- rid?: string | number
325
- ): Promise<{ rid: string | number; pid: number } | null> {
326
- try {
327
- const res = await fetch(`${API_V1}/tables/${encodeURIComponent(tableKey)}/rows`, {
328
- method: "POST",
329
- credentials: CREDENTIALS,
330
- headers: JSON_HEADERS,
331
- body: JSON.stringify(rid === undefined ? { values } : { rid, values }),
332
- });
333
- if (handledSession(res)) return null;
334
- const body = (await readJson(res)) as { rid?: string | number; pid?: number;
335
- detail?: { message?: string } } | null;
336
- if (!res.ok) {
337
- // The server states WHY (a row cap, a store refusal). Surfacing its sentence beats a
338
- // generic failure toast, and staying silent would be the worst of the three.
339
- const why = typeof body?.detail?.message === "string"
340
- ? body.detail.message
341
- : `The server answered ${res.status}.`;
342
- signal(TOAST_EVENT, why);
343
- return null;
344
- }
345
- if (body?.rid === undefined || typeof body?.pid !== "number") {
346
- signal(DATA_ERROR_EVENT, "The server did not say which row it created.");
347
- return null;
348
- }
349
- rowsCache.delete(`tables/${tableKey}/rows`);
350
- return { rid: body.rid, pid: body.pid };
351
- } catch {
352
- signal(DATA_ERROR_EVENT, "Cannot reach the server.");
353
- return null;
354
- }
355
- }
356
-
357
- /**
358
- * ⭐ WAVE 21 item 7 (contract C2) β€” ADD A COLUMN TO A USER DATABASE'S **DEFINITION**,
359
- * not to this user's overlay.
360
- *
361
- * β›” THE BUG THIS EXISTS FOR. Every column the grid creates has always gone out as a
362
- * `field_upsert` EVENT, which lands in the per-user workspace stratum
363
- * (`<key>_table_workspace`). That is right for the connector surfaces, where a custom
364
- * column IS one person's annotation of somebody else's data. It is wrong for a user
365
- * database, where the columns ARE the table β€” and it is why the automation editor's
366
- * "Automation column" picker was empty by construction: that picker reads the DEFINITION
367
- * (`user_tables`), which the grid had never written to. Two stores, one word, and the two
368
- * surfaces disagreed with no error anywhere. The route has existed since wave 18 with no
369
- * client caller; this is that caller, and `routes_tables.py`'s own comment above it names
370
- * this exact defect.
371
- *
372
- * ⚠ SCOPED TO THE AUTOMATION KIND THIS WAVE (C2, deliberately narrow). Moving EVERY kind
373
- * onto the definition is the right end state and is booked as debt: it changes who can see
374
- * a column (everyone with the table, not just its author), which is a visible change to
375
- * shipped behaviour on three surfaces and does not belong in a wave that is fixing this
376
- * one's blindness.
377
- *
378
- * ⚠ THE SERVER'S FIELD IS THE ANSWER, never the request. `add_field` re-slugs and
379
- * re-types on the way in, so the caller inserts what came BACK β€” a column ordered under a
380
- * key the store does not hold is a column that renders nowhere.
381
- */
382
- /**
383
- * ⭐⭐ 2026-08-07 (D-79's last half) β€” PATCH one column's DEFINITION on a user table.
384
- *
385
- * β›” WHY IT EXISTS: wave 25 shipped the profile flag's READER everywhere β€” the column-menu line,
386
- * the cell validator, R6's clear-on-write, the one-per-table refusal β€” and never its WRITER. The
387
- * route (`PATCH /tables/{key}/fields/{fkey}`) has accepted `profile` since that wave with no
388
- * client caller, so the enrich action could not be bound to any database a person made
389
- * themselves. This is that caller.
390
- *
391
- * ⚠ THE SERVER'S FIELD IS THE ANSWER, never the request β€” the same rule `addTableField` carries.
392
- * `_clean_field` enforces `type: 'text'` for a profile flag and refuses a SECOND one with a
393
- * sentence naming the column that already has it, so the refusal is worth surfacing verbatim.
394
- */
395
- /**
396
- * ⭐ 2026-08-07 β€” the sibling databases a `link` column may point at, WITH their fields.
397
- *
398
- * `GET /tables` already returns exactly this (key, label, fields) filtered by `may_open`, so
399
- * there is no new route and no new wall: a database you cannot open is a database you cannot
400
- * link to, decided by the same predicate that decides whether you can see it at all.
401
- *
402
- * ⚠ Returns `[]` on any failure rather than throwing. The link editor renders "no other
403
- * databases yet" for an empty list, which is the truthful reading of both an empty workspace
404
- * and an unreachable server β€” and a create pane that explodes because a picker could not fetch
405
- * is worse than one that says it has nothing to offer.
406
- */
407
- export async function fetchLinkTargets(): Promise<LinkTarget[]> {
408
- try {
409
- const res = await fetch(`${API_V1}/tables`, { credentials: CREDENTIALS });
410
- if (handledSession(res) || !res.ok) return [];
411
- const body = (await readJson(res)) as { tables?: LinkTarget[] } | null;
412
- return Array.isArray(body?.tables) ? body.tables : [];
413
- } catch {
414
- return [];
415
- }
416
- }
417
-
418
- export interface LinkTarget {
419
- key: string;
420
- label: string;
421
- fields: Field[];
422
- }
423
-
424
- export async function patchTableField(
425
- tableKey: string,
426
- fieldKey: string,
427
- patch: Record<string, unknown>
428
- ): Promise<Field | null> {
429
- try {
430
- const res = await fetch(
431
- `${API_V1}/tables/${encodeURIComponent(tableKey)}/fields/${encodeURIComponent(fieldKey)}`,
432
- { method: "PATCH", credentials: CREDENTIALS, headers: JSON_HEADERS,
433
- body: JSON.stringify(patch) }
434
- );
435
- if (handledSession(res)) return null;
436
- const body = (await readJson(res)) as
437
- | { field?: Field; detail?: { message?: string } }
438
- | null;
439
- if (!res.ok) {
440
- signal(TOAST_EVENT, typeof body?.detail?.message === "string"
441
- ? body.detail.message
442
- : `The server answered ${res.status}.`);
443
- return null;
444
- }
445
- return body?.field ?? null;
446
- } catch {
447
- signal(TOAST_EVENT, "That change did not reach the server.");
448
- return null;
449
- }
450
- }
451
-
452
- export async function addTableField(
453
- tableKey: string,
454
- field: { key: string; label: string; type: string }
455
- ): Promise<Field | null> {
456
- try {
457
- const res = await fetch(`${API_V1}/tables/${encodeURIComponent(tableKey)}/fields`, {
458
- method: "POST",
459
- credentials: CREDENTIALS,
460
- headers: JSON_HEADERS,
461
- body: JSON.stringify(field),
462
- });
463
- if (handledSession(res)) return null;
464
- const body = (await readJson(res)) as
465
- | { field?: Field; detail?: { message?: string } }
466
- | null;
467
- if (!res.ok) {
468
- // The server states WHY β€” the column cap, an unknown type (this is the 400 to expect
469
- // until `UT_FIELD_TYPES` learns `automation`, C2's other half), a wall refusal. Its
470
- // sentence beats anything invented here.
471
- const why = typeof body?.detail?.message === "string"
472
- ? body.detail.message
473
- : `The server answered ${res.status}.`;
474
- signal(TOAST_EVENT, why);
475
- return null;
476
- }
477
- const made = body?.field;
478
- if (!made || typeof made.key !== "string") {
479
- signal(DATA_ERROR_EVENT, "The server did not say which column it created.");
480
- return null;
481
- }
482
- // The definition feeds BOTH payloads for a user table (`ut_assembly` builds the rows
483
- // envelope and `/workspace` from the same `fields_base`), so the cached rows are now a
484
- // schema behind. Cheap to drop, and a stale schema is not a stale number β€” it is a
485
- // column that exists on the server and nowhere on screen.
486
- rowsCache.delete(`tables/${tableKey}/rows`);
487
- return made;
488
- } catch {
489
- signal(DATA_ERROR_EVENT, "Cannot reach the server.");
490
- return null;
491
- }
492
- }
493
-
494
- /**
495
- * ⭐ WAVE 21 item 11 (ruling R10, contract C5) β€” parse an uploaded `.xlsx`/`.csv` and
496
- * hand back its COLUMNS AND VALUES as strings.
497
- *
498
- * β›” THE FILE NEVER BECOMES DATA. This reads a spreadsheet to find out which records the
499
- * user means; nothing here writes a row, creates a field, or keeps the file. The server
500
- * parses (openpyxl is not a thing a browser has) and answers with strings β€” the matching
501
- * itself is client-side, over rows this browser already holds (`fileSelect.ts`).
502
- *
503
- * ⚠ `truncated` IS THE CONTRACT'S POINT. C5 caps a column at 20k values, and a cap the
504
- * caller cannot see is a silent wrong answer: a 30k-row file would report ten thousand
505
- * records "not found" that are simply past the cap. The dialog surfaces this flag as its
506
- * own sentence, never folded into the miss count ([[no-unverifiable-aggregates]]).
507
- */
508
- export interface TabularUpload {
509
- columns: string[];
510
- rows: number;
511
- values: Record<string, string[]>;
512
- truncated: boolean;
513
- }
514
-
515
- export async function uploadTabular(file: File): Promise<TabularUpload | null> {
516
- const form = new FormData();
517
- form.append("file", file);
518
- try {
519
- // ⚠ NO Content-Type HEADER. The browser must set it, because only the browser knows
520
- // the multipart boundary it generated β€” writing `multipart/form-data` by hand omits
521
- // the boundary and the server cannot parse the body at all.
522
- const res = await fetch(`${API_V1}/uploads/tabular`, {
523
- method: "POST",
524
- credentials: CREDENTIALS,
525
- body: form,
526
- });
527
- if (handledSession(res)) return null;
528
- const body = (await readJson(res)) as
529
- | { columns?: unknown; rows?: unknown; values?: unknown; truncated?: unknown;
530
- detail?: { message?: string } }
531
- | null;
532
- if (!res.ok) {
533
- const why = typeof body?.detail?.message === "string"
534
- ? body.detail.message
535
- : `The file could not be read (the server answered ${res.status}).`;
536
- signal(TOAST_EVENT, why);
537
- return null;
538
- }
539
- const columns = Array.isArray(body?.columns)
540
- ? body.columns.filter((c): c is string => typeof c === "string")
541
- : [];
542
- const rawValues = (body?.values && typeof body.values === "object" ? body.values : {}) as
543
- Record<string, unknown>;
544
- const values: Record<string, string[]> = {};
545
- for (const col of columns)
546
- values[col] = Array.isArray(rawValues[col])
547
- ? (rawValues[col] as unknown[]).map((v) => (v === null || v === undefined ? "" : String(v)))
548
- : [];
549
- if (!columns.length) {
550
- signal(TOAST_EVENT, "That file has no readable columns.");
551
- return null;
552
- }
553
- return {
554
- columns,
555
- rows: typeof body?.rows === "number" ? body.rows : 0,
556
- values,
557
- truncated: body?.truncated === true,
558
- };
559
- } catch {
560
- signal(DATA_ERROR_EVENT, "Cannot reach the server.");
561
- return null;
562
- }
563
- }
564
-
565
- /** C-UNDO's other half: drop a row this session just added. Same cache rule as the append. */
566
- export async function deleteTableRow(
567
- tableKey: string,
568
- rid: string | number
569
- ): Promise<boolean> {
570
- try {
571
- const res = await fetch(
572
- `${API_V1}/tables/${encodeURIComponent(tableKey)}/rows/${encodeURIComponent(String(rid))}`,
573
- { method: "DELETE", credentials: CREDENTIALS }
574
- );
575
- if (handledSession(res)) return false;
576
- if (res.ok) rowsCache.delete(`tables/${tableKey}/rows`);
577
- return res.ok;
578
- } catch {
579
- return false;
580
- }
581
- }
582
-
583
- /**
584
- * Item 7 (contract C-TS) β€” `POST /api/v1/grid/timeseries`.
585
- *
586
- * A READ that is a POST, because the request body carries a pid list that can run to
587
- * thousands: a URL cannot hold the caller's book, and putting customer ids in a query string
588
- * would also put them in every access log. The server intersects those pids with the caller's
589
- * `allowed_pids` regardless, so the body narrows the answer and can never widen it.
590
- *
591
- * ⚠ RETURNS `null` ON EVERY FAILURE, AND THE PANEL MUST SAY SO RATHER THAN DRAW ZEROES.
592
- * That is the difference between this and a payload of empty buckets: "we could not ask" and
593
- * "the answer is nothing" look identical on a chart and are not the same claim. A 400 is the
594
- * honest "narrow the span"; a 403 is `out_of_scope` (every requested pid was outside the
595
- * caller's book); a 401 ends the session like everywhere else.
596
- */
597
- export async function fetchTimeseries(
598
- body: TsRequest,
599
- scope: SurfaceScope = surfaceScope
600
- ): Promise<{ payload: TsPayload | null; status: number }> {
601
- try {
602
- const res = await fetch(
603
- `${API_V1}/grid/timeseries?scope=${encodeURIComponent(scope)}`,
604
- {
605
- method: "POST",
606
- credentials: CREDENTIALS,
607
- headers: JSON_HEADERS,
608
- body: JSON.stringify(body),
609
- }
610
- );
611
- if (handledSession(res)) return { payload: null, status: res.status };
612
- if (!res.ok) return { payload: null, status: res.status };
613
- const json = (await readJson(res)) as TsPayload | null;
614
- // Shape-check before trusting it: `columns` and `rows` are indexed POSITIONALLY against
615
- // each other, so a payload missing either would index into undefined all the way down.
616
- if (!json || !Array.isArray(json.columns) || !Array.isArray(json.rows))
617
- return { payload: null, status: res.status };
618
- return { payload: json, status: res.status };
619
- } catch {
620
- return { payload: null, status: 0 };
621
- }
622
- }
623
-
624
- /**
625
- * wave17 owner item 9 / ruling R4 (contract C-CAL) β€” per-DAY measure values.
626
- *
627
- * β›” WHY A SECOND CHANNEL rather than a parameter on the time-series one: **every group carries
628
- * its OWN pid set.** A calendar day holds whatever records the date field placed there, so the
629
- * subject changes from cell to cell; one `allowed_pids` for the whole request would compute
630
- * every day over everybody, which is a different question with the same shape.
631
- *
632
- * The caps are the SERVER's and it answers a 400 with its reason rather than trimming: 31 days,
633
- * 6 fields, 186 cells. The client's job is to send no more than that, never to guess what was
634
- * dropped. `values[field][day] === null` is UNANSWERED β€” a day past the tenant's today, a day
635
- * whose pids are all outside the reader's book, or a measure that could not resolve β€” and the
636
- * renderer draws "β€”" for all three. Never 0.
637
- */
638
- export interface CalMetricsResponse {
639
- values: Record<string, Record<string, number | null>>;
640
- today?: string;
641
- /** C-TSWIN at day grain: which window each metric was resolved under. */
642
- windows?: Record<string, { kind: string; label: string }>;
643
- dropped?: { field: string; reason: string }[];
644
- problems?: string[];
645
- }
646
-
647
- export async function fetchCalendarMetrics(
648
- body: { groups: { key: string; pids: number[] }[]; fields: string[] },
649
- scope: SurfaceScope = surfaceScope
650
- ): Promise<{ payload: CalMetricsResponse | null; status: number }> {
651
- try {
652
- const res = await fetch(
653
- `${API_V1}/grid/calendar_metrics?scope=${encodeURIComponent(scope)}`,
654
- {
655
- method: "POST",
656
- credentials: CREDENTIALS,
657
- headers: JSON_HEADERS,
658
- body: JSON.stringify(body),
659
- }
660
- );
661
- if (handledSession(res)) return { payload: null, status: res.status };
662
- if (!res.ok) return { payload: null, status: res.status };
663
- const json = (await readJson(res)) as CalMetricsResponse | null;
664
- // Shape-checked before it is trusted, the `fetchTimeseries` rule: every lookup below is
665
- // `values[field][day]`, so a payload without `values` would read `undefined` as a value and
666
- // paint blanks that look like real "β€”" refusals.
667
- if (!json || !json.values || typeof json.values !== "object")
668
- return { payload: null, status: res.status };
669
- return { payload: json, status: res.status };
670
- } catch {
671
- return { payload: null, status: 0 };
672
- }
673
- }
674
-
675
- // --- the WRITE (CP-C) ------------------------------------------------------
676
-
677
- /**
678
- * The replay bound. The SAME 24 the Streamlit value slot uses, for a different
679
- * reason: there, the window exists because one slot gets clobbered by a second
680
- * write; here, it bounds how much a failed POST may carry forward. Both rest on
681
- * the same server guarantee β€” X2 dedups by event id β€” so re-sending is free and
682
- * losing an event is not.
683
- */
684
- export const MAX_REPLAY = 24;
685
-
686
- let pending: HostEvent[] = [];
687
- let inFlight = false;
688
-
689
- /**
690
- * Which SURFACE this grid is drawing β€” `customer` (the whole scoped pool),
691
- * `cohort` (hand-curated sets) or `product` (the SKU table β€” wave 16 C-TOPIC).
692
- * It lives at module scope beside the queue on purpose: the queue is already
693
- * module-level, and an event must carry the scope it was CREATED under even if
694
- * it drains after a route change.
695
- *
696
- * ⚠ The server refuses an unknown value rather than defaulting it, so this is
697
- * never a free-text field. Read and write must agree on what a scope is: the
698
- * grid asks `/workspace?scope=X` and every event it then emits says `X`.
699
- */
700
- export type SurfaceScope = "customer" | "cohort" | "product" | `ut_${string}`;
701
- let surfaceScope: SurfaceScope = "customer";
702
-
703
- export function setSurfaceScope(scope: SurfaceScope): void {
704
- surfaceScope = scope;
705
- }
706
-
707
- /** For the gate: the queue is module state and a test needs a known start. */
708
- export function _resetQueue(): void {
709
- pending = [];
710
- inFlight = false;
711
- }
712
-
713
- export function _pendingIds(): string[] {
714
- return pending.map((e) => e.id);
715
- }
716
-
717
- /**
718
- * ONE request in flight at a time, and the queue drains in order.
719
- *
720
- * ORDER IS NOT COSMETIC: `field_upsert` then `view_upsert` is a field that
721
- * exists and a view that shows it; the other way round is a view referencing a
722
- * column the store has never heard of. Parallel POSTs would race exactly there.
723
- *
724
- * ⚠ A FAILED BATCH IS REQUEUED BUT NOT RETRIED ON A TIMER. It goes back at the
725
- * FRONT (it happened first) and rides the next emit. A self-scheduling retry
726
- * against a server that is down is a tight loop, and a timer is a background
727
- * behaviour nobody asked for; the next user action is a perfectly good clock.
728
- * The cost is bounded and known: with no further action, the last batch stays
729
- * unsent β€” which is strictly better than today, where standalone drops every
730
- * event on the floor.
731
- */
732
- async function drain(): Promise<void> {
733
- if (inFlight || pending.length === 0) return;
734
- inFlight = true;
735
- const batch = pending;
736
- pending = [];
737
- let requeue = false;
738
- try {
739
- const res = await fetch(`${API_V1}/grid/events`, {
740
- method: "POST",
741
- credentials: CREDENTIALS,
742
- headers: JSON_HEADERS,
743
- body: JSON.stringify({ events: batch, scopeKey: surfaceScope }),
744
- });
745
- // A dead session is NOT retryable β€” replaying into a 401 forever would
746
- // turn one expired cookie into an unbounded request loop.
747
- if (!handledSession(res)) {
748
- if (res.ok) applyEventResult(await readJson(res));
749
- else requeue = true;
750
- }
751
- } catch {
752
- requeue = true;
753
- } finally {
754
- inFlight = false;
755
- if (requeue) {
756
- // Front, and bounded: events that arrived during the flight are NEWER.
757
- pending = [...batch, ...pending].slice(-MAX_REPLAY);
758
- } else if (pending.length) {
759
- void drain();
760
- }
761
- }
762
- }
763
-
764
- /**
765
- * X2's `{results:[{id, rerender}], doc?, toast?}`.
766
- *
767
- * `results` needs nothing: the client is already optimistic and the server's
768
- * per-event ack carries no correction. `toast` is surfaced β€” for the
769
- * echo-dependent events it is the only feedback that exists in standalone.
770
- * `doc` is deliberately unhandled; see this file's header.
771
- */
772
- function applyEventResult(body: unknown): void {
773
- const b = body as {
774
- toast?: unknown; rerender?: unknown; results?: unknown; derived?: unknown;
775
- } | null;
776
- const toast = b?.toast;
777
- if (typeof toast === "string" && toast.trim() !== "") signal(TOAST_EVENT, toast.trim());
778
-
779
- // owner item 2 β€” a measure column's values, computed server-side right after the write and
780
- // returned here rather than waiting for the `/workspace` re-read below. Raised BEFORE the
781
- // stale signal so the numbers paint on the earlier of the two, whichever the server sent.
782
- // Shape-checked, because a shortcut that corrupts rows is worse than no shortcut.
783
- if (b?.derived && typeof b.derived === "object" && !Array.isArray(b.derived))
784
- signal(DERIVED_CELLS_EVENT, b.derived);
785
-
786
- // The server already tells us when a write changed durable state β€” `rerender`
787
- // is exactly the flag the Streamlit adapter uses to trigger its own rerun. We
788
- // were dropping it, which is why cohort membership, list adds and folder moves
789
- // never appeared in standalone until a manual reload.
790
- const results = Array.isArray(b?.results) ? (b.results as { rerender?: unknown }[]) : [];
791
- if (b?.rerender === true || results.some((r) => r?.rerender === true)) {
792
- signal(WORKSPACE_STALE_EVENT);
793
- }
794
- }
795
-
796
- /** The sink hostBridge falls through to in standalone. Returns true: the event
797
- * is now OURS (queued and owned), which is all the caller ever needed. */
798
- function standaloneSink(event: HostEvent): boolean {
799
- pending.push(event);
800
- if (pending.length > MAX_REPLAY) pending.shift();
801
- void drain();
802
- return true;
803
- }
804
-
805
- /** Called from main.tsx in standalone ONLY. Never in the embed β€” that is the
806
- * whole of "standalone-only branches", enforced at the one wiring point. */
807
- export function installStandaloneBridge(): void {
808
- setStandaloneSink(standaloneSink);
809
- }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ // ---------------------------------------------------------------------------
2
+ // customer-grid / apiBridge.ts β€” the STANDALONE half of the data path (X2).
3
+ //
4
+ // The branch beside hostBridge.ts. Where `hostBridge` speaks the Streamlit
5
+ // Components v1 protocol to a surrounding Python app, this speaks HTTP to the
6
+ // API β€” the SAME event objects, the same ids, the same optimism semantics on
7
+ // the client. Which one is live is decided once, by mode, and never mixed:
8
+ // `installStandaloneBridge()` is called from main.tsx ONLY in standalone, so
9
+ // the embed bundle never registers a sink and its behaviour is byte-unchanged
10
+ // (pinned by verify_bridge.py, which was written BEFORE this file existed).
11
+ //
12
+ // ⚠ WHAT THIS WAVE'S WRITE PATH CAN AND CANNOT DO β€” read before extending.
13
+ // The client's whole write model is "emit β†’ the host reruns β†’ a fresh payload
14
+ // arrives β†’ the UI reflects it" (CustomerGrid's own words: *"the emit itself
15
+ // triggers the host rerun, so the fresh name/list arrives with the next payload
16
+ // β€” no local mutation to drift from the store"*). Standalone has no rerun and
17
+ // X2 fixes `GET /api/v1/customers` at the EXACT current shape β€” `{fields, rows,
18
+ // today, pulled_at}`, with no `workspace`, no `cohorts`, no `docs`. So:
19
+ //
20
+ // VISIBLE + DURABLE view_upsert Β· view_delete Β· field_* Β· overlay_patch
21
+ // (the client already holds these in state + localStorage;
22
+ // the POST is what makes them durable server-side)
23
+ // DURABLE, INVISIBLE cohort_* Β· add_to_list Β· folder_* Β· item_move
24
+ // (they reach the store, but nothing re-reads them until
25
+ // the payload carries that state β€” EXIT-5). The server's
26
+ // `toast` is the only feedback the user gets, which is
27
+ // why it is wired through and not dropped.
28
+ // NOT REACHABLE doc_add / doc_fetch / doc_delete β€” the Documents panel
29
+ // is driven by `payload.docs`, which standalone never
30
+ // receives, so no doc event can be emitted at all. X2's
31
+ // `doc` field is therefore deliberately NOT handled here:
32
+ // wiring a response nobody can trigger is untestable dead
33
+ // code. When `/customers` grows `docs`, feed `doc` into
34
+ // the same `payload.docPayload` slot the embed uses.
35
+ //
36
+ // NOTHING HERE FALLS BACK TO `sample_customers.json`. That affordance ("the
37
+ // grid still renders with no backend") is unreachable now β€” the frame requires
38
+ // a session before CustomerGrid mounts at all β€” and behind a login, sample
39
+ // revenue rendered after a server hiccup is fabricated data on a screen the
40
+ // user has every reason to trust. An honest failure, always.
41
+ // ---------------------------------------------------------------------------
42
+
43
+ import {
44
+ API_V1,
45
+ CREDENTIALS,
46
+ DATA_ERROR_EVENT,
47
+ DERIVED_CELLS_EVENT,
48
+ ROWS_STALE_EVENT,
49
+ TOAST_EVENT,
50
+ WORKSPACE_STALE_EVENT,
51
+ UNAUTHORIZED_EVENT,
52
+ checkTenant,
53
+ signal,
54
+ } from "../apiContract";
55
+ import { setStandaloneSink } from "./hostBridge";
56
+ import { CUSTOMER_TOPIC } from "./types";
57
+ import type { CustomersPayload, Field, GridWorkspace, HostEvent, Row, TopicConfig } from "./types";
58
+ import type { TsPayload, TsRequest } from "./timeSeriesData";
59
+
60
+ const JSON_HEADERS = { "Content-Type": "application/json" };
61
+
62
+ async function readJson(res: Response): Promise<unknown> {
63
+ try {
64
+ return await res.json();
65
+ } catch {
66
+ return null;
67
+ }
68
+ }
69
+
70
+ /** 401 is the one status that is never a data problem. Raised once, centrally,
71
+ * so every call site cannot forget to. Returns true when it handled it.
72
+ * The rows caches die with the session β€” the next signed-in user may not
73
+ * be the same person, and a cached book must never cross that boundary. */
74
+ function handledUnauthorized(status: number): boolean {
75
+ if (status !== 401) return false;
76
+ rowsCache.clear();
77
+ signal(UNAUTHORIZED_EVENT);
78
+ return true;
79
+ }
80
+
81
+ /**
82
+ * The session check every authenticated call makes: the 401 above, PLUS the TENANT swap the
83
+ * 401 path structurally cannot see.
84
+ *
85
+ * β›” WHY `handledUnauthorized` WAS NEVER ENOUGH. Its own note says the rows cache "dies with the
86
+ * session β€” the next signed-in user may not be the same person". That is the USER boundary, and
87
+ * it is enforced on 401. A TENANT swap produces no 401 at all: `aios_session` is one cookie per
88
+ * ORIGIN, so signing into another tenant in a second tab repoints this one, and every request
89
+ * here keeps returning a cheerful 200 full of somebody else's data. `checkTenant` reads the
90
+ * server's own stamp and reloads the frame; see `apiContract.ts::TENANT_HEADER`.
91
+ */
92
+ function handledSession(res: Response): boolean {
93
+ if (checkTenant(res)) return true;
94
+ return handledUnauthorized(res.status);
95
+ }
96
+
97
+ // --- the READ (CP-B) -------------------------------------------------------
98
+
99
+ /**
100
+ * The last good `/customers` payload, reused across surface switches. The rows
101
+ * are a 15-minute-cached Odoo pull SERVER-side, but every Customer⇄Cohort
102
+ * route change remounts the grid and re-downloaded the whole ~1 MB payload β€”
103
+ * three full transfers in one short session, measured live. Within this window
104
+ * a remount reuses the copy; the workspace (the cheap per-scope call) is always
105
+ * re-fetched, so views/cohorts stay live.
106
+ *
107
+ * β›” THE CACHE IS SESSION-SCOPED OR IT IS A LEAK. It is cleared on ANY 401 and
108
+ * on sign-out/sign-in (Shell calls `clearCustomersCache`) β€” one browser, two
109
+ * accounts, and a cached book served across the boundary would be the exact
110
+ * cross-user leak EXIT-3b exists to prevent, client-side this time.
111
+ *
112
+ * Own edits stay visible ([[the NO-BLIP law]]): `patchCustomer` writes accepted
113
+ * values through into the cached rows, so a scope switch after an edit shows
114
+ * the edit, not the pre-edit pull.
115
+ */
116
+ const CUSTOMERS_FRESH_MS = 5 * 60_000;
117
+ // Wave 16 C-TOPIC: ONE cache map keyed by the topic's rows path, so the product pull and the
118
+ // customer pull each get the same 5-minute remount window without ever serving each other.
119
+ const rowsCache = new Map<string, { at: number; payload: CustomersPayload }>();
120
+
121
+ export function clearCustomersCache(): void {
122
+ rowsCache.clear();
123
+ }
124
+
125
+ /**
126
+ * X2 `GET /api/v1/<topic.rowsPath>` β†’ `{fields, rows, today, pulled_at}`, rows already
127
+ * scoped to the session's BU pool (EXIT-3b). The cookie replaces HTTP Basic.
128
+ *
129
+ * Returns null on ANY failure, having raised the matching signal. A null is not
130
+ * an empty table: the frame renders a failure, not "this tenant has no
131
+ * records", because those two look identical and only one of them is true.
132
+ */
133
+ export async function fetchTopicRows(topic: TopicConfig): Promise<CustomersPayload | null> {
134
+ const cached = rowsCache.get(topic.rowsPath);
135
+ if (cached && Date.now() - cached.at < CUSTOMERS_FRESH_MS) {
136
+ return cached.payload;
137
+ }
138
+ let res: Response;
139
+ try {
140
+ res = await fetch(`${API_V1}/${topic.rowsPath}`, { credentials: CREDENTIALS });
141
+ } catch {
142
+ signal(DATA_ERROR_EVENT, "Cannot reach the server.");
143
+ return null;
144
+ }
145
+ if (handledSession(res)) return null;
146
+ if (!res.ok) {
147
+ signal(DATA_ERROR_EVENT, `The server answered ${res.status}.`);
148
+ return null;
149
+ }
150
+ const body = (await readJson(res)) as CustomersPayload | null;
151
+ if (!body || !Array.isArray(body.rows) || !Array.isArray(body.fields)) {
152
+ signal(DATA_ERROR_EVENT, "The server sent an unreadable payload.");
153
+ return null;
154
+ }
155
+ rowsCache.set(topic.rowsPath, { at: Date.now(), payload: body });
156
+ return body;
157
+ }
158
+
159
+ /** The customer topic's fetch β€” kept under its own name because half the write-path notes in
160
+ * this repo cite it; it IS `fetchTopicRows(CUSTOMER_TOPIC)`. */
161
+ export function fetchCustomers(): Promise<CustomersPayload | null> {
162
+ return fetchTopicRows(CUSTOMER_TOPIC);
163
+ }
164
+
165
+ /**
166
+ * `GET /api/v1/workspace` β€” the saved views, custom fields, folders and cohorts
167
+ * for the session user (S1's amendment, 2026-07-30). It exists because X2 pins
168
+ * `/customers` at the EXACT current shape, which carries no `workspace`, and
169
+ * without one the standalone write path is WRITE-ONLY: a saved view reaches the
170
+ * store and is gone from the screen on reload.
171
+ *
172
+ * The body is `{workspace: GridWorkspace}` β€” the client's own built type
173
+ * (types.ts `GridWorkspace`), which is what the embed's host already projects.
174
+ *
175
+ * ⚠ ABSENT IS FINE AND MEANS TODAY'S BEHAVIOUR. A 404 (route not shipped) or a
176
+ * body carrying no workspace return null and the grid runs exactly as it does
177
+ * now β€” views in memory, no `storageKey`, nothing durable. It degrades to the
178
+ * honest gap rather than failing the whole read, because the workspace is an
179
+ * enhancement of the table, not a precondition for it. A 401 still ends the
180
+ * session: that is the one status that is never about this route.
181
+ *
182
+ * β›” WAVE 21 item 3 (3c) β€” BUT A REFUSAL IS NOT AN ABSENCE, and conflating the
183
+ * two is half of the "RI fields on a new database" report.
184
+ *
185
+ * This used to answer `null` to every unhappy path alike: route-not-shipped,
186
+ * 403, 500, connection refused, garbage body. The caller cannot tell those
187
+ * apart, so it did the only thing it could β€” carried on without a workspace β€”
188
+ * and `CustomerGrid` then had no `storageKey` and fell back to a bucket shared
189
+ * with every other surface (see the note at its `storageKey`). A tenant whose
190
+ * `/workspace` 403s for `ut_*` therefore got ANOTHER TABLE'S views and fields,
191
+ * silently, with the grid looking entirely healthy.
192
+ *
193
+ * `announceFailure` is the caller saying "I am the INITIAL load; if this fails
194
+ * for a reason that is not 'no workspace here', say so". It raises the same
195
+ * `DATA_ERROR_EVENT` a failed rows read raises, so the frame renders its honest
196
+ * failure card instead of a grid furnished with somebody else's schema.
197
+ *
198
+ * ⚠ IT IS OFF BY DEFAULT, and that is load-bearing rather than cautious.
199
+ * `reread()` calls this on every workspace-stale signal, and its own rule is
200
+ * "absent stays absent β€” never blank a live panel"; a signal from there would
201
+ * let one dropped packet replace a working table with an error card. The flag
202
+ * is passed explicitly at the one call site that owns the first paint.
203
+ */
204
+ export async function fetchWorkspace(
205
+ scope: SurfaceScope = "customer",
206
+ opts: { announceFailure?: boolean } = {},
207
+ ): Promise<GridWorkspace | null> {
208
+ const loud = opts.announceFailure === true;
209
+ let res: Response;
210
+ try {
211
+ res = await fetch(`${API_V1}/workspace?scope=${encodeURIComponent(scope)}`,
212
+ { credentials: CREDENTIALS });
213
+ } catch {
214
+ if (loud) signal(DATA_ERROR_EVENT, "Cannot reach the server.");
215
+ return null;
216
+ }
217
+ if (handledSession(res)) return null;
218
+ if (!res.ok) {
219
+ // 404 is the ONE status that means "this host does not serve a workspace",
220
+ // which is the enhancement posture above and stays quiet. Everything else
221
+ // is a server that had an answer and would not give it.
222
+ if (loud && res.status !== 404)
223
+ signal(DATA_ERROR_EVENT,
224
+ `The server answered ${res.status} for this table's saved views and columns.`);
225
+ return null;
226
+ }
227
+ const ws = (await readJson(res)) as { workspace?: GridWorkspace } | null;
228
+ const w = ws?.workspace;
229
+ // A 200 carrying no workspace is still an ABSENCE, not a refusal β€” an older
230
+ // host, or a scope this one has nothing stored for. Unchanged, and quiet.
231
+ return w && typeof w.storageKey === "string" && Array.isArray(w.views) ? w : null;
232
+ }
233
+
234
+ /**
235
+ * X2 `PATCH /api/v1/<topic.rowsPath>/{pid}` β€” the overlay stratum only (Odoo is never
236
+ * written). Kept as its own route rather than folded into the events log
237
+ * because X2 keeps it: cell edits are the one write with a per-row identity and
238
+ * a rollback the caller already implements.
239
+ */
240
+ export async function patchTopicRow(
241
+ topic: TopicConfig,
242
+ pid: number,
243
+ updates: Partial<Row>,
244
+ /**
245
+ * ⭐ Wave-25 (C3/R6) β€” what the SERVER says this row now holds, for callers that keep an
246
+ * optimistic copy. Two things a cell PATCH can now report that the request cannot predict:
247
+ * Β· a CANONICALISED value β€” a profile column stores `nurilab` for `@Nurilab` or a pasted
248
+ * profile URL, so the optimistic copy is a value the store never took;
249
+ * Β· CLEARED cells β€” blanking a profile handle also clears that row's enriched columns, and
250
+ * the client never typed those, so nothing else would ever repaint them.
251
+ * An optional callback rather than a richer return type, deliberately: the boolean IS the
252
+ * rollback contract every existing caller is written against, and widening it would make the
253
+ * two call sites that ignore this the ones most likely to get it wrong.
254
+ */
255
+ onAccepted?: (accepted: Partial<Row>, cleared: string[]) => void,
256
+ ): Promise<boolean> {
257
+ try {
258
+ const res = await fetch(`${API_V1}/${topic.rowsPath}/${pid}`, {
259
+ method: "PATCH",
260
+ credentials: CREDENTIALS,
261
+ headers: JSON_HEADERS,
262
+ body: JSON.stringify(updates),
263
+ });
264
+ if (handledSession(res)) return false;
265
+ const cached = rowsCache.get(topic.rowsPath);
266
+ if (res.ok) {
267
+ // Write the ACCEPTED values (the server's own report, not the request)
268
+ // through into the cached rows, so a remount within the cache window
269
+ // shows this edit instead of the pre-edit pull.
270
+ const body = (await readJson(res)) as
271
+ { updates?: Record<string, unknown>; cleared?: unknown } | null;
272
+ const accepted = body?.updates;
273
+ const cleared = Array.isArray(body?.cleared)
274
+ ? (body.cleared as unknown[]).filter((k): k is string => typeof k === "string")
275
+ : [];
276
+ const blanks: Partial<Row> = {};
277
+ for (const k of cleared) blanks[k] = "";
278
+ if (accepted && typeof accepted === "object") {
279
+ const row = cached?.payload.rows.find((r) => r.pid === pid);
280
+ if (row) Object.assign(row, accepted, blanks);
281
+ onAccepted?.({ ...(accepted as Partial<Row>), ...blanks }, cleared);
282
+ }
283
+ // A user-table edit may change a reciprocal Link or a Rollup in another database. Their
284
+ // row payloads share no cache key with this table, so invalidate the user-table family and
285
+ // let the active grid re-read the server's materialised relationship cells.
286
+ if (topic.rowsPath.startsWith("tables/")) {
287
+ for (const key of [...rowsCache.keys()])
288
+ if (key.startsWith("tables/")) rowsCache.delete(key);
289
+ signal(ROWS_STALE_EVENT);
290
+ }
291
+ }
292
+ return res.ok;
293
+ } catch {
294
+ return false;
295
+ }
296
+ }
297
+
298
+ /** The customer topic's patch, by its historical name. */
299
+ export function patchCustomer(pid: number, updates: Partial<Row>): Promise<boolean> {
300
+ return patchTopicRow(CUSTOMER_TOPIC, pid, updates);
301
+ }
302
+
303
+ /**
304
+ * ⭐ Wave-20 owner item 4 / contract C-ADDROW β€” **APPEND A ROW TO A USER DATABASE**, and
305
+ * (C-UNDO) restore a deleted one under its old id.
306
+ *
307
+ * USER TABLES ONLY, and the refusal is structural rather than checked here: no other scope has
308
+ * a `/tables/{key}/rows` endpoint at all. A connector's rows are read-synced from its source β€”
309
+ * R8 is explicit that a "+" which must refuse is a fake affordance, so the caller never renders
310
+ * one there.
311
+ *
312
+ * ⚠ THE ANSWER IS THE ID THAT WAS STORED, never the one that was asked for. An undo that
313
+ * requests `rid` may find that id re-used, and the server's own note says it answers with what
314
+ * it actually wrote; the caller re-anchors on the returned value rather than assuming.
315
+ *
316
+ * ⚠ The rows cache is CLEARED on success. `fetchTopicRows` holds a 5-minute window per topic,
317
+ * so a re-read straight after an append would serve the payload from before it β€” the new row
318
+ * would appear minutes later, which reads as "the button did nothing". Same reason the shell's
319
+ * retired Add-record bar cleared it (wave 20 item 4 moved that door here).
320
+ */
321
+ export async function addTableRow(
322
+ tableKey: string,
323
+ values: Record<string, unknown> = {},
324
+ rid?: string | number
325
+ ): Promise<{ rid: string | number; pid: number } | null> {
326
+ try {
327
+ const res = await fetch(`${API_V1}/tables/${encodeURIComponent(tableKey)}/rows`, {
328
+ method: "POST",
329
+ credentials: CREDENTIALS,
330
+ headers: JSON_HEADERS,
331
+ body: JSON.stringify(rid === undefined ? { values } : { rid, values }),
332
+ });
333
+ if (handledSession(res)) return null;
334
+ const body = (await readJson(res)) as { rid?: string | number; pid?: number;
335
+ detail?: { message?: string } } | null;
336
+ if (!res.ok) {
337
+ // The server states WHY (a row cap, a store refusal). Surfacing its sentence beats a
338
+ // generic failure toast, and staying silent would be the worst of the three.
339
+ const why = typeof body?.detail?.message === "string"
340
+ ? body.detail.message
341
+ : `The server answered ${res.status}.`;
342
+ signal(TOAST_EVENT, why);
343
+ return null;
344
+ }
345
+ if (body?.rid === undefined || typeof body?.pid !== "number") {
346
+ signal(DATA_ERROR_EVENT, "The server did not say which row it created.");
347
+ return null;
348
+ }
349
+ rowsCache.delete(`tables/${tableKey}/rows`);
350
+ return { rid: body.rid, pid: body.pid };
351
+ } catch {
352
+ signal(DATA_ERROR_EVENT, "Cannot reach the server.");
353
+ return null;
354
+ }
355
+ }
356
+
357
+ /**
358
+ * ⭐ WAVE 21 item 7 (contract C2) β€” ADD A COLUMN TO A USER DATABASE'S **DEFINITION**,
359
+ * not to this user's overlay.
360
+ *
361
+ * β›” THE BUG THIS EXISTS FOR. Every column the grid creates has always gone out as a
362
+ * `field_upsert` EVENT, which lands in the per-user workspace stratum
363
+ * (`<key>_table_workspace`). That is right for the connector surfaces, where a custom
364
+ * column IS one person's annotation of somebody else's data. It is wrong for a user
365
+ * database, where the columns ARE the table β€” and it is why the automation editor's
366
+ * "Automation column" picker was empty by construction: that picker reads the DEFINITION
367
+ * (`user_tables`), which the grid had never written to. Two stores, one word, and the two
368
+ * surfaces disagreed with no error anywhere. The route has existed since wave 18 with no
369
+ * client caller; this is that caller, and `routes_tables.py`'s own comment above it names
370
+ * this exact defect.
371
+ *
372
+ * ⚠ SCOPED TO THE AUTOMATION KIND THIS WAVE (C2, deliberately narrow). Moving EVERY kind
373
+ * onto the definition is the right end state and is booked as debt: it changes who can see
374
+ * a column (everyone with the table, not just its author), which is a visible change to
375
+ * shipped behaviour on three surfaces and does not belong in a wave that is fixing this
376
+ * one's blindness.
377
+ *
378
+ * ⚠ THE SERVER'S FIELD IS THE ANSWER, never the request. `add_field` re-slugs and
379
+ * re-types on the way in, so the caller inserts what came BACK β€” a column ordered under a
380
+ * key the store does not hold is a column that renders nowhere.
381
+ */
382
+ /**
383
+ * ⭐⭐ 2026-08-07 (D-79's last half) β€” PATCH one column's DEFINITION on a user table.
384
+ *
385
+ * β›” WHY IT EXISTS: wave 25 shipped the profile flag's READER everywhere β€” the column-menu line,
386
+ * the cell validator, R6's clear-on-write, the one-per-table refusal β€” and never its WRITER. The
387
+ * route (`PATCH /tables/{key}/fields/{fkey}`) has accepted `profile` since that wave with no
388
+ * client caller, so the enrich action could not be bound to any database a person made
389
+ * themselves. This is that caller.
390
+ *
391
+ * ⚠ THE SERVER'S FIELD IS THE ANSWER, never the request β€” the same rule `addTableField` carries.
392
+ * `_clean_field` enforces `type: 'text'` for a profile flag and refuses a SECOND one with a
393
+ * sentence naming the column that already has it, so the refusal is worth surfacing verbatim.
394
+ */
395
+ /**
396
+ * ⭐ 2026-08-07 β€” the sibling databases a `link` column may point at, WITH their fields.
397
+ *
398
+ * `GET /tables` already returns exactly this (key, label, fields) filtered by `may_open`, so
399
+ * there is no new route and no new wall: a database you cannot open is a database you cannot
400
+ * link to, decided by the same predicate that decides whether you can see it at all.
401
+ *
402
+ * ⚠ Returns `[]` on any failure rather than throwing. The link editor renders "no other
403
+ * databases yet" for an empty list, which is the truthful reading of both an empty workspace
404
+ * and an unreachable server β€” and a create pane that explodes because a picker could not fetch
405
+ * is worse than one that says it has nothing to offer.
406
+ */
407
+ export async function fetchLinkTargets(): Promise<LinkTarget[]> {
408
+ try {
409
+ const res = await fetch(`${API_V1}/tables`, { credentials: CREDENTIALS });
410
+ if (handledSession(res) || !res.ok) return [];
411
+ const body = (await readJson(res)) as { tables?: LinkTarget[] } | null;
412
+ return Array.isArray(body?.tables) ? body.tables : [];
413
+ } catch {
414
+ return [];
415
+ }
416
+ }
417
+
418
+ export interface LinkTarget {
419
+ key: string;
420
+ label: string;
421
+ fields: Field[];
422
+ }
423
+
424
+ /**
425
+ * ⭐⭐ 2026-08-09 β€” the READ-THROUGH rollup's offer: which governed Odoo topic, which metric of
426
+ * it, grouped by which dimension, over which date window.
427
+ *
428
+ * β›” WHY THIS IS A DIFFERENT LIST FROM `fetchLinkTargets`. A LINK rollup folds rows that live in
429
+ * the workspace, so its offer is "the other databases". A SOURCE rollup folds rows that were
430
+ * never copied here at all β€” 256,810 order lines answered by one grouped SQL query β€” so its
431
+ * offer is the semantic model, and the server derives it from `model/topics/*.yml` +
432
+ * `model/metrics/*.yml` rather than from anything the client knows.
433
+ *
434
+ * ⚠ Returns an EMPTY offer on any failure rather than throwing, for the same reason the link
435
+ * picker does: the editor then renders "no live sources are available", which is the truthful
436
+ * reading of both a tenant with no Odoo mirror and an unreachable server.
437
+ */
438
+ export async function fetchRollupSources(): Promise<RollupSourceOffer> {
439
+ try {
440
+ const res = await fetch(`${API_V1}/tables/rollup-sources`, { credentials: CREDENTIALS });
441
+ if (handledSession(res) || !res.ok) return { topics: [], windows: [] };
442
+ const body = (await readJson(res)) as Partial<RollupSourceOffer> | null;
443
+ return {
444
+ topics: Array.isArray(body?.topics) ? body!.topics! : [],
445
+ windows: Array.isArray(body?.windows) ? body!.windows! : [],
446
+ };
447
+ } catch {
448
+ return { topics: [], windows: [] };
449
+ }
450
+ }
451
+
452
+ export interface RollupSourceOffer {
453
+ topics: RollupSourceTopic[];
454
+ windows: { key: string; label: string }[];
455
+ }
456
+
457
+ export interface RollupSourceTopic {
458
+ key: string;
459
+ label: string;
460
+ grain: string;
461
+ /** `keyedBy` says whether the group key is an Odoo ID or the dimension's own value β€” which is
462
+ * what decides which column on THIS database it should be matched against. */
463
+ dims: { key: string; label: string; keyedBy: "id" | "value" }[];
464
+ measures: { key: string; label: string; format: string; description: string }[];
465
+ }
466
+
467
+ export async function patchTableField(
468
+ tableKey: string,
469
+ fieldKey: string,
470
+ patch: Record<string, unknown>
471
+ ): Promise<Field | null> {
472
+ try {
473
+ const res = await fetch(
474
+ `${API_V1}/tables/${encodeURIComponent(tableKey)}/fields/${encodeURIComponent(fieldKey)}`,
475
+ { method: "PATCH", credentials: CREDENTIALS, headers: JSON_HEADERS,
476
+ body: JSON.stringify(patch) }
477
+ );
478
+ if (handledSession(res)) return null;
479
+ const body = (await readJson(res)) as
480
+ | { field?: Field; detail?: { message?: string } }
481
+ | null;
482
+ if (!res.ok) {
483
+ signal(TOAST_EVENT, typeof body?.detail?.message === "string"
484
+ ? body.detail.message
485
+ : `The server answered ${res.status}.`);
486
+ return null;
487
+ }
488
+ return body?.field ?? null;
489
+ } catch {
490
+ signal(TOAST_EVENT, "That change did not reach the server.");
491
+ return null;
492
+ }
493
+ }
494
+
495
+ export async function addTableField(
496
+ tableKey: string,
497
+ field: { key: string; label: string; type: string }
498
+ ): Promise<Field | null> {
499
+ try {
500
+ const res = await fetch(`${API_V1}/tables/${encodeURIComponent(tableKey)}/fields`, {
501
+ method: "POST",
502
+ credentials: CREDENTIALS,
503
+ headers: JSON_HEADERS,
504
+ body: JSON.stringify(field),
505
+ });
506
+ if (handledSession(res)) return null;
507
+ const body = (await readJson(res)) as
508
+ | { field?: Field; detail?: { message?: string } }
509
+ | null;
510
+ if (!res.ok) {
511
+ // The server states WHY β€” the column cap, an unknown type (this is the 400 to expect
512
+ // until `UT_FIELD_TYPES` learns `automation`, C2's other half), a wall refusal. Its
513
+ // sentence beats anything invented here.
514
+ const why = typeof body?.detail?.message === "string"
515
+ ? body.detail.message
516
+ : `The server answered ${res.status}.`;
517
+ signal(TOAST_EVENT, why);
518
+ return null;
519
+ }
520
+ const made = body?.field;
521
+ if (!made || typeof made.key !== "string") {
522
+ signal(DATA_ERROR_EVENT, "The server did not say which column it created.");
523
+ return null;
524
+ }
525
+ // The definition feeds BOTH payloads for a user table (`ut_assembly` builds the rows
526
+ // envelope and `/workspace` from the same `fields_base`), so the cached rows are now a
527
+ // schema behind. Cheap to drop, and a stale schema is not a stale number β€” it is a
528
+ // column that exists on the server and nowhere on screen.
529
+ rowsCache.delete(`tables/${tableKey}/rows`);
530
+ return made;
531
+ } catch {
532
+ signal(DATA_ERROR_EVENT, "Cannot reach the server.");
533
+ return null;
534
+ }
535
+ }
536
+
537
+ /**
538
+ * ⭐ WAVE 21 item 11 (ruling R10, contract C5) β€” parse an uploaded `.xlsx`/`.csv` and
539
+ * hand back its COLUMNS AND VALUES as strings.
540
+ *
541
+ * β›” THE FILE NEVER BECOMES DATA. This reads a spreadsheet to find out which records the
542
+ * user means; nothing here writes a row, creates a field, or keeps the file. The server
543
+ * parses (openpyxl is not a thing a browser has) and answers with strings β€” the matching
544
+ * itself is client-side, over rows this browser already holds (`fileSelect.ts`).
545
+ *
546
+ * ⚠ `truncated` IS THE CONTRACT'S POINT. C5 caps a column at 20k values, and a cap the
547
+ * caller cannot see is a silent wrong answer: a 30k-row file would report ten thousand
548
+ * records "not found" that are simply past the cap. The dialog surfaces this flag as its
549
+ * own sentence, never folded into the miss count ([[no-unverifiable-aggregates]]).
550
+ */
551
+ export interface TabularUpload {
552
+ columns: string[];
553
+ rows: number;
554
+ values: Record<string, string[]>;
555
+ truncated: boolean;
556
+ }
557
+
558
+ export async function uploadTabular(file: File): Promise<TabularUpload | null> {
559
+ const form = new FormData();
560
+ form.append("file", file);
561
+ try {
562
+ // ⚠ NO Content-Type HEADER. The browser must set it, because only the browser knows
563
+ // the multipart boundary it generated β€” writing `multipart/form-data` by hand omits
564
+ // the boundary and the server cannot parse the body at all.
565
+ const res = await fetch(`${API_V1}/uploads/tabular`, {
566
+ method: "POST",
567
+ credentials: CREDENTIALS,
568
+ body: form,
569
+ });
570
+ if (handledSession(res)) return null;
571
+ const body = (await readJson(res)) as
572
+ | { columns?: unknown; rows?: unknown; values?: unknown; truncated?: unknown;
573
+ detail?: { message?: string } }
574
+ | null;
575
+ if (!res.ok) {
576
+ const why = typeof body?.detail?.message === "string"
577
+ ? body.detail.message
578
+ : `The file could not be read (the server answered ${res.status}).`;
579
+ signal(TOAST_EVENT, why);
580
+ return null;
581
+ }
582
+ const columns = Array.isArray(body?.columns)
583
+ ? body.columns.filter((c): c is string => typeof c === "string")
584
+ : [];
585
+ const rawValues = (body?.values && typeof body.values === "object" ? body.values : {}) as
586
+ Record<string, unknown>;
587
+ const values: Record<string, string[]> = {};
588
+ for (const col of columns)
589
+ values[col] = Array.isArray(rawValues[col])
590
+ ? (rawValues[col] as unknown[]).map((v) => (v === null || v === undefined ? "" : String(v)))
591
+ : [];
592
+ if (!columns.length) {
593
+ signal(TOAST_EVENT, "That file has no readable columns.");
594
+ return null;
595
+ }
596
+ return {
597
+ columns,
598
+ rows: typeof body?.rows === "number" ? body.rows : 0,
599
+ values,
600
+ truncated: body?.truncated === true,
601
+ };
602
+ } catch {
603
+ signal(DATA_ERROR_EVENT, "Cannot reach the server.");
604
+ return null;
605
+ }
606
+ }
607
+
608
+ /** C-UNDO's other half: drop a row this session just added. Same cache rule as the append. */
609
+ export async function deleteTableRow(
610
+ tableKey: string,
611
+ rid: string | number
612
+ ): Promise<boolean> {
613
+ try {
614
+ const res = await fetch(
615
+ `${API_V1}/tables/${encodeURIComponent(tableKey)}/rows/${encodeURIComponent(String(rid))}`,
616
+ { method: "DELETE", credentials: CREDENTIALS }
617
+ );
618
+ if (handledSession(res)) return false;
619
+ if (res.ok) rowsCache.delete(`tables/${tableKey}/rows`);
620
+ return res.ok;
621
+ } catch {
622
+ return false;
623
+ }
624
+ }
625
+
626
+ /**
627
+ * Item 7 (contract C-TS) β€” `POST /api/v1/grid/timeseries`.
628
+ *
629
+ * A READ that is a POST, because the request body carries a pid list that can run to
630
+ * thousands: a URL cannot hold the caller's book, and putting customer ids in a query string
631
+ * would also put them in every access log. The server intersects those pids with the caller's
632
+ * `allowed_pids` regardless, so the body narrows the answer and can never widen it.
633
+ *
634
+ * ⚠ RETURNS `null` ON EVERY FAILURE, AND THE PANEL MUST SAY SO RATHER THAN DRAW ZEROES.
635
+ * That is the difference between this and a payload of empty buckets: "we could not ask" and
636
+ * "the answer is nothing" look identical on a chart and are not the same claim. A 400 is the
637
+ * honest "narrow the span"; a 403 is `out_of_scope` (every requested pid was outside the
638
+ * caller's book); a 401 ends the session like everywhere else.
639
+ */
640
+ export async function fetchTimeseries(
641
+ body: TsRequest,
642
+ scope: SurfaceScope = surfaceScope
643
+ ): Promise<{ payload: TsPayload | null; status: number }> {
644
+ try {
645
+ const res = await fetch(
646
+ `${API_V1}/grid/timeseries?scope=${encodeURIComponent(scope)}`,
647
+ {
648
+ method: "POST",
649
+ credentials: CREDENTIALS,
650
+ headers: JSON_HEADERS,
651
+ body: JSON.stringify(body),
652
+ }
653
+ );
654
+ if (handledSession(res)) return { payload: null, status: res.status };
655
+ if (!res.ok) return { payload: null, status: res.status };
656
+ const json = (await readJson(res)) as TsPayload | null;
657
+ // Shape-check before trusting it: `columns` and `rows` are indexed POSITIONALLY against
658
+ // each other, so a payload missing either would index into undefined all the way down.
659
+ if (!json || !Array.isArray(json.columns) || !Array.isArray(json.rows))
660
+ return { payload: null, status: res.status };
661
+ return { payload: json, status: res.status };
662
+ } catch {
663
+ return { payload: null, status: 0 };
664
+ }
665
+ }
666
+
667
+ /**
668
+ * wave17 owner item 9 / ruling R4 (contract C-CAL) β€” per-DAY measure values.
669
+ *
670
+ * β›” WHY A SECOND CHANNEL rather than a parameter on the time-series one: **every group carries
671
+ * its OWN pid set.** A calendar day holds whatever records the date field placed there, so the
672
+ * subject changes from cell to cell; one `allowed_pids` for the whole request would compute
673
+ * every day over everybody, which is a different question with the same shape.
674
+ *
675
+ * The caps are the SERVER's and it answers a 400 with its reason rather than trimming: 31 days,
676
+ * 6 fields, 186 cells. The client's job is to send no more than that, never to guess what was
677
+ * dropped. `values[field][day] === null` is UNANSWERED β€” a day past the tenant's today, a day
678
+ * whose pids are all outside the reader's book, or a measure that could not resolve β€” and the
679
+ * renderer draws "β€”" for all three. Never 0.
680
+ */
681
+ export interface CalMetricsResponse {
682
+ values: Record<string, Record<string, number | null>>;
683
+ today?: string;
684
+ /** C-TSWIN at day grain: which window each metric was resolved under. */
685
+ windows?: Record<string, { kind: string; label: string }>;
686
+ dropped?: { field: string; reason: string }[];
687
+ problems?: string[];
688
+ }
689
+
690
+ export async function fetchCalendarMetrics(
691
+ body: { groups: { key: string; pids: number[] }[]; fields: string[] },
692
+ scope: SurfaceScope = surfaceScope
693
+ ): Promise<{ payload: CalMetricsResponse | null; status: number }> {
694
+ try {
695
+ const res = await fetch(
696
+ `${API_V1}/grid/calendar_metrics?scope=${encodeURIComponent(scope)}`,
697
+ {
698
+ method: "POST",
699
+ credentials: CREDENTIALS,
700
+ headers: JSON_HEADERS,
701
+ body: JSON.stringify(body),
702
+ }
703
+ );
704
+ if (handledSession(res)) return { payload: null, status: res.status };
705
+ if (!res.ok) return { payload: null, status: res.status };
706
+ const json = (await readJson(res)) as CalMetricsResponse | null;
707
+ // Shape-checked before it is trusted, the `fetchTimeseries` rule: every lookup below is
708
+ // `values[field][day]`, so a payload without `values` would read `undefined` as a value and
709
+ // paint blanks that look like real "β€”" refusals.
710
+ if (!json || !json.values || typeof json.values !== "object")
711
+ return { payload: null, status: res.status };
712
+ return { payload: json, status: res.status };
713
+ } catch {
714
+ return { payload: null, status: 0 };
715
+ }
716
+ }
717
+
718
+ // --- the WRITE (CP-C) ------------------------------------------------------
719
+
720
+ /**
721
+ * The replay bound. The SAME 24 the Streamlit value slot uses, for a different
722
+ * reason: there, the window exists because one slot gets clobbered by a second
723
+ * write; here, it bounds how much a failed POST may carry forward. Both rest on
724
+ * the same server guarantee β€” X2 dedups by event id β€” so re-sending is free and
725
+ * losing an event is not.
726
+ */
727
+ export const MAX_REPLAY = 24;
728
+
729
+ let pending: HostEvent[] = [];
730
+ let inFlight = false;
731
+
732
+ /**
733
+ * Which SURFACE this grid is drawing β€” `customer` (the whole scoped pool),
734
+ * `cohort` (hand-curated sets) or `product` (the SKU table β€” wave 16 C-TOPIC).
735
+ * It lives at module scope beside the queue on purpose: the queue is already
736
+ * module-level, and an event must carry the scope it was CREATED under even if
737
+ * it drains after a route change.
738
+ *
739
+ * ⚠ The server refuses an unknown value rather than defaulting it, so this is
740
+ * never a free-text field. Read and write must agree on what a scope is: the
741
+ * grid asks `/workspace?scope=X` and every event it then emits says `X`.
742
+ */
743
+ export type SurfaceScope = "customer" | "cohort" | "product" | `ut_${string}`;
744
+ let surfaceScope: SurfaceScope = "customer";
745
+
746
+ export function setSurfaceScope(scope: SurfaceScope): void {
747
+ surfaceScope = scope;
748
+ }
749
+
750
+ /** For the gate: the queue is module state and a test needs a known start. */
751
+ export function _resetQueue(): void {
752
+ pending = [];
753
+ inFlight = false;
754
+ }
755
+
756
+ export function _pendingIds(): string[] {
757
+ return pending.map((e) => e.id);
758
+ }
759
+
760
+ /**
761
+ * ONE request in flight at a time, and the queue drains in order.
762
+ *
763
+ * ORDER IS NOT COSMETIC: `field_upsert` then `view_upsert` is a field that
764
+ * exists and a view that shows it; the other way round is a view referencing a
765
+ * column the store has never heard of. Parallel POSTs would race exactly there.
766
+ *
767
+ * ⚠ A FAILED BATCH IS REQUEUED BUT NOT RETRIED ON A TIMER. It goes back at the
768
+ * FRONT (it happened first) and rides the next emit. A self-scheduling retry
769
+ * against a server that is down is a tight loop, and a timer is a background
770
+ * behaviour nobody asked for; the next user action is a perfectly good clock.
771
+ * The cost is bounded and known: with no further action, the last batch stays
772
+ * unsent β€” which is strictly better than today, where standalone drops every
773
+ * event on the floor.
774
+ */
775
+ async function drain(): Promise<void> {
776
+ if (inFlight || pending.length === 0) return;
777
+ inFlight = true;
778
+ const batch = pending;
779
+ pending = [];
780
+ let requeue = false;
781
+ try {
782
+ const res = await fetch(`${API_V1}/grid/events`, {
783
+ method: "POST",
784
+ credentials: CREDENTIALS,
785
+ headers: JSON_HEADERS,
786
+ body: JSON.stringify({ events: batch, scopeKey: surfaceScope }),
787
+ });
788
+ // A dead session is NOT retryable β€” replaying into a 401 forever would
789
+ // turn one expired cookie into an unbounded request loop.
790
+ if (!handledSession(res)) {
791
+ if (res.ok) applyEventResult(await readJson(res));
792
+ else requeue = true;
793
+ }
794
+ } catch {
795
+ requeue = true;
796
+ } finally {
797
+ inFlight = false;
798
+ if (requeue) {
799
+ // Front, and bounded: events that arrived during the flight are NEWER.
800
+ pending = [...batch, ...pending].slice(-MAX_REPLAY);
801
+ } else if (pending.length) {
802
+ void drain();
803
+ }
804
+ }
805
+ }
806
+
807
+ /**
808
+ * X2's `{results:[{id, rerender}], doc?, toast?}`.
809
+ *
810
+ * `results` needs nothing: the client is already optimistic and the server's
811
+ * per-event ack carries no correction. `toast` is surfaced β€” for the
812
+ * echo-dependent events it is the only feedback that exists in standalone.
813
+ * `doc` is deliberately unhandled; see this file's header.
814
+ */
815
+ function applyEventResult(body: unknown): void {
816
+ const b = body as {
817
+ toast?: unknown; rerender?: unknown; results?: unknown; derived?: unknown;
818
+ } | null;
819
+ const toast = b?.toast;
820
+ if (typeof toast === "string" && toast.trim() !== "") signal(TOAST_EVENT, toast.trim());
821
+
822
+ // owner item 2 β€” a measure column's values, computed server-side right after the write and
823
+ // returned here rather than waiting for the `/workspace` re-read below. Raised BEFORE the
824
+ // stale signal so the numbers paint on the earlier of the two, whichever the server sent.
825
+ // Shape-checked, because a shortcut that corrupts rows is worse than no shortcut.
826
+ if (b?.derived && typeof b.derived === "object" && !Array.isArray(b.derived))
827
+ signal(DERIVED_CELLS_EVENT, b.derived);
828
+
829
+ // The server already tells us when a write changed durable state β€” `rerender`
830
+ // is exactly the flag the Streamlit adapter uses to trigger its own rerun. We
831
+ // were dropping it, which is why cohort membership, list adds and folder moves
832
+ // never appeared in standalone until a manual reload.
833
+ const results = Array.isArray(b?.results) ? (b.results as { rerender?: unknown }[]) : [];
834
+ if (b?.rerender === true || results.some((r) => r?.rerender === true)) {
835
+ signal(WORKSPACE_STALE_EVENT);
836
+ }
837
+ }
838
+
839
+ /** The sink hostBridge falls through to in standalone. Returns true: the event
840
+ * is now OURS (queued and owned), which is all the caller ever needed. */
841
+ function standaloneSink(event: HostEvent): boolean {
842
+ pending.push(event);
843
+ if (pending.length > MAX_REPLAY) pending.shift();
844
+ void drain();
845
+ return true;
846
+ }
847
+
848
+ /** Called from main.tsx in standalone ONLY. Never in the embed β€” that is the
849
+ * whole of "standalone-only branches", enforced at the one wiring point. */
850
+ export function installStandaloneBridge(): void {
851
+ setStandaloneSink(standaloneSink);
852
+ }
web/src/customer-grid/types.ts CHANGED
The diff for this file is too large to render. See raw diff