fsanyoto commited on
Commit
c14ceee
Β·
verified Β·
1 Parent(s): 4748aae

Deploy AIOS web (React glide grid + FastAPI slice)

Browse files
This view is limited to 50 files because it contains too many changes. Β  See raw diff
Files changed (50) hide show
  1. Dockerfile +68 -68
  2. README.md +1 -1
  3. api/deps.py +1 -1
  4. api/main.py +364 -364
  5. api/pages.py +1 -1
  6. api/routes_assets.py +165 -165
  7. platform/aios_grid.py +1870 -0
  8. platform/aios_grid_fields.json +413 -0
  9. platform/core/assets.py +73 -0
  10. platform/core/auth_google.py +91 -0
  11. platform/core/context.py +59 -0
  12. platform/core/grid_events.py +1334 -0
  13. platform/core/keychain.py +182 -0
  14. platform/core/links.py +59 -0
  15. platform/core/management_pl.py +161 -0
  16. platform/core/measure_resolve.py +190 -0
  17. platform/core/odoo.py +201 -0
  18. platform/core/periods.py +155 -0
  19. platform/core/perm_migrate.py +156 -0
  20. platform/core/perm_scope.py +286 -0
  21. platform/core/perms.py +215 -0
  22. platform/core/record_comments.py +155 -0
  23. platform/core/registry.py +199 -0
  24. platform/core/store.py +369 -0
  25. platform/core/store_backend.py +85 -0
  26. platform/core/store_pg.py +249 -0
  27. platform/core/table_store.py +479 -0
  28. platform/core/user_tables.py +292 -0
  29. platform/core/users.py +361 -0
  30. platform/evals/analyst_golden.yml +104 -0
  31. platform/harness/__init__.py +3 -0
  32. platform/harness/analyst.py +318 -0
  33. platform/harness/base.py +123 -0
  34. platform/harness/canonical.py +101 -0
  35. platform/harness/chatlog.py +102 -0
  36. platform/harness/connectors/__init__.py +1 -0
  37. platform/harness/connectors/odoo.py +147 -0
  38. platform/harness/connectors/quickbooks.py +37 -0
  39. platform/harness/datastore.py +750 -0
  40. platform/harness/evals.py +134 -0
  41. platform/harness/filter_eval.py +485 -0
  42. platform/harness/filter_sql.py +673 -0
  43. platform/harness/measure_filter.py +788 -0
  44. platform/harness/pg/schema.sql +180 -0
  45. platform/harness/routines.py +259 -0
  46. platform/harness/runtime.py +248 -0
  47. platform/harness/semantic.py +936 -0
  48. platform/harness/skills/__init__.py +1 -0
  49. platform/harness/skills/sales.py +168 -0
  50. platform/harness/tables.py +129 -0
Dockerfile CHANGED
@@ -1,68 +1,68 @@
1
- # AIOS web β€” the React shell + FastAPI over the royalimports_os data layer. THE CANONICAL IMAGE:
2
- # this is the one that actually runs, and the one a host migration re-points (C1e).
3
- #
4
- # β›” WHY A SECOND DOCKERFILE EXISTS (ops/Dockerfile.api), stated here so nobody "tidies" the wrong
5
- # one away. It is NOT a duplicated service β€” it is the SAME service built from a DIFFERENT BUILD
6
- # CONTEXT:
7
- # * HF Spaces: the context is the Space repo root, which `deploy_web.py` uploads FLATTENED β€”
8
- # `api/`, `web/`, `royalimports_os/`, `requirements.txt` all at top level. Hence the paths here.
9
- # * ops/compose.yml: the context is the local repo root, where the same files live at
10
- # `aios-web/api/`, `aios-web/web/`, `royalimports_os/`. Different prefixes, same content.
11
- # β‡’ THE REAL FIX, and it is a follow-up rather than something to do the day of a first deploy:
12
- # make `deploy_web.py` upload with the REPO layout so both contexts are identical, then this file
13
- # serves both and `ops/Dockerfile.api` is deleted. Until then `ops/verify_portability.py` gates
14
- # the two against each other on every property that matters (same app object, same launcher
15
- # contract, non-root, healthchecked, no baked secrets), so they cannot drift silently.
16
- #
17
- # Two stages: node builds the React bundle, python serves it + the JSON API. Login is the app's own
18
- # branded page over a signed session cookie (wave 1 removed the HTTP Basic gate).
19
-
20
- # ---- stage 1: build the React bundle ----
21
- FROM node:22-slim AS web
22
- WORKDIR /web
23
- # Resolve deps FRESH from the public registry (the local lockfile pinned a mirror-only lodash
24
- # 4.18.1 that isn't on npmjs.org; a fresh install gets the public 4.17.x that glide accepts).
25
- COPY web/package.json ./
26
- RUN npm install --no-audit --no-fund --no-package-lock
27
- COPY web/ ./
28
- RUN rm -f package-lock.json && npm run build # -> /web/dist
29
-
30
- # ---- stage 2: python API serving the bundle ----
31
- FROM python:3.12-slim
32
- WORKDIR /app
33
- COPY requirements.txt ./
34
- RUN pip install --no-cache-dir -r requirements.txt
35
- COPY api/ ./api/
36
- # the reused data layer (modules/core/harness)
37
- # β›” COMMENT ON ITS OWN LINE. Docker does NOT support a trailing comment on an instruction:
38
- # it parsed `# the reused data layer ...` as extra SOURCE PATHS and the first build ever run
39
- # died with `failed to calculate checksum ... "/the": not found`. Only RUN tolerates it, and
40
- # only because the shell eats the `#`. Gated in ops/verify_portability.py.
41
- COPY royalimports_os/ ./royalimports_os/
42
- COPY --from=web /web/dist ./web/dist
43
-
44
- # Run unprivileged β€” the C2 security baseline, and matched to ops/Dockerfile.api so the two images
45
- # do not differ on a property that would only surface after a migration. uid 1000 is what HF
46
- # Spaces expects.
47
- RUN useradd --create-home --uid 1000 user && chown -R user:user /app
48
- USER user
49
-
50
- ENV RI_DIR=/app/royalimports_os \
51
- PORT=7860 \
52
- PYTHONUNBUFFERED=1 \
53
- AIOS_PREWARM=1
54
- # β›” AIOS_INSECURE_SSL is deliberately NOT set. It disables TLS verification for every outbound
55
- # call in the process and exists for one developer laptop. ops/verify_portability.py fails if any
56
- # deploy config sets it.
57
- EXPOSE 7860
58
-
59
- # Liveness via stdlib rather than curl β€” it keeps the image free of an apt layer that exists only
60
- # for the healthcheck.
61
- HEALTHCHECK --interval=30s --timeout=5s --start-period=60s --retries=3 \
62
- CMD python -c "import os,urllib.request,sys; sys.exit(0 if urllib.request.urlopen(f'http://127.0.0.1:{os.environ.get(\"PORT\",\"7860\")}/api/health',timeout=4).status==200 else 1)"
63
-
64
- # ⚠ THE PORT COMES FROM THE ENVIRONMENT, ALWAYS. The application itself reads PORT nowhere β€” it is
65
- # a launcher argument β€” which is what makes moving hosts a config change instead of a code change.
66
- # Secrets (Odoo creds, APP_PASSWORD, AIOS_SESSION_SECRET, APP_BASE_URL) arrive as env at RUN time
67
- # and are never baked; the repo-root .dockerignore drops **/.env from the build context.
68
- CMD ["sh", "-c", "uvicorn api.main:app --host 0.0.0.0 --port ${PORT}"]
 
1
+ # AIOS web β€” the React shell + FastAPI over the platform data layer. THE CANONICAL IMAGE:
2
+ # this is the one that actually runs, and the one a host migration re-points (C1e).
3
+ #
4
+ # β›” WHY A SECOND DOCKERFILE EXISTS (ops/Dockerfile.api), stated here so nobody "tidies" the wrong
5
+ # one away. It is NOT a duplicated service β€” it is the SAME service built from a DIFFERENT BUILD
6
+ # CONTEXT:
7
+ # * HF Spaces: the context is the Space repo root, which `deploy_web.py` uploads FLATTENED β€”
8
+ # `api/`, `web/`, `platform/`, `requirements.txt` all at top level. Hence the paths here.
9
+ # * ops/compose.yml: the context is the local repo root, where the same files live at
10
+ # `aios-web/api/`, `aios-web/web/`, `platform/`. Different prefixes, same content.
11
+ # β‡’ THE REAL FIX, and it is a follow-up rather than something to do the day of a first deploy:
12
+ # make `deploy_web.py` upload with the REPO layout so both contexts are identical, then this file
13
+ # serves both and `ops/Dockerfile.api` is deleted. Until then `ops/verify_portability.py` gates
14
+ # the two against each other on every property that matters (same app object, same launcher
15
+ # contract, non-root, healthchecked, no baked secrets), so they cannot drift silently.
16
+ #
17
+ # Two stages: node builds the React bundle, python serves it + the JSON API. Login is the app's own
18
+ # branded page over a signed session cookie (wave 1 removed the HTTP Basic gate).
19
+
20
+ # ---- stage 1: build the React bundle ----
21
+ FROM node:22-slim AS web
22
+ WORKDIR /web
23
+ # Resolve deps FRESH from the public registry (the local lockfile pinned a mirror-only lodash
24
+ # 4.18.1 that isn't on npmjs.org; a fresh install gets the public 4.17.x that glide accepts).
25
+ COPY web/package.json ./
26
+ RUN npm install --no-audit --no-fund --no-package-lock
27
+ COPY web/ ./
28
+ RUN rm -f package-lock.json && npm run build # -> /web/dist
29
+
30
+ # ---- stage 2: python API serving the bundle ----
31
+ FROM python:3.12-slim
32
+ WORKDIR /app
33
+ COPY requirements.txt ./
34
+ RUN pip install --no-cache-dir -r requirements.txt
35
+ COPY api/ ./api/
36
+ # the reused data layer (modules/core/harness)
37
+ # β›” COMMENT ON ITS OWN LINE. Docker does NOT support a trailing comment on an instruction:
38
+ # it parsed `# the reused data layer ...` as extra SOURCE PATHS and the first build ever run
39
+ # died with `failed to calculate checksum ... "/the": not found`. Only RUN tolerates it, and
40
+ # only because the shell eats the `#`. Gated in ops/verify_portability.py.
41
+ COPY platform/ ./platform/
42
+ COPY --from=web /web/dist ./web/dist
43
+
44
+ # Run unprivileged β€” the C2 security baseline, and matched to ops/Dockerfile.api so the two images
45
+ # do not differ on a property that would only surface after a migration. uid 1000 is what HF
46
+ # Spaces expects.
47
+ RUN useradd --create-home --uid 1000 user && chown -R user:user /app
48
+ USER user
49
+
50
+ ENV RI_DIR=/app/platform \
51
+ PORT=7860 \
52
+ PYTHONUNBUFFERED=1 \
53
+ AIOS_PREWARM=1
54
+ # β›” AIOS_INSECURE_SSL is deliberately NOT set. It disables TLS verification for every outbound
55
+ # call in the process and exists for one developer laptop. ops/verify_portability.py fails if any
56
+ # deploy config sets it.
57
+ EXPOSE 7860
58
+
59
+ # Liveness via stdlib rather than curl β€” it keeps the image free of an apt layer that exists only
60
+ # for the healthcheck.
61
+ HEALTHCHECK --interval=30s --timeout=5s --start-period=60s --retries=3 \
62
+ CMD python -c "import os,urllib.request,sys; sys.exit(0 if urllib.request.urlopen(f'http://127.0.0.1:{os.environ.get(\"PORT\",\"7860\")}/api/health',timeout=4).status==200 else 1)"
63
+
64
+ # ⚠ THE PORT COMES FROM THE ENVIRONMENT, ALWAYS. The application itself reads PORT nowhere β€” it is
65
+ # a launcher argument β€” which is what makes moving hosts a config change instead of a code change.
66
+ # Secrets (Odoo creds, APP_PASSWORD, AIOS_SESSION_SECRET, APP_BASE_URL) arrive as env at RUN time
67
+ # and are never baked; the repo-root .dockerignore drops **/.env from the build context.
68
+ CMD ["sh", "-c", "uvicorn api.main:app --host 0.0.0.0 --port ${PORT}"]
README.md CHANGED
@@ -9,5 +9,5 @@ pinned: false
9
  ---
10
 
11
  Loopable β€” the AIOS platform shell: a React front end (Airtable-grade grid + dashboard pages)
12
- over a FastAPI service reusing the royalimports_os data layer. Login-gated: a branded login page over an HMAC-signed session cookie, with per-user
13
  business-unit scope and module grants. Separate from the Streamlit app.
 
9
  ---
10
 
11
  Loopable β€” the AIOS platform shell: a React front end (Airtable-grade grid + dashboard pages)
12
+ over a FastAPI service reusing the platform data layer. Login-gated: a branded login page over an HMAC-signed session cookie, with per-user
13
  business-unit scope and module grants. Separate from the Streamlit app.
api/deps.py CHANGED
@@ -19,7 +19,7 @@ from fastapi import Depends, HTTPException, Request, Response
19
 
20
  import aios_session
21
 
22
- _RI = Path(os.environ.get("RI_DIR") or Path(__file__).resolve().parents[2] / "royalimports_os")
23
  if str(_RI) not in sys.path:
24
  sys.path.insert(0, str(_RI))
25
 
 
19
 
20
  import aios_session
21
 
22
+ _RI = Path(os.environ.get("RI_DIR") or Path(__file__).resolve().parents[2] / "platform")
23
  if str(_RI) not in sys.path:
24
  sys.path.insert(0, str(_RI))
25
 
api/main.py CHANGED
@@ -1,364 +1,364 @@
1
- """AIOS web API β€” the ONE shared, stateless process the Streamlit exit is aimed at.
2
-
3
- It REUSES the `royalimports_os/` data layer verbatim (nothing re-implemented): the canonical,
4
- reconciled Odoo model stays server-side and the browser only ever sees derived JSON. Serves the
5
- JSON API under `/api/*` and the built React bundle (`aios-web/web/dist`) at `/`.
6
-
7
- WHAT CHANGED IN EXIT WAVE 1 (2026-07-30) β€” three things, all of them load-bearing:
8
-
9
- * **Real sessions replace HTTP Basic.** The whole app used to sit behind one shared
10
- `APP_PASSWORD` with the browser's native Basic prompt, which means every caller was the same
11
- anonymous principal and no route could scope anything. Now a signed stateless cookie carries
12
- a real `core/users` identity (X3), and every v1 route resolves BU + own-book scope from the
13
- user RECORD. `/api/health` stays unauthenticated (liveness only).
14
- * **The overlay fork is deleted.** `aios-web/api/data/overlay.json` was a second writable home
15
- for user-owned fields the Streamlit app keeps in the tenant store. ONE store now (C1c).
16
- * **The SSL shim is env-gated.** It used to run at import, unconditionally.
17
-
18
- ⚠ TENANT RESOLUTION IS PER REQUEST (X7). Nothing tenant-shaped is held at module level; the
19
- session's tenant claim resolves through `harness.runtime.get_runtime`, which is LRU-bounded. This
20
- is the rule the ~30–80 MB/tenant target depends on β€” EXIT-0 measured the alternative at a ~0.6 GB
21
- commit FLOOR per tenant, duplicated within 2% per tenant, nothing shared.
22
- """
23
- import os
24
- import sys
25
- from pathlib import Path
26
-
27
- # --- the local Odoo SSL quirk, now BEHIND A GATE -------------------------------------------------
28
- # The Windows trust store reports the (valid) Odoo cert as expired, so local runs need an
29
- # unverified default context; the HF Space and the container are unaffected. This used to run
30
- # unconditionally at import β€” i.e. the shipped container disabled TLS verification for every
31
- # outbound HTTPS call it ever made, to Odoo and to everything else, forever. That is a
32
- # man-in-the-middle away from being someone else's data. It is now opt-in, must be set
33
- # deliberately, and is never on by default.
34
- # β›” NEVER set AIOS_INSECURE_SSL in a deployed environment. It exists for one developer laptop.
35
- if os.environ.get("AIOS_INSECURE_SSL") == "1":
36
- import ssl
37
- ssl._create_default_https_context = ssl._create_unverified_context # noqa: S323
38
-
39
- # --- reuse the tenant #0 data layer verbatim. RI_DIR overrides the location in the container
40
- # (where the layout differs from the local sibling-dir default). ---
41
- _HERE = Path(__file__).resolve()
42
- _RI = Path(os.environ.get("RI_DIR") or (_HERE.parents[2] / "royalimports_os"))
43
- for _p in (str(_RI), str(_HERE.parent)):
44
- if _p not in sys.path:
45
- sys.path.insert(0, _p)
46
-
47
- from dotenv import load_dotenv # noqa: E402
48
- load_dotenv(_RI / ".env") # Odoo creds + APP_PASSWORD, git-ignored, never committed/printed
49
-
50
- from fastapi import Body, Depends, FastAPI, Request # noqa: E402
51
- from fastapi.middleware.gzip import GZipMiddleware # noqa: E402
52
- from fastapi.responses import JSONResponse # noqa: E402
53
- from fastapi.staticfiles import StaticFiles # noqa: E402
54
- from starlette.exceptions import HTTPException as StarletteHTTPException # noqa: E402
55
-
56
- import aios_session # noqa: E402
57
- import routes_admin # noqa: E402
58
- import routes_assets # noqa: E402 (wave 18 C2-ASSET β€” catalog product imagery)
59
- import routes_auth # noqa: E402
60
- import routes_automation # noqa: E402 (wave 18 C4-AUTO β€” SESSION D's router, mounted by A)
61
- import routes_customers # noqa: E402
62
- import routes_grid # noqa: E402
63
- import routes_keychain # noqa: E402 (wave 18 C7 β€” keychain + connectors admin surfaces)
64
- import routes_nav # noqa: E402
65
- import routes_pages # noqa: E402
66
- import routes_products # noqa: E402 (wave 15 C-TOPIC β€” gated, not yet in the nav)
67
- import routes_records # noqa: E402
68
- import routes_tables # noqa: E402 (wave 18 C3-UT β€” user-created databases over the wire)
69
- from core import grid_events # noqa: E402
70
- from deps import Session, module_gate # noqa: E402
71
-
72
- _WEB_DIST = _HERE.parents[1] / "web" / "dist"
73
-
74
- # --- THE CANONICAL FIELD CONTRACT: a startup assertion, and the referee gate's subject ----------
75
- # Every field tags its semantic TYPE and its SOURCE. `source='odoo'` is READ-ONLY (Odoo is never
76
- # written); `source='overlay'` is the editable stratum that lives OUTSIDE Odoo. Loaded from the
77
- # ONE canonical file shared with the embedded host (`royalimports_os/aios_grid_fields.json`), so
78
- # embed == standalone by construction.
79
- #
80
- # TWO REASONS THIS IS HERE and not folded into the routes:
81
- # 1. It FAILS LOUDLY at import when the file is missing β€” which means the deploy or the RI_DIR
82
- # layout is broken, and discovering that from a 500 on the first customer request instead of
83
- # at startup costs a debugging session. (The pre-wave main.py had this guard; the EXIT-2a
84
- # rewrite dropped it and this restores it.)
85
- # 2. `aios-web/verify_fields_contract.py` β€” the cross-side REFEREE β€” reads `FIELDS` and
86
- # `_PASSTHROUGH_KEYS` from this module to prove the canonical file, `aios_grid.py` and this
87
- # API have not drifted. The rewrite removed them and the referee went red; retargeting the
88
- # gate would have been the wrong repair ([[gate-can-report-green-on-nothing]]: retarget, do
89
- # not delete β€” but only when the subject genuinely moved. Here it should not have moved).
90
- #
91
- # ⚠ THE ROUTES SERVE A SUPERSET OF THIS. `routes_customers._payload` derives its field list from
92
- # `aios_grid.fields_from_workspace(ws)`, which is `FIELDS` PLUS the session user's own custom_ and
93
- # measure_ columns β€” the same list the Streamlit host renders. That is the point: the standalone
94
- # shell now sees the user's own columns instead of the bare base contract. `FIELDS` is the
95
- # canonical FLOOR, asserted below to be exactly what `aios_grid` starts from.
96
- import json as _json_contract # noqa: E402
97
- _FIELDS_PATH = _RI / "aios_grid_fields.json"
98
- if not _FIELDS_PATH.is_file():
99
- raise FileNotFoundError(
100
- f"AIOS web API: canonical field contract missing at {_FIELDS_PATH}. Set RI_DIR to the "
101
- "royalimports_os root (it also carries the data layer this API imports)."
102
- )
103
- _contract = _json_contract.loads(_FIELDS_PATH.read_text(encoding="utf-8"))
104
- FIELDS = _contract["fields"] if isinstance(_contract, dict) else _contract
105
- # text/status/select/date pass through untouched; every OTHER odoo field is numeric -> rounded.
106
- # Derived from field TYPE (not a hand-kept key list) so a new text/date field can never be
107
- # wrongly rounded. `select` joined 2026-08-02 (dba) β€” a choice label rounded would be garbage.
108
- _PASSTHROUGH_KEYS = {f["key"] for f in FIELDS
109
- if f["source"] == "odoo" and f["type"] in ("text", "status", "select",
110
- "date")}
111
-
112
- app = FastAPI(title="AIOS web API")
113
-
114
- # The customers payload measured 1.15 MB of JSON on the live Space, shipped UNCOMPRESSED β€” with
115
- # the 754 KB bundle behind it, most of "the app is slow" was bytes on the wire. gzip takes the
116
- # payload to ~10–15% of that. minimum_size spares the tiny acks the overhead.
117
- app.add_middleware(GZipMiddleware, minimum_size=1024)
118
-
119
-
120
- @app.exception_handler(StarletteHTTPException)
121
- def _error_shape(request: Request, exc: StarletteHTTPException):
122
- """ONE error shape for every non-2xx (X2): `{"error": {"code", "message"}}`.
123
-
124
- `deps.err` already raises detail in that shape; anything FastAPI raises on its own (a 404, a
125
- 422 from a malformed path param) is wrapped here so a client never has to branch on two
126
- different error bodies.
127
- """
128
- detail = exc.detail
129
- if isinstance(detail, dict) and "error" in detail:
130
- body = detail
131
- else:
132
- body = {"error": {"code": f"http_{exc.status_code}", "message": str(detail)}}
133
- return JSONResponse(body, status_code=exc.status_code,
134
- headers=getattr(exc, "headers", None))
135
-
136
-
137
- @app.exception_handler(grid_events.StoreUnavailable)
138
- def _store_unavailable(request: Request, exc: grid_events.StoreUnavailable):
139
- """THE SAFETY NET for "the store is down" β†’ 503, from anywhere.
140
-
141
- ⚠ WHY THIS IS APP-LEVEL AND NOT A `try` PER ROUTE. It was a try per route first, and a
142
- `StoreUnavailable` raised while BUILDING THE PAYLOAD β€” before the route reached its own
143
- try/except β€” surfaced as a 500. A store outage is a normal operational state and every route
144
- here touches the store at least twice (the workspace read, then the write), so "remember to
145
- wrap it" is a rule that gets forgotten once and then reports the wrong thing. One handler
146
- means a store outage can only ever be a 503, whichever call raised it.
147
-
148
- The seam raises this only when the caller passed no `fallback_ws` β€” i.e. exactly on this
149
- adapter, which has no durable session dict to degrade into. A 200 over a write that
150
- evaporated is the failure the whole rule exists to prevent.
151
- """
152
- return JSONResponse(
153
- {"error": {"code": "store_unavailable",
154
- "message": "the tenant store is unavailable β€” no change was saved"}},
155
- status_code=503)
156
-
157
-
158
- @app.get("/api/health")
159
- def health():
160
- """Unauthenticated LIVENESS only β€” it must answer before anyone can sign in, so it may not
161
- reveal anything about the deployment beyond "the process is up". No version, no tenant list,
162
- no config: a health endpoint is the one URL every scanner finds first."""
163
- return {"ok": True}
164
-
165
-
166
- app.include_router(routes_auth.router)
167
- app.include_router(routes_nav.router)
168
- app.include_router(routes_customers.router)
169
- app.include_router(routes_products.router)
170
- app.include_router(routes_assets.router)
171
- app.include_router(routes_tables.router)
172
- app.include_router(routes_grid.router)
173
- app.include_router(routes_records.router)
174
- # EXIT wave 2: the Y1 page-data envelope (one route for every ported dashboard) and Y4's user
175
- # administration. `routes_pages` imports `pages`, which lazily imports each `pages_*` builder β€” so
176
- # a new page is a builder module plus one registry line, and nothing here changes.
177
- app.include_router(routes_pages.router)
178
- app.include_router(routes_admin.router)
179
- app.include_router(routes_automation.router)
180
- app.include_router(routes_keychain.router)
181
-
182
-
183
- # --- DEPRECATED ALIASES (removed when S2's shell flips; kept so the current bundle keeps working)
184
- # They are the v1 handlers with the v1 session requirement β€” NOT the old unauthenticated Basic
185
- # behavior. An alias that kept the old auth would be a bypass of everything above it.
186
- _GATE = module_gate(routes_customers.MODULE)
187
-
188
-
189
- @app.get("/api/customers", deprecated=True)
190
- def _customers_alias(session: Session = Depends(_GATE)):
191
- return routes_customers._payload(session)
192
-
193
-
194
- @app.patch("/api/customers/{pid}", deprecated=True)
195
- def _patch_alias(pid: int, body: dict = Body(default=None),
196
- session: Session = Depends(_GATE)):
197
- return routes_customers.patch_customer(pid, body, session)
198
-
199
-
200
- def _assert_contract_floor():
201
- """The canonical FIELDS must be exactly what `aios_grid` starts an empty workspace from.
202
-
203
- This is what stops `FIELDS` becoming a constant that exists only to satisfy a gate. If the
204
- canonical JSON and `aios_grid.fields_from_workspace({})` ever disagree, the standalone API and
205
- the embedded host are serving two different schemas and the grid's own contract has forked β€”
206
- the exact drift `verify_fields_contract.py` was written to catch, now also caught at startup
207
- on whatever machine is actually running.
208
- """
209
- import aios_grid
210
- base = [f for f in aios_grid.fields_from_workspace({}) if not f.get("custom")]
211
- if [f["key"] for f in base] != [f["key"] for f in FIELDS]:
212
- raise RuntimeError(
213
- "AIOS web API: the canonical field contract and aios_grid.fields_from_workspace({}) "
214
- "disagree on the base field set β€” embed and standalone would serve different schemas. "
215
- "Run aios-web/verify_fields_contract.py.")
216
-
217
-
218
- def _startup_notes():
219
- """Say the two things an operator must know, once, at import β€” never in a response body."""
220
- if aios_session.EPHEMERAL_SECRET:
221
- print("[aios-api] AIOS_SESSION_SECRET is not set β€” signing with a random per-process "
222
- "key. Sessions will not survive a restart and will not work across workers. "
223
- "Set it in production.")
224
- if os.environ.get("AIOS_INSECURE_SSL") == "1":
225
- print("[aios-api] AIOS_INSECURE_SSL=1 β€” TLS verification is DISABLED for outbound "
226
- "requests. Local development only; never in a deployed environment.")
227
-
228
-
229
- _assert_contract_floor()
230
- _startup_notes()
231
-
232
-
233
- # β›” THE STATIC MOUNT IS NOW UNAUTHENTICATED, and that is a change this wave made on purpose.
234
- # Before EXIT-3a, `BasicAuth` middleware gated the WHOLE app including this mount. A branded login
235
- # page cannot live behind a password prompt, so the shell's own assets must be public β€” which they
236
- # are: `index.html`, the JS/CSS bundle and the favicon reveal nothing.
237
- #
238
- # ⚠ WHAT THAT SILENTLY DE-GATED, caught in review rather than in production. `web/dist/` also
239
- # carries `sample_customers.json`, a DEV FIXTURE copied from `web/public/`. Its 8 customers are
240
- # synthetic, but the `agent` column holds REAL EMPLOYEE NAMES, and it went from Basic-gated to
241
- # publicly fetchable in this commit. Nothing needs it: both the API bridge and `useCustomerData`
242
- # deleted their sample fallback on purpose ("NOTHING HERE FALLS BACK TO sample_customers.json"),
243
- # and it survives only because it sits in `web/public/`. So it is refused here β€” 404, the same
244
- # answer as any other path that is not part of the app.
245
- #
246
- # The right long-term fix is deleting it from `web/public/` (S2's lane β€” flagged in the mailbox);
247
- # this guard is what makes the API safe regardless of what the bundle happens to contain.
248
- _DEV_FIXTURES = {"sample_customers.json"}
249
-
250
-
251
- class _AppStatic(StaticFiles):
252
- async def get_response(self, path, scope):
253
- if Path(path).name in _DEV_FIXTURES:
254
- raise StarletteHTTPException(status_code=404, detail="Not Found")
255
- resp = await super().get_response(path, scope)
256
- # Vite content-hashes everything under assets/ (a change is a NEW url), so those are
257
- # immutable β€” a repeat visit re-downloads zero bytes instead of the whole 750 KB bundle.
258
- # index.html must stay revalidated or a deploy would strand returning browsers on the old
259
- # bundle; ETag/304 makes that revalidation a header exchange, not a transfer.
260
- # ⚠ Normalised first: on a Windows host StaticFiles hands this path with backslashes,
261
- # and `startswith("assets/")` silently skipped every asset (measured on the local probe).
262
- if path.replace("\\", "/").lstrip("/").startswith("assets/"):
263
- resp.headers["Cache-Control"] = "public, max-age=31536000, immutable"
264
- else:
265
- resp.headers["Cache-Control"] = "no-cache"
266
- return resp
267
-
268
-
269
- # static bundle LAST so /api/* wins; html=True serves index.html at / plus the built assets
270
- if _WEB_DIST.is_dir():
271
- app.mount("/", _AppStatic(directory=str(_WEB_DIST), html=True), name="web")
272
-
273
-
274
- # --- boot prewarm (AIOS_PREWARM=1 β€” set by the Dockerfile, never by tests) -----------------------
275
- # Without this, the first visitor after every deploy/restart pays the full Odoo pool build in
276
- # their request. The thread warms the CONSOLIDATED scope (None, None) + every registered page's
277
- # default envelope; scoped users still pay their own scope's first build, once.
278
- # Env-gated rather than a startup event so importing `api.main` in a gate (verify_api and friends
279
- # run against fakes) can never fire a live Odoo pull.
280
- def _seed_and_sync_store():
281
- """Bring the analytical store (harness.datastore) LIVE for this container β€” the app.py
282
- bootstrap, mirrored (2026-07-31, owner item 1).
283
-
284
- `harness.datastore` powers every measure column/condition, and ONLY app.py used to call
285
- `ensure_seed()` β€” so on a fresh Space disk this container resolved measures against a store
286
- that never existed and every measure cell served blank. Seeding alone was NOT enough either,
287
- and that was measured live the same day: "datastore seeded in 1.7s" followed by an endless
288
- `api:measure-column: ModelError the data cache is still warming up` β€” `ready()` demands
289
- EVERY entity at phase 'live', and a seed that predates a newer entity leaves it un-synced
290
- forever in a process with no sync loop. The SYNC SPRINT after the seed is what closes the
291
- write_date gap and backfills anything the seed lacks (app.py:7779's exact pattern, bounded
292
- passes). Fail-quiet throughout: no seed/token/Odoo β†’ the columns stay blank, the rows still
293
- serve.
294
- """
295
- import time as _t
296
- t0 = _t.time()
297
- try:
298
- from harness import datastore as _ds
299
- if _ds.ensure_seed():
300
- print(f"[aios-api] datastore seeded in {_t.time() - t0:.1f}s")
301
- # DEDICATED Odoo connection for this thread (the W6 postmortem rule): the sync's
302
- # search_reads must never interleave on the shared client's xmlrpc transport.
303
- import core.odoo as _odoo
304
- try:
305
- _odoo._tlocal.client = _odoo.OdooClient()
306
- except Exception:
307
- pass
308
- res = {}
309
- for i in range(12):
310
- res = _ds.sync_all(log=lambda *a, **k: None)
311
- print(f"[aios-api] datastore sync pass {i + 1}: "
312
- + ", ".join(f"{k}={v.get('phase')}" for k, v in sorted(res.items())))
313
- if all(v.get("phase") == "live" for v in res.values()):
314
- break
315
- print(f"[aios-api] datastore sync done in {_t.time() - t0:.1f}s "
316
- f"(ready={_ds.ready()})")
317
- except Exception as e: # noqa: BLE001
318
- print(f"[aios-api] datastore seed/sync skipped: {e}")
319
-
320
-
321
- def _store_resync_loop():
322
- """Keep the analytical mirror current β€” the API-process stand-in for the re-sync the
323
- Streamlit app piggybacks on page renders. Measure memos key on the pool stamp, so a
324
- refreshed pool re-reads the freshly synced store."""
325
- import time as _t
326
- while True:
327
- _t.sleep(1800)
328
- try:
329
- from harness import datastore as _ds
330
- import core.odoo as _odoo
331
- try:
332
- _odoo._tlocal.client = _odoo.OdooClient()
333
- except Exception:
334
- pass
335
- _ds.sync_all(log=lambda *a, **k: None)
336
- except Exception as e: # noqa: BLE001
337
- print(f"[aios-api] store resync failed: {e}")
338
-
339
-
340
- def _prewarm():
341
- import time as _t
342
- t0 = _t.time()
343
- # ⚠ The store sync runs in its OWN thread, never ahead of the pool warm: a stale seed can
344
- # take many minutes of XML-RPC to close, and the first live probe of this arrangement
345
- # showed the pool warm (42s) silently queued behind it β€” the whole app cold for every
346
- # visitor while a background column channel caught up. Measure cells retry via the
347
- # transient rule until the sync lands; nothing else waits on it.
348
- import threading as _th
349
- _th.Thread(target=_seed_and_sync_store, daemon=True, name="store-seed-sync").start()
350
- try:
351
- from harness import runtime as _runtime
352
- rt = _runtime.get_runtime("royal-imports")
353
- routes_customers.warm_default(rt)
354
- import pages as _pages
355
- _pages.warm_default(rt)
356
- print(f"[aios-api] prewarm done in {_t.time() - t0:.1f}s")
357
- except Exception as e: # noqa: BLE001 β€” boot must not die on a warm-up
358
- print(f"[aios-api] prewarm skipped: {e}")
359
-
360
-
361
- if os.environ.get("AIOS_PREWARM") == "1":
362
- import threading as _threading
363
- _threading.Thread(target=_prewarm, daemon=True, name="prewarm").start()
364
- _threading.Thread(target=_store_resync_loop, daemon=True, name="store-resync").start()
 
1
+ """AIOS web API β€” the ONE shared, stateless process the Streamlit exit is aimed at.
2
+
3
+ It REUSES the `platform/` data layer verbatim (nothing re-implemented): the canonical,
4
+ reconciled Odoo model stays server-side and the browser only ever sees derived JSON. Serves the
5
+ JSON API under `/api/*` and the built React bundle (`aios-web/web/dist`) at `/`.
6
+
7
+ WHAT CHANGED IN EXIT WAVE 1 (2026-07-30) β€” three things, all of them load-bearing:
8
+
9
+ * **Real sessions replace HTTP Basic.** The whole app used to sit behind one shared
10
+ `APP_PASSWORD` with the browser's native Basic prompt, which means every caller was the same
11
+ anonymous principal and no route could scope anything. Now a signed stateless cookie carries
12
+ a real `core/users` identity (X3), and every v1 route resolves BU + own-book scope from the
13
+ user RECORD. `/api/health` stays unauthenticated (liveness only).
14
+ * **The overlay fork is deleted.** `aios-web/api/data/overlay.json` was a second writable home
15
+ for user-owned fields the Streamlit app keeps in the tenant store. ONE store now (C1c).
16
+ * **The SSL shim is env-gated.** It used to run at import, unconditionally.
17
+
18
+ ⚠ TENANT RESOLUTION IS PER REQUEST (X7). Nothing tenant-shaped is held at module level; the
19
+ session's tenant claim resolves through `harness.runtime.get_runtime`, which is LRU-bounded. This
20
+ is the rule the ~30–80 MB/tenant target depends on β€” EXIT-0 measured the alternative at a ~0.6 GB
21
+ commit FLOOR per tenant, duplicated within 2% per tenant, nothing shared.
22
+ """
23
+ import os
24
+ import sys
25
+ from pathlib import Path
26
+
27
+ # --- the local Odoo SSL quirk, now BEHIND A GATE -------------------------------------------------
28
+ # The Windows trust store reports the (valid) Odoo cert as expired, so local runs need an
29
+ # unverified default context; the HF Space and the container are unaffected. This used to run
30
+ # unconditionally at import β€” i.e. the shipped container disabled TLS verification for every
31
+ # outbound HTTPS call it ever made, to Odoo and to everything else, forever. That is a
32
+ # man-in-the-middle away from being someone else's data. It is now opt-in, must be set
33
+ # deliberately, and is never on by default.
34
+ # β›” NEVER set AIOS_INSECURE_SSL in a deployed environment. It exists for one developer laptop.
35
+ if os.environ.get("AIOS_INSECURE_SSL") == "1":
36
+ import ssl
37
+ ssl._create_default_https_context = ssl._create_unverified_context # noqa: S323
38
+
39
+ # --- reuse the tenant #0 data layer verbatim. RI_DIR overrides the location in the container
40
+ # (where the layout differs from the local sibling-dir default). ---
41
+ _HERE = Path(__file__).resolve()
42
+ _RI = Path(os.environ.get("RI_DIR") or (_HERE.parents[2] / "platform"))
43
+ for _p in (str(_RI), str(_HERE.parent)):
44
+ if _p not in sys.path:
45
+ sys.path.insert(0, _p)
46
+
47
+ from dotenv import load_dotenv # noqa: E402
48
+ load_dotenv(_RI / ".env") # Odoo creds + APP_PASSWORD, git-ignored, never committed/printed
49
+
50
+ from fastapi import Body, Depends, FastAPI, Request # noqa: E402
51
+ from fastapi.middleware.gzip import GZipMiddleware # noqa: E402
52
+ from fastapi.responses import JSONResponse # noqa: E402
53
+ from fastapi.staticfiles import StaticFiles # noqa: E402
54
+ from starlette.exceptions import HTTPException as StarletteHTTPException # noqa: E402
55
+
56
+ import aios_session # noqa: E402
57
+ import routes_admin # noqa: E402
58
+ import routes_assets # noqa: E402 (wave 18 C2-ASSET β€” catalog product imagery)
59
+ import routes_auth # noqa: E402
60
+ import routes_automation # noqa: E402 (wave 18 C4-AUTO β€” SESSION D's router, mounted by A)
61
+ import routes_customers # noqa: E402
62
+ import routes_grid # noqa: E402
63
+ import routes_keychain # noqa: E402 (wave 18 C7 β€” keychain + connectors admin surfaces)
64
+ import routes_nav # noqa: E402
65
+ import routes_pages # noqa: E402
66
+ import routes_products # noqa: E402 (wave 15 C-TOPIC β€” gated, not yet in the nav)
67
+ import routes_records # noqa: E402
68
+ import routes_tables # noqa: E402 (wave 18 C3-UT β€” user-created databases over the wire)
69
+ from core import grid_events # noqa: E402
70
+ from deps import Session, module_gate # noqa: E402
71
+
72
+ _WEB_DIST = _HERE.parents[1] / "web" / "dist"
73
+
74
+ # --- THE CANONICAL FIELD CONTRACT: a startup assertion, and the referee gate's subject ----------
75
+ # Every field tags its semantic TYPE and its SOURCE. `source='odoo'` is READ-ONLY (Odoo is never
76
+ # written); `source='overlay'` is the editable stratum that lives OUTSIDE Odoo. Loaded from the
77
+ # ONE canonical file shared with the embedded host (`platform/aios_grid_fields.json`), so
78
+ # embed == standalone by construction.
79
+ #
80
+ # TWO REASONS THIS IS HERE and not folded into the routes:
81
+ # 1. It FAILS LOUDLY at import when the file is missing β€” which means the deploy or the RI_DIR
82
+ # layout is broken, and discovering that from a 500 on the first customer request instead of
83
+ # at startup costs a debugging session. (The pre-wave main.py had this guard; the EXIT-2a
84
+ # rewrite dropped it and this restores it.)
85
+ # 2. `aios-web/verify_fields_contract.py` β€” the cross-side REFEREE β€” reads `FIELDS` and
86
+ # `_PASSTHROUGH_KEYS` from this module to prove the canonical file, `aios_grid.py` and this
87
+ # API have not drifted. The rewrite removed them and the referee went red; retargeting the
88
+ # gate would have been the wrong repair ([[gate-can-report-green-on-nothing]]: retarget, do
89
+ # not delete β€” but only when the subject genuinely moved. Here it should not have moved).
90
+ #
91
+ # ⚠ THE ROUTES SERVE A SUPERSET OF THIS. `routes_customers._payload` derives its field list from
92
+ # `aios_grid.fields_from_workspace(ws)`, which is `FIELDS` PLUS the session user's own custom_ and
93
+ # measure_ columns β€” the same list the Streamlit host renders. That is the point: the standalone
94
+ # shell now sees the user's own columns instead of the bare base contract. `FIELDS` is the
95
+ # canonical FLOOR, asserted below to be exactly what `aios_grid` starts from.
96
+ import json as _json_contract # noqa: E402
97
+ _FIELDS_PATH = _RI / "aios_grid_fields.json"
98
+ if not _FIELDS_PATH.is_file():
99
+ raise FileNotFoundError(
100
+ f"AIOS web API: canonical field contract missing at {_FIELDS_PATH}. Set RI_DIR to the "
101
+ "platform root (it also carries the data layer this API imports)."
102
+ )
103
+ _contract = _json_contract.loads(_FIELDS_PATH.read_text(encoding="utf-8"))
104
+ FIELDS = _contract["fields"] if isinstance(_contract, dict) else _contract
105
+ # text/status/select/date pass through untouched; every OTHER odoo field is numeric -> rounded.
106
+ # Derived from field TYPE (not a hand-kept key list) so a new text/date field can never be
107
+ # wrongly rounded. `select` joined 2026-08-02 (dba) β€” a choice label rounded would be garbage.
108
+ _PASSTHROUGH_KEYS = {f["key"] for f in FIELDS
109
+ if f["source"] == "odoo" and f["type"] in ("text", "status", "select",
110
+ "date")}
111
+
112
+ app = FastAPI(title="AIOS web API")
113
+
114
+ # The customers payload measured 1.15 MB of JSON on the live Space, shipped UNCOMPRESSED β€” with
115
+ # the 754 KB bundle behind it, most of "the app is slow" was bytes on the wire. gzip takes the
116
+ # payload to ~10–15% of that. minimum_size spares the tiny acks the overhead.
117
+ app.add_middleware(GZipMiddleware, minimum_size=1024)
118
+
119
+
120
+ @app.exception_handler(StarletteHTTPException)
121
+ def _error_shape(request: Request, exc: StarletteHTTPException):
122
+ """ONE error shape for every non-2xx (X2): `{"error": {"code", "message"}}`.
123
+
124
+ `deps.err` already raises detail in that shape; anything FastAPI raises on its own (a 404, a
125
+ 422 from a malformed path param) is wrapped here so a client never has to branch on two
126
+ different error bodies.
127
+ """
128
+ detail = exc.detail
129
+ if isinstance(detail, dict) and "error" in detail:
130
+ body = detail
131
+ else:
132
+ body = {"error": {"code": f"http_{exc.status_code}", "message": str(detail)}}
133
+ return JSONResponse(body, status_code=exc.status_code,
134
+ headers=getattr(exc, "headers", None))
135
+
136
+
137
+ @app.exception_handler(grid_events.StoreUnavailable)
138
+ def _store_unavailable(request: Request, exc: grid_events.StoreUnavailable):
139
+ """THE SAFETY NET for "the store is down" β†’ 503, from anywhere.
140
+
141
+ ⚠ WHY THIS IS APP-LEVEL AND NOT A `try` PER ROUTE. It was a try per route first, and a
142
+ `StoreUnavailable` raised while BUILDING THE PAYLOAD β€” before the route reached its own
143
+ try/except β€” surfaced as a 500. A store outage is a normal operational state and every route
144
+ here touches the store at least twice (the workspace read, then the write), so "remember to
145
+ wrap it" is a rule that gets forgotten once and then reports the wrong thing. One handler
146
+ means a store outage can only ever be a 503, whichever call raised it.
147
+
148
+ The seam raises this only when the caller passed no `fallback_ws` β€” i.e. exactly on this
149
+ adapter, which has no durable session dict to degrade into. A 200 over a write that
150
+ evaporated is the failure the whole rule exists to prevent.
151
+ """
152
+ return JSONResponse(
153
+ {"error": {"code": "store_unavailable",
154
+ "message": "the tenant store is unavailable β€” no change was saved"}},
155
+ status_code=503)
156
+
157
+
158
+ @app.get("/api/health")
159
+ def health():
160
+ """Unauthenticated LIVENESS only β€” it must answer before anyone can sign in, so it may not
161
+ reveal anything about the deployment beyond "the process is up". No version, no tenant list,
162
+ no config: a health endpoint is the one URL every scanner finds first."""
163
+ return {"ok": True}
164
+
165
+
166
+ app.include_router(routes_auth.router)
167
+ app.include_router(routes_nav.router)
168
+ app.include_router(routes_customers.router)
169
+ app.include_router(routes_products.router)
170
+ app.include_router(routes_assets.router)
171
+ app.include_router(routes_tables.router)
172
+ app.include_router(routes_grid.router)
173
+ app.include_router(routes_records.router)
174
+ # EXIT wave 2: the Y1 page-data envelope (one route for every ported dashboard) and Y4's user
175
+ # administration. `routes_pages` imports `pages`, which lazily imports each `pages_*` builder β€” so
176
+ # a new page is a builder module plus one registry line, and nothing here changes.
177
+ app.include_router(routes_pages.router)
178
+ app.include_router(routes_admin.router)
179
+ app.include_router(routes_automation.router)
180
+ app.include_router(routes_keychain.router)
181
+
182
+
183
+ # --- DEPRECATED ALIASES (removed when S2's shell flips; kept so the current bundle keeps working)
184
+ # They are the v1 handlers with the v1 session requirement β€” NOT the old unauthenticated Basic
185
+ # behavior. An alias that kept the old auth would be a bypass of everything above it.
186
+ _GATE = module_gate(routes_customers.MODULE)
187
+
188
+
189
+ @app.get("/api/customers", deprecated=True)
190
+ def _customers_alias(session: Session = Depends(_GATE)):
191
+ return routes_customers._payload(session)
192
+
193
+
194
+ @app.patch("/api/customers/{pid}", deprecated=True)
195
+ def _patch_alias(pid: int, body: dict = Body(default=None),
196
+ session: Session = Depends(_GATE)):
197
+ return routes_customers.patch_customer(pid, body, session)
198
+
199
+
200
+ def _assert_contract_floor():
201
+ """The canonical FIELDS must be exactly what `aios_grid` starts an empty workspace from.
202
+
203
+ This is what stops `FIELDS` becoming a constant that exists only to satisfy a gate. If the
204
+ canonical JSON and `aios_grid.fields_from_workspace({})` ever disagree, the standalone API and
205
+ the embedded host are serving two different schemas and the grid's own contract has forked β€”
206
+ the exact drift `verify_fields_contract.py` was written to catch, now also caught at startup
207
+ on whatever machine is actually running.
208
+ """
209
+ import aios_grid
210
+ base = [f for f in aios_grid.fields_from_workspace({}) if not f.get("custom")]
211
+ if [f["key"] for f in base] != [f["key"] for f in FIELDS]:
212
+ raise RuntimeError(
213
+ "AIOS web API: the canonical field contract and aios_grid.fields_from_workspace({}) "
214
+ "disagree on the base field set β€” embed and standalone would serve different schemas. "
215
+ "Run aios-web/verify_fields_contract.py.")
216
+
217
+
218
+ def _startup_notes():
219
+ """Say the two things an operator must know, once, at import β€” never in a response body."""
220
+ if aios_session.EPHEMERAL_SECRET:
221
+ print("[aios-api] AIOS_SESSION_SECRET is not set β€” signing with a random per-process "
222
+ "key. Sessions will not survive a restart and will not work across workers. "
223
+ "Set it in production.")
224
+ if os.environ.get("AIOS_INSECURE_SSL") == "1":
225
+ print("[aios-api] AIOS_INSECURE_SSL=1 β€” TLS verification is DISABLED for outbound "
226
+ "requests. Local development only; never in a deployed environment.")
227
+
228
+
229
+ _assert_contract_floor()
230
+ _startup_notes()
231
+
232
+
233
+ # β›” THE STATIC MOUNT IS NOW UNAUTHENTICATED, and that is a change this wave made on purpose.
234
+ # Before EXIT-3a, `BasicAuth` middleware gated the WHOLE app including this mount. A branded login
235
+ # page cannot live behind a password prompt, so the shell's own assets must be public β€” which they
236
+ # are: `index.html`, the JS/CSS bundle and the favicon reveal nothing.
237
+ #
238
+ # ⚠ WHAT THAT SILENTLY DE-GATED, caught in review rather than in production. `web/dist/` also
239
+ # carries `sample_customers.json`, a DEV FIXTURE copied from `web/public/`. Its 8 customers are
240
+ # synthetic, but the `agent` column holds REAL EMPLOYEE NAMES, and it went from Basic-gated to
241
+ # publicly fetchable in this commit. Nothing needs it: both the API bridge and `useCustomerData`
242
+ # deleted their sample fallback on purpose ("NOTHING HERE FALLS BACK TO sample_customers.json"),
243
+ # and it survives only because it sits in `web/public/`. So it is refused here β€” 404, the same
244
+ # answer as any other path that is not part of the app.
245
+ #
246
+ # The right long-term fix is deleting it from `web/public/` (S2's lane β€” flagged in the mailbox);
247
+ # this guard is what makes the API safe regardless of what the bundle happens to contain.
248
+ _DEV_FIXTURES = {"sample_customers.json"}
249
+
250
+
251
+ class _AppStatic(StaticFiles):
252
+ async def get_response(self, path, scope):
253
+ if Path(path).name in _DEV_FIXTURES:
254
+ raise StarletteHTTPException(status_code=404, detail="Not Found")
255
+ resp = await super().get_response(path, scope)
256
+ # Vite content-hashes everything under assets/ (a change is a NEW url), so those are
257
+ # immutable β€” a repeat visit re-downloads zero bytes instead of the whole 750 KB bundle.
258
+ # index.html must stay revalidated or a deploy would strand returning browsers on the old
259
+ # bundle; ETag/304 makes that revalidation a header exchange, not a transfer.
260
+ # ⚠ Normalised first: on a Windows host StaticFiles hands this path with backslashes,
261
+ # and `startswith("assets/")` silently skipped every asset (measured on the local probe).
262
+ if path.replace("\\", "/").lstrip("/").startswith("assets/"):
263
+ resp.headers["Cache-Control"] = "public, max-age=31536000, immutable"
264
+ else:
265
+ resp.headers["Cache-Control"] = "no-cache"
266
+ return resp
267
+
268
+
269
+ # static bundle LAST so /api/* wins; html=True serves index.html at / plus the built assets
270
+ if _WEB_DIST.is_dir():
271
+ app.mount("/", _AppStatic(directory=str(_WEB_DIST), html=True), name="web")
272
+
273
+
274
+ # --- boot prewarm (AIOS_PREWARM=1 β€” set by the Dockerfile, never by tests) -----------------------
275
+ # Without this, the first visitor after every deploy/restart pays the full Odoo pool build in
276
+ # their request. The thread warms the CONSOLIDATED scope (None, None) + every registered page's
277
+ # default envelope; scoped users still pay their own scope's first build, once.
278
+ # Env-gated rather than a startup event so importing `api.main` in a gate (verify_api and friends
279
+ # run against fakes) can never fire a live Odoo pull.
280
+ def _seed_and_sync_store():
281
+ """Bring the analytical store (harness.datastore) LIVE for this container β€” the app.py
282
+ bootstrap, mirrored (2026-07-31, owner item 1).
283
+
284
+ `harness.datastore` powers every measure column/condition, and ONLY app.py used to call
285
+ `ensure_seed()` β€” so on a fresh Space disk this container resolved measures against a store
286
+ that never existed and every measure cell served blank. Seeding alone was NOT enough either,
287
+ and that was measured live the same day: "datastore seeded in 1.7s" followed by an endless
288
+ `api:measure-column: ModelError the data cache is still warming up` β€” `ready()` demands
289
+ EVERY entity at phase 'live', and a seed that predates a newer entity leaves it un-synced
290
+ forever in a process with no sync loop. The SYNC SPRINT after the seed is what closes the
291
+ write_date gap and backfills anything the seed lacks (app.py:7779's exact pattern, bounded
292
+ passes). Fail-quiet throughout: no seed/token/Odoo β†’ the columns stay blank, the rows still
293
+ serve.
294
+ """
295
+ import time as _t
296
+ t0 = _t.time()
297
+ try:
298
+ from harness import datastore as _ds
299
+ if _ds.ensure_seed():
300
+ print(f"[aios-api] datastore seeded in {_t.time() - t0:.1f}s")
301
+ # DEDICATED Odoo connection for this thread (the W6 postmortem rule): the sync's
302
+ # search_reads must never interleave on the shared client's xmlrpc transport.
303
+ import core.odoo as _odoo
304
+ try:
305
+ _odoo._tlocal.client = _odoo.OdooClient()
306
+ except Exception:
307
+ pass
308
+ res = {}
309
+ for i in range(12):
310
+ res = _ds.sync_all(log=lambda *a, **k: None)
311
+ print(f"[aios-api] datastore sync pass {i + 1}: "
312
+ + ", ".join(f"{k}={v.get('phase')}" for k, v in sorted(res.items())))
313
+ if all(v.get("phase") == "live" for v in res.values()):
314
+ break
315
+ print(f"[aios-api] datastore sync done in {_t.time() - t0:.1f}s "
316
+ f"(ready={_ds.ready()})")
317
+ except Exception as e: # noqa: BLE001
318
+ print(f"[aios-api] datastore seed/sync skipped: {e}")
319
+
320
+
321
+ def _store_resync_loop():
322
+ """Keep the analytical mirror current β€” the API-process stand-in for the re-sync the
323
+ Streamlit app piggybacks on page renders. Measure memos key on the pool stamp, so a
324
+ refreshed pool re-reads the freshly synced store."""
325
+ import time as _t
326
+ while True:
327
+ _t.sleep(1800)
328
+ try:
329
+ from harness import datastore as _ds
330
+ import core.odoo as _odoo
331
+ try:
332
+ _odoo._tlocal.client = _odoo.OdooClient()
333
+ except Exception:
334
+ pass
335
+ _ds.sync_all(log=lambda *a, **k: None)
336
+ except Exception as e: # noqa: BLE001
337
+ print(f"[aios-api] store resync failed: {e}")
338
+
339
+
340
+ def _prewarm():
341
+ import time as _t
342
+ t0 = _t.time()
343
+ # ⚠ The store sync runs in its OWN thread, never ahead of the pool warm: a stale seed can
344
+ # take many minutes of XML-RPC to close, and the first live probe of this arrangement
345
+ # showed the pool warm (42s) silently queued behind it β€” the whole app cold for every
346
+ # visitor while a background column channel caught up. Measure cells retry via the
347
+ # transient rule until the sync lands; nothing else waits on it.
348
+ import threading as _th
349
+ _th.Thread(target=_seed_and_sync_store, daemon=True, name="store-seed-sync").start()
350
+ try:
351
+ from harness import runtime as _runtime
352
+ rt = _runtime.get_runtime("royal-imports")
353
+ routes_customers.warm_default(rt)
354
+ import pages as _pages
355
+ _pages.warm_default(rt)
356
+ print(f"[aios-api] prewarm done in {_t.time() - t0:.1f}s")
357
+ except Exception as e: # noqa: BLE001 β€” boot must not die on a warm-up
358
+ print(f"[aios-api] prewarm skipped: {e}")
359
+
360
+
361
+ if os.environ.get("AIOS_PREWARM") == "1":
362
+ import threading as _threading
363
+ _threading.Thread(target=_prewarm, daemon=True, name="prewarm").start()
364
+ _threading.Thread(target=_store_resync_loop, daemon=True, name="store-resync").start()
api/pages.py CHANGED
@@ -30,7 +30,7 @@ not interchangeable: `FMTS` for kpi/table/validation, and viz `FieldType` for a
30
  """
31
  import time
32
 
33
- # `deps` FIRST, deliberately: it is what puts `royalimports_os/` on `sys.path`, so importing it
34
  # ahead of any `core.*` makes this module importable in whatever order a caller reaches it.
35
  from deps import Session, err, perms # noqa: I001
36
 
 
30
  """
31
  import time
32
 
33
+ # `deps` FIRST, deliberately: it is what puts `platform/` on `sys.path`, so importing it
34
  # ahead of any `core.*` makes this module importable in whatever order a caller reaches it.
35
  from deps import Session, err, perms # noqa: I001
36
 
api/routes_assets.py CHANGED
@@ -1,165 +1,165 @@
1
- """routes_assets.py β€” product image service (wave 18, contract C2-ASSET).
2
-
3
- Serves the catalog's product imagery. Two sources, checked in order:
4
-
5
- 1. `AIOS_ASSET_DIR` β€” a local directory of master images whose FILENAME STEM is the product
6
- `code` (Odoo default_code, case-insensitive). Dev mode: the owner's master folder on this
7
- machine. Derivatives are generated on demand into a disk cache.
8
- 2. `AIOS_ASSET_REPO` β€” a private HF dataset (default `royal-imports/product-assets`) laid out
9
- by `royalimports_os/ingest_product_assets.py`:
10
- assets/products/orig/<CODE><ext> originals, archived verbatim (R9)
11
- assets/products/web/<CODE>.png ~800px alpha-preserving derivative
12
- manifest.json {"codes": {"<CODE>": {"ext": ".png"}}, ...}
13
- Files download once per container into the HF cache; responses carry immutable cache
14
- headers so the browser re-fetches nothing.
15
-
16
- QUALITY (R9): `q=web` = the low-res derivative the app displays; `q=print` and `q=orig` BOTH
17
- serve the ORIGINAL bytes β€” v1's "high quality" IS the master file (dated amendment in the wave
18
- doc). The gate is the PRODUCT grant: images decorate the product surface, so whoever may open
19
- `product_data` may fetch them; nobody else may.
20
-
21
- Codes are validated against a closed charset before they touch a filesystem or a repo path β€”
22
- a path separator can never reach the lookup, so traversal is refused at the door (400), and an
23
- unknown code is 404 `asset_not_found`, never an empty 200.
24
- """
25
- import os
26
- import re
27
- import tempfile
28
- import time
29
- from pathlib import Path
30
-
31
- from fastapi import Depends
32
- from fastapi import APIRouter
33
- from fastapi.responses import FileResponse
34
-
35
- from deps import Session, err, module_gate
36
-
37
- router = APIRouter(prefix="/api/v1")
38
-
39
- #: Same grant as the product surface these images decorate (routes_products.MODULE, restated
40
- #: here to avoid importing that module for one string).
41
- MODULE = "product_data"
42
-
43
- _WEB_EDGE = 800 # max edge (px) of the in-app derivative
44
- _QUALITIES = ("web", "print", "orig")
45
- _EXT_MEDIA = {".png": "image/png", ".jpg": "image/jpeg", ".jpeg": "image/jpeg"}
46
- #: closed charset β€” no slashes, no backslashes, no colons, so a code can never be a path.
47
- _CODE_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9 ._+&()\-]{0,63}$")
48
- _IMMUTABLE = "public, max-age=31536000, immutable"
49
-
50
- #: per-process index of the local master dir: rebuilt when the dir path or its mtime changes.
51
- _DIR_INDEX = {"path": None, "mtime": None, "map": {}}
52
-
53
-
54
- def _norm(code):
55
- return str(code or "").strip().upper()
56
-
57
-
58
- def _local_dir():
59
- d = os.environ.get("AIOS_ASSET_DIR", "").strip()
60
- if not d:
61
- return None
62
- p = Path(d)
63
- return p if p.is_dir() else None
64
-
65
-
66
- def _cache_root():
67
- root = Path(os.environ.get("AIOS_ASSET_CACHE", "").strip()
68
- or Path(tempfile.gettempdir()) / "aios_asset_cache")
69
- root.mkdir(parents=True, exist_ok=True)
70
- return root
71
-
72
-
73
- def _dir_index(d):
74
- """{CODE: Path} over the master dir. Two files sharing an upper-cased stem keep the
75
- lexicographically first β€” the same rule the ingest script applies, so dir mode and repo
76
- mode resolve identically."""
77
- try:
78
- mtime = d.stat().st_mtime_ns
79
- except OSError:
80
- return {}
81
- if _DIR_INDEX["path"] == str(d) and _DIR_INDEX["mtime"] == mtime:
82
- return _DIR_INDEX["map"]
83
- idx = {}
84
- for p in sorted(d.iterdir()):
85
- if not p.is_file() or p.name.startswith("."):
86
- continue
87
- if p.suffix.lower() not in _EXT_MEDIA:
88
- continue
89
- idx.setdefault(p.stem.upper(), p)
90
- _DIR_INDEX.update(path=str(d), mtime=mtime, map=idx)
91
- return idx
92
-
93
-
94
- def _web_derivative(src, code):
95
- """The ~800px alpha-preserving PNG for one master file, built lazily into the disk cache.
96
- Rebuilt when the master is newer than the cached copy."""
97
- out = _cache_root() / "web"
98
- out.mkdir(parents=True, exist_ok=True)
99
- dst = out / f"{code}.png"
100
- try:
101
- if dst.is_file() and dst.stat().st_mtime >= src.stat().st_mtime:
102
- return dst
103
- except OSError:
104
- pass
105
- try:
106
- from PIL import Image
107
- except ImportError:
108
- raise err(503, "pillow_missing",
109
- "image derivatives need Pillow β€” `pip install pillow` on this host")
110
- with Image.open(src) as im:
111
- im.load()
112
- if im.mode not in ("RGBA", "RGB", "LA", "L"):
113
- im = im.convert("RGBA")
114
- im.thumbnail((_WEB_EDGE, _WEB_EDGE))
115
- im.save(dst, format="PNG", optimize=True)
116
- return dst
117
-
118
-
119
- def _assets():
120
- """The asset SOURCE lives in `royalimports_os/core/assets.py`, not here.
121
-
122
- β›” NOT TIDINESS β€” a CONTRACT. `ops/verify_portability.py` B1/B2 assert the HuggingFace SDK
123
- appears only in its declared homes and NEVER under `aios-web/api/`, because this process is
124
- meant to lift to Hetzner/EC2 without a hub client. The first version of this file imported
125
- `huggingface_hub` directly and the gate caught it."""
126
- import core.assets as assets
127
- return assets
128
-
129
-
130
- @router.get("/assets/products")
131
- def asset_manifest(session: Session = Depends(module_gate(MODULE))):
132
- """The codes that HAVE an image β€” so the catalog designer can mark gaps instead of
133
- rendering broken tiles. `source` names which backend answered."""
134
- d = _local_dir()
135
- if d is not None:
136
- return {"codes": sorted(_dir_index(d).keys()), "source": "dir"}
137
- codes = _assets().manifest()
138
- if codes:
139
- return {"codes": sorted(codes.keys()), "source": "repo"}
140
- return {"codes": [], "source": "none"}
141
-
142
-
143
- @router.get("/assets/products/{code}")
144
- def product_asset(code: str, q: str = "web",
145
- session: Session = Depends(module_gate(MODULE))):
146
- if q not in _QUALITIES:
147
- raise err(400, "bad_quality", f"q must be one of {', '.join(_QUALITIES)}")
148
- norm = _norm(code)
149
- if not _CODE_RE.match(norm):
150
- raise err(400, "bad_code", "that is not a product code")
151
-
152
- path = None
153
- d = _local_dir()
154
- if d is not None:
155
- src = _dir_index(d).get(norm)
156
- if src is not None:
157
- path = _web_derivative(src, norm) if q == "web" else src
158
- if path is None:
159
- path = _assets().fetch(norm, q)
160
- if path is None or not Path(path).is_file():
161
- raise err(404, "asset_not_found", f"no image for {norm}")
162
-
163
- media = _EXT_MEDIA.get(Path(path).suffix.lower(), "application/octet-stream")
164
- return FileResponse(str(path), media_type=media,
165
- headers={"Cache-Control": _IMMUTABLE})
 
1
+ """routes_assets.py β€” product image service (wave 18, contract C2-ASSET).
2
+
3
+ Serves the catalog's product imagery. Two sources, checked in order:
4
+
5
+ 1. `AIOS_ASSET_DIR` β€” a local directory of master images whose FILENAME STEM is the product
6
+ `code` (Odoo default_code, case-insensitive). Dev mode: the owner's master folder on this
7
+ machine. Derivatives are generated on demand into a disk cache.
8
+ 2. `AIOS_ASSET_REPO` β€” a private HF dataset (default `royal-imports/product-assets`) laid out
9
+ by `platform/ingest_product_assets.py`:
10
+ assets/products/orig/<CODE><ext> originals, archived verbatim (R9)
11
+ assets/products/web/<CODE>.png ~800px alpha-preserving derivative
12
+ manifest.json {"codes": {"<CODE>": {"ext": ".png"}}, ...}
13
+ Files download once per container into the HF cache; responses carry immutable cache
14
+ headers so the browser re-fetches nothing.
15
+
16
+ QUALITY (R9): `q=web` = the low-res derivative the app displays; `q=print` and `q=orig` BOTH
17
+ serve the ORIGINAL bytes β€” v1's "high quality" IS the master file (dated amendment in the wave
18
+ doc). The gate is the PRODUCT grant: images decorate the product surface, so whoever may open
19
+ `product_data` may fetch them; nobody else may.
20
+
21
+ Codes are validated against a closed charset before they touch a filesystem or a repo path β€”
22
+ a path separator can never reach the lookup, so traversal is refused at the door (400), and an
23
+ unknown code is 404 `asset_not_found`, never an empty 200.
24
+ """
25
+ import os
26
+ import re
27
+ import tempfile
28
+ import time
29
+ from pathlib import Path
30
+
31
+ from fastapi import Depends
32
+ from fastapi import APIRouter
33
+ from fastapi.responses import FileResponse
34
+
35
+ from deps import Session, err, module_gate
36
+
37
+ router = APIRouter(prefix="/api/v1")
38
+
39
+ #: Same grant as the product surface these images decorate (routes_products.MODULE, restated
40
+ #: here to avoid importing that module for one string).
41
+ MODULE = "product_data"
42
+
43
+ _WEB_EDGE = 800 # max edge (px) of the in-app derivative
44
+ _QUALITIES = ("web", "print", "orig")
45
+ _EXT_MEDIA = {".png": "image/png", ".jpg": "image/jpeg", ".jpeg": "image/jpeg"}
46
+ #: closed charset β€” no slashes, no backslashes, no colons, so a code can never be a path.
47
+ _CODE_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9 ._+&()\-]{0,63}$")
48
+ _IMMUTABLE = "public, max-age=31536000, immutable"
49
+
50
+ #: per-process index of the local master dir: rebuilt when the dir path or its mtime changes.
51
+ _DIR_INDEX = {"path": None, "mtime": None, "map": {}}
52
+
53
+
54
+ def _norm(code):
55
+ return str(code or "").strip().upper()
56
+
57
+
58
+ def _local_dir():
59
+ d = os.environ.get("AIOS_ASSET_DIR", "").strip()
60
+ if not d:
61
+ return None
62
+ p = Path(d)
63
+ return p if p.is_dir() else None
64
+
65
+
66
+ def _cache_root():
67
+ root = Path(os.environ.get("AIOS_ASSET_CACHE", "").strip()
68
+ or Path(tempfile.gettempdir()) / "aios_asset_cache")
69
+ root.mkdir(parents=True, exist_ok=True)
70
+ return root
71
+
72
+
73
+ def _dir_index(d):
74
+ """{CODE: Path} over the master dir. Two files sharing an upper-cased stem keep the
75
+ lexicographically first β€” the same rule the ingest script applies, so dir mode and repo
76
+ mode resolve identically."""
77
+ try:
78
+ mtime = d.stat().st_mtime_ns
79
+ except OSError:
80
+ return {}
81
+ if _DIR_INDEX["path"] == str(d) and _DIR_INDEX["mtime"] == mtime:
82
+ return _DIR_INDEX["map"]
83
+ idx = {}
84
+ for p in sorted(d.iterdir()):
85
+ if not p.is_file() or p.name.startswith("."):
86
+ continue
87
+ if p.suffix.lower() not in _EXT_MEDIA:
88
+ continue
89
+ idx.setdefault(p.stem.upper(), p)
90
+ _DIR_INDEX.update(path=str(d), mtime=mtime, map=idx)
91
+ return idx
92
+
93
+
94
+ def _web_derivative(src, code):
95
+ """The ~800px alpha-preserving PNG for one master file, built lazily into the disk cache.
96
+ Rebuilt when the master is newer than the cached copy."""
97
+ out = _cache_root() / "web"
98
+ out.mkdir(parents=True, exist_ok=True)
99
+ dst = out / f"{code}.png"
100
+ try:
101
+ if dst.is_file() and dst.stat().st_mtime >= src.stat().st_mtime:
102
+ return dst
103
+ except OSError:
104
+ pass
105
+ try:
106
+ from PIL import Image
107
+ except ImportError:
108
+ raise err(503, "pillow_missing",
109
+ "image derivatives need Pillow β€” `pip install pillow` on this host")
110
+ with Image.open(src) as im:
111
+ im.load()
112
+ if im.mode not in ("RGBA", "RGB", "LA", "L"):
113
+ im = im.convert("RGBA")
114
+ im.thumbnail((_WEB_EDGE, _WEB_EDGE))
115
+ im.save(dst, format="PNG", optimize=True)
116
+ return dst
117
+
118
+
119
+ def _assets():
120
+ """The asset SOURCE lives in `platform/core/assets.py`, not here.
121
+
122
+ β›” NOT TIDINESS β€” a CONTRACT. `ops/verify_portability.py` B1/B2 assert the HuggingFace SDK
123
+ appears only in its declared homes and NEVER under `aios-web/api/`, because this process is
124
+ meant to lift to Hetzner/EC2 without a hub client. The first version of this file imported
125
+ `huggingface_hub` directly and the gate caught it."""
126
+ import core.assets as assets
127
+ return assets
128
+
129
+
130
+ @router.get("/assets/products")
131
+ def asset_manifest(session: Session = Depends(module_gate(MODULE))):
132
+ """The codes that HAVE an image β€” so the catalog designer can mark gaps instead of
133
+ rendering broken tiles. `source` names which backend answered."""
134
+ d = _local_dir()
135
+ if d is not None:
136
+ return {"codes": sorted(_dir_index(d).keys()), "source": "dir"}
137
+ codes = _assets().manifest()
138
+ if codes:
139
+ return {"codes": sorted(codes.keys()), "source": "repo"}
140
+ return {"codes": [], "source": "none"}
141
+
142
+
143
+ @router.get("/assets/products/{code}")
144
+ def product_asset(code: str, q: str = "web",
145
+ session: Session = Depends(module_gate(MODULE))):
146
+ if q not in _QUALITIES:
147
+ raise err(400, "bad_quality", f"q must be one of {', '.join(_QUALITIES)}")
148
+ norm = _norm(code)
149
+ if not _CODE_RE.match(norm):
150
+ raise err(400, "bad_code", "that is not a product code")
151
+
152
+ path = None
153
+ d = _local_dir()
154
+ if d is not None:
155
+ src = _dir_index(d).get(norm)
156
+ if src is not None:
157
+ path = _web_derivative(src, norm) if q == "web" else src
158
+ if path is None:
159
+ path = _assets().fetch(norm, q)
160
+ if path is None or not Path(path).is_file():
161
+ raise err(404, "asset_not_found", f"no image for {norm}")
162
+
163
+ media = _EXT_MEDIA.get(Path(path).suffix.lower(), "application/octet-stream")
164
+ return FileResponse(str(path), media_type=media,
165
+ headers={"Cache-Control": _IMMUTABLE})
platform/aios_grid.py ADDED
@@ -0,0 +1,1870 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """aios_grid β€” embed the AIOS React/glide Airtable-style grid inside Streamlit.
2
+
3
+ This is REUSABLE MODULE INFRASTRUCTURE: the same self-contained grid that runs as the
4
+ standalone aios-web app is inlined into a single HTML file and hosted inside a Streamlit
5
+ component. The React bundle reads its data from `window.__AIOS_GRID__` (an object
6
+ `{fields, rows}`) that we inject into the page <head> before the app's module script runs
7
+ β€” so there is NO backend and NO /api call in the Streamlit container; the browser only ever
8
+ sees the derived JSON we hand it.
9
+
10
+ Usage (from any page):
11
+ import aios_grid
12
+ aios_grid.render(aios_grid.rows_from_pool(pool_rows), aios_grid.FIELDS)
13
+
14
+ Design notes:
15
+ * ZERO app-internal imports (no `modules.*` / `core.*`). FIELDS is a literal and
16
+ `rows_from_pool` takes already-built rows β€” so this helper is tenant/module-agnostic and
17
+ sidesteps deploy_hf.py's import guard entirely.
18
+ * The built HTML ships as `aios_grid_embed.html` in THIS directory (added to
19
+ deploy_hf.py INCLUDE). Build it with `npm run build:embed` in aios-web/web, then copy
20
+ dist-embed/index.html -> platform/aios_grid_embed.html (see build_embed.py).
21
+ For LOCAL dev before that copy, we fall back to reading dist-embed/index.html directly.
22
+ * The preferred host is a Streamlit Components v1 bridge. It sends data/view/schema args
23
+ into the React app and returns guarded events for persistence in the tenant store. The old
24
+ injected-HTML path remains a read/local-write fallback when component assets are absent.
25
+ """
26
+ import json
27
+ import re
28
+ from pathlib import Path
29
+
30
+ _HERE = Path(__file__).resolve().parent
31
+
32
+ # Where the inlined single-file build lives. FIRST match wins:
33
+ # 1. the shipped copy in this dir (what deploy_hf.py uploads to the Space)
34
+ # 2. the raw build output in the sibling aios-web tree (local dev, pre-copy)
35
+ _EMBED_CANDIDATES = [
36
+ _HERE / "aios_grid_embed.html",
37
+ _HERE.parent / "aios-web" / "web" / "dist-embed" / "index.html",
38
+ ]
39
+ _COMPONENT_CANDIDATES = [
40
+ _HERE / "aios_grid_component",
41
+ _HERE.parent / "aios-web" / "web" / "dist-embed",
42
+ ]
43
+ _DECLARED_COMPONENTS = {}
44
+
45
+ # --- the FIELD CONTRACT β€” loaded from the CANONICAL source `aios_grid_fields.json` in THIS
46
+ # directory (the SINGLE source of truth, shared with aios-web/api/main.py). source='odoo' is
47
+ # READ-ONLY; source='overlay' is the editable stratum (notes/tags) that lives OUTSIDE Odoo.
48
+ #
49
+ # Why a sibling JSON and NOT an import: aios_grid.py's contract is ZERO app-internal imports so
50
+ # it stays tenant/module-agnostic and sidesteps deploy_hf.py's import guard. A JSON next to the
51
+ # module preserves that exactly β€” no import, no guard interaction β€” while still single-sourcing
52
+ # the values (a build-time copy from another tree would reintroduce the drift we're removing).
53
+ # The file ships to the Space via deploy_hf.py INCLUDE. Edit the JSON, then run
54
+ # aios-web/verify_fields_contract.py. ---
55
+ _FIELDS_PATH = _HERE / "aios_grid_fields.json"
56
+
57
+
58
+ def _load_fields():
59
+ if not _FIELDS_PATH.is_file():
60
+ raise FileNotFoundError(
61
+ f"aios_grid: canonical field contract missing at {_FIELDS_PATH}. It is the single "
62
+ "source of truth for the grid schema and MUST ship (deploy_hf.py INCLUDE lists it)."
63
+ )
64
+ doc = json.loads(_FIELDS_PATH.read_text(encoding="utf-8"))
65
+ fields = doc.get("fields") if isinstance(doc, dict) else doc
66
+ if not isinstance(fields, list) or not fields:
67
+ raise ValueError(f"aios_grid: {_FIELDS_PATH} has no 'fields' list.")
68
+ return fields
69
+
70
+
71
+ FIELDS = _load_fields()
72
+
73
+
74
+ def product_fields():
75
+ """The PRODUCT table's canonical contract (wave 15 C-TOPIC, `product_data` key in the same
76
+ JSON). A separate accessor rather than a second module constant so the one file-read and the
77
+ one failure mode stay shared with `FIELDS`."""
78
+ doc = json.loads(_FIELDS_PATH.read_text(encoding="utf-8"))
79
+ fields = (doc.get("product_data") or {}).get("fields")
80
+ if not isinstance(fields, list) or not fields:
81
+ raise ValueError(f"aios_grid: {_FIELDS_PATH} has no product_data.fields list.")
82
+ return fields
83
+
84
+ # text/date fields pass through untouched; every OTHER odoo (numeric) field is rounded β€”
85
+ # mirrors aios-web/api/main.py _payload() exactly so embed == standalone byte-for-byte.
86
+
87
+
88
+ def _round(v):
89
+ return round(v) if isinstance(v, (int, float)) and not isinstance(v, bool) else v
90
+
91
+
92
+ # Types a USER may create from the column menu (owner item 7, 2026-07-26). Mirrors
93
+ # customer-grid/types.ts CREATABLE_TYPES; verify_fields_contract.py holds the two in step.
94
+ # select β€” a single-select with its own `options` (the "Status" a user wants to add; distinct
95
+ # from the Odoo-sourced `status` lifecycle, which is not user-defined)
96
+ # user β€” an assignee, whose choices come from the HOST's real user list, never from here
97
+ #: `multiselect` (wave-2 item 5, 2026-07-27): the Airtable-style MULTI select β€” declared
98
+ #: options like `select`, but the cell holds a comma-joined SET and the row belongs to every
99
+ #: member of it (the `multi` grouping/cell contract the Cohorts column established).
100
+ #: Wave-5 item 11 (2026-07-27): `checkbox` (cell = bool; the overlay stores '1' or '') Β·
101
+ #: `phone` / `email` / `url` (text-family with per-type rendering) Β· `rating` (top-level
102
+ #: `max`, SVG stars client-side β€” never emoji) Β· `created_time` (READ-ONLY, renders the row's
103
+ #: `_created`) Β· `formula` (READ-ONLY, client-computed from the row's other cells).
104
+ CUSTOM_FIELD_TYPES = {"text", "select", "multiselect", "user", "int", "currency", "pct", "date",
105
+ "checkbox", "phone", "email", "url", "rating", "created_time", "formula",
106
+ "automation"}
107
+ #: User-created types whose CELLS are read-only: their values are computed (formula β€” client
108
+ #: side, any error degrades to BLANK) or system-owned (created_time = the row's `_created`).
109
+ #: Emitted with the cohort column's read-only mechanism β€” `source: 'odoo'` + `derived` β€” so
110
+ #: the client never offers an editor and the host never accepts a cell write for them.
111
+ READONLY_CUSTOM_TYPES = {"created_time", "formula"}
112
+ MAX_FIELD_OPTIONS = 50
113
+ MAX_FORMULA_LEN = 500
114
+ #: `rating` bounds. Airtable caps at 10; below 2 a rating is a checkbox.
115
+ RATING_MAX_DEFAULT, RATING_MAX_MIN, RATING_MAX_MAX = 5, 2, 10
116
+
117
+ #: Key prefix of a FORMULA-MEASURE column (owner item 7, 2026-07-27): a user-created field that
118
+ #: IS a measure over a window β€” `Sales Β· the last 90 days` as a column. Mirrors the client's
119
+ #: `measure_` keys in CustomerGrid.createField. Distinct from `custom_` because the two strata
120
+ #: could not be more different: `custom_` is the EDITABLE overlay (user-typed values), while a
121
+ #: measure field is READ-ONLY and its values are computed by the host per render.
122
+ MEASURE_FIELD_PREFIX = "measure_"
123
+ #: The numeric types a measure can render as (semantic._FORMAT_TYPE's range).
124
+ MEASURE_FIELD_TYPES = {"currency", "int", "pct"}
125
+
126
+
127
+ def _clean_options(raw):
128
+ """Choices for a `select`: strings, trimmed, de-duplicated case-insensitively, capped.
129
+ Mirrors types.ts parseOptions β€” a choice list that means one thing in the picker and
130
+ another in the filter dropdown is a column with two vocabularies."""
131
+ out, seen = [], set()
132
+ for v in list(raw or [])[:MAX_FIELD_OPTIONS * 2]:
133
+ if not isinstance(v, (str, int, float)) or isinstance(v, bool):
134
+ continue
135
+ s = str(v).strip()[:120]
136
+ if not s or s.lower() in seen:
137
+ continue
138
+ seen.add(s.lower())
139
+ out.append(s)
140
+ return out[:MAX_FIELD_OPTIONS]
141
+
142
+
143
+ def _clean_option_colors(raw, options):
144
+ """Choice-label -> #RRGGBB, limited to the field's canonical option vocabulary."""
145
+ if not isinstance(raw, dict):
146
+ return {}
147
+ supplied = {}
148
+ for label, color in raw.items():
149
+ if not isinstance(label, str) or not isinstance(color, str):
150
+ continue
151
+ clean = color.strip().upper()
152
+ if re.fullmatch(r"#[0-9A-F]{6}", clean):
153
+ supplied[label.strip().lower()] = clean
154
+ out = {}
155
+ for option in options or []:
156
+ color = supplied.get(str(option).strip().lower())
157
+ if color:
158
+ out[str(option)] = color
159
+ return out
160
+
161
+
162
+ def _choice_appearance(raw, options):
163
+ """Validated select-family appearance. Absent colour toggle means legacy-on."""
164
+ if not isinstance(raw, dict):
165
+ return {}
166
+ out = {}
167
+ if isinstance(raw.get("colorCodeOptions"), bool):
168
+ out["colorCodeOptions"] = raw["colorCodeOptions"]
169
+ colors = _clean_option_colors(raw.get("optionColors"), options)
170
+ if colors:
171
+ out["optionColors"] = colors
172
+ return out
173
+
174
+
175
+ def _clean_rating_max(raw):
176
+ """A rating's star count, bounded. Anything unusable is the default, not a refusal β€” the
177
+ field still holds its 1..max integers either way."""
178
+ try:
179
+ return max(RATING_MAX_MIN, min(int(raw), RATING_MAX_MAX))
180
+ except (TypeError, ValueError):
181
+ return RATING_MAX_DEFAULT
182
+
183
+
184
+ #: The field types a number-style display format may apply to. `formula` is here because its
185
+ #: RESULT is a number the client renders; `pct` already renders in points and takes decimals.
186
+ _NUMBER_FORMAT_TYPES = {"int", "currency", "pct", "formula"}
187
+
188
+
189
+ def _clean_format(raw, ftype):
190
+ """Per-type DISPLAY format (wave-5 item 10), fail-closed: unknown keys are DROPPED, wrong
191
+ types return None (the property is simply absent). Rendering-only β€” a format can change how
192
+ a value reads, never what it is, which is why this needs no parity gate of its own."""
193
+ if not isinstance(raw, dict):
194
+ return None
195
+ out = {}
196
+ if ftype in _NUMBER_FORMAT_TYPES:
197
+ if isinstance(raw.get("thousands"), bool):
198
+ out["thousands"] = raw["thousands"]
199
+ if raw.get("decimals") is not None:
200
+ try:
201
+ d = int(raw["decimals"])
202
+ except (TypeError, ValueError):
203
+ d = None
204
+ if d is not None and 0 <= d <= 4:
205
+ out["decimals"] = d
206
+ if isinstance(raw.get("abbrev"), bool):
207
+ out["abbrev"] = raw["abbrev"]
208
+ elif ftype in ("date", "created_time"):
209
+ if isinstance(raw.get("time"), bool):
210
+ out["time"] = raw["time"]
211
+ if raw.get("tz") in ("local", "utc"):
212
+ out["tz"] = raw["tz"]
213
+ return out or None
214
+
215
+
216
+ def _clean_permissions(raw):
217
+ """`{edit: 'everyone' | 'creator' | 'admins'}` or None (wave-5 item 1). WHO may set it is
218
+ the host handler's business (creator/admin, enforced fail-closed there); this validates
219
+ only the shape, like every other property here."""
220
+ if isinstance(raw, dict) and raw.get("edit") in ("everyone", "creator", "admins"):
221
+ return {"edit": raw["edit"]}
222
+ return None
223
+
224
+
225
+ #: β›” THE CHARSET MUST ADMIT EVERY TOKEN THE CLIENT ENGINE PARSES, or a legal formula is
226
+ #: refused by a wall that is supposed to be structural (2026-08-03).
227
+ #:
228
+ #: This regex was written when a formula was arithmetic over refs. On 2026-07-31 the client
229
+ #: engine (owner item 2) gained STRING literals, `&` concatenation and `^` β€” CONCATENATE, TEXT,
230
+ #: LEFT/RIGHT/MID, and any `IF(cond, "yes", "no")`. This list was never widened to match, so
231
+ #: every such formula died here: `field_upsert` refused the create, and `fields_from_workspace`
232
+ #: dropped the column on read. Nothing went red β€” a refused create looks like a quiet failure
233
+ #: and a dropped column looks like a column nobody made.
234
+ #:
235
+ #: Found by trying to ship the owner's own Buy signal formula, which is `IF(..., "Buy now",
236
+ #: "OK")` and could not be created through the product UI at all.
237
+ #:
238
+ #: ⚠ IT IS STILL STRUCTURAL, and deliberately not a second grammar β€” that is the filter_sql-class
239
+ #: drift risk this file's docstring names. The charset says which characters may appear; the
240
+ #: engine says what they mean. `_quotes_balanced` below is the one structural rule the quote
241
+ #: character brings with it.
242
+ _FORMULA_CHARS = re.compile(r"^[\w\s{}()+\-*/.,<>=!^&\"]*$")
243
+ _FORMULA_REF = re.compile(r"\{([^{}]*)\}")
244
+
245
+
246
+ def _quotes_balanced(s):
247
+ """An even number of `"` β€” the structural half of string support.
248
+
249
+ Sound because the engine's own escape is Excel's: `""` inside a string is one quote, and it
250
+ contributes TWO characters. So a well-formed expression always has an even count and an
251
+ unterminated string always has an odd one. What a balanced pair MEANS is the engine's
252
+ business, exactly as with parentheses.
253
+ """
254
+ return s.count('"') % 2 == 0
255
+
256
+
257
+ #: Wave-18 C5-AUTOFIELD. `kind` is a whitelist because an unknown kind would be a column that
258
+ #: silently never runs; `source` names where the run's subject comes from.
259
+ AUTOMATION_KINDS = {"instagram_profile"}
260
+ AUTOMATION_SOURCES = {"record_url_field", "self"}
261
+ MAX_AUTOMATION_SETTINGS = 12
262
+
263
+
264
+ def _clean_automation(raw, valid_keys=None):
265
+ """Validate an `automation` config bag. Returns the clean dict, or None when there is
266
+ nothing valid to store (the column then renders as unconfigured β€” never invented)."""
267
+ if not isinstance(raw, dict):
268
+ return None
269
+ kind = str(raw.get('kind') or '').strip()
270
+ if kind not in AUTOMATION_KINDS:
271
+ return None
272
+ source = str(raw.get('source') or 'record_url_field').strip()
273
+ if source not in AUTOMATION_SOURCES:
274
+ source = 'record_url_field'
275
+ out = {'kind': kind, 'source': source}
276
+ url_field = str(raw.get('urlField') or '').strip()[:80]
277
+ # fail closed on a ref that does not exist, exactly as _clean_formula does at WRITE time
278
+ if url_field and (valid_keys is None or url_field in valid_keys):
279
+ out['urlField'] = url_field
280
+ settings = {}
281
+ for k, v in list((raw.get('settings') or {}).items())[:MAX_AUTOMATION_SETTINGS]:
282
+ if isinstance(v, bool) or isinstance(v, (int, float)):
283
+ settings[str(k)[:40]] = v
284
+ elif isinstance(v, str):
285
+ settings[str(k)[:40]] = v[:200]
286
+ if settings:
287
+ out['settings'] = settings
288
+ return out
289
+
290
+
291
+ def _clean_formula(raw, valid_keys=None):
292
+ """STRUCTURAL passthrough for a formula field's expression (wave-5 item 9).
293
+
294
+ Meaning is NOT checked here: the CLIENT engine owns the grammar (arithmetic over `{field}`
295
+ refs, ABS/ROUND/MIN/MAX/IF), and any evaluation error degrades to a BLANK cell β€” never a
296
+ wrong number. That is the `_clean_window` split one stratum up, and deliberately NOT a
297
+ Python mirror of the grammar: a second engine is the filter_sql-class drift risk.
298
+ Structure IS checked β€” charset, length, balanced parens, BALANCED QUOTES, well-formed
299
+ non-empty `{refs}` β€” and at WRITE time (`valid_keys` given) every ref must name a field this
300
+ table has, fail closed. At READ time refs are left alone: a referenced field deleted later
301
+ must blank the CELLS, not vaporise the column.
302
+
303
+ ⚠ A `{ref}` INSIDE A STRING LITERAL is still checked against `valid_keys` at write time, so
304
+ `IF(x, "see {notafield}", "")` is refused. That is a false rejection and it is the
305
+ fail-closed direction: the alternative is teaching this function where strings begin and
306
+ end, which is the second grammar the paragraph above refuses to write.
307
+ """
308
+ if not isinstance(raw, str):
309
+ return None
310
+ s = raw.strip()
311
+ if not s or len(s) > MAX_FORMULA_LEN or not _FORMULA_CHARS.match(s):
312
+ return None
313
+ depth = 0
314
+ for ch in s:
315
+ if ch == "(":
316
+ depth += 1
317
+ elif ch == ")":
318
+ depth -= 1
319
+ if depth < 0:
320
+ return None
321
+ if depth:
322
+ return None
323
+ if not _quotes_balanced(s):
324
+ return None
325
+ refs = _FORMULA_REF.findall(s)
326
+ leftover = _FORMULA_REF.sub("", s)
327
+ if "{" in leftover or "}" in leftover: # unbalanced / nested braces
328
+ return None
329
+ if any(not r.strip() for r in refs): # a `{}` ref names nothing
330
+ return None
331
+ if valid_keys is not None and any(r not in valid_keys for r in refs):
332
+ return None
333
+ return s
334
+
335
+
336
+ def _field_extras(saved, ftype):
337
+ """createdBy / permissions / format / scope β€” the validated passthrough the created strata
338
+ share (wave 5). `createdBy` is only ever WRITTEN host-side (the handler stamps it); here it
339
+ is carried so the client can gate its menus and the handler can enforce against it.
340
+ `scope` (wave-6 item 9): 'cohort' marks a field created as cohort-specific β€” the handler
341
+ stamps it at create (cohort page only) and preserves it like createdBy; carried here so the
342
+ client can label the field, filtered OUT of other pages by fields_from_workspace."""
343
+ out = {}
344
+ who = saved.get("createdBy")
345
+ if isinstance(who, str) and who.strip():
346
+ out["createdBy"] = who.strip()[:80]
347
+ perms = _clean_permissions(saved.get("permissions"))
348
+ if perms:
349
+ out["permissions"] = perms
350
+ fmt = _clean_format(saved.get("format"), ftype)
351
+ if fmt:
352
+ out["format"] = fmt
353
+ if saved.get("scope") == "cohort":
354
+ out["scope"] = "cohort"
355
+ corrected_from = saved.get("labelCorrectedFrom")
356
+ correction_id = saved.get("labelCorrectionId")
357
+ if (isinstance(corrected_from, str) and corrected_from.strip()
358
+ and isinstance(correction_id, str) and correction_id.strip()):
359
+ out["labelCorrectedFrom"] = corrected_from.strip()[:120]
360
+ out["labelCorrectionId"] = correction_id.strip()[:180]
361
+ return out
362
+
363
+
364
+ #: The DERIVED column listing the cohorts a customer is in (owner, 2026-07-27).
365
+ #:
366
+ #: NOT in `aios_grid_fields.json`, deliberately. That contract is per-TABLE and shared with the
367
+ #: standalone API and the dev sample; a cohort is per-USER, so the column exists exactly when the
368
+ #: caller has cohorts β€” the same condition under which the `__cohort__` FILTER field is offered.
369
+ #: Putting it in the canonical contract would mean an always-present column that is empty for
370
+ #: everyone else, plus three consumers to keep in step for a value none of them can produce.
371
+ COHORT_COLUMN = 'cohorts'
372
+
373
+
374
+ def cohort_field(label='Locked views'):
375
+ """The derived membership column's descriptor.
376
+
377
+ ⚠ WAVE 17 item 14 / C-STR β€” THE LABEL AND THE NOTE SPEAK THE NEW VOCABULARY; THE KEY DOES
378
+ NOT. `COHORT_COLUMN` is still `'cohorts'` and the function is still `cohort_field`, because
379
+ every stored view that shows or groups by this column names it by KEY, and every gate in
380
+ two runtimes names the function. The owner renamed a CONCEPT ("we should stop calling it
381
+ Cohort, but locked instead"), which is a change to what a reader sees β€” renaming the
382
+ identifiers would break saved views to change a word nobody reads.
383
+
384
+ `source: 'odoo'` is doing ONE job here and it is not provenance: the client keys editability
385
+ off `source == 'overlay'`, and `_cl_handle_grid_event` accepts cell writes only for overlay
386
+ keys. So 'odoo' is what makes this column READ-ONLY at both ends. `derived: True` is what
387
+ stops the column menu calling it a "source field" on that basis.
388
+
389
+ ⚠ `filterable: False`, and the replacement is the `Where [Cohort] […]` CONDITION, not another
390
+ column. Text ops over a joined string would ALMOST work and disagree at the edges β€” `contains
391
+ "VIP"` also matches a cohort called "VIP club" β€” and a filter that is nearly right is worse
392
+ than one that is absent.
393
+ """
394
+ return {
395
+ 'key': COHORT_COLUMN, 'label': label, 'type': 'text', 'source': 'odoo',
396
+ 'default': False, 'filterable': False, 'derived': True, 'multi': True,
397
+ 'note': 'The locked views this customer is in, newest first. A locked view holds a SET, '
398
+ 'so grouping by this column puts a customer under EVERY view they are in β€” the '
399
+ 'group counts therefore add up to more than the record count, which stays the '
400
+ 'number of distinct customers. Read-only: membership changes only by adding or '
401
+ 'removing customers on the locked view itself.',
402
+ }
403
+
404
+
405
+ def cohort_cells(cohorts, allowed_pids=None):
406
+ """`{pid: {'cohorts': 'Q3 calls, Lost'}}` from `[{id,name,pids}, ...]`.
407
+
408
+ Built per render from the caller's OWN cohorts and handed to `rows_from_pool` as `derived`,
409
+ never merged into the cached pool rows β€” those are shared across users, and stamping one
410
+ user's cohorts onto them would leak the membership to everybody else on the next render.
411
+ """
412
+ cells = {}
413
+ for c in cohorts or []:
414
+ # ⚠ The COMMA is the separator the client splits on to group a customer into EVERY
415
+ # cohort they are in, so it cannot also occur inside a name. Cohort names are free text
416
+ # ("Q3 calls, west" is a name somebody will type), so a comma is replaced here rather
417
+ # than left to break the split silently β€” one group called "Q3 calls" and another called
418
+ # "west" would be two lists that do not exist. The cost is cosmetic and confined to the
419
+ # cell; the Cohort page still shows the name the user typed.
420
+ name = str(c.get('name') or c.get('id') or '').replace(',', ' ').strip()
421
+ for pid in c.get('pids') or ():
422
+ if allowed_pids is not None and pid not in allowed_pids:
423
+ continue
424
+ cells.setdefault(pid, []).append(name)
425
+ return {pid: {COHORT_COLUMN: ', '.join(names)} for pid, names in cells.items()}
426
+
427
+
428
+ def clean_measure_field(raw, offered):
429
+ """Validate one UNTRUSTED formula-measure field (owner item 7) against the caller's OFFER.
430
+
431
+ `offered` is `{measure key: {label, type}}` from `measure_filter.measure_fields(team_id)` β€”
432
+ the same admission the condition builder uses, so a field can only name a measure this
433
+ caller could also filter by. Returns the canonical stored shape, or None (fail closed).
434
+
435
+ `source:'odoo'` + `derived:True` is the cohort column's read-only mechanism, reused:
436
+ the client keys editability off `source == 'overlay'` and the host accepts cell writes only
437
+ for overlay keys, so a measure column cannot be typed into at either end. `filterable:False`
438
+ because the REPLACEMENT is the measure CONDITION with the same measure and window β€” the
439
+ governed, gate-proved path (CG-8/CG-12) β€” not text ops over a derived cell.
440
+ """
441
+ if not isinstance(raw, dict):
442
+ return None
443
+ key = str(raw.get("key") or "")
444
+ if not key.startswith(MEASURE_FIELD_PREFIX) or len(key) > 80:
445
+ return None
446
+ spec = raw.get("measure")
447
+ if not isinstance(spec, dict):
448
+ return None
449
+ m = (offered or {}).get(spec.get("key"))
450
+ if not m:
451
+ return None # not admitted for this caller -> fail closed
452
+ window = _clean_window(spec.get("window"))
453
+ if window is None:
454
+ return None # a measure column with no period is not a column
455
+ mtype = m.get("type") if m.get("type") in MEASURE_FIELD_TYPES else "currency"
456
+ return {
457
+ "key": key,
458
+ "label": str(raw.get("label") or m.get("label") or "Measure")[:120],
459
+ "type": mtype,
460
+ "source": "odoo",
461
+ "default": True,
462
+ "custom": True,
463
+ "derived": True,
464
+ "filterable": False,
465
+ "agg": "sum" if mtype in ("currency", "int") else None,
466
+ "note": str(raw.get("note") or "")[:2000],
467
+ "measure": {"key": str(spec.get("key"))[:80], "window": window},
468
+ }
469
+
470
+
471
+ def measure_fields_of(fields):
472
+ """The formula-measure columns among `fields` β€” the ones whose values the host must compute
473
+ per render (see `rows_from_pool`'s `derived`)."""
474
+ return [f for f in fields or [] if isinstance(f.get("measure"), dict)]
475
+
476
+
477
+ def fields_from_workspace(workspace=None, cohorts=False, scope_key=None, fields_base=None):
478
+ """Overlay persisted notes/custom fields onto the immutable source-field contract.
479
+
480
+ `cohorts=True` appends the derived cohort column β€” see `cohort_field`. Appended LAST and
481
+ `default: False`, so it never displaces a column somebody already reads; the Fields menu is
482
+ where you turn it on.
483
+
484
+ Three saved strata pass through: notes on base fields, `custom_` overlay fields (editable),
485
+ and `measure_` formula-measure fields (read-only, host-computed β€” owner item 7). A measure
486
+ field was validated against the caller's measure OFFER when it was written
487
+ (`clean_measure_field`); here only its SHAPE is re-checked, because this module has zero
488
+ app-internal imports and cannot know the offer. A measure that has since become
489
+ unanswerable (a BU scope on a company-level measure) degrades to a BLANK column at value
490
+ time, never to an error.
491
+
492
+ `scope_key` (wave-6 item 9) names the PAGE doing the asking ('customer' / 'cohort'). A
493
+ saved def carrying `scope` is emitted only when it matches β€” so a cohort-specific field
494
+ never appears on the Customer table. FAIL-CLOSED: a caller that passes no scope_key sees
495
+ only unscoped (global) fields; base contract fields are never scoped.
496
+
497
+ `fields_base` (wave 16 C-TOPIC) β€” the canonical contract to overlay onto. Absent = the
498
+ CUSTOMER contract (`FIELDS`), byte-identical to before the parameter existed; the product
499
+ surface passes `product_fields()`. The workspace dict a caller hands in must already be the
500
+ matching table object's bucket β€” this function cannot tell a customer overlay from a
501
+ product one, which is exactly why the buckets are separate stores.
502
+ """
503
+ saved = dict((workspace or {}).get("fields") or {})
504
+ out = []
505
+ base_fields = fields_base if fields_base is not None else FIELDS
506
+ base_keys = {field["key"] for field in base_fields}
507
+ for base in base_fields:
508
+ meta = saved.get(base["key"]) or {}
509
+ field = dict(base)
510
+ if isinstance(meta.get("note"), str):
511
+ field["note"] = meta["note"][:2000]
512
+ # Wave-5 item 10: a saved DISPLAY format on any base field (a preset included) β€” how a
513
+ # number or date READS, per user. Rendering-only, so this is the whole acceptance.
514
+ fmt = _clean_format(meta.get("format"), base.get("type"))
515
+ if fmt:
516
+ field["format"] = fmt
517
+ # PRESET measure fields (wave-2 item 8): a preset+measure base may take a saved
518
+ # window/label override. Wave 6 deleted every preset member (the owner's
519
+ # no-buildable-presets rule) so this branch is currently MEMBERLESS β€” kept as the
520
+ # measure_ path's twin for any future preset-carrying contract, and because deleting
521
+ # it would silently change what a re-added preset means.
522
+ if base.get("preset") and isinstance(base.get("measure"), dict):
523
+ saved_measure = meta.get("measure") if isinstance(meta.get("measure"), dict) else {}
524
+ window = _clean_window(saved_measure.get("window"))
525
+ if window is not None:
526
+ field["measure"] = {"key": base["measure"]["key"], "window": window}
527
+ if isinstance(meta.get("label"), str) and meta["label"].strip():
528
+ field["label"] = meta["label"][:120]
529
+ out.append(field)
530
+ for key, field in saved.items():
531
+ if key in base_keys or not isinstance(field, dict):
532
+ continue
533
+ if field.get("scope") and field.get("scope") != scope_key:
534
+ continue # a cohort-specific field on another page (wave-6 item 9)
535
+ if (str(key).startswith(MEASURE_FIELD_PREFIX)
536
+ and isinstance(field.get("measure"), dict)):
537
+ window = _clean_window(field["measure"].get("window"))
538
+ mkey = str(field["measure"].get("key") or "")
539
+ if window is None or not mkey:
540
+ continue
541
+ mtype = field.get("type") if field.get("type") in MEASURE_FIELD_TYPES else "currency"
542
+ out.append({
543
+ "key": str(key)[:80],
544
+ "label": str(field.get("label") or "Measure")[:120],
545
+ "type": mtype,
546
+ "source": "odoo",
547
+ "default": bool(field.get("default", True)),
548
+ "custom": True,
549
+ "derived": True,
550
+ "filterable": False,
551
+ "agg": "sum" if mtype in ("currency", "int") else None,
552
+ "note": str(field.get("note") or "")[:2000],
553
+ "measure": {"key": mkey[:80], "window": window},
554
+ **_field_extras(field, mtype),
555
+ })
556
+ continue
557
+ if not str(key).startswith("custom_"):
558
+ continue
559
+ ftype = field.get("type")
560
+ if ftype not in CUSTOM_FIELD_TYPES:
561
+ continue
562
+ if ftype in READONLY_CUSTOM_TYPES:
563
+ # Wave-5 items 9/11: the read-only user-created pair. Emitted with the cohort
564
+ # column's mechanism (source 'odoo' + derived) so the client never offers an
565
+ # editor and the host's overlay-write guard excludes them by construction.
566
+ # FILTERABLE since wave 6 (owner item 6): their values live client-side
567
+ # (formula computes over the row, created_time renders `_created`), this
568
+ # table's counts are client-mode, and the windowed count path never sees
569
+ # these tables β€” so the client engine answers them soundly.
570
+ entry = {
571
+ "key": str(key)[:80],
572
+ "label": str(field.get("label") or "Untitled")[:120],
573
+ "type": ftype,
574
+ "source": "odoo",
575
+ "derived": True,
576
+ "filterable": True,
577
+ "default": bool(field.get("default", True)),
578
+ "custom": True,
579
+ "note": str(field.get("note") or "")[:2000],
580
+ **_field_extras(field, ftype),
581
+ }
582
+ if ftype == "formula":
583
+ formula = _clean_formula(field.get("formula"))
584
+ if formula is None:
585
+ continue # a formula field without a formula is nothing
586
+ entry["formula"] = formula
587
+ out.append(entry)
588
+ continue
589
+ if field.get("source") != "overlay":
590
+ continue
591
+ options = (_clean_options(field.get("options"))
592
+ if ftype in ("select", "multiselect") else [])
593
+ if ftype in ("select", "multiselect") and not options:
594
+ # A select with no surviving choices can never hold a value. Dropping the COLUMN
595
+ # would lose the user's data; degrading it to text keeps every stored value
596
+ # readable and lets them re-add choices.
597
+ ftype = "text"
598
+ out.append({
599
+ "key": str(key)[:80],
600
+ "label": str(field.get("label") or "Untitled")[:120],
601
+ "type": ftype,
602
+ "source": "overlay",
603
+ "default": bool(field.get("default", True)),
604
+ "custom": True,
605
+ "agg": field.get("agg") if field.get("agg") in {"sum"} else None,
606
+ "note": str(field.get("note") or "")[:2000],
607
+ **({"options": options} if ftype in ("select", "multiselect") else {}),
608
+ **(_choice_appearance(field, options)
609
+ if ftype in ("select", "multiselect") else {}),
610
+ # comma-joined SET semantics (the Cohorts column's contract): the row belongs to
611
+ # every member, groups count it under each, the toolbar count stays distinct.
612
+ **({"multi": True} if ftype == "multiselect" else {}),
613
+ **({"max": _clean_rating_max(field.get("max"))} if ftype == "rating" else {}),
614
+ **({"automation": _clean_automation(field.get("automation"))}
615
+ if ftype == "automation" and _clean_automation(field.get("automation")) else {}),
616
+ **_field_extras(field, ftype),
617
+ })
618
+ if cohorts:
619
+ out.append(cohort_field())
620
+ return out
621
+
622
+
623
+ def rows_from_pool(pool_rows, fields=None, overlays=None, derived=None):
624
+ """Map customer_data.pool() dicts -> the API row shape the grid expects:
625
+ pid + each Odoo field (numeric fields rounded, text/date passed through) + the
626
+ persisted external overlay. Mirrors the standalone API payload contract.
627
+
628
+ `derived` is `{pid: {key: value}}` for columns the HOST computes per render rather than
629
+ reads off the pool row β€” today just the cohort column. A separate argument from `overlays`
630
+ on purpose: `overlays` is the PERSISTED user stratum, and putting a value there that is
631
+ never written back would make the dict mean two things.
632
+ """
633
+ fields = fields or FIELDS
634
+ odoo_fields = [field for field in fields if field["source"] == "odoo"]
635
+ overlay_fields = [field for field in fields if field["source"] == "overlay"]
636
+ derived_keys = [field["key"] for field in fields if field.get("derived")]
637
+ overlays = overlays or {}
638
+ derived = derived or {}
639
+ out = []
640
+ for r in pool_rows:
641
+ pid = r.get("pid")
642
+ # `_created` (wave-5 item 11) rides every row like `pid` does β€” the datum the
643
+ # `created_time` field type renders, regardless of that field's own key. Not a Field:
644
+ # it has no column of its own until a user creates one. `lat`/`lon` (wave-7 W11) ride
645
+ # the same way: the Map VIEW's data, nullable, deliberately not a column.
646
+ row = {"pid": pid, "_created": r.get("_created") or "",
647
+ "lat": r.get("lat"), "lon": r.get("lon")}
648
+ for field in odoo_fields:
649
+ k = field["key"]
650
+ if field.get("derived"):
651
+ continue # not on the pool row β€” filled from `derived` below
652
+ v = r.get(k)
653
+ row[k] = v if field["type"] in {"text", "status", "date"} else _round(v)
654
+ saved = overlays.get(str(pid), {}) or {}
655
+ for field in overlay_fields:
656
+ row[field["key"]] = saved.get(field["key"], "")
657
+ got = derived.get(pid) or {}
658
+ for k in derived_keys:
659
+ # '' not None: a customer in no cohort has an EMPTY cohort list, and `is empty` on a
660
+ # text column is the question somebody will ask of it.
661
+ row[k] = got.get(k, "")
662
+ out.append(row)
663
+ return out
664
+
665
+
666
+ # --- the FILTER-TREE contract (mirrors customer-grid/types.ts) ---------------
667
+ # Ops the Airtable-parity condition builder can emit. isEmpty/isNotEmpty are
668
+ # VALUE-FREE (they legitimately carry no value and must never be dropped for it).
669
+ FILTER_OPS = {'contains', 'doesNotContain', 'eq', 'neq', 'isEmpty', 'isNotEmpty',
670
+ 'gt', 'gte', 'lt', 'lte', 'between', 'within',
671
+ # Wave 2026-08-02 (C-OPS): RANK operators β€” evaluated as a SET pass over the
672
+ # sibling-filtered domain by the client engine (useVisibleRows). The validator
673
+ # accepts them like any op (structural, not semantic); filter_sql REFUSES to
674
+ # compile them to row SQL (a per-row WHERE cannot express Top-N). aboveAvg /
675
+ # belowAvg are VALUE-FREE; the rest encode their argument in `value` as a
676
+ # string int (topN/bottomN 1..10000, inTopPct/inBottomPct 1..100,
677
+ # inQuartile 1..4, inDecile 1..10). Deliberately NOT in MEASURE_OPS: on a
678
+ # measure-carrying column they rank the field's own derived values.
679
+ 'topN', 'bottomN', 'inTopPct', 'inBottomPct',
680
+ 'aboveAvg', 'belowAvg', 'inQuartile', 'inDecile'}
681
+ #: The RESERVED pseudo-column of a cohort-membership leaf (owner item 5, 2026-07-26). It is not
682
+ #: a Field and never will be β€” a cohort is a hand-curated SET, so making it a column would mean
683
+ #: a cell per row per cohort. Mirrors customer-grid/types.ts COHORT_FIELD.
684
+ COHORT_FIELD = '__cohort__'
685
+ #: Ops a cohort leaf may carry (owner, 2026-07-27): set operators over a SET of cohorts.
686
+ #: Anything else is dropped. Deliberately DISJOINT from FILTER_OPS β€” see types.ts COHORT_OPS:
687
+ #: a set op reaching a column leaf would fall through the client engine's switch to "no
688
+ #: narrowing", so keeping the vocabularies apart makes the existing fail-closed drop do the work.
689
+ COHORT_OPS = {'anyOf', 'allOf', 'noneOf'}
690
+ #: The single-cohort ops this leaf shipped with, kept as PERMANENT aliases and REWRITTEN here:
691
+ #: `is part of [one]` is `is any of [that one]`, so a saved view keeps answering and upgrades the
692
+ #: next time it is written. Mirrors types.ts COHORT_OP_ALIASES.
693
+ COHORT_OP_ALIASES = {'eq': 'anyOf', 'neq': 'noneOf'}
694
+ #: How many cohorts one condition may name. Mirrors types.ts MAX_COHORT_IDS.
695
+ MAX_COHORT_IDS = 20
696
+
697
+
698
+ def parse_cohort_ids(value):
699
+ """The cohorts a leaf names, parsed out of `value`. Mirrors types.ts `cohortIds()`.
700
+
701
+ Comma-separated in one string because `FilterRule.value` is what all four layers persist and
702
+ round-trip, and a one-element list is byte-identical to what the single-cohort leaf already
703
+ stored β€” so every shipped view parses with no migration. Safe because a cohort id is built
704
+ from `[a-z0-9_]` only (modules/cohort.new_id), so a comma cannot occur inside one.
705
+ """
706
+ out = []
707
+ for raw in ('' if value is None else str(value)).split(','):
708
+ cid = raw.strip()[:120]
709
+ if not cid or cid in out:
710
+ continue
711
+ out.append(cid)
712
+ if len(out) >= MAX_COHORT_IDS:
713
+ break
714
+ return out
715
+ # Airtable allows 3 nesting levels (root conditions -> group -> group), then grays
716
+ # the button out. MAX_FILTER_DEPTH in types.ts must stay in lock-step with this.
717
+ MAX_FILTER_DEPTH = 3
718
+ MAX_FILTER_NODES = 100 # total nodes across the whole tree
719
+ MAX_FILTER_SIBLINGS = 50 # per level
720
+
721
+
722
+ #: Shape of a measure condition's date window. aios_grid has ZERO app-internal imports by
723
+ #: design, so it does NOT know the window VOCABULARY β€” `harness/windows.py` owns that, mirrored
724
+ #: in `customer-grid/windows.ts`, and a third copy here is exactly the drift those two already
725
+ #: need a gate to prevent. This validates SHAPE only.
726
+ WINDOW_MAX_N = 3650
727
+
728
+
729
+ def _clean_window(raw):
730
+ """Structural passthrough for a measure condition's `{kind, n?, from?, to?}` window.
731
+
732
+ Meaning is NOT checked here: an unrecognised `kind` survives this function and is REFUSED by
733
+ `harness.measure_filter.resolve_rule`, the layer that owns the vocabulary. Splitting it this
734
+ way keeps the grid module reusable and keeps one definition of what "last quarter" means.
735
+ """
736
+ if not isinstance(raw, dict):
737
+ return None
738
+ kind = raw.get('kind')
739
+ if not isinstance(kind, str) or not kind or len(kind) > 40:
740
+ return None
741
+ out = {'kind': kind}
742
+ if raw.get('n') is not None:
743
+ try:
744
+ out['n'] = max(1, min(int(raw['n']), WINDOW_MAX_N))
745
+ except (TypeError, ValueError):
746
+ return None
747
+ for side in ('from', 'to'):
748
+ if raw.get(side) not in (None, ''):
749
+ out[side] = str(raw[side])[:32]
750
+ return out
751
+
752
+
753
+ def _clean_rhs(raw, valid_keys):
754
+ """CG-9 β€” validate `{kind, colId, window?}`, the "compare against another attribute" side.
755
+
756
+ SHAPE and KEY only: `colId` must be something this table has (the caller widens `valid_keys`
757
+ with the measure keys, exactly as it does for the left side), and a measure rhs must carry a
758
+ window. What the window MEANS is `harness/windows.py`'s business, same split as `_clean_window`.
759
+ """
760
+ if not isinstance(raw, dict):
761
+ return None
762
+ kind = raw.get('kind')
763
+ if kind not in ('field', 'measure', 'stat'):
764
+ return None
765
+ if kind == 'stat':
766
+ # A STATISTIC carries no column: the population is the comparand. Shape only β€” which
767
+ # statistics exist is `harness/measure_filter.STATS`'s business, and an unrecognised one
768
+ # is REFUSED there rather than guessed at, exactly like an unrecognised window kind.
769
+ stat = raw.get('stat')
770
+ if not isinstance(stat, str) or not stat or len(stat) > 24:
771
+ return None
772
+ return {'kind': 'stat', 'stat': stat}
773
+ col = raw.get('colId')
774
+ if col not in valid_keys:
775
+ return None
776
+ out = {'kind': kind, 'colId': col}
777
+ if kind == 'measure':
778
+ window = _clean_window(raw.get('window'))
779
+ if window is None:
780
+ return None # a measure comparand with no period is not a question
781
+ out['window'] = window
782
+ return out
783
+
784
+
785
+ #: View DISPLAY MODES beside the grid (wave-6 item 10; 'map' wave-7 W11; 'dashboard' wave-8
786
+ #: I19). Mirrors customer-grid/types.ts DISPLAY_MODES; 'grid' is what an absent/unknown
787
+ #: display means, so it is never stored.
788
+ #: ⚠ 'dashboard' is RETAINED FOREVER (wave-9 I10, contract C2). The owner renamed the mode to
789
+ #: "Chart" (a Dashboard MODULE is coming and the two would collide), but this set is the
790
+ #: gatekeeper for a STORED value: `_clean_display` DROPS an unknown mode, so removing
791
+ #: 'dashboard' here would silently downgrade every already-saved chart view to grid β€” and live
792
+ #: views are sitting in mode:'dashboard' right now (wave 8's own close-out records one). The
793
+ #: rename is therefore a stored-value MIGRATION, not a constant rename: accept 'dashboard' on
794
+ #: READ forever, only ever WRITE 'chart'.
795
+ DISPLAY_MODES = {'grid', 'list', 'calendar', 'kanban', 'map', 'dashboard', 'chart',
796
+ 'timeseries', 'catalog'}
797
+
798
+ #: C-DISP (wave 2026-08-02): the time-series view's bucket vocabulary and caps, plus the
799
+ #: calendar-summary metric cap. types.ts mirrors these as TS_BUCKETS / TS_MAX_LAST_N /
800
+ #: TS_MAX_FIELDS / MAX_CALENDAR_METRICS, and cleanDisplay applies the same per-entry drops,
801
+ #: so an accepted save reads back byte-identically on both engines.
802
+ TS_BUCKETS = {'week', 'month', 'quarter', 'year'}
803
+ TS_MAX_LAST_N = 120
804
+ TS_MAX_FIELDS = 12
805
+ MAX_CALENDAR_METRICS = 4
806
+ _ISO_DAY = re.compile(r'^\d{4}-\d{2}-\d{2}$')
807
+
808
+ #: C6-CATALOG (wave 18) β€” the catalog view's vocabulary and caps. types.ts mirrors every name
809
+ #: below, and `cleanDisplay` applies the same drops in the SAME ORDER, so an accepted save reads
810
+ #: back identically on both engines. The code budget is the order-sensitive one β€” see
811
+ #: `_clean_catalogs`.
812
+ MAX_CATALOGS = 12
813
+ MAX_CATALOG_PAGES = 40
814
+ MAX_CATALOG_CODES = 500 # cumulative across ONE catalog's pages, spent in PAGE ORDER
815
+ CATALOG_PAPERS = {'letter', 'a4', 'tabloid'}
816
+ CATALOG_ORIENTATIONS = {'portrait', 'landscape'}
817
+ CATALOG_QUALITIES = {'web', 'print'}
818
+ CATALOG_PAGE_KINDS = {'cover', 'intro', 'section', 'gallery'}
819
+ CATALOG_COLS = (2, 3, 4)
820
+ CATALOG_ID_MAX, CATALOG_NAME_MAX = 40, 80
821
+ CATALOG_TITLE_MAX, CATALOG_BODY_MAX, CATALOG_CODE_MAX = 120, 2000, 60
822
+ _HEX6 = re.compile(r'^#[0-9A-Fa-f]{6}$')
823
+
824
+ #: Wave 14 C-ACC ([[loopable-wave14-split]]; rulings R2/R3). Mirrored by types.ts
825
+ #: TS_DELTA_KINDS / TS_MAX_CUSTOM_ROWS / TS_MAX_STYLES β€” the C-DISP byte-identical law.
826
+ TS_DELTA_KINDS = ('abs', 'pct', 'yoy', 'ytd')
827
+ TS_MAX_CUSTOM_ROWS = 12
828
+ TS_MAX_STYLES = 200
829
+ #: R2 β€” a formula row's `expr` is stored VERBATIM and NEVER parsed here (evaluation is client
830
+ #: law; the client refuses unknown refs/cycles/div-zero itself). The charset wall is the whole
831
+ #: server-side contract: row refs `[...]`, arithmetic, numbers β€” no markup, no control chars.
832
+ _TS_EXPR_OK = re.compile(r'^[A-Za-z0-9_ .+\-*/()\[\]]+$')
833
+
834
+ #: Old wire value -> the value we store today. Applied AFTER the membership test so an unknown
835
+ #: mode is still rejected rather than accidentally aliased.
836
+ _LEGACY_MODES = {'dashboard': 'chart'}
837
+
838
+ #: Chart kinds a chart-mode view may hold (wave-8 I19, contract C2). Mirrors the client's
839
+ #: union. Deliberately small: the owner asked to "start with simple charts" and expand, and a
840
+ #: kind the client cannot draw is worse than one that does not exist yet.
841
+ #: Wave-16 C-CHARTCAP: + 'table' β€” the group-by aggregate table (by-rep / by-BU /
842
+ #: top-customers, the third Sales block shape). Client renderer: DashboardView's
843
+ #: GroupTableView over salesParity.tableFromSpec.
844
+ CHART_KINDS = {'bar', 'line', 'area', 'donut', 'kpi', 'table'}
845
+ CHART_AGGS = {'sum', 'avg', 'count', 'min', 'max'}
846
+ MAX_CHARTS = 12 #: per view β€” a dashboard, not an unbounded render loop
847
+ MAX_CHART_TITLE = 60
848
+
849
+ #: Wave-9 I11 (contract C2) β€” chart customisation, host-validated.
850
+ #:
851
+ #: `palette` names a colour JOB, never a colour. A browser must not be able to post a raw hex:
852
+ #: the four names below map to the four jobs a palette can do (identity / magnitude / polarity)
853
+ #: and resolve to brand ramps client-side, so a tenant restyle cannot be defeated by a stored
854
+ #: literal. STATUS colours (good/warning/serious/critical) are deliberately NOT selectable β€”
855
+ #: they are reserved signal, and reusing them as "series 4" is how a chart starts lying.
856
+ CHART_PALETTES = {'brand', 'categorical', 'sequential', 'diverging'}
857
+ CHART_FORMATS = {'auto', 'number', 'currency', 'percent', 'compact'}
858
+ MAX_AXIS_LABEL = 40
859
+ #: `size` is the I10 drag. Width is in GRID COLUMNS (a 12-column board), height in px.
860
+ CHART_W_RANGE = (1, 12)
861
+ CHART_H_RANGE = (120, 800)
862
+
863
+
864
+ def _clean_chart(raw, valid_keys):
865
+ """One dashboard chart, fail-closed. Returns None if the chart cannot be drawn.
866
+
867
+ A chart's `y` is optional (absent = count of rows, which is what "how many customers per
868
+ state" means). `x` is NOT: a chart with no category axis has nothing to plot against, and
869
+ silently keeping it would put an empty card on the dashboard with no way to tell why.
870
+ """
871
+ if not isinstance(raw, dict):
872
+ return None
873
+ kind = raw.get('kind')
874
+ if kind not in CHART_KINDS:
875
+ return None
876
+ x = raw.get('x')
877
+ if x not in valid_keys:
878
+ return None # dead category ref -> the chart goes, not the board
879
+ cid = raw.get('id')
880
+ if not isinstance(cid, str) or not cid.strip():
881
+ return None # the client owns chart ids; an unidentified card
882
+ # cannot be edited or removed, so it must not persist
883
+ out = {'id': cid.strip()[:64], 'kind': kind, 'x': x,
884
+ 'agg': raw.get('agg') if raw.get('agg') in CHART_AGGS else 'sum'}
885
+ if raw.get('y') in valid_keys:
886
+ out['y'] = raw['y']
887
+ else:
888
+ # no measurable column -> the only honest aggregation left is "how many rows"
889
+ out['agg'] = 'count'
890
+ title = raw.get('title')
891
+ if isinstance(title, str) and title.strip():
892
+ out['title'] = title.strip()[:MAX_CHART_TITLE]
893
+
894
+ # ── wave-9 I11 (contract C2): customisation ────────────────────────────────────────────
895
+ # `splitBy` is the field whose values become the SERIES. It is deliberately not called
896
+ # `colorBy`: that name already means two other things here (`config.colorBy` = row
897
+ # colouring, `display.colorField` = map pin colour) and a third sense would be unreadable.
898
+ if raw.get('splitBy') in valid_keys and raw['splitBy'] != x:
899
+ out['splitBy'] = raw['splitBy']
900
+ # Stacking is only a question once there are series to stack, and only for the two kinds
901
+ # that can express it. Anywhere else it is dropped rather than stored as a lie the client
902
+ # would have to re-decide.
903
+ if out.get('splitBy') and kind in ('bar', 'area') and raw.get('stacked') is True:
904
+ out['stacked'] = True
905
+ if raw.get('palette') in CHART_PALETTES:
906
+ out['palette'] = raw['palette']
907
+
908
+ axis = raw.get('axis')
909
+ if isinstance(axis, dict):
910
+ # β›” ONE y-scale, always. There is no second-axis key here and there must never be:
911
+ # two y-scales on one frame can manufacture any correlation you like by rescaling, and
912
+ # the honest alternatives are two charts, small multiples, or indexing to a common base.
913
+ # Ruled explicitly in contract C2 against the "Tableau versatility" brief.
914
+ clean_axis = {}
915
+ for side in ('x', 'y'):
916
+ spec = axis.get(side)
917
+ if not isinstance(spec, dict):
918
+ continue
919
+ one = {}
920
+ lab = spec.get('label')
921
+ if isinstance(lab, str) and lab.strip():
922
+ one['label'] = lab.strip()[:MAX_AXIS_LABEL]
923
+ if spec.get('format') in CHART_FORMATS:
924
+ one['format'] = spec['format']
925
+ if one:
926
+ clean_axis[side] = one
927
+ if clean_axis:
928
+ out['axis'] = clean_axis
929
+
930
+ size = raw.get('size')
931
+ if isinstance(size, dict):
932
+ one = {}
933
+ for key, (lo, hi) in (('w', CHART_W_RANGE), ('h', CHART_H_RANGE)):
934
+ try:
935
+ one[key] = max(lo, min(hi, int(size[key])))
936
+ except (KeyError, TypeError, ValueError):
937
+ pass # a partial size is fine: the client defaults the missing axis
938
+ if one:
939
+ out['size'] = one
940
+
941
+ # ── Wave 14 R3 ([[loopable-wave14-split]]): a METRIC chart may carry a PERIOD β€” the
942
+ # trend-over-buckets encoding. Kept only when the chart's value field is measure-backed:
943
+ # a category column has no time dimension, and a stored period on it would promise a
944
+ # trend the TS channel must refuse. `span` is meaningful only beside `bucket`.
945
+ if isinstance(out.get('y'), str) and out['y'].startswith('measure_'):
946
+ if raw.get('bucket') in TS_BUCKETS:
947
+ out['bucket'] = raw['bucket']
948
+ sp = raw.get('span')
949
+ if isinstance(sp, dict):
950
+ n = sp.get('lastN')
951
+ if (isinstance(n, int) and not isinstance(n, bool)
952
+ and 1 <= n <= TS_MAX_LAST_N):
953
+ out['span'] = {'lastN': n}
954
+ # ── Wave-16 C-CHARTCAP: the YoY companion. Kept ONLY where it can mean something β€”
955
+ # beside a kept bucket (the compare series) or on a sum-of-metric KPI (the delta
956
+ # line). Anything else is a stored claim the renderer would have to re-refuse.
957
+ # Mirrors the client's cleanCharts rule key for key.
958
+ if raw.get('compare') == 'prior_year' and (
959
+ out.get('bucket') or (kind == 'kpi' and out.get('agg') == 'sum')):
960
+ out['compare'] = 'prior_year'
961
+ return out
962
+
963
+
964
+ def _clean_catalog_page(raw, budget):
965
+ """C6-CATALOG β€” one page of a catalog. Returns `(page | None, codes_spent)`.
966
+
967
+ `budget` is what is LEFT of the catalog's 500-code allowance. Codes are deduped WITHIN a
968
+ page and not across the catalog: a product legitimately appears on a gallery page and again
969
+ in its section listing, and de-duplicating globally would silently delete the second
970
+ appearance. The budget is spent in page order, so a catalog that runs out loses the TAIL of
971
+ its last pages β€” never a random scatter, and never a page (a page with no products is a
972
+ heading the user can still see and fix).
973
+ """
974
+ if not isinstance(raw, dict):
975
+ return None, 0
976
+ page_id = str(raw.get('id') or '')[:CATALOG_ID_MAX]
977
+ kind = raw.get('kind')
978
+ if not page_id or kind not in CATALOG_PAGE_KINDS:
979
+ return None, 0
980
+ out = {'id': page_id, 'kind': kind}
981
+ for key, cap in (('title', CATALOG_TITLE_MAX), ('body', CATALOG_BODY_MAX),
982
+ ('imageCode', CATALOG_CODE_MAX)):
983
+ v = raw.get(key)
984
+ if isinstance(v, str) and v:
985
+ out[key] = v[:cap]
986
+ products = raw.get('products')
987
+ if isinstance(products, list) and budget > 0:
988
+ clean_p, seen_p = [], set()
989
+ for c in products:
990
+ if not isinstance(c, str) or not c:
991
+ continue
992
+ c = c[:CATALOG_CODE_MAX]
993
+ if c in seen_p:
994
+ continue
995
+ seen_p.add(c)
996
+ clean_p.append(c)
997
+ if len(clean_p) >= budget:
998
+ break
999
+ if clean_p:
1000
+ out['products'] = clean_p
1001
+ layout = raw.get('layout')
1002
+ if isinstance(layout, dict):
1003
+ clean_l = {}
1004
+ cols = layout.get('cols')
1005
+ if isinstance(cols, int) and not isinstance(cols, bool) and cols in CATALOG_COLS:
1006
+ clean_l['cols'] = cols
1007
+ # The kanbanClamp/tsSparkline asymmetry, one per direction: pack and colour SHOW by
1008
+ # default (the 2027 catalogue shows both), price does NOT (it shows no prices at all).
1009
+ # So only the opt-OUT is storable for the first two and only the opt-IN for the third β€”
1010
+ # a second spelling of a default is how a round trip starts churning.
1011
+ if layout.get('showPack') is False:
1012
+ clean_l['showPack'] = False
1013
+ if layout.get('showColor') is False:
1014
+ clean_l['showColor'] = False
1015
+ if layout.get('showPrice') is True:
1016
+ clean_l['showPrice'] = True
1017
+ if clean_l:
1018
+ out['layout'] = clean_l
1019
+ return out, len(out.get('products') or ())
1020
+
1021
+
1022
+ def _clean_catalogs(raw, valid_keys):
1023
+ """C6-CATALOG (wave 18) β€” `display.catalogs`, fail-closed. Returns a list or None.
1024
+
1025
+ A catalog is a PRINT artifact, so the two structural keys that decide how it paginates
1026
+ (`paper`, `orientation`) are NORMALISED WITH A DEFAULT rather than dropped: a page box with
1027
+ no size is not a smaller catalog, it is an unrenderable one. Everything else follows the
1028
+ house rules β€” unknown keys dropped, per-entry drops never cost the neighbours, empty
1029
+ sub-objects omitted entirely (`brand`, `fields`, `layout`, `products`) so an absent key and
1030
+ an empty one are not two spellings of the same nothing.
1031
+
1032
+ `fields` binds the listing lines to real columns (the 2027 listing prints description / SKU /
1033
+ pack / colour, and `product_data` carries no pack or colour of its own β€” the user binds
1034
+ custom fields). Refs are checked against `valid_keys` HERE and not on the client, the same
1035
+ split `dateField`/`stackField` already run.
1036
+ """
1037
+ if not isinstance(raw, list):
1038
+ return None
1039
+ out = []
1040
+ for c in raw:
1041
+ if len(out) >= MAX_CATALOGS:
1042
+ break
1043
+ if not isinstance(c, dict):
1044
+ continue
1045
+ cat_id = str(c.get('id') or '')[:CATALOG_ID_MAX]
1046
+ name = c.get('name')
1047
+ # An EMPTY name is legal (the user cleared the box and will type again) β€” an ABSENT one
1048
+ # is a malformed record. The `tsRows` label rule, same reasoning.
1049
+ if not cat_id or not isinstance(name, str):
1050
+ continue
1051
+ cat = {'id': cat_id, 'name': name[:CATALOG_NAME_MAX]}
1052
+ cat['paper'] = c['paper'] if c.get('paper') in CATALOG_PAPERS else 'letter'
1053
+ cat['orientation'] = (c['orientation']
1054
+ if c.get('orientation') in CATALOG_ORIENTATIONS else 'portrait')
1055
+ if c.get('quality') in CATALOG_QUALITIES:
1056
+ cat['quality'] = c['quality']
1057
+ brand = c.get('brand')
1058
+ if isinstance(brand, dict):
1059
+ clean_b = {}
1060
+ for k in ('primary', 'accent'):
1061
+ v = brand.get(k)
1062
+ if isinstance(v, str) and _HEX6.match(v):
1063
+ clean_b[k] = v
1064
+ company = brand.get('company')
1065
+ if isinstance(company, str) and company:
1066
+ clean_b['company'] = company[:CATALOG_NAME_MAX]
1067
+ # An asset CODE (resolved through C2-ASSET), never a URL: an arbitrary host inside
1068
+ # print CSS is exactly the tokens-not-values rule this contract carries.
1069
+ logo = brand.get('logo')
1070
+ if isinstance(logo, str) and logo:
1071
+ clean_b['logo'] = logo[:CATALOG_CODE_MAX]
1072
+ if clean_b:
1073
+ cat['brand'] = clean_b
1074
+ binds = c.get('fields')
1075
+ if isinstance(binds, dict):
1076
+ clean_bind = {k: binds[k] for k in ('name', 'pack', 'color', 'price')
1077
+ if binds.get(k) in valid_keys}
1078
+ if clean_bind:
1079
+ cat['fields'] = clean_bind
1080
+ pages, budget = [], MAX_CATALOG_CODES
1081
+ raw_pages = c.get('pages')
1082
+ if isinstance(raw_pages, list):
1083
+ for p in raw_pages:
1084
+ if len(pages) >= MAX_CATALOG_PAGES:
1085
+ break
1086
+ page, spent = _clean_catalog_page(p, budget)
1087
+ if page is None:
1088
+ continue
1089
+ budget -= spent
1090
+ pages.append(page)
1091
+ # ALWAYS emitted, even empty: a catalog with no pages yet is the state every catalog
1092
+ # starts in, and dropping the key would make "new" and "corrupt" the same wire value.
1093
+ cat['pages'] = pages
1094
+ out.append(cat)
1095
+ return out or None
1096
+
1097
+
1098
+ def _clean_display(raw, valid_keys):
1099
+ """Structural passthrough for a view's `config.display` (wave-6 item 10), fail-closed.
1100
+
1101
+ `{mode, dateField?, stackField?, titleField?, colorField?, sizeField?, charts?}` β€” mode
1102
+ must be a known non-grid mode (grid is the absent default, so storing it would be a second
1103
+ way to say nothing); every field ref must name a field this table has (a ref to a deleted
1104
+ field is DROPPED and the client falls back to its per-mode default); unknown keys are
1105
+ dropped. What each mode MEANS β€” calendar wants a date-family field, kanban a select-family
1106
+ stack, map a single-select to colour by and a numeric to size by β€” is the client's
1107
+ business: it is the only layer that renders them, and a wrong-typed ref degrades to that
1108
+ surface's default rather than to an error (the `_clean_window` split).
1109
+
1110
+ Wave-8 (contract C2) adds the map encodings (`colorField` I3, `sizeField` I5) and
1111
+ dashboard `charts` (I19). A chart whose x/y names a deleted field is dropped INDIVIDUALLY β€”
1112
+ never the whole array, because losing one column should not cost the user a dashboard they
1113
+ spent time building.
1114
+ """
1115
+ if not isinstance(raw, dict):
1116
+ return None
1117
+ mode = raw.get('mode')
1118
+ if mode not in DISPLAY_MODES or mode == 'grid':
1119
+ return None
1120
+ # Wave-9 I10 (C2): normalise the legacy wire value AFTER the membership test, so an unknown
1121
+ # mode is still rejected rather than accidentally aliased into a real one. Every already
1122
+ # saved 'dashboard' view reads back as 'chart' from here on; nothing writes 'dashboard'.
1123
+ mode = _LEGACY_MODES.get(mode, mode)
1124
+ out = {'mode': mode}
1125
+ for ref in ('dateField', 'stackField', 'titleField', 'colorField', 'sizeField'):
1126
+ if raw.get(ref) in valid_keys:
1127
+ out[ref] = raw[ref]
1128
+ # ── C-DISP (wave 2026-08-02) ─────────────────────────────────────────────────────────
1129
+ # kanbanClamp: stored ONLY as the literal opt-OUT. Absent means clamped β€” the new
1130
+ # standardized default β€” so storing True would be a second way to say nothing (the same
1131
+ # rule that keeps mode:'grid' out of the store).
1132
+ if raw.get('kanbanClamp') is False:
1133
+ out['kanbanClamp'] = False
1134
+ if raw.get('calendarMode') in ('records', 'summary'):
1135
+ out['calendarMode'] = raw['calendarMode']
1136
+ metrics = raw.get('calendarMetrics')
1137
+ if isinstance(metrics, list):
1138
+ clean_m, seen_m = [], set()
1139
+ for m in metrics[:MAX_CALENDAR_METRICS]:
1140
+ # Dropped INDIVIDUALLY (the charts precedent): one dead metric must not cost the
1141
+ # user the summary card they configured around it.
1142
+ if not isinstance(m, dict):
1143
+ continue
1144
+ mid = str(m.get('id') or '')[:40]
1145
+ if (not mid or mid in seen_m or m.get('field') not in valid_keys
1146
+ or m.get('agg') not in CHART_AGGS):
1147
+ continue
1148
+ seen_m.add(mid)
1149
+ clean_m.append({'id': mid, 'field': m['field'], 'agg': m['agg']})
1150
+ if clean_m:
1151
+ out['calendarMetrics'] = clean_m
1152
+ if raw.get('tsBucket') in TS_BUCKETS:
1153
+ out['tsBucket'] = raw['tsBucket']
1154
+ span = raw.get('tsSpan')
1155
+ if isinstance(span, dict):
1156
+ clean_span = {}
1157
+ n = span.get('lastN')
1158
+ if isinstance(n, int) and not isinstance(n, bool) and 1 <= n <= TS_MAX_LAST_N:
1159
+ clean_span['lastN'] = n
1160
+ else:
1161
+ f, t = span.get('from'), span.get('to')
1162
+ f = f if isinstance(f, str) and _ISO_DAY.match(f) else None
1163
+ t = t if isinstance(t, str) and _ISO_DAY.match(t) else None
1164
+ if f and t and f > t:
1165
+ f, t = t, f
1166
+ if f:
1167
+ clean_span['from'] = f
1168
+ if t:
1169
+ clean_span['to'] = t
1170
+ if clean_span:
1171
+ out['tsSpan'] = clean_span
1172
+ ts_fields = raw.get('tsFields')
1173
+ if isinstance(ts_fields, list):
1174
+ clean_f, seen_f = [], set()
1175
+ for k in ts_fields[:TS_MAX_FIELDS]:
1176
+ if k in valid_keys and k not in seen_f:
1177
+ seen_f.add(k)
1178
+ clean_f.append(k)
1179
+ if clean_f:
1180
+ out['tsFields'] = clean_f
1181
+ # ── Wave 14 C-ACC ([[loopable-wave14-split]] R2; items 17/18) ────────────────────────
1182
+ deltas = raw.get('tsDeltas')
1183
+ if isinstance(deltas, list):
1184
+ clean_d, seen_d = [], set()
1185
+ for d in deltas:
1186
+ if d in TS_DELTA_KINDS and d not in seen_d:
1187
+ seen_d.add(d)
1188
+ clean_d.append(d)
1189
+ if clean_d:
1190
+ out['tsDeltas'] = clean_d
1191
+ # Gridlines: stored ONLY as the literal opt-OUT (absent = shown), sparkline ONLY as the
1192
+ # literal opt-IN (absent = off) β€” the kanbanClamp asymmetry, one per direction.
1193
+ if raw.get('tsGridlines') is False:
1194
+ out['tsGridlines'] = False
1195
+ if raw.get('tsSparkline') is True:
1196
+ out['tsSparkline'] = True
1197
+ rows = raw.get('tsRows')
1198
+ if isinstance(rows, list):
1199
+ clean_r, seen_r = [], set()
1200
+ for r in rows[:TS_MAX_CUSTOM_ROWS]:
1201
+ if not isinstance(r, dict):
1202
+ continue
1203
+ rid = str(r.get('id') or '')[:40]
1204
+ r_kind = r.get('kind')
1205
+ if not rid or rid in seen_r or r_kind not in ('note', 'formula'):
1206
+ continue
1207
+ label = r.get('label')
1208
+ if not isinstance(label, str):
1209
+ continue # ABSENT label = malformed; an EMPTY one is a legal spacer row
1210
+ # (GRID's dated asymmetry amendments, 2026-08-02)
1211
+ one = {'id': rid, 'kind': r_kind, 'label': label.strip()[:120]}
1212
+ if r_kind == 'formula':
1213
+ expr = r.get('expr')
1214
+ if (isinstance(expr, str) and expr.strip()
1215
+ and len(expr) <= 200 and _TS_EXPR_OK.match(expr)):
1216
+ one['expr'] = expr
1217
+ # else: keep the ROW, drop the EXPR β€” it renders "β€”". Vanishing the row
1218
+ # would delete the user's label to punish their arithmetic (GRID's dated
1219
+ # asymmetry amendment; the calendarMetrics per-entry-drop precedent).
1220
+ seen_r.add(rid)
1221
+ clean_r.append(one)
1222
+ if clean_r:
1223
+ out['tsRows'] = clean_r
1224
+ styles = raw.get('tsStyles')
1225
+ if isinstance(styles, dict):
1226
+ clean_s = {}
1227
+ for s_key, s_val in styles.items():
1228
+ if len(clean_s) >= TS_MAX_STYLES:
1229
+ break # capped, not truncated silently: the gate names this
1230
+ if not isinstance(s_key, str) or not s_key or len(s_key) > 96:
1231
+ continue # key = rowId or "rowId:colKey" β€” the client's grammar
1232
+ if not isinstance(s_val, dict):
1233
+ continue
1234
+ one = {}
1235
+ if s_val.get('bold') is True:
1236
+ one['bold'] = True
1237
+ if s_val.get('line') is True:
1238
+ one['line'] = True
1239
+ if one:
1240
+ clean_s[s_key] = one
1241
+ if clean_s:
1242
+ out['tsStyles'] = clean_s
1243
+ charts = raw.get('charts')
1244
+ if isinstance(charts, list):
1245
+ clean = [c for c in (_clean_chart(x, valid_keys) for x in charts[:MAX_CHARTS]) if c]
1246
+ # de-dupe by id: two cards sharing an id are one card as far as the client's keyed
1247
+ # render is concerned, and the second would silently shadow the first
1248
+ seen, uniq = set(), []
1249
+ for c in clean:
1250
+ if c['id'] in seen:
1251
+ continue
1252
+ seen.add(c['id'])
1253
+ uniq.append(c)
1254
+ if uniq:
1255
+ out['charts'] = uniq
1256
+ # ── C6-CATALOG (wave 18) ─────────────────────────────────────────────────────────────
1257
+ catalogs = _clean_catalogs(raw.get('catalogs'), valid_keys)
1258
+ if catalogs:
1259
+ out['catalogs'] = catalogs
1260
+ return out
1261
+
1262
+
1263
+ #: FOLDERS over the saved views / cohorts sidebars (wave-8 I11, contract C4).
1264
+ #:
1265
+ #: ⚠ Folder membership is stored as a SIDE MAP (`itemFolders`), not as a `folderId` ON each
1266
+ #: view or cohort β€” a deliberate amendment to C4's first wording, recorded in the split doc.
1267
+ #: Two reasons. (1) A cohort lives in a DIFFERENT store (`customer_cohorts`, keyed by cohort
1268
+ #: id) and adding a `folders` key beside those ids would collide with a cohort whose generated
1269
+ #: id happened to be 'folders'. (2) Folder placement is a per-user ORGANISING act, not part of
1270
+ #: what a view IS: keeping it out of the view config means duplicating or exporting a view does
1271
+ #: not drag a folder reference along with it. One map, one home, both surfaces.
1272
+ FOLDER_SURFACES = {"views", "cohorts"}
1273
+ MAX_FOLDERS = 60
1274
+ MAX_FOLDER_NAME = 80
1275
+
1276
+ #: Wave-9 I15 (contract C5) β€” a user-chosen folder icon, as {shape, tone}.
1277
+ #:
1278
+ #: Both halves are WHITELISTS, never free values: `shape` names geometry the client already
1279
+ #: draws (one source, `iconShapes.ts`, read by both painters) and `tone` names a palette token,
1280
+ #: not a colour β€” so a browser cannot post a hex and defeat a tenant restyle, and a shape the
1281
+ #: client cannot render can never reach the store.
1282
+ #: ⚠ MIRRORS CLIENT'S `iconShapes.ts` ENUMERATION EXACTLY (C5: CLIENT enumerates, HOST mirrors β€”
1283
+ #: posted in the split doc 2026-07-29, HOST adopted it the same day, replacing a provisional
1284
+ #: 12-shape guess of mine that contained shapes the client cannot draw). Do not extend this set
1285
+ #: without the matching client geometry: an unknown shape falls back to the default folder mark,
1286
+ #: which is also I14's "existing folders get the folder icon" for every pre-wave-9 folder.
1287
+ FOLDER_ICON_SHAPES = {"folder", "star", "flag", "tag", "bookmark", "box", "circle", "square"}
1288
+ #: Tones are the C1 pastels β€” FILLS ONLY, never text (the standing palette rule). 'grey' is the
1289
+ #: default, and is CLIENT's key name: not 'neutral', which is what HOST first guessed.
1290
+ FOLDER_ICON_TONES = {"blue", "green", "yellow", "red", "grey"}
1291
+ FOLDER_ICON_DEFAULT_TONE = "grey"
1292
+
1293
+
1294
+ #: Wave-9 I17 (contract C4) β€” who may EDIT a saved view.
1295
+ #:
1296
+ #: ⚠ READ THIS BEFORE BUILDING ON IT. Views are stored PER USER today
1297
+ #: (`core/table_store.TableStore.workspace` reads `store.get(table_key)[username]`), so one
1298
+ #: user's views are invisible to every other user and "collaborative" has nothing to act on
1299
+ #: yet. This validator is therefore CORRECT-BUT-INERT plumbing: it makes the setting durable
1300
+ #: and fail-closed now, so that when shared views land the permission does not need a data
1301
+ #: migration and no saved view is retro-restricted. It does NOT make anything shared, and
1302
+ #: nothing in the app currently reads it to grant or deny cross-user access.
1303
+ #: Recorded as the C4 amendment in .claude/wiki/research/grid-wave9-split.md.
1304
+ VIEW_EDIT_MODES = {'personal', 'collaborative', 'users'}
1305
+ MAX_VIEW_USERS = 50
1306
+
1307
+
1308
+ def clean_view_permissions(raw, default, known_users=None):
1309
+ """{edit, users?} β€” fail-closed on both halves.
1310
+
1311
+ `default` is supplied by the CALLER because it splits by path, and that split is a
1312
+ permission rule rather than a formatting one: absent on a view that already exists means a
1313
+ pre-wave-9 view and must stay 'collaborative' (retro-restricting somebody's saved view is a
1314
+ silent takeaway), while absent on CREATE must be 'personal' (a new view must never be
1315
+ anyone-can-edit purely by omission).
1316
+
1317
+ `known_users` (when given) is the real account list: an unknown name is DROPPED, and an
1318
+ 'users' grant left with nobody in it collapses to 'personal' rather than to everyone.
1319
+ """
1320
+ mode = (raw or {}).get('edit') if isinstance(raw, dict) else None
1321
+ if mode not in VIEW_EDIT_MODES:
1322
+ mode = default if default in VIEW_EDIT_MODES else 'personal'
1323
+ if mode != 'users':
1324
+ return {'edit': mode}
1325
+ names, seen = [], set()
1326
+ for u in list((raw or {}).get('users') or [])[:MAX_VIEW_USERS]:
1327
+ u = str(u or '').strip()
1328
+ if not u or u.lower() in seen:
1329
+ continue
1330
+ if known_users is not None and u not in known_users:
1331
+ continue # fail-closed: a name we cannot resolve grants nothing
1332
+ seen.add(u.lower())
1333
+ names.append(u)
1334
+ if not names:
1335
+ return {'edit': 'personal'} # an empty grant is NOT "everyone"
1336
+ return {'edit': 'users', 'users': names}
1337
+
1338
+
1339
+ def clean_folder_icon(raw):
1340
+ """{shape, tone} or None. Fail-closed on both halves, independently.
1341
+
1342
+ A folder with a valid shape but a junk tone keeps the shape and defaults the tone rather
1343
+ than losing the icon entirely β€” losing a user's pick because one half was wrong is the kind
1344
+ of silent data loss the folder events already avoid elsewhere.
1345
+ """
1346
+ if not isinstance(raw, dict):
1347
+ return None
1348
+ shape = raw.get("shape")
1349
+ if shape not in FOLDER_ICON_SHAPES:
1350
+ return None
1351
+ tone = raw.get("tone")
1352
+ return {"shape": shape,
1353
+ "tone": tone if tone in FOLDER_ICON_TONES else FOLDER_ICON_DEFAULT_TONE}
1354
+
1355
+
1356
+ def clean_folders(raw):
1357
+ """Validate the per-surface folder lists, fail-closed. {surface: [{id, name, order}]}."""
1358
+ out = {}
1359
+ for surface in FOLDER_SURFACES:
1360
+ items, seen = [], set()
1361
+ for f in list((raw or {}).get(surface) or [])[:MAX_FOLDERS]:
1362
+ if not isinstance(f, dict):
1363
+ continue
1364
+ fid = str(f.get("id") or "").strip()[:80]
1365
+ name = str(f.get("name") or "").strip()[:MAX_FOLDER_NAME]
1366
+ if not fid or not name or fid in seen:
1367
+ continue # an unidentified or unnamed folder cannot be shown or edited
1368
+ seen.add(fid)
1369
+ try:
1370
+ order = int(f.get("order", len(items)))
1371
+ except (TypeError, ValueError):
1372
+ order = len(items)
1373
+ row = {"id": fid, "name": name, "order": order}
1374
+ icon = clean_folder_icon(f.get("icon")) # wave-9 I15 (C5); absent = default mark
1375
+ if icon:
1376
+ row["icon"] = icon
1377
+ items.append(row)
1378
+ items.sort(key=lambda x: x["order"])
1379
+ for i, f in enumerate(items):
1380
+ f["order"] = i # re-index so `order` is always dense and total
1381
+ if items:
1382
+ out[surface] = items
1383
+ return out
1384
+
1385
+
1386
+ def clean_item_folders(raw, folders, valid_ids):
1387
+ """{surface: {itemId: folderId}} β€” dropping any placement whose ITEM or FOLDER is gone.
1388
+
1389
+ This is what makes a deleted folder's contents fall back to the root rather than vanish:
1390
+ nothing stores "this item is in no folder", so an unresolvable placement simply disappears
1391
+ and the item renders at the top level. Same for an item that was deleted elsewhere β€” its
1392
+ stale placement can never resurrect it, because the sidebars render ITEMS and consult this
1393
+ map, never the other way round.
1394
+ """
1395
+ out = {}
1396
+ for surface in FOLDER_SURFACES:
1397
+ fids = {f["id"] for f in (folders or {}).get(surface, [])}
1398
+ ok = {}
1399
+ for item_id, fid in ((raw or {}).get(surface) or {}).items():
1400
+ if not isinstance(item_id, str) or not isinstance(fid, str):
1401
+ continue
1402
+ if item_id in (valid_ids or {}).get(surface, ()) and fid in fids:
1403
+ ok[item_id[:120]] = fid
1404
+ if ok:
1405
+ out[surface] = ok
1406
+ return out
1407
+
1408
+
1409
+ def clean_filter_tree(raw, valid_keys, depth=1, budget=None, cohort_ids=None):
1410
+ """Recursively validate an UNTRUSTED filter tree (conditions + nested groups).
1411
+
1412
+ Returns a clean tree of leaf conditions ({colId, op, value, value2}) and
1413
+ groups ({conj, children}). Module-agnostic on purpose: any module embedding
1414
+ the grid validates its own view state through this one function.
1415
+
1416
+ Fail-closed PER NODE: anything unrecognised is DROPPED rather than raised β€”
1417
+ the same contract the rest of the view sanitiser follows, so one bad rule can
1418
+ never cost a user their whole saved view. Depth, per-level width and total
1419
+ node count are all capped: the tree is re-evaluated for every row on every
1420
+ render, so an unbounded structure would be a persistent client-side DoS.
1421
+ Empty groups are dropped (they carry no meaning once persisted).
1422
+
1423
+ `cohort_ids` is the set of cohorts the CALLER may see. A cohort leaf naming anything else is
1424
+ DROPPED here rather than left for the engine β€” a deleted cohort would otherwise leave a
1425
+ condition that can only match nothing, so `List is not [deleted]` would show an empty table
1426
+ forever with no way to tell why. `None` means this host has no cohorts, and then every
1427
+ cohort leaf is dropped: fail-closed, like every other unknown key.
1428
+ """
1429
+ if budget is None:
1430
+ budget = [MAX_FILTER_NODES]
1431
+ out = []
1432
+ for node in list(raw or [])[:MAX_FILTER_SIBLINGS]:
1433
+ if budget[0] <= 0:
1434
+ break
1435
+ if not isinstance(node, dict):
1436
+ continue
1437
+ if isinstance(node.get('children'), list): # a condition GROUP
1438
+ if depth >= MAX_FILTER_DEPTH:
1439
+ continue # too deep -> drop
1440
+ budget[0] -= 1
1441
+ children = clean_filter_tree(node['children'], valid_keys,
1442
+ depth + 1, budget, cohort_ids)
1443
+ if children:
1444
+ out.append({'conj': 'or' if node.get('conj') == 'or' else 'and',
1445
+ 'children': children})
1446
+ continue
1447
+ if node.get('colId') == COHORT_FIELD: # a cohort-membership leaf
1448
+ op = COHORT_OP_ALIASES.get(node.get('op'), node.get('op'))
1449
+ named = parse_cohort_ids(node.get('value'))
1450
+ # ALL of them, or the leaf goes. A set that quietly lost a member asks a DIFFERENT
1451
+ # question, and for `noneOf` a strictly wider one: `is none of [A, B]` degrading to
1452
+ # `is none of [A]` would show every row in B under a count nobody would doubt. This
1453
+ # is the same all-or-nothing the single-cohort leaf already had, extended to a set.
1454
+ if op in COHORT_OPS and named and all(c in (cohort_ids or ()) for c in named):
1455
+ budget[0] -= 1
1456
+ out.append({'colId': COHORT_FIELD, 'op': op,
1457
+ 'value': ','.join(named), 'value2': ''})
1458
+ continue
1459
+ if node.get('colId') in valid_keys and node.get('op') in FILTER_OPS:
1460
+ budget[0] -= 1
1461
+ # `or ''` would be wrong here: it maps every FALSY value to '', and '' is the
1462
+ # signal for "inactive". A numeric 0 (or 0.0, or False) is a real value the client
1463
+ # treats as active β€” `0 === ""` is false in TS β€” so `revenue = 0` would silently
1464
+ # stop filtering and show every row instead of the zero-revenue ones.
1465
+ val, val2 = node.get('value'), node.get('value2')
1466
+ leaf = {'colId': node['colId'], 'op': node['op'],
1467
+ 'value': ('' if val is None else str(val))[:500],
1468
+ 'value2': ('' if val2 is None else str(val2))[:500]}
1469
+ # CG-8. A MEASURE condition ("Sales, in the last 90 days, > 5,000") carries two
1470
+ # extra members: a stable client-generated `id`, which is how the server's answer
1471
+ # finds its way back to the condition that asked (positional matching silently
1472
+ # re-associates every answer the moment a user deletes a condition), and the
1473
+ # `window`. Emitted ONLY when the input has them β€” a column condition's cleaned
1474
+ # shape is unchanged, so every persisted view deserialises byte-identically and
1475
+ # `clean_filter_tree` stays idempotent (verify_filter_engine.py asserts that by
1476
+ # exact structural comparison).
1477
+ rid = node.get('id')
1478
+ if rid not in (None, ''):
1479
+ leaf['id'] = str(rid)[:64]
1480
+ window = _clean_window(node.get('window'))
1481
+ if window is not None:
1482
+ leaf['window'] = window
1483
+ # Owner items 3 + 4, carried under the SAME rule as CG-8's `id`/`window`: emitted
1484
+ # only when the input has them, so a plain column condition's cleaned shape is
1485
+ # byte-identical to what it always was and `clean_filter_tree` stays idempotent
1486
+ # (verify_filter_engine.py asserts that by exact structural comparison). Drop the
1487
+ # carry-through and the next autosave silently strips a date condition back to a
1488
+ # bare comparison against an empty value β€” i.e. back to INACTIVE.
1489
+ date_window = _clean_window(node.get('dateWindow'))
1490
+ if date_window is not None:
1491
+ leaf['dateWindow'] = date_window
1492
+ mode = node.get('dateMode')
1493
+ # SHAPE only. An unrecognised mode survives here and is refused by
1494
+ # `windows.resolve_anchor`, which returns None and makes the condition match
1495
+ # NOTHING β€” the same split as `_clean_window`, and the reason this module can stay
1496
+ # free of the date vocabulary it would otherwise have to keep in step.
1497
+ if isinstance(mode, str) and 0 < len(mode) <= 40:
1498
+ leaf['dateMode'] = mode
1499
+ rhs = _clean_rhs(node.get('rhs'), valid_keys)
1500
+ if rhs is not None:
1501
+ leaf['rhs'] = rhs
1502
+ out.append(leaf)
1503
+ return out
1504
+
1505
+
1506
+ def _default_view_config(fields):
1507
+ shown = [field["key"] for field in fields
1508
+ if field.get("default") is not False or field["source"] == "overlay"]
1509
+ hidden = [field["key"] for field in fields if field["key"] not in shown]
1510
+ return {
1511
+ # `filters` is the ROOT of the filter tree: leaf conditions and/or nested
1512
+ # condition groups ({conj, children}); `filterConj` joins the root level.
1513
+ "filters": [], "filterConj": "and",
1514
+ "sorts": [], "groupBy": None, "colorBy": None,
1515
+ "rowHeightMode": "short", "order": shown + hidden, "visible": shown,
1516
+ "widths": {}, "memberPids": [],
1517
+ }
1518
+
1519
+
1520
+ #: Wave 17 R1 / C-LOCKV β€” the `kind` a PROJECTED locked view wears. A cohort is not a separate
1521
+ #: kind of object any more: it is a saved view whose rows are a hand-curated set.
1522
+ LOCKED_VIEW_KIND = 'locked'
1523
+
1524
+
1525
+ def locked_view_projection(entry, base_config):
1526
+ """One cohort -> the saved-view row that IS it (wave 17 R1, contract C-LOCKV).
1527
+
1528
+ β›” THE LOCK IS THE VIEW'S IDENTITY, NOT ITS CONFIGURATION. `config.cohortLock` names the
1529
+ view's OWN id, which is what makes the shipped engine law (`useVisibleRows`: intersect the
1530
+ named set FIRST, unconditionally, and match NOTHING when the membership is unresolvable) do
1531
+ all the work with no second mechanism. Membership itself is NEVER copied in here β€” it stays
1532
+ in `customer_cohorts` and travels as `workspace.lists`, because a per-reader-scoped
1533
+ collection inside a client-writable `config` is deleted by the next autosave (see the
1534
+ contract's reason 2).
1535
+
1536
+ ⚠ `locked: True` is the LEGACY "undeletable/mode-frozen" flag and is deliberately NOT set:
1537
+ these views are ordinary in every respect the owner asked for β€” reorder, folder, sort,
1538
+ filter, change display mode. The lock mark in the rail is driven by `kind`.
1539
+ """
1540
+ return {
1541
+ 'id': entry['id'],
1542
+ 'name': entry.get('name') or entry['id'],
1543
+ 'kind': LOCKED_VIEW_KIND,
1544
+ 'config': {**base_config, 'cohortLock': entry['id']},
1545
+ }
1546
+
1547
+
1548
+ def views_from_defs(defs, saved_views, fields, system_name="All customers", locked_lists=None):
1549
+ """Convert legacy list formulas into the shared serializable SavedView contract.
1550
+
1551
+ `system_name` (wave 16 C-TOPIC) labels the system view per TOPIC ("All products" on the
1552
+ product surface). The ID stays "all-customers" on every topic β€” the client pins it
1553
+ (UNDELETABLE_VIEW_IDS, the landing default), and an id that varies by surface would fork
1554
+ that contract for a label's sake.
1555
+
1556
+ `locked_lists` (wave 17 R1) are the caller's cohorts, each PROJECTED as a saved view whose
1557
+ id IS the cohort id β€” so every stored reference to that id (a `cohortLock` on another view,
1558
+ an `is part of` condition, a folder placement) keeps pointing at the same thing and no
1559
+ rewrite map is needed. Saved config OVERLAYS the projection through the same mechanism the
1560
+ `list:` views have always used, which is what gives a locked view its own sort, filter,
1561
+ columns and display mode with no new storage."""
1562
+ base = _default_view_config(fields)
1563
+ views = [{
1564
+ "id": "all-customers", "name": system_name, "kind": "system",
1565
+ "locked": True, "config": dict(base),
1566
+ }]
1567
+ op_map = {">=": "gte", ">": "gt", "<=": "lte", "<": "lt", "=": "eq",
1568
+ "contains": "contains"}
1569
+ for name, definition in (defs or {}).items():
1570
+ filters = []
1571
+ for rule in definition.get("rules") or []:
1572
+ if rule.get("field") not in {field["key"] for field in fields}:
1573
+ continue
1574
+ filters.append({
1575
+ "colId": rule["field"],
1576
+ "op": op_map.get(rule.get("op"), "eq"),
1577
+ "value": str(rule.get("value") if rule.get("value") is not None else ""),
1578
+ })
1579
+ sort = str(definition.get("sort") or "")
1580
+ sorts = ([{"colId": sort.lstrip("-"),
1581
+ "dir": "desc" if sort.startswith("-") else "asc"}]
1582
+ if sort.lstrip("-") in {field["key"] for field in fields} else [])
1583
+ views.append({
1584
+ "id": "list:" + str(name),
1585
+ "name": str(name),
1586
+ "kind": "list",
1587
+ "note": str(definition.get("note") or ""),
1588
+ "config": {
1589
+ **base, "filters": filters, "sorts": sorts,
1590
+ "memberPids": [int(pid) for pid in definition.get("members") or []
1591
+ if isinstance(pid, int) or str(pid).isdigit()],
1592
+ },
1593
+ })
1594
+ # Wave 17 R1 β€” the cohorts, as ordinary views. Appended BEFORE the saved-config overlay
1595
+ # below so a user's own edits to a locked view (its sort, its columns, its display mode)
1596
+ # land on the projection instead of creating a second row with the same id.
1597
+ for _entry in (locked_lists or []):
1598
+ if isinstance(_entry, dict) and _entry.get('id'):
1599
+ views.append(locked_view_projection(_entry, base))
1600
+ index = {view["id"]: i for i, view in enumerate(views)}
1601
+ for view_id, saved in (saved_views or {}).items():
1602
+ if not isinstance(saved, dict) or not isinstance(saved.get("config"), dict):
1603
+ continue
1604
+ clean = dict(saved)
1605
+ clean["id"] = str(view_id)
1606
+ if view_id in index:
1607
+ views[index[view_id]] = clean
1608
+ else:
1609
+ views.append(clean)
1610
+ # β›” WAVE 17 R1 β€” RE-STAMP THE LOCK AFTER THE OVERLAY. The loop above REPLACES a projected
1611
+ # view with its saved record, and a saved record that omits `cohortLock` would hand back a
1612
+ # view that shows the WHOLE BOOK under a locked view's name. That is not hypothetical: the
1613
+ # client rebuilds `config` on every autosave (a column resize is enough), and the lock is
1614
+ # identity here, not something the browser is the source of truth for. Read-side rather than
1615
+ # write-side-only on purpose β€” this also repairs any record already written by another path.
1616
+ _locked_ids = {e['id']: e for e in (locked_lists or [])
1617
+ if isinstance(e, dict) and e.get('id')}
1618
+ if _locked_ids:
1619
+ for _v in views:
1620
+ _entry = _locked_ids.get(_v.get('id'))
1621
+ if not _entry:
1622
+ continue
1623
+ _v['kind'] = LOCKED_VIEW_KIND
1624
+ _v['config'] = {**(_v.get('config') or {}), 'cohortLock': _v['id']}
1625
+ # One thing, one name: the cohort store owns it (the rename event routes there), so
1626
+ # a stale `name` on the saved record can never fork into a second title.
1627
+ _v['name'] = _entry.get('name') or _v['id']
1628
+ return views
1629
+
1630
+
1631
+ def workspace_wire(ws, uname, pool_pids, defs=None, scope_key='customer', storage_key=None,
1632
+ fields_base=None, with_cohorts=True):
1633
+ """The client's `GridWorkspace` WIRE SHAPE from the stored table workspace β€” the ONE
1634
+ projection, shared by both servers (app.py's `_table_grid` and the API's `/workspace`).
1635
+
1636
+ β›” WHY THIS EXISTS (2026-07-30). The API route used to return the STORE shape with no
1637
+ `storageKey` β€” and the client validator (`fetchWorkspace`) requires one, so the standalone
1638
+ shell silently discarded the whole workspace: saved views never rendered and `cohortMode`
1639
+ never arrived (the Cohort route drew the Customer surface). Duplicating the host's inline
1640
+ projection into the route would have re-created the same drift one wave later; extracting it
1641
+ means the wire can only be one thing.
1642
+
1643
+ Returns `(workspace, fields, views, cohort_lists)` β€” the extra three because the host
1644
+ interleaves further work (docs, derived cells, measure sets) that consumes them.
1645
+
1646
+ HOST-ONLY extras stay with the host: `docs`/`docPayload`, `pool`, `hideViews`,
1647
+ `cohortMode`/`scopeChoice` (the API stamps its own from `?scope=`).
1648
+
1649
+ Wave 16 C-TOPIC: `fields_base` selects the canonical contract (absent = customer,
1650
+ byte-identical); `with_cohorts=False` is the PRODUCT surface β€” cohorts are customer
1651
+ machinery (`modules/cohort` stores partner pids), and resolving them against product pids
1652
+ would intersect two unrelated id spaces and render a plausible, meaningless member count.
1653
+ """
1654
+ import modules.cohort as cohort_mod
1655
+
1656
+ cohort_lists = []
1657
+ if with_cohorts:
1658
+ for cid, c in sorted(cohort_mod.visible(uname, pool_pids).items(),
1659
+ key=lambda kv: (kv[1].get('name') or '').lower()):
1660
+ members = [p for p in (c.get('members') or []) if p in pool_pids]
1661
+ entry = {'id': cid, 'name': c.get('name') or cid, 'pids': members}
1662
+ # Rule 8b: a member can drop out of the 24-month pool without the cohort being
1663
+ # wrong, and a silently smaller cohort is exactly what the unverifiable-count rule
1664
+ # forbids.
1665
+ missing = len(c.get('members') or []) - len(members)
1666
+ if missing:
1667
+ entry['missing'] = missing
1668
+ cohort_lists.append(entry)
1669
+
1670
+ fields = fields_from_workspace(ws, cohorts=bool(cohort_lists), scope_key=scope_key,
1671
+ fields_base=fields_base)
1672
+ views = views_from_defs(defs or {}, ws.get('views'), fields,
1673
+ system_name=("All products" if scope_key == 'product'
1674
+ else "All customers"),
1675
+ locked_lists=cohort_lists)
1676
+ workspace = {'storageKey': storage_key, 'views': views, 'lists': cohort_lists}
1677
+ # Owner item 3 (2026-07-31): where this user left off. The client's own localStorage copy
1678
+ # wins when present; this is the server's answer for a FRESH browser, which used to fall
1679
+ # all the way to the system default view (and whatever display mode was stored on it).
1680
+ if ws.get('activeViewId'):
1681
+ workspace['activeViewId'] = str(ws['activeViewId'])
1682
+
1683
+ # FOLDERS (owner item 11, contract C4), re-validated at SERVE time: a view or cohort can be
1684
+ # deleted by a path that knows nothing about folders, and the placement map must not outlive
1685
+ # the thing it points at.
1686
+ _folders = clean_folders(ws.get('folders'))
1687
+ _view_ids = {v['id'] for v in (views or []) if isinstance(v, dict) and v.get('id')}
1688
+ # ── WAVE 17 R1 (C-LOCKV amendment 2026-08-03): the two folder surfaces become ONE, AT
1689
+ # SERVE TIME rather than by a store migration. A locked view is an ordinary view now, so
1690
+ # its folder has to be an ordinary view folder β€” but rewriting the stored map would be a
1691
+ # one-shot write that has to be got right once, while this is a projection that is right
1692
+ # every time it runs. New drags write to `views` anyway (the client only knows that
1693
+ # surface), so `cohorts` drains on its own and never needs a second pass.
1694
+ # ⚠ A cohorts-surface folder whose id ALREADY names a views folder is DROPPED, not merged:
1695
+ # re-parenting somebody's list into a folder that merely shares an id is a worse outcome
1696
+ # than the list appearing at the root, where it is visible and one drag from home.
1697
+ _cf = list(_folders.get('cohorts') or [])
1698
+ if _cf:
1699
+ _vf = list(_folders.get('views') or [])
1700
+ _taken = {f['id'] for f in _vf}
1701
+ _order = len(_vf)
1702
+ for _f in _cf:
1703
+ if _f['id'] in _taken:
1704
+ continue
1705
+ _vf.append({**_f, 'order': _order})
1706
+ _order += 1
1707
+ _folders['views'] = _vf
1708
+ _raw_item_folders = dict(ws.get('itemFolders') or {})
1709
+ if _raw_item_folders.get('cohorts'):
1710
+ # Cohort placements now describe VIEWS (same ids β€” that is the point of preserving them).
1711
+ # A placement already stored on the views surface WINS: it is the more recent act.
1712
+ _raw_item_folders['views'] = {**dict(_raw_item_folders.get('cohorts') or {}),
1713
+ **dict(_raw_item_folders.get('views') or {})}
1714
+ _placed = clean_item_folders(
1715
+ _raw_item_folders, _folders,
1716
+ {'views': _view_ids, 'cohorts': {c['id'] for c in cohort_lists}})
1717
+ if _folders.get('views'):
1718
+ workspace['folders'] = _folders['views']
1719
+ # β›” `cohortFolders` IS NO LONGER EMITTED. The rail has no cohorts section to fold, and a
1720
+ # wire that still described one would invite a second rendering of rows that are now views.
1721
+ _vplaced = _placed.get('views') or {}
1722
+ for _v in (views or []):
1723
+ if isinstance(_v, dict) and _v.get('id') in _vplaced:
1724
+ _v['folderId'] = _vplaced[_v['id']]
1725
+ _cplaced = _placed.get('cohorts') or {}
1726
+ for _c in cohort_lists:
1727
+ if _c['id'] in _cplaced:
1728
+ _c['folderId'] = _cplaced[_c['id']]
1729
+
1730
+ # RECORD LAYOUT (wave 2026-08-02, C-LAYOUT): the per-user record-detail field order,
1731
+ # re-validated at SERVE time exactly like folders β€” a field can be deleted by a path
1732
+ # that knows nothing about this stratum, and a stale key must not outlive its field.
1733
+ _rl = ws.get('recordLayout')
1734
+ if isinstance(_rl, dict) and isinstance(_rl.get('order'), list):
1735
+ _fkeys = {f['key'] for f in fields if isinstance(f, dict) and f.get('key')}
1736
+ _order, _seen = [], set()
1737
+ for _k in _rl['order'][:200]:
1738
+ _k = str(_k or '')
1739
+ if _k and _k in _fkeys and _k not in _seen:
1740
+ _seen.add(_k)
1741
+ _order.append(_k)
1742
+ if _order:
1743
+ workspace['recordLayout'] = {'order': _order}
1744
+
1745
+ return workspace, fields, views, cohort_lists
1746
+
1747
+
1748
+ def embed_html_path():
1749
+ """The first existing candidate path for the inlined single-file build, or None."""
1750
+ for p in _EMBED_CANDIDATES:
1751
+ if p.is_file():
1752
+ return p
1753
+ return None
1754
+
1755
+
1756
+ def scope_counts(shown, matched, total):
1757
+ """The honest 'N of M' a SERVER-WINDOWED table must carry (CG-2).
1758
+
1759
+ `matched` and `total` MUST come from their own queries over the whole scope. Never pass
1760
+ `len(rows)` as `matched` β€” that is the silent [:N] this exists to prevent: the page would
1761
+ report the window size as though it were the result size.
1762
+
1763
+ Refuses the shapes that could only be a mistake, because a wrong count here is invisible on
1764
+ screen (it looks like a smaller dataset, not like an error).
1765
+ """
1766
+ shown, matched, total = int(shown), int(matched), int(total)
1767
+ if matched > total:
1768
+ raise ValueError(f"matched ({matched}) exceeds total ({total}) β€” a filter cannot match "
1769
+ f"more rows than the scope holds")
1770
+ if shown > matched:
1771
+ raise ValueError(f"shown ({shown}) exceeds matched ({matched}) β€” the window cannot hold "
1772
+ f"more rows than the filter matched")
1773
+ return {"shown": shown, "matched": matched, "total": total, "windowed": True}
1774
+
1775
+
1776
+ def build_html(rows, fields, *, workspace=None, path=None, counts=None):
1777
+ """PURE: read the inlined single-file build and inject the data handshake.
1778
+
1779
+ Returns a self-contained HTML string with
1780
+ <script>window.__AIOS_GRID__ = {"fields": ..., "rows": ...};</script>
1781
+ inserted immediately after the opening <head> tag β€” a CLASSIC (non-module) script, so it
1782
+ executes during parse BEFORE the app's deferred `type="module"` script reads the global.
1783
+ Raises FileNotFoundError if no build is present (callers should try/except and fall back).
1784
+ """
1785
+ p = Path(path) if path else embed_html_path()
1786
+ if p is None or not Path(p).is_file():
1787
+ raise FileNotFoundError(
1788
+ "aios_grid: no embed build found. Run `npm run build:embed` in aios-web/web and "
1789
+ "copy dist-embed/index.html -> platform/aios_grid_embed.html "
1790
+ "(python build_embed.py). Looked in: "
1791
+ + " ; ".join(str(c) for c in _EMBED_CANDIDATES))
1792
+ html = Path(p).read_text(encoding="utf-8")
1793
+ # ensure_ascii=False keeps names readable; default=str is a safety net for stray
1794
+ # date/Decimal; the </ escape prevents a value containing "</script>" from breaking out.
1795
+ data = {"fields": fields, "rows": rows, "workspace": workspace or {}}
1796
+ # Only a WINDOWED table carries counts. Omitting the key leaves the client counting the rows
1797
+ # it holds, which is correct when it holds all of them.
1798
+ if counts:
1799
+ data["counts"] = counts
1800
+ payload = json.dumps(data, ensure_ascii=False, default=str).replace("</", "<\\/")
1801
+ tag = "<script>window.__AIOS_GRID__ = " + payload + ";</script>"
1802
+ new_html, n = re.subn(r"(<head\b[^>]*>)", lambda m: m.group(1) + tag, html, count=1)
1803
+ if n == 0: # no <head> (shouldn't happen) β€” still run it first
1804
+ new_html = tag + html
1805
+ return new_html
1806
+
1807
+
1808
+ def component_dir():
1809
+ for path in _COMPONENT_CANDIDATES:
1810
+ if (path / "index.html").is_file():
1811
+ return path
1812
+ return None
1813
+
1814
+
1815
+ def render(rows, fields, *, workspace=None, height=680, key="aios_grid",
1816
+ user_options=None, user_avatars=None, measures=None, measure_sets=None,
1817
+ today=None, viewer=None):
1818
+ """Render the Airtable-style grid and return its latest guarded host event.
1819
+
1820
+ The preferred custom-component path forwards `key` and supports two-way persistence.
1821
+ If only the legacy HTML build exists, it renders safely with no return event.
1822
+
1823
+ `user_options` are the assignable people for `user`-typed overlay columns. They come from
1824
+ the HOST (the tenant's real accounts) rather than from anything the grid could invent, so
1825
+ an assignee is always someone who can actually log in and see the row.
1826
+
1827
+ `measures` / `measure_sets` are CG-8. A measure is FILTERABLE BUT NOT DISPLAYABLE: "Sales in
1828
+ the last 90 days" is a question about a window, not a column of this table, and rendering it
1829
+ as a column would silently pick a window for the user. The HOST resolves each condition to a
1830
+ set of row ids (`{ruleId: [pid, ...]}`) and the client treats that set as a membership test β€”
1831
+ which is why this module can offer measure filtering while keeping ZERO app-internal imports:
1832
+ it never computes a measure, it only carries one.
1833
+
1834
+ `today` is the TENANT'S current date as an ISO string, and every relative date condition
1835
+ ("is within the past month", "is before one month ago") resolves against it. It comes from
1836
+ the host rather than from `new Date()` in the browser for the same three reasons
1837
+ harness/windows.py takes it as a parameter: a viewer a day ahead of the server would resolve
1838
+ a different month than the server summed, a window must not shift between the count and the
1839
+ rows, and the tenant's day boundary is the tenant's. Omit it and those conditions match
1840
+ NOTHING with a visible marker β€” never everything.
1841
+ """
1842
+ import streamlit.components.v1 as _c
1843
+ directory = component_dir()
1844
+ payload = {"fields": fields, "rows": rows, "workspace": workspace or {},
1845
+ "userOptions": [str(u)[:120] for u in (user_options or [])][:500],
1846
+ # Wave 14 C-AVATAR β€” display name -> data URL, the userOptions vocabulary's
1847
+ # companion. Same caps; a value that is not a data: URL is dropped here too
1848
+ # (defence in depth beside the API route's write-side wall).
1849
+ "userAvatars": {str(k)[:120]: str(v)
1850
+ for k, v in list((user_avatars or {}).items())[:500]
1851
+ if str(v).startswith("data:image/")},
1852
+ "measures": list(measures or []),
1853
+ "today": str(today) if today else None,
1854
+ # Wave-5 item 1: who is looking, host-computed β€” what gates the client's
1855
+ # "Edit field permissions" menu entry. The HOST-side enforcement is the wall;
1856
+ # this is the courtesy that keeps the menu honest.
1857
+ "viewer": ({"name": str(viewer.get("name") or "")[:80],
1858
+ "isAdmin": bool(viewer.get("isAdmin"))}
1859
+ if isinstance(viewer, dict) else None),
1860
+ "measureSets": {str(k): list(v) for k, v in (measure_sets or {}).items()}}
1861
+ if directory is not None:
1862
+ cache_key = str(directory)
1863
+ component = _DECLARED_COMPONENTS.get(cache_key)
1864
+ if component is None:
1865
+ component = _c.declare_component("aios_customer_grid", path=cache_key)
1866
+ _DECLARED_COMPONENTS[cache_key] = component
1867
+ return component(payload=payload, height=height, key=key, default=None)
1868
+ html = build_html(rows, fields, workspace=workspace)
1869
+ _c.html(html, height=height, scrolling=False)
1870
+ return None
platform/aios_grid_fields.json ADDED
@@ -0,0 +1,413 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "_comment": "CANONICAL field contract for the AIOS Airtable-style grid \u2014 the SINGLE source of truth. Consumed by platform/aios_grid.py (embed/Space host) and aios-web/api/main.py (standalone API), and regenerated into aios-web/web/public/sample_customers.json. Edit HERE only, then run aios-web/verify_fields_contract.py. source=odoo is READ-ONLY; source=overlay is the editable stratum (notes/tags) outside Odoo. type in {text,status,select,currency,int,date,pct} (select = a fixed-choice READ-ONLY brand attribute; dba is the first, wave 2026-08-02). `description` (wave 5) is the CANONICAL per-field description \u2014 every field must carry one, and since wave 7 (owner W8, 2026-07-28) every description is ONE SHORT PLAIN sentence (two only when a fact would otherwise mislead): what the field IS, nothing else \u2014 no filter tips, no '(none)' coaching, no rationale; the user's workspace NOTE overrides it in the (i) hover, never in this file. BUILDER FACT (documented here, deliberately NOT in user-facing text): blank text attributes display as '(none)', so `is '(none)'` \u2014 not `is empty` \u2014 finds the blanks on agent/city/state/country/zip/payment_terms/pricelist/tags. filterable:false = the CONDITION BUILDER does not offer it (still displayed, still sortable); every such field must have a replacement declared in aios-web/verify_fields_contract.py. 2026-07-27 partner attributes: country/zip/payment_terms/pricelist/tags/customer_since all ship default:false. zip is TEXT because a postal code has leading zeros. Odoo's credit_limit (1% populated) and user_id salesperson (2%) are deliberately ABSENT; agent_ids is the salesperson field and AR is where credit exposure comes from. Wave-5 item 8 (2026-07-27): ltm_rev and at_risk are DELETED \u2014 LTM's replacement is a creatable Sales measure column (the demo column IS Sales \u00b7 the last 12 months), at_risk's replacement is a formula field, e.g. MAX(0, {revenue_ly} - {revenue_ytd}). Wave-6 item 8 (2026-07-27, the no-buildable-presets rule): revenue_ytd, revenue_ly, orders_24m, aov and yoy_pct are DELETED \u2014 every one is self-buildable, so a frozen pre-set beside the builder was two ways to ask one question. Replacements (recorded in verify_fields_contract.py): creatable measure columns for Sales / Orders / Avg order $ over any period (harness/measure_filter.py ADMITTED carries revenue, orders and the composite aov), and a formula over two measure columns for YoY, e.g. ({sales_ytd} - {sales_ly}) / {sales_ly}. Stale view colIds naming the five self-heal on the next autosave (the established rule).",
3
+ "fields": [
4
+ {
5
+ "key": "customer",
6
+ "label": "Customer",
7
+ "type": "text",
8
+ "source": "odoo",
9
+ "pinned": true,
10
+ "default": true,
11
+ "description": "The customer's name in Odoo. One row per customer who ordered in the last 24 months."
12
+ },
13
+ {
14
+ "key": "odoo_status",
15
+ "label": "Odoo record",
16
+ "type": "status",
17
+ "source": "odoo",
18
+ "default": false,
19
+ "description": "Whether this customer still exists in Odoo. Archived means deleted there."
20
+ },
21
+ {
22
+ "key": "agent",
23
+ "label": "Agent",
24
+ "type": "text",
25
+ "source": "odoo",
26
+ "default": true,
27
+ "description": "The sales agent who owns this account."
28
+ },
29
+ {
30
+ "key": "dba",
31
+ "label": "DBA",
32
+ "type": "select",
33
+ "source": "odoo",
34
+ "default": false,
35
+ "options": [
36
+ "Fisch",
37
+ "Royal",
38
+ "Both"
39
+ ],
40
+ "description": "The brand this customer buys from - Fisch, Royal, or both. Amazon-channel orders are not a DBA."
41
+ },
42
+ {
43
+ "key": "salesperson",
44
+ "label": "Salesperson",
45
+ "type": "text",
46
+ "source": "odoo",
47
+ "default": false,
48
+ "description": "Who keyed in most of this customer's orders \u2014 not the Agent, who owns the account."
49
+ },
50
+ {
51
+ "key": "city",
52
+ "label": "City",
53
+ "type": "text",
54
+ "source": "odoo",
55
+ "default": true,
56
+ "description": "City on the customer's Odoo address."
57
+ },
58
+ {
59
+ "key": "state",
60
+ "label": "State",
61
+ "type": "text",
62
+ "source": "odoo",
63
+ "default": true,
64
+ "description": "State or province on the customer's Odoo address."
65
+ },
66
+ {
67
+ "key": "country",
68
+ "label": "Country",
69
+ "type": "text",
70
+ "source": "odoo",
71
+ "default": false,
72
+ "description": "Country on the customer's Odoo address."
73
+ },
74
+ {
75
+ "key": "zip",
76
+ "label": "ZIP",
77
+ "type": "text",
78
+ "source": "odoo",
79
+ "default": false,
80
+ "description": "Postal code on the customer's Odoo address."
81
+ },
82
+ {
83
+ "key": "customer_since",
84
+ "label": "Customer since",
85
+ "type": "date",
86
+ "source": "odoo",
87
+ "default": false,
88
+ "description": "When this customer was first set up in Odoo."
89
+ },
90
+ {
91
+ "key": "tags",
92
+ "label": "Tags",
93
+ "type": "text",
94
+ "source": "odoo",
95
+ "default": false,
96
+ "description": "Odoo labels on this customer, comma-separated."
97
+ },
98
+ {
99
+ "key": "pricelist",
100
+ "label": "Price list",
101
+ "type": "text",
102
+ "source": "odoo",
103
+ "default": false,
104
+ "description": "The price list this customer buys on."
105
+ },
106
+ {
107
+ "key": "payment_terms",
108
+ "label": "Payment terms",
109
+ "type": "text",
110
+ "source": "odoo",
111
+ "default": false,
112
+ "description": "Payment terms on this customer's account \u2014 Net 30, for example."
113
+ },
114
+ {
115
+ "key": "last_order",
116
+ "label": "Last order",
117
+ "type": "date",
118
+ "source": "odoo",
119
+ "default": true,
120
+ "description": "Date of the most recent confirmed order."
121
+ },
122
+ {
123
+ "key": "overdue_days",
124
+ "label": "Overdue days",
125
+ "type": "int",
126
+ "source": "odoo",
127
+ "default": true,
128
+ "description": "How many days late this customer is running against their own usual ordering rhythm."
129
+ },
130
+ {
131
+ "_note": "filterable:false \u2014 DERIVED ANALYTIC: est_missed is min(cycles missed, 3) x AOV, a score we compute rather than an object the business has, so a condition on it would read as a fact about the customer when it is a fact about our arithmetic. It still displays and still sorts. Until wave 6 this flag also covered the frozen-window presets (revenue_ytd / revenue_ly / orders_24m / aov / yoy_pct); those are now DELETED outright under the owner's no-buildable-presets rule \u2014 see _comment. est_missed itself STAYS: no creatable measure or formula reproduces the cadence model behind it.",
132
+ "key": "est_missed",
133
+ "label": "Est. missed $",
134
+ "type": "currency",
135
+ "source": "odoo",
136
+ "default": true,
137
+ "agg": "sum",
138
+ "filterable": false,
139
+ "description": "Estimated sales missed while quiet: missed orders (capped at 3) times average order value. An estimate, not money owed."
140
+ },
141
+ {
142
+ "key": "ar_open",
143
+ "label": "AR open $",
144
+ "type": "currency",
145
+ "source": "odoo",
146
+ "default": false,
147
+ "description": "Invoiced money owed, not yet overdue."
148
+ },
149
+ {
150
+ "key": "ar_overdue",
151
+ "label": "AR overdue $",
152
+ "type": "currency",
153
+ "source": "odoo",
154
+ "default": false,
155
+ "description": "Invoiced money past due \u2014 same basis as the Collections page."
156
+ },
157
+ {
158
+ "key": "ar_exposure",
159
+ "label": "Credit exposure $",
160
+ "type": "currency",
161
+ "source": "odoo",
162
+ "default": false,
163
+ "description": "The most you could be out if they stopped paying today: open, overdue, draft and not-yet-invoiced."
164
+ },
165
+ {
166
+ "key": "ar_aged_1_30",
167
+ "label": "1-30 days $",
168
+ "type": "currency",
169
+ "source": "odoo",
170
+ "default": false,
171
+ "description": "Overdue between 1 and 30 days. The four aging buckets sum to AR overdue $."
172
+ },
173
+ {
174
+ "key": "ar_aged_31_60",
175
+ "label": "31-60 days $",
176
+ "type": "currency",
177
+ "source": "odoo",
178
+ "default": false,
179
+ "description": "Overdue between 31 and 60 days. The four aging buckets sum to AR overdue $."
180
+ },
181
+ {
182
+ "key": "ar_aged_61_90",
183
+ "label": "61-90 days $",
184
+ "type": "currency",
185
+ "source": "odoo",
186
+ "default": false,
187
+ "description": "Overdue between 61 and 90 days. The four aging buckets sum to AR overdue $."
188
+ },
189
+ {
190
+ "key": "ar_aged_90_plus",
191
+ "label": "90+ days $",
192
+ "type": "currency",
193
+ "source": "odoo",
194
+ "default": false,
195
+ "description": "Overdue by more than 90 days. The four aging buckets sum to AR overdue $."
196
+ },
197
+ {
198
+ "key": "days_to_pay",
199
+ "label": "Days to pay",
200
+ "type": "int",
201
+ "source": "odoo",
202
+ "default": false,
203
+ "description": "Average days to pay an invoice in full. Blank means no fully paid invoice yet."
204
+ },
205
+ {
206
+ "key": "top_category",
207
+ "label": "Top category",
208
+ "type": "text",
209
+ "source": "odoo",
210
+ "default": false,
211
+ "description": "The category this customer spent the most on in the last 12 months."
212
+ },
213
+ {
214
+ "key": "top_category_pct",
215
+ "label": "Top category %",
216
+ "type": "pct",
217
+ "source": "odoo",
218
+ "default": false,
219
+ "description": "Share of last-12-months spend that went to the top category."
220
+ },
221
+ {
222
+ "key": "sku_count",
223
+ "label": "SKUs bought",
224
+ "type": "int",
225
+ "source": "odoo",
226
+ "default": false,
227
+ "description": "Distinct products bought in the last 12 months."
228
+ },
229
+ {
230
+ "key": "top_sku",
231
+ "label": "Top SKU",
232
+ "type": "text",
233
+ "source": "odoo",
234
+ "default": false,
235
+ "description": "The product this customer spent the most on in the last 12 months."
236
+ },
237
+ {
238
+ "key": "days_since",
239
+ "label": "Days since order",
240
+ "type": "int",
241
+ "source": "odoo",
242
+ "default": false,
243
+ "description": "Days since the last confirmed order."
244
+ },
245
+ {
246
+ "key": "typical_gap_days",
247
+ "label": "Typical gap days",
248
+ "type": "int",
249
+ "source": "odoo",
250
+ "default": false,
251
+ "description": "Days this customer usually goes between orders, from their own history."
252
+ },
253
+ {
254
+ "key": "notes",
255
+ "label": "Notes",
256
+ "type": "text",
257
+ "source": "overlay",
258
+ "default": false,
259
+ "description": "Your notes on this customer. Saved in this app only, visible only to you."
260
+ }
261
+ ],
262
+ "_product_comment": "ADDITIVE, wave 15 C-TOPIC. The PRODUCT table's field contract. Kept as a SEPARATE top-level key rather than restructuring `fields` into {customer_data, product_data}: both existing readers (aios_grid._load_fields, aios-web/api/main.py) index doc['fields'] directly, and reshaping that mid-wave would break the embed for a cosmetic gain. The keyed shape can arrive when both readers move in ONE commit; until then this is the product half and `fields` is the customer half.",
263
+ "_product_removed_buy_now": "OWNER, 2026-08-03: 'Buy signal' (key buy_now, a select of Buy now / OK) is NO LONGER A PRESET FIELD. It never earned one: it is a formula over two columns that are both still right here, and the platform has a formula field type for exactly that. THE FORMULA, which reproduces the retired column row for row (modules/product_data.validate proves the equivalence, and goes red if it ever stops holding): IF({lead_days} > 0, IF({dos} < {lead_days}, \"Buy now\", \"OK\"), \"\") . Every branch matches the old server rule, including the blanks - the formula engine refuses a comparison against a blank rather than coercing it to 0, so a SKU with no days-of-supply or no lead time comes out empty, which is 'we do not know' and not 'you are fine'. NOTE the column is still COMPUTED in product_data.pool(): it ships nowhere (rows_from_pool projects strictly through this contract, so no Field means no cell on the wire) and exists only as validate()'s oracle. A formula field is PER-USER, so nothing shared may filter on it - the Buy list view filters on dos/lead_days directly (_seed_wave17).",
264
+ "product_data": {
265
+ "identity": "pid",
266
+ "business_key": "code",
267
+ "fields": [
268
+ {
269
+ "key": "code",
270
+ "label": "SKU",
271
+ "type": "text",
272
+ "source": "odoo",
273
+ "pinned": true,
274
+ "default": true,
275
+ "description": "The SKU code \u2014 the product's real business key. `pid` is a stable CRC32 of it because the grid keys on an integer."
276
+ },
277
+ {
278
+ "key": "product",
279
+ "label": "Product",
280
+ "type": "text",
281
+ "source": "odoo",
282
+ "default": true,
283
+ "description": "Product name as it appears in Odoo."
284
+ },
285
+ {
286
+ "key": "category",
287
+ "label": "Category",
288
+ "type": "select",
289
+ "source": "odoo",
290
+ "default": true,
291
+ "description": "Product category; '(uncategorized)' when Odoo carries none."
292
+ },
293
+ {
294
+ "key": "supplier",
295
+ "label": "Supplier",
296
+ "type": "text",
297
+ "source": "odoo",
298
+ "default": true,
299
+ "description": "Who makes it, from the inventory mastersheet. Not an Odoo field."
300
+ },
301
+ {
302
+ "key": "origin_country",
303
+ "label": "Country",
304
+ "type": "text",
305
+ "source": "odoo",
306
+ "default": false,
307
+ "description": "Country of origin, from the inventory mastersheet."
308
+ },
309
+ {
310
+ "key": "lead_days",
311
+ "label": "Lead time (days)",
312
+ "type": "int",
313
+ "source": "odoo",
314
+ "default": true,
315
+ "description": "Order-to-arrival days for this supplier. Drives the buy signal."
316
+ },
317
+ {
318
+ "key": "first_cost",
319
+ "label": "First cost",
320
+ "type": "currency",
321
+ "source": "odoo",
322
+ "default": false,
323
+ "description": "Quoted unit cost at origin, before freight and duty."
324
+ },
325
+ {
326
+ "key": "rev_ytd",
327
+ "label": "Revenue YTD",
328
+ "type": "currency",
329
+ "source": "odoo",
330
+ "default": true,
331
+ "description": "Year-to-date revenue for this SKU, BU-scoped when the caller is."
332
+ },
333
+ {
334
+ "key": "rev_ly",
335
+ "label": "Revenue LY",
336
+ "type": "currency",
337
+ "source": "odoo",
338
+ "description": "Same period last year \u2014 seasonal wholesale compares like for like."
339
+ },
340
+ {
341
+ "key": "yoy_pct",
342
+ "label": "YoY %",
343
+ "type": "pct",
344
+ "source": "odoo",
345
+ "description": "Year-over-year change; null when last year was zero (a ratio to zero is not a number)."
346
+ },
347
+ {
348
+ "key": "qty_ytd",
349
+ "label": "Units YTD",
350
+ "type": "int",
351
+ "source": "odoo",
352
+ "description": "Units sold year to date."
353
+ },
354
+ {
355
+ "key": "orders_ytd",
356
+ "label": "Orders YTD",
357
+ "type": "int",
358
+ "source": "odoo",
359
+ "description": "Distinct orders containing this SKU, year to date."
360
+ },
361
+ {
362
+ "key": "on_hand",
363
+ "label": "On hand",
364
+ "type": "int",
365
+ "source": "odoo",
366
+ "description": "Units in stock. CONSOLIDATED \u2014 one physical warehouse, not brand-tagged, so this column is ABSENT for a BU-scoped caller rather than silently company-wide."
367
+ },
368
+ {
369
+ "key": "unit_cost",
370
+ "label": "Unit cost",
371
+ "type": "currency",
372
+ "source": "odoo",
373
+ "description": "Inventory unit cost. Consolidated; absent for a BU-scoped caller."
374
+ },
375
+ {
376
+ "key": "inv_value",
377
+ "label": "Stock value",
378
+ "type": "currency",
379
+ "source": "odoo",
380
+ "description": "On-hand value at cost. Consolidated; absent for a BU-scoped caller."
381
+ },
382
+ {
383
+ "key": "qty_ltm",
384
+ "label": "Units LTM",
385
+ "type": "int",
386
+ "source": "odoo",
387
+ "description": "Units sold in the last twelve months. Consolidated; absent for a BU-scoped caller."
388
+ },
389
+ {
390
+ "key": "dos",
391
+ "label": "Days of supply",
392
+ "type": "int",
393
+ "source": "odoo",
394
+ "description": "Days of supply at the LTM rate; null means it never sells through. Consolidated; absent for a BU-scoped caller."
395
+ },
396
+ {
397
+ "key": "cover_gap_d",
398
+ "label": "Cover gap (days)",
399
+ "type": "int",
400
+ "source": "odoo",
401
+ "default": false,
402
+ "description": "Days of supply minus lead time. Negative means it runs out before a reorder lands."
403
+ },
404
+ {
405
+ "key": "stock_bucket",
406
+ "label": "Stock status",
407
+ "type": "select",
408
+ "source": "odoo",
409
+ "description": "Dead / excess / healthy bucket from the inventory module. Consolidated; absent for a BU-scoped caller."
410
+ }
411
+ ]
412
+ }
413
+ }
platform/core/assets.py ADDED
@@ -0,0 +1,73 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """core/assets.py β€” the product-image ASSET SOURCE (wave 18, C2-ASSET).
2
+
3
+ WHY THIS FILE EXISTS AT ALL, given the route is three functions long: the API process is
4
+ HOST-AGNOSTIC by contract (`ops/verify_portability.py` checks B1/B2 β€” the HuggingFace SDK may
5
+ appear ONLY in its declared homes, all of which live under `platform/`, so the FastAPI
6
+ service can be lifted to Hetzner/EC2 without dragging a hub client with it). The first version
7
+ of `routes_assets.py` imported `huggingface_hub` directly and the gate caught it. The fetch is
8
+ a STORE-shaped concern anyway, so it belongs beside `core/store.py`.
9
+
10
+ Two sources, checked by the caller in order:
11
+ * a local master directory (`AIOS_ASSET_DIR`) β€” dev, and the fastest path on the owner's box;
12
+ * the private HF dataset `AIOS_ASSET_REPO` (default `royal-imports/product-assets`), laid out
13
+ by `ingest_product_assets.py`:
14
+ assets/products/orig/<CODE><ext> masters, archived verbatim (owner ruling R9)
15
+ assets/products/web/<CODE>.png ~800px derivatives (what the app displays)
16
+ manifest.json {"codes": {"<CODE>": {"ext": ".png"}}, ...}
17
+
18
+ Every failure returns None/{} rather than raising: a missing image is "no picture for that SKU",
19
+ never a broken catalog page.
20
+ """
21
+ import os
22
+ import time
23
+ from pathlib import Path
24
+
25
+ DEFAULT_REPO = 'royal-imports/product-assets'
26
+ _MANIFEST_TTL = 300
27
+ #: (fetched_at, {"CODE": {"ext": ".png"}}) β€” one manifest read per process per TTL.
28
+ _MANIFEST = {'at': 0.0, 'codes': None}
29
+
30
+
31
+ def repo_id():
32
+ return os.environ.get('AIOS_ASSET_REPO', '').strip() or DEFAULT_REPO
33
+
34
+
35
+ def _token():
36
+ return os.environ.get('HF_TOKEN') or None
37
+
38
+
39
+ def manifest():
40
+ """{CODE: {'ext': '.png'}} for the configured asset repo, or {} when unreachable."""
41
+ now = time.time()
42
+ if _MANIFEST['codes'] is not None and now - _MANIFEST['at'] < _MANIFEST_TTL:
43
+ return _MANIFEST['codes']
44
+ try:
45
+ import json
46
+
47
+ from huggingface_hub import hf_hub_download
48
+ p = hf_hub_download(repo_id(), 'manifest.json', repo_type='dataset', token=_token())
49
+ codes = (json.loads(Path(p).read_text(encoding='utf-8')) or {}).get('codes') or {}
50
+ except Exception:
51
+ codes = {}
52
+ _MANIFEST.update(at=now, codes=codes)
53
+ return codes
54
+
55
+
56
+ def fetch(code, quality='web'):
57
+ """A local Path to one asset, or None. The hub cache dedupes across requests, so a container
58
+ downloads each file at most once."""
59
+ meta = manifest().get(code)
60
+ if not meta:
61
+ return None
62
+ rel = (f'assets/products/web/{code}.png' if quality == 'web'
63
+ else f"assets/products/orig/{code}{meta.get('ext', '.png')}")
64
+ try:
65
+ from huggingface_hub import hf_hub_download
66
+ return Path(hf_hub_download(repo_id(), rel, repo_type='dataset', token=_token()))
67
+ except Exception:
68
+ return None
69
+
70
+
71
+ def reset_cache():
72
+ """Drop the memoized manifest β€” for gates and for a post-ingest refresh."""
73
+ _MANIFEST.update(at=0.0, codes=None)
platform/core/auth_google.py ADDED
@@ -0,0 +1,91 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Sign in with Google β€” TEMPLATE ONLY (wave-7 W10, 2026-07-28). Nothing here runs in
2
+ production yet: every flow function raises until the wiring checklist in
3
+ docs/GOOGLE_LOGIN_TODO.md is completed and GOOGLE_LOGIN_ENABLED=1 is set.
4
+
5
+ DESIGN (recorded now so the full build has a contract to meet):
6
+ - OAuth 2.0 authorization-code flow with PKCE against Google Identity
7
+ (https://accounts.google.com/.well-known/openid-configuration). No SDK dependency β€”
8
+ three HTTPS calls (authorize redirect, token exchange, JWKS fetch) keep the surface
9
+ auditable and the requirements.txt unchanged until we commit.
10
+ - Google is an IDENTITY, not an ACCOUNT SOURCE. A Google sign-in maps to an EXISTING
11
+ core/users.py account via its `email` attribute (set_access(..., email=...)); an
12
+ unknown email FAILS CLOSED with "no account for this Google identity" β€” Google login
13
+ never creates users, so the per-BU/per-module grant model stays the only door.
14
+ - The password gate stays byte-for-byte intact beside it. APP_PASSWORD remains the
15
+ bootstrap master; removing password auth is a separate, owner-approved step.
16
+
17
+ ENV (placeholders β€” see .env.example additions in docs/GOOGLE_LOGIN_TODO.md):
18
+ GOOGLE_OAUTH_CLIENT_ID OAuth client id from Google Cloud console
19
+ GOOGLE_OAUTH_CLIENT_SECRET its secret (Space secret / .env β€” never committed)
20
+ GOOGLE_OAUTH_REDIRECT_URI e.g. https://royal-imports-cfo-os.hf.space/ (must be
21
+ registered VERBATIM in the console; localhost:8501 for dev)
22
+ GOOGLE_LOGIN_ENABLED '1' arms the flow; anything else keeps this a template
23
+ """
24
+ from __future__ import annotations
25
+
26
+ import os
27
+
28
+ AUTH_ENDPOINT = "https://accounts.google.com/o/oauth2/v2/auth"
29
+ TOKEN_ENDPOINT = "https://oauth2.googleapis.com/token"
30
+ JWKS_URI = "https://www.googleapis.com/oauth2/v3/certs"
31
+ SCOPES = "openid email profile"
32
+
33
+
34
+ def configured() -> bool:
35
+ """True when the console credentials exist β€” the LOGIN BUTTON renders iff this is True.
36
+ (Separate from enabled(): a configured-but-disabled deploy shows nothing either, so a
37
+ half-finished setup never paints a dead button.)"""
38
+ return bool(os.environ.get("GOOGLE_OAUTH_CLIENT_ID")) and enabled()
39
+
40
+
41
+ def enabled() -> bool:
42
+ return os.environ.get("GOOGLE_LOGIN_ENABLED", "") == "1"
43
+
44
+
45
+ def auth_url(state: str, code_challenge: str) -> str:
46
+ """The Google consent-screen URL for the authorization-code + PKCE flow.
47
+
48
+ `state` must be an unguessable per-session token the callback VERIFIES (CSRF);
49
+ `code_challenge` is BASE64URL(SHA256(code_verifier)) with the verifier held in the
50
+ session. Template guard: raises until the TODO checklist lands.
51
+ """
52
+ raise NotImplementedError(
53
+ "Google login is a template β€” complete docs/GOOGLE_LOGIN_TODO.md, then implement "
54
+ "auth_url() (urlencode client_id, redirect_uri, response_type=code, scope, state, "
55
+ "code_challenge, code_challenge_method=S256 onto AUTH_ENDPOINT)."
56
+ )
57
+
58
+
59
+ def exchange_code(code: str, code_verifier: str) -> dict:
60
+ """POST the authorization code to TOKEN_ENDPOINT β†’ {'id_token', 'access_token', ...}.
61
+ Template guard: raises until implemented (requests.post with client_id/secret,
62
+ redirect_uri, grant_type=authorization_code, code, code_verifier)."""
63
+ raise NotImplementedError(
64
+ "Google login is a template β€” implement exchange_code() per docs/GOOGLE_LOGIN_TODO.md."
65
+ )
66
+
67
+
68
+ def verify_id_token(id_token: str) -> dict:
69
+ """Validate the JWT against Google's JWKS (signature, iss, aud=client_id, exp) and
70
+ return its claims ({'email', 'email_verified', 'name', ...}). MUST reject
71
+ email_verified=False. Template guard: raises until implemented."""
72
+ raise NotImplementedError(
73
+ "Google login is a template β€” implement verify_id_token() per docs/GOOGLE_LOGIN_TODO.md."
74
+ )
75
+
76
+
77
+ def account_for_claims(claims: dict):
78
+ """Map verified Google claims β†’ the core/users.py account, FAIL-CLOSED.
79
+
80
+ The one function already real: the mapping rule is the design's heart and testable
81
+ today. Returns the users.registry() record whose `email` equals the verified claim
82
+ (case-insensitive), only if the account is active; None otherwise β€” never a new user.
83
+ """
84
+ email = str(claims.get("email") or "").strip().lower()
85
+ if not email or claims.get("email_verified") is not True:
86
+ return None
87
+ from core import users
88
+ for username, u in (users.registry() or {}).items():
89
+ if str(u.get("email") or "").strip().lower() == email and u.get("active", True):
90
+ return users._public(username, u)
91
+ return None
platform/core/context.py ADDED
@@ -0,0 +1,59 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Request context β€” the single object that carries the GLOBAL filters through a page render.
2
+
3
+ Instead of threading date_from / date_to / team_id / ... through every function, a page builds
4
+ one `Ctx` from the sidebar controls and passes it (or its primitives) down. Modules read only
5
+ what they need. Add a new global filter once here and every module can opt into it.
6
+
7
+ The DBA (doing-business-as) brand filter lives here as `team_id`:
8
+ None -> All / consolidated (Fisch + Royal)
9
+ 5 -> Fisch
10
+ 6 -> Royal
11
+ HQ / company-level modules (cash flow, balance sheet, AR/AP, consolidated P&L) ignore brand and
12
+ call `ctx.consolidated()`; operational modules (sales, customers, products, margins) honour it.
13
+ """
14
+ from dataclasses import dataclass, replace
15
+ import core.odoo as O
16
+ import core.periods as P
17
+
18
+ # label -> team_id used by the sidebar Brand (DBA) selector
19
+ BRANDS = {'All': None, 'Fisch': 5, 'Royal': 6}
20
+
21
+
22
+ @dataclass(frozen=True)
23
+ class Ctx:
24
+ date_from: str
25
+ date_to: str
26
+ period_label: str = 'Year to date'
27
+ team_id: int | None = None # None = All DBAs (consolidated); 5 = Fisch; 6 = Royal
28
+ doc_mode: str = 'order' # reserved: 'order' vs 'invoice' recognition basis
29
+
30
+ @property
31
+ def brand(self) -> str:
32
+ return O.TEAM_NAMES.get(self.team_id, 'All') if self.team_id else 'All'
33
+
34
+ @property
35
+ def teams(self) -> list:
36
+ """Team ids in scope (single brand, or both when consolidated)."""
37
+ return [self.team_id] if self.team_id else list(O.TEAM_IDS)
38
+
39
+ @property
40
+ def window(self) -> tuple:
41
+ return self.date_from, self.date_to
42
+
43
+ def consolidated(self) -> 'Ctx':
44
+ """All-DBA copy β€” HQ modules use this so the brand filter never scopes company numbers."""
45
+ return replace(self, team_id=None)
46
+
47
+ def with_period(self, date_from, date_to, label=None) -> 'Ctx':
48
+ return replace(self, date_from=date_from, date_to=date_to,
49
+ period_label=label or self.period_label)
50
+
51
+
52
+ def from_ui(period_label, period_window, brand_label='All', doc_mode='order') -> Ctx:
53
+ df, dt = period_window
54
+ return Ctx(df, dt, period_label, BRANDS.get(brand_label), doc_mode)
55
+
56
+
57
+ def default(t=None) -> Ctx:
58
+ df, dt = P.ytd(t)
59
+ return Ctx(df, dt, 'Year to date', None)
platform/core/grid_events.py ADDED
@@ -0,0 +1,1334 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """core/grid_events.py β€” THE WRITE SEAM, lifted out of Streamlit (EXIT-1a, X1, 2026-07-30).
2
+
3
+ This is the grid's event handler: the code that takes an UNTRUSTED event object from the
4
+ browser component and decides what may be persisted. It was `app.py:_cl_handle_grid_event` /
5
+ `_cl_handle_one` and moved here as a PURE MOVE β€” no behavior change inside a move (MA principle
6
+ 5). Every body below is the app.py body verbatim; the ONLY mechanical rewrites are the
7
+ Streamlit couplings the contract names:
8
+
9
+ st.session_state['_cl_grid_seen_ids'] -> ctx.seen_ids (adapter-owned dedup state)
10
+ st.session_state['_cl_doc_payload'] -> ctx.out.doc (EventResult.doc)
11
+ st.session_state['_cohort_toast'] -> ctx.out.toast (EventResult.toast)
12
+ st.session_state['cl_table_workspace'] -> ctx.fallback_ws (store-unavailable fallback)
13
+ is_admin() (the bare call inside doc_delete, and _cl_table_workspace's admin=None default)
14
+ -> ctx.admin
15
+
16
+ There is NO `import streamlit` in this file and there must never be one β€” `verify_seam.py`
17
+ enforces that with a `sys.meta_path` block, which is the only proof that survives refactoring.
18
+
19
+ TWO ADAPTERS, ONE HANDLER:
20
+ * the Streamlit adapter β€” app.py keeps `_cl_handle_grid_event(event, uname, allowed_pids, …)`
21
+ with its old name and signature, builds an EventCtx whose `seen_ids` and `fallback_ws` are
22
+ session-state-backed dicts, and applies the Result to session state.
23
+ * the HTTP adapter β€” `aios-web/api` builds an EventCtx with `fallback_ws=None`. A None
24
+ fallback with the store unavailable is a **503**, never a silent in-memory workspace: an
25
+ API request has no durable session dict to fall back to, so pretending to persist would
26
+ lose the write with a 200 on it.
27
+
28
+ ⚠ LAYERING, stated honestly. `core/` is not supposed to import `modules/` (ARCHITECTURE §1's
29
+ one-directional rule). This handler genuinely straddles: it is the WRITE half of the customer
30
+ table and its persistence verbs live in `modules.customer_data` / `modules.cohort`. The
31
+ contracted path for the strangler is `core/grid_events.py` (both adapters import it from here),
32
+ so the import stays and the violation is RECORDED rather than hidden. It is the one file in
33
+ `core/` that points up; when EXIT-5 ports the surfaces, the persistence verbs are what move
34
+ down, and this note is the marker for that work.
35
+ """
36
+ import datetime as dt
37
+ from dataclasses import dataclass, field
38
+ from typing import MutableMapping, Optional
39
+
40
+ import core.store as store
41
+ import core.table_store as _tstore # wave-9 I17: the shared-view authorisation predicates
42
+ import core.users as users
43
+ import harness.telemetry as _tel # _tel.error() = THE error sink (all caught errors log)
44
+ import modules.cohort as cohort_mod
45
+ import modules.customer_data as cl_mod
46
+
47
+ #: C5 (owner item 12): customer documents. The cap is enforced HOST-side on the DECODED byte
48
+ #: count β€” the client checks the same number first, but a client check only spares the user a
49
+ #: pointless upload; this is what bounds what the tenant's dataset can be made to hold.
50
+ #: MOVED from app.py:1410–1412 with the handler (app.py re-imports these during the strangler).
51
+ _DOCS_KEY = 'customer_docs'
52
+ _DOC_MAX_BYTES = 5 * 1024 * 1024
53
+ _DOC_MAX_PER_ROW = 50
54
+
55
+ #: The props of a field def the CLIENT actually draws β€” the wave-6 no-blip comparison set.
56
+ #: Host-only stamps (createdBy, permissions) are deliberately absent: the client renders
57
+ #: nothing from its own copy of those, and a permissions refusal is answered explicitly.
58
+ _FIELD_ECHO_PROPS = ('label', 'type', 'note', 'options', 'colorCodeOptions',
59
+ 'optionColors', 'max', 'formula', 'agg', 'scope', 'format', 'measure')
60
+
61
+
62
+ class StoreUnavailable(RuntimeError):
63
+ """The store is down AND the caller offered no durable fallback (HTTP adapter β†’ 503).
64
+
65
+ The Streamlit adapter never sees this: it passes a session-state-backed `fallback_ws`, so
66
+ its store-down path degrades to the same captioned in-memory workspace it always had. An
67
+ API caller has no such dict, and the honest answer to "persist this" with nowhere to
68
+ persist it is an error β€” not a 200 over a write that evaporates.
69
+ """
70
+
71
+
72
+ @dataclass
73
+ class EventResult:
74
+ """What the HANDLER needs to tell its adapter β€” the three things it cannot do itself.
75
+
76
+ `rerender` is the old return value with its old meaning: "this page must re-render from
77
+ server state", NOT "handled" (see `handle_one`'s docstring β€” the distinction is the
78
+ difference between a snappy grid and a sluggish one).
79
+ """
80
+ rerender: bool = False
81
+ doc: Optional[dict] = None # doc_fetch's answer β€” the bridge has no response channel
82
+ toast: Optional[str] = None # add_to_list's confirmation
83
+
84
+
85
+ @dataclass
86
+ class EventCtx:
87
+ """Everything the handler is allowed to know, and nothing about how it is being served.
88
+
89
+ ⚠ `allowed_pids` is a SECURITY parameter: it is the pool this caller may touch at all, and
90
+ every row-scoped event (overlay_patch, doc_*, cohort membership, add_to_list, memberPids)
91
+ is intersected with it. `admin` and `uname` are likewise the HOST's statement of identity β€”
92
+ never anything the browser said about itself.
93
+
94
+ `seen_ids` is ADAPTER-OWNED dedup state (the event log resends its recent window on every
95
+ emit, so consumed ids must stay consumed): Streamlit passes a session-state dict, HTTP
96
+ passes the caller's per-session mapping.
97
+
98
+ `fallback_ws` is the store-unavailable workspace home, the OUTER dict keyed by username
99
+ (exactly what `st.session_state.setdefault('cl_table_workspace', {})` is). HTTP passes
100
+ None, and None + store down raises StoreUnavailable (β†’ 503).
101
+
102
+ `out` is the sink for the two parked answers, so `handle_one` can keep returning a bare
103
+ bool exactly as it did in app.py while doc/toast still escape. (Additive amendment to X1's
104
+ field list, 2026-07-30 β€” defaulted; recorded in the S1 mailbox.)
105
+ """
106
+ uname: str
107
+ allowed_pids: frozenset
108
+ fields: list
109
+ measure_keys: frozenset = frozenset()
110
+ resolved_ids: frozenset = frozenset()
111
+ cohort_ids: frozenset = frozenset()
112
+ measure_offer: tuple = ()
113
+ visible_views: tuple = ()
114
+ admin: bool = False
115
+ #: β›” WAVE 15 C-PERM β€” the fields this caller's PERMANENT PERMISSIONS hide, transitive
116
+ #: closure included (`core.perm_scope.hidden_keys`). A SECURITY parameter in the same class
117
+ #: as `allowed_pids`, and here for the same reason: the read paths strip a hidden field from
118
+ #: both wires, so a caller cannot SEE it β€” but without this they could still WRITE it by
119
+ #: naming the key, because `overlay_keys` only asks whether the column exists. Empty means
120
+ #: "nothing hidden", which is every legacy record and therefore every caller today.
121
+ hidden_keys: frozenset = frozenset()
122
+ scope_key: str = 'customer'
123
+ #: Wave 16 C-TOPIC β€” the TABLE OPS this ctx reads and writes (a `core.table_store.make`
124
+ #: object). None = the CUSTOMER table (`modules.customer_data.TABLE_OPS`), which keeps
125
+ #: every existing caller byte-identical. The PRODUCT surface passes
126
+ #: `modules.product_data.TABLE_OPS` so its views/fields/overlays land in the product
127
+ #: bucket β€” never the customer one, whose pids are a different id space entirely.
128
+ table: Optional[object] = None
129
+ seen_ids: Optional[MutableMapping] = None
130
+ fallback_ws: Optional[MutableMapping] = None
131
+ out: EventResult = field(default_factory=EventResult)
132
+
133
+ def __post_init__(self):
134
+ # A caller that forgets `seen_ids` gets per-call dedup rather than a crash β€” the SET is
135
+ # what makes re-delivery idempotent, and its absence must not be the thing that decides
136
+ # whether an event is processed at all.
137
+ if self.seen_ids is None:
138
+ self.seen_ids = {}
139
+
140
+
141
+ def _tops(ctx):
142
+ """The table-ops object this ctx operates on β€” the customer table unless the caller chose
143
+ another topic. ONE resolution point so 'which bucket does this write land in' has exactly
144
+ one answer per event."""
145
+ return ctx.table if ctx.table is not None else cl_mod.TABLE_OPS
146
+
147
+
148
+ # ------------------------------------------------------------------ the workspace family (X1 amd.)
149
+ def table_workspace(ctx, allowed_pids=None, consume_corrections=True):
150
+ """Durable Airtable-style view/schema/overlay state, with an honest session fallback.
151
+
152
+ Wave-9 I17: the returned `views` are this user's OWN views MERGED with every view SHARED
153
+ with them. Own views win a same-id collision β€” a personal view is the user's own object and
154
+ must never be shadowed by somebody else's share.
155
+
156
+ ⚠ `allowed_pids` is a SECURITY parameter, not a convenience. A shared view carries
157
+ `memberPids`, which the writer's scope validated on the way IN β€” but the READER may have a
158
+ narrower scope. Handing a Fisch-scoped user a view whose member list was built by a
159
+ full-access user would leak the other BU's customers straight through the strict-isolation
160
+ rule. So membership is re-filtered against the READER on the way OUT. Pass it wherever the
161
+ result reaches a renderer or an event handler; omitting it drops memberPids entirely rather
162
+ than trusting the stored list (fail-closed: a smaller view, never a wider one).
163
+
164
+ MOVED from app.py:5323 (`_cl_table_workspace`). `admin` was a parameter defaulting to a live
165
+ `is_admin()` call; it now always comes from `ctx.admin`, which is identical at the one call
166
+ site because app.py computes `admin = bool(is_admin())` once per render. `allowed_pids`
167
+ stays an EXPLICIT argument because the folder leg deliberately omits it.
168
+ """
169
+ uname = ctx.uname
170
+ if store.available():
171
+ ws = _tops(ctx).workspace(
172
+ uname, consume_corrections=bool(consume_corrections))
173
+ try:
174
+ shared = _tops(ctx).shared_views(uname, bool(ctx.admin))
175
+ except Exception as _se:
176
+ _tel.error('sharedviews:read', _se)
177
+ shared = {}
178
+ if shared:
179
+ merged = dict(shared)
180
+ merged.update(ws.get('views') or {}) # own views win the id collision
181
+ ws['views'] = {vid: scope_view(v, uname, allowed_pids)
182
+ for vid, v in merged.items()}
183
+ return ws
184
+ # Store down. Streamlit degrades to its captioned session workspace exactly as before; an
185
+ # API caller has no durable dict and must be told, not quietly served memory.
186
+ if ctx.fallback_ws is None:
187
+ raise StoreUnavailable('table workspace unavailable: no store and no session fallback')
188
+ return ctx.fallback_ws.setdefault(
189
+ uname, {'views': {}, 'fields': {}, 'overlays': {}})
190
+
191
+
192
+ def scope_view(view, uname, allowed_pids):
193
+ """Re-scope a view to the READER. A no-op for a view the reader created.
194
+
195
+ Two things travel inside a shared view that are SCOPE, not presentation:
196
+ 1. `memberPids` β€” a customer list. Re-filtered against the reader's permitted pool.
197
+ 2. cohort conditions β€” a cohort id is a permission handle owned by ONE user. For a reader
198
+ who does not own it the leaf is UNANSWERABLE, and partial evaluation of an unanswerable
199
+ leaf WIDENS `is none of` ([[cg-cohort-set-conditions]]) β€” i.e. it would show MORE rows,
200
+ not fewer. So the whole filter tree is dropped for a foreign reader rather than
201
+ evaluated with a hole in it. A visibly unfiltered shared view is recoverable; a
202
+ silently widened one is a leak.
203
+ """
204
+ if not isinstance(view, dict) or view.get('createdBy') == uname:
205
+ return view
206
+ out = dict(view)
207
+ cfg = dict(out.get('config') or {})
208
+ mp = cfg.get('memberPids')
209
+ if isinstance(mp, list):
210
+ cfg['memberPids'] = ([p for p in mp if p in (allowed_pids or ())]
211
+ if allowed_pids is not None else [])
212
+ if tree_has_cohort(cfg.get('filters')):
213
+ cfg['filters'] = []
214
+ cfg['_scopedOut'] = 'cohort' # the client says so rather than pretending
215
+ out['config'] = cfg
216
+ return out
217
+
218
+
219
+ def tree_has_cohort(node):
220
+ """True when a filter tree references the Cohorts column anywhere (any depth)."""
221
+ import aios_grid as _agc
222
+ col = _agc.COHORT_COLUMN
223
+ if isinstance(node, list):
224
+ return any(tree_has_cohort(n) for n in node)
225
+ if not isinstance(node, dict):
226
+ return False
227
+ if node.get('colId') == col or node.get('field') == col:
228
+ return True
229
+ return tree_has_cohort(node.get('conditions') or node.get('children'))
230
+
231
+
232
+ def measure_leaves(nodes):
233
+ """Every leaf of a filter tree that carries a date WINDOW β€” i.e. every measure condition.
234
+
235
+ Keyed off the window rather than the column name on purpose: a window is the one thing no
236
+ column condition has, so this cannot mistake `revenue_ytd > 5000` for a measure.
237
+ """
238
+ out = []
239
+ for node in nodes or []:
240
+ if not isinstance(node, dict):
241
+ continue
242
+ if isinstance(node.get('children'), list):
243
+ out.extend(measure_leaves(node['children']))
244
+ elif node.get('window') is not None:
245
+ out.append(node)
246
+ return out
247
+
248
+
249
+ # ------------------------------------------------------------------ private handler helpers
250
+ def _field_echo_equal(accepted, raw):
251
+ """Does the ACCEPTED def match what the client already renders? (wave-6 item 3)
252
+
253
+ The no-blip law: a field_upsert whose acceptance changed NOTHING the client draws needs no
254
+ re-render β€” the client is the ORIGIN of the change, its optimistic copy is already right,
255
+ and the rerun it would buy costs a full payload re-serialisation plus an iframe remount
256
+ (the "blip" the owner named). STRICT per-prop equality after collapsing the empty forms
257
+ ('' / [] / {} / absent all render identically): any normalisation a validator applied is a
258
+ real difference the client must be SHOWN. Fail-closed β€” unsure means not equal means rerun.
259
+ """
260
+ empty = ('', [], {})
261
+ for k in _FIELD_ECHO_PROPS:
262
+ a, b = accepted.get(k), raw.get(k) if isinstance(raw, dict) else None
263
+ if k == 'scope':
264
+ # 'global' and absent are the SAME rendering (an unscoped field) β€” the event may
265
+ # say 'global' explicitly while the stored def says it by omission.
266
+ a = None if a == 'global' else a
267
+ b = None if b == 'global' else b
268
+ if (None if a in empty else a) != (None if b in empty else b):
269
+ return False
270
+ return True
271
+
272
+
273
+ def _unique_name(wanted, existing, *, fallback='Untitled', max_len=120):
274
+ """Return a durable, case-insensitively unique field/view display name."""
275
+ limit = max(1, int(max_len))
276
+
277
+ def _clean(value):
278
+ return ' '.join(str(value or '').split())
279
+
280
+ base = (_clean(wanted) or _clean(fallback) or 'Untitled')[:limit].rstrip()
281
+ taken = {_clean(value).casefold() for value in existing if _clean(value)}
282
+ if base.casefold() not in taken:
283
+ return base
284
+ index = 2
285
+ while True:
286
+ suffix = f' {index}'
287
+ stem = base[:max(0, limit - len(suffix))].rstrip()
288
+ candidate = f'{stem}{suffix}' if stem else str(index)[-limit:]
289
+ if candidate.casefold() not in taken:
290
+ return candidate
291
+ index += 1
292
+
293
+
294
+ def _known_usernames():
295
+ """Real account names, for C4's fail-closed `users` grant. None = "could not resolve".
296
+
297
+ None means DO NOT FILTER, deliberately. Returning an empty set on a transient registry
298
+ error would silently collapse a user's stored 'specific users' grant to 'personal' on the
299
+ very next autosave β€” destroying a setting because a read blipped. The grant is inert today
300
+ anyway (views are per-user; nothing reads it to allow cross-user access), so the honest
301
+ trade is: never invent access, never destroy a stored intent either.
302
+ """
303
+ try:
304
+ reg = set(users.registry() or {})
305
+ except Exception as _ue:
306
+ _tel.error('viewperms:users', _ue)
307
+ return None
308
+ # ⚠ EMPTY IS "UNRESOLVED", NOT "NOBODY EXISTS". registry() returns {} rather than raising
309
+ # when the user store cannot be read (a store blip logs `store:get:users` and hands back an
310
+ # empty dict) β€” and a working app ALWAYS has at least the bootstrap admin. Treating {} as a
311
+ # real answer would filter every name out of a stored grant and collapse it to 'personal'
312
+ # on the next autosave. Caught by testing this in BARE mode with no store token.
313
+ return reg or None
314
+
315
+
316
+ # ------------------------------------------------------------------ THE SEAM
317
+ def handle_events(events, ctx):
318
+ """The event LOG entry point β†’ one EventResult. The `[-24:]` window lives here.
319
+
320
+ ⚠ THE VALUE IS AN EVENT LOG `{'events': [...]}`, NOT ONE EVENT (2026-07-27). A component
321
+ has exactly one value slot, and two quick emits share it: creating a field emits
322
+ `field_upsert` and the insertColumn autosave emits `view_upsert` 420ms later β€” with the
323
+ server still inside the first rerun, the second write clobbered the slot before any run
324
+ read it, and the field creation VANISHED with no error anywhere (measured live; the same
325
+ race sat latent under overlay-field creation, masked by localStorage). The client now
326
+ sends its recent window of events; each is processed exactly once, keyed by its id. The
327
+ single-event shape is still accepted β€” it is what an older client in a stale tab sends.
328
+
329
+ Accepts everything the component value can be: the log dict, a bare list of events (what
330
+ the HTTP body's `events` array unwraps to), or one legacy single-event object.
331
+
332
+ `EventResult.rerender` is True when ANY processed event demands a re-render from server
333
+ state β€” the same OR-fold `_cl_handle_grid_event` did.
334
+ """
335
+ if isinstance(events, dict) and isinstance(events.get('events'), list):
336
+ seq = events['events'][-24:]
337
+ elif isinstance(events, list):
338
+ seq = events[-24:]
339
+ else:
340
+ seq = [events]
341
+ rerun = False
342
+ for one in seq:
343
+ rerun = handle_one(one, ctx) or rerun
344
+ ctx.out.rerender = rerun
345
+ return ctx.out
346
+
347
+
348
+ def handle_one(event, ctx):
349
+ """Validate ONE untrusted component event.
350
+
351
+ Odoo-source fields are never accepted as cell updates. View and schema objects are
352
+ user-scoped; row patches additionally require the pid to be in the currently permitted
353
+ Customer List pool.
354
+
355
+ RETURN VALUE = "this page must re-render from server state", NOT "handled".
356
+ The distinction is the difference between a snappy grid and a sluggish one.
357
+ Persisting is not by itself a reason to rerun: Streamlit has ALREADY re-run the
358
+ script once simply to deliver the component value, and the client is the ORIGIN
359
+ of the change, so its own state is correct before the server hears about it. An
360
+ extra st.rerun() therefore buys nothing and costs a second full page run.
361
+ Measured on the 1,550-row book (2026-07-25): one filter tweak produced TWO
362
+ page_customer_data runs ~2.8s apart, each re-serialising the whole ~576 KB row
363
+ payload β€” and the handler on the second run is a guaranteed no-op, because the
364
+ event-id dedupe two lines below has already consumed that id.
365
+
366
+ THE WAVE-6 NO-BLIP LAW (item 3, 2026-07-27) β€” True ONLY when the next render
367
+ genuinely differs from what the client already draws:
368
+ field_upsert β€” False when the accepted def ECHOES the client's copy
369
+ (_field_echo_equal); True on any validator mutation, any
370
+ permissions refusal, and ALWAYS when a measure window changed
371
+ (values must be re-resolved server-side).
372
+ overlay_patch β€” False when every requested update was accepted verbatim;
373
+ True the moment anything was refused or truncated.
374
+ field_delete β€” False: the client removed the column optimistically and the
375
+ store scrub is invisible to the next render.
376
+ field_duplicate β€” True only when host-held VALUES must flow back (custom_ cell
377
+ copies, measure_ recomputes); formula/created_time clones
378
+ compute client-side.
379
+ view_delete β€” False (the deleted view simply stops being served).
380
+ view_upsert β€” unchanged: False, except an unresolved measure condition.
381
+ cohort_* / add_to_list β€” True: the lists payload is server truth, and these are
382
+ not the hot path.
383
+
384
+ The id dedupe is a SET (not a last-id slot): the event log resends the whole recent
385
+ window on every emit, so every id must stay consumed, not just the latest one.
386
+ """
387
+ # The ctx fields the moved body reads as plain locals β€” bound once here so every branch
388
+ # below is textually the app.py body (which is what makes this move reviewable).
389
+ uname = ctx.uname
390
+ allowed_pids = ctx.allowed_pids
391
+ fields = ctx.fields
392
+ measure_keys = ctx.measure_keys
393
+ resolved_ids = ctx.resolved_ids
394
+ cohort_ids = ctx.cohort_ids
395
+ measure_offer = ctx.measure_offer
396
+ visible_views = ctx.visible_views
397
+ admin = ctx.admin
398
+ scope_key = ctx.scope_key
399
+
400
+ if not isinstance(event, dict):
401
+ return False
402
+ event_id = str(event.get('id') or '')[:180]
403
+ seen = ctx.seen_ids
404
+ if not event_id or event_id in seen:
405
+ return False
406
+ seen[event_id] = True
407
+ if len(seen) > 400: # a session-lifetime set, kept bounded
408
+ for stale in list(seen)[:200]:
409
+ seen.pop(stale, None)
410
+ kind = event.get('type')
411
+ field_by_key = {f['key']: f for f in fields}
412
+ valid_keys = set(field_by_key)
413
+ overlay_keys = {f['key'] for f in fields if f.get('source') == 'overlay'}
414
+ # Event validation is not a render payload. It must not consume a one-shot label
415
+ # correction before the browser has had a chance to receive it.
416
+ ws = table_workspace(
417
+ ctx, allowed_pids=allowed_pids, consume_corrections=False)
418
+
419
+ def _session_ready():
420
+ ws.setdefault('views', {})
421
+ ws.setdefault('fields', {})
422
+ ws.setdefault('overlays', {})
423
+
424
+ if kind == 'view_select':
425
+ # Owner item 3 (2026-07-31): remember WHERE THE USER IS so a fresh browser resumes
426
+ # there instead of the system default view. Presentation state only β€” the read side
427
+ # re-validates the id against what the caller may see, so nothing here needs an
428
+ # authorisation wall beyond the length cap. Never a re-render: the client is already
429
+ # on that view.
430
+ vid = str(event.get('viewId') or '').strip()[:120]
431
+ if vid and store.available():
432
+ try:
433
+ _tops(ctx).save_active_view(uname, vid)
434
+ except Exception as _ae:
435
+ _tel.error('grid:view-select', _ae)
436
+ return False
437
+
438
+ if kind == 'record_layout':
439
+ # Wave 2026-08-02 (C-LAYOUT): the per-user record-detail field order. Presentation
440
+ # state for one surface; grid column order is untouched. Unknown keys are pruned
441
+ # against the LIVE field set, dupes collapse, and the wire re-validates at serve
442
+ # time β€” so a client racing a field delete degrades to default order, never an error.
443
+ raw_order = event.get('order')
444
+ if not isinstance(raw_order, list):
445
+ return False
446
+ order, seen = [], set()
447
+ for k in raw_order[:200]:
448
+ k = str(k or '').strip()[:80]
449
+ if k and k in valid_keys and k not in seen:
450
+ seen.add(k)
451
+ order.append(k)
452
+ if store.available():
453
+ _tops(ctx).save_record_layout(uname, order)
454
+ else:
455
+ _session_ready()
456
+ if order:
457
+ ws['recordLayout'] = {'order': order}
458
+ else:
459
+ ws.pop('recordLayout', None)
460
+ # Autosave hot path, like view_select: the client already renders the order it asked
461
+ # for, and the wire echoes it back on the next workspace read.
462
+ return False
463
+
464
+ if kind == 'view_upsert':
465
+ raw = event.get('view')
466
+ if not isinstance(raw, dict) or not isinstance(raw.get('config'), dict):
467
+ return False
468
+ view_id = str(raw.get('id') or '').strip()[:120]
469
+ name = str(raw.get('name') or '').strip()[:120]
470
+ requested_name = name
471
+ if not view_id or not name:
472
+ return False
473
+ cfg = raw['config']
474
+ # The filter TREE (leaf conditions and/or nested condition groups) is
475
+ # validated by the module-agnostic grid contract helper β€” imported lazily
476
+ # here for the same reason the page does it: a missing embed build must
477
+ # never break page import.
478
+ import aios_grid as _ag
479
+ # A MEASURE key is valid in a FILTER and nowhere else (CG-8): it may not be ordered,
480
+ # shown, grouped or coloured by, because it is not a column of this table. Hence the
481
+ # widened key set here and the plain `valid_keys` everywhere below.
482
+ # `cohort_ids` is a PERMISSION list, not a convenience: a cohort rule naming anything
483
+ # else is dropped here. Leaving it in would persist a condition the engine can only
484
+ # answer with "nothing", so `Cohort is not [a list I deleted]` would show an empty table
485
+ # forever with nothing on screen explaining why.
486
+ filters = _ag.clean_filter_tree(cfg.get('filters'),
487
+ valid_keys | set(measure_keys),
488
+ cohort_ids=set(cohort_ids))
489
+ sorts = []
490
+ for rule in list(cfg.get('sorts') or [])[:20]:
491
+ if (isinstance(rule, dict) and rule.get('colId') in valid_keys
492
+ and rule.get('dir') in ('asc', 'desc')):
493
+ sorts.append({'colId': rule['colId'], 'dir': rule['dir']})
494
+ order = list(dict.fromkeys(
495
+ key for key in list(cfg.get('order') or []) if key in valid_keys))
496
+ order += [field['key'] for field in fields if field['key'] not in order]
497
+ visible = list(dict.fromkeys(
498
+ key for key in list(cfg.get('visible') or []) if key in valid_keys))
499
+ widths = {}
500
+ raw_widths = cfg.get('widths') if isinstance(cfg.get('widths'), dict) else {}
501
+ for key, value in raw_widths.items():
502
+ if key in valid_keys:
503
+ try:
504
+ widths[key] = max(60, min(600, round(float(value))))
505
+ except (TypeError, ValueError):
506
+ pass
507
+ # C-LOCK (wave 2026-08-02, item 12's mechanism): a view may be LOCKED to one cohort.
508
+ # `cohort_ids` is the same permission list the cohort filter leaf uses β€” a lock naming
509
+ # a cohort this session cannot see is DROPPED at write, exactly like a cohort rule.
510
+ # (Readers of a SHARED locked view resolve the id against their OWN visible sets; an
511
+ # unresolvable lock matches NOTHING client-side β€” the cohort-leaf law, never a widen.)
512
+ cohort_lock = str(cfg.get('cohortLock') or '').strip()[:120]
513
+ # β›” WAVE 17 R1 (C-LOCKV guard a) β€” A PROJECTED LOCKED VIEW RE-STAMPS ITS OWN LOCK.
514
+ # A locked view IS its set: its id and its lock are the same fact. The client rebuilds
515
+ # `config` from scratch on every autosave (a column resize is enough), so taking the
516
+ # lock from the browser would mean one omitted key turns "the 40 accounts we agreed to
517
+ # call" into the whole book, under the same name, with nothing going red. The read side
518
+ # re-stamps too (aios_grid.views_from_defs) β€” this keeps the STORE from accumulating the
519
+ # wrong record in the first place.
520
+ _is_locked_view = view_id in set(cohort_ids)
521
+ if _is_locked_view:
522
+ cohort_lock = view_id
523
+ config = {
524
+ 'filters': filters,
525
+ # conjunction joining the ROOT-level conditions; anything but an
526
+ # explicit 'or' means 'and' (legacy views had an implicit AND).
527
+ 'filterConj': 'or' if cfg.get('filterConj') == 'or' else 'and',
528
+ **({'cohortLock': cohort_lock}
529
+ if cohort_lock and cohort_lock in set(cohort_ids) else {}),
530
+ 'sorts': sorts,
531
+ 'groupBy': cfg.get('groupBy') if cfg.get('groupBy') in valid_keys else None,
532
+ 'colorBy': cfg.get('colorBy') if cfg.get('colorBy') in valid_keys else None,
533
+ 'rowHeightMode': (cfg.get('rowHeightMode')
534
+ if cfg.get('rowHeightMode') in ('short', 'medium', 'tall')
535
+ else 'short'),
536
+ 'order': order,
537
+ 'visible': visible,
538
+ 'widths': widths,
539
+ 'memberPids': [pid for pid in list(cfg.get('memberPids') or [])
540
+ if isinstance(pid, int) and pid in allowed_pids],
541
+ }
542
+ # Wave-6 item 10: a non-grid display mode rides the view config, validated
543
+ # structurally (mode whitelist, refs must be this table's fields).
544
+ display = _ag._clean_display(cfg.get('display'), valid_keys)
545
+ if display:
546
+ config['display'] = display
547
+ # Wave-6 item 11 (pin): how many leading columns stay frozen while scrolling
548
+ # sideways. Clamped 1..8 β€” 0 would unfreeze the identity column the whole
549
+ # surface pivots on; absent = the legacy single frozen column.
550
+ if cfg.get('frozenCount') is not None:
551
+ try:
552
+ config['frozenCount'] = max(1, min(8, int(cfg['frozenCount'])))
553
+ except (TypeError, ValueError):
554
+ pass
555
+ # ── wave-9 C3 (lock) + C4 (permissions) ────────────────────────────────────────────
556
+ # The PRIOR stored view decides both: whether this is a create or an update (C4's
557
+ # default splits on exactly that) and who is allowed to change the lock (C3).
558
+ prior = (ws.get('views') or {}).get(view_id) or {}
559
+ # ⚠ THE SHARED WALL (wave-9 I17). `ws` is already visibility-filtered, so a view the
560
+ # caller may not SEE reads as absent β€” which would look like a CREATE and let anybody
561
+ # overwrite somebody else's shared view by guessing its id. Ask the store for the raw
562
+ # record and refuse before deciding anything else. Fail-closed on both branches.
563
+ _raw_shared = _tops(ctx).shared_view(view_id) if store.available() else None
564
+ if _raw_shared is not None and not prior:
565
+ return False # exists, not visible to this caller: refuse
566
+ if _raw_shared is not None and not _tstore._may_edit(_raw_shared, uname, admin):
567
+ return False # visible but not editable by this caller
568
+ is_new = not prior
569
+ # C4: the creator is HOST-STAMPED and never read from the browser β€” the same rule as
570
+ # the field-level createdBy. An existing view keeps whoever made it.
571
+ creator = prior.get('createdBy') or uname
572
+ # Only the creator or an admin may change the PERMISSIONS themselves. A collaborator who
573
+ # could rewrite them could grant themselves sole ownership of someone else's view, or
574
+ # quietly widen a users-scoped view to everyone β€” privilege escalation by edit.
575
+ if not is_new and not _tstore._may_administer(prior, uname, admin):
576
+ raw = dict(raw)
577
+ raw['permissions'] = prior.get('permissions')
578
+ raw['locked'] = prior.get('locked')
579
+ # ── I12/C3 BELT (2026-07-31): a LOCKED view's DISPLAY MODE is frozen HOST-SIDE too.
580
+ # The client's switcher already refuses (setDisplayMode), and its comment has claimed
581
+ # "the host refuses too" since wave 9 β€” this makes that claim true for direct API
582
+ # callers. ⚠ MIRRORS types.ts `isModeFrozen` EXACTLY: SYSTEM views and the undeletable
583
+ # list:/cohort: projections are EXEMPT β€” their `locked` is legacy "undeletable", not
584
+ # mode-frozen, and freezing them would trap the default view in whatever mode it last
585
+ # saved (the owner's stuck-in-Map report is that trap, client-made). An upsert that
586
+ # UNLOCKS in the same write (authorised above) may change display freely.
587
+ _mode_frozen = (prior.get('locked') and raw.get('locked') is not False
588
+ and prior.get('kind') != 'system'
589
+ and view_id != 'all-customers')
590
+ if _mode_frozen:
591
+ if prior.get('config', {}).get('display'):
592
+ config['display'] = dict(prior['config']['display'])
593
+ else:
594
+ config.pop('display', None)
595
+ # ⚠ A SYSTEM VIEW IS PER-USER STATE AND MUST NEVER BE SHARED. Caught in the wave-9
596
+ # integration pass, before deploy: `all-customers` is every user's DEFAULT view and it
597
+ # carries their own column widths, visible fields and row height. C4's read-default
598
+ # ('collaborative' when permissions are absent, so nobody's saved view is
599
+ # retro-restricted) is right for a view somebody AUTHORED β€” but applied to the system
600
+ # view it routed `all-customers` into the shared bucket, and the first user to resize a
601
+ # column would have published their layout to everyone. Verified in the store:
602
+ # shared_table_views() was returning ['all-customers'].
603
+ # System views are pinned personal, so they stay in each user's own workspace exactly as
604
+ # before this wave. `kind` is host-validated two lines below; read it here from `raw` and
605
+ # treat the stored kind as authoritative for an existing view.
606
+ _kind = (prior.get('kind') if not is_new
607
+ else (raw.get('kind') if raw.get('kind') in ('system', 'list', 'custom')
608
+ else 'custom'))
609
+ if _kind == 'system' or view_id == 'all-customers':
610
+ perms = {'edit': 'personal'}
611
+ else:
612
+ perms = _ag.clean_view_permissions(
613
+ raw.get('permissions'),
614
+ default='personal' if is_new else 'collaborative',
615
+ known_users=_known_usernames())
616
+ # C3: only the creator or an admin may SET or CLEAR the lock; anyone else keeps whatever
617
+ # is already stored. Enforced here rather than hidden in the client. This composes with
618
+ # C4 and does not override it: C4 governs who may edit the CONTENTS, C3 who may freeze
619
+ # the display MODE.
620
+ # ⚠ Inert in practice TODAY β€” a user can only ever write their own per-user workspace,
621
+ # so `uname == creator` always holds. Correct plumbing for shared views, not a live gate.
622
+ may_lock = admin or uname == creator
623
+ locked = bool(raw.get('locked')) if may_lock else bool(prior.get('locked'))
624
+ # C3: the LOCK's actual job β€” a locked view's display mode is frozen, so a locked Kanban
625
+ # stays Kanban. This half IS live per-user and is the part the owner asked for. Filters,
626
+ # sorts and columns stay editable by design.
627
+ if _mode_frozen:
628
+ was = (prior.get('config') or {}).get('display')
629
+ now = config.get('display')
630
+ if (was or {}).get('mode') != (now or {}).get('mode'):
631
+ if was:
632
+ config['display'] = was
633
+ else:
634
+ config.pop('display', None)
635
+ # WAVE 17 R1 (C-LOCKV guard b) β€” ONE THING, ONE NAME. Renaming a locked view renames
636
+ # the SET it is, in the store that owns it; the view record must not grow a second title
637
+ # that the read side would then have to arbitrate. Routed rather than duplicated.
638
+ if _is_locked_view and store.available():
639
+ _cur = (cohort_mod.all_for(uname).get(view_id) or {}).get('name')
640
+ if name and name != _cur:
641
+ try:
642
+ cohort_mod.rename(uname, view_id, name)
643
+ except ValueError as _e:
644
+ _tel.error('cohort:rename-via-view', _e)
645
+ view = {'id': view_id, 'name': name,
646
+ # ⚠ DERIVED FROM THE ID, never taken from the browser: `kind` is what paints the
647
+ # lock mark, so a client claiming it for an ordinary view would be claiming a
648
+ # guarantee the engine is not making about that view's rows.
649
+ 'kind': _ag.LOCKED_VIEW_KIND if _is_locked_view
650
+ else raw.get('kind') if raw.get('kind') in ('system', 'list', 'custom')
651
+ else 'custom',
652
+ 'locked': locked,
653
+ 'createdBy': creator,
654
+ 'permissions': perms,
655
+ 'note': str(raw.get('note') or '')[:2000],
656
+ 'config': config}
657
+ # System/list projections are not all persisted in this table store, but custom views
658
+ # may not impersonate their visible names. Persisted view names are allocated
659
+ # tenant-globally by TableStore inside the write transaction.
660
+ _reserved_view_names = [
661
+ value.get('name') for value in visible_views
662
+ if (isinstance(value, dict) and value.get('id') != view_id
663
+ and value.get('kind') in ('system', 'list'))
664
+ ]
665
+ if view_id != 'all-customers':
666
+ _reserved_view_names.append('All customers')
667
+ if store.available():
668
+ view = (_tops(ctx).save_view(
669
+ uname, view, reserved_names=_reserved_view_names, is_admin=admin) or view)
670
+ else:
671
+ _session_ready()
672
+ _view_names = list(_reserved_view_names)
673
+ _view_names.extend(
674
+ value.get('name')
675
+ for candidate_id, value in ws['views'].items()
676
+ if candidate_id != view_id and isinstance(value, dict)
677
+ )
678
+ view['name'] = _unique_name(view.get('name'), _view_names)
679
+ ws['views'][view_id] = view
680
+ name_changed = view.get('name') != requested_name
681
+ # PERSISTED, but deliberately NO rerun β€” see the return-value contract in
682
+ # this function's docstring. This is the hot path: every filter, sort,
683
+ # group, colour, column-resize and column-toggle autosaves through here.
684
+ #
685
+ # CG-8 is the ONE exception, and it is self-limiting. A measure condition is answered
686
+ # server-side, so if this view carries one whose answer was NOT in the payload we just
687
+ # shipped, the client is showing a pending marker with nothing coming. Ask for exactly
688
+ # one more run β€” and only then, so the common case (a column filter) still costs zero.
689
+ # If the top-of-run read of the component's own value already covered it, this is False
690
+ # and nothing extra happens; if that read is ever stale, this is what stops the
691
+ # condition hanging unresolved.
692
+ #
693
+ # ⚠ COMPLETE rules only (wave-6 item 3, found by the live roundtrip on this rev):
694
+ # picking a measure FIELD mints a valueless rule whose autosave used to land here as
695
+ # "unresolved" and trigger a rerun that could resolve NOTHING β€” `rule_complete` refuses
696
+ # half-asked questions β€” so the only effect was a remount that closed the builder
697
+ # popover mid-build (the wave-5 "gate flake" runs, now explained). A rule still being
698
+ # typed stays client-side PENDING by design; the run that matters fires when its VALUE
699
+ # arrives and the question becomes answerable.
700
+ from harness import measure_filter as _mfc
701
+ unresolved = {str(r.get('id') or '') for r in measure_leaves(filters)
702
+ if _mfc.rule_complete(r)} - set(resolved_ids)
703
+ return name_changed or bool(unresolved - {''})
704
+
705
+ if kind == 'view_delete':
706
+ view_id = str(event.get('viewId') or '')[:120]
707
+ if not view_id or view_id == 'all-customers':
708
+ return False
709
+ # WAVE 17 R1 (C-LOCKV guard c) β€” deleting a LOCKED view deletes the set it is. Without
710
+ # this the store would drop the saved config and the projection would rebuild the view
711
+ # on the very next render: the row would come back, minus the user's sort and columns,
712
+ # which reads as a corruption rather than as a refusal. Own sets only (`all_for`), the
713
+ # same wall `cohort_delete` rides.
714
+ if store.available() and view_id in cohort_mod.all_for(uname):
715
+ cohort_mod.delete(uname, view_id)
716
+ _tops(ctx).delete_view(uname, view_id)
717
+ # True, unlike an ordinary view_delete: the lists payload IS server truth and the
718
+ # client cannot know the membership channel changed.
719
+ return True
720
+ # Wave-9 I17: deleting a SHARED view is narrower than editing it β€” creator or admin
721
+ # only. A collaborator can change what a shared view shows; they cannot destroy
722
+ # everybody else's copy of it, because there is only one copy.
723
+ if store.available():
724
+ _raw_shared = _tops(ctx).shared_view(view_id)
725
+ if _raw_shared is not None and not _tstore._may_administer(
726
+ _raw_shared, uname, admin):
727
+ return False
728
+ _tops(ctx).delete_view(uname, view_id)
729
+ else:
730
+ _session_ready()
731
+ ws['views'].pop(view_id, None)
732
+ # Wave-6 item 3: the client removed the view optimistically; a deleted view simply
733
+ # stops being served, so the next render differs in nothing the client still shows.
734
+ return False
735
+
736
+ if kind == 'field_upsert':
737
+ raw = event.get('field')
738
+ if not isinstance(raw, dict):
739
+ return False
740
+ key = str(raw.get('key') or '')[:80]
741
+ import aios_grid as _ag2
742
+ if key.startswith(_ag2.MEASURE_FIELD_PREFIX):
743
+ # A FORMULA-MEASURE column (owner item 7). Validated against THIS caller's measure
744
+ # offer β€” the same admission the condition builder uses β€” so a field can only name a
745
+ # measure this user could also filter by. Fail closed on anything else.
746
+ field = _ag2.clean_measure_field(raw, {m['key']: m for m in measure_offer})
747
+ if field is None:
748
+ return False
749
+ fmt = _ag2._clean_format(raw.get('format'), field.get('type'))
750
+ if fmt:
751
+ field['format'] = fmt
752
+ elif key in field_by_key and not key.startswith('custom_'):
753
+ base = field_by_key[key]
754
+ field = {**base, 'note': str(raw.get('note') or '')[:2000]}
755
+ # Wave-5 item 10: a DISPLAY format on any base field (presets included) β€” how a
756
+ # number/date reads, per user. Rendering-only; validated fail-closed per type.
757
+ fmt = _ag2._clean_format(raw.get('format'), base.get('type'))
758
+ if fmt:
759
+ field['format'] = fmt
760
+ else:
761
+ field.pop('format', None) # cleared -> back to the type's default render
762
+ # PRESET measure fields (wave-2 item 8): the browser may change the PERIOD (and the
763
+ # auto-renamed label) of revenue_ytd / revenue_ly / orders_24m β€” nothing else. The
764
+ # whitelist IS the base contract: only a field that shipped with preset+measure can
765
+ # take a window here, and the measure KEY is pinned to the base's (fail closed).
766
+ if (base.get('preset') and isinstance(base.get('measure'), dict)
767
+ and isinstance(raw.get('measure'), dict)):
768
+ window = _ag2._clean_window(raw['measure'].get('window'))
769
+ if window is not None:
770
+ field['measure'] = {'key': base['measure']['key'], 'window': window}
771
+ if isinstance(raw.get('label'), str) and raw['label'].strip():
772
+ field['label'] = str(raw['label'])[:120]
773
+ elif key.startswith('custom_') and raw.get('type') in _ag2.CUSTOM_FIELD_TYPES:
774
+ ftype = raw['type']
775
+ readonly = ftype in _ag2.READONLY_CUSTOM_TYPES
776
+ # The editable family must SAY it is overlay (the client's own contract); the
777
+ # read-only pair (formula / created_time) is stored under the cohort column's
778
+ # mechanism instead, so no cell write can ever be accepted for it.
779
+ if not readonly and raw.get('source') != 'overlay':
780
+ return False
781
+ field = {'key': key, 'label': str(raw.get('label') or 'Untitled')[:120],
782
+ 'type': ftype, 'source': 'odoo' if readonly else 'overlay',
783
+ 'default': True, 'custom': True,
784
+ 'note': str(raw.get('note') or '')[:2000]}
785
+ if readonly:
786
+ field['derived'] = True
787
+ # FILTERABLE since wave 6 (owner item 6): formula/created_time values live
788
+ # client-side, and this table's counts are client-mode β€” the client engine
789
+ # answers conditions on them soundly. Mirrors fields_from_workspace.
790
+ field['filterable'] = True
791
+ if raw.get('agg') == 'sum' and not readonly:
792
+ field['agg'] = 'sum'
793
+ # A `select`/`multiselect` carries its own vocabulary; a `user` does NOT β€” its
794
+ # choices are the tenant's accounts, resolved fresh on every render, so persisting
795
+ # a snapshot here would go stale the moment somebody joins or leaves.
796
+ if ftype in ('select', 'multiselect'):
797
+ field['options'] = _ag2._clean_options(raw.get('options'))
798
+ if not field['options']:
799
+ return False # a select with no choices can never hold a value
800
+ field.update(_ag2._choice_appearance(raw, field['options']))
801
+ if ftype == 'rating':
802
+ field['max'] = _ag2._clean_rating_max(raw.get('max'))
803
+ if ftype == 'formula':
804
+ # WRITE-time is where refs must exist (fail closed); read-time leaves them
805
+ # alone so a later-deleted ref blanks the CELLS, not the column.
806
+ formula = _ag2._clean_formula(raw.get('formula'), valid_keys)
807
+ if formula is None:
808
+ return False
809
+ field['formula'] = formula
810
+ if ftype == 'automation':
811
+ # C5-AUTOFIELD (wave 18): persist the validated config bag. No `return False`
812
+ # when absent β€” an unconfigured automation column is a legitimate intermediate
813
+ # state (create the column, then configure it in the gear).
814
+ auto = _ag2._clean_automation(raw.get('automation'), valid_keys)
815
+ if auto:
816
+ field['automation'] = auto
817
+ fmt = _ag2._clean_format(raw.get('format'), ftype)
818
+ if fmt:
819
+ field['format'] = fmt
820
+ else:
821
+ return False
822
+ # Wave-5 item 1 β€” createdBy + permissions on the CREATED strata (custom_/measure_).
823
+ # `createdBy` is stamped HOST-SIDE at create and preserved on every later write: the
824
+ # client never supplies it, so a browser cannot claim someone else's field. A
825
+ # permissions CHANGE is accepted only from the creator or an admin; a legacy field
826
+ # with no createdBy is admin-only (fail closed, both directions). The table workspace
827
+ # is per-user today, so this wall becomes load-bearing when sharing arrives β€” but the
828
+ # admin path and the legacy path are enforceable (and NC'd) right now.
829
+ needs_values = False
830
+ refused_perms = False
831
+ if key.startswith('custom_') or key.startswith(_ag2.MEASURE_FIELD_PREFIX):
832
+ prior = ws.get('fields', {}).get(key)
833
+ prior = prior if isinstance(prior, dict) else None
834
+ if prior is None:
835
+ field['createdBy'] = uname
836
+ # Wave-6 item 9: scope is chosen at CREATE, on the cohort page only β€”
837
+ # 'cohort' marks the field cohort-specific; 'global' or absent stays
838
+ # unscoped, so an old bundle that sends no scope keeps today's behavior.
839
+ if scope_key == 'cohort' and raw.get('scope') == 'cohort':
840
+ field['scope'] = 'cohort'
841
+ else:
842
+ if isinstance(prior.get('createdBy'), str) and prior['createdBy'].strip():
843
+ field['createdBy'] = prior['createdBy']
844
+ # Scope is PRESERVED like createdBy: the client copy never moves an
845
+ # existing field between scopes.
846
+ if prior.get('scope') == 'cohort':
847
+ field['scope'] = 'cohort'
848
+ want = _ag2._clean_permissions(raw.get('permissions'))
849
+ have = _ag2._clean_permissions((prior or {}).get('permissions'))
850
+ owner = field.get('createdBy')
851
+ if want is not None and (admin or (owner and owner == uname)):
852
+ field['permissions'] = want
853
+ else:
854
+ if have is not None:
855
+ field['permissions'] = have
856
+ # Asked for a permissions change and did not get it: the client's
857
+ # optimistic copy is a lie that must be repainted (wave-6 item 3).
858
+ refused_perms = want is not None and want != have
859
+ # A NEW measure column, or an existing one whose window moved: the VALUES
860
+ # are host-computed, so the next render genuinely differs.
861
+ if key.startswith(_ag2.MEASURE_FIELD_PREFIX):
862
+ needs_values = prior is None or prior.get('measure') != field.get('measure')
863
+ # Canonical names are reserved outside the durable workspace. Persisted field names are
864
+ # allocated by TableStore from the transaction's current state, not this stale snapshot.
865
+ _workspace_field_keys = set((ws.get('fields') or {}))
866
+ _reserved_field_names = [
867
+ value.get('label') for candidate_key, value in field_by_key.items()
868
+ if (candidate_key != key and candidate_key not in _workspace_field_keys
869
+ and isinstance(value, dict))
870
+ ]
871
+ if store.available():
872
+ field = (_tops(ctx).save_field(
873
+ uname, field, reserved_names=_reserved_field_names,
874
+ correction_id=event_id) or field)
875
+ else:
876
+ _session_ready()
877
+ _field_names = list(_reserved_field_names)
878
+ _field_names.extend(
879
+ value.get('label')
880
+ for candidate_key, value in ws['fields'].items()
881
+ if candidate_key != key and isinstance(value, dict)
882
+ )
883
+ requested_label = ' '.join(
884
+ str(field.get('label') or 'Untitled').split())[:120].rstrip()
885
+ field.pop('labelCorrectedFrom', None)
886
+ field.pop('labelCorrectionId', None)
887
+ field['label'] = _unique_name(requested_label, _field_names)
888
+ if field['label'] != requested_label and event_id:
889
+ field['labelCorrectedFrom'] = requested_label
890
+ field['labelCorrectionId'] = str(event_id)[:180]
891
+ if not field.get('custom') and not str(field.get('note') or '').strip():
892
+ ws['fields'].pop(key, None)
893
+ else:
894
+ ws['fields'][key] = field
895
+ if needs_values or refused_perms:
896
+ return True
897
+ # THE NO-BLIP RETURN (wave-6 item 3): an accepted def that echoes what the client
898
+ # already renders needs no rerun β€” see _field_echo_equal and the docstring's law.
899
+ return not _field_echo_equal(field, raw)
900
+
901
+ if kind in ('cohort_add', 'cohort_remove', 'cohort_rename', 'cohort_delete'):
902
+ # Cohort membership/lifecycle edits from the grid's cohort panel (wave-2 item 2 β€”
903
+ # replaces the retired Manage popover, so rename/delete keep a surface). Fail-closed:
904
+ # only the caller's OWN cohorts (all_for), and member pids are intersected with the
905
+ # caller's permitted pool β€” the same guard add_to_list rides.
906
+ if not store.available():
907
+ return False
908
+ cohort_id = str(event.get('cohortId') or '').strip()[:120]
909
+ if not cohort_id or cohort_id not in cohort_mod.all_for(uname):
910
+ return False
911
+ try:
912
+ if kind == 'cohort_rename':
913
+ name = str(event.get('name') or '').strip()[:120]
914
+ if not name:
915
+ return False
916
+ cohort_mod.rename(uname, cohort_id, name)
917
+ elif kind == 'cohort_delete':
918
+ cohort_mod.delete(uname, cohort_id)
919
+ else:
920
+ raw_pids = event.get('pids')
921
+ if not isinstance(raw_pids, list):
922
+ return False
923
+ pids = [p for p in raw_pids if isinstance(p, int) and p in allowed_pids]
924
+ if not pids:
925
+ return False
926
+ if kind == 'cohort_add':
927
+ cohort_mod.add_members(uname, cohort_id, pids)
928
+ else:
929
+ cohort_mod.remove_members(uname, cohort_id, pids)
930
+ except ValueError as e:
931
+ _tel.error('cohort:panel-edit', e)
932
+ return False
933
+ # True: the lists payload (names, membership, counts) must reflect this on the next
934
+ # render β€” the panel is showing server truth, not local optimism.
935
+ return True
936
+
937
+ if kind in ('doc_add', 'doc_delete', 'doc_fetch'):
938
+ # DOCUMENTS on a customer row (owner item 12, contract C5). Metadata in the store key
939
+ # `customer_docs`; the BYTES live at their own path in the tenant dataset, never in a
940
+ # JSON blob β€” see core.store.upload_bytes for why that distinction is load-bearing.
941
+ import base64 as _b64c
942
+ import re as _re_mod
943
+ if not store.available():
944
+ return False
945
+ pid = event.get('pid')
946
+ doc_id = str(event.get('docId') or '').strip()[:80]
947
+ # The pid wall is the SAME one every other row-scoped event rides: a caller may only
948
+ # touch customers inside their own permitted pool. Without it, documents would be the
949
+ # one surface where a scoped agent could read another book's rows.
950
+ if not isinstance(pid, int) or pid not in allowed_pids or not doc_id:
951
+ return False
952
+ meta_all = store.get(_DOCS_KEY) or {}
953
+ row = list((meta_all.get(str(pid)) or []))
954
+
955
+ if kind == 'doc_add':
956
+ raw_b64 = event.get('data_b64')
957
+ name = str(event.get('name') or 'document')[:200]
958
+ if not isinstance(raw_b64, str) or not raw_b64:
959
+ return False
960
+ try:
961
+ blob = _b64c.b64decode(raw_b64, validate=True)
962
+ except Exception:
963
+ return False # not base64 -> not a file we will store
964
+ # Re-check the DECODED size host-side. The client checks it too, but a client check
965
+ # is a courtesy: this is the number that bounds what the tenant's repo can be made
966
+ # to hold by anyone who can reach the endpoint.
967
+ if not blob or len(blob) > _DOC_MAX_BYTES:
968
+ return False
969
+ if len(row) >= _DOC_MAX_PER_ROW or any(d.get('id') == doc_id for d in row):
970
+ return False
971
+ safe = _re_mod.sub(r'[^A-Za-z0-9._-]', '_', name)[:80] or 'file'
972
+ path = f'docs/{pid}/{doc_id}_{safe}'
973
+ try:
974
+ store.upload_bytes(path, blob, message=f'doc {doc_id} for customer {pid}')
975
+ except Exception as _ue:
976
+ _tel.error('docs:upload', _ue)
977
+ return False
978
+ row.append({'id': doc_id, 'name': name,
979
+ 'mime': str(event.get('mime') or 'application/octet-stream')[:120],
980
+ 'size': len(blob), 'by': uname,
981
+ 'ts': dt.datetime.now().strftime('%Y-%m-%dT%H:%M:%S'),
982
+ 'path': path})
983
+ elif kind == 'doc_delete':
984
+ hit = next((d for d in row if d.get('id') == doc_id), None)
985
+ if not hit:
986
+ return False
987
+ # THE WALL: the uploader or an admin. Enforced here regardless of what the client
988
+ # chose to render β€” `canDelete` in the payload only hides a button.
989
+ if hit.get('by') != uname and not admin:
990
+ return False
991
+ store.delete_path(hit.get('path') or '')
992
+ row = [d for d in row if d.get('id') != doc_id]
993
+ else: # doc_fetch
994
+ hit = next((d for d in row if d.get('id') == doc_id), None)
995
+ if not hit:
996
+ return False
997
+ blob = store.download_bytes(hit.get('path') or '')
998
+ if blob is None:
999
+ return False
1000
+ # The bridge is a fire-and-forget event LOG with no response channel, so the answer
1001
+ # is PARKED on the Result and rides the next render's payload (C5).
1002
+ ctx.out.doc = {
1003
+ 'pid': pid, 'docId': doc_id, 'name': hit.get('name') or 'document',
1004
+ 'mime': hit.get('mime') or 'application/octet-stream',
1005
+ 'data_b64': _b64c.b64encode(blob).decode('ascii')}
1006
+ return True # re-render to deliver it
1007
+
1008
+ def _write(cur):
1009
+ cur[str(pid)] = row
1010
+ return cur
1011
+ store.update(_DOCS_KEY, _write, flush='async')
1012
+ return True
1013
+
1014
+ if kind in ('folder_create', 'folder_rename', 'folder_delete', 'folder_duplicate',
1015
+ 'item_move'):
1016
+ # FOLDERS over the Views / Cohorts sidebars (owner item 11, contract C4). All five
1017
+ # events touch ONE home β€” the table workspace's `folders` + `itemFolders` β€” so a cohort
1018
+ # can be filed without the cohort STORE learning about folders at all.
1019
+ import aios_grid as _agf
1020
+ if not store.available():
1021
+ return False
1022
+ surface = str(event.get('surface') or '')
1023
+ if surface not in _agf.FOLDER_SURFACES:
1024
+ return False
1025
+ # The ids this user may actually file: their own views, their own cohorts. Fail-closed β€”
1026
+ # an item_move naming somebody else's cohort is dropped by clean_item_folders anyway,
1027
+ # but refusing here means we never write it in the first place.
1028
+ def _own_ids(sfc, cur_ws):
1029
+ if sfc == 'views':
1030
+ return set((cur_ws.get('views') or {}))
1031
+ return set(cohort_mod.all_for(uname))
1032
+
1033
+ def _mutate(cur_ws):
1034
+ folders = _agf.clean_folders(cur_ws.get('folders'))
1035
+ placed = dict(cur_ws.get('itemFolders') or {})
1036
+ lst = list(folders.get(surface) or [])
1037
+ # ⚠ the wire field is `folderId` on EVERY folder event (CLIENT's types.ts is the
1038
+ # built contract; the split doc's first draft said `id`). Converged here rather
1039
+ # than asking the client to re-emit β€” see the C4 reconciliation note in the doc.
1040
+ fid = str(event.get('folderId') or '').strip()[:80]
1041
+ if kind == 'folder_create':
1042
+ name = str(event.get('name') or '').strip()[:_agf.MAX_FOLDER_NAME]
1043
+ if not fid or not name or len(lst) >= _agf.MAX_FOLDERS:
1044
+ return None
1045
+ if any(f['id'] == fid for f in lst):
1046
+ return None
1047
+ row = {'id': fid, 'name': name, 'order': len(lst)}
1048
+ icon = _agf.clean_folder_icon(event.get('icon')) # wave-9 I15 (C5)
1049
+ if icon:
1050
+ row['icon'] = icon
1051
+ lst.append(row)
1052
+ elif kind == 'folder_rename':
1053
+ name = str(event.get('name') or '').strip()[:_agf.MAX_FOLDER_NAME]
1054
+ hit = next((f for f in lst if f['id'] == fid), None)
1055
+ if not hit or not name:
1056
+ return None
1057
+ hit['name'] = name
1058
+ # wave-9 I15 (C5): the rename pane is also where the icon is changed. An event
1059
+ # that carries NO icon key leaves the existing one alone (a rename must not
1060
+ # silently strip a chosen icon); an explicit null clears it back to default.
1061
+ if 'icon' in event:
1062
+ icon = _agf.clean_folder_icon(event.get('icon'))
1063
+ if icon:
1064
+ hit['icon'] = icon
1065
+ else:
1066
+ hit.pop('icon', None)
1067
+ elif kind == 'folder_delete':
1068
+ if not any(f['id'] == fid for f in lst):
1069
+ return None
1070
+ lst = [f for f in lst if f['id'] != fid]
1071
+ # CONTENTS MOVE TO ROOT, never delete. Dropping the placements is exactly that:
1072
+ # an item with no placement renders at the top level.
1073
+ placed[surface] = {k: v for k, v in (placed.get(surface) or {}).items()
1074
+ if v != fid}
1075
+ elif kind == 'folder_duplicate':
1076
+ new_id = str(event.get('newId') or '').strip()[:80]
1077
+ src = next((f for f in lst if f['id'] == fid), None)
1078
+ if not src or not new_id or len(lst) >= _agf.MAX_FOLDERS:
1079
+ return None
1080
+ if any(f['id'] == new_id for f in lst):
1081
+ return None
1082
+ lst.append({'id': new_id, 'name': f"{src['name']} copy"[:_agf.MAX_FOLDER_NAME],
1083
+ 'order': len(lst)})
1084
+ # ⚠ The folder's CONTENTS are duplicated by the client emitting the ordinary
1085
+ # per-item duplicate events it already owns (view_upsert with a fresh id, the
1086
+ # cohort copy path) and then item_move-ing them here. Copying views/cohorts
1087
+ # inside this handler would be a SECOND implementation of "duplicate an item",
1088
+ # and the two would drift the first time either one changed.
1089
+ else: # item_move
1090
+ item_id = str(event.get('itemId') or '').strip()[:120]
1091
+ target = event.get('folderId')
1092
+ if not item_id or item_id not in _own_ids(surface, cur_ws):
1093
+ return None
1094
+ cur = dict(placed.get(surface) or {})
1095
+ if target is None:
1096
+ cur.pop(item_id, None) # back to the root
1097
+ else:
1098
+ tid = str(target)[:80]
1099
+ if not any(f['id'] == tid for f in lst):
1100
+ return None
1101
+ cur[item_id] = tid
1102
+ placed[surface] = cur
1103
+ folders[surface] = lst
1104
+ clean = _agf.clean_folders(folders)
1105
+ return clean, _agf.clean_item_folders(
1106
+ placed, clean,
1107
+ {sfc: _own_ids(sfc, cur_ws) for sfc in _agf.FOLDER_SURFACES})
1108
+
1109
+ # ⚠ NO allowed_pids ON PURPOSE β€” this is app.py's bare `_cl_table_workspace(uname)`
1110
+ # call at the old :6268, preserved exactly. Omitting the pool drops memberPids from
1111
+ # any foreign shared view (fail-closed), which is harmless here because the folder
1112
+ # legs read only view IDS and the folder maps. Passing the pool would be a behavior
1113
+ # change inside a pure move.
1114
+ _res = _mutate(table_workspace(ctx, consume_corrections=False))
1115
+ if _res is None:
1116
+ return False # refused: nothing is written
1117
+ _tops(ctx).save_folders(uname, _res[0], _res[1])
1118
+ # True: the sidebars render from server state (which folder holds what), not from local
1119
+ # optimism β€” the same contract the cohort panel's membership edits ride.
1120
+ return True
1121
+
1122
+ if kind == 'field_delete':
1123
+ # Delete a USER-CREATED column outright (owner gap closed 2026-07-27). The prefix IS
1124
+ # the permission: only the created strata (`custom_` overlay fields, `measure_` formula
1125
+ # columns) are deletable β€” a base field's key fails the prefix test no matter what the
1126
+ # browser claims, so the table's contract cannot be deleted from the client. The store
1127
+ # scrubs the field's stored cell values with it (delete_table_field); views naming the
1128
+ # key self-heal on their next autosave, the rule every stale colId already rides.
1129
+ key = str(event.get('key') or '')[:80]
1130
+ import aios_grid as _ag3
1131
+ if not (key.startswith('custom_') or key.startswith(_ag3.MEASURE_FIELD_PREFIX)):
1132
+ return False
1133
+ if store.available():
1134
+ _tops(ctx).delete_field(uname, key)
1135
+ else:
1136
+ _session_ready()
1137
+ ws['fields'].pop(key, None)
1138
+ for _row in ws['overlays'].values():
1139
+ if isinstance(_row, dict):
1140
+ _row.pop(key, None)
1141
+ # Wave-6 item 3: the client dropped the column optimistically; the store scrub is
1142
+ # invisible to the next render (the values leave WITH the column).
1143
+ return False
1144
+
1145
+ if kind == 'field_duplicate':
1146
+ # Wave-5 item 1. The DESTINATION key is client-generated (the established pattern β€”
1147
+ # the client must know it synchronously to place the clone right of the source), and
1148
+ # both keys must sit in the SAME created stratum: base odoo fields offer no Duplicate,
1149
+ # because duplicating the source-of-truth into an editable copy would need a
1150
+ # value-snapshot mechanism nobody asked for. Overlay VALUES are copied only for
1151
+ # `custom_` sources (a measure clone recomputes; a formula clone re-evaluates).
1152
+ # The duplicator becomes the clone's creator β€” it is a NEW field they made.
1153
+ src_key = str(event.get('sourceKey') or '')[:80]
1154
+ new_key = str(event.get('key') or '')[:80]
1155
+ import aios_grid as _ag5
1156
+ prefix = ('custom_' if src_key.startswith('custom_')
1157
+ else _ag5.MEASURE_FIELD_PREFIX
1158
+ if src_key.startswith(_ag5.MEASURE_FIELD_PREFIX) else None)
1159
+ if (prefix is None or not new_key.startswith(prefix) or new_key == src_key
1160
+ or new_key in field_by_key):
1161
+ return False
1162
+ src = ws.get('fields', {}).get(src_key)
1163
+ if not isinstance(src, dict):
1164
+ return False
1165
+ clone = dict(src)
1166
+ clone['key'] = new_key
1167
+ label = str(event.get('label') or '').strip()[:120]
1168
+ requested_label = label or (str(src.get('label') or 'Untitled')[:110] + ' copy')
1169
+ _workspace_field_keys = set((ws.get('fields') or {}))
1170
+ _reserved_field_names = [
1171
+ value.get('label') for candidate_key, value in field_by_key.items()
1172
+ if (candidate_key != new_key and candidate_key not in _workspace_field_keys
1173
+ and isinstance(value, dict))
1174
+ ]
1175
+ clone['label'] = requested_label
1176
+ clone['createdBy'] = uname
1177
+ # Wave-6 item 9: the clone inherits the source's scope via dict(src); on the cohort
1178
+ # page the duplicate surface may explicitly place it instead (same create-time rule
1179
+ # as field_upsert β€” a page that is not the cohort page cannot mint a cohort field).
1180
+ if scope_key == 'cohort' and event.get('scope') in ('cohort', 'global'):
1181
+ if event['scope'] == 'cohort':
1182
+ clone['scope'] = 'cohort'
1183
+ else:
1184
+ clone.pop('scope', None)
1185
+ if store.available():
1186
+ clone = (_tops(ctx).duplicate_field(
1187
+ uname, src_key, new_key, clone,
1188
+ reserved_names=_reserved_field_names,
1189
+ correction_id=event_id) or clone)
1190
+ else:
1191
+ _session_ready()
1192
+ _field_names = list(_reserved_field_names)
1193
+ _field_names.extend(
1194
+ value.get('label')
1195
+ for candidate_key, value in ws['fields'].items()
1196
+ if candidate_key != new_key and isinstance(value, dict)
1197
+ )
1198
+ clone.pop('labelCorrectedFrom', None)
1199
+ clone.pop('labelCorrectionId', None)
1200
+ clone['label'] = _unique_name(requested_label, _field_names)
1201
+ if clone['label'] != requested_label and event_id:
1202
+ clone['labelCorrectedFrom'] = requested_label
1203
+ clone['labelCorrectionId'] = str(event_id)[:180]
1204
+ ws['fields'][new_key] = clone
1205
+ if src_key.startswith('custom_'):
1206
+ for _row in ws['overlays'].values():
1207
+ if isinstance(_row, dict) and src_key in _row:
1208
+ _row[new_key] = _row[src_key]
1209
+ # Wave-6 item 3: True only when host-held VALUES must flow back β€” copied overlay
1210
+ # cells (custom_ sources) or host-computed measure values. formula/created_time
1211
+ # clones compute client-side, so the optimistic insert is already complete.
1212
+ label_changed = clone['label'] != requested_label
1213
+ return label_changed or src.get('type') not in ('formula', 'created_time')
1214
+
1215
+ if kind == 'add_to_list':
1216
+ # "Add to list" (owner item 6): a view's current matches become fixed cohort members.
1217
+ #
1218
+ # ⚠ `pids` arrives from the BROWSER. It is intersected with `allowed_pids` β€” the pool this
1219
+ # user may actually see β€” before anything is stored. The client computing the set is a
1220
+ # correctness decision (it is the only engine that already knows the answer, and asking a
1221
+ # second engine the same question is the drift verify_filter_engine.py exists to prevent);
1222
+ # it is NOT a trust decision.
1223
+ raw_pids = event.get('pids')
1224
+ if not isinstance(raw_pids, list):
1225
+ return False
1226
+ pids = [p for p in raw_pids if isinstance(p, int) and p in allowed_pids]
1227
+ if not pids:
1228
+ return False
1229
+ cohort_id = str(event.get('cohortId') or '').strip()[:120]
1230
+ name = str(event.get('name') or '').strip()[:120]
1231
+ if not store.available():
1232
+ return False
1233
+ try:
1234
+ if not cohort_id:
1235
+ if not name:
1236
+ return False
1237
+ cohort_id = cohort_mod.create(uname, name, pids)
1238
+ else:
1239
+ if cohort_id not in cohort_mod.all_for(uname):
1240
+ return False # not this user's cohort β€” fail closed, do not create
1241
+ cohort_mod.add_members(uname, cohort_id, pids)
1242
+ except ValueError as e:
1243
+ _tel.error('cohort:add_to_list', e)
1244
+ return False
1245
+ # True: the Cohort page's list and counts must reflect this, and the toast below only
1246
+ # renders on a fresh run.
1247
+ ctx.out.toast = (
1248
+ f"Added {len(pids):,} customer{'' if len(pids) == 1 else 's'} to "
1249
+ f"{name or 'the list'}.")
1250
+ return True
1251
+
1252
+ if kind == 'overlay_patch':
1253
+ pid = event.get('pid')
1254
+ if not isinstance(pid, int) or pid not in allowed_pids:
1255
+ return False
1256
+ updates = {}
1257
+ refused = False
1258
+ raw_updates = event.get('updates') if isinstance(event.get('updates'), dict) else {}
1259
+ for key, value in raw_updates.items():
1260
+ if key not in overlay_keys or not isinstance(value, (str, int, float)):
1261
+ refused = True
1262
+ continue
1263
+ # C-PERM: a field this caller's permissions HIDE cannot be written, only refused.
1264
+ # `overlay_keys` asks whether the column exists, not whether this person may touch
1265
+ # it β€” so without this line a restricted user who knows a hidden key could PATCH a
1266
+ # value they are not allowed to read back. Refused LOUDLY (repaint) so the client
1267
+ # reverts rather than showing a value the store never took.
1268
+ if key in (ctx.hidden_keys or frozenset()):
1269
+ refused = True
1270
+ continue
1271
+ # Wave-5 item 1: `permissions.edit` is enforced HERE, per key β€” the client hiding
1272
+ # its editor is courtesy, this is the wall. Fail-closed: a restricted field with
1273
+ # no readable createdBy admits only admins.
1274
+ perms = (field_by_key.get(key) or {}).get('permissions') or {}
1275
+ edit = perms.get('edit')
1276
+ if edit == 'admins' and not admin:
1277
+ refused = True
1278
+ continue
1279
+ if edit == 'creator' and not (
1280
+ admin or (field_by_key.get(key) or {}).get('createdBy') == uname):
1281
+ refused = True
1282
+ continue
1283
+ # C5-AUTOFIELD (wave 18): an automation cell is MACHINE-WRITTEN. The client marks
1284
+ # the type read-only (READONLY_CELL_TYPES) but that is courtesy β€” this is the wall.
1285
+ # Without it the UI hides the editor while the API still takes a PATCH, which is
1286
+ # exactly the shape a "read-only" type must not have.
1287
+ if (field_by_key.get(key) or {}).get('type') == 'automation':
1288
+ refused = True
1289
+ continue
1290
+ clean = str(value)[:10000]
1291
+ if isinstance(value, str) and clean != value:
1292
+ # Truncated: the stored truth is shorter than the client's copy. (An
1293
+ # int/float arriving as its string form renders identically β€” not a refusal.)
1294
+ refused = True
1295
+ definition = field_by_key.get(key) or {}
1296
+ if definition.get('type') in ('select', 'multiselect') and clean:
1297
+ options = [str(v) for v in definition.get('options') or []]
1298
+ canonical = {v.strip().lower(): v for v in options}
1299
+ if definition.get('type') == 'select':
1300
+ normalized = canonical.get(clean.strip().lower())
1301
+ if normalized is None:
1302
+ refused = True
1303
+ continue
1304
+ else:
1305
+ normalized_parts = []
1306
+ seen = set()
1307
+ invalid = False
1308
+ for part in clean.split(','):
1309
+ match = canonical.get(part.strip().lower())
1310
+ if match is None:
1311
+ invalid = True
1312
+ break
1313
+ if match.lower() not in seen:
1314
+ seen.add(match.lower())
1315
+ normalized_parts.append(match)
1316
+ if invalid:
1317
+ refused = True
1318
+ continue
1319
+ normalized = ','.join(normalized_parts)
1320
+ if normalized != clean:
1321
+ refused = True
1322
+ clean = normalized
1323
+ updates[key] = clean
1324
+ if updates:
1325
+ if store.available():
1326
+ _tops(ctx).patch_overlay(uname, pid, updates)
1327
+ else:
1328
+ _session_ready()
1329
+ ws['overlays'].setdefault(str(pid), {}).update(updates)
1330
+ # Wave-6 item 3: the client's cell already shows the typed value β€” rerun ONLY when
1331
+ # some of what it shows was refused or clamped and must be repainted to the truth.
1332
+ return refused
1333
+
1334
+ return False
platform/core/keychain.py ADDED
@@ -0,0 +1,182 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """core/keychain.py β€” per-tenant CREDENTIAL STORE (wave 18, contract C7 / ruling R3).
2
+
3
+ An admin stores a data source's secrets (Royal's Odoo API key today; any tenant's sources
4
+ tomorrow) encrypted at rest in the TENANT's store bucket. The runtime posture is R3's
5
+ "live with env fallback": the connector layer asks `odoo_creds(rt)` FIRST and falls back to
6
+ the process environment (tenant #0's `.env` / Space secrets), so Royal keeps working with an
7
+ empty keychain and a new tenant needs no code to bring keys. The FULL cutover (env retired)
8
+ is staged for a later wave β€” R3, verbatim.
9
+
10
+ CRYPTO: Fernet (AES128-CBC + HMAC, the `cryptography` package) under ONE platform key,
11
+ `AIOS_KEYCHAIN_KEY` (a `Fernet.generate_key()` value; a Space secret, never in the store).
12
+ No key β‡’ the keychain is LOCKED: metadata still lists (labels are not secrets), but nothing
13
+ decrypts and writes are refused β€” fail closed, never a plaintext fallback.
14
+
15
+ SHAPE (bucket `keychain`, per-tenant via the runtime handle):
16
+
17
+ {entries: {id: {id, label, type, enc, preview, created, createdBy}}}
18
+
19
+ `enc` = Fernet token over the JSON fields dict. `preview` = a masked hint ("Β·Β·Β·Β·1234")
20
+ computed ONCE at write from the entry's most secret-looking field β€” the ONLY part of a
21
+ secret that ever leaves this module through a route. `read_fields` exists for the CONNECTOR
22
+ LAYER; no route returns its output.
23
+ """
24
+ import datetime as _dt
25
+ import json
26
+ import os
27
+ import secrets as _secrets
28
+
29
+ KEY = 'keychain'
30
+ ENTRY_TYPES = {'odoo', 'generic'}
31
+ #: which stored field feeds the masked preview, per type (first present wins)
32
+ _PREVIEW_FIELDS = ('api_key', 'value', 'password', 'token')
33
+ MAX_ENTRIES = 40
34
+ MAX_FIELD_LEN = 500
35
+
36
+
37
+ class KeychainLocked(Exception):
38
+ """Raised when AIOS_KEYCHAIN_KEY is absent/wrong β€” callers answer 503, never {}."""
39
+
40
+
41
+ def _fernet():
42
+ key = (os.environ.get('AIOS_KEYCHAIN_KEY') or '').strip()
43
+ if not key:
44
+ raise KeychainLocked('AIOS_KEYCHAIN_KEY is not configured')
45
+ try:
46
+ from cryptography.fernet import Fernet
47
+ return Fernet(key.encode('ascii'))
48
+ except Exception as e:
49
+ raise KeychainLocked(f'keychain key unusable: {type(e).__name__}')
50
+
51
+
52
+ def unlocked():
53
+ try:
54
+ _fernet()
55
+ return True
56
+ except KeychainLocked:
57
+ return False
58
+
59
+
60
+ def _bucket(rt):
61
+ return rt.get(KEY) or {}
62
+
63
+
64
+ def list_entries(rt):
65
+ """Metadata ONLY β€” never a decrypted field, never `enc` itself."""
66
+ out = []
67
+ for eid, e in sorted((_bucket(rt).get('entries') or {}).items()):
68
+ if not isinstance(e, dict):
69
+ continue
70
+ out.append({'id': eid, 'label': e.get('label') or eid,
71
+ 'type': e.get('type') or 'generic',
72
+ 'preview': e.get('preview') or '',
73
+ 'created': e.get('created') or '',
74
+ 'createdBy': e.get('createdBy') or ''})
75
+ return out
76
+
77
+
78
+ def add_entry(rt, label, etype, fields, username):
79
+ """Encrypt + store. Returns the metadata row, or raises ValueError on shape problems and
80
+ KeychainLocked when there is no key (a secret must never be stored in the clear)."""
81
+ label = ' '.join(str(label or '').split())[:80]
82
+ etype = str(etype or '').strip().lower()
83
+ if not label:
84
+ raise ValueError('give the key a label')
85
+ if etype not in ENTRY_TYPES:
86
+ raise ValueError(f"type must be one of {', '.join(sorted(ENTRY_TYPES))}")
87
+ if not isinstance(fields, dict) or not fields:
88
+ raise ValueError('fields must be a non-empty object')
89
+ clean = {}
90
+ for k, v in list(fields.items())[:12]:
91
+ k = str(k).strip()[:40]
92
+ if k and isinstance(v, (str, int, float, bool)):
93
+ clean[k] = str(v)[:MAX_FIELD_LEN]
94
+ if not clean:
95
+ raise ValueError('no usable fields')
96
+ token = _fernet().encrypt(json.dumps(clean, ensure_ascii=False).encode('utf-8'))
97
+ secretish = next((clean[k] for k in _PREVIEW_FIELDS if clean.get(k)), '')
98
+ preview = ('Β·Β·Β·Β·' + secretish[-4:]) if len(secretish) >= 4 else ('Β·Β·Β·Β·' if secretish else '')
99
+ eid = f'k_{_secrets.token_hex(6)}'
100
+ row = {'id': eid, 'label': label, 'type': etype, 'enc': token.decode('ascii'),
101
+ 'preview': preview,
102
+ 'created': _dt.datetime.now().strftime('%Y-%m-%dT%H:%M:%S'),
103
+ 'createdBy': str(username or '')}
104
+
105
+ def _up(cur):
106
+ entries = cur.setdefault('entries', {})
107
+ if len(entries) >= MAX_ENTRIES:
108
+ raise ValueError(f'this keychain is at its {MAX_ENTRIES}-entry cap')
109
+ entries[eid] = row
110
+ return cur
111
+
112
+ rt.update(KEY, _up, flush='sync')
113
+ return {'id': eid, 'label': label, 'type': etype, 'preview': preview,
114
+ 'created': row['created'], 'createdBy': row['createdBy']}
115
+
116
+
117
+ def delete_entry(rt, entry_id):
118
+ def _drop(cur):
119
+ (cur.get('entries') or {}).pop(str(entry_id), None)
120
+ return cur
121
+ rt.update(KEY, _drop, flush='sync')
122
+ return True
123
+
124
+
125
+ def read_fields(rt, entry_id):
126
+ """Decrypt ONE entry's fields β€” for the CONNECTOR LAYER only; no route returns this.
127
+ None when the entry does not exist; KeychainLocked when the key is absent/wrong (a wrong
128
+ key must be loud β€” silently empty creds would read as 'not configured')."""
129
+ e = (_bucket(rt).get('entries') or {}).get(str(entry_id))
130
+ if not isinstance(e, dict) or not e.get('enc'):
131
+ return None
132
+ try:
133
+ raw = _fernet().decrypt(str(e['enc']).encode('ascii'))
134
+ except KeychainLocked:
135
+ raise
136
+ except Exception:
137
+ raise KeychainLocked('this entry does not decrypt under the configured key')
138
+ try:
139
+ data = json.loads(raw.decode('utf-8'))
140
+ return data if isinstance(data, dict) else None
141
+ except Exception:
142
+ return None
143
+
144
+
145
+ def odoo_creds(rt):
146
+ """R3's resolver seam: the FIRST odoo-type entry's fields, else None (the caller falls
147
+ back to the environment). Deterministic order = insertion-id sort, so 'first' is stable."""
148
+ for row in list_entries(rt):
149
+ if row['type'] == 'odoo':
150
+ return read_fields(rt, row['id'])
151
+ return None
152
+
153
+
154
+ def test_entry(rt, entry_id, timeout=8):
155
+ """A cheap, READ-ONLY liveness probe. odoo: xmlrpc `common.version()` (no auth, no data);
156
+ generic: shape-only. Always answers {ok, message} β€” an exception here is a result, not a
157
+ crash."""
158
+ try:
159
+ e = (_bucket(rt).get('entries') or {}).get(str(entry_id))
160
+ if not isinstance(e, dict):
161
+ return {'ok': False, 'message': 'no such entry'}
162
+ fields = read_fields(rt, entry_id) or {}
163
+ if e.get('type') != 'odoo':
164
+ return {'ok': bool(fields), 'message': f'{len(fields)} field(s) stored'}
165
+ url = str(fields.get('url') or '').rstrip('/')
166
+ if not url.startswith(('http://', 'https://')):
167
+ return {'ok': False, 'message': 'url must start with http(s)://'}
168
+ import socket
169
+ import xmlrpc.client
170
+ old = socket.getdefaulttimeout()
171
+ socket.setdefaulttimeout(timeout)
172
+ try:
173
+ common = xmlrpc.client.ServerProxy(f'{url}/xmlrpc/2/common', allow_none=True)
174
+ ver = common.version()
175
+ finally:
176
+ socket.setdefaulttimeout(old)
177
+ sv = (ver or {}).get('server_version') if isinstance(ver, dict) else None
178
+ return {'ok': True, 'message': f'Odoo answered (server {sv or "unknown"})'}
179
+ except KeychainLocked as e:
180
+ return {'ok': False, 'message': str(e)}
181
+ except Exception as e:
182
+ return {'ok': False, 'message': f'{type(e).__name__}: {str(e)[:120]}'}
platform/core/links.py ADDED
@@ -0,0 +1,59 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Deep links into the app β€” ?page=<key>&entity=<kind>:<id>&src=<source>.
2
+
3
+ The app (app.py) consumes these params once per session right after login and routes to the page,
4
+ opening the entity drawer when an entity is present. Digest/alert/mention emails build their links
5
+ here so every emailed number lands the reader on the exact row it came from. `src` tags where the
6
+ click came from (digest / alert / mention) β€” the app logs it as the click-through read receipt.
7
+ """
8
+ import os
9
+ import sys
10
+ from urllib.parse import quote
11
+
12
+ #: β›” THE ONE HOST LITERAL LEFT IN THE RUNTIME PATH, and it is a MIGRATION HAZARD β€” named here
13
+ #: rather than buried in a default argument, because the day we change hosts this is what silently
14
+ #: keeps mailing customers a link to the DEAD one. Found by `ops/verify_portability.py` on its
15
+ #: first run (2026-07-30).
16
+ #:
17
+ #: Why it is still here instead of raising: `deeplink()` has exactly one family of callers
18
+ #: (`modules/digest.py`, 7 sites, all building email hrefs), `DIGEST_ENABLED` defaults to ON, and
19
+ #: no deploy sets `APP_BASE_URL` today β€” so refusing to guess would break a LIVE feature to fix a
20
+ #: latent one. Instead: both deploy scripts now push `APP_BASE_URL`, which makes the fallback
21
+ #: unreachable in a deployed environment, and reaching it prints an operator warning.
22
+ #:
23
+ #: β‡’ EXIT CONDITION for this constant: once a deploy has set `APP_BASE_URL` and it is confirmed
24
+ #: present in the running environment, delete the fallback and let `deeplink()` raise instead. It
25
+ #: is declared in the portability gate's RESIDUALS so the gate reports it honestly rather than
26
+ #: reporting green over it.
27
+ _LEGACY_HF_FALLBACK = 'https://royal-imports-cfo-os.hf.space'
28
+
29
+ BASE_URL = (os.environ.get('APP_BASE_URL') or _LEGACY_HF_FALLBACK).rstrip('/')
30
+
31
+ #: Warned LAZILY and ONCE β€” at the first link actually built, not at import. An import-time print
32
+ #: fires in every gate, every subprocess and every unit test that so much as touches `core`, which
33
+ #: is how a real warning becomes noise nobody reads.
34
+ _WARNED = [False]
35
+
36
+
37
+ def _warn_once_if_defaulted():
38
+ if _WARNED[0] or os.environ.get('APP_BASE_URL'):
39
+ return
40
+ _WARNED[0] = True
41
+ print(f"[links] APP_BASE_URL is not set - emailed deep links will point at "
42
+ f"{_LEGACY_HF_FALLBACK}, which is correct ONLY while that is where the app lives. "
43
+ f"Set APP_BASE_URL in the deployed environment.", file=sys.stderr)
44
+
45
+ # entity kinds the URL scheme supports (must stay in sync with app.py's _desc_from_param)
46
+ LINK_KINDS = ('customer', 'sku', 'agent', 'account')
47
+
48
+
49
+ def deeplink(page=None, kind=None, ident=None, src=None):
50
+ """An absolute app URL. deeplink('customer_data', 'customer', 1234, src='digest')."""
51
+ _warn_once_if_defaulted()
52
+ parts = []
53
+ if page:
54
+ parts.append(f"page={quote(str(page), safe='')}")
55
+ if kind in LINK_KINDS and ident not in (None, ''):
56
+ parts.append(f"entity={quote(f'{kind}:{ident}', safe=':')}")
57
+ if src:
58
+ parts.append(f"src={quote(str(src), safe='')}")
59
+ return BASE_URL + ('/?' + '&'.join(parts) if parts else '/')
platform/core/management_pl.py ADDED
@@ -0,0 +1,161 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """core/management_pl.py β€” reproduce the operating model's MANAGEMENT P&Ls
2
+ (FFS-M / RI-M / GD-M / BU-M) monthly from RAW ODOO, Sep-2024 cutover onward. READ-ONLY.
3
+
4
+ Decoded from v42 (see 17. Application development/financial_analysis/MODEL_LOGIC_MAP.md Β§10):
5
+
6
+ -M[entity][account] = clean_entity[account] # live from Odoo (PROVEN to the cent)
7
+ + hq_overlay[entity][account] # HQ cost allocated to the entity
8
+ + adj_overrides[entity][account] # tiny, transition-month only
9
+
10
+ clean_entity = account.analytic.line grouped by general_account_id (GL row) x account_id
11
+ (analytic = ENTITY column). Analytic ids: Fisch=1 Royal=2 GiftwareDeals=3
12
+ HQ=4 Internal=5. Sections by account_type. This is build_income_statements.py
13
+ (verified exact). Proven live: Oct-2025 acct 51000 -> Fisch 327,531.85 etc.
14
+
15
+ hq_overlay / adj_overrides come from data/mgmt_overlay.json (extracted read-only from v42 by
16
+ build_mgmt_overlay.py). The HQ allocation is HARDCODED in the model (PNL (HQ-*) tabs are literal
17
+ values, not formulas), so storing it is faithful to how the model is actually maintained.
18
+
19
+ Consolidation: BU-M = FFS-M + RI-M + GD-M (the Internal analytic is eliminated; HQ is allocated out).
20
+
21
+ Pure: data in -> plain dicts out. No Streamlit, no writes.
22
+ """
23
+ from __future__ import annotations
24
+ import json
25
+ import os
26
+ from functools import lru_cache
27
+
28
+ from core import odoo as O
29
+
30
+ # ---- entity <-> Odoo analytic account ids (confirmed live) --------------------------------------
31
+ ENTITY_ANALYTIC = {'FFS': 1, 'RI': 2, 'GD': 3, 'HQ': 4, 'Internal': 5}
32
+ ANALYTIC_ENTITY = {v: k for k, v in ENTITY_ANALYTIC.items()}
33
+ BUS = ['FFS', 'RI', 'GD'] # the business units that get a -M statement
34
+ ENTITY_LABEL = {'FFS': 'Fisch (FFS)', 'RI': 'Royal (RI)', 'GD': 'Giftware Deals (GD)', 'BU': 'Consolidated (BU)'}
35
+
36
+ # P&L sections by Odoo account_type (mirrors build_income_statements.py)
37
+ INCOME_TYPES = {'income'}
38
+ OTHER_INCOME_TYPES = {'income_other'}
39
+ COGS_TYPES = {'expense_direct_cost'}
40
+ EXPENSE_TYPES = {'expense'}
41
+ DEPREC_TYPES = {'expense_depreciation'}
42
+ PL_TYPES = INCOME_TYPES | OTHER_INCOME_TYPES | COGS_TYPES | EXPENSE_TYPES | DEPREC_TYPES
43
+
44
+ _OVERLAY_PATH = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))),
45
+ 'data', 'mgmt_overlay.json')
46
+
47
+
48
+ def month_bounds(year: int, month: int) -> tuple[str, str]:
49
+ import calendar
50
+ last = calendar.monthrange(year, month)[1]
51
+ return f'{year}-{month:02d}-01', f'{year}-{month:02d}-{last:02d}'
52
+
53
+
54
+ @lru_cache(maxsize=1)
55
+ def account_master() -> dict:
56
+ """{gl_account_id: {'code','name','type'}} for every account."""
57
+ rows = O.search_read('account.account', [], ['id', 'code', 'name', 'account_type'])
58
+ return {r['id']: {'code': r.get('code') or '', 'name': r.get('name') or '',
59
+ 'type': r.get('account_type')} for r in rows}
60
+
61
+
62
+ def _display(acc_type: str, raw: float) -> float:
63
+ """Income shown raw; costs/expenses/depreciation flipped positive (model convention)."""
64
+ if acc_type in INCOME_TYPES or acc_type in OTHER_INCOME_TYPES:
65
+ return raw
66
+ return -raw
67
+
68
+
69
+ @lru_cache(maxsize=64)
70
+ def clean_entities(year: int, month: int) -> dict:
71
+ """The clean per-entity P&L straight from Odoo analytic lines, for one month.
72
+
73
+ Returns {entity_key: {account_code: display_amount}} for FFS/RI/GD/HQ/Internal, restricted to
74
+ P&L account types. This is the LIVE, proven-exact foundation (= build_income_statements.py).
75
+ """
76
+ start, end = month_bounds(year, month)
77
+ rows = O.read_group('account.analytic.line',
78
+ domain=[('date', '>=', start), ('date', '<=', end)],
79
+ fields=['amount:sum'], groupby=['general_account_id', 'account_id'], lazy=False)
80
+ am = account_master()
81
+ out = {k: {} for k in ENTITY_ANALYTIC}
82
+ for r in rows:
83
+ gen = r.get('general_account_id'); ana = r.get('account_id')
84
+ if not gen or not ana:
85
+ continue
86
+ gid, aid = gen[0], ana[0]
87
+ ent = ANALYTIC_ENTITY.get(aid)
88
+ acc = am.get(gid)
89
+ if ent is None or not acc or acc['type'] not in PL_TYPES:
90
+ continue
91
+ amt = _display(acc['type'], r.get('amount') or 0.0)
92
+ if abs(amt) < 0.005:
93
+ continue
94
+ out[ent][acc['code']] = out[ent].get(acc['code'], 0.0) + amt
95
+ return out
96
+
97
+
98
+ @lru_cache(maxsize=1)
99
+ def _overlay() -> dict:
100
+ """The stored HQ-allocation + transition-reconciliation overlay (build_mgmt_overlay.py).
101
+ Shape: {'YYYY-MM': {'FFS': {acct: amt}, ...}}. These are financial figures, so they live in the
102
+ PRIVATE HF Dataset store on the Space (never in the public Space repo); locally they load from the
103
+ committed JSON. File first, so local runs need no token. Empty -> -M = clean entity only."""
104
+ try:
105
+ with open(_OVERLAY_PATH, encoding='utf-8') as f:
106
+ return json.load(f)
107
+ except (FileNotFoundError, json.JSONDecodeError):
108
+ pass
109
+ try:
110
+ from core import store
111
+ return store.get('mgmt_overlay') or {}
112
+ except Exception:
113
+ return {}
114
+
115
+
116
+ def management_pl(year: int, month: int, entity: str) -> dict:
117
+ """The management P&L for one entity-month: {account_code: amount} = clean + overlay.
118
+
119
+ entity in {'FFS','RI','GD'}; 'BU' returns the consolidation (sum of the three).
120
+ """
121
+ if entity == 'BU':
122
+ agg: dict = {}
123
+ for bu in BUS:
124
+ for code, amt in management_pl(year, month, bu).items():
125
+ agg[code] = agg.get(code, 0.0) + amt
126
+ return agg
127
+ clean = dict(clean_entities(year, month).get(entity, {}))
128
+ ov = _overlay().get(f'{year}-{month:02d}', {}).get(entity, {})
129
+ for code, amt in ov.items():
130
+ clean[code] = clean.get(code, 0.0) + amt
131
+ return clean
132
+
133
+
134
+ # ---- section roll-ups (for the statement view) --------------------------------------------------
135
+ def _bucket(code: str) -> str | None:
136
+ acc = next((a for a in account_master().values() if a['code'] == code), None)
137
+ if not acc:
138
+ return None
139
+ t = acc['type']
140
+ if t in INCOME_TYPES: return 'income'
141
+ if t in OTHER_INCOME_TYPES: return 'other_income'
142
+ if t in COGS_TYPES: return 'cogs'
143
+ if t in EXPENSE_TYPES: return 'expense'
144
+ if t in DEPREC_TYPES: return 'deprec'
145
+ return None
146
+
147
+
148
+ def statement(year: int, month: int, entity: str) -> dict:
149
+ """Section subtotals + net for one entity-month (income / cogs / gross / opex / deprec / net)."""
150
+ lines = management_pl(year, month, entity)
151
+ sec = {'income': 0.0, 'other_income': 0.0, 'cogs': 0.0, 'expense': 0.0, 'deprec': 0.0}
152
+ for code, amt in lines.items():
153
+ b = _bucket(code)
154
+ if b:
155
+ sec[b] += amt
156
+ income = sec['income'] + sec['other_income']
157
+ gross = income - sec['cogs']
158
+ net = gross - sec['expense'] - sec['deprec']
159
+ return {'income': income, 'cogs': sec['cogs'], 'gross_profit': gross,
160
+ 'operating_expense': sec['expense'], 'depreciation': sec['deprec'], 'net_profit': net,
161
+ 'lines': lines}
platform/core/measure_resolve.py ADDED
@@ -0,0 +1,190 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """core/measure_resolve.py β€” HOST-NEUTRAL measure resolution (EXIT-5, SHELL.md ladder step 3).
2
+
3
+ The `_cl_measure_offer` / `_cl_resolve_measures` / `_cl_measure_columns` family, OUT of app.py:
4
+ one implementation, two hosts. The Streamlit adapter passes its `st.session_state` memos and its
5
+ pending-view read; the API passes runtime-level dicts and no pending view (over HTTP the
6
+ `view_upsert` lands via `/grid/events` BEFORE the workspace re-read, so the saved views already
7
+ carry the freshest rule β€” there is no mid-run component value to read).
8
+
9
+ No streamlit import, no app import β€” harness only, so both adapters can call it.
10
+
11
+ Memo discipline (unchanged from the app.py originals):
12
+ * keys carry (stamp, scope, POOL IDENTITY, question) β€” the pool's identity, never its size,
13
+ because Cohort passes a different pool per cohort and two same-sized cohorts must not be
14
+ handed each other's answers.
15
+ * `column_values` memoises permanent failures as None but NEVER a transient one (datastore
16
+ still warming after a restart) β€” a memoised transient means blank measure cells that stay
17
+ blank for the whole session long after the data landed.
18
+ * both memos are bounded by clearing wholesale past a cap β€” session/runtime-lifetime caches,
19
+ not leaks.
20
+ """
21
+ import hashlib
22
+
23
+
24
+ def offer(team_id=None, on_error=None):
25
+ """The measures the condition builder / measure columns may use, or [] if the semantic
26
+ store is unavailable. The caller's BU narrows the offer (a company-level measure cannot be
27
+ answered for one business unit)."""
28
+ try:
29
+ from harness import measure_filter as _mf
30
+ return _mf.measure_fields(team_id)
31
+ except Exception as e: # no store / model problem -> offer nothing
32
+ if on_error:
33
+ on_error('measures', e)
34
+ return []
35
+
36
+
37
+ def _pool_key(allowed_pids):
38
+ """The pool's identity, hashed. Part of every ANSWER, not just the question: the zero-group
39
+ is reconstructed from it (a customer with no orders in the window has no row for SQL to
40
+ group, so `Sales < 100` would miss exactly the lapsed customers the filter is for)."""
41
+ return hashlib.blake2b(
42
+ repr(sorted(allowed_pids)).encode(), digest_size=8).hexdigest()
43
+
44
+
45
+ def condition_sets(view_configs, pending_cfg, team_id, allowed_pids, today, stamp, memo,
46
+ on_error=None):
47
+ """Every measure condition across `view_configs` (+ the optional pending config, which goes
48
+ LAST and overwrites β€” the freshest statement of what the user wants) -> `{ruleId: [pid,…]}`.
49
+
50
+ A rule that is incomplete (mid-edit), unresolvable, or errors is left OUT of the answer β€”
51
+ the client reads a missing id as PENDING and says "Calculating…" rather than showing a count
52
+ it cannot stand behind. Never resolve an incomplete rule: `to_num(None)` is 0 by the
53
+ engines' shared contract, so a valueless `Sales > …` would become `Sales > 0` β€” a wrong set
54
+ under a confident count.
55
+ """
56
+ from harness import measure_filter as _mf
57
+ by_id = {}
58
+ for cfg in list(view_configs or []) + [pending_cfg or {}]:
59
+ for rule in _mf.collect((cfg or {}).get('filters')):
60
+ rid = str(rule.get('id') or '')
61
+ if rid:
62
+ by_id[rid] = rule
63
+ rules = list(by_id.values())
64
+ if not rules:
65
+ return {}
66
+ pk = _pool_key(allowed_pids)
67
+ out = {}
68
+ for rule in rules:
69
+ if not _mf.rule_complete(rule):
70
+ continue
71
+ key = (stamp, team_id, pk, _mf.signature(rule))
72
+ if key not in memo:
73
+ try:
74
+ memo[key] = _mf.resolve_rule(rule, today, team_id, allowed_pids)
75
+ except Exception as e:
76
+ # An unresolvable condition must NOT become "everything" or "nothing" silently.
77
+ if on_error:
78
+ on_error('measure-resolve', e)
79
+ continue
80
+ out[str(rule['id'])] = sorted(memo[key])
81
+ if len(memo) > 200: # a lifetime cache, not a leak
82
+ memo.clear()
83
+ return out
84
+
85
+
86
+ def series_values(fields, bucket, buckets, team_id, allowed_pids, today, stamp,
87
+ memo, on_error=None):
88
+ """`{field key: {'values': {bucket_start: v}, 'agg': kind}}` for every measure column
89
+ among `fields`, over `buckets` = [(start_iso, end_iso), …] β€” the C-TS channel, C-TSWIN
90
+ semantics since wave 14: each field's OWN window slides across the buckets
91
+ (`today := bucket end`), so two fields on the same measure with different windows are
92
+ DIFFERENT questions β€” the memo key carries the window signature, which the wave-13 key
93
+ (measure-only) silently conflated. `custom` fixed-range windows are the CALLER's to drop
94
+ (`window_fixed`) before calling; one that slips through memoises as a permanent failure.
95
+
96
+ Same memo discipline as `column_values`: keys carry (stamp, scope, POOL IDENTITY, the
97
+ question); a permanent failure memoises as None (the caller reports that field as
98
+ dropped), a TRANSIENT one (datastore still warming) never memoises, so the next call
99
+ retries instead of freezing a blank panel for the session's lifetime.
100
+ """
101
+ out = {}
102
+ mfields = [f for f in fields or [] if isinstance(f.get('measure'), dict)]
103
+ if not mfields or not buckets:
104
+ return out
105
+ from harness import measure_filter as _mf
106
+ from harness import windows as _wn
107
+ pk = _pool_key(allowed_pids)
108
+ span_from, span_to = str(buckets[0][0]), str(buckets[-1][1])
109
+ for f in mfields:
110
+ spec = f.get('measure') or {}
111
+ mkey = spec.get('key')
112
+ w = _wn.normalize(spec.get('window'))
113
+ wsig = None if w is None else tuple(sorted(w.items()))
114
+ key = (stamp, team_id, pk, mkey, bucket, span_from, span_to, wsig)
115
+ if key not in memo:
116
+ try:
117
+ memo[key] = _mf.resolve_series_slid(mkey, spec.get('window'), buckets,
118
+ today, team_id, allowed_pids)
119
+ except Exception as e:
120
+ if on_error:
121
+ on_error('measure-series', e)
122
+ transient = False
123
+ try:
124
+ from harness import datastore as _ds
125
+ transient = not _ds.ready()
126
+ except Exception:
127
+ transient = False
128
+ if transient:
129
+ continue # no memo entry -> the next call tries again
130
+ memo[key] = None
131
+ ans = memo[key]
132
+ if ans is None:
133
+ continue
134
+ out[f['key']] = ans
135
+ if len(memo) > 200:
136
+ memo.clear()
137
+ return out
138
+
139
+
140
+ def column_values(fields, team_id, allowed_pids, today, stamp, memo, on_error=None):
141
+ """Values for every FORMULA-MEASURE column among `fields` -> `{pid: {field key: value}}`,
142
+ handed to `rows_from_pool(derived=…)` beside the cohort cells.
143
+
144
+ A column that cannot be computed (store down, BU scope on a company-level measure,
145
+ unresolvable window) degrades to BLANK β€” the value dict simply omits that field key, and
146
+ the client renders empty cells, never $0: blank is "could not compute", 0 is a real zero.
147
+ """
148
+ mfields = [f for f in fields or [] if isinstance(f.get('measure'), dict)]
149
+ if not mfields:
150
+ return {}
151
+ from harness import measure_filter as _mf
152
+ from harness import windows as _wn
153
+ pk = _pool_key(allowed_pids)
154
+ out = {}
155
+ for f in mfields:
156
+ spec = f['measure']
157
+ w = _wn.normalize(spec.get('window'))
158
+ key = (stamp, team_id, pk, spec.get('key'),
159
+ None if w is None else tuple(sorted(w.items())))
160
+ if key not in memo:
161
+ try:
162
+ memo[key] = _mf.resolve_values(spec.get('key'), spec.get('window'),
163
+ today, team_id, allowed_pids)
164
+ except Exception as e:
165
+ if on_error:
166
+ on_error('measure-column', e)
167
+ # ⚠ ONLY A PERMANENT FAILURE MAY BE MEMOISED. "The data cache is still warming
168
+ # up after a restart" is TRANSIENT and self-healing β€” memoising it means a user
169
+ # who opened the page during that window gets blank measure cells that STAY
170
+ # blank for the whole session. A permanent failure (an unadmitted measure, an
171
+ # unresolvable window) still memoises, so the retry-storm this guard was
172
+ # written for cannot come back.
173
+ transient = False
174
+ try:
175
+ from harness import datastore as _ds
176
+ transient = not _ds.ready()
177
+ except Exception:
178
+ transient = False
179
+ if transient:
180
+ continue # no memo entry -> the next call tries again
181
+ memo[key] = None
182
+ vals = memo[key]
183
+ if vals is None:
184
+ continue
185
+ fkey = f['key']
186
+ for pid, v in vals.items():
187
+ out.setdefault(pid, {})[fkey] = v
188
+ if len(memo) > 100:
189
+ memo.clear()
190
+ return out
platform/core/odoo.py ADDED
@@ -0,0 +1,201 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """CFO-OS shared Odoo data layer (READ-ONLY).
2
+
3
+ Reuses the existing write-guarded client (ffs_dashboard/odoo_client.py) so there is
4
+ a single source of truth for the connection. Adds:
5
+ - cached singleton client
6
+ - read_group / search_read passthroughs with light cleanup
7
+ - business constants (teams, excluded partners)
8
+ - the "sales universe" base domain (confirmed orders, RI+FFS only)
9
+
10
+ NOTHING here writes to Odoo β€” the underlying client hard-blocks writes.
11
+ """
12
+ import os
13
+ import sys
14
+ import queue as _queue
15
+ import threading
16
+ from concurrent.futures import ThreadPoolExecutor
17
+ from functools import lru_cache
18
+ from pathlib import Path
19
+
20
+ # Self-contained read-only client lives at the app root (platform/odoo_client.py) with its
21
+ # own .env β€” so this whole folder can be moved/hosted anywhere without external paths.
22
+ _ROOT = Path(__file__).resolve().parents[1] # platform/
23
+ if str(_ROOT) not in sys.path:
24
+ sys.path.insert(0, str(_ROOT))
25
+ from odoo_client import OdooClient # noqa: E402 (read-only, write-guarded)
26
+
27
+ # ---- business constants ----
28
+ TEAM_IDS = [5, 6]
29
+ TEAM_NAMES = {5: 'Fisch', 6: 'Royal'}
30
+ # Partners to exclude from analytics (not part of RI/FFS). Sourced from env so no
31
+ # customer name is ever hardcoded in the (potentially public) repo; set as a Space secret.
32
+ EXCLUDE_PARTNER_NAMES = {n.strip() for n in os.environ.get('EXCLUDE_PARTNER_NAMES', '').split(',') if n.strip()}
33
+
34
+ # Confirmed sale.order.line scope for RI+FFS (the analytics "sales universe").
35
+ SALE_LINE_BASE = [
36
+ ('state', 'in', ['sale', 'done']),
37
+ ('order_id.team_id', 'in', TEAM_IDS),
38
+ ('product_id', '!=', False),
39
+ ]
40
+ # Posted customer invoices/credit notes (AR + revenue recognised).
41
+ INVOICE_BASE = [
42
+ ('move_type', 'in', ['out_invoice', 'out_refund']),
43
+ ('state', '=', 'posted'),
44
+ ]
45
+
46
+
47
+ @lru_cache(maxsize=1)
48
+ def _global_client():
49
+ """The single read-only client used for normal (main-thread) calls."""
50
+ return OdooClient()
51
+
52
+
53
+ # ---- concurrent read pool -------------------------------------------------------------------
54
+ # Each Odoo XML-RPC call is a round-trip; a heavy drawer fires a dozen of them and they add up to
55
+ # many seconds when run one-by-one. `parallel()` runs independent READ tasks across a small pool of
56
+ # extra connections, with each task transparently using its own connection via a thread-local β€” so
57
+ # the existing read_group/search_read/sum_field helpers need no change. READ-ONLY only.
58
+ _POOL_MAX = 8
59
+ _tlocal = threading.local()
60
+ _pool_q = _queue.Queue()
61
+ _pool_n = 0
62
+ _pool_lock = threading.Lock()
63
+
64
+
65
+ def _checkout():
66
+ global _pool_n
67
+ try:
68
+ return _pool_q.get_nowait()
69
+ except _queue.Empty:
70
+ with _pool_lock:
71
+ grow = _pool_n < _POOL_MAX
72
+ if grow:
73
+ _pool_n += 1
74
+ return OdooClient() if grow else _pool_q.get() # else block until one frees
75
+
76
+
77
+ def _checkin(c):
78
+ _pool_q.put(c)
79
+
80
+
81
+ def get_odoo():
82
+ """The active read-only client. Inside a parallel() task this is that task's pooled connection;
83
+ otherwise it's the shared singleton. Transparent to every read_group/search_read/... caller."""
84
+ c = getattr(_tlocal, 'client', None)
85
+ return c if c is not None else _global_client()
86
+
87
+
88
+ def set_doc_mode(mode):
89
+ """Recognition basis for THIS thread's domain builds: 'order' (all confirmed orders) or 'invoice'
90
+ (fully-invoiced orders only). Cached bundles call this before pulling; parallel() propagates it to
91
+ each worker. Thread-local, so concurrent user sessions never race on each other's basis."""
92
+ _tlocal.doc_mode = 'invoice' if mode == 'invoice' else 'order'
93
+
94
+
95
+ def doc_mode():
96
+ return getattr(_tlocal, 'doc_mode', 'order')
97
+
98
+
99
+ def parallel(tasks):
100
+ """Run independent zero-arg READ callables concurrently across the connection pool; returns
101
+ results in order. Inside each task, get_odoo() (hence read_group/search_read/sum_field/
102
+ search_count) uses that task's own connection. Never use for writes."""
103
+ tasks = list(tasks)
104
+ if not tasks:
105
+ return []
106
+ if len(tasks) == 1:
107
+ return [tasks[0]()]
108
+
109
+ caller_mode = doc_mode() # propagate the submitter's basis to each worker
110
+ def _run(fn):
111
+ c = _checkout()
112
+ _tlocal.client = c
113
+ _tlocal.doc_mode = caller_mode
114
+ try:
115
+ return fn()
116
+ finally:
117
+ _tlocal.client = None
118
+ _tlocal.doc_mode = 'order'
119
+ _checkin(c)
120
+ with ThreadPoolExecutor(max_workers=min(len(tasks), _POOL_MAX)) as ex:
121
+ return list(ex.map(_run, tasks))
122
+
123
+
124
+ def parallel_map(task_dict):
125
+ """parallel() over a {name: callable} mapping -> {name: result}."""
126
+ keys = list(task_dict)
127
+ return dict(zip(keys, parallel([task_dict[k] for k in keys])))
128
+
129
+
130
+ def m2o_id(v):
131
+ return v[0] if isinstance(v, list) and v else None
132
+
133
+
134
+ def m2o_name(v):
135
+ return v[1] if isinstance(v, list) and len(v) > 1 else ''
136
+
137
+
138
+ @lru_cache(maxsize=1)
139
+ def excluded_partner_ids():
140
+ """Resolve the excluded partner names (from EXCLUDE_PARTNER_NAMES) to ids, for fast
141
+ domain filtering. Cached for the process."""
142
+ o = get_odoo()
143
+ if not EXCLUDE_PARTNER_NAMES:
144
+ return tuple()
145
+ rows = o.search_read('res.partner',
146
+ [('name', 'in', list(EXCLUDE_PARTNER_NAMES))], ['id'])
147
+ return tuple(r['id'] for r in rows)
148
+
149
+
150
+ def sale_line_domain(date_from=None, date_to=None, team_id=None, extra=None, partner_ids=None):
151
+ """Build a sale.order.line domain in the RI+FFS confirmed-order scope, excluding
152
+ the configured house accounts, with optional date window / single team / extra clauses.
153
+ partner_ids (a collection, possibly empty) restricts to those customers β€” the carrier for
154
+ the Customer module's Agent filter; None = no partner restriction (an empty set matches none)."""
155
+ dom = list(SALE_LINE_BASE)
156
+ if team_id is not None:
157
+ dom = [d for d in dom if not (isinstance(d, tuple) and d[0] == 'order_id.team_id')]
158
+ dom.append(('order_id.team_id', '=', team_id))
159
+ if date_from:
160
+ dom.append(('order_id.date_order', '>=', f'{date_from} 00:00:00'))
161
+ if date_to:
162
+ dom.append(('order_id.date_order', '<=', f'{date_to} 23:59:59'))
163
+ ex = excluded_partner_ids()
164
+ if ex:
165
+ dom.append(('order_partner_id', 'not in', list(ex)))
166
+ if partner_ids is not None:
167
+ dom.append(('order_partner_id', 'in', list(partner_ids)))
168
+ if doc_mode() == 'invoice':
169
+ dom.append(('order_id.invoice_status', '=', 'invoiced'))
170
+ if extra:
171
+ dom.extend(extra)
172
+ return dom
173
+
174
+
175
+ def read_group(model, domain, fields, groupby, **kw):
176
+ return get_odoo().read_group(model, domain=domain, fields=fields, groupby=groupby, **kw)
177
+
178
+
179
+ def search_read(model, domain=None, fields=None, **kw):
180
+ return get_odoo().search_read(model, domain=domain or [], fields=fields or [], **kw)
181
+
182
+
183
+ def sum_field(model, domain, field):
184
+ """One-line aggregate sum of `field` over a domain (uses read_group, no row fetch).
185
+
186
+ Robust to the empty match set: a groupby=[] read_group over zero rows makes Odoo return a
187
+ None aggregate, which the server's XML-RPC layer cannot marshal ("cannot marshal None").
188
+ An empty set simply means the sum is 0.0, so we treat that specific Fault as zero."""
189
+ try:
190
+ g = get_odoo().read_group(model, domain=domain, fields=[f'{field}:sum'], groupby=[], lazy=False)
191
+ except Exception as e: # noqa: BLE001 β€” narrow check below
192
+ if 'cannot marshal None' in str(e):
193
+ return 0.0
194
+ raise
195
+ return (g[0].get(field) or 0.0) if g else 0.0
196
+
197
+
198
+ def distinct_count(model, domain, field):
199
+ """Distinct count of a (m2o) field over a domain via grouped read_group."""
200
+ g = get_odoo().read_group(model, domain=domain, fields=[field], groupby=[field], lazy=False)
201
+ return sum(1 for r in g if r.get(field))
platform/core/periods.py ADDED
@@ -0,0 +1,155 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Period helpers β€” floral wholesale is highly seasonal, so YoY / same-period
2
+ comparisons are first-class. Everything returns ISO date strings (yyyy-mm-dd).
3
+
4
+ `today` is injectable for testing/reproducibility.
5
+ """
6
+ import datetime as dt
7
+
8
+
9
+ def _d(x):
10
+ return x.isoformat()
11
+
12
+
13
+ def today():
14
+ return dt.date.today()
15
+
16
+
17
+ def ytd(t=None):
18
+ """(start, end) for year-to-date through `t`."""
19
+ t = t or today()
20
+ return _d(dt.date(t.year, 1, 1)), _d(t)
21
+
22
+
23
+ def ytd_last_year(t=None):
24
+ """Same calendar window one year earlier (Jan 1 LY β†’ same month/day LY).
25
+ Clamps Feb 29 β†’ Feb 28 in non-leap years."""
26
+ t = t or today()
27
+ try:
28
+ end = dt.date(t.year - 1, t.month, t.day)
29
+ except ValueError:
30
+ end = dt.date(t.year - 1, t.month, t.day - 1)
31
+ return _d(dt.date(t.year - 1, 1, 1)), end if isinstance(end, str) else _d(end)
32
+
33
+
34
+ def ltm(t=None):
35
+ """Trailing 12 months ending `t`."""
36
+ t = t or today()
37
+ return _d(t - dt.timedelta(days=365)), _d(t)
38
+
39
+
40
+ def prior_ltm(t=None):
41
+ t = t or today()
42
+ return _d(t - dt.timedelta(days=730)), _d(t - dt.timedelta(days=365))
43
+
44
+
45
+ def mtd(t=None):
46
+ t = t or today()
47
+ return _d(dt.date(t.year, t.month, 1)), _d(t)
48
+
49
+
50
+ def wtd(t=None):
51
+ """(Monday of this ISO week, t) β€” week-to-date."""
52
+ t = t or today()
53
+ return _d(t - dt.timedelta(days=t.weekday())), _d(t)
54
+
55
+
56
+ def qtd(t=None):
57
+ """(first day of this calendar quarter, t) β€” quarter-to-date."""
58
+ t = t or today()
59
+ q_start_month = ((t.month - 1) // 3) * 3 + 1
60
+ return _d(dt.date(t.year, q_start_month, 1)), _d(t)
61
+
62
+
63
+ def quarter_range(t=None):
64
+ """(first day, last day) of the calendar quarter containing t."""
65
+ t = t or today()
66
+ qs = ((t.month - 1) // 3) * 3 + 1
67
+ first = dt.date(t.year, qs, 1)
68
+ nxt = dt.date(t.year + (1 if qs + 3 > 12 else 0), ((qs + 2) % 12) + 1, 1)
69
+ return _d(first), _d(nxt - dt.timedelta(days=1))
70
+
71
+
72
+ def shift_year(date_from, date_to, weeks=False):
73
+ """Same window one year earlier. For week-aligned comparisons (weeks=True) shift by exactly 52
74
+ weeks (364 days) so the weekday lines up; otherwise shift by calendar year (clamping Feb 29)."""
75
+ f = dt.date.fromisoformat(date_from)
76
+ t = dt.date.fromisoformat(date_to)
77
+ if weeks:
78
+ return _d(f - dt.timedelta(days=364)), _d(t - dt.timedelta(days=364))
79
+
80
+ def back(d):
81
+ try:
82
+ return dt.date(d.year - 1, d.month, d.day)
83
+ except ValueError:
84
+ return dt.date(d.year - 1, d.month, d.day - 1)
85
+ return _d(back(f)), _d(back(t))
86
+
87
+
88
+ def last_n_days(n, t=None):
89
+ t = t or today()
90
+ return _d(t - dt.timedelta(days=n)), _d(t)
91
+
92
+
93
+ def year_range(year):
94
+ return f'{year}-01-01', f'{year}-12-31'
95
+
96
+
97
+ def month_range(year, month):
98
+ """(first day, last day) of a given calendar month, ISO strings."""
99
+ first = dt.date(year, month, 1)
100
+ nxt = dt.date(year + (1 if month == 12 else 0), 1 if month == 12 else month + 1, 1)
101
+ return _d(first), _d(nxt - dt.timedelta(days=1))
102
+
103
+
104
+ def period_options(t=None, n_months=15):
105
+ """Ordered {label: (date_from, date_to)} for the close/reconciliation period picker."""
106
+ t = t or today()
107
+ opts = {}
108
+ opts['This month (to date)'] = (_d(dt.date(t.year, t.month, 1)), _d(t))
109
+ lm_year, lm_month = (t.year, t.month - 1) if t.month > 1 else (t.year - 1, 12)
110
+ lf, lt = month_range(lm_year, lm_month)
111
+ opts[f'Last month ({lf[:7]})'] = (lf, lt)
112
+ opts['Quarter to date'] = (_d(dt.date(t.year, ((t.month - 1) // 3) * 3 + 1, 1)), _d(t))
113
+ opts['Year to date'] = (_d(dt.date(t.year, 1, 1)), _d(t))
114
+ for label, mf, mt in reversed(month_starts(n_months, t)):
115
+ if label == f'{t.year}-{t.month:02d}' or label == lf[:7]:
116
+ continue
117
+ opts[label] = (mf, mt)
118
+ return opts
119
+
120
+
121
+ def month_starts(n_back=13, t=None):
122
+ """List of (yyyy-mm, start, end) for the last n_back months including current."""
123
+ t = t or today()
124
+ out = []
125
+ y, m = t.year, t.month
126
+ for _ in range(n_back):
127
+ start = dt.date(y, m, 1)
128
+ end = (dt.date(y + (m // 12), (m % 12) + 1, 1) - dt.timedelta(days=1))
129
+ out.append((f'{y}-{m:02d}', _d(start), _d(end)))
130
+ m -= 1
131
+ if m == 0:
132
+ m = 12
133
+ y -= 1
134
+ return list(reversed(out))
135
+
136
+
137
+ def week_starts(n_back=12, t=None):
138
+ """List of (label, start, end) for the last n_back ISO weeks (Mon–Sun) including the current
139
+ (partial) week. label = the Monday as 'MM-DD' (compact, chart-friendly); end is clamped to t
140
+ for the current week so weekly totals never include future days."""
141
+ t = t or today()
142
+ this_mon = t - dt.timedelta(days=t.weekday())
143
+ out = []
144
+ for i in range(n_back):
145
+ mon = this_mon - dt.timedelta(weeks=i)
146
+ sun = mon + dt.timedelta(days=6)
147
+ end = min(sun, t)
148
+ out.append((f'{mon.month:02d}-{mon.day:02d}', _d(mon), _d(end)))
149
+ return list(reversed(out))
150
+
151
+
152
+ def yoy_pct(this_v, last_v):
153
+ if last_v:
154
+ return (this_v - last_v) / last_v * 100.0
155
+ return None
platform/core/perm_migrate.py ADDED
@@ -0,0 +1,156 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """core/perm_migrate.py β€” R1's one-time migration: `bus`/`agent` become a PERMANENT FILTER.
2
+
3
+ Owner ruling R1: BU access stops being its own field and becomes an ordinary permanent filter
4
+ under the permissioning engine, declared and edited exactly like "Agent is Tara". This module
5
+ builds that filter from the legacy record and β€” the part that matters β€” PROVES the swap is a
6
+ no-op before anything relies on it.
7
+
8
+ β›” WHY THE PROOF IS ABOUT VALUES AND NOT ABOUT ROWS (C-PERM amendment 3).
9
+ `modules/customer_data._pool_build` passes `team_id` into `cust._cust_rev` three times (YTD, LY,
10
+ LTM) and into `cust._cadence_bulk`. It therefore decides what `rev`, `ly`, `ltm`, `aov`,
11
+ `est_missed` and the derived `status` MEAN β€” it is not a row filter. Swap it for a post-filter
12
+ and a Fisch-only user keeps a plausible row LIST while every number on it quietly becomes
13
+ Fisch+Royal, worst for `dba = Both` customers, who are precisely the ones a BU filter admits.
14
+ A pid-set reconciliation reports GREEN on that. So `reconcile()` compares VALUES.
15
+
16
+ The pushdown survives as `perm_scope.derive_pool_scope` β€” a DERIVATION OF the filter rather than
17
+ a wall beside it β€” so a correctly migrated record produces the IDENTICAL pool call, and the
18
+ reconciliation below should be exact rather than merely close. If it is not exact, that is a
19
+ finding for the owner, not a tolerance to widen.
20
+ """
21
+ import core.perm_scope as perm_scope
22
+ import core.perms as perms
23
+
24
+ MODULE = 'customer_data'
25
+
26
+ #: `_dba_attrs` emits exactly {'Fisch','Royal','Both'} (blank = no brand attributable in 24
27
+ #: months). A BU grant admits its own brand AND the dual-brand customers: a Both customer IS a
28
+ #: Fisch customer. Mirrors `perm_scope._DBA_TEAM`, inverted.
29
+ #:
30
+ #: ⚠ TWO VALUES, AND THE COLUMN VOCABULARY HAS NO "IS ANY OF". `FILTER_OPS` is single-valued by
31
+ #: design (`eq`, `neq`, `contains`, …) β€” `anyOf` belongs to the COHORT vocabulary and is
32
+ #: deliberately DISJOINT from it, so a column leaf carrying `anyOf` is refused by every engine
33
+ #: (`filter_eval` returns False, `clean_filter_tree` drops it). This migration's first draft
34
+ #: emitted exactly that and `reconcile()` caught it as two lost customers. So the BU condition
35
+ #: is an OR GROUP of `eq` leaves, which is what the condition builder itself produces.
36
+ _TEAM_DBA = {5: ('Fisch', 'Both'), 6: ('Royal', 'Both')}
37
+
38
+
39
+ def filter_for(rec):
40
+ """The permanent filter equivalent to this record's legacy `bus` + `agent`, or None when the
41
+ record was unrestricted (an admin, or `bus:'all'` with no agent link).
42
+
43
+ AND-conjoined at the root, and top-level by construction β€” which is also what makes it
44
+ pushdown-derivable (`derive_pool_scope` reads top-level AND leaves only).
45
+ """
46
+ nodes = []
47
+ team_id = perms.scope_team_id(rec)
48
+ if team_id in _TEAM_DBA:
49
+ nodes.append({'conj': 'or',
50
+ 'children': [{'colId': 'dba', 'op': 'eq', 'value': v, 'value2': ''}
51
+ for v in _TEAM_DBA[team_id]]})
52
+ agent = perms.scope_agent(rec)
53
+ if agent:
54
+ nodes.append({'colId': 'agent', 'op': 'eq', 'value': str(agent), 'value2': ''})
55
+ if not nodes:
56
+ return None
57
+ return {'conj': 'and', 'nodes': nodes}
58
+
59
+
60
+ def perms_for(rec, modules=(MODULE,)):
61
+ """The `perms` block this legacy record becomes.
62
+
63
+ ⚠ ACCESS IS COPIED FROM THE OLD GRANT, NOT ASSUMED. `may_open` is what decided reachability
64
+ before this wave, so migrating a record must not widen it β€” and on a migrated record an
65
+ undeclared module DENIES (amendment 4), so every module the user could reach has to be
66
+ declared here or the migration is a lockout.
67
+ """
68
+ out = {}
69
+ for key in modules:
70
+ out[key] = {'access': bool(perms.may_open(rec, key)),
71
+ 'filter': filter_for(rec),
72
+ 'hiddenFields': []}
73
+ return out
74
+
75
+
76
+ def plan(registry_dict, modules=(MODULE,)):
77
+ """`{username: perms_block}` for every record that is not already migrated.
78
+
79
+ Admins are included and get an all-access block with no filter: they bypass `perm_scope`
80
+ anyway, but leaving them un-migrated would keep the legacy reader alive for the one account
81
+ most likely to be inspected when something looks wrong.
82
+ """
83
+ out = {}
84
+ for uname, rec in (registry_dict or {}).items():
85
+ if not isinstance(rec, dict) or perm_scope.is_migrated(rec):
86
+ continue
87
+ out[uname] = perms_for(rec, modules)
88
+ return out
89
+
90
+
91
+ #: The row members whose meaning depends on `team_id` β€” the ones amendment 3 is about. A
92
+ #: reconciliation that skipped these would be the pid-only gate that cannot see the leak.
93
+ VALUE_KEYS = ('rev', 'ly', 'ltm', 'aov', 'est_missed', 'status', 'orders', 'last_order')
94
+
95
+
96
+ def reconcile(rec, pool_fn, modules=(MODULE,)):
97
+ """Prove the migrated record sees EXACTLY what the legacy record saw.
98
+
99
+ `pool_fn(agent, team_id) -> rows` is injected so this is testable without Odoo and so the
100
+ caller decides whether to hit the live builder or a fixture.
101
+
102
+ Returns `{ok, legacy_scope, derived_scope, lost, gained, value_diffs}`:
103
+ * `lost` β€” pids the legacy wall admitted and the migrated one drops. Expected to be
104
+ EMPTY. One narrow class is known and must be REPORTED rather than tolerated:
105
+ `_dba_attrs` uses a 731-day window while `_cust_rev`'s LY window can reach a
106
+ little further back, so a customer whose only in-scope revenue predates that
107
+ window carries a BLANK `dba` and fails a `dba` leaf. Blank is the honest "no
108
+ brand attributable" state, so such a customer is NAMED here, never rounded
109
+ away.
110
+ * `gained` β€” pids the migrated wall admits and the legacy one did not. Must be empty: it
111
+ is the widening direction, and there is no acceptable non-empty value.
112
+ * `value_diffs` β€” per-pid disagreements on VALUE_KEYS. The assertion amendment 3 exists
113
+ for; a non-empty list means the pushdown was not applied and the numbers are
114
+ consolidated.
115
+ """
116
+ legacy_scope = (perms.scope_agent(rec), perms.scope_team_id(rec))
117
+ migrated = dict(rec)
118
+ migrated['perms'] = perms_for(rec, modules)
119
+ migrated['perms_v'] = perm_scope.PERMS_VERSION
120
+ team_id, agent = perm_scope.derive_pool_scope(migrated, MODULE)
121
+ derived_scope = (agent, team_id)
122
+
123
+ legacy_rows = pool_fn(legacy_scope[0], legacy_scope[1])
124
+ derived_rows = pool_fn(agent, team_id)
125
+ fields = _fields_of(legacy_rows)
126
+ kept = perm_scope.apply_row_scope(derived_rows, migrated, MODULE, fields)
127
+
128
+ legacy_by = {r.get('pid'): r for r in legacy_rows}
129
+ kept_by = {r.get('pid'): r for r in kept}
130
+ lost = sorted(set(legacy_by) - set(kept_by))
131
+ gained = sorted(set(kept_by) - set(legacy_by))
132
+
133
+ value_diffs = []
134
+ for pid in sorted(set(legacy_by) & set(kept_by)):
135
+ a, b = legacy_by[pid], kept_by[pid]
136
+ for k in VALUE_KEYS:
137
+ if k in a and a.get(k) != b.get(k):
138
+ value_diffs.append({'pid': pid, 'key': k,
139
+ 'legacy': a.get(k), 'migrated': b.get(k)})
140
+ return {'ok': not lost and not gained and not value_diffs,
141
+ 'legacy_scope': legacy_scope, 'derived_scope': derived_scope,
142
+ 'lost': lost, 'gained': gained, 'value_diffs': value_diffs,
143
+ 'names': {r.get('pid'): r.get('customer') for r in legacy_rows
144
+ if r.get('pid') in set(lost)}}
145
+
146
+
147
+ def _fields_of(rows):
148
+ """A field list good enough to EVALUATE with, inferred from the assembled rows.
149
+
150
+ The permanent filters this module builds name `dba` (text) and `agent` (text) only, so the
151
+ inference is exact for them. It is deliberately not a schema: `permits()` needs types to
152
+ dispatch comparisons, and inferring `text` for an unknown key would be a guess with
153
+ consequences β€” so anything not recognised is left out, which makes its leaf unanswerable and
154
+ therefore DENYING rather than silently mistyped.
155
+ """
156
+ return [{'key': 'dba', 'type': 'text'}, {'key': 'agent', 'type': 'text'}]
platform/core/perm_scope.py ADDED
@@ -0,0 +1,286 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """core/perm_scope.py β€” the permission WALL for table modules (wave 15, C-PERM).
2
+
3
+ ONE place answers the three questions a restricted account raises on a grid surface:
4
+
5
+ may_access(user, module) may they open it at all?
6
+ visible_fields(fields, u, mod) which COLUMNS may they receive?
7
+ apply_row_scope(rows, u, mod, …) which ROWS may they receive?
8
+
9
+ plus one that exists only because of how the pool is built:
10
+
11
+ derive_pool_scope(user, module) which (team_id, agent) must the pool be BUILT with?
12
+
13
+ Both hosts call these β€” `aios-web/api` (`grid_assembly`) and `app.py` (`_table_grid`) β€” because
14
+ a wall that exists on one runtime and not the other is not a wall. `core/perms.py` stays what it
15
+ is (module GRANTS + the legacy BU derivation); this module is the row/field/pushdown layer that
16
+ sits on top, and it is deliberately a separate file so the legacy readers can keep their
17
+ semantics untouched while this one fails closed.
18
+
19
+ ────────────────────────────────────────────────────────────────────────────────────────────
20
+ THE RECORD
21
+
22
+ user['perms'] = {'<module_key>': {'access': bool,
23
+ 'filter': {'conj'?: 'and'|'or', 'nodes': [...]} | None,
24
+ 'hiddenFields': ['<field key>', ...]}}
25
+ user['perms_v'] = 1 stamped by the migration and by every write
26
+
27
+ `perms_v` is the EXPLICIT-RESOLUTION marker, and it is here because of `permissioning.md`
28
+ Part II gap #5: "`allowed_modules() is None` is fail-open by default … make 'resolved:
29
+ unrestricted' an explicit value so absence/uncertainty DENIES." The same class already shipped
30
+ twice in this codebase (`modules: []` and `bus: [<unknown id>]` both read as UNRESTRICTED β€”
31
+ `routes_admin.py`'s own docstring documents both). So:
32
+
33
+ * `perms_v` ABSENT β†’ the record is UN-MIGRATED, and the LEGACY wall applies unchanged
34
+ (`core.perms` module grants + the `bus`/`agent` query scope). That is not fail-open: it is
35
+ today's real wall, and it bounds the rollout window to "until the migration runs".
36
+ * `perms_v == 1` and the module has NO entry β†’ **DENY**. Absence now means what it says.
37
+ * `role == 'admin'` bypasses all of it β€” which is also what keeps BREAK-GLASS alive.
38
+ `deps._user_for` hands back a hardcoded master dict on a store outage
39
+ (`{'username':'admin','role':'admin','bus':'all','modules':'all'}`) that will never carry a
40
+ perms block; without this clause an explicit-marker scheme locks the owner out of their own
41
+ product at exactly the moment the store is broken.
42
+ """
43
+ import re
44
+
45
+ import core.perms as perms
46
+
47
+ #: `aios_grid._FORMULA_REF`'s pattern, restated rather than imported: this module is imported by
48
+ #: the API's request path and `aios_grid` pulls in the whole grid stack. Same regex, one line,
49
+ #: and `verify_api` asserts the two agree so it cannot drift into a different grammar.
50
+ _FORMULA_REF = re.compile(r"\{([^{}]*)\}")
51
+
52
+ PERMS_VERSION = 1
53
+
54
+
55
+ def _rec(user):
56
+ return user if isinstance(user, dict) else {}
57
+
58
+
59
+ def is_migrated(user):
60
+ """True once this record carries an explicit resolution. See the module docstring."""
61
+ return int(_rec(user).get('perms_v') or 0) >= PERMS_VERSION
62
+
63
+
64
+ def entry(user, module):
65
+ """This user's declared permissions for `module`, or None if nothing is declared.
66
+
67
+ None is AMBIGUOUS on purpose and every caller must resolve it against `is_migrated`:
68
+ on a migrated record it means DENY, on a legacy one it means "ask the old wall".
69
+ """
70
+ p = _rec(user).get('perms')
71
+ if not isinstance(p, dict):
72
+ return None
73
+ e = p.get(module)
74
+ return e if isinstance(e, dict) else None
75
+
76
+
77
+ def may_access(user, module):
78
+ """May this account open `module` at all? Fail-closed on a migrated record."""
79
+ if perms.is_admin(user):
80
+ return True
81
+ e = entry(user, module)
82
+ if e is not None:
83
+ return bool(e.get('access', True))
84
+ if is_migrated(user):
85
+ # Migrated and undeclared = denied. This is the whole point of the marker.
86
+ return False
87
+ return perms.may_open(user, module) # legacy record: the old grant wall
88
+
89
+
90
+ # ── FIELDS ───────────────────────────────────────────────────────────────────────────────────
91
+ def hidden_keys(user, module, fields):
92
+ """The TRANSITIVE closure of hidden field keys (C-PERM amendment 5).
93
+
94
+ β›” WHY A CLOSURE AND NOT A SET DIFFERENCE. A formula field is computed in the BROWSER
95
+ (`formulaEngine.ts`, injected by `computedRows`) from `{ref}`s into other columns, and a
96
+ measure column's value arrives precomputed in `derived`. So hiding field X has exactly three
97
+ possible outcomes and only one of them is coherent:
98
+
99
+ strip X, keep formulas β†’ every formula over X computes blank or wrong, silently
100
+ keep X's value for them β†’ X has leaked, wearing a formula's name
101
+ strip X AND its dependents→ the only honest answer
102
+
103
+ So a hidden field drags every formula that references it β€” and every formula that references
104
+ THAT formula, hence the fixpoint loop β€” out of the payload with it.
105
+
106
+ ⚠ This runs on every assembly, so it is a fixpoint over a handful of custom fields, not a
107
+ graph library. `MAX_PASSES` bounds a reference cycle the client would refuse to evaluate
108
+ anyway; without it a self-referential pair would spin here.
109
+ """
110
+ e = entry(user, module)
111
+ if perms.is_admin(user) or not e:
112
+ return frozenset()
113
+ hidden = {str(k) for k in (e.get('hiddenFields') or ()) if k}
114
+ if not hidden:
115
+ return frozenset()
116
+
117
+ refs = {}
118
+ for f in fields or ():
119
+ if not isinstance(f, dict) or not f.get('key'):
120
+ continue
121
+ expr = f.get('formula')
122
+ if isinstance(expr, str) and expr:
123
+ refs[f['key']] = {m.strip() for m in _FORMULA_REF.findall(expr) if m.strip()}
124
+
125
+ MAX_PASSES = 12
126
+ for _ in range(MAX_PASSES):
127
+ grew = False
128
+ for key, deps in refs.items():
129
+ if key not in hidden and deps & hidden:
130
+ hidden.add(key)
131
+ grew = True
132
+ if not grew:
133
+ break
134
+ return frozenset(hidden)
135
+
136
+
137
+ def visible_fields(fields, user, module):
138
+ """`fields` minus the hidden closure. Order preserved β€” the column order is the user's."""
139
+ hide = hidden_keys(user, module, fields)
140
+ if not hide:
141
+ return list(fields or ())
142
+ return [f for f in (fields or ())
143
+ if not (isinstance(f, dict) and f.get('key') in hide)]
144
+
145
+
146
+ def strip_row(row, hide):
147
+ """Drop hidden keys from ONE assembled row. Cheap enough to run per row, and it must run
148
+ per row: the field list and the row payload are two different wires, and stripping only the
149
+ first would leave the value sitting in the second where anyone can read it."""
150
+ if not hide or not isinstance(row, dict):
151
+ return row
152
+ return {k: v for k, v in row.items() if k not in hide}
153
+
154
+
155
+ # ── ROWS ─────────────────────────────────────────────────────────────────────────────────────
156
+ def apply_row_scope(rows, user, module, fields, ctx=None):
157
+ """The rows this account may receive: `filter_eval.permits` over the permanent filter.
158
+
159
+ `permits`, never `matches` β€” an unanswerable permanent filter DENIES rather than being
160
+ ignored. See `harness/filter_eval`'s docstring for the field-rename walkthrough that makes
161
+ the difference a leak rather than a preference.
162
+ """
163
+ if perms.is_admin(user):
164
+ return list(rows or ())
165
+ e = entry(user, module)
166
+ tree = (e or {}).get('filter')
167
+ if not tree:
168
+ return list(rows or ())
169
+ from harness import filter_eval as fe
170
+ return [r for r in (rows or ()) if fe.permits(tree, r, fields, ctx)]
171
+
172
+
173
+ # ── THE PUSHDOWN ─────────────────────────────────────────────────────────────────────────────
174
+ #: `dba` value sets that pin a BU, and the Odoo team they pin. `_dba_attrs` emits exactly
175
+ #: {'Fisch','Royal','Both'} (blank for "no brand attributable in 24 months"), so a permission
176
+ #: filter admitting Fisch admits {Fisch, Both} β€” a Both customer IS a Fisch customer.
177
+ _DBA_TEAM = ((frozenset({'fisch', 'both'}), 5), (frozenset({'royal', 'both'}), 6))
178
+
179
+ #: Only equality pushes down. `FILTER_OPS` is single-valued (there is no column-level "is any
180
+ #: of" β€” that vocabulary belongs to cohort leaves and is disjoint), so a multi-value BU
181
+ #: condition arrives as an OR GROUP of `eq` leaves, handled by `_group_dba_team` below.
182
+ _PUSHDOWN_OPS = frozenset({'eq'})
183
+
184
+
185
+ def derive_pool_scope(user, module):
186
+ """`(team_id, agent)` the POOL must be BUILT with, derived from the permanent filter.
187
+
188
+ β›” THIS EXISTS BECAUSE `team_id` SHAPES VALUES, NOT ROW MEMBERSHIP (C-PERM amendment 3).
189
+ `modules/customer_data._pool_build` passes `team_id` into `cust._cust_rev` three times (YTD,
190
+ LY, LTM) and into `cust._cadence_bulk`, so it decides what `rev`, `ly`, `ltm`, `aov`,
191
+ `est_missed` and the derived `status` MEAN. Enforce a BU purely as a post-filter and a
192
+ Fisch-only user keeps a correct-looking row LIST while every number on it silently becomes
193
+ Fisch+Royal β€” worst for `dba = Both` customers, who are exactly the ones a BU filter admits.
194
+ A pid-level reconciliation cannot see that; the values one can, and does.
195
+
196
+ So the query-level pushdown SURVIVES β€” but as a DERIVATION OF the permanent filter rather
197
+ than a second wall beside it, which is what keeps R1's "BU access is just a filter" true at
198
+ the level the owner asked for it (one declaration, one UI, one engine).
199
+
200
+ Pure, and recomputed per request rather than stored: a stored derivation drifts from the
201
+ filter it came from, and then two things disagree about what an account may see.
202
+
203
+ Reads TOP-LEVEL AND-conjunction leaves ONLY. A leaf under `or` guarantees no narrowing β€”
204
+ `dba is Fisch OR revenue > 10` must not pin the pool to Fisch β€” so it never pushes down.
205
+ Anything not recognised here simply is not pushed down; `apply_row_scope` still applies the
206
+ whole tree, so the wall is unchanged either way. Belt AND braces, deliberately: the pushdown
207
+ is what makes the VALUES right, `permits()` is what makes the ROWS right.
208
+ """
209
+ if perms.is_admin(user):
210
+ return None, None
211
+ e = entry(user, module)
212
+ tree = (e or {}).get('filter')
213
+ if not tree:
214
+ # Un-migrated records still answer through the legacy derivation, so the old wall keeps
215
+ # working until the migration has run.
216
+ if not is_migrated(user):
217
+ return perms.scope_team_id(user), perms.scope_agent(user)
218
+ return None, None
219
+
220
+ from harness import filter_eval as fe
221
+ nodes, conj = fe.tree_parts(tree)
222
+ if conj == 'or':
223
+ return None, None
224
+
225
+ team_id, agent = None, None
226
+ for n in nodes:
227
+ if not isinstance(n, dict):
228
+ continue
229
+ if isinstance(n.get('children'), list):
230
+ # A top-level OR GROUP under an AND root IS a guaranteed narrowing β€” every row must
231
+ # satisfy it β€” so it may push down, unlike a leaf under an OR ROOT (refused above).
232
+ # This is the shape a multi-value BU condition actually takes; see `_group_dba_team`.
233
+ tid = _group_dba_team(n)
234
+ if tid is not None:
235
+ team_id = tid if team_id in (None, tid) else None
236
+ continue
237
+ if n.get('op') not in _PUSHDOWN_OPS:
238
+ continue
239
+ col = n.get('colId')
240
+ raw = n.get('value')
241
+ if col == 'dba':
242
+ vals = {v.strip().lower() for v in str(raw or '').split(',') if v.strip()}
243
+ if not vals:
244
+ continue
245
+ for allowed, tid in _DBA_TEAM:
246
+ if vals <= allowed:
247
+ # Both BUs named = no narrowing to push; leave it to the post-filter.
248
+ team_id = tid if team_id in (None, tid) else None
249
+ break
250
+ elif col == 'agent':
251
+ v = str(raw or '').strip()
252
+ # A SET of agents cannot become the pool's single `agent_name`; the post-filter
253
+ # handles it. Only an unambiguous single value pushes down.
254
+ if v and ',' not in v:
255
+ agent = v
256
+ return team_id, agent
257
+
258
+
259
+ def _group_dba_team(group):
260
+ """The team a top-level `or` group pins, or None.
261
+
262
+ Recognises ONLY the exact shape "every child is a `dba eq <brand>` leaf" β€” the group the
263
+ condition builder emits for a multi-value BU condition, and the one `perm_migrate` writes.
264
+ Every OTHER group returns None and is left entirely to the post-filter: a group mixing `dba`
265
+ with another column, or containing a nested group, does not pin a BU on its own, and
266
+ guessing that it does would build the pool from the wrong book. Narrow by construction β€”
267
+ the pushdown may only ever be an OPTIMISATION of a constraint the filter already expresses.
268
+ """
269
+ if group.get('conj') != 'or':
270
+ return None
271
+ children = group.get('children') or []
272
+ if not children:
273
+ return None
274
+ vals = set()
275
+ for c in children:
276
+ if (not isinstance(c, dict) or isinstance(c.get('children'), list)
277
+ or c.get('colId') != 'dba' or c.get('op') != 'eq'):
278
+ return None
279
+ v = str(c.get('value') or '').strip().lower()
280
+ if not v:
281
+ return None
282
+ vals.add(v)
283
+ for allowed, tid in _DBA_TEAM:
284
+ if vals <= allowed:
285
+ return tid
286
+ return None
platform/core/perms.py ADDED
@@ -0,0 +1,215 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """core/perms.py β€” the permission + scope predicates as PURE functions of a user record.
2
+
3
+ EXIT-3b (2026-07-30). `ui/session.py` holds the same predicates keyed off `st.session_state`;
4
+ these take the user record explicitly, so the API can answer "what may this session see?" without
5
+ a Streamlit session. ONE implementation of each rule, two callers β€” and `verify_api.py` asserts
6
+ lock-step against `ui/session.py`'s source (the [[date-window-vocabulary]] discipline: a mirrored
7
+ vocabulary that is not gated has already drifted).
8
+
9
+ ⚠ THE TRAP THIS FILE EXISTS TO AVOID. `ui/session.brand_team()` reads the sidebar's BU selector
10
+ and defaults to `'All'` β†’ `team_id=None` β†’ CONSOLIDATED. Mirroring it literally on the API would
11
+ be a fail-OPEN: an API session has no selector, so a Royal-only user would be handed Fisch rows β€”
12
+ the exact leak strict isolation exists to prevent. The scope must be derived from the USER RECORD
13
+ (`bus`), never from a UI default. `scope_team_id()` below is that derivation.
14
+
15
+ Fail-closed everywhere: an unknown module key is denied, an unreadable `bus` narrows rather than
16
+ widens, and a user with no grant sees nothing rather than everything.
17
+ """
18
+ import core.context as ctxlib
19
+ import core.registry as registry
20
+ import core.users as users
21
+
22
+ # Old module keys migrate ON READ β€” grants, Library prefs and ?page= deep links keep working
23
+ # without ever editing a stored user record. MUST stay identical to ui/session._LEGACY_KEYS
24
+ # (gated in verify_api.py): 'myday' shipped before the Customer List rename, 'customer_list'
25
+ # before the 2026-07-26 rename to 'customer_data', 'map' joined in wave 8 when the Map PAGE
26
+ # became a VIEW of the customer table. A chain is fine only if every hop resolves to the CURRENT
27
+ # key, so 'myday' points at the END of the chain, not at the middle.
28
+ # ⚠ 'customers' is deliberately NOT here. It is a real registry key β€” the parent of
29
+ # customer_data + cohort β€” and may_open() gives a parent grant to its children. Mapping it away
30
+ # would silently narrow every "customers" grant to one child.
31
+ _LEGACY_KEYS = {'myday': 'customer_data', 'customer_list': 'customer_data',
32
+ 'map': 'customer_data', 'library': 'settings',
33
+ # wave 16 (item 5, R10): Cohort folded into the Customer rail as locked
34
+ # views β€” a cohort grant or ?page=cohort deep link lands on the grid that
35
+ # now hosts them.
36
+ 'cohort': 'customer_data',
37
+ # wave 16 (item 7, R11): the Agents PAGE is deleted; an 'agent'-granted
38
+ # login's surviving surface is the Customer grid over their OWN m2m book β€”
39
+ # the legacy query-scope (scope_agent reads the record's agent LINK, not
40
+ # this list) keeps that book exact, shared customers included. The R1
41
+ # migration deliberately SKIPS agent-linked records: `agent is X` cannot
42
+ # express m2m membership (a shared customer displays its primary agent),
43
+ # so their wall stays legacy until a membership predicate exists.
44
+ 'agent': 'customer_data',
45
+ # wave 17 (item 15, R2): the Collections PAGE is gone β€” its worklist is the
46
+ # shared "Collections" view on the Customer grid. An `ar` grant (or an
47
+ # ?page=ar deep link) lands on the surface that now does the job. The registry
48
+ # row survives as `validate_only`, so this is a grant migration, not a deletion:
49
+ # nobody loses access, they arrive somewhere that still exists.
50
+ 'ar': 'customer_data',
51
+ # wave 17 (item 13, R3): Procurement folded into the Product grid β€” the buy
52
+ # list is a view over formula fields there.
53
+ 'procurement': 'product_data'}
54
+
55
+
56
+ def is_admin(user):
57
+ return (user or {}).get('role') == 'admin'
58
+
59
+
60
+ def allowed_modules(user):
61
+ """Registry keys this user may open β€” None means ALL (admins / unrestricted users)."""
62
+ m = (user or {}).get('modules', 'all')
63
+ if not m or m == 'all':
64
+ return None
65
+ return {_LEGACY_KEYS.get(k, k) for k in m}
66
+
67
+
68
+ def may_open(user, key):
69
+ """True iff `user` may OPEN module `key`. A PARENT grant covers its sub-modules (granting
70
+ 'customers' grants 'customer_data'). Fail-closed β€” an unknown key stays denied."""
71
+ am = allowed_modules(user)
72
+ if am is None or key in am:
73
+ return True
74
+ parent = registry.BY_KEY.get(key, {}).get('parent')
75
+ return parent in am if parent else False
76
+
77
+
78
+ # ------------------------------------------------------------------ BU (business unit) scope
79
+ def allowed_bu_labels(user):
80
+ """The BU labels this user may see β€” `core.users.allowed_bus_labels`, which is already a
81
+ pure function of the record. Named here so callers have one import for the whole rule set."""
82
+ return users.allowed_bus_labels(user)
83
+
84
+
85
+ def allowed_team_ids(user):
86
+ """The concrete Odoo team_ids this user may see, as a frozenset. `All` expands to every BU."""
87
+ labels = allowed_bu_labels(user)
88
+ if 'All' in labels:
89
+ return frozenset(t for t in ctxlib.BRANDS.values() if t is not None)
90
+ return frozenset(ctxlib.BRANDS[l] for l in labels
91
+ if l in ctxlib.BRANDS and ctxlib.BRANDS[l] is not None)
92
+
93
+
94
+ def scope_team_id(user):
95
+ """The `team_id` an API request must query with β€” the ONE number that enforces BU isolation.
96
+
97
+ None means CONSOLIDATED, and it is returned ONLY when the user may genuinely see every BU
98
+ (`bus='all'`, or a multi-BU grant, both of which put 'All' in the label list). A user
99
+ permitted exactly one BU gets that BU's id PINNED, never None.
100
+
101
+ ⚠ This is the opposite default from `ui/session.brand_team()`, which falls back to 'All'
102
+ because a UI selector has to start somewhere. Here there is no selector to read, so the
103
+ fallback would BE the leak. A single-BU user's rows are pinned at the query, not filtered
104
+ afterwards, so there is no window in which the other BU's rows exist in the response.
105
+ """
106
+ labels = allowed_bu_labels(user)
107
+ if 'All' in labels:
108
+ return None
109
+ ids = sorted(allowed_team_ids(user))
110
+ return ids[0] if len(ids) == 1 else None
111
+
112
+
113
+ def scope_agent(user):
114
+ """The sales agent whose OWN BOOK this session is confined to, or None for the whole book.
115
+
116
+ ⚠ Mirrors `page_customer_data` (app.py), which reads `u.get('agent')` with NO admin
117
+ exemption β€” deliberately different from `page_agent`'s `None if is_admin() else …`. The
118
+ customer TABLE is the surface `/api/v1/customers` serves, so its rule is the one that
119
+ travels: an account linked to an agent sees that agent's book, admin or not. Getting this
120
+ wrong in the permissive direction would hand a restricted sales-agent login the whole book.
121
+ """
122
+ return (user or {}).get('agent') or None
123
+
124
+
125
+ # ------------------------------------------------------------------ nav projection
126
+ def nav_pages(user):
127
+ """The nav this user may see: registry order, `may_open`-filtered, archived EXCLUDED.
128
+
129
+ Shape (X2's `{key,label,source?,parent?}` plus ONE additive field, `chrome`):
130
+ [{key, label, source?, chrome: 'main'|'utility'}, …]
131
+
132
+ Four rules, each of which has bitten before:
133
+
134
+ * ARCHIVED IS INVISIBLE TO EVERYONE ([[module-archive-semantics]]) β€” not "hidden unless
135
+ admin". 18 of the 28 registry rows are archived and none may appear here.
136
+
137
+ * `group_only` ROWS ARE EXCLUDED ENTIRELY, and therefore **no `parent` is ever emitted**.
138
+ Wave-9 I8 flipped the host nav to a FLAT list: children became top-level and the folder
139
+ key was dropped, because a `group_only` key names a FAMILY and has no `PAGE_FUNCS` entry
140
+ β€” rendering a row for it is a white screen one click away (app.py:8182-8187 says exactly
141
+ this). `customers` is the only such row. Emitting `parent: 'customers'` on its children
142
+ while the parent itself is filtered out would hand the client a DANGLING reference to a
143
+ row that is not in the payload, which is worse than a flat list β€” so the flat list is
144
+ what ships, matching the surface a user actually sees today.
145
+
146
+ * `nav: False` IS NOT `archived`. `settings`, `analyst` and `dictionary` are LIVE surfaces
147
+ the host renders outside the module list (Analyst is the unrestricted landing page,
148
+ Settings sits in the chrome). Dropping them would leave the shell unable to reach two of
149
+ the nine live surfaces, so they ship flagged `chrome: 'utility'` and the client places
150
+ them where the host does. `chrome: 'main'` is the module list.
151
+
152
+ * NOT filtered by `ui/session.user_enabled_modules()`. That is a PREFERENCE read from
153
+ `platform/data/store/prefs.json` β€” a local file on one box. Consuming it here
154
+ would put per-user FILESYSTEM state inside a process EXIT-4 requires to be stateless,
155
+ and the same user's nav would differ per container. It is a Library setting, not a
156
+ permission; the permission gate is `may_open`, and that is what this applies.
157
+ """
158
+ out = []
159
+ for m in registry.REGISTRY:
160
+ if m.get('archived') or m.get('group_only'):
161
+ continue
162
+ # WAVE 17 (R2 + R8) β€” `validate_only` rows have NO SURFACE ANYWHERE. Not the module
163
+ # list, not the utility group behind the account menu, not a page. They exist so
164
+ # `validate.py` keeps proving contracts that a LIVE surface depends on:
165
+ # Β· `ar` β€” the Collections page is gone (its worklist is a saved view on the
166
+ # Customer grid now), but `ar.validate()` is what reconciles the
167
+ # AR columns that view is built from. Archiving the row would have
168
+ # skipped it (validate.py skips archived), deleting the proof while
169
+ # keeping the numbers β€” the worst of both.
170
+ # Β· `dictionary` β€” the semantic-layer contracts, the Analyst's grounding proof.
171
+ # `api_surface: False` is the NARROWER neighbour: the row still has a Streamlit page,
172
+ # but it is not a surface of the PRODUCT. `ar` is the case β€” its Collections worklist
173
+ # is a saved view on the Customer grid now, while the admin-only Statements sender
174
+ # (the one sanctioned Odoo writer) keeps its host page.
175
+ # ⚠ This is NOT `nav: False`. That flag means "a live surface the HOST renders outside
176
+ # the module list" and ships as `chrome: 'utility'`, which is exactly the account-menu
177
+ # row R8 says to delete.
178
+ if m.get('validate_only') or m.get('api_surface') is False:
179
+ continue
180
+ # WAVE 17 R8, the other half β€” `settings` STOPS being offered to the API's clients.
181
+ # It shipped as `chrome: 'utility'`, which the React shell rendered as "App settings":
182
+ # a hand-off that opens the STREAMLIT settings page in a new tab, beside the shell's
183
+ # own Profile modal that does the same job better. Two doors to one room, one of them
184
+ # leading out of the application. The owner's words were "I don't even know what it
185
+ # does β€” delete them".
186
+ # ⚠ The ROW STAYS, and so does the Streamlit page behind it: this host's own account
187
+ # menu still opens it, and the dispatch guard uses `settings` as its always-renderable
188
+ # last resort for a user whose whole grant has been retired. What changes is that the
189
+ # API no longer advertises it as a surface of the PRODUCT.
190
+ if m['key'] == 'settings':
191
+ continue
192
+ if not may_open(user, m['key']):
193
+ continue
194
+ row = {'key': m['key'], 'label': m['label'],
195
+ 'chrome': 'main' if m.get('nav', True) else 'utility'}
196
+ if m.get('source'):
197
+ row['source'] = m['source']
198
+ out.append(row)
199
+ return out
200
+
201
+
202
+ def landing_page(user):
203
+ """Where a restricted user lands: the first MAIN-chrome page in their grant. None for
204
+ unrestricted users (the client lands on the Analyst, as the host does).
205
+
206
+ Mirrors `ui/session.first_allowed_module` minus its archived last-resort leg β€” that leg
207
+ exists so a misconfigured Streamlit login is not bricked; an API caller gets an explicit
208
+ answer instead, and serving an archived key would contradict the rule above.
209
+ """
210
+ if allowed_modules(user) is None:
211
+ return None
212
+ for row in nav_pages(user):
213
+ if row['chrome'] == 'main':
214
+ return row['key']
215
+ return None
platform/core/record_comments.py ADDED
@@ -0,0 +1,155 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Durable comments attached to customer records.
2
+
3
+ The service is deliberately unaware of HTTP and of the concrete store backend.
4
+ It receives a tenant-bound ``TenantRuntime`` handle, so the same logical key can
5
+ never cross a tenant boundary. Odoo remains read-only: comments live only in
6
+ AIOS's writable tenant store.
7
+ """
8
+ from __future__ import annotations
9
+
10
+ import datetime as dt
11
+ import uuid
12
+
13
+ STORE_KEY = "customer_record_comments"
14
+ MAX_BODY = 4000
15
+ MAX_PER_RECORD = 500
16
+
17
+
18
+ class CommentsUnavailable(RuntimeError):
19
+ """The tenant store cannot currently provide durable comments."""
20
+
21
+
22
+ class CommentForbidden(RuntimeError):
23
+ """The caller tried to remove another user's comment."""
24
+
25
+
26
+ def _now_iso():
27
+ return dt.datetime.now(dt.timezone.utc).replace(microsecond=0).isoformat().replace("+00:00", "Z")
28
+
29
+
30
+ def _record_key(pid):
31
+ return str(int(pid))
32
+
33
+
34
+ def _clean_comment(raw):
35
+ if not isinstance(raw, dict):
36
+ return None
37
+ cid = str(raw.get("id") or "").strip()
38
+ body = str(raw.get("body") or "").strip()
39
+ author_key = str(raw.get("authorKey") or "").strip().lower()
40
+ author = str(raw.get("author") or author_key).strip()
41
+ created_at = str(raw.get("createdAt") or "").strip()
42
+ if not cid or not body or not author_key or not created_at:
43
+ return None
44
+ return {
45
+ "id": cid,
46
+ "body": body[:MAX_BODY],
47
+ "authorKey": author_key,
48
+ "author": author[:120] or author_key,
49
+ "createdAt": created_at,
50
+ }
51
+
52
+
53
+ def _read(runtime):
54
+ if not runtime.available():
55
+ raise CommentsUnavailable("the tenant store is unavailable")
56
+ try:
57
+ data = runtime.get(STORE_KEY)
58
+ except Exception as exc:
59
+ raise CommentsUnavailable("comments could not be read") from exc
60
+ return data if isinstance(data, dict) else {}
61
+
62
+
63
+ def list_comments(runtime, pid):
64
+ """Return one record's comments oldest-first, dropping malformed legacy entries."""
65
+ rows = _read(runtime).get(_record_key(pid), [])
66
+ if not isinstance(rows, list):
67
+ return []
68
+ cleaned = [comment for raw in rows if (comment := _clean_comment(raw)) is not None]
69
+ cleaned.sort(key=lambda comment: (comment["createdAt"], comment["id"]))
70
+ return cleaned
71
+
72
+
73
+ def add_comment(runtime, pid, body, author_key, author_name="", now=None, comment_id=None):
74
+ """Append one server-authored comment atomically and synchronously."""
75
+ text = str(body or "").strip()
76
+ if not text:
77
+ raise ValueError("write a comment before posting")
78
+ if len(text) > MAX_BODY:
79
+ raise ValueError(f"comments may be at most {MAX_BODY} characters")
80
+ uname = str(author_key or "").strip().lower()
81
+ if not uname:
82
+ raise ValueError("a signed-in author is required")
83
+ comment = {
84
+ "id": str(comment_id or uuid.uuid4().hex),
85
+ "body": text,
86
+ "authorKey": uname,
87
+ "author": str(author_name or uname).strip()[:120] or uname,
88
+ "createdAt": str(now or _now_iso()),
89
+ }
90
+ if not runtime.available():
91
+ raise CommentsUnavailable("the tenant store is unavailable")
92
+
93
+ def _append(data):
94
+ if not isinstance(data, dict):
95
+ data = {}
96
+ key = _record_key(pid)
97
+ rows = data.get(key)
98
+ if not isinstance(rows, list):
99
+ rows = []
100
+ if len(rows) >= MAX_PER_RECORD:
101
+ raise ValueError(f"a record may hold at most {MAX_PER_RECORD} comments")
102
+ rows.append(comment)
103
+ data[key] = rows
104
+ return data
105
+
106
+ try:
107
+ runtime.update(STORE_KEY, _append, flush="sync")
108
+ except ValueError:
109
+ raise
110
+ except Exception as exc:
111
+ raise CommentsUnavailable("the comment was not saved") from exc
112
+ return dict(comment)
113
+
114
+
115
+ def delete_comment(runtime, pid, comment_id, actor_key, admin=False):
116
+ """Delete a comment owned by ``actor_key`` (or by an administrator)."""
117
+ cid = str(comment_id or "").strip()
118
+ actor = str(actor_key or "").strip().lower()
119
+ if not cid:
120
+ return False
121
+ if not runtime.available():
122
+ raise CommentsUnavailable("the tenant store is unavailable")
123
+ state = {"found": False, "forbidden": False}
124
+
125
+ def _delete(data):
126
+ if not isinstance(data, dict):
127
+ return {}
128
+ key = _record_key(pid)
129
+ rows = data.get(key)
130
+ if not isinstance(rows, list):
131
+ return data
132
+ kept = []
133
+ for raw in rows:
134
+ comment = _clean_comment(raw)
135
+ if comment is None or comment["id"] != cid:
136
+ kept.append(raw)
137
+ continue
138
+ state["found"] = True
139
+ if not admin and comment["authorKey"] != actor:
140
+ state["forbidden"] = True
141
+ kept.append(raw)
142
+ if kept:
143
+ data[key] = kept
144
+ else:
145
+ data.pop(key, None)
146
+ return data
147
+
148
+ try:
149
+ runtime.update(STORE_KEY, _delete, flush="sync")
150
+ except Exception as exc:
151
+ raise CommentsUnavailable("the comment was not deleted") from exc
152
+ if state["forbidden"]:
153
+ raise CommentForbidden("only the author or an administrator may delete this comment")
154
+ return bool(state["found"])
155
+
platform/core/registry.py ADDED
@@ -0,0 +1,199 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Module registry β€” the single source of truth for the dashboards.
2
+
3
+ `app.py` builds the sidebar nav from this, and decides whether the global Brand (DBA) filter
4
+ applies to a page. `validate.py` runs each module's validate() from this. Adding a module =
5
+ add one row here + a page function in app.py + a module in modules/.
6
+
7
+ Fields:
8
+ key stable id (matches the modules/<key>.py file and the app page_<key> function)
9
+ label sidebar/title text (no emojis)
10
+ brand True -> DBA-filterable (honours the Fisch/Royal selector)
11
+ False -> consolidated/company-level (ignores the brand selector)
12
+ hq True -> HQ / company-level (cash flow, balance sheet, AR/AP, consolidated)
13
+ validate True -> module exposes validate() and is included in validate.py
14
+ parent set -> this row is a SUB-MODULE of <parent key>: it leaves the Workflows dropdown and
15
+ renders as a sub-nav button when its parent (or a sibling) is the active module; a
16
+ parent grant covers its sub-modules (owner nav redesign 2026-07-23)
17
+ archived True -> RETIRED from the UI for EVERYONE (owner 2026-07-27, supersedes the
18
+ 2026-07-23 per-user-restore semantics): no nav entry, no Settings>Modules checkbox,
19
+ prewarm and validate.py skip it. Code and data layer stay in the repo but are NOT
20
+ maintained β€” spend no effort on archived modules until the owner un-archives.
21
+ 2026-07-23 pass: backorders, pricecomp, o2c, bookings, spend, expenses.
22
+ 2026-07-27 pass (keep = Sales + Customers family + Agents + Procurement +
23
+ Collections + Analyst/plumbing): products, assortment, financial, management,
24
+ pricing, inventory, vendor, warehouse, returns, outreach, complexity, health.
25
+ note one-line scope note shown in the UI / docs
26
+ """
27
+ # WAVE-9 I8 β€” `'source'`: which connected system a module's data comes from.
28
+ #
29
+ # The owner's model: every nav entry is "a piece of database from a source", so the nav shows the
30
+ # source beside the name ("Sales Β· Odoo"). It is a REGISTRY FACT rather than a string in the nav
31
+ # code precisely so the nav needs no edit when a second connector lands β€” a user-created blank
32
+ # table simply carries a different source, and a module with no `source` (Metric Dictionary,
33
+ # Settings, the Analyst) renders no badge because it is not a database at all.
34
+ # See [[connector-onboarding]]: the plug-in seam is datastore.py, and Odoo has no delegated
35
+ # OAuth, so the "+ New" connect flow is the credential-form shape, not an OAuth redirect.
36
+ REGISTRY = [
37
+ # ARCHIVED wave 16 (owner item 7, R3/R11, 2026-08-02): the PAGE went on the strength of
38
+ # C-CHARTCAP β€” every Sales block shape (YoY compare series, KPI delta tiles, group-by
39
+ # tables) is now assemblable BY HAND from the grid's chart/dashboard views, which is what
40
+ # R3 required before deletion. `modules/sales.py` (the data layer) STAYS β€” drawers and the
41
+ # briefing still read it, and pages_sales.py survives UNREGISTERED as the Y1 envelope's
42
+ # template + verify_api's fixture.
43
+ {'key': 'sales', 'label': 'Sales', 'brand': True, 'hq': False, 'validate': True, 'source': 'Odoo', 'archived': True,
44
+ 'note': 'Revenue, YoY, seasonality, reps, customers, SKUs β€” by BU. Retired wave 16: '
45
+ 'rebuild as grid chart/dashboard views (compare series, KPI deltas, tables).'},
46
+ # ARCHIVED AS A PAGE (owner item 14, wave 8): "archive the current form of the Customer
47
+ # dashboard completely, i want to redesign it, exactly with the backend we have". So the
48
+ # PAGE goes and the DATA LAYER stays β€” modules/customers.py still powers the Customer table's
49
+ # metrics and still validates against Odoo, which is what the redesign will be built on.
50
+ #
51
+ # ⚠ NOT 'archived': True. Archived means invisible to everyone, and this row is the parent of
52
+ # customer_data + cohort β€” archiving it would take the whole family out of the nav and strand
53
+ # its children. 'group_only' says exactly what is true: this key names a FOLDER, never a
54
+ # destination. It has no PAGE_FUNCS entry, the nav never renders it as a leaf, and a stale
55
+ # deep link to it redirects to the family's first visible member.
56
+ {'key': 'customers', 'label': 'Customers', 'brand': True, 'hq': False, 'validate': True,
57
+ 'group_only': True,
58
+ 'note': 'Folder for the customer surfaces (Customer table, Cohort). The old Customers '
59
+ 'dashboard was retired for redesign (owner item 14, 2026-07-28); its metric layer '
60
+ '(modules/customers.py) still backs the table and still validates against Odoo.'},
61
+ # LABEL renamed 'Customer List' -> 'Data' (owner 2026-07-25) -> 'Customer' (owner item 5,
62
+ # 2026-07-27); KEY renamed 'customer_list' -> 'customer_data' (owner 2026-07-26). The key
63
+ # rename is safe ONLY because _LEGACY_KEYS (ui/session.py) migrates grants, Library prefs
64
+ # and ?page= deep links ON READ β€” stored user records are never edited, exactly as
65
+ # myday->customer_list was done. The STORE keys are deliberately NOT renamed:
66
+ # 'customer_table_workspace' and 'customer_lists' hold real user data, and renaming them
67
+ # would orphan every saved view. Label-only changes need none of that machinery.
68
+ {'key': 'customer_data', 'label': 'Customer', 'brand': True, 'hq': False, 'validate': False, 'source': 'Odoo', 'parent': 'customers',
69
+ 'note': 'Build, filter and save customer lists: the whole scoped book with per-customer '
70
+ 'metrics; saved lists are visible, editable formulas (Call list and Win-back ship '
71
+ 'as templates) plus hand-picked members. Sub-module of Customers (owner nav redesign 2026-07-23; replaced My Day).'},
72
+ # Cohort (owner item 6, 2026-07-26): the SAME table as Data over a FIXED set. The
73
+ # difference is membership semantics, not layout β€” a saved view re-populates as the
74
+ # data moves, a cohort only changes when a person edits it. validate:False because
75
+ # there is no Odoo aggregate to reconcile a hand-picked list against; its metrics are
76
+ # the customers module's, already validated there.
77
+ # ARCHIVED wave 16 (owner item 5, R10 β€” the fold-in): every cohort now projects as a
78
+ # LOCKED VIEW in the Customer rail's "Cohorts" section (wave-15 C-LOCK), so the separate
79
+ # nav destination is gone. ALL cohort machinery stays: `scope=cohort` still works
80
+ # (routes_grid._SCOPES, storage keys, events), page_cohort remains in app.py, and
81
+ # _LEGACY_KEYS maps cohort→customer_data so grants + ?page= deep links land on the grid
82
+ # that now hosts the cohorts.
83
+ {'key': 'cohort', 'label': 'Cohort', 'brand': True, 'hq': False, 'validate': False, 'source': 'Odoo', 'parent': 'customers', 'archived': True,
84
+ 'note': 'Hand-curated, unchanging customer lists β€” folded into the Customer rail as '
85
+ 'locked views (wave 16). The set is fixed: it changes only when someone adds '
86
+ 'or removes a member. Open them from Customer > Views > Cohorts.'},
87
+ # ARCHIVED wave 16 (owner item 7, R11, 2026-08-02) beside Sales. The agent DRAWER and the
88
+ # data layer (modules/agent.py, agent_* caches) STAY β€” agent entities still open from
89
+ # customer surfaces; an own-book agent login's home is the Customer grid narrowed by the
90
+ # R1 permanent filter ("Agent is X"), not this page.
91
+ {'key': 'agent', 'label': 'Agents', 'brand': True, 'hq': False, 'validate': True, 'source': 'Odoo', 'archived': True,
92
+ 'note': 'Per-agent book: sales (custom dates), returns, top SKUs, full customer list incl. inactive. '
93
+ 'Retired wave 16: agent drawers + the R1 own-book filter carry the use case.'},
94
+ # Wave 16 C-TOPIC (owner items 9+10, R4): the PRODUCT table family β€” the second object on
95
+ # the table-page factory. `products_family` is the folder head (the `customers` pattern:
96
+ # group_only, never a destination); `product_data` is the grid over the SKU dataset, with
97
+ # its OWN workspace bucket ('product_table_workspace') and the /products + scope=product
98
+ # seams. The archived Streamlit `products` (SKU) row below stays archived and untouched,
99
+ # exactly as R4 rules.
100
+ {'key': 'products_family', 'label': 'Products', 'brand': True, 'hq': False, 'validate': False,
101
+ 'group_only': True,
102
+ 'note': 'Folder for the product surfaces (Product table). The archived SKU dashboard is '
103
+ 'not part of this family.'},
104
+ {'key': 'product_data', 'label': 'Product', 'brand': True, 'hq': False, 'validate': True, 'source': 'Odoo', 'parent': 'products_family',
105
+ 'note': 'The SKU catalogue as a grid: per-product revenue, units and (consolidated) '
106
+ 'stock columns, with saved views and custom fields. Identity = the SKU code.'},
107
+ {'key': 'products', 'label': 'SKU', 'brand': True, 'hq': False, 'validate': True, 'archived': True,
108
+ 'note': 'SKU movers, zombies, velocity, coverage, drawers β€” by BU (sales-derived).'},
109
+ {'key': 'assortment', 'label': 'Assortment', 'brand': True, 'hq': False, 'validate': True, 'archived': True,
110
+ 'note': 'Facet-level performance (category/color/occasion/collection/season) + season readiness β€” by BU.'},
111
+ {'key': 'financial', 'label': 'Financial', 'brand': True, 'hq': False, 'validate': True, 'archived': True,
112
+ 'note': 'Gross margin by BU/category/SKU. Cash-conversion cycle is HQ-consolidated.'},
113
+ {'key': 'pricing', 'label': 'Pricing', 'brand': True, 'hq': False, 'validate': True, 'archived': True,
114
+ 'note': 'Per-SKU margin/markup/LTM sales + allocated net P&L (channel-rate cost-to-SKU) for pricing decisions.'},
115
+ {'key': 'management', 'label': 'Management P&L', 'brand': False, 'hq': True, 'validate': True, 'archived': True,
116
+ 'nav': False, # rendered INSIDE the Financial module (view toggle) β€” archived WITH financial
117
+ 'note': 'Operating-model management income statements (FFS/RI/GD/BU-M) reproduced from raw Odoo, Sep-2024+.'},
118
+ {'key': 'inventory', 'label': 'Inventory', 'brand': False, 'hq': False, 'validate': True, 'archived': True,
119
+ 'note': 'On-hand stock is physically consolidated (shared warehouse); not DBA-split.'},
120
+ {'key': 'procurement', 'label': 'Procurement', 'brand': False, 'hq': False, 'validate': True, 'source': 'Odoo',
121
+ 'validate_only': True,
122
+ 'note': 'PAGE RETIRED 2026-08-03 (owner wave-17 item 13: "We should be able to replace '
123
+ 'Procurement completely, and add it as part of the Product database"). The buy '
124
+ 'list is now a saved VIEW on the Product grid, filtered on a FORMULA field over '
125
+ 'the supplier/lead-time columns and the demand measure β€” the owner\'s ruling R3: '
126
+ '"Buy list is just a View, with a Filter from a Formula field that taps into '
127
+ 'Metrics Fields... the 8-month demand baseline etc. is just math in a Formula '
128
+ 'field." The ROW STAYS validate_only: `modules/procurement.py` still owns the '
129
+ 'supplier master map (procurement_suppliers.json, 3,963 SKUs) that seeds those '
130
+ 'columns, and its validate() is the proof they reconcile.'},
131
+ {'key': 'vendor', 'label': 'Vendors', 'brand': False, 'hq': False, 'validate': False, 'archived': True,
132
+ 'note': 'Vendor master data + the SKUs each supplies; multiple-vendor / cheapest-price view.'},
133
+ {'key': 'warehouse', 'label': 'Warehouse', 'brand': False, 'hq': False, 'validate': True, 'archived': True,
134
+ 'note': 'Movement & efficiency: on-time ship, Pick-Pack-Ship cycle times, throughput, backlog age, shrinkage. Physical ops (consolidated).'},
135
+ {'key': 'ar', 'label': 'Collections', 'brand': False, 'hq': True, 'validate': True,
136
+ 'source': 'Odoo', 'api_surface': False, 'admin_only': True,
137
+ 'note': 'PAGE RETIRED 2026-08-03 (owner wave-17 item 15: "Collections should also be '
138
+ 'entirely replicable as just a View under Customer"). The worklist is the shared '
139
+ '"Collections" view on the Customer grid β€” same numbers, from this module\'s own '
140
+ 'reconciled blocks (ar_open/ar_overdue/ar_exposure/days_to_pay + the four aging '
141
+ 'buckets). The ROW STAYS validate_only so validate.py keeps running ar.validate(), '
142
+ 'which is the proof those columns rest on; archiving it would have skipped the '
143
+ 'reconciliation while the numbers kept shipping. ⚠ NOT `validate_only`, and the '
144
+ 'difference is a LIVE WORKFLOW: this page also mounts the admin-gated STATEMENTS '
145
+ 'sender (app._collections_statements -> modules/collections_send), the ONE '
146
+ 'sanctioned Odoo writer, which the same ruling says stays untouched. So the row '
147
+ 'keeps a Streamlit page for ADMINS ONLY (`admin_only`) and leaves the API payload '
148
+ 'entirely (`api_surface: False`) β€” no "Collections" in the React nav, no second '
149
+ 'worklist, and the biweekly send keeps its door.'},
150
+ {'key': 'returns', 'label': 'Returns', 'brand': False, 'hq': True, 'validate': True, 'archived': True,
151
+ 'note': 'Credit-note lens: refund concentration by SKU (quality) and customer (behavior). Company-level.'},
152
+ {'key': 'outreach', 'label': 'Outreach', 'brand': True, 'hq': False, 'validate': False, 'archived': True,
153
+ 'note': 'Campaign emails to customer segments: templates, suppression, send log, revenue attribution. Sending is admin-gated behind SAFE_MODE.'},
154
+ {'key': 'backorders', 'label': 'Backorders', 'brand': True, 'hq': False, 'validate': True, 'archived': True,
155
+ 'note': 'Confirmed-undelivered order lines aged vs promise date, valued, with a supply-aware next action per row (ship / expedite / call). Wholesale scope.'},
156
+ {'key': 'pricecomp', 'label': 'Price Compliance', 'brand': True, 'hq': False, 'validate': True, 'archived': True,
157
+ 'note': 'Selling below the customer pricelist tier (LTM, per customer x SKU): the pocket-price floor worklist with annualized leak $. Sub-30% ratios usually mean a stale or pack-basis RULE β€” fix the rule, not the rep.'},
158
+ {'key': 'o2c', 'label': 'Cash Timing', 'brand': True, 'hq': False, 'validate': True, 'archived': True,
159
+ 'note': 'Order-to-cash stage decomposition (order-to-ship / ship-to-invoice / invoice-to-paid, each with its owner) + the terms-gap rollup: contractual vs actual days per payment term with the free-credit $ it strands.'},
160
+ {'key': 'bookings', 'label': 'Order Book', 'brand': True, 'hq': False, 'validate': True, 'archived': True,
161
+ 'note': 'Pre-season booking coverage: cumulative booked $ for Aug-Dec delivery vs last year same-week; category TY-vs-LY-as-of-date. The container-program decision, months ahead.'},
162
+ {'key': 'spend', 'label': 'Spend & Payables', 'brand': False, 'hq': True, 'validate': True, 'archived': True,
163
+ 'note': 'Vendor-bill side: payment-terms capture (early-pay cash surrendered), duplicate-bill review, the opex spend cube (fragmentation, consolidate+rebid) and freight recovery. Company-level.'},
164
+ {'key': 'complexity', 'label': 'SKU Complexity', 'brand': False, 'hq': False, 'validate': True, 'archived': True,
165
+ 'note': 'BCG tail rationalization: every SKU re-costed by ACTIVITY (lines, picks, returns) + 25%/yr carrying; KILL candidates rest on hard math (GM minus carrying), REVIEW on the pooled activity estimate. All-channel.'},
166
+ {'key': 'expenses', 'label': 'Expenses', 'brand': False, 'hq': True, 'validate': True, 'archived': True,
167
+ 'note': 'Operating expense from the GL (all expense-type accounts; COGS excluded): trend, operating leverage (opex % of revenue), the YoY cost bridge, XmR control-limit spike watch list, fixed/variable split and a drill-to-ledger category directory. Company-level.'},
168
+ {'key': 'health', 'label': 'Data Health', 'brand': False, 'hq': True, 'validate': True, 'archived': True,
169
+ 'note': 'Close/reconciliation scan; period-filtered, mostly company-level. ARCHITECTURE '
170
+ '(owner 2026-07-23): every discrepancy / potential-error marker lives here β€” '
171
+ 'procurement mapping gaps, count-trust (unverified on-hand counts), untracked-on-'
172
+ 'order β€” so operational workflows stay clean for doing the work.'},
173
+ {'key': 'dictionary', 'label': 'Metric Dictionary', 'brand': False, 'hq': True, 'validate': True,
174
+ 'nav': False, 'validate_only': True,
175
+ 'note': 'PAGE RETIRED 2026-07-23 (owner: broken/not customer-facing) β€” row kept ONLY so '
176
+ 'validate.py keeps running the semantic-layer contracts (the Analyst grounding '
177
+ 'proof). No nav, no page. Wave 17 R8 ("I don\'t even know what it does β€” delete '
178
+ 'them") made that literal: `validate_only` takes it out of the account menu too, '
179
+ 'which is the last place it was still visible. The proof survives; the door does not.'},
180
+ {'key': 'automation', 'label': 'Automation', 'brand': False, 'hq': True, 'validate': False,
181
+ 'nav': True,
182
+ 'note': 'Wave 18 (C-AUTONAV): the Automation surface β€” scheduled jobs that create and '
183
+ 'refresh user databases (website scrape-to-DB, the Instagram field). React-only '
184
+ 'surface (no PAGE_FUNCS entry, the no-new-Streamlit rule); admins hold it via '
185
+ '"all", other users need the explicit grant β€” fail-closed default.'},
186
+ {'key': 'settings', 'label': 'Settings', 'brand': False, 'hq': True, 'validate': False,
187
+ 'nav': False, # sidebar sentinel (account group) β€” owner IA 2026-07-12
188
+ 'note': 'User scope settings: the Business Unit toggle (strict isolation) and the Data basis '
189
+ 'toggle (Orders vs Invoiced) moved here from the sidebar.'},
190
+ {'key': 'analyst', 'label': 'AIOS Analyst', 'brand': False, 'hq': True, 'validate': False,
191
+ 'nav': False, # sidebar sentinel button; eval gate = harness/evals.py (run pre-ship, NOT in validate.py β€” live LLM cost)
192
+ 'note': 'Ask the business a question in plain language: a small AI model calls governed tools over the semantic layer β€” answers carry their query trace and drill links. AI-generated output (Art. 50 labeled).'},
193
+ ]
194
+
195
+ BY_KEY = {m['key']: m for m in REGISTRY}
196
+
197
+
198
+ def brand_filterable(key) -> bool:
199
+ return bool(BY_KEY.get(key, {}).get('brand'))
platform/core/store.py ADDED
@@ -0,0 +1,369 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Tiny JSON persistence on a PRIVATE Hugging Face Dataset repo β€” the platform's ONLY writable store
2
+ (Odoo stays strictly read-only). Holds the user registry, customer notes and per-user tags.
3
+
4
+ Auth uses HF_TOKEN (a Space secret / .env value with write access to the royal-imports org). The
5
+ repo is created private on first write. Reads use a short in-process cache that writes refresh, so
6
+ within a running instance reads are consistent with writes; a fresh instance re-fetches the latest.
7
+
8
+ WAVE 18 (C1-TENANT, R2): the store is now a CLASS, one instance per repo. The module-level
9
+ functions below delegate to the default instance (`OS_DATA_REPO`, tenant #0) β€” every existing
10
+ `import core.store as store` caller is byte-for-byte unaffected, which the gate battery proves.
11
+ `for_repo(repo_id)` hands back the bound instance for a tenant that owns its OWN dataset repo
12
+ (nurilab, gtmlab, …); `harness.runtime.TenantRuntime` binds it from the tenant record. Each
13
+ instance carries its own cache / dirty-marks / flush workers / commit-gap clock, so one tenant's
14
+ write burst can never starve another's flush window.
15
+ """
16
+ import io
17
+ import os
18
+ import json
19
+ import threading
20
+ import time as _time
21
+ from pathlib import Path
22
+
23
+ try:
24
+ from dotenv import load_dotenv
25
+ load_dotenv(Path(__file__).resolve().parents[1] / '.env')
26
+ except Exception:
27
+ pass
28
+
29
+ from huggingface_hub import HfApi, hf_hub_download
30
+ from huggingface_hub.utils import EntryNotFoundError, RepositoryNotFoundError
31
+
32
+ REPO = os.environ.get('OS_DATA_REPO', 'royal-imports/cfo-os-data')
33
+ _FLUSH_DELAY = 2.0 # coalesce a burst of edits into one commit
34
+ _FLUSH_MIN_GAP = 20.0 # floor between commits of one key (the 256-commits/hr budget)
35
+
36
+
37
+ def _token():
38
+ return os.environ.get('HF_TOKEN') or None
39
+
40
+
41
+ class Store:
42
+ """One repo's persistence: JSON-per-key + raw byte paths, with the wave-7 async flush.
43
+
44
+ Every method body is the pre-wave-18 module function, verbatim except globals→fields.
45
+ The behaviour contracts that matter are stated on each method and are UNCHANGED:
46
+ strict-read-guards-the-write, exists()-is-True-on-uncertainty, lenient display reads.
47
+ """
48
+
49
+ def __init__(self, repo):
50
+ self.repo = str(repo)
51
+ self._lock = threading.RLock()
52
+ self._cache = {}
53
+ # --- async flush state (wave-7 W3, 2026-07-28) ------------------------------------
54
+ # One typed character in a Notes cell used to cost THREE synchronous network calls
55
+ # inside the component round-trip. update(..., flush='async') applies the mutation to
56
+ # the in-process cache synchronously (read-your-writes: every reader consults the
57
+ # cache first) and a background worker coalesces uploads. The restart-wipe protection
58
+ # is UNCHANGED where it matters: the FIRST update of a key in a fresh process still
59
+ # strict-reads. Known, accepted: with a second concurrent writer the stale window
60
+ # widens from ~one round-trip to ~the flush gap.
61
+ self._owned = {} # name -> True once cache reflects our own last accepted write
62
+ self._dirty = {} # name -> cache is ahead of the hub
63
+ self._flushers = {} # name -> live worker thread
64
+ self._repo_ok = False # create_repo(exist_ok=True) confirmed once per instance
65
+ self._last_flush = {} # name -> monotonic time of the last completed upload
66
+
67
+ def available(self):
68
+ """True when a token is configured β€” callers degrade gracefully without one."""
69
+ return bool(_token())
70
+
71
+ def _download(self, name):
72
+ """Parse {name}.json into a dict. Returns {} ONLY when the file is genuinely absent.
73
+ Any other failure (network, auth, rate-limit, corrupt JSON) RAISES β€” a transient read
74
+ error must never be mistaken for an empty store, or a caller could overwrite real data
75
+ with an empty dict (this is what wiped the user registry on restart)."""
76
+ try:
77
+ path = hf_hub_download(self.repo, f'{name}.json', repo_type='dataset',
78
+ token=_token(), force_download=True)
79
+ with open(path, encoding='utf-8') as f:
80
+ return json.load(f)
81
+ except (EntryNotFoundError, RepositoryNotFoundError, FileNotFoundError):
82
+ return {}
83
+ except ValueError as e:
84
+ # Wave 18 (R2, found by SESSION D): current huggingface_hub wraps a missing REPO in
85
+ # ValueError("Force download failed…") with the 404 as __cause__ β€” so a brand-new
86
+ # tenant repo's very first strict read raised here and aborted its very first write.
87
+ # A genuinely-absent repo is the same fact as a genuinely-absent file: {}.
88
+ cause = getattr(e, '__cause__', None)
89
+ if isinstance(cause, (EntryNotFoundError, RepositoryNotFoundError)):
90
+ return {}
91
+ raise
92
+
93
+ def get(self, name, fresh=False):
94
+ """Load {name}.json as a dict. LENIENT (for display reads): on a transient fetch error,
95
+ hand back the last known-good cached value if we have one, else {}. Mutating callers go
96
+ through update()/_read_strict, which RAISE instead."""
97
+ with self._lock:
98
+ if not fresh and name in self._cache:
99
+ return json.loads(json.dumps(self._cache[name])) # hand back a copy
100
+ try:
101
+ data = self._download(name)
102
+ except Exception as e:
103
+ try: # lazy import β€” core stays layer-clean
104
+ import harness.telemetry as _tel # noqa: PLC0415
105
+ _tel.error(f'store:get:{name}', e,
106
+ fallback='cached' if name in self._cache else 'empty')
107
+ except Exception:
108
+ pass
109
+ if name in self._cache:
110
+ return json.loads(json.dumps(self._cache[name]))
111
+ return {}
112
+ self._cache[name] = data
113
+ return json.loads(json.dumps(data))
114
+
115
+ def _read_strict(self, name):
116
+ """Like get(fresh=True) but RAISES on any transient read error ({} only for a genuinely
117
+ absent file), so a failed read ABORTS a read-modify-write instead of merging into {}."""
118
+ with self._lock:
119
+ data = self._download(name)
120
+ self._cache[name] = data
121
+ return json.loads(json.dumps(data))
122
+
123
+ def exists(self, name):
124
+ """True if {name}.json is confirmed present, False if confirmed absent, and True on ANY
125
+ uncertainty β€” a seeder must never clobber a registry we merely failed to reach."""
126
+ try:
127
+ files = HfApi(token=_token()).list_repo_files(self.repo, repo_type='dataset')
128
+ return f'{name}.json' in files
129
+ except RepositoryNotFoundError:
130
+ return False
131
+ except Exception:
132
+ return True
133
+
134
+ def _upload(self, name, data):
135
+ """The raw hub write. create_repo is confirmed ONCE per instance β€” it was a full extra
136
+ API round-trip on every put(), pure latency after the first success."""
137
+ api = HfApi(token=_token())
138
+ if not self._repo_ok:
139
+ api.create_repo(self.repo, repo_type='dataset', private=True, exist_ok=True)
140
+ self._repo_ok = True
141
+ buf = io.BytesIO(json.dumps(data, ensure_ascii=False, indent=2).encode('utf-8'))
142
+ api.upload_file(path_or_fileobj=buf, path_in_repo=f'{name}.json', repo_id=self.repo,
143
+ repo_type='dataset', commit_message=f'update {name}')
144
+
145
+ def upload_bytes(self, path_in_repo, data, message=None):
146
+ """Persist RAW BYTES at an arbitrary path in the tenant dataset (wave-8 C5, documents).
147
+ Bytes are never cached in-process: large, cold, fetched on explicit user action."""
148
+ if not self.available():
149
+ raise RuntimeError('No HF_TOKEN configured β€” persistence is unavailable.')
150
+ with self._lock:
151
+ api = HfApi(token=_token())
152
+ if not self._repo_ok:
153
+ api.create_repo(self.repo, repo_type='dataset', private=True, exist_ok=True)
154
+ self._repo_ok = True
155
+ api.upload_file(path_or_fileobj=io.BytesIO(data), path_in_repo=path_in_repo,
156
+ repo_id=self.repo, repo_type='dataset',
157
+ commit_message=message or f'add {path_in_repo}')
158
+
159
+ def download_bytes(self, path_in_repo):
160
+ """Raw bytes back, or None when absent/unreachable β€” a missing attachment degrades to
161
+ "that file is gone", never to a broken page."""
162
+ if not self.available():
163
+ return None
164
+ try:
165
+ local = hf_hub_download(self.repo, path_in_repo, repo_type='dataset',
166
+ token=_token())
167
+ return Path(local).read_bytes()
168
+ except (EntryNotFoundError, RepositoryNotFoundError):
169
+ return None
170
+ except Exception:
171
+ return None
172
+
173
+ def delete_path(self, path_in_repo):
174
+ """Remove a stored file. Absent is SUCCESS β€” deleting what is already gone is the goal."""
175
+ if not self.available():
176
+ return False
177
+ try:
178
+ HfApi(token=_token()).delete_file(path_in_repo=path_in_repo, repo_id=self.repo,
179
+ repo_type='dataset',
180
+ commit_message=f'delete {path_in_repo}')
181
+ return True
182
+ except (EntryNotFoundError, RepositoryNotFoundError):
183
+ return True
184
+ except Exception:
185
+ return False
186
+
187
+ def put(self, name, data):
188
+ """Persist {name}.json (creates the private repo on first write). Updates the cache."""
189
+ if not self.available():
190
+ raise RuntimeError('No HF_TOKEN configured β€” persistence is unavailable.')
191
+ with self._lock:
192
+ self._upload(name, data)
193
+ self._cache[name] = json.loads(json.dumps(data))
194
+ self._owned[name] = True
195
+ self._last_flush[name] = _time.monotonic()
196
+ return self._cache[name]
197
+
198
+ def _flush_worker(self, name):
199
+ """Coalescing uploader for one key. Runs while dirty; retries with backoff on failure β€”
200
+ the cache stays the truth and the dirty mark survives, so nothing is lost silently."""
201
+ backoff = 2.0
202
+ while True:
203
+ _time.sleep(_FLUSH_DELAY)
204
+ with self._lock:
205
+ gap = _time.monotonic() - self._last_flush.get(name, 0.0)
206
+ if gap < _FLUSH_MIN_GAP:
207
+ wait = _FLUSH_MIN_GAP - gap
208
+ elif not self._dirty.get(name):
209
+ self._flushers.pop(name, None)
210
+ return
211
+ else:
212
+ self._dirty[name] = False
213
+ snapshot = json.loads(json.dumps(self._cache.get(name, {})))
214
+ wait = None
215
+ if wait is not None:
216
+ _time.sleep(wait)
217
+ continue
218
+ try:
219
+ self._upload(name, snapshot)
220
+ with self._lock:
221
+ self._last_flush[name] = _time.monotonic()
222
+ backoff = 2.0
223
+ except Exception as e:
224
+ with self._lock:
225
+ self._dirty[name] = True # keep it queued; cache is still the truth
226
+ try:
227
+ import harness.telemetry as _tel # noqa: PLC0415
228
+ _tel.error(f'store:flush:{name}', e)
229
+ except Exception:
230
+ pass
231
+ _time.sleep(backoff)
232
+ backoff = min(backoff * 2, 60.0)
233
+
234
+ def _schedule_flush(self, name):
235
+ """Mark dirty + ensure exactly one live worker for the key. Caller holds the lock."""
236
+ self._dirty[name] = True
237
+ t = self._flushers.get(name)
238
+ if t is not None and t.is_alive():
239
+ return
240
+ t = threading.Thread(target=self._flush_worker, args=(name,), daemon=True,
241
+ name=f'store-flush-{self.repo}:{name}')
242
+ self._flushers[name] = t
243
+ t.start()
244
+
245
+ def flush(self, name=None, timeout=30.0):
246
+ """Block until pending async writes for `name` (or ALL keys) reach the hub β€” QA gates
247
+ and shutdown hooks use this; the app itself never needs to."""
248
+ deadline = _time.monotonic() + timeout
249
+ names = [name] if name else None
250
+ while _time.monotonic() < deadline:
251
+ with self._lock:
252
+ pending = {k for k, v in self._dirty.items() if v} | {
253
+ k for k, t in self._flushers.items() if t.is_alive()}
254
+ if names is not None:
255
+ pending &= set(names)
256
+ if not pending:
257
+ return True
258
+ _time.sleep(0.2)
259
+ return False
260
+
261
+ def _flush_now_at_exit(self):
262
+ """Process-exit safety net: push any dirty keys synchronously, bypassing the coalescing
263
+ gap β€” a Space restart must not eat the last seconds of the async window."""
264
+ with self._lock:
265
+ dirty = [k for k, v in self._dirty.items() if v]
266
+ snaps = {k: json.loads(json.dumps(self._cache.get(k, {}))) for k in dirty}
267
+ for k in dirty:
268
+ self._dirty[k] = False
269
+ for k, d in snaps.items():
270
+ try:
271
+ self._upload(k, d)
272
+ except Exception:
273
+ pass # exiting anyway β€” nothing left to reschedule
274
+
275
+ def update(self, name, fn, flush='sync'):
276
+ """Read-modify-write. STRICT first read (raises on a transient error rather than
277
+ starting from an empty dict), apply fn(data)->data, persist.
278
+
279
+ flush='sync' (default) = fresh strict read + blocking upload. flush='async' (the
280
+ table-workspace hot path, wave-7 W3) = the mutation applies to the in-process cache and
281
+ returns immediately; a coalescing worker commits. The strict read still guards the
282
+ FIRST async update of a key per process."""
283
+ with self._lock:
284
+ if flush == 'async' and self._owned.get(name) and name in self._cache:
285
+ data = json.loads(json.dumps(self._cache[name]))
286
+ else:
287
+ data = self._read_strict(name)
288
+ result = fn(data)
289
+ data = result if result is not None else data
290
+ if flush == 'async':
291
+ if not self.available():
292
+ raise RuntimeError('No HF_TOKEN configured β€” persistence is unavailable.')
293
+ self._cache[name] = json.loads(json.dumps(data))
294
+ self._owned[name] = True
295
+ self._schedule_flush(name)
296
+ return data
297
+ self.put(name, data)
298
+ return data
299
+
300
+
301
+ # ------------------------------------------------------------------- instances + module API
302
+ _INSTANCES = {}
303
+ _INSTANCES_LOCK = threading.RLock()
304
+
305
+
306
+ def for_repo(repo_id):
307
+ """The Store bound to `repo_id` β€” one instance per repo per process (each with its own
308
+ cache and flush state). The default instance IS `for_repo(REPO)`."""
309
+ rid = str(repo_id or '').strip() or REPO
310
+ with _INSTANCES_LOCK:
311
+ inst = _INSTANCES.get(rid)
312
+ if inst is None:
313
+ inst = Store(rid)
314
+ _INSTANCES[rid] = inst
315
+ return inst
316
+
317
+
318
+ _DEFAULT = for_repo(REPO)
319
+
320
+
321
+ def available():
322
+ return _DEFAULT.available()
323
+
324
+
325
+ def get(name, fresh=False):
326
+ return _DEFAULT.get(name, fresh=fresh)
327
+
328
+
329
+ def _read_strict(name):
330
+ return _DEFAULT._read_strict(name)
331
+
332
+
333
+ def exists(name):
334
+ return _DEFAULT.exists(name)
335
+
336
+
337
+ def upload_bytes(path_in_repo, data, message=None):
338
+ return _DEFAULT.upload_bytes(path_in_repo, data, message=message)
339
+
340
+
341
+ def download_bytes(path_in_repo):
342
+ return _DEFAULT.download_bytes(path_in_repo)
343
+
344
+
345
+ def delete_path(path_in_repo):
346
+ return _DEFAULT.delete_path(path_in_repo)
347
+
348
+
349
+ def put(name, data):
350
+ return _DEFAULT.put(name, data)
351
+
352
+
353
+ def flush(name=None, timeout=30.0):
354
+ return _DEFAULT.flush(name=name, timeout=timeout)
355
+
356
+
357
+ def update(name, fn, flush='sync'):
358
+ return _DEFAULT.update(name, fn, flush=flush)
359
+
360
+
361
+ def _flush_all_at_exit():
362
+ with _INSTANCES_LOCK:
363
+ instances = list(_INSTANCES.values())
364
+ for inst in instances:
365
+ inst._flush_now_at_exit()
366
+
367
+
368
+ import atexit # noqa: E402 (registered after the class it needs)
369
+ atexit.register(_flush_all_at_exit)
platform/core/store_backend.py ADDED
@@ -0,0 +1,85 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """core/store_backend.py β€” which store is active: `STORE_BACKEND=hf|pg` (X4 / EXIT-2b).
2
+
3
+ import core.store_backend as store # new code
4
+ store.get('users') # goes to whichever backend is selected
5
+
6
+ β›” WHAT THIS DOES **NOT** DO, stated first because assuming otherwise would be a silent data
7
+ split. Flipping `STORE_BACKEND=pg` does NOT redirect the ~40 existing callers that say
8
+ `import core.store as store` β€” they are bound to the HF module directly. Rewiring them is task
9
+ **C-4** (the HF→Postgres cutover with its dual-read window), and C-4 cannot start until the
10
+ database exists (owner blocker **B-3**). Until then this selector is the SEAM, exercised by new
11
+ code and by the gate, and the default is `hf`, which is exactly today's behaviour.
12
+
13
+ Doing it any other way would be worse. The tempting shortcut β€” have `core/store.py` itself
14
+ delegate β€” would mean every existing caller silently changes backend the moment an env var is set
15
+ in some shell, with no dual-read window and no way to compare the two stores' contents first.
16
+ "One store" is the goal; getting there through a flag nobody audited would produce two.
17
+
18
+ THE DEFAULT IS `hf` AND STAYS `hf` until C-4 says otherwise. An unrecognised value is a
19
+ configuration error and RAISES rather than falling back β€” a typo'd backend name that quietly
20
+ served the old store would be discovered by data going missing.
21
+ """
22
+ import os
23
+
24
+ import core.store as _hf
25
+
26
+ _NAMES = ('available', 'get', 'exists', 'put', 'update', 'upload_bytes', 'download_bytes',
27
+ 'delete_path', 'flush')
28
+
29
+
30
+ def name():
31
+ """The selected backend name, validated. `hf` unless explicitly told otherwise."""
32
+ raw = (os.environ.get('STORE_BACKEND') or 'hf').strip().lower()
33
+ if raw not in ('hf', 'pg'):
34
+ raise RuntimeError(
35
+ f"STORE_BACKEND={raw!r} is not a backend. Use 'hf' (the HF Dataset store, the "
36
+ f"default) or 'pg' (Postgres, needs DATABASE_URL). Refusing to guess: a typo that "
37
+ f"silently served the other store is how data ends up in two places.")
38
+ return raw
39
+
40
+
41
+ def active():
42
+ """The backend MODULE. Resolved per call, so a test can flip the env var without a reimport."""
43
+ if name() == 'pg':
44
+ import core.store_pg as _pg
45
+ return _pg
46
+ return _hf
47
+
48
+
49
+ # --- the nine delegates. Written out rather than generated with setattr so that reading this file
50
+ # tells you the whole interface, and so a new store operation cannot be added to core/store.py
51
+ # without something here failing to match (verify_store_pg.py compares the signatures).
52
+ def available():
53
+ return active().available()
54
+
55
+
56
+ def get(name_, fresh=False):
57
+ return active().get(name_, fresh=fresh)
58
+
59
+
60
+ def exists(name_):
61
+ return active().exists(name_)
62
+
63
+
64
+ def put(name_, data):
65
+ return active().put(name_, data)
66
+
67
+
68
+ def update(name_, fn, flush='sync'):
69
+ return active().update(name_, fn, flush=flush)
70
+
71
+
72
+ def upload_bytes(path_in_repo, data, message=None):
73
+ return active().upload_bytes(path_in_repo, data, message=message)
74
+
75
+
76
+ def download_bytes(path_in_repo):
77
+ return active().download_bytes(path_in_repo)
78
+
79
+
80
+ def delete_path(path_in_repo):
81
+ return active().delete_path(path_in_repo)
82
+
83
+
84
+ def flush(name_=None, timeout=30.0):
85
+ return active().flush(name_, timeout=timeout)
platform/core/store_pg.py ADDED
@@ -0,0 +1,249 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """core/store_pg.py β€” `core/store.py`'s interface, backed by Postgres (X4 / EXIT-2b, 2026-07-30).
2
+
3
+ THE INTERFACE IS THE CONTRACT, and it is `core/store.py`'s nine functions:
4
+
5
+ available() Β· get(name, fresh=False) Β· exists(name) Β· put(name, data)
6
+ update(name, fn, flush='sync') Β· upload_bytes(path, data, message=None)
7
+ download_bytes(path) Β· delete_path(path) Β· flush(name=None, timeout=30.0)
8
+
9
+ Same names, same signatures, same return types, same failure semantics β€” so switching backends is
10
+ an env var (`STORE_BACKEND=hf|pg`) and not a rewrite of every caller. `core/store_backend.py`
11
+ does the selection; nothing above it needs to know which store it is talking to.
12
+
13
+ β›” NOT VERIFIED AGAINST A REAL SERVER β€” owner blocker **B-3** (choose + provision managed
14
+ Postgres, US-region, co-located with Ashburn). `verify_store_pg.py` is written and it SKIPS LOUDLY
15
+ without `DATABASE_URL`; it is not reported as passing, and this module is not the default backend.
16
+ The schema is `harness/pg/schema.sql`. Sequenced around the blocker, never stubbed past it.
17
+
18
+ WHAT POSTGRES FIXES, precisely (C1c β€” these are the reasons, not a preference):
19
+ * **`get()` is cache-first on the HF store**, so a write from another process is invisible to a
20
+ running app until restart. Observed live in wave 11. Here a read is a SELECT: cross-process
21
+ coherence is the default rather than a thing to remember.
22
+ * **`update()` was read-modify-write with no concurrency control** β€” two writers raced and the
23
+ loser vanished silently. Here it runs inside ONE transaction with `SELECT … FOR UPDATE`, so
24
+ the second writer waits and then sees the first writer's value.
25
+ * **A write was an HTTP commit** against a 256-commits/hour repo budget, which is why the hot
26
+ path needed a coalescing async flusher at all. A Postgres write is a write; `flush='async'`
27
+ stays in the signature and becomes a no-op, because there is nothing to coalesce.
28
+
29
+ ⚠ ONE DELIBERATE DIFFERENCE FROM THE HF BACKEND, and it is a fix. `store.get()` swallows transient
30
+ read failures and hands back a cached-or-empty dict, because a display read must not break a page.
31
+ That leniency is exactly what wiped the user registry once (an empty read merged into a write), and
32
+ `_read_strict` exists to opt out of it. Postgres has no equivalent "the network blipped" state
33
+ worth hiding: a connection failure RAISES from `get()` here. Callers that must not break on a read
34
+ should catch it β€” silently returning `{}` from a database that is simply unreachable is how an
35
+ empty dict gets written back over real data.
36
+ """
37
+ import json
38
+ import os
39
+ import threading
40
+
41
+ _URL_ENV = 'DATABASE_URL'
42
+ _LOCK = threading.RLock()
43
+ _POOL = {'pool': None, 'url': None, 'schema': None}
44
+
45
+
46
+ def _url():
47
+ return os.environ.get(_URL_ENV) or None
48
+
49
+
50
+ def _schema_for(tenant_slug=None):
51
+ """`t_<slug>` with `-` normalised to `_` (a hyphen is not legal in an unquoted identifier).
52
+
53
+ Mirrors `control.provision_tenant_schema` exactly β€” the two MUST agree or a write lands in a
54
+ schema the DDL never created. `AIOS_TENANT` names the tenant this process serves; it defaults
55
+ to tenant #0 so the default behaviour is unchanged, the same rule `harness.datastore.path_for`
56
+ follows for the DuckDB filename.
57
+ """
58
+ slug = (tenant_slug or os.environ.get('AIOS_TENANT') or 'royal-imports').strip().lower()
59
+ return 't_' + slug.replace('-', '_')
60
+
61
+
62
+ def _pool():
63
+ """The psycopg connection pool, opened once per process.
64
+
65
+ psycopg is imported LAZILY and only here: `STORE_BACKEND=hf` must not require the dependency
66
+ to be installed at all, which is what keeps this file safe to commit before B-3 unblocks.
67
+ """
68
+ with _LOCK:
69
+ url = _url()
70
+ if not url:
71
+ raise RuntimeError(
72
+ f'{_URL_ENV} is not set β€” the Postgres store backend has nothing to connect to.')
73
+ if _POOL['pool'] is not None and _POOL['url'] == url:
74
+ return _POOL['pool']
75
+ try:
76
+ from psycopg_pool import ConnectionPool
77
+ except ImportError as e: # pragma: no cover - depends on B-3
78
+ raise RuntimeError(
79
+ 'STORE_BACKEND=pg needs psycopg[binary,pool]. It is deliberately NOT in '
80
+ 'requirements.txt until the database exists (owner blocker B-3), so that the '
81
+ 'default hf backend installs nothing it does not use.') from e
82
+ if _POOL['pool'] is not None:
83
+ try:
84
+ _POOL['pool'].close()
85
+ except Exception:
86
+ pass
87
+ # min_size=0 so a process that never touches the store opens no connection at all β€” the
88
+ # shared API serves plenty of requests (health, static, a cached payload) that never read.
89
+ _POOL['pool'] = ConnectionPool(url, min_size=0, max_size=8, open=True, timeout=10.0)
90
+ _POOL['url'] = url
91
+ return _POOL['pool']
92
+
93
+
94
+ def available():
95
+ """True when a connection can actually be MADE, not merely when a URL is configured.
96
+
97
+ `core.store.available()` only checks for a token, which is all it can cheaply do. Here the
98
+ honest answer needs a round-trip, so the result is memoised per process: callers use this to
99
+ decide whether to degrade the UI, and a `SELECT 1` per render would be absurd.
100
+ """
101
+ if not _url():
102
+ return False
103
+ if _POOL.get('ok'):
104
+ return True
105
+ try:
106
+ with _pool().connection() as con:
107
+ con.execute('SELECT 1')
108
+ _POOL['ok'] = True
109
+ return True
110
+ except Exception:
111
+ return False
112
+
113
+
114
+ def _table(kind, tenant_slug=None):
115
+ """`"t_slug"."store_kv"` β€” the qualified table, safely quoted."""
116
+ from psycopg import sql
117
+ return sql.SQL('{}.{}').format(sql.Identifier(_schema_for(tenant_slug)),
118
+ sql.Identifier(kind))
119
+
120
+
121
+ def _q(template, kind, tenant_slug=None):
122
+ """Compose `template` with `{tbl}` = the QUALIFIED table and `{bare}` = the table name alone.
123
+
124
+ ⚠ THE TWO ARE NOT INTERCHANGEABLE, and getting it wrong is a runtime syntax error nothing
125
+ local would catch (no database here β€” owner blocker B-3). Inside `ON CONFLICT … DO UPDATE`,
126
+ Postgres refers to the conflicting row by the table's ALIAS, which defaults to the bare
127
+ table name: `store_kv.rev` is valid, `t_royal_imports.store_kv.rev` is not. So the FROM
128
+ position takes `{tbl}` and the DO-UPDATE position takes `{bare}`.
129
+
130
+ Identifiers go through `psycopg.sql.Identifier`, so the schema name β€” derived from a tenant
131
+ slug β€” cannot be an injection vector even though it is interpolated. The slug is ALSO
132
+ regex-constrained at its source (`control.tenants`), which is the belt to this brace.
133
+ """
134
+ from psycopg import sql
135
+ return sql.SQL(template).format(tbl=_table(kind, tenant_slug),
136
+ bare=sql.Identifier(kind))
137
+
138
+
139
+ def get(name, fresh=False, tenant_slug=None):
140
+ """The value stored under `name`, or `{}` when the key is genuinely absent.
141
+
142
+ `fresh` is accepted and IGNORED: it exists in the HF signature to bypass a process cache, and
143
+ there is no cache here β€” every read is a SELECT. Keeping the parameter means callers do not
144
+ branch on the backend.
145
+ """
146
+ with _pool().connection() as con:
147
+ row = con.execute(
148
+ _q('SELECT value FROM {tbl} WHERE key = %s', 'store_kv', tenant_slug),
149
+ (str(name),)).fetchone()
150
+ return dict(row[0]) if row and isinstance(row[0], dict) else (row[0] if row else {})
151
+
152
+
153
+ def exists(name, tenant_slug=None):
154
+ """True/False β€” and unlike the HF backend this is NEVER "True on uncertainty".
155
+
156
+ `core.store.exists` returns True on any error so a caller that would seed the store cannot
157
+ clobber a registry it merely failed to reach. That guard exists because a FILE HOST cannot
158
+ distinguish "absent" from "unreachable". Postgres can: an absent row is a fact, and an
159
+ unreachable server raises instead of answering.
160
+ """
161
+ with _pool().connection() as con:
162
+ row = con.execute(
163
+ _q('SELECT 1 FROM {tbl} WHERE key = %s', 'store_kv', tenant_slug),
164
+ (str(name),)).fetchone()
165
+ return row is not None
166
+
167
+
168
+ def put(name, data, tenant_slug=None):
169
+ with _pool().connection() as con:
170
+ con.execute(
171
+ _q('INSERT INTO {tbl} (key, value) VALUES (%s, %s::jsonb) '
172
+ 'ON CONFLICT (key) DO UPDATE SET value = EXCLUDED.value, '
173
+ 'updated_at = now(), rev = {bare}.rev + 1', 'store_kv', tenant_slug),
174
+ (str(name), json.dumps(data)))
175
+ return data
176
+
177
+
178
+ def update(name, fn, flush='sync', tenant_slug=None):
179
+ """Read-modify-write inside ONE transaction, with the row LOCKED for the duration.
180
+
181
+ This is the operation the HF backend could not make safe. There, `update` read over HTTP,
182
+ applied `fn`, and uploaded β€” so two concurrent writers both read the old value and the second
183
+ upload silently discarded the first writer's change. `SELECT … FOR UPDATE` makes the second
184
+ writer WAIT and then apply `fn` to the first writer's result, which is what "read-modify-write"
185
+ was always supposed to mean.
186
+
187
+ `flush` is accepted and ignored (see the module docstring): there is no commit budget to
188
+ coalesce against, so 'async' has nothing to defer.
189
+ """
190
+ with _pool().connection() as con:
191
+ with con.transaction():
192
+ row = con.execute(
193
+ _q('SELECT value FROM {tbl} WHERE key = %s FOR UPDATE',
194
+ 'store_kv', tenant_slug),
195
+ (str(name),)).fetchone()
196
+ data = dict(row[0]) if row and isinstance(row[0], dict) else (row[0] if row else {})
197
+ result = fn(data)
198
+ data = result if result is not None else data
199
+ con.execute(
200
+ _q('INSERT INTO {tbl} (key, value) VALUES (%s, %s::jsonb) '
201
+ 'ON CONFLICT (key) DO UPDATE SET value = EXCLUDED.value, '
202
+ 'updated_at = now(), rev = {bare}.rev + 1', 'store_kv', tenant_slug),
203
+ (str(name), json.dumps(data)))
204
+ return data
205
+
206
+
207
+ def upload_bytes(path_in_repo, data, message=None, tenant_slug=None):
208
+ with _pool().connection() as con:
209
+ con.execute(
210
+ _q('INSERT INTO {tbl} (path, bytes, message) VALUES (%s, %s, %s) '
211
+ 'ON CONFLICT (path) DO UPDATE SET bytes = EXCLUDED.bytes, '
212
+ 'message = EXCLUDED.message, updated_at = now()', 'store_blobs', tenant_slug),
213
+ (str(path_in_repo), bytes(data), message))
214
+
215
+
216
+ def download_bytes(path_in_repo, tenant_slug=None):
217
+ """Raw bytes, or None when absent β€” the same contract as the HF backend: a missing attachment
218
+ degrades to "that file is gone", never to a broken page."""
219
+ with _pool().connection() as con:
220
+ row = con.execute(
221
+ _q('SELECT bytes FROM {tbl} WHERE path = %s', 'store_blobs', tenant_slug),
222
+ (str(path_in_repo),)).fetchone()
223
+ return bytes(row[0]) if row else None
224
+
225
+
226
+ def delete_path(path_in_repo, tenant_slug=None):
227
+ """Absent is SUCCESS β€” deleting what is already gone is the goal (HF backend's contract)."""
228
+ with _pool().connection() as con:
229
+ con.execute(_q('DELETE FROM {tbl} WHERE path = %s', 'store_blobs', tenant_slug),
230
+ (str(path_in_repo),))
231
+ return True
232
+
233
+
234
+ def flush(name=None, timeout=30.0):
235
+ """A no-op that returns True: every write above is already committed when it returns. Kept in
236
+ the interface because QA gates and shutdown hooks call it and must not branch on the backend."""
237
+ return True
238
+
239
+
240
+ def close():
241
+ """Release the pool (tests, and a process that is shutting down cleanly)."""
242
+ with _LOCK:
243
+ if _POOL['pool'] is not None:
244
+ try:
245
+ _POOL['pool'].close()
246
+ finally:
247
+ _POOL['pool'] = None
248
+ _POOL['url'] = None
249
+ _POOL.pop('ok', None)
platform/core/table_store.py ADDED
@@ -0,0 +1,479 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """The generic per-user TABLE WORKSPACE store β€” the persistence half of the table-page factory.
2
+
3
+ One durable store key holds one table OBJECT's per-user Airtable-style state:
4
+
5
+ {username: {'views': {view_id: SavedView},
6
+ 'fields': {field_key: Field}, # notes + custom_ + measure_ strata
7
+ 'overlays': {str(pid): {field_key: value}}}}
8
+
9
+ `make(table_key)` returns the six operations a table page's host loop needs, closed over that
10
+ key. The Customer table's ops (`modules/customer_data.py`, key 'customer_table_workspace') are
11
+ these exact functions β€” the logic MOVED here 2026-07-27 so that duplicating the Customer table
12
+ pattern to a new object is a registry row + a config, not a copy of the store plumbing
13
+ (owner directive: the table-page factory).
14
+
15
+ A LIST (membership/formula semantics) is deliberately a different store from a VIEW
16
+ (presentation/query state) β€” see modules/customer_data.py's customer_lists key.
17
+ """
18
+ import core.store as store
19
+
20
+ #: Wave-9 I17 β€” the SHARED bucket. Views whose permissions make them visible to anyone but
21
+ #: their creator live here instead of in a personal workspace, under a key that cannot collide
22
+ #: with a username (usernames come from core/users.py and are never dunder-wrapped; `is_shared`
23
+ #: guards it anyway). ONE HOME PER VIEW, never both: a view moved to personal is REMOVED from
24
+ #: here, and a view shared is removed from its creator's workspace. Two homes would mean two
25
+ #: divergent copies the moment either was edited.
26
+ SHARED_KEY = '__shared__'
27
+
28
+
29
+ def _may_see(view, viewer, is_admin=False):
30
+ """Visibility for ONE shared view, fail-closed.
31
+
32
+ 'collaborative' = everyone who can already open the module (the caller has gated that).
33
+ 'users' = the named users, plus the creator, plus admins β€” an admin who could not
34
+ see a view could not administer it either.
35
+ Anything unrecognised returns False rather than defaulting open: an unreadable permission
36
+ must never widen access ([[aios-permissioning]] β€” no fail-open defaults).
37
+ """
38
+ if not isinstance(view, dict):
39
+ return False
40
+ if view.get('createdBy') == viewer or is_admin:
41
+ return True
42
+ perms = view.get('permissions') or {}
43
+ edit = perms.get('edit')
44
+ if edit == 'collaborative':
45
+ return True
46
+ if edit == 'users':
47
+ return viewer in set(perms.get('users') or ())
48
+ return False # 'personal', absent, or junk
49
+
50
+
51
+ def _may_edit(view, viewer, is_admin=False):
52
+ """Who may WRITE a shared view. Same set as visibility today β€” the owner's item asks 'who
53
+ can edit' and lists who 'can have access', i.e. seeing and editing are one grant. Kept as a
54
+ separate function so they can diverge (a future read-only share) without hunting callers."""
55
+ return _may_see(view, viewer, is_admin)
56
+
57
+
58
+ def _may_administer(view, viewer, is_admin=False):
59
+ """Who may change a view's PERMISSIONS, or delete it: the creator or an admin ONLY.
60
+
61
+ Deliberately narrower than _may_edit. If a collaborator could rewrite `permissions` they
62
+ could grant themselves sole ownership of somebody else's view, or quietly widen a
63
+ users-scoped view to everyone β€” the classic privilege-escalation-by-edit hole.
64
+ """
65
+ if not isinstance(view, dict):
66
+ return False
67
+ return bool(is_admin) or view.get('createdBy') == viewer
68
+
69
+
70
+ def is_shared(view):
71
+ """A view belongs in the shared bucket when its permissions reach beyond its creator."""
72
+ return ((view or {}).get('permissions') or {}).get('edit') in ('collaborative', 'users')
73
+
74
+
75
+ def _unique_name(wanted, existing, *, fallback='Untitled', max_len=120):
76
+ """Allocate one human-facing name inside a store.update transaction.
77
+
78
+ Keys/ids remain structural identity. Names compare case-insensitively after collapsing
79
+ whitespace, because those variants are indistinguishable in the UI. This helper belongs
80
+ in the store layer: allocating from a pre-write snapshot lets two concurrent requests both
81
+ choose the same free name before either write lands.
82
+ """
83
+ limit = max(1, int(max_len))
84
+
85
+ def _clean(value):
86
+ return ' '.join(str(value or '').split())
87
+
88
+ base = (_clean(wanted) or _clean(fallback) or 'Untitled')[:limit].rstrip()
89
+ taken = {_clean(value).casefold() for value in existing if _clean(value)}
90
+ if base.casefold() not in taken:
91
+ return base
92
+ index = 2
93
+ while True:
94
+ suffix = f' {index}'
95
+ stem = base[:max(0, limit - len(suffix))].rstrip()
96
+ candidate = f'{stem}{suffix}' if stem else str(index)[-limit:]
97
+ if candidate.casefold() not in taken:
98
+ return candidate
99
+ index += 1
100
+
101
+
102
+ class TableStore:
103
+ """The six store operations for one table object's workspace, closed over its store key.
104
+
105
+ `st` (wave 18, C3-UT) is the STORE HANDLE β€” anything exposing `get(name)` /
106
+ `update(name, fn, flush=)`. Default = `core.store` (tenant #0, every existing caller,
107
+ zero behaviour change). The API passes the session's `TenantRuntime`, whose accessors
108
+ apply the tenant prefix / repo binding β€” which is what makes a user table created by a
109
+ Nurilab admin land in Nurilab's store instead of Royal's.
110
+ """
111
+
112
+ def __init__(self, table_key, st=None):
113
+ self.table_key = table_key
114
+ self._st = st if st is not None else store
115
+
116
+ # ---------------------------------------------------------------- read
117
+ def workspace(self, username, consume_corrections=True):
118
+ """One user's durable workspace: always the full three-strata shape."""
119
+ try:
120
+ data = self._st.get(self.table_key) or {}
121
+ ws = data.get(username, {}) or {}
122
+ # A collision acknowledgement is protocol state, not part of a field definition.
123
+ # Consume it with the first fresh workspace payload after the correcting write,
124
+ # then splice a bounded copy into that payload only. Keeping it out of `fields`
125
+ # prevents an old request id surviving forever and overriding a later rename.
126
+ corrections = {}
127
+ if consume_corrections and ws.get('fieldCorrections'):
128
+ def _take(current):
129
+ current_ws = current.get(username) or {}
130
+ pending = current_ws.get('fieldCorrections') or {}
131
+ corrections.update({
132
+ str(key)[:80]: dict(value)
133
+ for key, value in pending.items()
134
+ if isinstance(value, dict)
135
+ })
136
+ current_ws.pop('fieldCorrections', None)
137
+ return current
138
+
139
+ data = self._st.update(self.table_key, _take, flush='async')
140
+ ws = (data or {}).get(username, {}) or {}
141
+ except Exception:
142
+ ws = {}
143
+ corrections = {}
144
+ fields = {
145
+ key: dict(value) if isinstance(value, dict) else value
146
+ for key, value in (ws.get('fields') or {}).items()
147
+ }
148
+ for key, ack in corrections.items():
149
+ field = fields.get(key)
150
+ accepted_label = str(ack.get('label') or '')[:120]
151
+ requested_label = str(ack.get('labelCorrectedFrom') or '')[:120]
152
+ correction_id = str(ack.get('labelCorrectionId') or '')[:180]
153
+ # A newer field write clears/replaces the pending ack in the SAME transaction.
154
+ # The label check is an extra belt against ever attaching a stale ack to a newer
155
+ # definition if a future store implementation weakens that ordering.
156
+ if (isinstance(field, dict) and accepted_label
157
+ and str(field.get('label') or '') == accepted_label
158
+ and requested_label and correction_id):
159
+ field['labelCorrectedFrom'] = requested_label
160
+ field['labelCorrectionId'] = correction_id
161
+ out = {
162
+ 'views': dict(ws.get('views') or {}),
163
+ 'fields': fields,
164
+ 'overlays': dict(ws.get('overlays') or {}),
165
+ # wave-8 I11 (C4): folders over the saved views / cohorts sidebars. A FOURTH
166
+ # stratum rather than a key on each item β€” see aios_grid.clean_folders for why
167
+ # (a cohort lives in another store, and filing is an organising act, not part of
168
+ # what a view is). Absent for every workspace saved before this wave, which is
169
+ # exactly "no folders yet".
170
+ 'folders': dict(ws.get('folders') or {}),
171
+ 'itemFolders': dict(ws.get('itemFolders') or {}),
172
+ }
173
+ # 2026-07-31 (owner item 3): WHERE THE USER LEFT OFF survives a new browser. The
174
+ # client's localStorage copy wins when present; this is the server's answer for a
175
+ # fresh profile, which used to fall all the way to the system default view.
176
+ if ws.get('activeViewId'):
177
+ out['activeViewId'] = str(ws['activeViewId'])
178
+ # Wave 2026-08-02 (C-LAYOUT): the per-user record-detail field order. A fifth
179
+ # stratum, absent until the user first reorders β€” exactly "default order".
180
+ if isinstance(ws.get('recordLayout'), dict):
181
+ out['recordLayout'] = dict(ws['recordLayout'])
182
+ return out
183
+
184
+ # ---------------------------------------------------------------- write
185
+ def _update(self, username, change):
186
+ def _up(data):
187
+ ws = data.setdefault(username, {})
188
+ ws.setdefault('views', {})
189
+ ws.setdefault('fields', {})
190
+ ws.setdefault('overlays', {})
191
+ ws.setdefault('folders', {})
192
+ ws.setdefault('itemFolders', {})
193
+ change(ws)
194
+ return data
195
+ # flush='async' (wave-7 W3): this is THE hot path β€” every autosaved filter tweak,
196
+ # column note and typed overlay cell lands here inside the component round-trip, and
197
+ # the historical synchronous hub commit cost seconds per edit. The mutation applies to
198
+ # the in-process cache (read-your-writes for every subsequent render); the hub write
199
+ # coalesces in the background. Registry/auth writes elsewhere stay flush='sync'.
200
+ return self._st.update(self.table_key, _up, flush='async')
201
+
202
+ def save_active_view(self, username, view_id):
203
+ """Remember which view this user last opened (owner item 3, 2026-07-31).
204
+
205
+ Presentation state, not authorisation: the READ side re-validates the id against what
206
+ the caller may actually see, so a stale or foreign id degrades to the default view
207
+ rather than granting anything. Stored per user like every other stratum.
208
+ """
209
+ vid = str(view_id or '').strip()[:120]
210
+ if not vid or username == SHARED_KEY:
211
+ return
212
+
213
+ def _set(ws):
214
+ ws['activeViewId'] = vid
215
+ self._update(username, _set)
216
+
217
+ def save_record_layout(self, username, order):
218
+ """The per-user RECORD-DETAIL field order (wave 2026-08-02, C-LAYOUT).
219
+
220
+ Presentation state for ONE surface β€” the record modal. Deliberately not view config:
221
+ the owner's ask is per-user, not per-view, and it must never reorder grid columns.
222
+ The event handler validated keys against the live field set; the wire re-validates at
223
+ serve time (aios_grid.workspace_wire), so a deleted field cannot outlive itself here.
224
+ An empty order clears the stratum back to "default order".
225
+ """
226
+ if username == SHARED_KEY:
227
+ return
228
+ clean, seen = [], set()
229
+ for key in (order or [])[:200]:
230
+ key = str(key or '').strip()[:80]
231
+ if key and key not in seen:
232
+ seen.add(key)
233
+ clean.append(key)
234
+
235
+ def _set(ws):
236
+ if clean:
237
+ ws['recordLayout'] = {'order': clean}
238
+ else:
239
+ ws.pop('recordLayout', None)
240
+
241
+ self._update(username, _set)
242
+
243
+ def save_folders(self, username, folders, item_folders):
244
+ """Replace the folder stratum wholesale (wave-8 I11).
245
+
246
+ Wholesale rather than per-folder because the caller has ALREADY validated the complete
247
+ picture through aios_grid.clean_folders / clean_item_folders, and those two are
248
+ interdependent: a placement is only legal while its folder exists, so committing them
249
+ separately would leave a window where a reader sees an item filed into a folder that is
250
+ not there yet. One write, one consistent state.
251
+ """
252
+ def _set(ws):
253
+ ws['folders'] = dict(folders or {})
254
+ ws['itemFolders'] = dict(item_folders or {})
255
+ self._update(username, _set)
256
+
257
+ def shared_views(self, viewer, is_admin=False):
258
+ """Every SHARED view this viewer may see, by id (wave-9 I17).
259
+
260
+ Read-only and independent of the viewer's own workspace: the caller merges. Returns
261
+ only what `_may_see` allows, so a caller cannot accidentally render somebody else's
262
+ personal view by forgetting to filter.
263
+ """
264
+ try:
265
+ bucket = ((self._st.get(self.table_key) or {}).get(SHARED_KEY) or {}).get('views') or {}
266
+ except Exception:
267
+ return {}
268
+ return {vid: dict(v) for vid, v in bucket.items()
269
+ if _may_see(v, viewer, is_admin)}
270
+
271
+ def shared_view(self, view_id):
272
+ """One shared view RAW β€” no visibility filter. For authorisation decisions only: a
273
+ caller must know a view exists and who owns it before it can decide whether the actor
274
+ may touch it. Never hand the result to a renderer without checking `_may_see`."""
275
+ try:
276
+ return ((self._st.get(self.table_key) or {}).get(SHARED_KEY) or {}
277
+ ).get('views', {}).get(str(view_id))
278
+ except Exception:
279
+ return None
280
+
281
+ def save_view(self, username, view, shared=None, reserved_names=(), is_admin=False):
282
+ """Upsert a SavedView into its ONE home β€” personal workspace or the shared bucket.
283
+
284
+ `shared` defaults to reading the view's own permissions (`is_shared`). Whichever home
285
+ it lands in, the view is REMOVED from the other, so a view can never exist as two
286
+ copies that diverge on the next edit.
287
+
288
+ ⚠ AUTHORISATION IS THE CALLER'S JOB and must happen BEFORE this is called β€” this layer
289
+ moves data and does not know who is asking. `_cl_handle_one` is the wall.
290
+ """
291
+ view_id = str((view or {}).get('id') or '').strip()
292
+ if not view_id:
293
+ raise ValueError('view id is required')
294
+ if username == SHARED_KEY:
295
+ raise ValueError('reserved username')
296
+ to_shared = is_shared(view) if shared is None else bool(shared)
297
+ requested = dict(view)
298
+ accepted = {}
299
+
300
+ def _up(data):
301
+ # View names are tenant-global: every personal workspace plus the shared bucket.
302
+ # This deliberately includes views the actor cannot see. The only disclosed fact
303
+ # is that a display name is already taken, while the categorical "no duplicate
304
+ # view names" contract remains true when a personal view is later shared.
305
+ names = list(reserved_names or ())
306
+ for workspace in data.values():
307
+ if not isinstance(workspace, dict):
308
+ continue
309
+ names.extend(
310
+ value.get('name')
311
+ for candidate_id, value in (workspace.get('views') or {}).items()
312
+ if candidate_id != view_id and isinstance(value, dict)
313
+ )
314
+ payload = dict(requested)
315
+ payload['name'] = _unique_name(payload.get('name'), names)
316
+ accepted.clear()
317
+ accepted.update(payload)
318
+ if to_shared:
319
+ bucket = data.setdefault(SHARED_KEY, {})
320
+ bucket.setdefault('views', {})[view_id] = payload
321
+ # it may have lived in the creator's workspace before being shared
322
+ owner = data.get(payload.get('createdBy') or username) or {}
323
+ (owner.get('views') or {}).pop(view_id, None)
324
+ else:
325
+ ws = data.setdefault(username, {})
326
+ ws.setdefault('views', {})[view_id] = payload
327
+ (data.get(SHARED_KEY, {}).get('views') or {}).pop(view_id, None)
328
+ return data
329
+
330
+ self._st.update(self.table_key, _up, flush='async')
331
+ return dict(accepted)
332
+
333
+ def delete_view(self, username, view_id):
334
+ """Delete a custom/list view override. The system all-rows view is guarded by caller.
335
+
336
+ Removes from BOTH homes: the caller has already authorised the delete, and leaving a
337
+ stale copy in the other bucket would resurrect the view on the next read.
338
+ """
339
+ vid = str(view_id)
340
+
341
+ def _up(data):
342
+ (data.get(username, {}).get('views') or {}).pop(vid, None)
343
+ (data.get(SHARED_KEY, {}).get('views') or {}).pop(vid, None)
344
+ return data
345
+
346
+ self._st.update(self.table_key, _up, flush='async')
347
+
348
+ def save_field(self, username, field, reserved_names=(), correction_id=None):
349
+ """Persist a column note or a user-created (custom_/measure_) field definition."""
350
+ key = str((field or {}).get('key') or '').strip()
351
+ if not key:
352
+ raise ValueError('field key is required')
353
+ requested = dict(field)
354
+ accepted = {}
355
+
356
+ def _save(ws):
357
+ names = list(reserved_names or ())
358
+ names.extend(
359
+ value.get('label')
360
+ for candidate_key, value in (ws.get('fields') or {}).items()
361
+ if candidate_key != key and isinstance(value, dict)
362
+ )
363
+ payload = dict(requested)
364
+ payload.pop('labelCorrectedFrom', None)
365
+ payload.pop('labelCorrectionId', None)
366
+ requested_label = ' '.join(
367
+ str(payload.get('label') or 'Untitled').split())[:120].rstrip()
368
+ payload['label'] = _unique_name(requested_label, names)
369
+ corrections = ws.setdefault('fieldCorrections', {})
370
+ corrections.pop(key, None)
371
+ if payload['label'] != requested_label and correction_id:
372
+ corrections[key] = {
373
+ 'label': payload['label'],
374
+ 'labelCorrectedFrom': requested_label,
375
+ 'labelCorrectionId': str(correction_id)[:180],
376
+ }
377
+ if not corrections:
378
+ ws.pop('fieldCorrections', None)
379
+ accepted.clear()
380
+ accepted.update(payload)
381
+ # A cleared note on an immutable source field returns to the canonical schema
382
+ # instead of leaving a meaningless override row. Custom fields remain even with
383
+ # an empty note β€” and so does a PRESET field carrying a measure-window override
384
+ # (wave-2 item 8) or a DISPLAY-format override (wave-5 item 10): the note may be
385
+ # empty but the period/format is real user state.
386
+ if (not payload.get('custom') and not str(payload.get('note') or '').strip()
387
+ and not isinstance(payload.get('measure'), dict)
388
+ and not isinstance(payload.get('format'), dict)):
389
+ ws['fields'].pop(key, None)
390
+ else:
391
+ ws['fields'][key] = payload
392
+
393
+ self._update(username, _save)
394
+ return dict(accepted)
395
+
396
+ def duplicate_field(self, username, source_key, new_key, field,
397
+ reserved_names=(), correction_id=None):
398
+ """Clone a user-created field in ONE store transaction (wave-5 item 1): the new
399
+ definition plus β€” for `custom_` overlay sources only β€” every stored cell value under
400
+ the source key. One transaction, because a def without its values (or values without a
401
+ def) is exactly the orphan state delete_field exists to prevent, in reverse.
402
+ The caller validated both keys (same created stratum) and stamped the clone's
403
+ createdBy; this layer only moves data."""
404
+ source_key = str(source_key or '').strip()
405
+ new_key = str(new_key or '').strip()
406
+ if not source_key or not new_key or source_key == new_key:
407
+ raise ValueError('duplicate_field needs two distinct keys')
408
+ requested = dict(field)
409
+ accepted = {}
410
+
411
+ def _dup(ws):
412
+ names = list(reserved_names or ())
413
+ names.extend(
414
+ value.get('label')
415
+ for candidate_key, value in (ws.get('fields') or {}).items()
416
+ if candidate_key != new_key and isinstance(value, dict)
417
+ )
418
+ payload = dict(requested)
419
+ payload.pop('labelCorrectedFrom', None)
420
+ payload.pop('labelCorrectionId', None)
421
+ requested_label = ' '.join(
422
+ str(payload.get('label') or 'Untitled').split())[:120].rstrip()
423
+ payload['label'] = _unique_name(requested_label, names)
424
+ corrections = ws.setdefault('fieldCorrections', {})
425
+ corrections.pop(new_key, None)
426
+ if payload['label'] != requested_label and correction_id:
427
+ corrections[new_key] = {
428
+ 'label': payload['label'],
429
+ 'labelCorrectedFrom': requested_label,
430
+ 'labelCorrectionId': str(correction_id)[:180],
431
+ }
432
+ if not corrections:
433
+ ws.pop('fieldCorrections', None)
434
+ accepted.clear()
435
+ accepted.update(payload)
436
+ ws['fields'][new_key] = payload
437
+ if source_key.startswith('custom_'):
438
+ for row in ws['overlays'].values():
439
+ if isinstance(row, dict) and source_key in row:
440
+ row[new_key] = row[source_key]
441
+
442
+ self._update(username, _dup)
443
+ return dict(accepted)
444
+
445
+ def delete_field(self, username, key):
446
+ """Delete a USER-CREATED field definition outright (owner gap closed 2026-07-27).
447
+
448
+ Only the created strata ever reach here (`custom_` overlay fields, `measure_` formula
449
+ columns β€” the caller enforces the prefix). The stored overlay VALUES for the key are
450
+ scrubbed with it: a deleted column's cells must not linger as orphan data that would
451
+ silently resurface if the key were ever reused. Views referencing the key self-heal on
452
+ their next autosave (an unknown colId is dropped) β€” the rule every stale key rides.
453
+ """
454
+ key = str(key or '').strip()
455
+ if not key:
456
+ return
457
+
458
+ def _drop(ws):
459
+ ws['fields'].pop(key, None)
460
+ for row in ws['overlays'].values():
461
+ if isinstance(row, dict):
462
+ row.pop(key, None)
463
+
464
+ self._update(username, _drop)
465
+
466
+ def patch_overlay(self, username, pid, updates):
467
+ """Patch only the external editable stratum; never writes to the source system."""
468
+ clean = dict(updates or {})
469
+ if not clean:
470
+ return
471
+
472
+ def _patch(ws):
473
+ ws['overlays'].setdefault(str(int(pid)), {}).update(clean)
474
+
475
+ self._update(username, _patch)
476
+
477
+
478
+ def make(table_key, st=None):
479
+ return TableStore(table_key, st=st)
platform/core/user_tables.py ADDED
@@ -0,0 +1,292 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """USER-CREATED TABLES (wave-9 I8 / contract C6) β€” databases that are not a connector.
2
+
3
+ The owner's "+ New" flow has two branches: connect a SOURCE (Odoo today) or *"create a completely
4
+ new database blank, where this time the user can actually insert row that they want."* The first
5
+ branch is the connector seam and lives in `harness/datastore.py`; THIS module is the second.
6
+
7
+ WHY A SEPARATE STORE AND NOT A REGISTRY ROW. `core/registry.py` is a static Python list read at
8
+ import time β€” it is the catalogue of modules the PRODUCT ships, and a tenant cannot append to it
9
+ at runtime without editing code. A user-created table is tenant data, so it lives in the tenant
10
+ store and is MERGED into the nav beside the registry rows. That also keeps the archived/allowed
11
+ machinery honest: a user table is never "archived", it is deleted, and it is never in
12
+ `allowed_modules` because it is not a module.
13
+
14
+ SHAPE (one store key, `user_tables`):
15
+
16
+ {table_key: {'key', 'label', 'source', 'createdBy', 'created', 'fields': [...], 'rows': {...}}}
17
+
18
+ `source` is deliberately NOT 'Odoo' β€” the nav badge is generated from it, so a blank table reads
19
+ "Blank" and can never be mistaken for connected data. `rows` is `{row_id: {field_key: value}}`,
20
+ which is the same overlay shape `TableStore` already uses, so the grid's write path needs no new
21
+ storage concept.
22
+
23
+ ⚠ ROW WRITES ARE ONLY EVER LEGAL HERE. An Odoo-sourced table is READ-ONLY at the source and the
24
+ row endpoints refuse unless `is_user_table(key)` β€” a user must not be able to invent a customer
25
+ in Odoo by typing into a grid. (Wave 18 note: the events seam has NO row event types; row
26
+ add/delete are REST endpoints on `aios-web/api/routes_tables.py`, and this predicate is their
27
+ wall.)
28
+
29
+ WAVE 18 (C3-UT): every function takes an optional `st` STORE HANDLE β€” anything exposing
30
+ `get(name)` / `update(name, fn, flush=)` / `exists(name)`. Default = `core.store` (tenant #0,
31
+ the Streamlit host, unchanged). The API passes the session's `TenantRuntime`, whose accessors
32
+ apply the tenant prefix / repo binding, so a Nurilab table lands in Nurilab's store.
33
+ """
34
+ import datetime as _dt
35
+ import re
36
+
37
+ import core.store as store
38
+
39
+ STORE_KEY = 'user_tables'
40
+ MAX_TABLES = 40
41
+ MAX_LABEL = 60
42
+ MAX_ROWS = 5000
43
+ MAX_FIELDS = 60
44
+ #: the badge the nav paints for a table with no connector behind it
45
+ BLANK_SOURCE = 'Blank'
46
+ #: automation-created tables carry their maker instead (wave 18 C4-AUTO)
47
+ AUTOMATION_SOURCE = 'Automation'
48
+ #: every user table's key is prefixed, so it can never collide with a registry module key
49
+ KEY_PREFIX = 'ut_'
50
+
51
+ #: The field types a user table may declare. ⚠ Kept a SUBSET of
52
+ #: `aios_grid.CUSTOM_FIELD_TYPES` (gated in verify_api's W18-UT section) β€” minus the two
53
+ #: machine-written kinds (`formula` computes client-side over a workspace stratum; a base
54
+ #: column of that type would have no author). Local literal rather than an aios_grid import so
55
+ #: this module stays dependency-light for the API's boot path.
56
+ UT_FIELD_TYPES = {'text', 'select', 'multiselect', 'user', 'int', 'currency', 'pct', 'date',
57
+ 'checkbox', 'phone', 'email', 'url', 'rating'}
58
+
59
+
60
+ def _st(st):
61
+ return st if st is not None else store
62
+
63
+
64
+ def _slug(label):
65
+ s = re.sub(r'[^a-z0-9]+', '_', str(label or '').strip().lower()).strip('_')
66
+ return (s or 'table')[:40]
67
+
68
+
69
+ def clean_fields(raw):
70
+ """Validate a caller-supplied field list into the stored shape β€” refuse junk, never store
71
+ it. Returns a list of `{key,label,type,source}` dicts (source is always 'overlay': every
72
+ base column of a user table is user-editable by construction), or None when nothing
73
+ survives. Unknown types are DROPPED, not coerced β€” a column silently retyped is a lie."""
74
+ out, seen = [], set()
75
+ for f in (raw or [])[:MAX_FIELDS]:
76
+ if not isinstance(f, dict):
77
+ continue
78
+ label = ' '.join(str(f.get('label') or f.get('key') or '').split())[:80]
79
+ key = re.sub(r'[^a-z0-9_]+', '_', str(f.get('key') or _slug(label)).strip().lower())
80
+ key = key.strip('_')[:60]
81
+ ftype = str(f.get('type') or 'text').strip().lower()
82
+ if not key or key in seen or ftype not in UT_FIELD_TYPES or not label:
83
+ continue
84
+ seen.add(key)
85
+ entry = {'key': key, 'label': label, 'type': ftype, 'source': 'overlay'}
86
+ if ftype in ('select', 'multiselect'):
87
+ opts = [' '.join(str(o).split())[:60] for o in (f.get('options') or [])
88
+ if str(o).strip()][:50]
89
+ if opts:
90
+ entry['options'] = opts
91
+ if f.get('default') is True:
92
+ entry['default'] = True
93
+ out.append(entry)
94
+ return out or None
95
+
96
+
97
+ def all_tables(st=None):
98
+ """{key: definition} β€” every user-created table in this tenant."""
99
+ try:
100
+ return dict(_st(st).get(STORE_KEY) or {})
101
+ except Exception:
102
+ return {}
103
+
104
+
105
+ def get(table_key, st=None):
106
+ return all_tables(st).get(str(table_key))
107
+
108
+
109
+ def is_user_table(table_key, st=None):
110
+ """THE permission predicate for row writes. Cheap and total: a key that is not in this store
111
+ is not a user table, so it cannot accept invented rows."""
112
+ return str(table_key or '').startswith(KEY_PREFIX) and get(table_key, st) is not None
113
+
114
+
115
+ def create(label, username, fields=None, source=None, st=None):
116
+ """Create a blank table. Returns its key, or None if refused.
117
+
118
+ Refuses rather than raises on the ordinary conditions (no name, duplicate, cap reached) β€”
119
+ the caller is a UI form and a refusal is a message, not an exception.
120
+ """
121
+ label = str(label or '').strip()[:MAX_LABEL]
122
+ if not label:
123
+ return None
124
+ existing = all_tables(st)
125
+ if len(existing) >= MAX_TABLES:
126
+ return None
127
+ key = KEY_PREFIX + _slug(label)
128
+ if key in existing: # a second table of the same name gets a suffix
129
+ for n in range(2, 50):
130
+ if f'{key}_{n}' not in existing:
131
+ key = f'{key}_{n}'
132
+ break
133
+ else:
134
+ return None
135
+ # A blank table still needs ONE column to be a table at all: a row with no fields cannot be
136
+ # displayed, edited or identified. 'name' is the identity column, the same role `customer`
137
+ # plays in the Customer table.
138
+ defn = {
139
+ 'key': key,
140
+ 'label': label,
141
+ 'source': (source if source in (BLANK_SOURCE, AUTOMATION_SOURCE) else BLANK_SOURCE),
142
+ 'createdBy': username,
143
+ 'created': _dt.datetime.now().strftime('%Y-%m-%dT%H:%M:%S'),
144
+ 'fields': clean_fields(fields) or [{'key': 'name', 'label': 'Name', 'type': 'text',
145
+ 'source': 'overlay', 'default': True}],
146
+ 'rows': {},
147
+ }
148
+
149
+ def _add(cur):
150
+ cur[key] = defn
151
+ return cur
152
+
153
+ _st(st).update(STORE_KEY, _add, flush='sync') # a creation is not a hot path; commit it
154
+ return key
155
+
156
+
157
+ def delete(table_key, st=None):
158
+ def _drop(cur):
159
+ cur.pop(str(table_key), None)
160
+ return cur
161
+ _st(st).update(STORE_KEY, _drop, flush='sync')
162
+
163
+
164
+ def set_fields(table_key, fields, st=None):
165
+ """Replace a table's base-field contract (wave 18 β€” the automation engine grows tables it
166
+ creates). Cleaned with the same validator as `create`; refuses to leave a table fieldless."""
167
+ clean = clean_fields(fields)
168
+ if not clean or not is_user_table(table_key, st):
169
+ return False
170
+
171
+ def _set(cur):
172
+ t = cur.get(str(table_key))
173
+ if t is not None:
174
+ t['fields'] = clean
175
+ return cur
176
+
177
+ _st(st).update(STORE_KEY, _set, flush='sync')
178
+ return True
179
+
180
+
181
+ def add_row(table_key, values=None, username=None, st=None):
182
+ """Append one row. Returns the new row id, or None if refused."""
183
+ if not is_user_table(table_key, st):
184
+ return None # ⚠ never on a connector-backed table
185
+ defn = get(table_key, st) or {}
186
+ rows = defn.get('rows') or {}
187
+ if len(rows) >= MAX_ROWS:
188
+ return None
189
+ rid = str(max((int(r) for r in rows if str(r).isdigit()), default=0) + 1)
190
+ valid = {f['key'] for f in (defn.get('fields') or [])}
191
+ clean = {k: str(v) for k, v in (values or {}).items() if k in valid}
192
+
193
+ def _add(cur):
194
+ t = cur.get(str(table_key))
195
+ if t is not None:
196
+ t.setdefault('rows', {})[rid] = clean
197
+ return cur
198
+
199
+ _st(st).update(STORE_KEY, _add, flush='async')
200
+ return rid
201
+
202
+
203
+ def upsert_rows(table_key, rows_by_key, key_field, username=None, st=None):
204
+ """Bulk UPSERT for the automation engine (wave 18 C4-AUTO): match on `key_field`'s value,
205
+ update matched rows, insert the rest. Returns `{'updated': n, 'inserted': n, 'orphans': n}`
206
+ where orphans = existing rows whose key value did not appear this run β€” COUNTED, never
207
+ deleted (the contract's no-silent-delete rule). Refuses on a non-user table."""
208
+ if not is_user_table(table_key, st):
209
+ return None
210
+ defn = get(table_key, st) or {}
211
+ valid = {f['key'] for f in (defn.get('fields') or [])}
212
+ if key_field not in valid:
213
+ return None
214
+ incoming = {}
215
+ for kv, vals in (rows_by_key or {}).items():
216
+ kv = str(kv).strip()
217
+ if kv:
218
+ incoming[kv] = {k: str(v) for k, v in (vals or {}).items() if k in valid}
219
+ counts = {'updated': 0, 'inserted': 0, 'orphans': 0}
220
+
221
+ def _apply(cur):
222
+ t = cur.get(str(table_key))
223
+ if t is None:
224
+ return cur
225
+ rows = t.setdefault('rows', {})
226
+ by_val = {}
227
+ for rid, row in rows.items():
228
+ v = str((row or {}).get(key_field) or '').strip()
229
+ if v and v not in by_val:
230
+ by_val[v] = rid
231
+ next_id = max((int(r) for r in rows if str(r).isdigit()), default=0)
232
+ for kv, vals in incoming.items():
233
+ rid = by_val.get(kv)
234
+ if rid is not None:
235
+ rows[rid].update(vals)
236
+ counts['updated'] += 1
237
+ elif len(rows) < MAX_ROWS:
238
+ next_id += 1
239
+ rows[str(next_id)] = vals
240
+ counts['inserted'] += 1
241
+ counts['orphans'] = sum(1 for v in by_val if v not in incoming)
242
+ return cur
243
+
244
+ _st(st).update(STORE_KEY, _apply, flush='sync')
245
+ return counts
246
+
247
+
248
+ def delete_row(table_key, row_id, st=None):
249
+ if not is_user_table(table_key, st):
250
+ return False
251
+
252
+ def _drop(cur):
253
+ t = cur.get(str(table_key))
254
+ if t is not None:
255
+ (t.get('rows') or {}).pop(str(row_id), None)
256
+ return cur
257
+
258
+ _st(st).update(STORE_KEY, _drop, flush='async')
259
+ return True
260
+
261
+
262
+ def may_open(table_key, viewer, is_admin=False, st=None):
263
+ """FAIL-CLOSED visibility: the creator, or an admin. Nothing else.
264
+
265
+ A user table cannot be gated by `allowed_modules` β€” a module grant written last month cannot
266
+ describe a table created this morning β€” so it needs its own rule, and the safe rule is the
267
+ narrow one. Tables are stored tenant-wide under one key, so without this ANY user (including
268
+ a BU-scoped sales agent) could open a table somebody else created just by knowing its key.
269
+ Sharing a user table with named colleagues is a follow-on, and it should reuse the shared-
270
+ VIEW vocabulary rather than inventing a second one.
271
+ """
272
+ t = get(table_key, st)
273
+ if not t:
274
+ return False
275
+ return bool(is_admin) or t.get('createdBy') == viewer
276
+
277
+
278
+ def nav_entries(viewer=None, is_admin=False, st=None):
279
+ """Registry-SHAPED dicts so the nav can render user tables beside real modules with no
280
+ special-casing: the same keys the flat nav reads (`key`, `label`, `source`).
281
+
282
+ Filtered by `may_open`, so the nav cannot offer a row the dispatcher would refuse.
283
+ """
284
+ out = []
285
+ for key, t in sorted(all_tables(st).items(),
286
+ key=lambda kv: (kv[1].get('label') or '').lower()):
287
+ if viewer is not None and not may_open(key, viewer, is_admin, st=st):
288
+ continue
289
+ out.append({'key': key, 'label': t.get('label') or key,
290
+ 'source': t.get('source') or BLANK_SOURCE,
291
+ 'user_table': True, 'nav': True, 'validate': False})
292
+ return out
platform/core/users.py ADDED
@@ -0,0 +1,361 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Per-user accounts for the platform, persisted in the HF Dataset store (users.json).
2
+
3
+ Passwords are salted + PBKDF2-HMAC-SHA256 (200k iterations) β€” never stored or logged in plaintext.
4
+ A bootstrap 'admin' account is seeded from APP_PASSWORD so the owner can always log in and create
5
+ users; APP_PASSWORD also works as an emergency master for 'admin' if the registry is unreachable.
6
+
7
+ Each account carries BU access ('all' or a list of team-ids [5=Fisch, 6=Royal]) which drives
8
+ allowed_bus() β€” the basis for per-Business-Unit permissioning (a Royal-only user never sees Fisch).
9
+ """
10
+ import os
11
+ import hmac
12
+ import hashlib
13
+ import secrets
14
+
15
+ import core.store as store
16
+
17
+ BU_LABELS = {5: 'Fisch', 6: 'Royal'}
18
+ _ITER = 200_000
19
+
20
+ #: Wave 15 C-PERM β€” the explicit-resolution marker. Mirrors `core.perm_scope.PERMS_VERSION`;
21
+ #: kept as a literal here so `users` does not import the permission layer it is read by.
22
+ PERMS_VERSION = 1
23
+
24
+
25
+ def _hash(pw, salt):
26
+ return hashlib.pbkdf2_hmac('sha256', str(pw).encode('utf-8'), bytes.fromhex(salt), _ITER).hex()
27
+
28
+
29
+ def _record(pw, name, role, bus, active=True, modules='all', agent=None, email=None,
30
+ perms=None, tenant='royal-imports'):
31
+ salt = secrets.token_hex(16)
32
+ rec = {'salt': salt, 'hash': _hash(pw, salt), 'name': name, 'role': role,
33
+ 'bus': bus, 'active': active, 'modules': modules,
34
+ 'agent': agent or None, 'email': email or None,
35
+ # Wave 18 (C1-TENANT, R1): the account's COMPANY. Absent == 'royal-imports' on every
36
+ # pre-wave record β€” no migration. Login binds the session to THIS value; the posted
37
+ # tenant field can hint but never override it.
38
+ 'tenant': str(tenant or 'royal-imports').strip().lower()}
39
+ if perms is not None:
40
+ # Wave 15 C-PERM. A record written WITH perms is migrated by construction β€” the marker
41
+ # and the block are set together, here, so no writer can create one without the other.
42
+ # (`core.perm_scope` reads an unmarked record as legacy, so a block without its marker
43
+ # would be silently ignored; a marker without a block would deny everything.)
44
+ rec['perms'] = perms
45
+ rec['perms_v'] = PERMS_VERSION
46
+ return rec
47
+
48
+
49
+ def registry():
50
+ return store.get('users')
51
+
52
+
53
+ def ensure_bootstrap():
54
+ """Seed an 'admin' account from APP_PASSWORD ONLY on a truly fresh store (no users file yet).
55
+ Idempotent; no-op if the store is unavailable (the app then falls back to the master-password
56
+ path in verify()).
57
+
58
+ Critically, this NEVER overwrites an existing registry: it seeds only when store.exists('users')
59
+ is definitively False. A transient read failure at startup used to return {} and make this
60
+ re-seed just {admin} over the real accounts β€” that is the bug that wiped users on restart."""
61
+ if not store.available():
62
+ return
63
+ if store.exists('users'): # present, or uncertain -> never clobber
64
+ return
65
+ try:
66
+ reg = store.get('users', fresh=True)
67
+ except Exception:
68
+ return
69
+ if reg:
70
+ return
71
+ master = os.environ.get('APP_PASSWORD', '')
72
+ if not master:
73
+ return
74
+ try:
75
+ store.put('users', {'admin': _record(master, 'Administrator', 'admin', 'all')})
76
+ except Exception:
77
+ pass
78
+
79
+
80
+ def _public(username, u):
81
+ return {'username': username, 'name': u.get('name', username),
82
+ 'role': u.get('role', 'user'), 'bus': u.get('bus', 'all'),
83
+ 'modules': u.get('modules', 'all'),
84
+ 'agent': u.get('agent'), 'email': u.get('email'),
85
+ # Wave 18 (C1-TENANT): the session's tenant binding travels on the projection or it
86
+ # does not travel β€” the same rule the perms block states below.
87
+ 'tenant': str(u.get('tenant') or 'royal-imports').strip().lower(),
88
+ # Wave 14 C-AVATAR: the profile photo is public-safe by definition (it is served
89
+ # to every grid session via the workspace map); without it here the API session's
90
+ # user record silently drops it and /me can never show your own photo.
91
+ 'avatar': u.get('avatar') or None,
92
+ # β›” WAVE 15 C-PERM β€” THE WALL TRAVELS ON THIS PROJECTION OR IT DOES NOT TRAVEL.
93
+ # `deps._user_for` builds every API session from `_public()`, so a `perms` block
94
+ # dropped here is a restricted account served as an unrestricted one β€” silently, on
95
+ # every route, with nothing to notice. `perms_v` must ride ALONG WITH it and for the
96
+ # same reason inverted: the marker without the block denies everything, the block
97
+ # without the marker is ignored. Two keys, one fact, never separated.
98
+ # `verify_api` asserts a restricted user's SESSION OBJECT carries both, at the mount
99
+ # rather than by grep β€” a projection is exactly the kind of wiring that looks
100
+ # present in three files and is absent in the one that runs.
101
+ **({'perms': u['perms']} if isinstance(u.get('perms'), dict) else {}),
102
+ **({'perms_v': int(u['perms_v'] or 0)} if u.get('perms_v') else {}),
103
+ 'epoch': int(u.get('epoch') or 0)}
104
+
105
+
106
+ # ------------------------------------------------------------------ session revocation (X3)
107
+ # The API's session cookie is SIGNED AND STATELESS: there is no server-side session table to
108
+ # delete from, so "log this user out everywhere" needs a number that lives with the account. The
109
+ # cookie carries the epoch it was minted under; bumping the account's epoch makes every
110
+ # outstanding cookie for that user fail verification on its next use. Absent == 0, so every
111
+ # record written before this wave is valid without a migration.
112
+ def epoch(username):
113
+ """The current session epoch for `username`. None when there is no such account.
114
+
115
+ None is NOT 0. 0 is "this account exists and has never been revoked"; None is "no record" β€”
116
+ which the session verifier must treat as a reason to refuse, not as a default to compare
117
+ against. (The APP_PASSWORD emergency-master admin has no record at all; the verifier handles
118
+ that case explicitly rather than inventing an epoch for it here.)
119
+ """
120
+ username = (username or '').strip().lower()
121
+ try:
122
+ u = (store.get('users') or {}).get(username)
123
+ except Exception:
124
+ return None
125
+ return int((u or {}).get('epoch') or 0) if u else None
126
+
127
+
128
+ def bump_epoch(username):
129
+ """Revoke every outstanding API session for this account."""
130
+ username = (username or '').strip().lower()
131
+
132
+ def _set(reg):
133
+ u = reg.get(username)
134
+ if u:
135
+ u['epoch'] = int(u.get('epoch') or 0) + 1
136
+ return reg
137
+ store.update('users', _set)
138
+
139
+
140
+ def verify(username, pw):
141
+ """Return a public user dict on success, else None. APP_PASSWORD is an emergency master for the
142
+ 'admin' login even if the store is unreachable, so the owner is never locked out."""
143
+ username = (username or '').strip().lower()
144
+ if not username or not pw:
145
+ return None
146
+ master = os.environ.get('APP_PASSWORD', '')
147
+ try:
148
+ # read fresh so accounts created moments ago (UI or out-of-band) are recognised at once
149
+ reg = store.get('users', fresh=True)
150
+ except Exception:
151
+ reg = {}
152
+ u = reg.get(username)
153
+ if u is None and '@' in username:
154
+ # Wave 18 (R1): the login box takes a username OR an email β€” admin@nurilab.id signs in
155
+ # without knowing the slug an admin chose. First case-insensitive email match wins;
156
+ # ambiguity is an admin data problem, not a login feature.
157
+ for k, r in reg.items():
158
+ if isinstance(r, dict) and str(r.get('email') or '').strip().lower() == username:
159
+ username, u = k, r
160
+ break
161
+ if u and u.get('active', True) and hmac.compare_digest(_hash(pw, u['salt']), u['hash']):
162
+ return _public(username, u)
163
+ # emergency master: admin + APP_PASSWORD always works (covers first run / store outage)
164
+ if username == 'admin' and master and hmac.compare_digest(str(pw), master):
165
+ # ⚠ CARRY THE RECORD'S CURRENT EPOCH when there is a record to read, so the session cookie
166
+ # the API mints from this dict AGREES with the stored account.
167
+ #
168
+ # This is not what stops the emergency lockout β€” `deps._user_for`'s master fallback does
169
+ # that, and a negative control confirmed the lockout is gone with or without this line.
170
+ # What it fixes is subtler and is a SCOPE question: a cookie whose epoch disagrees with the
171
+ # record falls through to that master fallback, which hands back a SYNTHETIC identity
172
+ # (`bus: 'all'`, `modules: 'all'`). An admin whose record narrows either field would
173
+ # therefore be silently WIDENED to consolidated, all-module access for the life of that
174
+ # session. Matching the epoch means the record branch wins and the account's real scope
175
+ # applies, leaving the master fallback as the true last resort it is meant to be.
176
+ # 0 when the store is unreachable, which is the case this branch was written for.
177
+ return {'username': 'admin', 'name': 'Administrator', 'role': 'admin', 'bus': 'all',
178
+ 'modules': 'all', 'epoch': int((reg.get('admin') or {}).get('epoch') or 0)}
179
+ return None
180
+
181
+
182
+ def create_user(username, pw, name, role='user', bus='all', modules='all',
183
+ agent=None, email=None, tenant=None):
184
+ """Create β€” or, from app.py's dialog, OVERWRITE β€” an account.
185
+
186
+ β›” X3: OVERWRITING AN ACCOUNT MUST NOT RESURRECT ITS OLD SESSIONS. `_record()` builds a fresh
187
+ record with no `epoch` key, i.e. absent == 0. So re-saving an existing username used to reset
188
+ the epoch to 0, and every cookie minted before that account's last password rotation started
189
+ verifying again β€” a silent un-revocation. `app.py`'s "Add / update a user" calls this function
190
+ for BOTH add and update, so the hole was reachable from the shipped UI.
191
+
192
+ Epoch revocation is only ever as strong as the NARROWEST write path that touches the record, so
193
+ the carry-and-bump lives here rather than in each caller: an overwrite is at least as
194
+ session-invalidating as a password change, and it usually IS one.
195
+ """
196
+ username = (username or '').strip().lower()
197
+ if not username or not pw:
198
+ raise ValueError('username and password are required')
199
+
200
+ def _add(reg):
201
+ prior = reg.get(username) or {}
202
+ rec = _record(pw, name or username, role, bus, modules=modules,
203
+ agent=agent, email=email,
204
+ # An overwrite that names no tenant KEEPS the account's company β€” a
205
+ # rename must never quietly move a user between tenants.
206
+ tenant=(tenant or prior.get('tenant') or 'royal-imports'))
207
+ if prior:
208
+ rec['epoch'] = int(prior.get('epoch') or 0) + 1
209
+ reg[username] = rec
210
+ return reg
211
+ store.update('users', _add)
212
+
213
+
214
+ def set_password(username, pw):
215
+ username = (username or '').strip().lower()
216
+
217
+ def _set(reg):
218
+ u = reg.get(username)
219
+ if u:
220
+ u['salt'] = secrets.token_hex(16)
221
+ u['hash'] = _hash(pw, u['salt'])
222
+ # X3: a password change revokes every outstanding API session for the account. Bumped
223
+ # INSIDE the same read-modify-write as the hash so the two can never disagree β€” a
224
+ # separate update() could rotate the password and leave old cookies live if the second
225
+ # write failed.
226
+ u['epoch'] = int(u.get('epoch') or 0) + 1
227
+ return reg
228
+ store.update('users', _set)
229
+
230
+
231
+ def set_active(username, active):
232
+ username = (username or '').strip().lower()
233
+
234
+ def _set(reg):
235
+ if username in reg:
236
+ reg[username]['active'] = bool(active)
237
+ # X3: DEACTIVATION must kill live sessions, not just future logins β€” otherwise a
238
+ # disabled account keeps working until its cookie expires. Bumped on reactivation too:
239
+ # cheap, and it means a re-enabled account never resurrects a stale cookie.
240
+ reg[username]['epoch'] = int(reg[username].get('epoch') or 0) + 1
241
+ return reg
242
+ store.update('users', _set)
243
+
244
+
245
+ def set_access(username, role=None, bus=None, modules=None, agent=None, email=None,
246
+ name=None, perms=None):
247
+ """Update access fields. agent/email: pass '' to clear, None to leave unchanged β€”
248
+ the user↔agent link scopes the Customer List page / digests to that agent's book.
249
+
250
+ `name` follows the same None-means-unchanged idiom. It is here because it had no setter at all:
251
+ a display name could previously only be changed by re-creating the record through
252
+ `create_user`, i.e. by also resetting the password (and, before the fix above, the session
253
+ epoch). Y4's `PATCH {name?}` needs the narrow write, not the destructive one."""
254
+ username = (username or '').strip().lower()
255
+
256
+ def _set(reg):
257
+ u = reg.get(username)
258
+ if u:
259
+ if name is not None:
260
+ u['name'] = name
261
+ if role is not None:
262
+ u['role'] = role
263
+ if bus is not None:
264
+ u['bus'] = bus
265
+ if modules is not None:
266
+ u['modules'] = modules
267
+ if agent is not None:
268
+ u['agent'] = agent or None
269
+ if email is not None:
270
+ u['email'] = email or None
271
+ if perms is not None:
272
+ # Wave 15 C-PERM. Writing perms MIGRATES the record: the marker goes on in the
273
+ # same read-modify-write, so a record can never end up with one and not the
274
+ # other (see `_record`). Whole-block replace, matching the PUT route's shape β€”
275
+ # a merge would make "remove this restriction" unexpressible.
276
+ u['perms'] = perms
277
+ u['perms_v'] = PERMS_VERSION
278
+ return reg
279
+ store.update('users', _set)
280
+
281
+
282
+ def allowed_bus_labels(user):
283
+ """BU labels this user may select. 'all' -> All+Fisch+Royal; a single BU -> just that BU (no
284
+ 'All', so the other BU is never reachable); multiple -> All + each."""
285
+ bus = (user or {}).get('bus', 'all')
286
+ if bus == 'all':
287
+ return ['All', 'Fisch', 'Royal']
288
+ labels = [BU_LABELS[b] for b in bus if b in BU_LABELS]
289
+ if not labels:
290
+ return ['All', 'Fisch', 'Royal']
291
+ return (['All'] + labels) if len(labels) > 1 else labels
292
+
293
+
294
+ def assignable_people(tenant=None):
295
+ """Display names for `user`-typed overlay columns β€” the tenant's ACTIVE accounts.
296
+
297
+ Moved from app.py (2026-07-31) so both hosts serve the same choices. Resolved on every
298
+ call rather than persisted with the column: a snapshot would keep offering people who
299
+ have left and never offer people who joined. Deactivated accounts are excluded; a value
300
+ already stored on a row is untouched β€” history should still say who owned something.
301
+
302
+ Wave 18 (C1-TENANT): pass `tenant` to scope the choices to ONE company β€” the user registry
303
+ is a global control-plane bucket, and a Nurilab picker offering Royal's staff is a
304
+ cross-tenant name leak. None = unscoped (the Streamlit host, tenant #0's process).
305
+ """
306
+ try:
307
+ reg = registry() or {}
308
+ except Exception:
309
+ return []
310
+ want = str(tenant or '').strip().lower()
311
+ out = []
312
+ for username, u in reg.items():
313
+ if not isinstance(u, dict) or u.get('active') is False:
314
+ continue
315
+ if want and str(u.get('tenant') or 'royal-imports').strip().lower() != want:
316
+ continue
317
+ out.append(str(u.get('name') or username))
318
+ return sorted(set(out))
319
+
320
+
321
+ def set_avatar(username, data_url):
322
+ """Set (or clear, with None/'') the user's profile photo β€” a data URL (wave 14 C-AVATAR,
323
+ [[loopable-wave14-split]] item 11). Stored VERBATIM; the API route owns validation (mime +
324
+ decoded size) because this value is served back to every grid session. Cleared by removing
325
+ the key, so records without a photo keep their pre-wave shape."""
326
+ username = (username or '').strip().lower()
327
+
328
+ def _set(reg):
329
+ u = reg.get(username)
330
+ if u:
331
+ if data_url:
332
+ u['avatar'] = str(data_url)
333
+ else:
334
+ u.pop('avatar', None)
335
+ return reg
336
+ store.update('users', _set)
337
+
338
+
339
+ def avatar_map(tenant=None):
340
+ """{display name -> avatar data URL} for ACTIVE accounts with a photo β€” the companion of
341
+ `assignable_people()`, keyed by the SAME vocabulary: a `user` cell stores the display
342
+ name, so the display name is the only join a renderer has. Two active accounts sharing a
343
+ display name share one option; the first WITH a photo wins the key rather than a coin
344
+ flip deciding whether the option has a face. `tenant` scopes it exactly as
345
+ `assignable_people(tenant)` does, and for the same leak."""
346
+ try:
347
+ reg = registry() or {}
348
+ except Exception:
349
+ return {}
350
+ want = str(tenant or '').strip().lower()
351
+ out = {}
352
+ for username, u in sorted(reg.items()):
353
+ if not isinstance(u, dict) or u.get('active') is False:
354
+ continue
355
+ if want and str(u.get('tenant') or 'royal-imports').strip().lower() != want:
356
+ continue
357
+ av = u.get('avatar')
358
+ nm = str(u.get('name') or username)
359
+ if av and nm not in out:
360
+ out[nm] = str(av)
361
+ return out
platform/evals/analyst_golden.yml ADDED
@@ -0,0 +1,104 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Analyst golden eval set v1 (OM-4 gate, 2026-07-11; truths REFRESHED 2026-07-17 after the
2
+ # Jul-15 Odoo mass-recompute + hard-delete reconciliation β€” closed windows DO drift when docs
3
+ # are edited/deleted; refresh truths from the parity-proven layer, never loosen tolerances).
4
+ # Original header: truths computed from the PARITY-PROVEN
5
+ # semantic layer over CLOSED windows (H1 2026 = 2026-01-01..2026-06-30) so live data drift cannot
6
+ # break them. Checks are DETERMINISTIC (number within tolerance + required substrings + tool-use
7
+ # assertions) β€” the validate() culture applied to AI answers. Run: harness/evals.py.
8
+ # expect.value tolerance: abs or rel (fraction). expect.contains: ALL substrings (case-insensitive).
9
+ # expect.tools: tool names that MUST appear in the trace (the answer must come FROM the tools).
10
+ # expect.not_tools: tool names that must NOT appear (the over-refusal guard β€” an answerable ask
11
+ # must never end in report_gap; a gap ask must never end in a forced wrong query).
12
+ items:
13
+ - id: rev_h1
14
+ ask: "What was our wholesale revenue for the first half of 2026 (Jan 1 through Jun 30)?"
15
+ expect: {value: 3501508.33, rel: 0.001, tools: [run_semantic_query]}
16
+ - id: rev_h1_fisch
17
+ ask: "Fisch revenue only, H1 2026 (Jan 1 - Jun 30)?"
18
+ expect: {value: 2494244.70, rel: 0.001, tools: [run_semantic_query]}
19
+ - id: rev_h1_royal
20
+ ask: "And Royal's revenue for the same H1 2026 window?"
21
+ expect: {value: 1007263.63, rel: 0.001, tools: [run_semantic_query]}
22
+ - id: rev_june
23
+ ask: "Revenue for June 2026?"
24
+ expect: {value: 535046.06, rel: 0.001, tools: [run_semantic_query]}
25
+ - id: margin_h1
26
+ ask: "Gross margin dollars for H1 2026 (Jan-Jun), wholesale?"
27
+ expect: {value: 2067509.45, rel: 0.001, tools: [run_semantic_query]}
28
+ - id: margin_pct_h1
29
+ ask: "What gross margin percentage did we run in H1 2026?"
30
+ expect: {value: 0.59, abs: 0.005, allow_pct_form: true, tools: [run_semantic_query]}
31
+ - id: orders_h1
32
+ ask: "How many confirmed wholesale orders did we take in H1 2026 (Jan 1 - Jun 30)?"
33
+ expect: {value: 4989, abs: 1, tools: [run_semantic_query]}
34
+ - id: customers_h1
35
+ ask: "How many distinct customers bought from us in H1 2026?"
36
+ expect: {value: 1037, abs: 1, tools: [run_semantic_query]}
37
+ - id: top_customer
38
+ ask: "Who was our biggest customer by revenue in H1 2026?"
39
+ expect: {contains: ["Poppy Flowers"], tools: [run_semantic_query], not_tools: [report_gap]}
40
+ - id: opex_h1
41
+ ask: "Total operating expense on the GL for H1 2026 (Jan 1 - Jun 30)?"
42
+ expect: {value: 4683428.81, rel: 0.001, tools: [run_semantic_query]}
43
+ - id: top_expense
44
+ ask: "What is our single largest expense account in H1 2026?"
45
+ expect: {contains: ["FBA"], tools: [run_semantic_query]}
46
+ - id: chart_request
47
+ ask: "Make me a line chart of monthly wholesale revenue for H1 2026."
48
+ expect: {artifact: chart, tools: [run_semantic_query, make_chart]}
49
+ - id: bu_guard
50
+ ask: "Show one single revenue line combining Fisch and Royal together per month for H1 2026."
51
+ expect: {tools: [run_semantic_query]} # combined-total ok; the guard is it must NOT error
52
+ - id: name_resolution
53
+ ask: "How much revenue did the customer 'flower' β€” I don't remember the exact name, something with FLOWERS in it β€” bring in H1 2026? Pick the top match."
54
+ expect: {tools: [get_field_values, run_semantic_query]}
55
+ - id: map_request
56
+ ask: "Put my top 20 customers by H1 2026 revenue on a map."
57
+ expect: {artifact: chart, tools: [run_semantic_query, make_chart]}
58
+ # --- the gap loop + location/YoY competence (added 2026-07-16 with the debug fixes) ---
59
+ - id: city_filter
60
+ ask: "Which customer in Brooklyn brought us the most revenue in H1 2026 (Jan 1 - Jun 30)?"
61
+ expect: {value: 48785.00, rel: 0.001, contains: ["J KAY"],
62
+ tools: [get_field_values, run_semantic_query], not_tools: [report_gap]}
63
+ - id: yoy_decliners
64
+ ask: "Which Fisch customer declined the most in revenue in H1 2026 (Jan 1 - Jun 30) versus the same window of 2025? Give the dollar decline."
65
+ expect: {value: 16451.25, rel: 0.005, contains: ["JUNE"],
66
+ tools: [run_semantic_query, transform_result], not_tools: [report_gap]}
67
+ # --- the agent dimension (added 2026-07-17: res.partner.agent_ids m2m sync β€” the #1 logged
68
+ # gap, closed). agent_book is VERBATIM the old gap_refusal ask: yesterday's honest refusal
69
+ # must become today's answer.
70
+ - id: agent_book
71
+ ask: "Who are the top customers handled by our salesperson Naomi in H1 2026?"
72
+ expect: {contains: ["Poppy Flowers"], tools: [get_field_values, run_semantic_query],
73
+ not_tools: [report_gap]}
74
+ - id: agent_revenue
75
+ ask: "How much revenue did agent Naomi's book of customers bring in H1 2026 (Jan 1 - Jun 30)?"
76
+ expect: {value: 872869.98, rel: 0.005, tools: [run_semantic_query], not_tools: [report_gap]}
77
+ # --- the TWO agent sources (added 2026-07-28). Odoo holds a customer-master book AND a
78
+ # per-invoice-line commission agent; they name different people, and the commission table also
79
+ # carries internal SALESPEOPLE who are not agents. These three lock the distinction: the
80
+ # unallocated bucket must be answerable at all, and Anna/Shantal must never be called agents.
81
+ - id: agent_unallocated
82
+ ask: "Using the invoice-line basis, how much of Fisch's 2025 sales was NOT allocated to an agent?"
83
+ expect: {value: 1447579.59, rel: 0.01, tools: [run_semantic_query], not_tools: [report_gap]}
84
+ - id: agent_not_salesperson
85
+ ask: "Who are our agents on the Fisch invoice lines in 2025, and what did each generate?"
86
+ expect: {contains: ["Pasternak"], not_contains: ["Anna", "Shantal"],
87
+ tools: [run_semantic_query], not_tools: [report_gap]}
88
+ - id: agent_allocated_2025
89
+ ask: "On the invoice-line basis, how much of Fisch's 2025 sales was allocated to a real agent?"
90
+ expect: {value: 2995664.75, rel: 0.01, tools: [run_semantic_query], not_tools: [report_gap]}
91
+ - id: gap_refusal
92
+ ask: "How many units of our best-selling SKU do we physically have in stock right now?"
93
+ expect: {tools: [report_gap]} # stock-on-hand has NO topic in the store β€” must self-report, never guess
94
+ # --- the analytics transform library (added 2026-07-18: 60-op Tableau-class set). Truths from
95
+ # the parity-proven semantic layer over the CLOSED H1 2026 window.
96
+ - id: abc_a_tier
97
+ ask: "Using an ABC / Pareto analysis of H1 2026 revenue (Jan 1 - Jun 30), how many customers fall in the A tier β€” the vital few that make up the first 80% of revenue?"
98
+ expect: {value: 311, abs: 6, tools: [run_semantic_query, transform_result], not_tools: [report_gap]}
99
+ - id: ytd_customers
100
+ ask: "How many distinct customers had we sold to cumulatively from Jan 1 through the end of March 2026?"
101
+ expect: {value: 790, abs: 1, tools: [run_semantic_query], not_tools: [report_gap]}
102
+ - id: concentration_top5
103
+ ask: "What percent of our H1 2026 revenue (Jan 1 - Jun 30) is accounted for by our top 5 customers?"
104
+ expect: {value: 7.84, abs: 0.5, tools: [run_semantic_query], not_tools: [report_gap]}
platform/harness/__init__.py ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ """Harness β€” the multi-source financial-dashboarding platform (connectors + canonical model + skills)."""
2
+ from .canonical import Capability, Entity, Query, Filter
3
+ from .base import Connector, Tenant, Skill, register_skill, skills_for
platform/harness/analyst.py ADDED
@@ -0,0 +1,318 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """harness/analyst.py β€” the AIOS Analyst loop (OM-4, 2026-07-11).
2
+
3
+ A CHEAP small model (via OpenRouter, OpenAI-compatible) drives the compounding tool registry
4
+ (harness/tools.py) to answer questions, build charts, and spawn dashboards over the semantic
5
+ layer. The model NEVER writes SQL β€” it picks registry keys, guided by the in-product SKILL
6
+ recipes (model/skills/*.skill.yml) injected into its system prompt. Plan: omni-adoption addendum.
7
+
8
+ Architecture (mirrors the Omni coordinator pattern):
9
+ ask(question) -> bounded tool loop: chat -> tool_calls -> tools.dispatch() -> results -> chat …
10
+ until a plain answer or MAX_ITERS. chat_fn is INJECTABLE: live = OpenRouter POST; tests = a
11
+ scripted double β€” the loop mechanics are provable offline (no key, no tokens).
12
+
13
+ Correctness posture: bounded iterations (never endless); tool errors go BACK to the model as
14
+ readable envelopes (it self-corrects); the final answer carries the tool trace (auditability +
15
+ the drill guarantee); exhausted-budget is reported AS exhausted, never as success (loop rules).
16
+ Cost posture: DEFAULT_MODEL is a cheap tool-calling model; override per-tenant/env; prompt = the
17
+ skills + topic summary (small, cacheable), NOT raw data.
18
+ """
19
+ import json
20
+ import os
21
+ from pathlib import Path
22
+
23
+ import harness.tools as T
24
+
25
+ # Provider LADDER (owner 2026-07-12: "use Groq and Cerebras"): all OpenAI-compatible chat
26
+ # endpoints; tried in order, failing over on payment/rate/server errors β€” an exhausted or
27
+ # rate-limited provider degrades to the next instead of taking the Analyst down (the
28
+ # OpenRouter-402 incident). Pin one with ANALYST_PROVIDER=groq|cerebras|openrouter; override
29
+ # its model with ANALYST_MODEL. Every model here must re-pass the eval gate (Β§4b).
30
+ PROVIDERS = [
31
+ {"name": "groq", "env": "GROQ_API_KEY",
32
+ "url": "https://api.groq.com/openai/v1/chat/completions",
33
+ "model": "llama-3.3-70b-versatile"},
34
+ {"name": "cerebras", "env": "CEREBRAS_API_KEY",
35
+ "url": "https://api.cerebras.ai/v1/chat/completions",
36
+ "model": "gpt-oss-120b"}, # this account's tool-capable model (from /v1/models)
37
+ {"name": "openrouter", "env": "OPENROUTER_API_KEY",
38
+ "url": "https://openrouter.ai/api/v1/chat/completions",
39
+ "model": "openai/gpt-4o-mini"},
40
+ ]
41
+ DEFAULT_MODEL = None # None = the selected provider's default model
42
+ MAX_ITERS = 8
43
+
44
+
45
+ def available_providers():
46
+ pin = os.environ.get("ANALYST_PROVIDER")
47
+ ps = [p for p in PROVIDERS if os.environ.get(p["env"])]
48
+ if pin:
49
+ ps = [p for p in ps if p["name"] == pin] or ps
50
+ return ps
51
+
52
+
53
+ def llm_configured():
54
+ return bool(available_providers())
55
+ SKILLS_DIR = Path(__file__).resolve().parents[1] / "model" / "skills"
56
+
57
+
58
+ def _skills_text():
59
+ parts = []
60
+ for f in sorted(SKILLS_DIR.glob("*.skill.yml")):
61
+ parts.append(f.read_text(encoding="utf-8"))
62
+ return "\n\n".join(parts) or "(no skill files installed)"
63
+
64
+
65
+ def system_prompt():
66
+ """The small model's standing instructions: the platform rules + the skill recipes.
67
+ Stable content (cache-friendly); per-question content goes in the user turn."""
68
+ topics = T.list_topics()
69
+ topic_summary = "\n".join(
70
+ f"- {t['topic']}: {t['label']} | metrics: {', '.join(t['metrics'])} | dims: {', '.join(t['dims'])}"
71
+ for t in topics)
72
+ return f"""You are the AIOS Analyst for a wholesale business. You answer data questions by
73
+ calling TOOLS over a governed semantic layer. THE RULES (non-negotiable):
74
+ 1. You NEVER write SQL or invent field names β€” only registry keys from the tools' schemas.
75
+ 2. Start unknown asks with list_topics/describe_topic. Resolve typed names (customers, products)
76
+ with get_field_values BEFORE filtering. Business units: Fisch=5, Royal=6 β€” never mix them in
77
+ one series unless explicitly comparing.
78
+ 3. Every number you present must come from a tool result and keep its result_id (drillable).
79
+ If a tool errors, read the error and correct your call β€” do not make numbers up, ever.
80
+ 4. Artifact actions (save_view, compose_dashboard, schedule_report, create_alert) need the
81
+ user's explicit confirmation first.
82
+ 5. Answer plainly and lead with the finding. State the window and scope you used β€” READ the
83
+ window from the tool result's 'window' field, never guess it: a query without dates covers
84
+ ALL recorded history (there is no hidden default window). Time-bound asks ("this year",
85
+ "past 4 weeks") must pass explicit date_from/date_to.
86
+ 6. A single-number question gets THE SINGLE NUMBER first (the total for the asked window/scope),
87
+ THEN any breakdown. Never answer a "what was X" question with only components: no monthly
88
+ lists without the total, no per-BU splits when the consolidated figure was asked. If the ask
89
+ is consolidated, query WITHOUT group_by (or sum before answering).
90
+ 7. NEVER state an aggregate you did not obtain from a tool result. If you want to mention a
91
+ total across the rows you fetched, either run the scalar query for it or ADD UP those exact
92
+ rows β€” a round "approximately" figure is a fabrication and forbidden. When in doubt, present
93
+ the rows without a total. Joining or diffing two result sets BY HAND (e.g. this year vs last
94
+ year per customer) is equally forbidden β€” that is transform_result's job ({{"op":"yoy"}} adds
95
+ <m>_ly, <m>_delta, <m>_yoy_pct per row); if no transform can compute it, say you cannot.
96
+ 8. THE GAP PROTOCOL. If β€” and only if β€” after checking the schema (list_topics/describe_topic)
97
+ no registered dimension, metric, transform or chart kind can answer, do NOT guess and do NOT
98
+ force a wrong query: call report_gap(kind, missing, question, workaround) ONCE, then answer
99
+ in one honest sentence what is missing and offer the nearest ask that IS answerable. Before
100
+ concluding a gap, check the TRANSFORMS: year-over-year / decline / growth comparisons (yoy),
101
+ rankings (top_n/sort/head), shares, running totals and distributions are ALL answerable via
102
+ transform_result β€” a derived-analytics question is never a gap. Both directions are
103
+ failures: refusing without report_gap, and calling report_gap for something the tools
104
+ support.
105
+ 9. NEVER emit markdown images, data: URIs, or links to charts β€” you cannot draw. Charts exist
106
+ ONLY through make_chart / make_table; the platform renders them below your answer. Refer to
107
+ a chart as "the chart below". Derived analytics (top-N, YoY, shares, running totals, moving
108
+ averages, ranks, bins) exist ONLY through transform_result β€” never arithmetic in your head.
109
+
110
+ CHART PICKER (the form follows the comparison β€” pick by what the question compares):
111
+ - trend over time -> line (few periods -> bar); seasonal/two-dim intensity -> heatmap
112
+ - this year vs last year by period -> transform_result [{{"op":"yoy"}}] then yoy_bars
113
+ - ranking / "top N" -> transform_result top_n then ranked_bar (sorted, labeled)
114
+ - part-to-whole -> pie|donut ONLY when <=6 slices (top_n first); over time -> stacked_bar;
115
+ shares of a shifting total -> stacked_pct; hierarchy of shares -> treemap
116
+ - what drove the change / contribution -> waterfall (signed steps; platform adds the Total)
117
+ - concentration ("do the top X carry the book?") -> pareto
118
+ - correlation of two measures -> scatter; + a third measure -> bubble (size)
119
+ - distribution of a measure -> histogram (or transform bin -> bar)
120
+ - stage conversion -> funnel; actual vs target -> bullet (y=actual, y2=target)
121
+ - level + rate together (e.g. GM$ + GM%) -> combo (y=bars left, y2=line right)
122
+ - same small chart per segment -> facet on line|bar|area|scatter
123
+ - exact figures, many columns, lists -> make_table (totals row included)
124
+ - customer geography -> map. When two forms tie, pick the simpler one.
125
+
126
+ DATASETS:
127
+ {topic_summary}
128
+
129
+ YOUR SKILL RECIPES (follow these plans; adjust the [bracketed] parameters):
130
+ {_skills_text()}"""
131
+
132
+
133
+ _DEAD_STATUS = {401, 402, 403} # payment/auth: skip the provider immediately, no retry
134
+ _RETRY_STATUS = {429, 500, 502, 503} # rate/server: retry with backoff before failing over
135
+ _MAX_ANSWER_TOKENS = 1500 # answers are short; also keeps low-credit providers usable
136
+ _COOLDOWN = {} # provider name -> unix ts until which we skip it (in-proc)
137
+
138
+
139
+ def _live_chat(messages, tools, model, tool_choice="auto"):
140
+ """POST down the provider ladder with LATENCY-FIRST failover (a chat user is waiting):
141
+ a rate-limited provider is SKIPPED IMMEDIATELY in favor of the next one β€” we only sleep
142
+ when the ENTIRE ladder is limited (then short ladder-level backoffs). A LONG Retry-After
143
+ (daily quota, e.g. Groq TPD) puts the provider in an in-process COOLDOWN; 401/402/403 skip
144
+ the provider outright."""
145
+ import time as _time
146
+
147
+ import requests
148
+ provs = available_providers()
149
+ if not provs:
150
+ raise RuntimeError("no LLM provider configured β€” set GROQ_API_KEY / CEREBRAS_API_KEY / "
151
+ "OPENROUTER_API_KEY in platform/.env")
152
+ last_err = None
153
+ # Two patience profiles: CHAT (default) fails over fast β€” a user is waiting; BATCH
154
+ # (ANALYST_PATIENT=1, set by the eval gate / insight runner) waits out per-minute rate
155
+ # windows across more ladder rounds instead of exhausting the ladder in seconds.
156
+ patient = os.environ.get("ANALYST_PATIENT") == "1"
157
+ round_sleeps = (5, 10, 20, 30, 45, 60) if patient else (2, 6, 12)
158
+ for rnd in range(len(round_sleeps)): # ladder rounds
159
+ saw_retryable = False
160
+ for p in provs:
161
+ if _COOLDOWN.get(p["name"], 0) > _time.time():
162
+ last_err = RuntimeError(f"{p['name']}: cooling down (quota)")
163
+ continue
164
+ try:
165
+ r = requests.post(p["url"], timeout=120,
166
+ headers={"Authorization": f"Bearer {os.environ[p['env']]}",
167
+ "HTTP-Referer": "https://aios.local",
168
+ "X-Title": "AIOS Analyst"},
169
+ json={"model": (model or os.environ.get("ANALYST_MODEL")
170
+ or p["model"]),
171
+ "messages": messages, "tools": tools,
172
+ "tool_choice": tool_choice, "temperature": 0.1,
173
+ "max_tokens": _MAX_ANSWER_TOKENS})
174
+ if r.status_code in _DEAD_STATUS:
175
+ last_err = RuntimeError(f"{p['name']}: HTTP {r.status_code}")
176
+ _COOLDOWN[p["name"]] = _time.time() + 300 # don't re-poke dead auth/payment
177
+ continue
178
+ if r.status_code in _RETRY_STATUS:
179
+ last_err = RuntimeError(f"{p['name']}: HTTP {r.status_code}")
180
+ try:
181
+ wait = float(r.headers.get("retry-after") or 0)
182
+ except ValueError:
183
+ wait = 0
184
+ if wait > 60: # daily/long quota β€” cool down
185
+ _COOLDOWN[p["name"]] = _time.time() + wait
186
+ else:
187
+ saw_retryable = True # NO sleep β€” next provider NOW
188
+ continue
189
+ r.raise_for_status()
190
+ usage = r.json().get("usage", {})
191
+ return r.json()["choices"][0]["message"], usage
192
+ except requests.RequestException as e:
193
+ last_err = e
194
+ continue # network blip β€” next provider
195
+ if not saw_retryable:
196
+ break # nothing retryable left anywhere
197
+ _time.sleep(round_sleeps[rnd]) # whole ladder limited β€” pause a round
198
+ raise RuntimeError(f"all LLM providers failed (last: {last_err})")
199
+
200
+
201
+ #: Wave 15 R2 β€” what a restricted account is told instead of an answer.
202
+ ANALYST_DENIED = ("The Analyst is not available on your account. It answers from the full "
203
+ "governed model, which is wider than what your account is permitted to see.")
204
+
205
+
206
+ def may_ask(user):
207
+ """β›” R2 β€” THE AI WALL, AND IT IS DELIBERATELY BLUNT.
208
+
209
+ Owner, verbatim: *"If the AI is triggered by a user, the AI ITSELF should not be able to see
210
+ the data."* Today the Analyst's tools run UNSCOPED β€” `harness/tools.run_semantic_query` takes
211
+ no subject and no filter, so every question it answers is answered over the whole tenant.
212
+ A restricted account asking "what were our top customers?" would therefore receive, in prose,
213
+ precisely the rows the grid's permanent filter exists to withhold. Prose is not a lesser leak
214
+ than a table; it is a harder one to notice.
215
+
216
+ So the wall is REFUSAL, not narrowing: an account with ANY declared restriction β€” or any
217
+ non-admin on a migrated record β€” does not reach the model at all. An AI that never runs
218
+ cannot leak what it never read. Fail-closed on an unreadable record for the same reason.
219
+
220
+ β­’ NAMED DEBT, owner-mandated (R2, doc Β§Debt, log 2026-08-02). The RIGHT answer is to thread
221
+ the subject's permanent filter into every semantic query the Analyst runs β€” `filter_sql`
222
+ already compiles that tree to SQL, so the shape exists; what does not exist is a per-subject
223
+ query path through `harness/semantic.py`, which takes `team_id` and nothing else. Until that
224
+ lands this refusal is the whole enforcement, and it must NOT be relaxed into "narrow the
225
+ prompt" or "tell the model what it may not say" β€” a prompt is not a permission boundary.
226
+ """
227
+ import core.perms as perms
228
+ import core.perm_scope as perm_scope
229
+
230
+ if not isinstance(user, dict):
231
+ return False # no identity resolved -> no Analyst
232
+ if perms.is_admin(user):
233
+ return True
234
+ if perm_scope.is_migrated(user):
235
+ # Migrated and not an admin: the account is governed by a wall the Analyst cannot honour.
236
+ return False
237
+ # Legacy record: restricted the old way is still restricted.
238
+ return (perms.allowed_modules(user) is None
239
+ and perms.scope_team_id(user) is None
240
+ and not perms.scope_agent(user))
241
+
242
+
243
+ def ask(question, history=None, chat_fn=None, model=DEFAULT_MODEL, max_iters=MAX_ITERS,
244
+ telemetry_kind="analyst", user=None):
245
+ """Run one Analyst turn. Returns {answer, artifacts, tool_trace, iterations, exhausted,
246
+ usage}. chat_fn(messages, tools, model) -> (assistant_message_dict, usage_dict).
247
+
248
+ `user` is the SUBJECT asking. Passing it engages the R2 wall (`may_ask`); omitting it keeps
249
+ the unscoped behaviour that `harness/evals.py` and `harness/routines.py` rely on β€” those run
250
+ as the SYSTEM, not on behalf of a person, and gating them on a user record they do not have
251
+ would break the eval ladder to no security benefit. Every call that IS on behalf of somebody
252
+ passes it, and `app.py`'s chat entry point does.
253
+ """
254
+ if user is not None and not may_ask(user):
255
+ return {"answer": ANALYST_DENIED, "artifacts": [], "tool_trace": [],
256
+ "iterations": 0, "exhausted": False, "denied": True,
257
+ "usage": {"prompt_tokens": 0, "completion_tokens": 0}}
258
+ chat = chat_fn or _live_chat
259
+ tools = T.openai_tools()
260
+ messages = [{"role": "system", "content": system_prompt()}]
261
+ messages += list(history or [])
262
+ # The model has no clock: anchor relative windows ("last week", "this month") to today.
263
+ # Lives in the USER turn so the system prompt stays byte-stable (prompt-cache friendly).
264
+ import datetime as _dt
265
+ messages.append({"role": "user",
266
+ "content": f"(today is {_dt.date.today().isoformat()})\n{question}"})
267
+ trace, artifacts, usage_total = [], [], {"prompt_tokens": 0, "completion_tokens": 0}
268
+
269
+ for i in range(max_iters):
270
+ msg, usage = chat(messages, tools, model)
271
+ for k in usage_total:
272
+ usage_total[k] += usage.get(k, 0) or 0
273
+ tool_calls = msg.get("tool_calls") or []
274
+ # Free-tier models INTERMITTENTLY answer a data question without calling any tool
275
+ # (~1 in 4 observed on gpt-oss-120b β€” the "no chart" defect). Deterministic fix: if
276
+ # the VERY FIRST turn used no tools, redo it once with tool_choice='required'. Meta
277
+ # questions just get a harmless schema lookup before answering.
278
+ if i == 0 and not tool_calls and chat is _live_chat:
279
+ msg, usage = _live_chat(messages, tools, model, tool_choice="required")
280
+ for k in usage_total:
281
+ usage_total[k] += usage.get(k, 0) or 0
282
+ tool_calls = msg.get("tool_calls") or []
283
+ messages.append({"role": "assistant", "content": msg.get("content") or "",
284
+ **({"tool_calls": tool_calls} if tool_calls else {})})
285
+ if not tool_calls:
286
+ answer = (msg.get("content") or "").strip()
287
+ if not answer:
288
+ # A model occasionally returns NO content and NO tool calls (observed live
289
+ # 2026-07-16: the reply bubble was simply empty and the next turn hallucinated).
290
+ # Never surface silence β€” report the failure honestly instead.
291
+ answer = ("(the model returned an empty answer β€” nothing was computed; "
292
+ "please rephrase or ask again)")
293
+ import harness.telemetry as _tel
294
+ _tel.log(telemetry_kind, question=question[:200],
295
+ model=model or (available_providers() or [{}])[0].get("name", "auto"),
296
+ iterations=i + 1,
297
+ tools=[t["tool"] for t in trace],
298
+ tokens=usage_total["prompt_tokens"] + usage_total["completion_tokens"])
299
+ return {"answer": answer, "artifacts": artifacts,
300
+ "tool_trace": trace, "iterations": i + 1, "exhausted": False,
301
+ "usage": usage_total}
302
+ for tc in tool_calls:
303
+ name = tc["function"]["name"]
304
+ args = tc["function"].get("arguments") or "{}"
305
+ res = T.dispatch(name, args)
306
+ trace.append({"tool": name, "args": args if isinstance(args, str) else json.dumps(args),
307
+ "ok": res["ok"], "error": res.get("error")})
308
+ if res["ok"] and isinstance(res["data"], dict) and (
309
+ "chart" in res["data"] or "kpi" in res["data"] or "table" in res["data"]
310
+ or "dashboard" in res["data"]):
311
+ artifacts.append(res["data"])
312
+ messages.append({"role": "tool", "tool_call_id": tc.get("id", name),
313
+ "content": json.dumps(res, default=str)[:8000]})
314
+ # budget exhausted β€” report honestly (never as success)
315
+ return {"answer": "(stopped: tool-call budget exhausted before a final answer β€” "
316
+ "narrow the question or raise max_iters)",
317
+ "artifacts": artifacts, "tool_trace": trace, "iterations": max_iters,
318
+ "exhausted": True, "usage": usage_total}
platform/harness/base.py ADDED
@@ -0,0 +1,123 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Harness core β€” the platform contract that turns 'a dashboard' into 'a product'.
2
+
3
+ Three concepts:
4
+ Connector a source adapter (Odoo / QuickBooks / Amazon / …). Declares its capabilities and
5
+ answers canonical Queries. Adding a source = subclass Connector.
6
+ Tenant one client (Royal Imports, …). Bundles the connectors they've hooked up + their scope
7
+ config (business units, excluded accounts, channel map, branding). Multi-tenant lives here.
8
+ Skill a portable dashboard/metric written ONCE against the canonical model; it declares the
9
+ capabilities it needs so a tenant only ever sees skills its sources can serve.
10
+
11
+ The UI (app.py) becomes a thin renderer over (tenant β†’ available skills). Nothing above the
12
+ connector layer knows what Odoo is.
13
+ """
14
+ from __future__ import annotations
15
+ from abc import ABC, abstractmethod
16
+ from dataclasses import dataclass, field
17
+ from typing import Callable
18
+ from .canonical import Capability, Query, Entity
19
+
20
+
21
+ class Connector(ABC):
22
+ """A data source. Implement `capabilities()` + `aggregate()` (and optionally `records()`), map the
23
+ canonical model to the source, and the source is usable by every compatible skill."""
24
+ key: str = 'connector'
25
+ name: str = 'Connector'
26
+
27
+ @abstractmethod
28
+ def capabilities(self) -> set[Capability]:
29
+ """The canonical capabilities this source can serve for the given account."""
30
+
31
+ @abstractmethod
32
+ def aggregate(self, q: Query) -> list[dict]:
33
+ """Answer an aggregate Query β†’ list of canonical dicts (keys = measures/group_by, + 'count')."""
34
+
35
+ def records(self, q: Query) -> list[dict]:
36
+ """Optional row-level fetch (canonical dicts). Default: not supported."""
37
+ raise NotImplementedError(f'{self.key}: row-level records() not implemented')
38
+
39
+ def distinct_count(self, q: Query, fieldname: str) -> int:
40
+ """Number of distinct values of a canonical field over the query's scope (e.g. active
41
+ customers). Default: derive from an aggregate group_by; connectors may override efficiently."""
42
+ from dataclasses import replace
43
+ rows = self.aggregate(replace(q, measures=(), aggregate='count', group_by=(fieldname,)))
44
+ return sum(1 for r in rows if r.get(fieldname) not in (None, False))
45
+
46
+ def supports(self, cap: Capability) -> bool:
47
+ return cap in self.capabilities()
48
+
49
+ def entities(self) -> set[Entity]:
50
+ """Canonical entities this connector can materialize (advertised by subclasses)."""
51
+ return set()
52
+
53
+
54
+ @dataclass
55
+ class Tenant:
56
+ """One client of the platform. `sources` maps a connector key β†’ Connector instance. `config` holds
57
+ the client's scope semantics (which is what used to be hard-coded in core/odoo.py):
58
+ business_units {label: source_scope_value} e.g. {'Fisch': 5, 'Royal': 6}
59
+ confirmed_statuses [...] which order states count as 'real'
60
+ excluded_customers [ids] house / inter-co accounts to drop
61
+ channels {label: match} e.g. {'Amazon': <partner>} (marketplace split)
62
+ branding {...} palette, name, logo
63
+ A skill reads tenant.config for scope and asks tenant.source(capability) for the right connector."""
64
+ key: str
65
+ name: str
66
+ sources: dict[str, Connector] = field(default_factory=dict)
67
+ config: dict = field(default_factory=dict)
68
+
69
+ def add(self, conn: Connector) -> 'Tenant':
70
+ self.sources[conn.key] = conn
71
+ return self
72
+
73
+ def capabilities(self) -> set[Capability]:
74
+ caps: set[Capability] = set()
75
+ for c in self.sources.values():
76
+ caps |= c.capabilities()
77
+ return caps
78
+
79
+ def source(self, cap: Capability, prefer: str | None = None) -> Connector:
80
+ """The connector that serves `cap` (a tenant may hook several sources; `prefer` picks one,
81
+ else first that supports it). Raises if no connected source provides the capability."""
82
+ if prefer and prefer in self.sources and self.sources[prefer].supports(cap):
83
+ return self.sources[prefer]
84
+ for c in self.sources.values():
85
+ if c.supports(cap):
86
+ return c
87
+ raise LookupError(f"tenant {self.key!r} has no source for capability {cap.value!r}")
88
+
89
+ def has(self, *caps: Capability) -> bool:
90
+ mine = self.capabilities()
91
+ return all(c in mine for c in caps)
92
+
93
+
94
+ # --------------------------------------------------------------------------- skill registry
95
+ @dataclass(frozen=True)
96
+ class Skill:
97
+ """A portable dashboard. `requires` are the capabilities it needs; the harness only offers it to a
98
+ tenant whose connected sources cover them. `render` is the UI entrypoint (given tenant + context)."""
99
+ key: str
100
+ label: str
101
+ requires: tuple[Capability, ...]
102
+ render: Callable | None = None
103
+ hq: bool = False # consolidated / company-level (ignores the business-unit filter)
104
+
105
+ def available_for(self, tenant: Tenant) -> bool:
106
+ return tenant.has(*self.requires)
107
+
108
+
109
+ _SKILLS: dict[str, Skill] = {}
110
+
111
+
112
+ def register_skill(skill: Skill) -> Skill:
113
+ _SKILLS[skill.key] = skill
114
+ return skill
115
+
116
+
117
+ def skills_for(tenant: Tenant) -> list[Skill]:
118
+ """The skills a tenant can actually run, given its connected sources β€” this is the dynamic nav."""
119
+ return [s for s in _SKILLS.values() if s.available_for(tenant)]
120
+
121
+
122
+ def all_skills() -> list[Skill]:
123
+ return list(_SKILLS.values())
platform/harness/canonical.py ADDED
@@ -0,0 +1,101 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Canonical financial data model β€” the SHARED vocabulary of the harness.
2
+
3
+ Skills speak ONLY in these terms; connectors (Odoo / QuickBooks / Amazon / …) translate them to
4
+ their own source. This is the seam that decouples the dashboards from any single system: add a new
5
+ source by writing a connector that maps its API onto these entities/fields β€” every skill that needs
6
+ only the capabilities that source provides then works, unchanged.
7
+
8
+ Design rules:
9
+ - Entities and canonical field names are source-agnostic (no `amount_untaxed`, no `sale.order`).
10
+ - A skill builds a `Query` (what business question), a connector answers it (how, per source).
11
+ - Capabilities let a skill declare what it needs and a tenant show only the dashboards its
12
+ connected sources can actually serve.
13
+ """
14
+ from __future__ import annotations
15
+ from dataclasses import dataclass, field
16
+ from enum import Enum
17
+ from typing import Any
18
+
19
+
20
+ class Capability(str, Enum):
21
+ """What a connector can serve. Skills require a set of these; a tenant's nav is the union of
22
+ what its connectors provide. (A pure-accounting source like QuickBooks has LEDGER+INVOICING+
23
+ CUSTOMERS but not SKU-level SALES velocity; Amazon has SALES but no LEDGER; Odoo has all.)"""
24
+ SALES = 'sales' # orders / order lines (SKU-level revenue, units)
25
+ INVOICING = 'invoicing' # customer invoices + credit notes (AR, returns)
26
+ LEDGER = 'ledger' # GL accounts + journal lines (P&L, balance sheet)
27
+ PAYABLES = 'payables' # vendor bills / AP
28
+ INVENTORY = 'inventory' # on-hand stock levels + valuation
29
+ PRODUCTS = 'products' # product master (SKU, category, cost)
30
+ CUSTOMERS = 'customers' # customer master (name, geo, assigned agent)
31
+
32
+
33
+ class Entity(str, Enum):
34
+ """Canonical business objects. Each connector declares which it can materialize."""
35
+ SALES_ORDER = 'sales_order'
36
+ SALES_ORDER_LINE = 'sales_order_line'
37
+ INVOICE = 'invoice' # customer invoice / credit note
38
+ JOURNAL_LINE = 'journal_line' # GL posting
39
+ GL_ACCOUNT = 'gl_account'
40
+ VENDOR_BILL = 'vendor_bill'
41
+ STOCK_LEVEL = 'stock_level'
42
+ PRODUCT = 'product'
43
+ CUSTOMER = 'customer'
44
+
45
+
46
+ # Canonical field names per entity β€” the ONLY field identifiers a skill may use. Connectors own the
47
+ # mapping to their native fields. `date` is the entity's primary time axis (used by Query windows).
48
+ FIELDS: dict[Entity, set[str]] = {
49
+ Entity.SALES_ORDER: {'date', 'amount', 'customer', 'salesperson', 'agent', 'status', 'business_unit', 'channel'},
50
+ Entity.SALES_ORDER_LINE: {'date', 'amount', 'qty', 'margin', 'product', 'category', 'customer', 'status', 'business_unit'},
51
+ Entity.INVOICE: {'date', 'amount', 'customer', 'doc_type', 'status', 'salesperson', 'business_unit'},
52
+ Entity.JOURNAL_LINE: {'date', 'amount', 'account', 'account_type', 'business_unit', 'status'},
53
+ Entity.GL_ACCOUNT: {'code', 'name', 'type'},
54
+ Entity.VENDOR_BILL: {'date', 'amount', 'vendor', 'status'},
55
+ Entity.STOCK_LEVEL: {'product', 'qty', 'value', 'location'},
56
+ Entity.PRODUCT: {'id', 'code', 'name', 'category', 'cost', 'active'},
57
+ Entity.CUSTOMER: {'id', 'name', 'city', 'state', 'agent', 'active'},
58
+ }
59
+
60
+ # Allowed filter/aggregate vocabulary (connectors translate these; they never see source operators).
61
+ OPS = frozenset({'eq', 'ne', 'in', 'not_in', 'gte', 'lte', 'gt', 'lt', 'like'})
62
+ AGGS = frozenset({'sum', 'count', 'avg', 'min', 'max'})
63
+ GRAINS = frozenset({'day', 'week', 'month', 'quarter', 'year'}) # time-bucket for a trend
64
+
65
+
66
+ @dataclass(frozen=True)
67
+ class Filter:
68
+ """A canonical predicate: (canonical_field, op, value). op ∈ OPS."""
69
+ fieldname: str
70
+ op: str
71
+ value: Any
72
+
73
+ def __post_init__(self):
74
+ if self.op not in OPS:
75
+ raise ValueError(f'unknown op {self.op!r}; allowed {sorted(OPS)}')
76
+
77
+
78
+ @dataclass(frozen=True)
79
+ class Query:
80
+ """A source-agnostic analytical request. `measures` are canonical fields aggregated by `aggregate`,
81
+ optionally grouped by canonical `group_by` fields, filtered by `filters`, over [date_from, date_to]
82
+ on the entity's `date` axis. A connector returns a list of plain dicts keyed by the canonical
83
+ measure/group names (+ 'count'). Skills post-process those dicts β€” never source rows."""
84
+ entity: Entity
85
+ measures: tuple[str, ...] = ()
86
+ aggregate: str = 'sum'
87
+ group_by: tuple[str, ...] = ()
88
+ filters: tuple[Filter, ...] = ()
89
+ date_from: str | None = None
90
+ date_to: str | None = None
91
+ grain: str | None = None # bucket the date axis into a trend; result rows carry 'period' (e.g. '2026-05')
92
+
93
+ def __post_init__(self):
94
+ if self.aggregate not in AGGS:
95
+ raise ValueError(f'unknown aggregate {self.aggregate!r}')
96
+ if self.grain is not None and self.grain not in GRAINS:
97
+ raise ValueError(f'unknown grain {self.grain!r}; allowed {sorted(GRAINS)}')
98
+ valid = FIELDS.get(self.entity, set())
99
+ for f in list(self.measures) + list(self.group_by) + [x.fieldname for x in self.filters]:
100
+ if f not in valid:
101
+ raise ValueError(f'{f!r} is not a canonical field of {self.entity.value} (have {sorted(valid)})')
platform/harness/chatlog.py ADDED
@@ -0,0 +1,102 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """harness/chatlog.py β€” durable Analyst conversations (2026-07-16).
2
+
3
+ Every Analyst conversation is a logged SESSION that survives reruns, page navigation and Space
4
+ restarts. Persistence rides the platform's only writable store (core/store.py β†’ the private HF
5
+ dataset), the SAME mechanism as users/notes β€” Odoo stays read-only.
6
+
7
+ Shape (store key 'analyst_sessions'): { username: [ session, ... ] } (newest first), each
8
+ session = {id, title, started_at, updated_at, ui:[[role,content,artifacts],...],
9
+ llm:[{role,content},...]}.
10
+ ui = the redisplay transcript; llm = the trimmed model-context transcript.
11
+
12
+ Write posture (per the design review): the LIVE turn stays in st.session_state for a snappy chat;
13
+ we persist in a BACKGROUND daemon thread (core/store.update is a full dataset commit β€” never block
14
+ the UI on it), and bound growth (MAX_SESSIONS per user, MAX_TURNS per session) so the JSON stays
15
+ small. A failed/absent store degrades to in-memory only (chat still works this session).
16
+ """
17
+ import threading
18
+ import time
19
+ import uuid
20
+
21
+ import core.store as store
22
+
23
+ KEY = "analyst_sessions"
24
+ MAX_SESSIONS = 40 # keep the last N conversations per user (bounds the JSON)
25
+ MAX_TURNS = 60 # ui/llm entries kept per conversation
26
+ TITLE_LEN = 60
27
+
28
+
29
+ def new_id():
30
+ return uuid.uuid4().hex[:12]
31
+
32
+
33
+ def new_session(sid=None):
34
+ now = time.strftime("%Y-%m-%d %H:%M:%S")
35
+ return {"id": sid or new_id(), "title": "", "started_at": now, "updated_at": now,
36
+ "ui": [], "llm": []}
37
+
38
+
39
+ def _safe(obj):
40
+ """JSON round-trip a value (chart/kpi artifacts) so only serializable content is stored;
41
+ anything that won't serialize is dropped rather than breaking the whole write."""
42
+ import json
43
+ try:
44
+ return json.loads(json.dumps(obj, default=str))
45
+ except Exception:
46
+ return None
47
+
48
+
49
+ def load_sessions(username):
50
+ """The user's conversations, newest first (lenient read β€” never raises on a store blip)."""
51
+ if not store.available():
52
+ return []
53
+ try:
54
+ all_s = store.get(KEY) or {}
55
+ except Exception:
56
+ return []
57
+ return list(all_s.get(username, []))
58
+
59
+
60
+ def touch(session, question):
61
+ """Stamp updated_at and derive a title from the first user question."""
62
+ session["updated_at"] = time.strftime("%Y-%m-%d %H:%M:%S")
63
+ if not session.get("title") and question:
64
+ t = " ".join(str(question).split())
65
+ session["title"] = t[:TITLE_LEN] + ("…" if len(t) > TITLE_LEN else "")
66
+ # bound in-memory transcript length (keep the most recent turns)
67
+ if len(session.get("ui", [])) > MAX_TURNS:
68
+ session["ui"] = session["ui"][-MAX_TURNS:]
69
+ if len(session.get("llm", [])) > MAX_TURNS:
70
+ session["llm"] = session["llm"][-MAX_TURNS:]
71
+ return session
72
+
73
+
74
+ def upsert(sessions, session):
75
+ """Return the sessions list with `session` moved to the front (deduped by id), capped."""
76
+ out = [s for s in (sessions or []) if s.get("id") != session.get("id")]
77
+ out.insert(0, session)
78
+ return out[:MAX_SESSIONS]
79
+
80
+
81
+ def _persist(username, session):
82
+ # sanitize artifacts for storage (charts are plain dicts; drop any non-serializable stragglers)
83
+ stored = dict(session)
84
+ stored["ui"] = [[role, content, [a for a in (_safe(x) for x in (arts or [])) if a is not None]]
85
+ for (role, content, arts) in session.get("ui", [])]
86
+
87
+ def _fn(data):
88
+ data = data or {}
89
+ data[username] = upsert(data.get(username, []), stored)
90
+ return data
91
+ try:
92
+ store.update(KEY, _fn)
93
+ except Exception:
94
+ pass # store blip / no token β†’ in-memory only this run
95
+
96
+
97
+ def save_async(username, session):
98
+ """Persist in a daemon thread so the chat never waits on a dataset commit."""
99
+ if not store.available():
100
+ return
101
+ threading.Thread(target=_persist, args=(username, dict(session)),
102
+ daemon=True, name="analyst-chatlog").start()
platform/harness/connectors/__init__.py ADDED
@@ -0,0 +1 @@
 
 
1
+ """Source connectors β€” one module per system (odoo, quickbooks, amazon, ...)."""
platform/harness/connectors/odoo.py ADDED
@@ -0,0 +1,147 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Odoo connector β€” the reference implementation.
2
+
3
+ ALL Odoo specifics (model names, field names, XML-RPC quirks, the read_group dot-path gotchas) live
4
+ here and nowhere else. It maps the canonical model onto the existing, battle-tested `core.odoo` data
5
+ layer (read-only, write-guarded) so nothing is re-plumbed β€” the harness just gives it a source-agnostic
6
+ face. Every other connector (QuickBooks, Amazon, …) is a sibling of this file.
7
+ """
8
+ from __future__ import annotations
9
+ import sys
10
+ from dataclasses import replace
11
+ from functools import lru_cache
12
+ from pathlib import Path
13
+ sys.path.insert(0, str(Path(__file__).resolve().parents[2])) # platform/
14
+ import core.odoo as O
15
+ from harness.canonical import Capability, Entity, Query, Filter
16
+
17
+
18
+ @lru_cache(maxsize=1)
19
+ def _product_category_map():
20
+ """product_id -> main category name (Odoo-side enrichment; reuses the existing resolver)."""
21
+ import modules.sales as _sales
22
+ return _sales._product_cat()
23
+
24
+
25
+ def _replace_group(q: Query, new_group_by):
26
+ return replace(q, group_by=new_group_by, grain=None)
27
+
28
+ _OPS = {'eq': '=', 'ne': '!=', 'in': 'in', 'not_in': 'not in',
29
+ 'gte': '>=', 'lte': '<=', 'gt': '>', 'lt': '<', 'like': 'ilike'}
30
+
31
+ # Canonical entity -> Odoo model + (canonical field -> Odoo field) + its time axis.
32
+ _MAP = {
33
+ Entity.SALES_ORDER: {
34
+ 'model': 'sale.order', 'date': 'date_order', 'dt': True,
35
+ 'f': {'amount': 'amount_untaxed', 'customer': 'partner_id', 'salesperson': 'user_id',
36
+ 'status': 'state', 'business_unit': 'team_id'}},
37
+ Entity.SALES_ORDER_LINE: {
38
+ 'model': 'sale.order.line', 'date': 'order_id.date_order', 'dt': True,
39
+ 'f': {'amount': 'price_subtotal', 'qty': 'product_uom_qty', 'margin': 'margin',
40
+ 'product': 'product_id', 'customer': 'order_partner_id',
41
+ 'status': 'order_id.state', 'business_unit': 'order_id.team_id'}},
42
+ Entity.INVOICE: {
43
+ 'model': 'account.move', 'date': 'invoice_date', 'dt': False,
44
+ 'f': {'amount': 'amount_untaxed_signed', 'customer': 'partner_id', 'doc_type': 'move_type',
45
+ 'status': 'state', 'salesperson': 'invoice_user_id', 'business_unit': 'team_id'}},
46
+ Entity.STOCK_LEVEL: {
47
+ 'model': 'stock.quant', 'date': None, 'dt': False,
48
+ 'f': {'product': 'product_id', 'qty': 'quantity', 'location': 'location_id'}},
49
+ Entity.PRODUCT: {
50
+ 'model': 'product.product', 'date': None, 'dt': False,
51
+ 'f': {'id': 'id', 'code': 'default_code', 'name': 'name', 'category': 'categ_id',
52
+ 'cost': 'standard_price', 'active': 'active'}},
53
+ Entity.CUSTOMER: {
54
+ 'model': 'res.partner', 'date': None, 'dt': False,
55
+ 'f': {'id': 'id', 'name': 'name', 'city': 'city', 'state': 'state_id',
56
+ 'agent': 'agent_ids', 'active': 'active'}},
57
+ }
58
+
59
+
60
+ class OdooConnector:
61
+ key = 'odoo'
62
+ name = 'Odoo ERP'
63
+
64
+ def __init__(self, doc_mode: str = 'order'):
65
+ self._doc_mode = doc_mode
66
+
67
+ def capabilities(self) -> set[Capability]:
68
+ return {Capability.SALES, Capability.INVOICING, Capability.LEDGER, Capability.PAYABLES,
69
+ Capability.INVENTORY, Capability.PRODUCTS, Capability.CUSTOMERS}
70
+
71
+ def entities(self) -> set[Entity]:
72
+ return set(_MAP)
73
+
74
+ def supports(self, cap: Capability) -> bool:
75
+ return cap in self.capabilities()
76
+
77
+ # ------------------------------------------------------------------ query translation
78
+ def _domain(self, q: Query, meta):
79
+ dom = []
80
+ if q.date_from and meta['date']:
81
+ dom.append((meta['date'], '>=', q.date_from + (' 00:00:00' if meta['dt'] else '')))
82
+ if q.date_to and meta['date']:
83
+ dom.append((meta['date'], '<=', q.date_to + (' 23:59:59' if meta['dt'] else '')))
84
+ for flt in q.filters:
85
+ dom.append((meta['f'].get(flt.fieldname, flt.fieldname), _OPS[flt.op], flt.value))
86
+ return dom
87
+
88
+ def aggregate(self, q: Query) -> list[dict]:
89
+ meta = _MAP[q.entity]
90
+ O.set_doc_mode(self._doc_mode)
91
+ # 'category' isn't a native line field (dot-path groupby fails) β€” enrich from the product master.
92
+ if 'category' in q.group_by:
93
+ return self._by_category(q, meta)
94
+ dom = self._domain(q, meta)
95
+ measures = [f"{meta['f'].get(m, m)}:{q.aggregate}" for m in q.measures]
96
+ groupby = [meta['f'].get(g, g) for g in q.group_by]
97
+ if q.grain and meta['date']: # time-bucket -> Odoo's date:grain groupby
98
+ groupby.append(f"{meta['date']}:{q.grain}")
99
+ rows = O.read_group(meta['model'], dom, measures, groupby, lazy=False)
100
+ return [self._canon_row(r, q, meta) for r in rows]
101
+
102
+ def _canon_row(self, r, q, meta):
103
+ rec = {}
104
+ for m in q.measures:
105
+ rec[m] = r.get(meta['f'].get(m, m)) or 0.0
106
+ for g in q.group_by:
107
+ v = r.get(meta['f'].get(g, g))
108
+ rec[g] = ({'id': O.m2o_id(v), 'name': O.m2o_name(v)} if isinstance(v, (list, tuple)) else v)
109
+ if q.grain and meta['date']:
110
+ rng = (r.get('__range') or {}).get(f"{meta['date']}:{q.grain}") or {}
111
+ rec['period'] = (rng.get('from') or '')[:7]
112
+ rec['count'] = r.get('__count') or 0
113
+ return rec
114
+
115
+ def _by_category(self, q, meta):
116
+ """Group SALES_ORDER_LINE by main product category by grouping on product then folding
117
+ product->category in-connector (source-specific enrichment lives here, not in the skill)."""
118
+ catmap = _product_category_map()
119
+ sub = _replace_group(q, tuple(g for g in q.group_by if g != 'category') + ('product',))
120
+ agg = {}
121
+ for r in self.aggregate(sub):
122
+ prod = r.get('product') or {}
123
+ cat = catmap.get(prod.get('id') if isinstance(prod, dict) else prod, '(uncategorized)')
124
+ e = agg.setdefault(cat, {'category': cat, 'count': 0})
125
+ for m in q.measures:
126
+ e[m] = e.get(m, 0.0) + (r.get(m) or 0.0)
127
+ e['count'] += r.get('count') or 0
128
+ return sorted(agg.values(), key=lambda x: -(x.get(q.measures[0]) if q.measures else 0))
129
+
130
+ def distinct_count(self, q: Query, fieldname: str) -> int:
131
+ meta = _MAP[q.entity]
132
+ O.set_doc_mode(self._doc_mode)
133
+ return O.distinct_count(meta['model'], self._domain(q, meta), meta['f'].get(fieldname, fieldname))
134
+
135
+ def records(self, q: Query) -> list[dict]:
136
+ meta = _MAP[q.entity]
137
+ O.set_doc_mode(self._doc_mode)
138
+ fields = [meta['f'].get(m, m) for m in (q.measures or tuple(meta['f']))]
139
+ rows = O.search_read(meta['model'], self._domain(q, meta), fields)
140
+ inv = {v: k for k, v in meta['f'].items()}
141
+ return [{inv.get(k, k): (O.m2o_name(v) if isinstance(v, (list, tuple)) else v)
142
+ for k, v in r.items()} for r in rows]
143
+
144
+ # convenience for tenant setup: resolve names β†’ ids for scope config
145
+ @staticmethod
146
+ def excluded_customer_ids() -> list[int]:
147
+ return list(O.excluded_partner_ids())
platform/harness/connectors/quickbooks.py ADDED
@@ -0,0 +1,37 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """QuickBooks Online connector β€” STUB (scaffolding for the next real integration).
2
+
3
+ It exists to prove the harness is source-agnostic: it declares the capabilities QBO actually has
4
+ (accounting + AR + customers, but NOT SKU-level sales velocity or warehouse inventory), so the
5
+ platform will automatically OFFER ledger/AR/customer skills on a QuickBooks-only tenant and HIDE the
6
+ SKU/inventory skills β€” no per-skill if-statements anywhere. Fill in aggregate() against the QBO
7
+ Reports/Query API (OAuth2) to make it live.
8
+ """
9
+ from __future__ import annotations
10
+ from harness.canonical import Capability, Entity, Query
11
+
12
+
13
+ class QuickBooksConnector:
14
+ key = 'quickbooks'
15
+ name = 'QuickBooks Online'
16
+
17
+ def __init__(self, realm_id: str | None = None, token: str | None = None):
18
+ self.realm_id = realm_id
19
+ self._token = token # OAuth2 access token (never logged)
20
+
21
+ def capabilities(self) -> set[Capability]:
22
+ # QBO gives you the books + receivables + the customer list; it does not model SKU-level
23
+ # order lines or physical stock the way an ERP/marketplace does.
24
+ return {Capability.LEDGER, Capability.INVOICING, Capability.PAYABLES, Capability.CUSTOMERS}
25
+
26
+ def entities(self) -> set[Entity]:
27
+ return {Entity.INVOICE, Entity.JOURNAL_LINE, Entity.GL_ACCOUNT, Entity.VENDOR_BILL, Entity.CUSTOMER}
28
+
29
+ def supports(self, cap: Capability) -> bool:
30
+ return cap in self.capabilities()
31
+
32
+ def aggregate(self, q: Query) -> list[dict]:
33
+ raise NotImplementedError(
34
+ "QuickBooksConnector.aggregate: TODO β€” translate the canonical Query to the QBO "
35
+ "Query/Reports API (GET /v3/company/{realm}/query) and map rows back to canonical dicts. "
36
+ "Entity map: INVOICE->Invoice/CreditMemo, JOURNAL_LINE->JournalEntry lines, "
37
+ "GL_ACCOUNT->Account, VENDOR_BILL->Bill, CUSTOMER->Customer.")
platform/harness/datastore.py ADDED
@@ -0,0 +1,750 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """harness/store.py β€” the tenant data store (OM-1, 2026-07-11).
2
+
3
+ Incremental Odoo β†’ DuckDB mirror: the local analytical store that saved views/dashboards (OM-3)
4
+ and the AI Analyst's tools (OM-4) query at warehouse speed, instead of hammering live XML-RPC
5
+ per question.
6
+ Plan: .claude/wiki/research/omni-adoption.md Part IV (OM-1) + [[odoo-open-source]] BUILD-NOW #1.
7
+
8
+ Design rules:
9
+ - The store is a RAW mirror (unscoped). Business scope (wholesale teams, GIFTWARE exclusion,
10
+ confirmed-only) is applied by the SEMANTIC layer at query time (OM-2) β€” one source of truth.
11
+ - Sync is CHECKPOINTED + BOUNDED + RESUMABLE (loop-library discipline): backfill paginates by
12
+ id; live mode advances a write_date cursor; every run is bounded by max_batches and safe to
13
+ kill/re-run (upserts are idempotent).
14
+ - validate() compares the store against LIVE Odoo (raw fidelity: counts + monthly sums to the
15
+ cent) β€” the store never validates itself. Odoo hard-deletes (unlink) don't move write_date;
16
+ the count checks are the drift detector for those.
17
+ - Client data: the .duckdb file lives under data/store/ (git-ignored). READ-ONLY on Odoo.
18
+ """
19
+ import datetime as dt
20
+ import json
21
+ import os
22
+ import sys
23
+ import time
24
+ from pathlib import Path
25
+
26
+ import duckdb
27
+
28
+ import core.odoo as O
29
+
30
+ _STORE_DIR = Path(__file__).resolve().parents[1] / "data" / "store"
31
+
32
+ # ⚠ MUTABLE ON PURPOSE, and read LATE by every function below (Python resolves a module global at
33
+ # CALL time), which is how `provision_tenant.py` already points a fresh instance at its own file.
34
+ # EXIT-4a (X7) formalises that: `AIOS_DUCKDB_PATH` lets a container name the file without an edit,
35
+ # `path_for()` owns the per-tenant naming convention, and `use_path()` is the SAFE way to switch β€”
36
+ # see its docstring for why a bare assignment is not.
37
+ # The DEFAULT is unchanged: <app root>/data/store/royal.duckdb.
38
+ DB_PATH = Path(os.environ.get("AIOS_DUCKDB_PATH") or (_STORE_DIR / "royal.duckdb"))
39
+
40
+
41
+ def path_for(tenant_key):
42
+ """The canonical DuckDB file for a tenant slug. `royal-imports` keeps the historical
43
+ `royal.duckdb` name so tenant #0's existing 175 MB file is not orphaned by a rename.
44
+
45
+ File-per-tenant IS the isolation model (C1c): DuckDB has no row-level security, so the only
46
+ boundary that holds is the operating system's β€” one file, one tenant.
47
+ """
48
+ key = str(tenant_key or "").strip().lower()
49
+ if key in ("", "royal-imports"):
50
+ return DB_PATH if key == "" else Path(
51
+ os.environ.get("AIOS_DUCKDB_PATH") or (_STORE_DIR / "royal.duckdb"))
52
+ safe = "".join(c if (c.isalnum() or c in "-_") else "_" for c in key)[:60]
53
+ return _STORE_DIR / f"{safe}.duckdb"
54
+
55
+
56
+ def use_path(path):
57
+ """Repoint the store at `path`, CLOSING the process singleton first.
58
+
59
+ ⚠ A bare `datastore.DB_PATH = …` is not enough and is the more dangerous half of this
60
+ operation. `_instance()` caches ONE open connection for the life of the process, so after a
61
+ plain reassignment every read keeps being served from the PREVIOUSLY opened file β€” a silent
62
+ cross-tenant read, returning real rows that belong to somebody else. Closing the connection
63
+ and clearing the readiness memo (which is also per-file) is what makes the switch honest.
64
+
65
+ ⚠ AND CLOSING THE CONNECTION IS STILL NOT ENOUGH ON ITS OWN, because `ro_con()` caches a
66
+ cursor in a `threading.local`: this function can only clear the CALLING thread's, and
67
+ `ro_con`'s liveness probe (`SELECT 1`) SUCCEEDS on a cursor whose underlying connection was
68
+ closed out from under it in some cases β€” so another thread would keep answering from the old
69
+ file with no error to notice. The generation counter below is what closes that: `ro_con`
70
+ compares the generation its cursor was opened under and reopens when it has moved, which is
71
+ a check no individual thread can forget to do.
72
+
73
+ Returns the new path. Callers: `provision_tenant.py`, and any single-tenant worker process.
74
+ NOT a per-request operation β€” one process serves one analytical store at a time, which is why
75
+ `harness.runtime` hands out the PATH rather than switching the global underneath a request.
76
+ """
77
+ global DB_PATH
78
+ with _INSTANCE_LOCK:
79
+ con = _INSTANCE.get("con")
80
+ if con is not None:
81
+ try:
82
+ con.close()
83
+ except Exception:
84
+ pass
85
+ _INSTANCE["con"] = None
86
+ _RO_TLS.__dict__.pop("cur", None)
87
+ _READY["ok"] = False
88
+ _GENERATION[0] += 1 # every other thread's cached cursor is now stale
89
+ DB_PATH = Path(path)
90
+ return DB_PATH
91
+
92
+ # Entity specs: Odoo model β†’ store table. m2o fields land as <name>_id + <name>_name.
93
+ # 'archivable' entities are pulled with active in [True, False] (the re-SKU/merge rule).
94
+ ENTITIES = {
95
+ "sale_order": {
96
+ "model": "sale.order", "archivable": False,
97
+ "fields": ["name", "date_order", "partner_id", "team_id", "user_id", "state",
98
+ "amount_untaxed", "invoice_status", "write_date"],
99
+ },
100
+ "sale_order_line": {
101
+ "model": "sale.order.line", "archivable": False,
102
+ "fields": ["order_id", "order_partner_id", "product_id", "product_uom_qty",
103
+ "price_subtotal", "margin", "purchase_price", "write_date"],
104
+ },
105
+ "res_partner": {
106
+ "model": "res.partner", "archivable": True,
107
+ # `agent` (bool, labelled "Creditor/Agent") is THE discriminator between a real sales
108
+ # AGENT and an internal SALESPERSON who merely carries commission lines β€” the commission
109
+ # table itself cannot tell them apart. `salesman_as_agent` ("convert salesman into agent")
110
+ # marks agents who came from staff; the owner ruling on whether those count as agents is
111
+ # OUTSTANDING, so both flags are synced and neither reading is baked in.
112
+ "fields": ["name", "team_id", "city", "state_id", "country_id", "active",
113
+ "agent", "salesman_as_agent", "write_date"],
114
+ "backfill_fields": ["agent", "salesman_as_agent"], # added 2026-07-28
115
+ # m2m fields land as <base>_id (the FIRST linked id β€” the Customers-module convention:
116
+ # a customer's assigned agent is agent_ids[0], ~one per customer, MECE) + <field> as a
117
+ # JSON list of ALL ids. Declared explicitly: a 2-element m2m read would otherwise be
118
+ # indistinguishable from an m2o (id, name) pair in _flatten.
119
+ "m2m": ["agent_ids"],
120
+ },
121
+ "product_product": {
122
+ "model": "product.product", "archivable": True,
123
+ "fields": ["default_code", "name", "categ_id", "type", "standard_price", "active",
124
+ "write_date"],
125
+ },
126
+ "account_move": {
127
+ "model": "account.move", "archivable": False,
128
+ "fields": ["name", "move_type", "state", "invoice_date", "invoice_date_due",
129
+ "partner_id", "amount_untaxed_signed", "amount_residual_signed",
130
+ "payment_state", "write_date"],
131
+ },
132
+ "account_move_line": {
133
+ "model": "account.move.line", "archivable": False,
134
+ # display_type/move_type/parent_state/price_subtotal/product_id added 2026-07-28 so the
135
+ # invoice-LINE grain is queryable (topic `invoice_lines`).
136
+ # ⚠ display_type values here are 'product', 'cogs', 'payment_term', 'line_note',
137
+ # 'line_section'. Commission rows attach to 'cogs' lines as well as 'product' ones
138
+ # (96,936 of 147,160 in 2025). Filtering to 'product' is a GRAIN guard: measured
139
+ # 2026-07-29, omitting it does not move revenue (non-product lines carry
140
+ # price_subtotal = 0) but it inflates commission ROW COUNTS ~80% and would corrupt any
141
+ # count-of-lines measure.
142
+ "fields": ["move_id", "account_id", "partner_id", "date", "debit", "credit",
143
+ "balance", "display_type", "move_type", "parent_state", "price_subtotal",
144
+ "product_id", "write_date"],
145
+ "backfill_fields": ["display_type", "move_type", "parent_state", "price_subtotal",
146
+ "product_id"], # added 2026-07-28
147
+ # the BU bridge: account.move carries NO business unit (every invoice sits on team 1 β€”
148
+ # see [[invoice-bu-attribution]]), so Fisch/Royal on the invoice basis is only reachable
149
+ # through the originating sale order. sale_line_id (the FIRST linked sale line) is
150
+ # lossless here: ZERO 2025 invoice lines span more than one team (measured).
151
+ "m2m": ["sale_line_ids"],
152
+ },
153
+ "account_invoice_line_agent": {
154
+ # The OCA sale-commission module: one row per (invoice line Γ— agent). This is the SECOND
155
+ # agent source β€” per-line commission attribution β€” and it names a different population
156
+ # than res_partner.agent_ids (the customer-master book). See topic `invoice_lines`.
157
+ # ⚠ TWO similarly-named FKs, do not confuse them:
158
+ # object_id -> account_move_line (THE grain; join on this)
159
+ # invoice_id -> account_move (the parent document)
160
+ "model": "account.invoice.line.agent", "archivable": False,
161
+ "fields": ["agent_id", "commission_id", "amount", "invoice_id", "object_id",
162
+ "invoice_date", "settled", "write_date"],
163
+ },
164
+ "account_account": {
165
+ # NOTE: no `active` field on account.account in this Odoo version (learned 2026-07-11 β€”
166
+ # a ('active','in',...) domain 500s); treat as non-archivable.
167
+ "model": "account.account", "archivable": False,
168
+ "fields": ["code", "name", "account_type", "write_date"],
169
+ },
170
+ }
171
+
172
+ BATCH = 2000
173
+
174
+ # Fields that are genuinely BOOLEAN. This list is load-bearing in TWO places: _cols types the
175
+ # column, and _flatten must NOT collapse a real False into NULL. Odoo returns False both for
176
+ # "empty" and for "boolean false", so a bool field missing from here silently becomes NULL β€” and
177
+ # for res_partner.agent that would erase the ONE flag separating an AGENT from a SALESPERSON
178
+ # (see [[invoice-line-agent-commission]]): every row would read "not an agent" indistinguishably
179
+ # from "unknown".
180
+ BOOL_FIELDS = ("active", "agent", "salesman_as_agent", "settled", "commission_free")
181
+
182
+
183
+ import threading as _thr
184
+
185
+ _RO_TLS = _thr.local()
186
+ _READY = {"ok": False}
187
+ _INSTANCE = {"con": None}
188
+ _INSTANCE_LOCK = _thr.Lock()
189
+ #: Bumped by `use_path()` whenever the store FILE changes. Every per-thread cursor is stamped with
190
+ #: the generation it was opened under, so a thread that never called `use_path` still cannot keep
191
+ #: reading the previous tenant's file β€” see `ro_con`. A plain "is the cursor alive?" probe cannot
192
+ #: answer that question, which is why this counter exists rather than more probing.
193
+ _GENERATION = [0]
194
+
195
+
196
+ def mark_ready():
197
+ """In-process signal that the store is fully synced (every entity live). The sync writer calls
198
+ this after an all-live pass so a reader never has to (re)discover usability by touching the
199
+ file. Belt-and-suspenders alongside the shared-instance model below."""
200
+ _READY["ok"] = True
201
+
202
+
203
+ def _instance():
204
+ """THE one process-wide read-write DuckDB connection. Readers take .cursor() off it and the
205
+ sync writer uses it too β€” DuckDB serves concurrent reads + a single writer from ONE instance
206
+ via MVCC, so a read NEVER contends with a sync pass for the file lock.
207
+
208
+ This is the fix for the 'connection error unless I refresh' bug (2026-07-16): an independent
209
+ read-only `duckdb.connect()` FAILS while the writer holds the lock ('Can't open a connection
210
+ to same database file with a different configuration than existing connections'); a cursor on
211
+ the shared instance does not. Openers must ensure the file is fully present first β€” ensure_seed
212
+ writes atomically (os.replace) and every read path guards on DB_PATH.exists() β€” so the instance
213
+ never binds a partial or absent file. (Trade-off: the app now holds the write lock for its whole
214
+ life, so a SEPARATE process β€” e.g. mcp_server.py β€” cannot open the same store file concurrently;
215
+ in production only the app runs, and that was already true during any sync pass.)"""
216
+ con = _INSTANCE["con"]
217
+ if con is not None:
218
+ return con
219
+ with _INSTANCE_LOCK:
220
+ if _INSTANCE["con"] is None:
221
+ DB_PATH.parent.mkdir(parents=True, exist_ok=True)
222
+ _INSTANCE["con"] = duckdb.connect(str(DB_PATH))
223
+ return _INSTANCE["con"]
224
+
225
+
226
+ def ro_cursor():
227
+ """A read cursor on the shared instance β€” the contention-free way to read the store."""
228
+ return _instance().cursor()
229
+
230
+
231
+ def ready():
232
+ """True once EVERY entity has completed its backfill (phase 'live'). Gate all store READS on
233
+ this: a mid-backfill store would return PARTIAL totals silently β€” worse than an error. Positive
234
+ results cache for the process (backfill never regresses). Reads via a shared-instance cursor,
235
+ so it is never mis-reported as 'not ready' just because a background sync pass is mid-flight."""
236
+ if _READY["ok"]:
237
+ return True
238
+ if not DB_PATH.exists():
239
+ return False
240
+ try:
241
+ cur = ro_cursor()
242
+ try:
243
+ phases = {r[0]: r[1] for r in
244
+ cur.execute("SELECT entity, phase FROM _sync_state").fetchall()}
245
+ finally:
246
+ cur.close()
247
+ except Exception:
248
+ return False
249
+ if all(phases.get(k) == "live" for k in ENTITIES):
250
+ _READY["ok"] = True
251
+ return True
252
+ return False
253
+
254
+
255
+ def ro_con():
256
+ """Per-thread cached read CURSOR on the shared instance for page/module store reads (the
257
+ open cost dominates repeated small queries — the Expenses retrofit measured 7.7s→1.0s). No
258
+ lock contention with the sync writer; a cursor that has gone bad is reopened once. REFUSES
259
+ until ready() β€” callers either fall back to live reads (modules) or relay a readable message
260
+ (Analyst).
261
+
262
+ ⚠ THE GENERATION CHECK IS A CORRECTNESS GUARD, NOT AN OPTIMISATION (EXIT-4a). The liveness
263
+ probe below asks "does this cursor still work?", which is a different question from "is this
264
+ cursor still pointing at the file we are supposed to be reading?". After `use_path()` switches
265
+ tenants, a cursor cached in ANOTHER thread's `threading.local` can still answer `SELECT 1` β€”
266
+ and would then serve that thread real, plausible rows from the PREVIOUS tenant's file, with no
267
+ exception anywhere. `use_path` bumps the generation; a cursor opened under an older one is
268
+ discarded here, which is the only place every thread is guaranteed to pass through."""
269
+ cur = getattr(_RO_TLS, "cur", None)
270
+ if cur is not None and getattr(_RO_TLS, "gen", None) != _GENERATION[0]:
271
+ try:
272
+ cur.close()
273
+ except Exception:
274
+ pass
275
+ cur, _RO_TLS.cur = None, None # the store moved underneath this thread
276
+ if cur is not None:
277
+ try:
278
+ cur.execute("SELECT 1") # cheap liveness probe
279
+ return cur
280
+ except Exception:
281
+ _RO_TLS.cur = None # stale β†’ reopen below
282
+ if not ready():
283
+ raise RuntimeError("the tenant data store is completing its FIRST sync on this "
284
+ "deployment β€” dashboards work meanwhile (live reads); retry "
285
+ "store queries in a few minutes")
286
+ cur = ro_cursor()
287
+ _RO_TLS.cur = cur
288
+ _RO_TLS.gen = _GENERATION[0] # stamp WHICH file this cursor belongs to
289
+ return cur
290
+
291
+
292
+ SEED_DATASET = os.environ.get("STORE_SEED_DATASET", "royal-imports/cfo-os-data")
293
+
294
+
295
+ def ensure_seed():
296
+ """Fresh/ephemeral deployment (HF Space disks reset on every rebuild): hydrate the store
297
+ from the PRIVATE dataset's seed snapshot β€” Analyst-ready in ~a minute instead of a 20-40 min
298
+ XML-RPC backfill; the auto-sync then closes the gap via write_date cursors. Also restores
299
+ the pilot views/routines when absent. Fail-quiet: no seed/token β†’ normal backfill."""
300
+ import shutil
301
+ if DB_PATH.exists():
302
+ return False
303
+ tok = os.environ.get("HF_TOKEN")
304
+ if not tok:
305
+ return False
306
+ try:
307
+ from huggingface_hub import hf_hub_download
308
+ DB_PATH.parent.mkdir(parents=True, exist_ok=True)
309
+ p = hf_hub_download(SEED_DATASET, "store_seed/royal.duckdb",
310
+ repo_type="dataset", token=tok)
311
+ # Atomic install: copy to a temp path then os.replace β€” a reader that guards on
312
+ # DB_PATH.exists() must never bind a half-written file into the shared instance.
313
+ tmp = DB_PATH.with_name(DB_PATH.name + ".tmp")
314
+ shutil.copyfile(p, tmp)
315
+ os.replace(tmp, DB_PATH)
316
+ for fn in ("views.json", "routines.json"):
317
+ tgt = DB_PATH.parent / fn
318
+ if not tgt.exists():
319
+ try:
320
+ q = hf_hub_download(SEED_DATASET, f"store_seed/{fn}",
321
+ repo_type="dataset", token=tok)
322
+ shutil.copyfile(q, tgt)
323
+ except Exception:
324
+ pass
325
+ return True
326
+ except Exception:
327
+ return False
328
+
329
+
330
+ def connect():
331
+ """A cursor on the shared instance for the SYNC WRITER (and status/validate). Returned as a
332
+ cursor so callers' con.close() frees the cursor without closing the process-wide instance;
333
+ one writer cursor is active at a time (the sync thread is sequential), readers use others."""
334
+ cur = _instance().cursor()
335
+ cur.execute("""CREATE TABLE IF NOT EXISTS _sync_state (
336
+ entity VARCHAR PRIMARY KEY, phase VARCHAR, last_id BIGINT,
337
+ cursor_wd VARCHAR, rows BIGINT, updated_at VARCHAR)""")
338
+ return cur
339
+
340
+
341
+ def _flatten(rec, fields, m2m=()):
342
+ out = {"id": rec["id"]}
343
+ for f in m2m:
344
+ v = rec.get(f) or []
345
+ base = f.removesuffix("_ids")
346
+ out[f"{base}_id"] = v[0] if v else None
347
+ out[f] = json.dumps(list(v)) if v else None
348
+ for f in fields:
349
+ v = rec.get(f)
350
+ if isinstance(v, (list, tuple)) and len(v) == 2 and isinstance(v[0], int):
351
+ out[f"{f.removesuffix('_id')}_id"] = v[0]
352
+ out[f"{f.removesuffix('_id')}_name"] = str(v[1])
353
+ elif isinstance(v, (list, tuple)):
354
+ out[f] = json.dumps(v)
355
+ elif v is False and f not in BOOL_FIELDS: # Odoo False = NULL for non-bool fields
356
+ out[f] = None
357
+ else:
358
+ out[f] = v
359
+ return out
360
+
361
+
362
+ def _cols(spec):
363
+ cols = ["id BIGINT PRIMARY KEY"]
364
+ for f in spec["fields"]:
365
+ base = f.removesuffix("_id")
366
+ if f.endswith("_id"):
367
+ cols += [f"{base}_id BIGINT", f"{base}_name VARCHAR"]
368
+ elif f in BOOL_FIELDS:
369
+ cols.append(f"{f} BOOLEAN")
370
+ elif f in ("amount_untaxed", "amount_untaxed_signed", "amount_residual_signed",
371
+ "price_subtotal", "margin", "purchase_price", "product_uom_qty",
372
+ "standard_price", "debit", "credit", "balance", "amount"):
373
+ cols.append(f"{f} DOUBLE")
374
+ else:
375
+ cols.append(f"{f} VARCHAR")
376
+ for f in spec.get("m2m") or []:
377
+ base = f.removesuffix("_ids")
378
+ cols += [f"{base}_id BIGINT", f"{f} VARCHAR"]
379
+ return cols
380
+
381
+
382
+ def _ensure_table(con, key, spec):
383
+ con.execute(f"CREATE TABLE IF NOT EXISTS {key} ({', '.join(_cols(spec))})")
384
+
385
+
386
+ def _ensure_columns(con, key, spec):
387
+ """Schema evolution: existing stores/seeds predate columns a newer spec introduces β€”
388
+ CREATE TABLE IF NOT EXISTS won't add them. Returns the newly added column names."""
389
+ have = {r[1] for r in con.execute(f"PRAGMA table_info('{key}')").fetchall()}
390
+ added = []
391
+ for c in _cols(spec):
392
+ name = c.split()[0]
393
+ if name not in have:
394
+ con.execute(f"ALTER TABLE {key} ADD COLUMN {c}")
395
+ added.append(name)
396
+ return added
397
+
398
+
399
+ def _field_cols(f):
400
+ """The store column(s) one spec field materialises into (m2o fields become _id + _name)."""
401
+ if f.endswith("_id"):
402
+ base = f.removesuffix("_id")
403
+ return [f"{base}_id", f"{base}_name"]
404
+ return [f]
405
+
406
+
407
+ def _backfill_columns(con, key, spec, log=print):
408
+ """Scalar/m2o schema evolution β€” the twin of _backfill_m2m, and for the same reason.
409
+
410
+ _ensure_columns ALTERs a new column in, but existing rows NEVER re-sync (WE moved the schema;
411
+ their write_date didn't move), so a newly added field stays NULL forever β€” silently, and NULL
412
+ is indistinguishable from a legitimately empty value. That is how a store starts answering
413
+ 'no rows match' instead of erroring. Re-pull just the affected fields for every row,
414
+ paginated by id, marked durably so a crash retries rather than declaring completion.
415
+
416
+ The fields are DECLARED in the spec (`backfill_fields`), never inferred from "which columns
417
+ did _ensure_columns just add" β€” that inference is wrong on the second run, when the ALTER has
418
+ already happened and the list comes back empty while the data is still missing. Declaring it
419
+ makes the migration re-entrant and reviewable. Leave the entry in place after it completes;
420
+ the durable marker, not the absence of the declaration, is what stops it re-running.
421
+ """
422
+ fields = [f for f in (spec.get("backfill_fields") or []) if f != "write_date"]
423
+ if not fields:
424
+ return
425
+ marker = f"{key}.cols.{','.join(sorted(fields))}"
426
+ st = con.execute("SELECT phase, last_id, rows FROM _sync_state WHERE entity=?",
427
+ [marker]).fetchone()
428
+ if st and st[0] == "done":
429
+ return
430
+ # RESUMABLE, not restart-from-zero: these passes run to ~1M rows, and a migration that loses
431
+ # an hour of work to one dropped connection gets skipped by the next person under time
432
+ # pressure. Progress is checkpointed every page; 'done' is written only at the end, so an
433
+ # interrupted run resumes at the last committed id instead of re-pulling or, worse, declaring
434
+ # completion it never reached.
435
+ last, n = (st[1] or 0, st[2] or 0) if st else (0, 0)
436
+ if last:
437
+ log(f" {key}: column backfill resuming at id {last:,} ({n:,} rows already done)")
438
+ cols = [c for f in fields for c in _field_cols(f)]
439
+ setter = ", ".join(f"{c}=?" for c in cols)
440
+ while True:
441
+ recs = O.search_read(spec["model"], _base_domain(spec) + [("id", ">", last)],
442
+ fields, limit=5000, order="id asc")
443
+ if not recs:
444
+ break
445
+ rows = [_flatten(r, fields) for r in recs]
446
+ con.executemany(f"UPDATE {key} SET {setter} WHERE id=?",
447
+ [[r.get(c) for c in cols] + [r["id"]] for r in rows])
448
+ last, n = recs[-1]["id"], n + len(recs)
449
+ con.execute("INSERT OR REPLACE INTO _sync_state VALUES (?,?,?,?,?,?)",
450
+ [marker, "running", last, None, n, time.strftime("%Y-%m-%d %H:%M:%S")])
451
+ log(f" {key}: column backfill {'+'.join(fields)} -> id {last:,} ({n:,} rows)")
452
+ con.execute("INSERT OR REPLACE INTO _sync_state VALUES (?,?,?,?,?,?)",
453
+ [marker, "done", last, None, n, time.strftime("%Y-%m-%d %H:%M:%S")])
454
+ log(f" {key}: column backfill complete -> {n:,} rows")
455
+
456
+
457
+ def _backfill_m2m(con, key, spec, log=print):
458
+ """One-time targeted backfill after a schema migration: existing rows never re-sync (their
459
+ write_date didn't move when WE added a column), so pull every record where the m2m field is
460
+ SET and update in place. Rows with the field empty keep NULL β€” correct and free. Completion
461
+ is a durable _sync_state marker ('<key>.m2m.<field>') β€” NOT 'column just added': a crash
462
+ between the ALTER and this backfill (seen live: the SSL-failed first run) must retry."""
463
+ for f in spec.get("m2m") or []:
464
+ marker = f"{key}.m2m.{f}"
465
+ if con.execute("SELECT 1 FROM _sync_state WHERE entity=?", [marker]).fetchone():
466
+ continue
467
+ base = f.removesuffix("_ids")
468
+ # PAGINATED by id. A single limit=100000 call silently TRUNCATED here: account_move_line
469
+ # has 228,067 rows with sale_line_ids set, so 56% of the BU bridge would have gone
470
+ # missing with no error β€” a partial backfill that reads as a complete one (the
471
+ # [[no-unverifiable-aggregates]] no-silent-caps rule).
472
+ rows, last, page = [], 0, 20000
473
+ while True:
474
+ recs = O.search_read(spec["model"],
475
+ _base_domain(spec) + [(f, "!=", False), ("id", ">", last)],
476
+ [f], limit=page, order="id asc")
477
+ if not recs:
478
+ break
479
+ rows += [(r[f][0], json.dumps(list(r[f])), r["id"]) for r in recs if r.get(f)]
480
+ last = recs[-1]["id"]
481
+ log(f" {key}: m2m {f} scanned to id {last:,} ({len(rows):,} rows)")
482
+ if rows:
483
+ con.executemany(f"UPDATE {key} SET {base}_id=?, {f}=? WHERE id=?", rows)
484
+ con.execute("INSERT OR REPLACE INTO _sync_state VALUES (?,?,?,?,?,?)",
485
+ [marker, "done", 0, None, len(rows), time.strftime("%Y-%m-%d %H:%M:%S")])
486
+ log(f" {key}: m2m backfill {f} -> {len(rows):,} rows updated")
487
+
488
+
489
+ def _upsert(con, key, spec, recs):
490
+ if not recs:
491
+ return
492
+ rows = [_flatten(r, spec["fields"], spec.get("m2m") or ()) for r in recs]
493
+ colnames = [c.split()[0] for c in _cols(spec)]
494
+ ids = [r["id"] for r in rows]
495
+ con.execute(f"DELETE FROM {key} WHERE id IN ({','.join(map(str, ids))})")
496
+ con.executemany(
497
+ f"INSERT INTO {key} ({', '.join(colnames)}) VALUES ({', '.join('?' for _ in colnames)})",
498
+ [[r.get(c) for c in colnames] for r in rows])
499
+
500
+
501
+ def _state(con, key):
502
+ row = con.execute("SELECT phase, last_id, cursor_wd, rows FROM _sync_state WHERE entity=?",
503
+ [key]).fetchone()
504
+ return {"phase": row[0], "last_id": row[1], "cursor_wd": row[2], "rows": row[3]} if row else None
505
+
506
+
507
+ def _save_state(con, key, phase, last_id, cursor_wd):
508
+ n = con.execute(f"SELECT count(*) FROM {key}").fetchone()[0]
509
+ con.execute("""INSERT OR REPLACE INTO _sync_state VALUES (?,?,?,?,?,?)""",
510
+ [key, phase, last_id, cursor_wd, n, time.strftime("%Y-%m-%d %H:%M:%S")])
511
+
512
+
513
+ def _base_domain(spec):
514
+ return [("active", "in", [True, False])] if spec["archivable"] else []
515
+
516
+
517
+ def sync_entity(key, max_batches=200, batch=BATCH, log=print):
518
+ """One bounded, resumable sync pass for an entity. Backfill (by id) β†’ live (by write_date).
519
+ Returns (phase, pulled_rows)."""
520
+ spec = ENTITIES[key]
521
+ con = connect()
522
+ _ensure_table(con, key, spec)
523
+ _ensure_columns(con, key, spec)
524
+ _backfill_columns(con, key, spec, log=log)
525
+ _backfill_m2m(con, key, spec, log=log)
526
+ st = _state(con, key) or {"phase": "backfill", "last_id": 0, "cursor_wd": None, "rows": 0}
527
+ fields = spec["fields"] + list(spec.get("m2m") or [])
528
+ pulled = 0
529
+ try:
530
+ if st["phase"] == "backfill":
531
+ last_id = st["last_id"] or 0
532
+ for i in range(max_batches):
533
+ dom = _base_domain(spec) + [("id", ">", last_id)]
534
+ recs = O.search_read(spec["model"], dom, ["id"] + fields,
535
+ limit=batch, order="id asc")
536
+ if not recs:
537
+ # backfill complete β†’ switch to live mode anchored at max write_date seen
538
+ wd = con.execute(f"SELECT max(write_date) FROM {key}").fetchone()[0]
539
+ _save_state(con, key, "live", last_id, wd or "1970-01-01 00:00:00")
540
+ log(f" {key}: BACKFILL COMPLETE ({st['rows'] + pulled:,} rows) -> live mode")
541
+ return "live", pulled
542
+ _upsert(con, key, spec, recs)
543
+ last_id = recs[-1]["id"]
544
+ pulled += len(recs)
545
+ _save_state(con, key, "backfill", last_id, None)
546
+ if (i + 1) % 10 == 0:
547
+ log(f" {key}: backfill …{pulled:,} rows (id≀{last_id})")
548
+ log(f" {key}: backfill PAUSED at {pulled:,} rows this run (bounded; resumes)")
549
+ return "backfill", pulled
550
+ # live mode: write_date >= cursor (idempotent overlap; upsert dedupes). MASS-STAMP
551
+ # FALLBACK (found live 2026-07-17): a batch job can stamp >batch rows with ONE
552
+ # write_date second (an Odoo recompute stamped ~88k account_move rows '2026-07-15
553
+ # 16:35:03'; stored values carry microseconds, XML-RPC strings don't) β€” then the
554
+ # write_date cursor can NEVER advance and every pass re-pulls the same first page
555
+ # forever. When a full batch leaves the cursor unchanged we switch to an ID-WALK over
556
+ # write_date >= cursor (order id asc, id > tie), checkpointed as 'cursor|<id>' in
557
+ # cursor_wd so a killed walk resumes; on completion the cursor jumps one second past
558
+ # the stamp (safe: the walk covered every row in that second, and the stamp is in the
559
+ # past β€” guarded by a 2-minute recency check).
560
+ raw = st["cursor_wd"] or "1970-01-01 00:00:00"
561
+ cursor, _pipe, _tie = raw.partition("|")
562
+ walking, tie_id = bool(_pipe), int(_tie or 0)
563
+ walk_max = cursor
564
+ for _ in range(max_batches):
565
+ if walking:
566
+ dom = _base_domain(spec) + [("write_date", ">=", cursor), ("id", ">", tie_id)]
567
+ recs = O.search_read(spec["model"], dom, ["id"] + fields,
568
+ limit=batch, order="id asc")
569
+ if not recs: # walk complete β†’ advance past the stamp
570
+ try:
571
+ wm = dt.datetime.strptime(walk_max, "%Y-%m-%d %H:%M:%S")
572
+ if wm < dt.datetime.utcnow() - dt.timedelta(minutes=2):
573
+ cursor = (wm + dt.timedelta(seconds=1)).strftime("%Y-%m-%d %H:%M:%S")
574
+ else:
575
+ cursor = walk_max
576
+ except ValueError:
577
+ cursor = walk_max
578
+ walking = False
579
+ log(f" {key}: id-walk complete -> cursor {cursor}")
580
+ break
581
+ _upsert(con, key, spec, recs)
582
+ pulled += len(recs)
583
+ tie_id = recs[-1]["id"]
584
+ walk_max = max(walk_max, max(str(r["write_date"]) for r in recs))
585
+ _save_state(con, key, "live", st["last_id"], f"{cursor}|{tie_id}")
586
+ continue
587
+ dom = _base_domain(spec) + [("write_date", ">=", cursor)]
588
+ recs = O.search_read(spec["model"], dom, ["id"] + fields,
589
+ limit=batch, order="write_date asc, id asc")
590
+ if not recs or (len(recs) == 1 and str(recs[0].get("write_date")) == cursor
591
+ and pulled == 0 and _already(con, key, recs[0])):
592
+ break
593
+ new_cursor = str(recs[-1]["write_date"])
594
+ if new_cursor == cursor and len(recs) == batch:
595
+ # full batch stuck on one second: enter the id-walk from id 0 (the write_date
596
+ # ordering shuffles ids within the stamp second, so the batch just pulled is
597
+ # NOT an id prefix β€” restart coverage by id; upserts make the overlap free)
598
+ walking, tie_id, walk_max = True, 0, cursor
599
+ _save_state(con, key, "live", st["last_id"], f"{cursor}|0")
600
+ log(f" {key}: mass-stamp second at {cursor} -> switching to id-walk")
601
+ continue
602
+ _upsert(con, key, spec, recs)
603
+ pulled += len(recs)
604
+ if new_cursor == cursor and len(recs) < batch:
605
+ cursor = new_cursor
606
+ break
607
+ cursor = new_cursor
608
+ # Checkpoint the cursor EVERY batch: a big delta (88k account_move rows after the
609
+ # 2026-07-15 recompute) means a killed pass otherwise re-pulls everything β€”
610
+ # upserts are idempotent, but the wasted RPC is real (learned 2026-07-17).
611
+ _save_state(con, key, "live", st["last_id"], cursor)
612
+ _save_state(con, key, "live", st["last_id"],
613
+ f"{cursor}|{tie_id}" if walking else cursor)
614
+ if pulled:
615
+ log(f" {key}: live sync +{pulled:,} rows (cursor {cursor}"
616
+ + (f" walking id>{tie_id}" if walking else "") + ")")
617
+ return "live", pulled
618
+ finally:
619
+ con.close()
620
+
621
+
622
+ def _already(con, key, rec):
623
+ return bool(con.execute(f"SELECT 1 FROM {key} WHERE id=?", [rec["id"]]).fetchone())
624
+
625
+
626
+ def sync_all(entities=None, max_batches=200, log=print):
627
+ out = {}
628
+ for key in (entities or list(ENTITIES)):
629
+ t0 = time.time()
630
+ phase, pulled = sync_entity(key, max_batches=max_batches, log=log)
631
+ out[key] = {"phase": phase, "pulled": pulled, "secs": round(time.time() - t0, 1)}
632
+ # A full pass that lands every entity in live mode latches process readiness in-process β€” so
633
+ # readers never race the writer's lock to (re)discover the store is usable.
634
+ if entities is None and out and all(v["phase"] == "live" for v in out.values()):
635
+ mark_ready()
636
+ return out
637
+
638
+
639
+ def reconcile_deletes(entities=None, log=print):
640
+ """Remove store rows hard-deleted in Odoo. unlink() doesn't move write_date, so the cursor
641
+ sync can never see a deletion β€” the count parity DETECTS the drift (by design); this repairs
642
+ it. Pulls the full live id set per entity in bounded id-ascending pages (id-only reads are
643
+ cheap) and deletes store ids not present live. Run from sync_runner / maintenance, not the
644
+ app's frequent passes."""
645
+ con = connect()
646
+ try:
647
+ out = {}
648
+ for key in (entities or list(ENTITIES)):
649
+ spec = ENTITIES[key]
650
+ live_ids, last = set(), 0
651
+ while True:
652
+ recs = O.search_read(spec["model"], _base_domain(spec) + [("id", ">", last)],
653
+ ["id"], limit=50000, order="id asc")
654
+ if not recs:
655
+ break
656
+ live_ids.update(r["id"] for r in recs)
657
+ last = recs[-1]["id"]
658
+ store_ids = {r[0] for r in con.execute(f"SELECT id FROM {key}").fetchall()}
659
+ dead = store_ids - live_ids
660
+ if dead:
661
+ con.execute("CREATE TEMP TABLE IF NOT EXISTS _dead(id BIGINT)")
662
+ con.execute("DELETE FROM _dead")
663
+ con.executemany("INSERT INTO _dead VALUES (?)", [[i] for i in dead])
664
+ con.execute(f"DELETE FROM {key} WHERE id IN (SELECT id FROM _dead)")
665
+ con.execute("DELETE FROM _dead")
666
+ n = con.execute(f"SELECT count(*) FROM {key}").fetchone()[0]
667
+ con.execute("UPDATE _sync_state SET rows=? WHERE entity=?", [n, key])
668
+ out[key] = len(dead)
669
+ log(f" {key}: reconciled {len(dead):,} hard-deleted rows")
670
+ return out
671
+ finally:
672
+ con.close()
673
+
674
+
675
+ def status():
676
+ con = connect()
677
+ try:
678
+ return {r[0]: {"phase": r[1], "cursor": r[3], "rows": r[4], "updated": r[5]}
679
+ for r in con.execute("SELECT * FROM _sync_state").fetchall()}
680
+ finally:
681
+ con.close()
682
+
683
+
684
+ # ------------------------------------------------------------------ parity (store vs LIVE Odoo)
685
+
686
+ def _check(name, ours, live, tol=0.01):
687
+ gap = abs((ours or 0) - (live or 0))
688
+ return {"check": name, "ok": gap <= tol, "gap": round(gap, 4), "ours": ours, "live": live}
689
+
690
+
691
+ def validate(pre=None, months=("2026-01-01", "2026-07-01")):
692
+ """Raw-fidelity parity: the store must mirror live Odoo β€” row counts per synced entity +
693
+ unscoped monetary sums over a window, to the cent. Only checks entities in LIVE phase."""
694
+ con = connect()
695
+ out = []
696
+ try:
697
+ live_state = status()
698
+ d0, d1 = months
699
+ for key, spec in ENTITIES.items():
700
+ if live_state.get(key, {}).get("phase") != "live":
701
+ continue
702
+ ours = con.execute(f"SELECT count(*) FROM {key}").fetchone()[0]
703
+ live = O.get_odoo().search_count(spec["model"], _base_domain(spec))
704
+ out.append(_check(f"store.{key} row count == live search_count", ours, live, tol=0))
705
+ if live_state.get("sale_order_line", {}).get("phase") == "live":
706
+ ours = con.execute(
707
+ "SELECT coalesce(sum(l.price_subtotal),0) FROM sale_order_line l "
708
+ "JOIN sale_order o ON o.id = l.order_id "
709
+ "WHERE o.date_order >= ? AND o.date_order < ?", [d0, d1]).fetchone()[0]
710
+ live = O.sum_field("sale.order.line",
711
+ [("order_id.date_order", ">=", f"{d0} 00:00:00"),
712
+ ("order_id.date_order", "<", f"{d1} 00:00:00")],
713
+ "price_subtotal")
714
+ out.append(_check(f"store Ξ£ line revenue [{d0}..{d1}) == live (unscoped)", ours, live))
715
+ if live_state.get("sale_order", {}).get("phase") == "live":
716
+ ours = con.execute(
717
+ "SELECT coalesce(sum(amount_untaxed),0) FROM sale_order "
718
+ "WHERE date_order >= ? AND date_order < ?", [d0, d1]).fetchone()[0]
719
+ live = O.sum_field("sale.order",
720
+ [("date_order", ">=", f"{d0} 00:00:00"),
721
+ ("date_order", "<", f"{d1} 00:00:00")], "amount_untaxed")
722
+ out.append(_check(f"store Ξ£ order amount_untaxed [{d0}..{d1}) == live (unscoped)",
723
+ ours, live))
724
+ finally:
725
+ con.close()
726
+ return out
727
+
728
+
729
+ if __name__ == "__main__":
730
+ args = sys.argv[1:]
731
+ if args and args[0] == "status":
732
+ for k, v in status().items():
733
+ print(f" {k:18s} {v['phase']:9s} rows={v['rows']:>9,} cursor={v['cursor']} ({v['updated']})")
734
+ elif args and args[0] == "reconcile":
735
+ ents = args[1].split(",") if len(args) > 1 else None
736
+ for k, n in reconcile_deletes(ents).items():
737
+ print(f" {k:18s} -{n:,} hard-deleted rows")
738
+ elif args and args[0] == "validate":
739
+ ok = True
740
+ for c in validate():
741
+ mark = "OK " if c["ok"] else "XX "
742
+ ok &= c["ok"]
743
+ print(f" {mark}{c['check'][:80]} gap={c['gap']}")
744
+ print("PARITY:", "GREEN" if ok else "RED")
745
+ sys.exit(0 if ok else 1)
746
+ else:
747
+ ents = args[0].split(",") if args else None
748
+ mb = int(args[1]) if len(args) > 1 else 200
749
+ for k, v in sync_all(ents, max_batches=mb).items():
750
+ print(f" {k:18s} {v['phase']:9s} +{v['pulled']:,} rows in {v['secs']}s")
platform/harness/evals.py ADDED
@@ -0,0 +1,134 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """harness/evals.py β€” the Analyst eval gate (OM-4, 2026-07-11).
2
+
3
+ Runs the golden set (evals/analyst_golden.yml β€” truths from the parity-proven semantic layer over
4
+ CLOSED windows) against the Analyst and scores DETERMINISTICALLY: expected number found in the
5
+ answer (tolerance-aware, handles $/commas/% forms), required substrings, required tools in the
6
+ trace, required artifacts. No LLM judge in v1 β€” the validate() culture applied to AI answers.
7
+
8
+ Gate rule (from the plan): run before any deploy that touches model/ or the Analyst; ship only on
9
+ pass-rate >= the bar (start 80%, tighten as the model/skills improve).
10
+
11
+ Usage:
12
+ python -X utf8 -c "import ssl; ssl._create_default_https_context=ssl._create_unverified_context;
13
+ import harness.evals as E; E.run()" # live (needs OPENROUTER_API_KEY)
14
+ E.run(chat_fn=scripted) # offline harness test
15
+ """
16
+ import re
17
+ from pathlib import Path
18
+
19
+ import yaml
20
+
21
+ GOLDEN = Path(__file__).resolve().parents[1] / "evals" / "analyst_golden.yml"
22
+
23
+
24
+ def _numbers(text):
25
+ """Every number in the answer, normalized: $1,234.56 -> 1234.56; '59%'/'59.0 %' -> 0.59 too."""
26
+ out = []
27
+ for m in re.finditer(r"\$?([\d,]+(?:\.\d+)?)\s*(%?)", text):
28
+ try:
29
+ v = float(m.group(1).replace(",", ""))
30
+ except ValueError:
31
+ continue
32
+ out.append(v)
33
+ if m.group(2): # percent form: also offer the fraction
34
+ out.append(v / 100.0)
35
+ return out
36
+
37
+
38
+ def _norm_ws(s):
39
+ """Collapse ALL unicode whitespace to single spaces. Models emit U+202F (narrow no-break
40
+ space) and friends inside names β€” 'Poppy\\u202fFlowers' failed a plain substring check on
41
+ two different models (the 'top_customer flake', diagnosed 2026-07-12)."""
42
+ return re.sub(r"\s+", " ", s)
43
+
44
+
45
+ def _check(item, result):
46
+ exp = item.get("expect") or {}
47
+ answer = _norm_ws(result.get("answer") or "")
48
+ reasons = []
49
+ if result.get("exhausted"):
50
+ reasons.append("tool budget exhausted")
51
+ if "value" in exp:
52
+ want = float(exp["value"])
53
+ tol = exp.get("abs", exp.get("rel", 0.001) * abs(want) or 0.01)
54
+ nums = _numbers(answer)
55
+ if exp.get("allow_pct_form"):
56
+ nums += [n for n in list(nums)]
57
+ if not any(abs(n - want) <= tol for n in nums):
58
+ reasons.append(f"expected {want:,.2f} (Β±{tol:,.2f}) not in answer")
59
+ for s in exp.get("contains", []):
60
+ if _norm_ws(s).lower() not in answer.lower():
61
+ reasons.append(f"missing substring {s!r}")
62
+ # WRONG-ANSWER guard (2026-07-28). `contains` can only prove an answer said the right thing;
63
+ # some failures are about saying a WRONG thing that a right answer never says β€” e.g. naming an
64
+ # internal salesperson as an agent ([[invoice-line-agent-commission]]). Those need a negative
65
+ # assertion or the eval passes on an answer that is confidently incorrect.
66
+ for s in exp.get("not_contains", []):
67
+ if _norm_ws(s).lower() in answer.lower():
68
+ reasons.append(f"forbidden substring {s!r} present")
69
+ used = [t["tool"] for t in result.get("tool_trace", [])]
70
+ for t in exp.get("tools", []):
71
+ if t not in used:
72
+ reasons.append(f"tool {t!r} not used (trace: {used})")
73
+ for t in exp.get("not_tools", []): # over-refusal guard: these must NOT appear
74
+ if t in used:
75
+ reasons.append(f"tool {t!r} used but forbidden here (trace: {used})")
76
+ if exp.get("artifact") == "chart":
77
+ if not any("chart" in a for a in result.get("artifacts", [])):
78
+ reasons.append("no chart artifact produced")
79
+ return (not reasons), reasons
80
+
81
+
82
+ def run(chat_fn=None, model=None, only=None, bar=0.8, verbose=True):
83
+ """Run the golden set. Returns {passed, total, pass_rate, gate_ok, results}."""
84
+ import harness.analyst as A
85
+ items = yaml.safe_load(GOLDEN.read_text(encoding="utf-8"))["items"]
86
+ if only:
87
+ items = [i for i in items if i["id"] in only]
88
+ results, passed = [], 0
89
+ import os as _os
90
+ import time as _time
91
+ _os.environ["ANALYST_PATIENT"] = "1" # batch mode: wait out rate windows, don't fail fast
92
+ infra_fails = 0
93
+ aborted = False
94
+ for item in items:
95
+ _time.sleep(6) # pace the free-tier per-minute windows (Groq/Cerebras)
96
+ kw = {"chat_fn": chat_fn} if chat_fn else {}
97
+ if model:
98
+ kw["model"] = model
99
+ try:
100
+ res = A.ask(item["ask"], telemetry_kind="analyst_eval", **kw)
101
+ ok, reasons = _check(item, res)
102
+ infra_fails = 0
103
+ except Exception as e:
104
+ res, ok, reasons = {"answer": "", "tool_trace": []}, False, [f"RUN ERROR: {e}"]
105
+ # Provider exhaustion is NOT a model failure: two in a row means the ladder is out
106
+ # of quota β€” abort and report INCONCLUSIVE instead of burning the remaining items
107
+ # into a misleading FAIL (learned 2026-07-16: 14 wasted items, 4/18 "22%").
108
+ if "all LLM providers failed" in str(e):
109
+ infra_fails += 1
110
+ if infra_fails >= 2:
111
+ aborted = True
112
+ passed += ok
113
+ results.append({"id": item["id"], "ok": ok, "reasons": reasons,
114
+ "usage": res.get("usage"), "iterations": res.get("iterations")})
115
+ if verbose:
116
+ mark = "OK " if ok else "XX "
117
+ print(f" {mark}{item['id']:18s}" + ("" if ok else f" β€” {'; '.join(reasons)[:90]}"))
118
+ if aborted:
119
+ break
120
+ if aborted:
121
+ ran = [r for r in results if not str(r["reasons"])[:60].startswith("['RUN ERROR")]
122
+ if verbose:
123
+ print(f"\nEVAL ABORTED β€” LLM providers exhausted after {len(results)} item(s); "
124
+ f"{passed}/{len(ran) or 1} of the items that actually ran passed. "
125
+ "Gate is INCONCLUSIVE (not a model failure) β€” re-run when quota recovers.")
126
+ return {"passed": passed, "total": len(items), "pass_rate": None, "gate_ok": False,
127
+ "inconclusive": True, "results": results}
128
+ rate = passed / len(items) if items else 0.0
129
+ gate = rate >= bar
130
+ if verbose:
131
+ print(f"\nEVAL: {passed}/{len(items)} = {rate:.0%} GATE({bar:.0%}): "
132
+ f"{'PASS' if gate else 'FAIL'}")
133
+ return {"passed": passed, "total": len(items), "pass_rate": rate, "gate_ok": gate,
134
+ "results": results}
platform/harness/filter_eval.py ADDED
@@ -0,0 +1,485 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """harness/filter_eval.py β€” the filter tree evaluated against ROWS, in Python (wave 15, C-EVAL).
2
+
3
+ THE THIRD ENGINE, and the one with a different job. There are now three readers of the same
4
+ filter tree and it is worth being precise about why:
5
+
6
+ * `customer-grid/useVisibleRows.ts` β€” the CLIENT, over in-memory rows. The reference.
7
+ * `harness/filter_sql.py` β€” compiles the tree to SQL for the DuckDB semantic tables.
8
+ * **this module** β€” evaluates the tree against wire-shaped row dicts in the
9
+ API/host process, where there is no SQL and the rows are
10
+ already in hand.
11
+
12
+ It exists because a PERMANENT (permission) filter has to be applied server-side, to rows the
13
+ server is about to send, on a surface whose columns are not a DuckDB table. Neither existing
14
+ engine can do that: the client one is TypeScript, and the SQL one needs a table.
15
+
16
+ β›” TWO ENTRY POINTS, OPPOSITE DEFAULTS, AND THE DIFFERENCE IS THE WHOLE POINT.
17
+
18
+ matches(...) DISPLAY semantics. A faithful mirror of the client, including the places
19
+ where the client IGNORES a leaf. Refereed against the client by
20
+ `aios-web/verify_filter_parity.py` β€” if this disagrees with the browser about
21
+ one row, a saved view means two different things in two places.
22
+
23
+ permits(...) PERMISSION semantics. Same evaluator, `strict=True`: anything the display
24
+ engine would IGNORE, this DENIES.
25
+
26
+ Why they cannot be the same function (C-EVAL amendment 1, and the reason this module has a
27
+ docstring this long). The client engine is TRI-STATE β€” `True` / `False` / `None`, where `None`
28
+ means "this leaf is INACTIVE, parent ignores it entirely". A leaf naming a column that does not
29
+ exist returns `None` (`useVisibleRows.ts:318`), and a group whose children are all inactive is
30
+ itself inactive (:273). For a DISPLAY filter that is exactly right: a half-built condition must
31
+ not zero the grid, and a stale column reference must not either.
32
+
33
+ For a PERMISSION filter it is a catastrophe. `agent is Tara` is the only thing standing between
34
+ a sales rep and the whole book; rename or delete the `agent` field and that leaf becomes
35
+ `None` β€” ignored β€” and the rep silently receives every customer. The wall would be one field
36
+ rename away from open, and nothing would go red. So `permits()` inverts every ignore into a
37
+ denial: an unanswerable permission filter is an OUTAGE, not a licence.
38
+
39
+ The asymmetry is a PARAMETER (`strict`) rather than a second implementation, deliberately: a
40
+ mirror must copy the original's asymmetries, and the only way to be sure it still does next
41
+ month is for both behaviours to come out of the same walk.
42
+
43
+ ⚠ WHAT THIS MODULE DOES NOT RE-IMPLEMENT. Activeness (`is_rule_active`), the JS-`Number`
44
+ coercion (`to_num`), the op vocabularies, cohort-id parsing and the date-window/anchor resolvers
45
+ all already exist in `filter_sql.py` and `windows.py`, and are already held against the client by
46
+ `verify_filter_engine.py` (525 checks) and `verify_windows.py` (123). Importing them is not
47
+ laziness β€” a second copy of "what counts as an active rule" is precisely the drift those gates
48
+ exist to catch, and it would be a copy the gates do not cover.
49
+ """
50
+ from harness.filter_sql import ( # noqa: F401 (COHORT_OPS re-exported for callers)
51
+ COHORT_FIELD, COHORT_OPS, COHORT_OP_ALIASES, FILTER_OPS, RANK_OPS,
52
+ is_rule_active, parse_cohort_ids, to_num, _resolve_anchor, _window_bounds,
53
+ )
54
+
55
+ #: The CLIENT's numeric family, which is WIDER than `filter_sql.NUMERIC_TYPES`. `rating` (an
56
+ #: integer 1..max) and `formula` (computed in the browser and injected at the field's key) compare
57
+ #: as numbers here because this engine sees the same assembled row the client does. They are
58
+ #: absent from the SQL port for a good reason β€” they are client-only strata that can never reach a
59
+ #: DuckDB table β€” so the two lists differing is correct, not drift. Mirrors `isNumericType`.
60
+ NUMERIC_TYPES = frozenset({'currency', 'int', 'pct', 'rating', 'formula'})
61
+
62
+ #: `created_time` is a DATETIME rendered read-only, and it filters as a DATE (truncated to the
63
+ #: day below). Mirrors `isDateType`.
64
+ DATE_TYPES = frozenset({'date', 'created_time'})
65
+
66
+ #: The date ops that actually compare against an ANCHOR. Everything else falls to the
67
+ #: active-and-always-true default WITHOUT resolving one β€” ordering matters, see `_match_date`.
68
+ DATE_CMP_OPS = frozenset({'eq', 'neq', 'lt', 'lte', 'gt', 'gte'})
69
+
70
+
71
+ class EvalCtx:
72
+ """Everything the engine needs besides the row and the columns.
73
+
74
+ All four members share one property β€” each is an answer a single ROW cannot compute β€” and
75
+ therefore share one failure mode: absent, the condition matches NOTHING rather than
76
+ everything. That is the widening sin the cohort/measure/rank leaves are each written to
77
+ avoid, and it is why none of these defaults to a permissive value.
78
+ """
79
+
80
+ __slots__ = ('cohort_sets', 'measure_sets', 'rank_sets', 'today')
81
+
82
+ def __init__(self, cohort_sets=None, measure_sets=None, rank_sets=None, today=None):
83
+ #: {cohort_id: set(pid)} β€” the cohorts THIS reader may see.
84
+ self.cohort_sets = cohort_sets or {}
85
+ #: {rule_id: set(pid)} β€” resolved measure conditions (CG-8).
86
+ self.measure_sets = measure_sets or {}
87
+ #: {id(leaf) or leaf-key: set(pid)} β€” resolved rank slices (C-OPS).
88
+ self.rank_sets = rank_sets or {}
89
+ #: The TENANT's today, ISO. Never the server clock at call time β€” a date filter that
90
+ #: means something different per process is not a filter.
91
+ self.today = today
92
+
93
+
94
+ _EMPTY_CTX = EvalCtx()
95
+
96
+
97
+ def _is_group(node):
98
+ return isinstance(node, dict) and isinstance(node.get('children'), list)
99
+
100
+
101
+ def _is_cohort(node):
102
+ return isinstance(node, dict) and node.get('colId') == COHORT_FIELD
103
+
104
+
105
+ def _is_measure(node):
106
+ """Mirrors `isMeasureRule`: a leaf carrying a `window` IS a measure condition."""
107
+ return isinstance(node, dict) and node.get('window') is not None
108
+
109
+
110
+ def _rhs_col(rhs):
111
+ """Mirrors `rhsColId`: a STATISTIC names no column."""
112
+ if not isinstance(rhs, dict) or rhs.get('kind') == 'stat':
113
+ return None
114
+ return rhs.get('colId')
115
+
116
+
117
+ def _as_text(v):
118
+ """The client's `String(raw ?? "").toLowerCase()`. `None` becomes '' (never 'None')."""
119
+ return ('' if v is None else str(v)).lower()
120
+
121
+
122
+ def _as_day(v):
123
+ """Day-granularity truncation. A `created_time` cell is a DATETIME and every comparand in
124
+ this vocabulary is a DATE β€” lexically the longer string sorts AFTER its own day, so without
125
+ this `on or before <today>` would silently exclude today's rows. Identity for a pure date."""
126
+ return ('' if v is None else str(v))[:10]
127
+
128
+
129
+ def _cmp_num(op, a, b):
130
+ """`default: True` is the documented ACTIVE-and-always-true case β€” an operator that exists
131
+ but does not apply to this type (`contains` on a number) narrows nothing and asks nothing.
132
+ An operator in NO vocabulary never reaches here; `_eval` refuses it first."""
133
+ if op == 'gt':
134
+ return a > b
135
+ if op == 'gte':
136
+ return a >= b
137
+ if op == 'lt':
138
+ return a < b
139
+ if op == 'lte':
140
+ return a <= b
141
+ if op == 'eq':
142
+ return a == b
143
+ if op == 'neq':
144
+ return a != b
145
+ return True
146
+
147
+
148
+ def _cmp_str(op, a, v):
149
+ if op == 'eq':
150
+ return a == v
151
+ if op == 'neq':
152
+ return a != v
153
+ if op == 'lt':
154
+ return a < v
155
+ if op == 'lte':
156
+ return a <= v
157
+ if op == 'gt':
158
+ return a > v
159
+ if op == 'gte':
160
+ return a >= v
161
+ return True
162
+
163
+
164
+ def _cmp_text(op, a, v):
165
+ if op == 'contains':
166
+ return v in a
167
+ if op == 'doesNotContain':
168
+ return v not in a
169
+ if op == 'eq':
170
+ return a == v
171
+ if op == 'neq':
172
+ return a != v
173
+ return True
174
+
175
+
176
+ def _is_blank(raw):
177
+ """Blank = None or ''. A numeric 0 is NOT blank β€” `isEmpty` must not swallow a real zero,
178
+ and the wire preserves null for a missing value."""
179
+ return raw is None or raw == ''
180
+
181
+
182
+ def _match_numeric(rule, raw, other, has_rhs):
183
+ a = to_num(raw)
184
+ op = rule.get('op')
185
+ if has_rhs:
186
+ return _cmp_num(op, a, to_num(other))
187
+ if op == 'between':
188
+ v, v2 = rule.get('value'), rule.get('value2')
189
+ if v == '' or v2 is None or v2 == '':
190
+ return True # both bounds required
191
+ lo, hi = to_num(v), to_num(v2)
192
+ if lo > hi:
193
+ lo, hi = hi, lo
194
+ return lo <= a <= hi
195
+ if rule.get('value') == '':
196
+ return True # inactive until a value is typed
197
+ return _cmp_num(op, a, to_num(rule.get('value')))
198
+
199
+
200
+ def _match_date(rule, raw, other, has_rhs, today):
201
+ a = _as_day(raw)
202
+ op = rule.get('op')
203
+
204
+ # "is within [the past month]" β€” a RANGE test, and the only date op reading `dateWindow`.
205
+ if op == 'within':
206
+ if not today:
207
+ return False
208
+ bounds = _window_bounds(rule.get('dateWindow'), today)
209
+ if not bounds:
210
+ return False # unresolvable range -> matches nothing
211
+ if a == '':
212
+ # A blank date is not "within" any range. Without this, '' >= a null lower bound is
213
+ # vacuously true and every never-ordered customer lands inside every window.
214
+ return False
215
+ lo, hi = bounds
216
+ return (lo is None or a >= lo) and (hi is None or a <= hi)
217
+
218
+ if has_rhs:
219
+ return _cmp_str(op, a, _as_day(other))
220
+ if op == 'between':
221
+ v, v2 = rule.get('value'), rule.get('value2')
222
+ if v == '' or v2 is None or v2 == '':
223
+ return True
224
+ lo, hi = (v, v2) if v < v2 else (v2, v)
225
+ return lo <= a <= hi
226
+
227
+ # An op that does not apply to a date (`contains`) is ACTIVE-and-always-true, decided BEFORE
228
+ # any anchor is resolved β€” there is no date in that question to resolve. Ordering this after
229
+ # the anchor would turn `contains` on a date from always-true into matches-nothing, which is
230
+ # a different answer under `or`.
231
+ if op not in DATE_CMP_OPS:
232
+ return True
233
+ if not rule.get('dateMode') and rule.get('value') == '':
234
+ return True # inactive until a value is typed
235
+ mode = rule.get('dateMode') or 'exact'
236
+ # `today` is required by a RELATIVE anchor and by nothing else, which is what keeps every
237
+ # pre-anchor saved view working in a payload that carries no today.
238
+ if mode != 'exact' and not today:
239
+ return False
240
+ v = _resolve_anchor(mode, rule.get('value'), today or '1970-01-01')
241
+ if v is None:
242
+ # Unresolvable (junk date, missing N) -> matches nothing. The honest answer to a
243
+ # question with no date in it.
244
+ return False
245
+ return _cmp_str(op, a, v)
246
+
247
+
248
+ def _match_filter(rule, row, columns, today):
249
+ """One LEAF against one row, dispatched on the field's declared type.
250
+
251
+ `isEmpty`/`isNotEmpty` are handled FIRST and type-independently: they must never fall
252
+ through to the "blank value -> inactive" guards below, which would turn them into
253
+ always-true no-ops.
254
+ """
255
+ field = columns.get(rule.get('colId')) or {}
256
+ ftype = field.get('type')
257
+ raw = row.get(rule.get('colId'))
258
+ op = rule.get('op')
259
+
260
+ if op == 'isEmpty':
261
+ return _is_blank(raw)
262
+ if op == 'isNotEmpty':
263
+ return not _is_blank(raw)
264
+
265
+ # A RANK op can never be answered from one row. `_eval` routes rank leaves to their resolved
266
+ # set and one should never arrive here; this is the SECOND layer, and it exists because of
267
+ # what the first layer's absence would do β€” `_cmp_num`'s `default: True` would make `topN`
268
+ # match EVERY row, a filter that widens under a count that agrees with it.
269
+ if op in RANK_OPS:
270
+ return False
271
+
272
+ rhs = rule.get('rhs')
273
+ has_rhs = rhs is not None
274
+ other = row.get(_rhs_col(rhs) or '') if has_rhs else None
275
+
276
+ if ftype in NUMERIC_TYPES:
277
+ return _match_numeric(rule, raw, other, has_rhs)
278
+ if ftype in DATE_TYPES:
279
+ return _match_date(rule, raw, other, has_rhs, today)
280
+
281
+ a = _as_text(raw)
282
+ if has_rhs:
283
+ return _cmp_text(op, a, _as_text(other))
284
+ if rule.get('value') == '':
285
+ return True
286
+ return _cmp_text(op, a, _as_text(rule.get('value')))
287
+
288
+
289
+ def _eval(node, row, columns, ctx, strict):
290
+ """One node against one row. `True` / `False` / `None`, where `None` = INACTIVE.
291
+
292
+ `strict` is the permission mode: every `None` this function would return becomes `False`
293
+ instead. Read the two behaviours side by side rather than trusting either alone β€” the
294
+ difference between them IS the security property (see the module docstring).
295
+ """
296
+ ignore = False if strict else None
297
+
298
+ if _is_group(node):
299
+ saw_active = False
300
+ all_true = True
301
+ any_true = False
302
+ for child in node['children']:
303
+ v = _eval(child, row, columns, ctx, strict)
304
+ if v is None:
305
+ continue # inactive -> ignored entirely
306
+ saw_active = True
307
+ all_true = all_true and v
308
+ any_true = any_true or v
309
+ if not saw_active:
310
+ # A group with no active children is itself inactive. In strict mode an empty or
311
+ # wholly-unanswerable permission group DENIES: a wall that evaluates to "nothing to
312
+ # say" must not mean "let them through".
313
+ return ignore
314
+ return any_true if node.get('conj') == 'or' else all_true
315
+
316
+ if not isinstance(node, dict):
317
+ return ignore
318
+
319
+ # A COHORT leaf, BEFORE the column lookup: `__cohort__` is not a column, so the lookup below
320
+ # would call it INACTIVE β€” and inactive means every row on screen under a count nobody doubts.
321
+ if _is_cohort(node):
322
+ if not is_rule_active(node, columns):
323
+ return ignore
324
+ ids = parse_cohort_ids(node.get('value'))
325
+ sets = [ctx.cohort_sets.get(cid) for cid in ids]
326
+ # ANY named cohort we were given no membership for makes the WHOLE condition
327
+ # unanswerable, and unanswerable matches nothing β€” for `is none of` as well. Answering
328
+ # "everything" to `is none of [a cohort we cannot see]` is the widening sin wearing a
329
+ # plausible face, and PARTIAL evaluation is worse: `is none of [A, B]` quietly becoming
330
+ # `is none of [A]` would show every row in B under a count nobody would doubt.
331
+ if any(s is None for s in sets):
332
+ return False
333
+ pid = row.get('pid')
334
+ op = node.get('op')
335
+ op = op if op in COHORT_OPS else COHORT_OP_ALIASES.get(op, 'anyOf')
336
+ if op == 'allOf':
337
+ return all(pid in s for s in sets)
338
+ in_any = any(pid in s for s in sets)
339
+ return (not in_any) if op == 'noneOf' else in_any
340
+
341
+ # A MEASURE leaf, also before the column lookup and for the same reason: a measure's colId
342
+ # ("revenue") is not a column of this table. An ACTIVE rule whose set has not been resolved
343
+ # is PENDING and matches NOTHING β€” never everything.
344
+ if _is_measure(node):
345
+ if not is_rule_active(node, columns):
346
+ return ignore
347
+ s = ctx.measure_sets.get(node.get('id')) if node.get('id') else None
348
+ if s is None:
349
+ return False
350
+ return row.get('pid') in s
351
+
352
+ # ── a column leaf ────────────────────────────────────────────────────────────────────────
353
+ if node.get('colId') not in columns:
354
+ # THE LEAF THIS WHOLE MODULE'S `strict` FLAG EXISTS FOR. Display: ignore it. Permission:
355
+ # deny β€” see the module docstring's field-rename walkthrough.
356
+ return ignore
357
+ if node.get('rhs') is not None and _rhs_col(node['rhs']) not in columns:
358
+ # An rhs naming a column this table does not have is ignored by the SAME rule as an
359
+ # unknown LEFT column. Treating one side as "ignore" and the other as "match nothing"
360
+ # would make `a > deleted` behave differently from `deleted > a`.
361
+ return ignore
362
+ if not is_rule_active(node, columns):
363
+ return ignore
364
+
365
+ # A RANK leaf is a MEMBERSHIP test, like a cohort or a measure: the set was computed in a
366
+ # prior pass over the other rows. No set = REFUSE, never "ignore" and never "true" β€” both of
367
+ # those SHOW rows.
368
+ if node.get('op') in RANK_OPS:
369
+ s = ctx.rank_sets.get(_rank_key(node))
370
+ return (row.get('pid') in s) if s is not None else False
371
+
372
+ # An operator in NO vocabulary at all. The comparators end in `default: True`, right for an
373
+ # operator that exists but does not apply to this type; NOT right for one that exists
374
+ # nowhere (`inTertile` out of a stale saved view), which would match every row.
375
+ if node.get('op') not in FILTER_OPS:
376
+ return False
377
+
378
+ return _match_filter(node, row, columns, ctx.today)
379
+
380
+
381
+ def _rank_key(node):
382
+ """The client keys `rankSets` by leaf IDENTITY (a Map keyed on the node object). Python
383
+ dicts cannot key on a dict, so the caller that resolves rank sets and this evaluator agree
384
+ on a derived key. Kept in ONE place so the two cannot drift apart."""
385
+ return (node.get('id') or '', node.get('colId') or '', node.get('op') or '',
386
+ str(node.get('value') if node.get('value') is not None else ''))
387
+
388
+
389
+ def _columns_map(fields):
390
+ """`[{key,type,…}]` -> `{key: field}`. Accepts a mapping unchanged, so callers holding
391
+ either shape need no adapter."""
392
+ if isinstance(fields, dict):
393
+ return fields
394
+ out = {}
395
+ for f in fields or ():
396
+ if isinstance(f, dict) and f.get('key'):
397
+ out[f['key']] = f
398
+ return out
399
+
400
+
401
+ def evaluate(tree, row, fields, ctx=None):
402
+ """The TRI-STATE the client actually has: `True` / `False` / `None`, where `None` means the
403
+ tree is INACTIVE β€” it asks nothing, so it narrows nothing.
404
+
405
+ Exposed because collapsing `None` into `True` (which `matches` does, correctly, for its own
406
+ purpose) destroys a distinction some callers must keep. The `memberPids` union is the one
407
+ that bit: the client's rule is `members.has(pid) || matchFilterTree(…)` and that union sits
408
+ INSIDE `if (filters.length)`, so an INACTIVE tree must show every row and ignore the member
409
+ list β€” whereas an ACTIVE tree that merely fails must still admit hand-picked members. A
410
+ caller holding only `matches` cannot tell those two apart and will narrow an inactive view
411
+ down to its hand-picked rows. (`filter_sql.compile_filter_tree` keeps the same distinction
412
+ by returning None rather than a predicate; this is the row-engine's equivalent.)
413
+
414
+ Activeness is ROW-INDEPENDENT β€” every `None` here comes from `is_rule_active` or an empty
415
+ group, none of which reads the row β€” so a caller may compute it once per tree if it wishes.
416
+ """
417
+ columns = _columns_map(fields)
418
+ nodes, conj = tree_parts(tree)
419
+ return _eval({'conj': conj, 'children': nodes}, row, columns, ctx or _EMPTY_CTX, False)
420
+
421
+
422
+ def matches(tree, row, fields, ctx=None):
423
+ """DISPLAY semantics β€” does `row` survive this filter tree? Mirrors the client exactly.
424
+
425
+ A wholly INACTIVE tree returns True: no narrowing, every row shows. That is what the browser
426
+ does with a half-built condition, and this function's whole job is to agree with the browser.
427
+ Use `evaluate()` instead when you need to tell "inactive" from "matched" (see its docstring).
428
+ """
429
+ v = evaluate(tree, row, fields, ctx)
430
+ return True if v is None else bool(v)
431
+
432
+
433
+ def visible_pids(tree, rows, fields, ctx=None, member_pids=None):
434
+ """The client's whole row pipeline for one tree, member union included β€” the shape a caller
435
+ should reach for instead of assembling `matches` in a loop.
436
+
437
+ Mirrors `useVisibleRows`: an INACTIVE tree yields every row (and IGNORES `member_pids`,
438
+ because the union lives inside `if (filters.length)`); an ACTIVE tree yields the rows it
439
+ matches UNION the hand-picked members.
440
+ """
441
+ ctx = ctx or _EMPTY_CTX
442
+ members = set(member_pids or ())
443
+ out, active = [], False
444
+ verdicts = []
445
+ for r in rows:
446
+ v = evaluate(tree, r, fields, ctx)
447
+ if v is not None:
448
+ active = True
449
+ verdicts.append((r, v))
450
+ for r, v in verdicts:
451
+ if not active:
452
+ out.append(r.get('pid'))
453
+ elif v or (members and r.get('pid') in members):
454
+ out.append(r.get('pid'))
455
+ return out
456
+
457
+
458
+ def permits(tree, row, fields, ctx=None):
459
+ """PERMISSION semantics β€” may this reader SEE `row` under this permanent filter?
460
+
461
+ Same evaluator, every ignore inverted into a denial. An empty/absent tree means "no
462
+ restriction declared" and admits (the caller decides whether a module with no entry is
463
+ reachable at all β€” that is `perm_scope`'s question, not this one); anything PRESENT but
464
+ unanswerable DENIES.
465
+ """
466
+ nodes, conj = tree_parts(tree)
467
+ if not nodes:
468
+ return True # nothing declared -> nothing to enforce here
469
+ columns = _columns_map(fields)
470
+ v = _eval({'conj': conj, 'children': nodes}, row, columns, ctx or _EMPTY_CTX, True)
471
+ return bool(v)
472
+
473
+
474
+ def tree_parts(tree):
475
+ """Read C-PERM amendment 2's `FilterTree` β€” `{conj?, nodes}` β€” into `(nodes, conj)`.
476
+
477
+ Accepts a BARE LIST too, because that is what `ViewConfig.filters` is and what
478
+ `clean_filter_tree` returns; an absent conj is `'and'`, mirroring `ViewConfig.filterConj`.
479
+ One reader, so the absent-conj default lives here instead of at every call site.
480
+ """
481
+ if isinstance(tree, dict):
482
+ nodes = tree.get('nodes')
483
+ conj = 'or' if tree.get('conj') == 'or' else 'and'
484
+ return (nodes if isinstance(nodes, list) else []), conj
485
+ return (tree if isinstance(tree, list) else []), 'and'
platform/harness/filter_sql.py ADDED
@@ -0,0 +1,673 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """harness/filter_sql.py β€” the grid's filter/sort engine, compiled to SQL (CG-1).
2
+
3
+ THE PORT. `customer-grid/useVisibleRows.ts` evaluates the Airtable-parity filter TREE and the
4
+ multi-level sort in TypeScript, per row, in the browser. That works at customer grain (1,550 rows,
5
+ 576 KB) and does NOT work at line grain, so a line-grain table has to filter server-side. This
6
+ module is the second implementation of that contract, emitting parameterized SQL instead of
7
+ booleans. `aios_grid.clean_filter_tree` (the validator) is the third. All three are held in
8
+ lock-step by `aios-web/verify_filter_engine.py`, which runs the SAME cases through the TS engine,
9
+ this compiler (executed on DuckDB), and the validator, and compares row sets.
10
+
11
+ Design: this file is PURE β€” no model, no store, no app imports. It takes a filter tree plus a
12
+ `columns` spec and returns SQL + params. `harness/semantic.store_columns` builds the spec from a
13
+ topic; nothing else here knows what a topic is.
14
+
15
+ columns = {colId: {"sql": <SQL expression>, "type": <field type>, "aggregate": bool}}
16
+
17
+ `sql` is a trusted IDENTIFIER/expression from the semantic model (never user input); every VALUE
18
+ is a bound parameter. That split is the whole injection story.
19
+
20
+ --------------------------------------------------------------------------------------------
21
+ TRI-STATE, AND WHY IT BECOMES COMPILE-TIME PRUNING
22
+ --------------------------------------------------------------------------------------------
23
+ `evalNode` in TS returns true | false | null, where null means "this node is INACTIVE and its
24
+ parent must IGNORE it". Get that wrong and a half-typed condition returns true, makes its `or`
25
+ group true, and shows every record.
26
+
27
+ Activeness reads ONLY the rule (op, value, value2, is-the-column-known) β€” never the row. So in
28
+ SQL it is not a third truth value at all: it is decided while COMPILING. An inactive node emits
29
+ no SQL, a group with no active children emits no SQL, and a root with nothing active emits NO
30
+ PREDICATE AT ALL (`compile_filter_tree` returns None) β€” which is exactly `matchFilterTree`
31
+ returning true for every row.
32
+
33
+ --------------------------------------------------------------------------------------------
34
+ THE FOUR PLACES A NAIVE PORT SILENTLY DIVERGES
35
+ --------------------------------------------------------------------------------------------
36
+ JavaScript coerces null to 0 / "" before comparing; SQL propagates NULL and drops the row.
37
+
38
+ 1. `toNum(null)` is 0, so a NULL revenue PASSES `>= 0`. SQL: NULL >= 0 -> NULL -> dropped.
39
+ 2. `String(null)` is "", so a NULL state PASSES `is not FL`. SQL: NULL <> 'fl' -> dropped.
40
+ 3. `"" <= "2026-01-01"` is true, so a NULL date PASSES `is before`. SQL: dropped.
41
+ 4. `contains` is a LITERAL substring test. `LIKE '%v%'` treats % and _ as wildcards, so a
42
+ value containing either matches the wrong rows. Hence `strpos`, not LIKE.
43
+
44
+ Every column is therefore read through `_value_sql`, which COALESCEs to the same zero-value
45
+ JavaScript would have coerced to. The one thing that must NOT be coalesced away is the
46
+ blank test itself β€” `isBlank` looks at the RAW value, and numeric 0 is NOT blank.
47
+
48
+ --------------------------------------------------------------------------------------------
49
+ AND THE FIFTH: `default: return true` IS ACTIVE, NOT INACTIVE
50
+ --------------------------------------------------------------------------------------------
51
+ `matchFilter`'s per-type switch ends in `default: return true`, reachable whenever an operator
52
+ does not apply to the field type β€” `contains` on a currency/date field, or `gt`/`between` on a
53
+ text field. A persisted tree can hold such a pair. That rule is ACTIVE and evaluates TRUE for
54
+ every row, which is NOT the same as inactive: under `or` it makes the whole group true and shows
55
+ everything. We emit `1=1` for those, and the gate asserts the difference is observable.
56
+
57
+ --------------------------------------------------------------------------------------------
58
+ INPUT CONTRACT
59
+ --------------------------------------------------------------------------------------------
60
+ Feed this CLEANED trees (`aios_grid.clean_filter_tree`). It is deliberately faithful to the TS
61
+ engine rather than fail-closed on garbage β€” an unknown OP compiles to `1=1` exactly as TS's
62
+ `default:` branch returns true β€” because the validator's job is to reject garbage and this
63
+ module's job is to mean the same thing as the client. The gate proves the composition.
64
+
65
+ Ported ahead of their consumer (CG-2) and gated at the same fidelity as the tree:
66
+ `compile_search` (`matchSearch`) and `member_ids` (the hand-picked-members union in
67
+ `useVisibleRows`).
68
+ """
69
+ import math
70
+ import re
71
+ from collections import namedtuple
72
+
73
+ # Mirrors customer-grid/types.ts FilterOp. verify_filter_engine.py asserts this set is
74
+ # byte-identical to aios_grid.FILTER_OPS β€” three copies of a vocabulary need a drift gate.
75
+ FILTER_OPS = frozenset({
76
+ 'contains', 'doesNotContain', 'eq', 'neq', 'isEmpty', 'isNotEmpty',
77
+ 'gt', 'gte', 'lt', 'lte', 'between', 'within',
78
+ 'topN', 'bottomN', 'inTopPct', 'inBottomPct',
79
+ 'aboveAvg', 'belowAvg', 'inQuartile', 'inDecile',
80
+ })
81
+ #: Wave 2026-08-02 (C-OPS): the RANK subset β€” set-ranked over the sibling-filtered domain.
82
+ #: Vocabulary here, execution NOWHERE in this module: a per-row WHERE cannot express Top-N,
83
+ #: and compiling to 1=0 would answer "nothing" to a question that has an answer, under a
84
+ #: row_count that still looks authoritative. `_compile_leaf` therefore REFUSES (ValueError),
85
+ #: the same server-side posture as a relative anchor with no `today`. The TS engine
86
+ #: evaluates these as a set pass in useVisibleRows (gated by web/verify_rank_filters.py).
87
+ RANK_OPS = frozenset({'topN', 'bottomN', 'inTopPct', 'inBottomPct',
88
+ 'aboveAvg', 'belowAvg', 'inQuartile', 'inDecile'})
89
+ #: The RESERVED pseudo-column of a cohort-membership leaf. Mirrors types.ts COHORT_FIELD.
90
+ COHORT_FIELD = '__cohort__'
91
+ #: Set operators over a SET of cohorts. Mirrors types.ts COHORT_OPS; DISJOINT from FILTER_OPS.
92
+ COHORT_OPS = frozenset({'anyOf', 'allOf', 'noneOf'})
93
+ #: The single-cohort ops the leaf shipped with, kept as permanent aliases. types.ts mirrors this.
94
+ COHORT_OP_ALIASES = {'eq': 'anyOf', 'neq': 'noneOf'}
95
+ #: Mirrors types.ts MAX_COHORT_IDS.
96
+ MAX_COHORT_IDS = 20
97
+
98
+
99
+ def parse_cohort_ids(value):
100
+ """The cohorts a leaf names. Mirrors types.ts `cohortIds()` and aios_grid.parse_cohort_ids."""
101
+ out = []
102
+ for raw in ('' if value is None else str(value)).split(','):
103
+ cid = raw.strip()[:120]
104
+ if not cid or cid in out:
105
+ continue
106
+ out.append(cid)
107
+ if len(out) >= MAX_COHORT_IDS:
108
+ break
109
+ return out
110
+ # types.ts VALUE_FREE_OPS: no value control is rendered, and they must never be judged
111
+ # "inactive because the value is blank" β€” that would turn them into no-ops.
112
+ VALUE_FREE_OPS = frozenset({'isEmpty', 'isNotEmpty', 'aboveAvg', 'belowAvg'})
113
+ # types.ts isNumericType()
114
+ NUMERIC_TYPES = frozenset({'currency', 'int', 'pct'})
115
+ TEXTUAL_TYPES = frozenset({'text', 'status'})
116
+
117
+ SQL_TRUE = '1=1'
118
+
119
+ _NUM_CMP = {'gt': '>', 'gte': '>=', 'lt': '<', 'lte': '<=', 'eq': '=', 'neq': '<>'}
120
+ _DATE_CMP = dict(_NUM_CMP)
121
+
122
+ #: sql = the predicate (already parenthesised); params = bound values, positional;
123
+ #: uses_aggregate = the predicate references an aggregate column, so it belongs in HAVING
124
+ #: rather than WHERE; columns_used = colIds actually referenced in the emitted SQL (an
125
+ #: always-true `1=1` branch references nothing, and must not force HAVING routing).
126
+ Compiled = namedtuple('Compiled', 'sql params uses_aggregate columns_used')
127
+
128
+
129
+ # --------------------------------------------------------------------------- value coercion
130
+
131
+ # JS `Number(string)` trims exactly this set (WhiteSpace + LineTerminator). Python's float() and
132
+ # str.strip() use a DIFFERENT set -- notably U+0085 NEL, which Python treats as whitespace and JS
133
+ # does not -- so the set is spelled out rather than left to strip().
134
+ _JS_WS = "".join(chr(c) for c in (9, 10, 11, 12, 13, 32, 160, 5760, 8232, 8233,
135
+ 8239, 8287, 12288, 65279)) + \
136
+ "".join(chr(c) for c in range(8192, 8203))
137
+ # ASCII-ONLY on purpose: `\d` in a Python str pattern matches every Unicode decimal digit, so
138
+ # `\d` here would parse the fullwidth "250" that JS rejects -- reintroducing the exact bug
139
+ # this function exists to avoid.
140
+ _JS_DEC = re.compile(r"^[+-]?(?:[0-9]+\.?[0-9]*|\.[0-9]+)(?:[eE][+-]?[0-9]+)?$")
141
+ _JS_INF = re.compile(r"^[+-]?Infinity$")
142
+ _JS_RADIX = ((re.compile(r"^0[xX][0-9a-fA-F]+$"), 16),
143
+ (re.compile(r"^0[oO][0-7]+$"), 8),
144
+ (re.compile(r"^0[bB][01]+$"), 2))
145
+
146
+
147
+ def to_num(v):
148
+ """Port of `toNum`: JS `Number(v)`, then `Number.isFinite` or 0.
149
+
150
+ NOT `float(v)`. Python's float() is MORE permissive than JS's Number() in ways that make SQL
151
+ narrow where the client shows everything -- the worse direction, because at line grain that
152
+ is a silent under-fetch sitting beside an honest-looking row_count:
153
+
154
+ "1_000" float() -> 1000.0 (PEP-515 underscores) Number() -> NaN -> 0
155
+ "250" float() -> 250.0 (any Unicode Nd) Number() -> NaN -> 0
156
+ "\\u0085" + "12" float() -> 12.0 (NEL is Python whitespace) Number() -> NaN -> 0
157
+
158
+ and LESS permissive in one family, which errs the other way:
159
+
160
+ "0x10" float() -> raises -> 0 Number() -> 16 (also 0b, 0o)
161
+
162
+ A tree carrying any of these is reachable: the React builder renders <input type="number">,
163
+ but persisted / seeded / imported view JSON goes through `clean_filter_tree`, which passes
164
+ the value string through untouched by design (it validates structure, not numeric syntax).
165
+
166
+ Agreed by construction on the rest: ""/None -> 0, "abc"/"12,000" -> 0, " 250 " -> 250,
167
+ ".5"/"5."/"+5", "1e400" -> Infinity -> 0, "-0" -> 0, and float64 truncation of
168
+ "9007199254740993" -> 9007199254740992.
169
+ """
170
+ if isinstance(v, bool):
171
+ return 1.0 if v else 0.0 # JS Number(true) === 1
172
+ if isinstance(v, (int, float)):
173
+ n = float(v)
174
+ return n if math.isfinite(n) else 0.0
175
+ if v is None:
176
+ return 0.0 # JS Number(null) === 0
177
+ s = str(v).strip(_JS_WS)
178
+ if s == "":
179
+ return 0.0 # JS Number("") === 0
180
+ if _JS_INF.match(s):
181
+ return 0.0 # Infinity is not finite -> 0
182
+ for pattern, base in _JS_RADIX:
183
+ if pattern.match(s):
184
+ return float(int(s[2:], base))
185
+ if not _JS_DEC.match(s):
186
+ return 0.0 # JS NaN -> 0
187
+ n = float(s)
188
+ return n if math.isfinite(n) else 0.0
189
+
190
+
191
+ def _value_sql(spec):
192
+ """The ONE expression for a column, used by BOTH the filter and the sort.
193
+
194
+ Two expressions would let two rows tie in the grid and not tie in SQL (dates truncated for
195
+ comparison but ordered by the raw timestamp is the obvious way to get that wrong), so the
196
+ next sort key would break the tie in one engine and not the other.
197
+
198
+ numeric: `round_even(x, 0)` reproduces `aios_grid._round`, which is Python's round() =
199
+ BANKER'S rounding. DuckDB's plain ROUND() rounds half AWAY from zero, so ROUND(1234.5)=1235
200
+ while the grid shows 1234 β€” `eq 1234` would then match on screen and miss in SQL. The grid
201
+ displays rounded values; filters must agree with what is on screen.
202
+ date: truncate to the 10-char ISO shape the payload carries, so lexical compare is
203
+ chronological and a timestamp column cannot smuggle ' 00:00:00' into the comparison.
204
+ text: lowercased, matching `String(raw ?? "").toLowerCase()`.
205
+ """
206
+ e = spec['sql']
207
+ t = spec['type']
208
+ if t in NUMERIC_TYPES:
209
+ return f"COALESCE(round_even(TRY_CAST({e} AS DOUBLE), 0), 0)"
210
+ if t == 'date':
211
+ return f"COALESCE(SUBSTR(CAST({e} AS VARCHAR), 1, 10), '')"
212
+ return f"LOWER(COALESCE(CAST({e} AS VARCHAR), ''))"
213
+
214
+
215
+ def _blank_sql(spec):
216
+ """Port of `isBlank`: null/undefined/"" are blank β€” numeric 0 is NOT.
217
+
218
+ Reads the RAW column, never `_value_sql` (which has already coalesced blanks into 0/"",
219
+ the very distinction this test exists to make).
220
+ """
221
+ e = spec['sql']
222
+ if spec['type'] in NUMERIC_TYPES:
223
+ # 0 stays non-blank: CAST(0 AS VARCHAR) is '0'. The '' arm covers a text-typed column
224
+ # holding "", which `_round` passes through unchanged into the payload.
225
+ return f"(({e}) IS NULL OR CAST({e} AS VARCHAR) = '')"
226
+ if spec['type'] == 'date':
227
+ return f"(COALESCE(SUBSTR(CAST({e} AS VARCHAR), 1, 10), '') = '')"
228
+ return f"(COALESCE(CAST({e} AS VARCHAR), '') = '')"
229
+
230
+
231
+ # --------------------------------------------------------------------------- activeness
232
+
233
+ def is_rule_active(rule, columns):
234
+ """Port of `isRuleActive` + `evalNode`'s two leaf guards. Row-independent by construction.
235
+
236
+ Unknown column -> inactive (evalNode returns null before matchFilter is ever called).
237
+ Value-free ops are always active. `between` needs BOTH bounds.
238
+
239
+ ⚠ The four VALUE-FREE ANCHOR MODES (`is before [today]`) carry no value, and the historical
240
+ "blank value = inactive" rule would IGNORE them β€” widening the result under a count nobody
241
+ would doubt. Same shape as the isEmpty/isNotEmpty no-op bug, arriving by a different door.
242
+ """
243
+ if not isinstance(rule, dict):
244
+ return False
245
+ col = rule.get('colId')
246
+ if col == COHORT_FIELD: # a cohort leaf has no column to look up
247
+ # Driven by the SET it names, so a value of "," is INACTIVE rather than active-and-
248
+ # unanswerable. Mirrors the cohort branch at the top of types.ts isRuleActive.
249
+ return bool(parse_cohort_ids(rule.get('value')))
250
+ if col not in columns:
251
+ return False
252
+ op = rule.get('op')
253
+ if op in VALUE_FREE_OPS:
254
+ return True
255
+ if op == 'within':
256
+ return _window_bounds(rule.get('dateWindow'), None, probe=True)
257
+ if rule.get('rhs') is not None:
258
+ rhs = rule['rhs']
259
+ # An rhs naming a column this table does not have is IGNORED, by the same rule as an
260
+ # unknown left column two lines above β€” for EITHER kind. Ignoring one side and hiding
261
+ # every row for the other would make the comparison asymmetric in a way nobody could
262
+ # predict. Doing it here rather than in _compile_leaf keeps activeness the single place
263
+ # that decides, which is what makes the compiled branch below provably unreachable.
264
+ if not isinstance(rhs, dict) or rhs.get('colId') not in columns:
265
+ return False
266
+ return _rhs_complete(rhs)
267
+ if rule.get('dateMode') in _ANCHOR_VALUE_FREE:
268
+ return True
269
+ value = _as_str(rule.get('value'))
270
+ if op == 'between':
271
+ v2 = rule.get('value2')
272
+ return value != '' and v2 is not None and _as_str(v2) != ''
273
+ return value != ''
274
+
275
+
276
+ def _rhs_complete(rhs):
277
+ """Port of types.ts isRhsComplete. An incomplete rhs makes the rule INACTIVE, exactly as a
278
+ half-typed value does."""
279
+ if not isinstance(rhs, dict) or not rhs.get('colId'):
280
+ return False
281
+ if rhs.get('kind') == 'measure':
282
+ return _windows().normalize(rhs.get('window')) is not None
283
+ return rhs.get('kind') == 'field'
284
+
285
+
286
+ def _windows():
287
+ """`harness.windows` imported lazily, so this module stays importable on its own.
288
+
289
+ filter_sql is otherwise PURE (no model/store/app imports) and that property is what lets
290
+ aios_grid embed the grid anywhere. windows.py is the one exception and it is a deliberate
291
+ one: the date vocabulary is a two-ENGINE contract, and a third copy of "what the past month
292
+ means" is precisely the drift verify_windows.py exists to prevent. It imports nothing but
293
+ datetime, so purity is preserved in substance.
294
+ """
295
+ from harness import windows as _w
296
+ return _w
297
+
298
+
299
+ #: Mirrors windows.ANCHOR_VALUE_FREE, resolved lazily on first use (see _windows).
300
+ class _AnchorValueFree:
301
+ def __contains__(self, mode):
302
+ return mode in _windows().ANCHOR_VALUE_FREE
303
+
304
+
305
+ _ANCHOR_VALUE_FREE = _AnchorValueFree()
306
+
307
+
308
+ def _window_bounds(spec, today, probe=False):
309
+ """Resolve a date-condition RANGE, or refuse.
310
+
311
+ `probe=True` answers only "is this a resolvable window?" without needing `today` β€” which is
312
+ what activeness has to know while compiling, before any date is in hand.
313
+
314
+ ⚠ A missing `today` on the real path RAISES rather than widening. The client's answer to the
315
+ same gap is to match nothing; the server's must be to refuse, because a server that quietly
316
+ drops a condition reports an authoritative row_count for a query nobody asked for. Same rule
317
+ as the sibling/depth caps: truncate on the client, refuse on the server.
318
+ """
319
+ w = _windows()
320
+ if probe:
321
+ return w.normalize(spec) is not None
322
+ if today is None:
323
+ raise ValueError(
324
+ 'a relative-date condition needs `today` β€” compile_filter_tree(today=...). '
325
+ 'Refusing rather than dropping the condition, which would widen the result while '
326
+ 'the count beside it still looked authoritative.')
327
+ return w.resolve(spec, today)
328
+
329
+
330
+ def _as_str(v):
331
+ """The builder stores values as raw input strings; be tolerant of a number that survived
332
+ an older payload. `str(500) != ''` is active and `to_num` reads it back, matching JS where
333
+ `500 === ""` is false."""
334
+ return '' if v is None else str(v)
335
+
336
+
337
+ # --------------------------------------------------------------------------- leaf
338
+
339
+ def _compile_leaf(rule, columns, state):
340
+ """Port of `matchFilter`. Returns (sql, params) or None when the rule is INACTIVE."""
341
+ if not is_rule_active(rule, columns):
342
+ return None
343
+ col_id = rule['colId']
344
+ op = rule.get('op')
345
+ value = _as_str(rule.get('value'))
346
+
347
+ # --- owner item 5: a COHORT leaf, before any column lookup -------------------------------
348
+ # `__cohort__` is not a column. Handled first for the same reason evalNode handles it first:
349
+ # falling through to the column path would make it INACTIVE, and inactive means every row.
350
+ if col_id == COHORT_FIELD:
351
+ return _compile_cohort(rule, state)
352
+
353
+ # --- C-OPS (wave 2026-08-02): a RANK leaf is structurally unanswerable at row grain ------
354
+ # Top-N / quantile membership is a property of the SET, not the row; the client engine
355
+ # answers it with a set pass over the sibling-filtered domain. Refuse loudly rather than
356
+ # compile something that means a different question (see RANK_OPS).
357
+ if op in RANK_OPS:
358
+ raise ValueError(
359
+ f'rank operator {op!r} cannot compile to row SQL - evaluate it as a set pass '
360
+ f'(the client engine) or pre-resolve the matching ids before compiling')
361
+
362
+ spec = columns[col_id]
363
+
364
+ def used(sql, params=()):
365
+ state['used'].add(col_id)
366
+ if spec.get('aggregate'):
367
+ state['aggregate'] = True
368
+ return (sql, list(params))
369
+
370
+ # --- value-free ops, BEFORE any value-based short-circuit (the no-op bug) ---------------
371
+ if op == 'isEmpty':
372
+ return used(_blank_sql(spec))
373
+ if op == 'isNotEmpty':
374
+ return used(f"NOT {_blank_sql(spec)}")
375
+
376
+ v = _value_sql(spec)
377
+ t = spec['type']
378
+
379
+ # --- CG-9: the comparand is another COLUMN, coerced through the LEFT field's type --------
380
+ # `_value_sql` on a spec whose `sql` is the RIGHT column and whose `type` is the LEFT one β€”
381
+ # exactly what TS does (`toNum(other)` for a numeric LHS, `String(other ?? "")` for a date,
382
+ # `.toLowerCase()` for text). Reading the right column through its OWN type would compare a
383
+ # rounded number against an unrounded one and disagree on the ties.
384
+ rhs_v = None
385
+ if rule.get('rhs') is not None:
386
+ # No `rspec is None` branch: `is_rule_active` already rejected an rhs naming a column
387
+ # this table does not have, so it cannot be missing here. A defensive fallback would be
388
+ # unreachable, and the negative control showed exactly that β€” deleting it changed
389
+ # nothing, which is how dead code hides behind a careful-looking line.
390
+ rspec = columns[rule['rhs']['colId']]
391
+ rhs_v = _value_sql({'sql': rspec['sql'], 'type': t})
392
+ state['used'].add(rule['rhs']['colId'])
393
+ if rspec.get('aggregate'):
394
+ state['aggregate'] = True
395
+
396
+ if t in NUMERIC_TYPES:
397
+ if rhs_v is not None:
398
+ return used(f"({v} {_NUM_CMP[op]} {rhs_v})") if op in _NUM_CMP else (SQL_TRUE, [])
399
+ if op == 'between':
400
+ a, b = to_num(value), to_num(rule.get('value2'))
401
+ return used(f"({v} >= ? AND {v} <= ?)", [min(a, b), max(a, b)])
402
+ if op in _NUM_CMP:
403
+ return used(f"({v} {_NUM_CMP[op]} ?)", [to_num(value)])
404
+ return (SQL_TRUE, []) # contains/doesNotContain on a number -> default: true
405
+
406
+ if t == 'date':
407
+ if op == 'within':
408
+ bounds = _window_bounds(rule.get('dateWindow'), state.get('today'))
409
+ if bounds is None:
410
+ return ('1=0', []) # unreachable: activeness probed it. Never widen.
411
+ lo, hi = bounds
412
+ # `{v} <> ''` is NOT redundant. `_value_sql` COALESCEs a NULL date to '', and
413
+ # '' <= any upper bound is true β€” so an open-ended window would sweep in every
414
+ # never-ordered customer. TS guards it with `if (a === "") return false`.
415
+ parts, params = [f"{v} <> ''"], []
416
+ if lo is not None:
417
+ parts.append(f"{v} >= ?")
418
+ params.append(lo)
419
+ if hi is not None:
420
+ parts.append(f"{v} <= ?")
421
+ params.append(hi)
422
+ return used('(' + ' AND '.join(parts) + ')', params)
423
+ if rhs_v is not None:
424
+ return used(f"({v} {_DATE_CMP[op]} {rhs_v})") if op in _DATE_CMP else (SQL_TRUE, [])
425
+ if op == 'between':
426
+ v2 = _as_str(rule.get('value2'))
427
+ lo, hi = (value, v2) if value < v2 else (v2, value)
428
+ return used(f"({v} >= ? AND {v} <= ?)", [lo, hi])
429
+ if op in _DATE_CMP:
430
+ anchor = _resolve_anchor(rule.get('dateMode'), value, state.get('today'))
431
+ if anchor is None:
432
+ return ('1=0', []) # unresolvable anchor -> matches nothing (TS: false)
433
+ return used(f"({v} {_DATE_CMP[op]} ?)", [anchor])
434
+ return (SQL_TRUE, []) # contains/doesNotContain on a date -> default: true
435
+
436
+ # text / status β€” case-insensitive. strpos, not LIKE: `includes` is a LITERAL substring
437
+ # test, so a value holding % or _ must not act as a wildcard.
438
+ #
439
+ # BOTH SIDES fold in SQL (`LOWER(?)`, never Python's str.lower()). Python and JS agree on
440
+ # case folding; DuckDB does NOT β€” LOWER('ISTANBUL' with a dotted capital I) drops the dot
441
+ # while both languages keep it as i + combining dot. Folding the needle in Python and the
442
+ # haystack in DuckDB therefore produced two different strings, and a value could fail to
443
+ # match ITSELF. One folder for both operands is the only version that cannot do that.
444
+ if rhs_v is not None:
445
+ # Both operands are already folded by `_value_sql` (text branch), so no LOWER(?) here.
446
+ if op == 'contains':
447
+ return used(f"(strpos({v}, {rhs_v}) > 0)")
448
+ if op == 'doesNotContain':
449
+ return used(f"(strpos({v}, {rhs_v}) = 0)")
450
+ if op == 'eq':
451
+ return used(f"({v} = {rhs_v})")
452
+ if op == 'neq':
453
+ return used(f"({v} <> {rhs_v})")
454
+ return (SQL_TRUE, [])
455
+ if op == 'contains':
456
+ return used(f"(strpos({v}, LOWER(?)) > 0)", [value])
457
+ if op == 'doesNotContain':
458
+ return used(f"(strpos({v}, LOWER(?)) = 0)", [value])
459
+ if op == 'eq':
460
+ return used(f"({v} = LOWER(?))", [value])
461
+ if op == 'neq':
462
+ return used(f"({v} <> LOWER(?))", [value])
463
+ return (SQL_TRUE, []) # gt/gte/lt/lte/between/within on text -> default: true
464
+
465
+
466
+ def _resolve_anchor(mode, value, today):
467
+ """A date condition's anchor -> one ISO date, or None. `today` may be absent ONLY for the
468
+ `exact` mode, which does not use it.
469
+
470
+ The sentinel is deliberate and load-bearing for BACK-COMPAT: every date filter that existed
471
+ before anchors is `exact`, and every caller that compiles one (store_query, store_rows)
472
+ passes no `today`. Requiring it unconditionally would turn a working filter into an
473
+ exception. A RELATIVE anchor still refuses β€” the server never widens.
474
+ """
475
+ mode = mode or 'exact'
476
+ if today is None:
477
+ if mode != 'exact':
478
+ raise ValueError(
479
+ f"the date anchor {mode!r} is relative and needs `today` β€” "
480
+ f"compile_filter_tree(today=...). Refusing rather than dropping the condition, "
481
+ f"which would widen the result under an authoritative-looking count.")
482
+ # `exact` ignores `today` entirely (verify_windows resolves it across 11 probe dates and
483
+ # gets the same answer every time), so any valid date serves as the unused argument.
484
+ today = '1970-01-01'
485
+ return _windows().resolve_anchor(mode, value, today)
486
+
487
+
488
+ def _compile_cohort(rule, state):
489
+ """Port of evalNode's cohort branch: membership in a SET of hand-curated sets of row ids.
490
+
491
+ Owner 2026-07-27: the leaf names one OR MORE cohorts and asks `anyOf` / `allOf` / `noneOf`.
492
+ A single id is a one-element set, so the shipped `is part of` compiles down this same path.
493
+
494
+ States, mirroring TS exactly:
495
+ every set present -> the union (anyOf / noneOf) or the intersection (allOf)
496
+ a set present but EMPTY -> that member contributes 1=0, because nobody is in an empty
497
+ cohort. So `allOf` collapses to 1=0 and `noneOf` to 1=1 β€”
498
+ which is what `set.has(pid)` on an empty Set does in TS.
499
+ ANY set ABSENT -> 1=0 for ALL THREE ops. The condition is unanswerable, and
500
+ unanswerable matches nothing: answering "everything" to
501
+ `is none of [a cohort we cannot see]` is the widening sin
502
+ wearing a plausible face, and answering it for the members we
503
+ DO have would silently ask a different question.
504
+
505
+ COALESCE(... , FALSE) because a NULL row id makes `id IN (...)` NULL, and `NOT NULL` is NULL
506
+ β€” SQL would drop the row where TS keeps it.
507
+ """
508
+ sets = state.get('cohort_sets')
509
+ if sets is None:
510
+ raise ValueError(
511
+ 'a cohort condition needs cohort_sets β€” compile_filter_tree(cohort_sets=...). '
512
+ 'Refusing rather than dropping it: a dropped condition widens the result while the '
513
+ 'count beside it still looks authoritative.')
514
+ id_sql = state.get('id_sql')
515
+ if not id_sql:
516
+ raise ValueError('a cohort condition needs id_sql (the row-identity column)')
517
+ op = COHORT_OP_ALIASES.get(rule.get('op'), rule.get('op'))
518
+ named = parse_cohort_ids(rule.get('value'))
519
+ members = [sets.get(cid) for cid in named]
520
+ if not named or any(m is None for m in members):
521
+ return ('1=0', [])
522
+
523
+ parts, params = [], []
524
+ for ids in members:
525
+ ids = [int(x) for x in ids]
526
+ if not ids:
527
+ parts.append('1=0')
528
+ continue
529
+ parts.append(f"COALESCE({id_sql} IN ({','.join('?' for _ in ids)}), FALSE)")
530
+ params.extend(ids)
531
+ if op == 'allOf':
532
+ return ('(' + ' AND '.join(parts) + ')', params)
533
+ union = '(' + ' OR '.join(parts) + ')'
534
+ return ((f"(NOT {union})" if op == 'noneOf' else union), params)
535
+
536
+
537
+ # --------------------------------------------------------------------------- tree
538
+
539
+ def _compile_node(node, columns, state):
540
+ """Port of `evalNode`. Returns (sql, params), or None when the node is INACTIVE.
541
+
542
+ A group combines only its ACTIVE children with its OWN conjunction; a group with no active
543
+ children is itself inactive and disappears. That is what stops a half-built group from
544
+ either zeroing the grid (`and`) or disabling it (`or`).
545
+ """
546
+ if not isinstance(node, dict):
547
+ return None
548
+ if isinstance(node.get('children'), list):
549
+ parts, params = [], []
550
+ for child in node['children']:
551
+ got = _compile_node(child, columns, state)
552
+ if got is None:
553
+ continue # inactive -> ignored entirely
554
+ parts.append(got[0])
555
+ params.extend(got[1])
556
+ if not parts:
557
+ return None
558
+ joiner = ' OR ' if node.get('conj') == 'or' else ' AND '
559
+ return ('(' + joiner.join(parts) + ')', params)
560
+ return _compile_leaf(node, columns, state)
561
+
562
+
563
+ def compile_filter_tree(nodes, conj='and', columns=None, member_ids=None, id_sql=None,
564
+ today=None, cohort_sets=None):
565
+ """Port of `matchFilterTree` (+ the `memberPids` union in `useVisibleRows`).
566
+
567
+ Returns a `Compiled`, or **None when nothing is active** β€” meaning no predicate at all, not
568
+ a false one. `matchFilterTree` returns true when the tree is inactive, so emitting anything
569
+ here would narrow rows the client would have shown.
570
+
571
+ `member_ids` reproduces `members.has(r.pid) || matchFilterTree(...)`. Note the union lives
572
+ INSIDE `if (filters.length)` and is OR'd with a tree that is true for everything when
573
+ inactive β€” so an inactive tree must emit NOTHING, never `pid IN (...)`, which would narrow
574
+ to just the hand-picked members. Same family of bug as the tri-state one, inverted.
575
+
576
+ `today` is the TENANT'S today, required by any relative date condition (`is within …`,
577
+ `is before [one month ago]`) and by nothing else β€” an `exact` date needs none, which is what
578
+ keeps every pre-anchor caller working unchanged. `cohort_sets` is `{cohortId: [pid, ...]}`
579
+ for cohort-membership leaves. Both REFUSE when needed and missing; neither defaults.
580
+ """
581
+ state = {'aggregate': False, 'used': set(),
582
+ 'today': today, 'cohort_sets': cohort_sets, 'id_sql': id_sql}
583
+ got = _compile_node({'conj': conj, 'children': list(nodes or [])}, columns or {}, state)
584
+ if got is None:
585
+ return None
586
+ sql, params = got
587
+ if member_ids:
588
+ if not id_sql:
589
+ raise ValueError('member_ids requires id_sql (the row-identity column)')
590
+ ids = [int(x) for x in member_ids]
591
+ sql = f"({sql} OR {id_sql} IN ({','.join('?' for _ in ids)}))"
592
+ params = list(params) + ids
593
+ return Compiled(sql, list(params), state['aggregate'], frozenset(state['used']))
594
+
595
+
596
+ def compile_search(q, columns):
597
+ """Port of `matchSearch`: the narrowing search box.
598
+
599
+ True if ANY human-readable (text/status) column contains `q`, case-insensitively. NUMERIC
600
+ columns are skipped on purpose so a query like "10" does not match revenue or order counts.
601
+ Dates are skipped too β€” `matchSearch` tests only text and status.
602
+
603
+ The caller ANDs this with the filter predicate, matching the TS pipeline order
604
+ (filter -> search). It is deliberately OUTSIDE the `member_ids` union: in `useVisibleRows`
605
+ the union applies to the filter step only, so a hand-picked member is still searchable away.
606
+ """
607
+ # `.strip(_JS_WS)`, not `.strip()`: the client trims with JS String.trim(), whose whitespace
608
+ # set differs from Python's in BOTH directions (Python strips U+0085 NEL and U+001C..1F,
609
+ # JS does not; JS strips U+FEFF, Python does not). A query of just those characters is
610
+ # blank to one engine and a real search to the other. Folding is left to SQL β€” see the
611
+ # LOWER(?) note in _compile_leaf.
612
+ q = (q or '').strip(_JS_WS)
613
+ if not q:
614
+ return None
615
+ parts, params, used = [], [], set()
616
+ aggregate = False
617
+ for col_id, spec in (columns or {}).items():
618
+ if spec['type'] not in TEXTUAL_TYPES:
619
+ continue
620
+ parts.append(f"(strpos({_value_sql(spec)}, LOWER(?)) > 0)")
621
+ params.append(q)
622
+ used.add(col_id)
623
+ aggregate = aggregate or bool(spec.get('aggregate'))
624
+ if not parts:
625
+ # No searchable column: `matchSearch` returns false for every row, so the search
626
+ # narrows to nothing. That is a real predicate, not an absent one.
627
+ return Compiled('1=0', [], False, frozenset())
628
+ return Compiled('(' + ' OR '.join(parts) + ')', params, aggregate, frozenset(used))
629
+
630
+
631
+ def compile_order_by(sorts, columns, tiebreak_sql=None):
632
+ """Port of `makeComparator`. Returns an ORDER BY body, or None when nothing sorts.
633
+
634
+ `tiebreak_sql` (the row-identity column) is appended as a final ASC key and is REQUIRED for
635
+ any windowed fetch. `Array.prototype.sort` is stable, so TS leaves tied rows in their
636
+ incoming order; SQL guarantees NOTHING for ties, so LIMIT/OFFSET paging over a non-total
637
+ order can repeat a row on one page and drop it from another. Appending row identity makes
638
+ the order total. It is a STRENGTHENING of the TS guarantee, not a divergence: the payload a
639
+ windowed fetch produces is itself in this order, so the client never re-sorts it differently.
640
+
641
+ BLANKS SORT LAST IN BOTH DIRECTIONS (Airtable's behavior, and the reason descending
642
+ "Last order" leads with the most recent rather than with never-ordered customers). Each key
643
+ contributes TWO terms: an always-ASC blank flag, then the value in the requested direction.
644
+ Two blanks tie on both terms, so the NEXT key decides β€” matching the comparator's
645
+ `if (aBlank && bBlank) continue`.
646
+
647
+ An unknown column is skipped: in TS its values are `undefined` on both sides, so both are
648
+ blank and the key is a no-op tiebreak.
649
+
650
+ Direction: `s.dir === "asc" ? c : -c` β€” only the exact string "asc" ascends, anything else
651
+ descends. Faithfully reproduced; do not "fix" it to default ASC.
652
+
653
+ Residual divergence, deliberate: TS uses `localeCompare`, we use LOWER() + DuckDB's binary
654
+ collation. They agree on case-insensitive ordering of ASCII and on ISO dates (unaffected by
655
+ LOWER), and differ on locale tie-breaks between case-variants of the same string ("a" vs
656
+ "A" sort adjacent under localeCompare, tie under LOWER) and on accent ordering.
657
+ """
658
+ terms = []
659
+ for s in (sorts or []):
660
+ if not isinstance(s, dict):
661
+ continue
662
+ spec = (columns or {}).get(s.get('colId'))
663
+ if spec is None:
664
+ continue
665
+ terms.append(f"CASE WHEN {_blank_sql(spec)} THEN 1 ELSE 0 END ASC")
666
+ terms.append(f"{_value_sql(spec)} {'ASC' if s.get('dir') == 'asc' else 'DESC'}")
667
+ # Append the tiebreak even when NO sort key survived (all unknown, or none given). A
668
+ # windowed fetch needs a total order regardless of what the user sorted by β€” an unsorted
669
+ # LIMIT/OFFSET is exactly as unstable as a tied one. Callers wanting pure client parity
670
+ # pass no tiebreak_sql and still get None.
671
+ if tiebreak_sql:
672
+ terms.append(f"{tiebreak_sql} ASC")
673
+ return ', '.join(terms) or None
platform/harness/measure_filter.py ADDED
@@ -0,0 +1,788 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """harness/measure_filter.py β€” answering `Where [Sales] [in the last 90 days] [>] [5,000]` (CG-8).
2
+
3
+ GATED 2026-07-26 by `platform/verify_measure_filter.py` β€” 119 assertions against the real
4
+ store (every resolved id set compared to a per-customer aggregate summed in Python from raw rows)
5
+ plus a synthetic store for the cases live data cannot produce. Negative control: 24 faithful
6
+ mutations, 24 caught. The gate found two things the reasoning below asserted and the code did not
7
+ do, both now fixed and both re-caught by the control:
8
+
9
+ - a rule with NO window resolved to ALL TIME. The error message four lines below says it
10
+ refuses to do exactly that; the line above it did it anyway.
11
+ - `sum()` over an all-NULL group is NULL, and `NULL > x` and `NULL <= x` are BOTH false, so
12
+ such a customer fell out of the two halves of the same question. Now COALESCEd to 0.
13
+
14
+ THE PROBLEM. `revenue_ytd` is a pre-computed column in `customer_data.pool()`. "Sales in the last
15
+ 90 days" is a DIFFERENT aggregate over a DIFFERENT window, and no column holds it. So a measure
16
+ condition cannot be answered by looking at the row in hand β€” it has to be a query.
17
+
18
+ THE SHAPE. Resolve each measure condition server-side to a **set of customer ids**, and let the
19
+ existing client engine treat that set as a membership test. Per condition, one query:
20
+
21
+ SELECT rp_id FROM <topic scope> WHERE <window> GROUP BY rp_id
22
+ HAVING <the metric's OWN aggregate expression> <op> ?
23
+
24
+ Three properties that fall out, none incidental:
25
+
26
+ * **No second definition of a number.** The aggregate comes from `semantic._measure_sql` β€” the
27
+ same expression the Analyst and every metric already use. This cannot disagree with "revenue"
28
+ elsewhere because it IS that computation.
29
+ * **The displayed columns are untouched.** `revenue_ytd` still comes from `pool()`. We ADD a
30
+ predicate; we do not re-source the table. So this needs no `pool()`β†’store migration, and does
31
+ not inherit that migration's parity problem.
32
+ * **Counts stay honest.** The client still holds every row, so the table stays whole-book and
33
+ `countLabel` keeps counting what it actually has (CG-3 `whole-book` mode).
34
+
35
+ ⚠ WHAT MAY BE OFFERED, AND WHY IT IS AN ALLOWLIST
36
+ `store_parity()` validates exactly five metrics against live Odoo. Every OTHER sum-metric on the
37
+ topic would get an expression with no independent check behind it β€” self-consistent with the rest
38
+ of the app and still wrong if the expression is wrong. Against "robust for EVERY numerically
39
+ driven object", that gap is load-bearing, so admission is gated on HAVING A CONTRACT and spelled
40
+ as a list. An allowlist fails CLOSED when a metric is added; iterating the topic's metrics would
41
+ fail open, silently offering an unvalidated number in the one place a number looks authoritative.
42
+
43
+ ⚠ AN UNRESOLVABLE CONDITION RAISES. It never returns "everything" or "nothing": both are answers,
44
+ and a wrong answer here is invisible β€” the grid just shows a different set of customers under a
45
+ count that looks authoritative. `ModelError` is the only safe failure.
46
+ """
47
+ from harness import semantic as _sem
48
+ from harness import windows as _win
49
+
50
+ #: metric key -> how it reads in the condition builder. ONLY metrics with an independent
51
+ #: validation contract (semantic.store_parity / a metric `validate:` block) appear here.
52
+ #: `orders` lives on sales_orders and `customers` is a property of a SET, not of a customer β€”
53
+ #: neither is a per-customer measure, so neither is offered.
54
+ #: Each entry carries its TOPIC, because admission is no longer single-topic. `orders` lives on
55
+ #: sales_orders (order-header grain) while the other three live on sales_lines β€” and the topic
56
+ #: decides the FROM clause, the scope SQL and which dim identifies a customer. Letting the caller
57
+ #: pass the topic (the old shape) meant a caller could pass a wrong one; deriving it from the
58
+ #: measure means there is nothing to get wrong.
59
+ ADMITTED = {
60
+ 'revenue': {'label': 'Sales', 'topic': 'sales_lines'},
61
+ 'margin': {'label': 'Gross Margin $', 'topic': 'sales_lines'},
62
+ 'units': {'label': 'Units', 'topic': 'sales_lines'},
63
+ # `orders` is admitted on the SAME rule as the other three and not by exception:
64
+ # `semantic.store_parity()` checks it store-vs-live per BU, and the metric carries an
65
+ # `order_count` validate contract. It is the replacement for the retired `Orders 24M`
66
+ # column β€” same question, with the period as a parameter instead of frozen at 24 months.
67
+ 'orders': {'label': 'Orders', 'topic': 'sales_orders'},
68
+ # ⚠ The first COMPOSITE measure (2026-07-27, owner's follow-up): a RATIO, not an aggregate.
69
+ # `aov` is `revenue` (line grain, sales_lines) over `orders` (header grain, sales_orders), so
70
+ # it has no single query and `_measure_sql` REFUSES it β€” which is the fail-closed shape that
71
+ # makes a mis-wired ratio raise instead of quietly computing something else. It is resolved
72
+ # by computing each BASE per customer and combining them through `semantic._post_compute`,
73
+ # the same function every other surface uses, so AOV here cannot disagree with AOV anywhere.
74
+ # `topic` names where the metric is DECLARED; the bases each resolve on their own.
75
+ 'aov': {'label': 'Avg order $', 'topic': 'sales_lines'},
76
+ # ⚠ `input_scale` SETTLES the question this measure was withheld over. The metric stays a
77
+ # FRACTION (margin/revenue, 0-1) β€” it is the semantic model's number and every other surface
78
+ # reads it β€” while the USER'S TYPED VALUE is divided by 100 on the way in, so
79
+ # `Gross Margin % > 20` compares against 0.20. The factor moves exactly once, at the input
80
+ # boundary, and never touches the metric. Scaling a COMPUTED comparand (a statistic, or the
81
+ # other side of a measure-vs-measure) would divide by 100 twice, so it is applied to the
82
+ # literal alone and the gate proves that by construction.
83
+ 'margin_pct': {'label': 'Gross Margin %', 'topic': 'sales_lines', 'input_scale': 100.0},
84
+ # ⚠ COMPANY-LEVEL (2026-07-27, owner's ask). Credit notes carry team_id = 1 for everything,
85
+ # so a business-unit total is not a thing that exists β€” `_query_parts` refuses one and the
86
+ # live domain builder refuses one. `measure_fields(team_id)` therefore stops OFFERING these
87
+ # to a BU-scoped caller rather than letting them build a condition that can only error: a
88
+ # dead entry in the picker is worse than an absent one, and hiding what cannot be answered
89
+ # is what `filterable: false` already does for columns.
90
+ 'returns': {'label': 'Returns $', 'topic': 'credit_notes'},
91
+ 'returns_pct': {'label': 'Return %', 'topic': 'credit_notes', 'input_scale': 100.0},
92
+ }
93
+
94
+ #: The metrics `semantic.store_parity()` actually exercises against live Odoo. ADMITTED must be
95
+ #: a SUBSET of this β€” that is the whole admission rule ("does this measure carry an independent
96
+ #: validation contract"), and stating it as data lets the gate assert it instead of trusting the
97
+ #: comment above. Adding a measure here without adding it to store_parity's loop fails the gate.
98
+ PARITY_CHECKED = frozenset({'revenue', 'margin', 'units', 'orders', 'customers',
99
+ 'returns', 'invoiced'})
100
+
101
+ #: numeric comparison ops, mirroring filter_sql._NUM_CMP. `between` is expressed as two
102
+ #: conditions by the caller rather than a third operator shape.
103
+ OPS = {'gt': '>', 'gte': '>=', 'lt': '<', 'lte': '<=', 'eq': '=', 'neq': '<>'}
104
+
105
+ #: CG-9 β€” the ops allowed when the right-hand side is ANOTHER MEASURE. `eq`/`neq` are withheld,
106
+ #: and not out of caution: both sides are sums of floats the user never sees, so `=` would match
107
+ #: nobody and `β‰ ` would match everybody, and both would LOOK like they worked. CG-8 could offer
108
+ #: `=` honestly because the other side was a number a person typed. Mirrors types.ts
109
+ #: MEASURE_PAIR_OPS; the gate asserts the two lists are the same.
110
+ PAIR_OPS = ('gte', 'gt', 'lte', 'lt')
111
+
112
+ #: The statistics a condition may compare against β€” "who is in the top quartile" (owner,
113
+ #: 2026-07-27). Each is computed over the CALLER'S POOL for the same measure and window.
114
+ #: The labels are the sentence the user reads; the client mirrors them.
115
+ STATS = {
116
+ 'avg': 'the average',
117
+ 'median': 'the median',
118
+ 'p10': 'the bottom 10%',
119
+ 'p25': 'the bottom quartile',
120
+ 'p75': 'the top quartile',
121
+ 'p90': 'the top 10%',
122
+ }
123
+ # ⚠ The owner's OWN words, and short enough to fit: "the 75th percentile (top quartile)" clipped
124
+ # to "the 75th perce" in the value select, and a truncated parenthetical reads as a rendering
125
+ # bug. `> the top quartile` is also how a person says it out loud, so the shorter label is the
126
+ # better one twice over. Seen in a live screenshot of a run whose every assertion was green.
127
+
128
+ #: ⚠ THE POPULATION IS THE CALLER'S POOL, INCLUDING ZEROS. A customer who bought nothing has a
129
+ #: measure of 0, not of nothing β€” that is the convention the zero-group already establishes
130
+ #: everywhere else, and "the median customer" means the median of the customers you can see.
131
+ #: Two consequences worth knowing before reading a result:
132
+ #: Β· it is a PERMISSION boundary as well as a definition β€” a scoped user's median is theirs,
133
+ #: and a statistic over the whole book would leak the shape of data they cannot open;
134
+ #: Β· on a measure most customers score 0 on, the 75th percentile IS 0, so "top quartile"
135
+ #: degenerates to "anyone with any activity at all". True, and surprising.
136
+
137
+
138
+ def percentile(values, q):
139
+ """The `q` quantile of `values` by LINEAR INTERPOLATION between closest ranks β€” the R-7
140
+ convention, which is what Excel's PERCENTILE.INC computes. `q` may also be a STATS key.
141
+
142
+ Pinned to one convention on purpose: nearest-rank and interpolation disagree on any even
143
+ population (the median of [1,2,3,4] is 2.5 here and 2 or 3 under nearest-rank), and the gate
144
+ checks it against hand-computed values rather than against `statistics.median` β€” a reference
145
+ borrowed from a library only proves we match that library.
146
+
147
+ Returns None for an empty population: there is no median of nobody, and 0 would be a number
148
+ the caller could act on.
149
+ """
150
+ if isinstance(q, str):
151
+ if q == 'avg':
152
+ xs = list(values)
153
+ return (sum(xs) / len(xs)) if xs else None
154
+ if q not in STATS:
155
+ raise _sem.ModelError(
156
+ f"{q!r} is not a statistic. Offered: {sorted(STATS)}")
157
+ q = 0.5 if q == 'median' else int(q[1:]) / 100.0
158
+ xs = sorted(float(v) for v in values)
159
+ if not xs:
160
+ return None
161
+ if len(xs) == 1:
162
+ return xs[0]
163
+ pos = (len(xs) - 1) * float(q)
164
+ lo = int(pos)
165
+ hi = min(lo + 1, len(xs) - 1)
166
+ return xs[lo] + (xs[hi] - xs[lo]) * (pos - lo)
167
+
168
+
169
+ #: The same six comparisons as OPS, as Python callables β€” used wherever the comparison happens in
170
+ #: Python rather than in a HAVING (a composite measure, or a two-measure pair).
171
+ _CMP = {'gt': lambda a, b: a > b, 'gte': lambda a, b: a >= b,
172
+ 'lt': lambda a, b: a < b, 'lte': lambda a, b: a <= b,
173
+ 'eq': lambda a, b: a == b, 'neq': lambda a, b: a != b}
174
+
175
+
176
+ def is_company_level(key):
177
+ """True when this measure cannot be scoped to a business unit.
178
+
179
+ Checked over every BASE the measure decomposes into, not just the declared topic: `Return %`
180
+ names credit_notes but also reads customer_invoices, and one un-scopable half is enough to
181
+ make the whole ratio un-scopable.
182
+ """
183
+ bases, _ = _sem._expand_measures([key])
184
+ tps = _sem.topics()
185
+ for b in bases:
186
+ store = (tps.get(_sem.metrics()[b].get('topic')) or {}).get('store') or {}
187
+ if not store.get('team_col'):
188
+ return True
189
+ return False
190
+
191
+
192
+ def measure_fields(team_id=None):
193
+ """The measures the condition builder may offer, in the payload's field shape.
194
+
195
+ These are FILTERABLE BUT NOT DISPLAYABLE β€” a measure is a question about a window, not a
196
+ column of the table, and rendering it as a column would silently pick a window for the user.
197
+
198
+ ⚠ `team_id` NARROWS THE OFFER. A COMPANY-LEVEL measure (returns, and anything else built on
199
+ documents that carry no trustworthy BU tag) cannot be answered for one business unit β€”
200
+ `_query_parts` refuses, and refusing is right. But a picker entry whose every use ends in an
201
+ error is worse than no entry, so a BU-scoped caller is not offered them at all. Same
202
+ principle as `filterable: false` on a column: do not offer what cannot be answered.
203
+ """
204
+ mts = _sem.metrics()
205
+ out = []
206
+ for key, spec in ADMITTED.items():
207
+ m = mts.get(key)
208
+ if not m:
209
+ continue
210
+ if team_id and is_company_level(key):
211
+ continue
212
+ out.append({'key': key, 'label': spec['label'],
213
+ 'type': _sem._FORMAT_TYPE.get(m.get('format'), 'currency')})
214
+ return out
215
+
216
+
217
+ def topic_for(key):
218
+ """The topic a measure is answered on. Raises for anything not admitted."""
219
+ spec = ADMITTED.get(key)
220
+ if not spec:
221
+ raise _sem.ModelError(
222
+ f"{key!r} is not an admitted measure. Offered: {sorted(ADMITTED)} β€” a measure "
223
+ f"without an independent validation contract is not filterable (see this module's "
224
+ f"docstring).")
225
+ return spec['topic']
226
+
227
+
228
+ def _entity_sql(topic):
229
+ """(from-clause, the id column customers are grouped by, the store binding) for a topic."""
230
+ t = _sem.topics().get(topic)
231
+ if not t or 'store' not in t:
232
+ raise _sem.ModelError(f"topic {topic!r} has no store binding")
233
+ s = t['store']
234
+ dims = s.get('dims') or {}
235
+ # sales_lines calls it `order_partner` (the line's parent order's partner); sales_orders
236
+ # calls the same thing `partner`. One name per topic, both meaning "the customer this row
237
+ # belongs to" β€” which is the only dim this module ever groups by.
238
+ dim = dims.get('order_partner') or dims.get('partner')
239
+ if not dim:
240
+ raise _sem.ModelError(f"topic {topic!r} has no customer dim to group by")
241
+ return f"{s['table']} {s['alias']} {s.get('join', '')}", dim['col'], s
242
+
243
+
244
+ def is_composite(key):
245
+ """True for a measure with no single aggregate expression β€” a `ratio` or a `derived`.
246
+
247
+ These are answered by computing each BASE per customer and combining in Python; they can
248
+ never be a HAVING, which is why `_measure_sql` refusing them is a feature.
249
+ """
250
+ return _sem.metrics().get(key, {}).get('agg') in ('ratio', 'derived')
251
+
252
+
253
+ def _base_topic(key):
254
+ """The topic a BASE metric aggregates on β€” read from the METRIC, never from ADMITTED.
255
+
256
+ A composite decomposes into bases that need not themselves be offerable (and whose topics may
257
+ differ from each other: `aov`'s numerator is on sales_lines and its denominator on
258
+ sales_orders). Resolving the topic from the allowlist would make that undecidable.
259
+ """
260
+ m = _sem.metrics().get(key)
261
+ if not m:
262
+ raise _sem.ModelError(f"unknown metric {key!r}")
263
+ topic = m.get('topic')
264
+ if not topic:
265
+ raise _sem.ModelError(f"metric {key!r} declares no topic to aggregate on")
266
+ return topic
267
+
268
+
269
+ def _query_parts(key, window, today, team_id):
270
+ """The FROM / GROUP-BY id / aggregate / WHERE for one (measure, window), shared by the
271
+ one-sided and the two-sided resolver so there is one definition of the scope."""
272
+ topic = _base_topic(key)
273
+ # NO DEFAULT WINDOW. A rule that carries no window is not "all time" β€” it is a rule whose
274
+ # window we do not know, and answering it over all time WIDENS the condition while the count
275
+ # beside it still looks authoritative. `all_time` is a legitimate ANSWER the user can pick;
276
+ # it is not a legitimate GUESS. (This line shipped as `rule.get('window') or {...}` and the
277
+ # gate caught it contradicting the error message two lines below.)
278
+ bounds = _win.resolve(window, today)
279
+ if bounds is None:
280
+ raise _sem.ModelError(
281
+ f"unresolvable window {window!r} β€” refusing rather than defaulting to all "
282
+ f"time, which would widen the condition while the count still looked authoritative")
283
+ date_from, date_to = bounds
284
+
285
+ frm, id_col, s = _entity_sql(topic)
286
+ m = _sem.metrics()[key]
287
+ # COALESCE, because SQL's sum() over an all-NULL group is NULL, and `NULL > x` and
288
+ # `NULL <= x` are BOTH false: the customer would vanish from the two halves of the same
289
+ # question, so `margin <= 0` and `margin > 0` would not cover the book. The zero-group below
290
+ # cannot rescue them either β€” they DO have rows in the window, just no values. Python's sum
291
+ # of the same rows is 0.0, and 0 is what the grid would show, so 0 is the answer.
292
+ agg = f"COALESCE({_sem._measure_sql(m, s['alias'])}, 0)"
293
+
294
+ where, params = [_sem.render_scope(s['scope_sql'].strip())], []
295
+ if date_from:
296
+ where.append(f"CAST({s['date_col']} AS TIMESTAMP) >= ?")
297
+ params.append(f"{date_from} 00:00:00")
298
+ if date_to:
299
+ where.append(f"CAST({s['date_col']} AS TIMESTAMP) <= ?")
300
+ params.append(f"{date_to} 23:59:59")
301
+ if team_id:
302
+ if not s.get('team_col'):
303
+ raise _sem.ModelError(f"topic {topic!r} is company-level β€” it has no business-unit "
304
+ f"filter, so a BU-scoped caller cannot be answered safely")
305
+ where.append(f"{s['team_col']} = ?")
306
+ params.append(int(team_id))
307
+ return frm, id_col, agg, ' AND '.join(where), params
308
+
309
+
310
+ def _scaled(key, value):
311
+ """A TYPED value in the units the metric actually holds.
312
+
313
+ `Gross Margin % > 20` is typed in percentage points and the metric is a fraction, so the 20
314
+ becomes 0.20. Applied to a LITERAL only β€” a statistic and the other side of a
315
+ measure-vs-measure are already on the metric's scale, and dividing them too would apply the
316
+ factor twice for a result that still looks plausible.
317
+ """
318
+ scale = ADMITTED.get(key, {}).get('input_scale')
319
+ return (value / scale) if scale else value
320
+
321
+
322
+ def resolve_stat(rule, today, team_id=None, allowed_pids=None):
323
+ """`Where [Sales] [this year] [>] [the 75th percentile]` β€” the top quartile of the pool.
324
+
325
+ One totals query for the measure, the statistic taken over the CALLER'S POOL (see the STATS
326
+ note on why the population is the pool and what that costs), then a comparison per customer.
327
+ """
328
+ stat = (rule.get('rhs') or {}).get('stat')
329
+ if stat not in STATS:
330
+ raise _sem.ModelError(f"{stat!r} is not a statistic. Offered: {sorted(STATS)}")
331
+ key = rule.get('colId')
332
+ topic_for(key)
333
+ op = rule.get('op')
334
+ if op not in PAIR_OPS:
335
+ raise _sem.ModelError(
336
+ f"{op!r} cannot compare a measure with a statistic ({sorted(PAIR_OPS)}). Equality "
337
+ f"against a computed cut-off matches nobody except by coincidence.")
338
+ pool = _pool_of(allowed_pids)
339
+ if pool is None:
340
+ raise _sem.ModelError(
341
+ "a statistic needs a population to be computed over, and none was supplied β€” "
342
+ "refusing rather than taking it over whichever customers happen to have rows, "
343
+ "which would be a different question with an authoritative-looking answer")
344
+ con = _sem._store_con()
345
+ try:
346
+ totals = _totals(con, key, rule.get('window'), today, team_id)
347
+ finally:
348
+ con.close()
349
+ # ⚠ THE POPULATION IS THE CUSTOMERS WHO HAVE THIS MEASURE AT ALL β€” owner's decision,
350
+ # 2026-07-27: *"top quartile of customers who actually has returned something"*.
351
+ #
352
+ # It was the whole pool including zeros, which is the convention everywhere else and was
353
+ # wrong HERE. The pool carries lapsed customers, so ~30% of it sits at zero for any 12-month
354
+ # window β€” that dragged the 25th percentile to exactly 0.0, made `< the bottom quartile`
355
+ # match NOBODY, and would have been worse for Return %, where most customers never return
356
+ # anything and "the top quartile" would have degenerated to "anyone with a single refund".
357
+ #
358
+ # NON-ZERO, not has-rows: a zero Return % means nothing was returned, which is precisely the
359
+ # customer the owner is excluding. It also reads as one sentence β€” the question is asked
360
+ # among customers who have that measure β€” and it generalises: "the median Sales" means the
361
+ # median among customers who sold something.
362
+ #
363
+ # The ANSWER is restricted to the same population, so `< the bottom quartile` returns a
364
+ # quarter of the returners rather than every customer who never returned.
365
+ population = {pid: v for pid, v in totals.items() if pid in pool and v}
366
+ cut = percentile(list(population.values()), stat)
367
+ if cut is None: # nobody has this measure -> nobody qualifies
368
+ return set()
369
+ return {pid for pid, v in population.items() if _CMP[op](v, cut)}
370
+
371
+
372
+ def _pool_of(allowed_pids):
373
+ """The caller's visible pool as a set of plain ints, or None.
374
+
375
+ ⚠ The pool arrives from `pool()`/a cohort and may hold numpy ints or id strings; the store
376
+ returns BIGINTs. A set difference across two int types silently finds nothing in common,
377
+ which here would look like "no lapsed customers" rather than an error.
378
+ """
379
+ return None if allowed_pids is None else {int(p) for p in allowed_pids if p is not None}
380
+
381
+
382
+ def rule_complete(rule):
383
+ """Is this rule's right-hand side FULLY present β€” a question that has actually been asked?
384
+
385
+ THE GAP THIS CLOSES (2026-07-27, the cold-container race). The component's view state
386
+ passes through the host while the user β€” or the client's own initialisation β€” is mid-edit,
387
+ and a rule can transiently arrive with its window, statistic or value missing. `to_num(None)`
388
+ is 0 by the engines' shared contract, so resolution used to ANSWER that half-asked question
389
+ as `<op> 0`: the id landed in `measureSets`, the client read the rule as answered and
390
+ cleared its pending marker, and the grid stood behind a count for a question nobody asked.
391
+ Refusing instead leaves the id OUT of the payload, which the client renders as PENDING
392
+ ("Calculating…") β€” the honest state for a question still being typed. Exactly the class the
393
+ module docstring already names: an unresolvable condition never becomes an answer.
394
+
395
+ INCOMPLETE is distinct from MALFORMED, on purpose. An ABSENT window / stat / value is a
396
+ normal transient (the caller skips it silently β€” mid-edit is not a telemetry event); a
397
+ PRESENT-but-bogus one is a real defect and still RAISES downstream (`unresolvable window`,
398
+ `not a statistic`, `not admitted`). `value` presence is textual, not numeric: "0" is a
399
+ typed answer, while parseability stays `to_num`'s business so the two engines keep one
400
+ contract for garbage input.
401
+ """
402
+ if not isinstance(rule, dict) or not rule.get('op'):
403
+ return False
404
+ if rule.get('window') is None:
405
+ return False
406
+ rhs = rule.get('rhs')
407
+ if isinstance(rhs, dict) and rhs.get('kind') == 'measure':
408
+ return bool(rhs.get('colId')) and rhs.get('window') is not None
409
+ if isinstance(rhs, dict) and rhs.get('kind') == 'stat':
410
+ return bool(rhs.get('stat'))
411
+ v = rule.get('value')
412
+ return v is not None and str(v).strip() != ''
413
+
414
+
415
+ def resolve_rule(rule, today, team_id=None, allowed_pids=None):
416
+ """One measure condition -> the SET OF CUSTOMER IDS satisfying it.
417
+
418
+ `rule` = {colId: <measure key>, window: <window spec>, op: <numeric op>, value: <str|num>},
419
+ or β€” CG-9 β€” the same with `rhs: {kind:'measure', colId, window}` instead of a value, which
420
+ hands off to `resolve_pair`.
421
+
422
+ ⚠ Customers with NO rows in the window are absent from the GROUP BY, so a condition like
423
+ `revenue < 100` would miss them even though 0 < 100 is true. That is the single most
424
+ dangerous thing about this query shape: the missing rows are exactly the lapsed customers a
425
+ "who has gone quiet" filter is FOR. So the caller passes `allowed_pids` (the pool) and the
426
+ zero-group is reconstructed explicitly below rather than left to the SQL.
427
+ """
428
+ if not rule_complete(rule):
429
+ # An absent value would coerce to 0 and resolve as a DIFFERENT question under an
430
+ # authoritative-looking count β€” the same failure class as defaulting a missing window
431
+ # to all time, caught by the same principle: refuse, never guess.
432
+ raise _sem.ModelError(
433
+ 'incomplete measure condition β€” its window, statistic or value is not present; '
434
+ 'refusing to answer a question that has not been fully asked')
435
+ rhs = rule.get('rhs')
436
+ if isinstance(rhs, dict) and rhs.get('kind') == 'measure':
437
+ return resolve_pair(rule, today, team_id, allowed_pids)
438
+ if isinstance(rhs, dict) and rhs.get('kind') == 'stat':
439
+ return resolve_stat(rule, today, team_id, allowed_pids)
440
+ key = rule.get('colId')
441
+ topic_for(key) # raises for anything not admitted
442
+ op = rule.get('op')
443
+ if op not in OPS:
444
+ raise _sem.ModelError(f"{op!r} is not a numeric comparison ({sorted(OPS)})")
445
+
446
+ from harness import filter_sql as _fs
447
+ # ⚠ THE SCALE IS APPLIED HERE, TO THE TYPED VALUE, AND NOWHERE ELSE.
448
+ #
449
+ # Its position relative to `_zero_satisfies` turns out NOT to matter, and the negative
450
+ # control is what established that rather than reasoning: dividing by a POSITIVE constant
451
+ # cannot change how a number compares to zero, so the zero-group is identical either way.
452
+ # (An earlier version of this comment claimed the opposite. It was wrong.) What DOES matter
453
+ # is that a COMPUTED comparand never passes through here β€” see `_scaled`.
454
+ value = _scaled(key, _fs.to_num(rule.get('value')))
455
+
456
+ pool = _pool_of(allowed_pids)
457
+ if is_composite(key):
458
+ # A ratio has no HAVING to push the comparison into: compute the value per customer, then
459
+ # compare in Python β€” the same shape `resolve_pair` uses, and for the same reason.
460
+ con = _sem._store_con()
461
+ try:
462
+ totals = _totals(con, key, rule.get('window'), today, team_id)
463
+ finally:
464
+ con.close()
465
+ # No pool means we can only speak about customers a base has seen. WITH a pool, everyone
466
+ # else scores 0 through the ratio's guard β€” which is what makes the zero-group work for a
467
+ # ratio exactly as it does for a sum ("AOV <= 250" includes those who bought nothing).
468
+ universe = pool if pool is not None else set(totals)
469
+ return {pid for pid in universe if _CMP[op](totals.get(pid, 0.0), value)}
470
+
471
+ frm, id_col, agg, where_sql, params = _query_parts(
472
+ key, rule.get('window'), today, team_id)
473
+ sql = (f"SELECT {id_col} AS pid, {agg} AS v FROM {frm} WHERE {where_sql} "
474
+ f"GROUP BY {id_col} HAVING {agg} {OPS[op]} ?")
475
+
476
+ con = _sem._store_con()
477
+ try:
478
+ matched = {int(r[0]) for r in con.execute(sql, params + [value]).fetchall()
479
+ if r[0] is not None}
480
+ # The zero-group: every customer in scope with NO rows in the window aggregates to 0, and
481
+ # SQL cannot return a group that has no rows. Reconstruct it from the pool rather than
482
+ # pretending those customers do not exist.
483
+ if pool is not None:
484
+ present = {int(r[0]) for r in con.execute(
485
+ f"SELECT DISTINCT {id_col} FROM {frm} WHERE {where_sql}",
486
+ params).fetchall() if r[0] is not None}
487
+ absent = pool - present
488
+ if absent and _zero_satisfies(op, value):
489
+ matched |= absent
490
+ finally:
491
+ con.close()
492
+ if pool is not None:
493
+ matched &= pool
494
+ return matched
495
+
496
+
497
+ def _totals(con, key, window, today, team_id):
498
+ """`{pid: value}` for one (measure, window), COMPOSITES included.
499
+
500
+ For a plain aggregate this is resolve_rule's query minus the HAVING. For a ratio or a derived
501
+ metric it is one query per BASE β€” each on that base's own topic β€” combined per customer by
502
+ `semantic._post_compute`, the same function `store_query` uses for the un-grouped number. So
503
+ AOV per customer is the app's AOV, computed once and defined once.
504
+
505
+ ⚠ The pid keys come from DIFFERENT dims for a cross-topic ratio (`l.order_partner_id` and
506
+ `o.partner_id`). The union is taken rather than the numerator's keys, so a customer either
507
+ side has seen gets a number; the ratio's own `den == 0 -> 0.0` guard covers the rest.
508
+ """
509
+ if is_composite(key):
510
+ bases, _ = _sem._expand_measures([key])
511
+ parts = {b: _totals(con, b, window, today, team_id) for b in bases}
512
+ pids = set()
513
+ for got in parts.values():
514
+ pids |= set(got)
515
+ return {pid: _sem._post_compute({b: parts[b].get(pid, 0.0) for b in bases}, key)
516
+ for pid in pids}
517
+ frm, id_col, agg, where_sql, params = _query_parts(key, window, today, team_id)
518
+ sql = (f"SELECT {id_col} AS pid, {agg} AS v FROM {frm} WHERE {where_sql} GROUP BY {id_col}")
519
+ # `float(r[1])`, not `float(r[1] or 0.0)`: `_query_parts` already wraps the aggregate in
520
+ # COALESCE(..., 0), so a NULL cannot arrive here β€” a second guard would be unreachable, and
521
+ # the negative control proved it (deleting it changed nothing). If one ever DID arrive, a
522
+ # TypeError is the right outcome: it would mean the SQL-level guard had been removed, which
523
+ # is the bug worth hearing about rather than papering over.
524
+ return {int(r[0]): float(r[1])
525
+ for r in con.execute(sql, params).fetchall() if r[0] is not None}
526
+
527
+
528
+ def resolve_pair(rule, today, team_id=None, allowed_pids=None):
529
+ """CG-9 β€” `Where [Sales] [this year] [>] [Sales] [year to date, last year]`.
530
+
531
+ TWO QUERIES, compared in Python. The obvious alternative β€” one query with two conditional
532
+ aggregates and a HAVING comparing them β€” would have to reach inside `_sem._measure_sql` to
533
+ inject a per-side window into an expression that is already a `sum(CASE WHEN … )` for
534
+ filtered measures. Two queries keep ONE definition of each number (the metric's own
535
+ aggregate, unmodified), let the two sides sit on DIFFERENT TOPICS, and make the zero-group
536
+ fall out for free: absent from a side's result means 0 on that side.
537
+
538
+ ⚠ The zero-group is LOUDER here than it is for a one-sided rule. Every customer with no rows
539
+ in EITHER window compares 0 against 0, so `<=` and `>=` match all of them β€” correct, and it
540
+ will look wrong on a narrow window. That is why `eq`/`neq` are not offered at all
541
+ (PAIR_OPS): two float sums are never exactly equal except at zero, where they always are.
542
+ """
543
+ rhs = rule.get('rhs') or {}
544
+ lkey, rkey = rule.get('colId'), rhs.get('colId')
545
+ topic_for(lkey), topic_for(rkey) # raises for anything not admitted
546
+ op = rule.get('op')
547
+ if op not in PAIR_OPS:
548
+ raise _sem.ModelError(
549
+ f"{op!r} cannot compare two measures ({sorted(PAIR_OPS)}). Equality between two sums "
550
+ f"of floats matches nobody and inequality matches everybody; neither answers the "
551
+ f"question, and both look like they work.")
552
+ cmp = _CMP[op]
553
+
554
+ pool = _pool_of(allowed_pids)
555
+ con = _sem._store_con()
556
+ try:
557
+ left = _totals(con, lkey, rule.get('window'), today, team_id)
558
+ right = _totals(con, rkey, rhs.get('window'), today, team_id)
559
+ finally:
560
+ con.close()
561
+ # With no pool we can only speak about customers at least one side has seen β€” the zero-vs-
562
+ # zero group is unknowable without being told who exists. With a pool, it is every member.
563
+ universe = pool if pool is not None else (set(left) | set(right))
564
+ return {pid for pid in universe if cmp(left.get(pid, 0.0), right.get(pid, 0.0))}
565
+
566
+
567
+ def _zero_satisfies(op, value):
568
+ """Does a customer with NO activity in the window (measure = 0) satisfy this condition?"""
569
+ return _CMP[op](0.0, value)
570
+
571
+
572
+ def resolve_values(key, window, today, team_id=None, allowed_pids=None):
573
+ """`{pid: display value}` for one (measure, window) over the caller's pool β€” the COLUMN
574
+ version of `resolve_rule` (owner item 7, 2026-07-27: a formula-measure FIELD).
575
+
576
+ Same `_totals` the filter conditions and statistics use, so the column and the condition
577
+ cannot disagree. Three display decisions live here, not in the client:
578
+
579
+ * ZERO-FILLED over the pool. A customer with no rows in the window has a measure of 0 β€”
580
+ the zero-group convention everywhere else β€” and 0 is a real value the cell shows. Blank
581
+ is reserved for "could not compute" (the caller omits the column entirely on failure).
582
+ * `input_scale` applied OUTWARD. `margin_pct` is a fraction (0-1) internally; the user
583
+ types percentage points into the filter (input divides by 100), so the CELL shows points
584
+ too (value Γ— 100) β€” one boundary, both directions, and `yoy_pct` beside it already reads
585
+ in points.
586
+ * Rounded like every other numeric column: whole numbers for money/counts (the pool's own
587
+ `_round` convention), one decimal for a percent.
588
+
589
+ Raises for anything not admitted / an unresolvable window β€” the caller degrades that column
590
+ to blank rather than to a number nobody computed.
591
+ """
592
+ topic_for(key) # raises for anything not admitted
593
+ # The window must RESOLVE before any query runs β€” same refusal as `_query_parts`, checked
594
+ # here too because a composite's `_totals` path resolves per base and would otherwise raise
595
+ # deeper with a less useful message.
596
+ if _win.resolve(window, today) is None:
597
+ raise _sem.ModelError(
598
+ f"unresolvable window {window!r} for the {key!r} column β€” refusing rather than "
599
+ f"defaulting to all time")
600
+ pool = _pool_of(allowed_pids)
601
+ con = _sem._store_con()
602
+ try:
603
+ totals = _totals(con, key, window, today, team_id)
604
+ finally:
605
+ con.close()
606
+ scale = ADMITTED.get(key, {}).get('input_scale') or 1.0
607
+ mtype = _sem._FORMAT_TYPE.get(_sem.metrics().get(key, {}).get('format'), 'currency')
608
+ nd = 1 if mtype == 'pct' else None # money/counts whole, percents one decimal
609
+ universe = pool if pool is not None else set(totals)
610
+ # round(x, None) returns an int β€” exactly the pool's `_round` convention for money/counts.
611
+ return {int(p): round(float(totals.get(p, 0.0)) * scale, nd) for p in universe}
612
+
613
+
614
+ #: The series channel's calendar buckets. `date_trunc` names in DuckDB; 'week' truncates to
615
+ #: Monday, matching the date-window vocabulary's week convention.
616
+ SERIES_BUCKETS = frozenset({'week', 'month', 'quarter', 'year'})
617
+
618
+
619
+ def resolve_series(key, bucket, date_from, date_to, today, team_id=None, allowed_pids=None):
620
+ """POOLED value of one measure per calendar bucket β€” the time-series channel (C-TS,
621
+ wave 2026-08-02). Returns `{'values': {bucket_start_iso: value}, 'agg': kind}`.
622
+
623
+ ONE grouped query per measure: the aggregate expression is `semantic._measure_sql`'s own
624
+ (no second definition of a number), evaluated over the CALLER'S POOL per `date_trunc`
625
+ bucket. That is the pooled aggregate β€” a non-additive measure (an average) is computed at
626
+ pool grain, never averaged over per-customer answers (the additivity law).
627
+
628
+ Composites (ratio/derived) are REFUSED: they have no single aggregate expression, and
629
+ combining their bases per bucket is a correctness project, not a default β€” the caller
630
+ degrades that field honestly rather than showing a number nobody computed.
631
+
632
+ Scale and rounding mirror `resolve_values` exactly, so a bucket cell and the grid's own
633
+ measure column can never disagree on units. Buckets with no rows are ABSENT from the
634
+ answer β€” zero-filling is the CALLER'S decision because it depends on the agg kind: an
635
+ empty month of a SUM is a real 0, an empty month of an AVERAGE is unanswerable.
636
+ """
637
+ if bucket not in SERIES_BUCKETS:
638
+ raise _sem.ModelError(
639
+ f"unknown series bucket {bucket!r} - offered: week, month, quarter, year")
640
+ topic_for(key) # raises for anything not admitted
641
+ if is_composite(key):
642
+ raise _sem.ModelError(
643
+ f"{key!r} is a ratio/derived measure - the series channel computes "
644
+ f"single-aggregate measures only")
645
+ window = {'kind': 'custom', 'from': str(date_from), 'to': str(date_to)}
646
+ frm, id_col, agg, where, params = _query_parts(key, window, today, team_id)
647
+ _, _, s = _entity_sql(_base_topic(key))
648
+ meta = _sem.metrics().get(key, {})
649
+ agg_kind = str(meta.get('agg') or 'sum')
650
+ pool = _pool_of(allowed_pids)
651
+ sql_params = list(params)
652
+ pool_sql = ''
653
+ if pool is not None:
654
+ if not pool:
655
+ return {'values': {}, 'agg': agg_kind}
656
+ marks = ','.join('?' for _ in pool)
657
+ pool_sql = f" AND {id_col} IN ({marks})"
658
+ sql_params.extend(int(p) for p in sorted(pool))
659
+ sql = (f"SELECT CAST(date_trunc('{bucket}', CAST({s['date_col']} AS TIMESTAMP)) AS DATE)"
660
+ f" AS b, {agg} AS v FROM {frm} WHERE {where}{pool_sql} GROUP BY b")
661
+ con = _sem._store_con()
662
+ try:
663
+ rows = con.execute(sql, sql_params).fetchall()
664
+ finally:
665
+ con.close()
666
+ scale = ADMITTED.get(key, {}).get('input_scale') or 1.0
667
+ mtype = _sem._FORMAT_TYPE.get(meta.get('format'), 'currency')
668
+ nd = 1 if mtype == 'pct' else None
669
+ values = {str(b)[:10]: round(float(v) * scale, nd) for b, v in rows if b is not None}
670
+ return {'values': values, 'agg': agg_kind}
671
+
672
+
673
+ def resolve_series_slid(key, window_spec, buckets, today, team_id=None, allowed_pids=None):
674
+ """AS-OF series (C-TSWIN, wave 14 β€” [[loopable-wave14-split]] R1): the measure's OWN
675
+ window re-resolved per bucket with `today := min(bucket end, real today)` β€” so `ytd`
676
+ becomes cumulative from Jan 1, `last_90_days` becomes trailing-90, `all_time` cumulative
677
+ ever. Returns `{'values': {bucket_start_iso: value}, 'agg': kind}`.
678
+
679
+ `buckets` is `[(start_iso, end_iso), …]`. One aggregate query per bucket rather than one
680
+ grouped query β€” deliberately: every part (scope, aggregate expression, BU wall, scale,
681
+ rounding) is `_query_parts`' own, so **bucket N is exactly what the grid's measure column
682
+ would show if the tenant's today were bucket N's end** β€” the two surfaces cannot disagree
683
+ by construction, which is the property the pooled/date_trunc shape could not give a
684
+ sliding window.
685
+
686
+ Absence semantics (the caller's fill law, same as `resolve_series`): a bucket whose
687
+ window holds NO ROWS is ABSENT from the answer (`COALESCE` would have manufactured a 0
688
+ for an empty AVERAGE, so the query carries a COUNT(*) to tell the two apart); a bucket
689
+ whose START is after `today` is absent too β€” an unstarted period's YTD is not 0, it is
690
+ unanswered. `custom` (fixed-range) windows cannot slide and are REFUSED β€” the caller
691
+ drops them as `window_fixed` before ever calling here.
692
+ """
693
+ topic_for(key) # raises for anything not admitted
694
+ if is_composite(key):
695
+ raise _sem.ModelError(
696
+ f"{key!r} is a ratio/derived measure - the series channel computes "
697
+ f"single-aggregate measures only")
698
+ spec = _win.normalize(window_spec)
699
+ if spec is None or spec.get('kind') == 'custom':
700
+ raise _sem.ModelError(
701
+ f"{key!r}: a fixed-range window cannot slide across buckets (window_fixed)")
702
+ meta = _sem.metrics().get(key, {})
703
+ agg_kind = str(meta.get('agg') or 'sum')
704
+ pool = _pool_of(allowed_pids)
705
+ if pool is not None and not pool:
706
+ return {'values': {}, 'agg': agg_kind}
707
+ scale = ADMITTED.get(key, {}).get('input_scale') or 1.0
708
+ mtype = _sem._FORMAT_TYPE.get(meta.get('format'), 'currency')
709
+ nd = 1 if mtype == 'pct' else None
710
+ today_s = str(today)[:10]
711
+ values = {}
712
+ con = _sem._store_con()
713
+ try:
714
+ for start, end in buckets:
715
+ start_s, end_s = str(start)[:10], str(end)[:10]
716
+ if start_s > today_s:
717
+ continue # unstarted period -> unanswered, never 0
718
+ eff = min(end_s, today_s)
719
+ frm, id_col, agg, where, params = _query_parts(key, spec, eff, team_id)
720
+ sql_params = list(params)
721
+ pool_sql = ''
722
+ if pool is not None:
723
+ marks = ','.join('?' for _ in pool)
724
+ pool_sql = f" AND {id_col} IN ({marks})"
725
+ sql_params.extend(int(p) for p in sorted(pool))
726
+ sql = f"SELECT {agg} AS v, COUNT(*) AS n FROM {frm} WHERE {where}{pool_sql}"
727
+ v, nrows = con.execute(sql, sql_params).fetchone()
728
+ if not nrows:
729
+ continue # empty window -> the caller's per-agg fill law
730
+ values[start_s] = round(float(v) * scale, nd)
731
+ finally:
732
+ con.close()
733
+ return {'values': values, 'agg': agg_kind}
734
+
735
+
736
+ def resolve_tree(rules, today, team_id=None, allowed_pids=None):
737
+ """{rule_id: sorted pids} for every measure condition in a tree.
738
+
739
+ Keyed by the rule's client-generated `id` so the browser can map an answer back to the
740
+ condition that asked it β€” positional matching would silently re-associate every answer the
741
+ moment a user deleted a condition.
742
+ """
743
+ out = {}
744
+ for r in rules or []:
745
+ rid = str(r.get('id') or '')
746
+ if not rid:
747
+ raise _sem.ModelError('a measure condition needs a stable id to carry its answer back')
748
+ out[rid] = sorted(resolve_rule(r, today, team_id, allowed_pids))
749
+ return out
750
+
751
+
752
+ def signature(rule):
753
+ """The identity of the QUESTION a rule asks, ignoring which condition row asked it.
754
+
755
+ Two conditions with the same measure, window, operator and value have the same answer, so
756
+ they cost one query between them β€” and a saved view re-rendered unchanged costs none, since
757
+ the caller can memoise on this. `id` is deliberately absent: it names the asker, not the
758
+ question.
759
+
760
+ ⚠ The RHS is part of the question. Without it, `Sales[ytd] > Sales[last year]` and
761
+ `Sales[ytd] > Sales[ytd_last_year]` would share one memo entry β€” same measure, same op, same
762
+ (empty) value β€” and the second would be served the first one's answer. Silent, and wrong in
763
+ exactly the comparison this was built for.
764
+ """
765
+ w = _win.normalize(rule.get('window'))
766
+ rhs = rule.get('rhs')
767
+ rsig = None
768
+ if isinstance(rhs, dict):
769
+ rw = _win.normalize(rhs.get('window'))
770
+ # `stat` rides the same tuple: `> the median` and `> the 75th percentile` are different
771
+ # questions with the same measure, window, op and (empty) value.
772
+ rsig = (rhs.get('kind'), rhs.get('colId'), rhs.get('stat'),
773
+ None if rw is None else tuple(sorted(rw.items())))
774
+ return (rule.get('colId'), rule.get('op'), str(rule.get('value') or ''),
775
+ None if w is None else tuple(sorted(w.items())), rsig)
776
+
777
+
778
+ def collect(nodes):
779
+ """Every MEASURE leaf in a filter tree, flattened (groups contribute their children)."""
780
+ found = []
781
+ for n in nodes or []:
782
+ if not isinstance(n, dict):
783
+ continue
784
+ if isinstance(n.get('children'), list):
785
+ found.extend(collect(n['children']))
786
+ elif n.get('colId') in ADMITTED:
787
+ found.append(n)
788
+ return found
platform/harness/pg/schema.sql ADDED
@@ -0,0 +1,180 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ -- harness/pg/schema.sql β€” the control DB + per-tenant schema (X4 / EXIT-2b, 2026-07-30).
2
+ --
3
+ -- WHY POSTGRES AT ALL (C1c, and it is not a preference). The HF Dataset store is a FILE HOST:
4
+ -- no transactions, no PITR, no row-level anything, and `store.get()` is cache-first β€” so a write
5
+ -- from another process is invisible to a running app until restart. That was observed LIVE in
6
+ -- wave 11, and it is the whole reason the store leaves the product. Application/control data
7
+ -- (users, sessions, tenants, audit, saved views, overlay fields, cohorts, folders, prefs,
8
+ -- billing mirror) is transactional and needs joins; the ANALYTICAL mirror stays in DuckDB,
9
+ -- one file per tenant, because file-per-tenant IS the isolation model there.
10
+ --
11
+ -- β›” NOT YET RUN AGAINST A REAL SERVER. Owner blocker **B-3** (choose + provision managed
12
+ -- Postgres, US-region, co-located with Ashburn). This file and `core/store_pg.py` are the
13
+ -- sequenced-around work; the integration test is written but PENDING and is NOT reported as
14
+ -- passing. Nothing here is claimed to be verified beyond "it parses and the shapes match the
15
+ -- interface `core/store.py` already exposes".
16
+ --
17
+ -- ⚠ CO-LOCATE (C1b). An Ashburn app with a European database adds ~90ms to every query and undoes
18
+ -- the reason the market pivot happened.
19
+ --
20
+ -- psql "$DATABASE_URL" -v ON_ERROR_STOP=1 -f harness/pg/schema.sql
21
+ -- psql "$DATABASE_URL" -v ON_ERROR_STOP=1 -v slug=royal_imports -f harness/pg/schema.sql
22
+ -- (the second form additionally provisions one tenant schema β€” see the bottom block)
23
+
24
+ BEGIN;
25
+
26
+ -- ─────────────────────────────────────────────────────────────────────────────────────────────
27
+ -- CONTROL: what the platform knows about itself. One row per tenant, one per account.
28
+ -- ─────────────────────────────────────────────────────────────────────────────────────────────
29
+ CREATE SCHEMA IF NOT EXISTS control;
30
+
31
+ CREATE TABLE IF NOT EXISTS control.tenants (
32
+ -- The SLUG is the identity everywhere: the session cookie's `t` claim, the store namespace,
33
+ -- the DuckDB filename and the schema name are all derived from it. Constrained to what is
34
+ -- safe in all four positions, which is stricter than Postgres alone would require.
35
+ slug text PRIMARY KEY CHECK (slug ~ '^[a-z0-9][a-z0-9-]{0,58}[a-z0-9]$'),
36
+ name text NOT NULL,
37
+ -- `harness/tenants.py`'s literal, as data. Its own docstring says this becomes "a tenants
38
+ -- store (DB / HF dataset)"; this is that store. jsonb rather than columns because the shape
39
+ -- is per-connector and adding a source must not be a migration.
40
+ config jsonb NOT NULL DEFAULT '{}'::jsonb,
41
+ -- Connector CREDENTIALS never live here in plaintext. The Odoo key is the customer's,
42
+ -- stored encrypted at rest by the secrets layer (C2: SOPS+age per tenant) and shown once.
43
+ -- This column holds only a REFERENCE to where the secret lives.
44
+ secret_ref text,
45
+ active boolean NOT NULL DEFAULT true,
46
+ created_at timestamptz NOT NULL DEFAULT now(),
47
+ updated_at timestamptz NOT NULL DEFAULT now()
48
+ );
49
+
50
+ CREATE TABLE IF NOT EXISTS control.users (
51
+ -- Mirrors `core/users.py`'s record exactly, so the HF→PG cutover (C-4) is a copy, not a
52
+ -- redesign: salt + PBKDF2-HMAC-SHA256 hash (200k), never a plaintext password.
53
+ tenant_slug text NOT NULL REFERENCES control.tenants(slug) ON DELETE CASCADE,
54
+ username text NOT NULL CHECK (username = lower(username)),
55
+ salt text NOT NULL,
56
+ hash text NOT NULL,
57
+ name text NOT NULL DEFAULT '',
58
+ role text NOT NULL DEFAULT 'user',
59
+ -- 'all' or a JSON array of Odoo team ids ([5]=Fisch, [6]=Royal) β€” the BU isolation input.
60
+ bus jsonb NOT NULL DEFAULT '"all"'::jsonb,
61
+ -- 'all' or a JSON array of registry keys β€” the module grant `may_open` reads.
62
+ modules jsonb NOT NULL DEFAULT '"all"'::jsonb,
63
+ agent text, -- own-book scope (res.partner.agent_ids name)
64
+ email text,
65
+ active boolean NOT NULL DEFAULT true,
66
+ -- X3's stateless-session revocation handle. A signed cookie cannot be deleted server-side,
67
+ -- so it carries the epoch it was minted under and every verification re-reads THIS number;
68
+ -- a password change or a deactivation bumps it and every outstanding cookie dies at once.
69
+ epoch integer NOT NULL DEFAULT 0,
70
+ created_at timestamptz NOT NULL DEFAULT now(),
71
+ updated_at timestamptz NOT NULL DEFAULT now(),
72
+ PRIMARY KEY (tenant_slug, username)
73
+ );
74
+
75
+ -- D2's SESSIONS MIRROR β€” an AUDIT view, deliberately NOT the source of truth (X3 keeps sessions
76
+ -- stateless so the API process holds no per-tenant state). It answers "who is signed in, from
77
+ -- where, since when", and it is what makes per-DEVICE revocation possible later. Arrives with
78
+ -- C-2; the table is defined now so the cutover does not need a second migration.
79
+ CREATE TABLE IF NOT EXISTS control.sessions (
80
+ id bigserial PRIMARY KEY,
81
+ tenant_slug text NOT NULL,
82
+ username text NOT NULL,
83
+ epoch integer NOT NULL,
84
+ issued_at timestamptz NOT NULL DEFAULT now(),
85
+ last_seen_at timestamptz NOT NULL DEFAULT now(),
86
+ absolute_expiry timestamptz NOT NULL,
87
+ -- Truncated/derived client facts only. A full UA string plus an exact IP is more personal
88
+ -- data than an audit trail needs, and the DPA is easier to keep when the row is smaller.
89
+ ip_prefix text,
90
+ user_agent text,
91
+ revoked_at timestamptz,
92
+ FOREIGN KEY (tenant_slug, username)
93
+ REFERENCES control.users(tenant_slug, username) ON DELETE CASCADE
94
+ );
95
+ CREATE INDEX IF NOT EXISTS sessions_live_idx
96
+ ON control.sessions (tenant_slug, username) WHERE revoked_at IS NULL;
97
+
98
+ CREATE TABLE IF NOT EXISTS control.audit (
99
+ id bigserial PRIMARY KEY,
100
+ at timestamptz NOT NULL DEFAULT now(),
101
+ tenant_slug text,
102
+ username text,
103
+ action text NOT NULL, -- 'login' | 'store.put' | 'user.deactivate' | …
104
+ target text,
105
+ detail jsonb NOT NULL DEFAULT '{}'::jsonb
106
+ );
107
+ CREATE INDEX IF NOT EXISTS audit_at_idx ON control.audit (at DESC);
108
+ CREATE INDEX IF NOT EXISTS audit_tenant_idx ON control.audit (tenant_slug, at DESC);
109
+
110
+ -- ─────────────────────────────────────────────────────────────────────────────────────────────
111
+ -- PER-TENANT: one schema `t_<slug>` per tenant, holding what the HF store holds today.
112
+ --
113
+ -- A SCHEMA PER TENANT, not a tenant_id column with RLS. Both are defensible; this one is chosen
114
+ -- because the isolation is then structural β€” a query that forgets its tenant predicate cannot
115
+ -- silently return another tenant's rows, it names a table that is not in the search_path. RLS
116
+ -- makes the same guarantee only while every policy is correct on every table, forever, and
117
+ -- [[aios-permissioning]] already records that a forgotten predicate is the failure mode we keep
118
+ -- writing gates against. The cost is DDL per tenant, which is one function call below.
119
+ -- ─────────────────────────────────────────────────────────────────────────────────────────────
120
+ CREATE OR REPLACE FUNCTION control.provision_tenant_schema(p_slug text)
121
+ RETURNS text
122
+ LANGUAGE plpgsql
123
+ AS $$
124
+ DECLARE
125
+ -- ⚠ `-` IS NOT LEGAL UNQUOTED, and the slug carries them (`royal-imports`). Normalising to
126
+ -- `_` here rather than quoting everywhere keeps every generated identifier plain, and
127
+ -- `format('%I')` is what makes the interpolation injection-safe regardless.
128
+ v_schema text := 't_' || replace(lower(p_slug), '-', '_');
129
+ BEGIN
130
+ IF p_slug IS NULL OR p_slug !~ '^[a-z0-9][a-z0-9-]{0,58}[a-z0-9]$' THEN
131
+ RAISE EXCEPTION 'refusing to provision an unsafe tenant slug: %', p_slug;
132
+ END IF;
133
+
134
+ EXECUTE format('CREATE SCHEMA IF NOT EXISTS %I', v_schema);
135
+
136
+ -- store_kv β€” the JSON-per-key store `core/store.py` exposes, one row per key. `rev` is what
137
+ -- the file host never had: an optimistic-concurrency counter, so a read-modify-write can
138
+ -- detect that somebody else wrote between its read and its write instead of overwriting
139
+ -- them. That is the concrete thing "no transactions" cost us.
140
+ EXECUTE format($f$
141
+ CREATE TABLE IF NOT EXISTS %I.store_kv (
142
+ key text PRIMARY KEY,
143
+ value jsonb NOT NULL,
144
+ updated_at timestamptz NOT NULL DEFAULT now(),
145
+ rev bigint NOT NULL DEFAULT 1
146
+ )$f$, v_schema);
147
+
148
+ -- store_blobs β€” `upload_bytes`/`download_bytes`/`delete_path`. Customer documents live here
149
+ -- rather than base64'd into a JSON value, for the reason `core.store.upload_bytes` gives:
150
+ -- the workspace blob is READ ON EVERY RENDER, so a few MB of base64 inside it would be paid
151
+ -- on every keystroke by every user of that table.
152
+ EXECUTE format($f$
153
+ CREATE TABLE IF NOT EXISTS %I.store_blobs (
154
+ path text PRIMARY KEY,
155
+ bytes bytea NOT NULL,
156
+ message text,
157
+ updated_at timestamptz NOT NULL DEFAULT now()
158
+ )$f$, v_schema);
159
+
160
+ EXECUTE format('CREATE INDEX IF NOT EXISTS %I ON %I.store_kv (updated_at DESC)',
161
+ v_schema || '_kv_updated_idx', v_schema);
162
+ RETURN v_schema;
163
+ END;
164
+ $$;
165
+
166
+ -- Tenant #0, always. Its slug is the one the session cookie already carries.
167
+ INSERT INTO control.tenants (slug, name)
168
+ VALUES ('royal-imports', 'Royal Imports')
169
+ ON CONFLICT (slug) DO NOTHING;
170
+ SELECT control.provision_tenant_schema('royal-imports');
171
+
172
+ -- Optional: `psql -v slug=<a-slug>` provisions one more tenant schema in the same transaction.
173
+ -- `:'slug'` is unset in the plain invocation, so this block is skipped there.
174
+ \if :{?slug}
175
+ INSERT INTO control.tenants (slug, name) VALUES (:'slug', :'slug')
176
+ ON CONFLICT (slug) DO NOTHING;
177
+ SELECT control.provision_tenant_schema(:'slug');
178
+ \endif
179
+
180
+ COMMIT;
platform/harness/routines.py ADDED
@@ -0,0 +1,259 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """harness/routines.py β€” OM-5 v0: scheduled reports + KPI alerts over the semantic layer.
2
+
3
+ Two of the four productization user jobs (Scheduled Reporting, KPI monitoring). Definitions live
4
+ in data/store/routines.json (tenant data, git-ignored); evaluation happens ON READ against the
5
+ tenant store through the same governed queries as everything else β€” an alert is a metric key +
6
+ a mode, never a hand-rolled query.
7
+
8
+ v0 boundaries (deliberate):
9
+ - Channel is 'in-app' only. Email/Slack delivery is OUTBOUND and ships only with owner approval
10
+ (the Odoo mail client is whitelisted for Collections statements ONLY β€” nothing else sends).
11
+ - No background scheduler: reports record cadence + show due-state in-app; alerts evaluate live
12
+ whenever the Routines page (or a tool) asks. The enterprise scheduler lands with OM-5 delivery.
13
+
14
+ Alert modes:
15
+ - 'xmr' Wheeler individuals chart on complete monthly values (promoted from
16
+ modules/warehouse._xmr, generalized: no 0/100 clamps β€” metrics aren't percentages).
17
+ Three detection rules only (more = alarm fatigue, per the warehouse brief).
18
+ - 'threshold' last complete month above/below a fixed value.
19
+ """
20
+ import json
21
+ import time
22
+ from datetime import date, timedelta
23
+
24
+ import harness.semantic as SEM
25
+ from harness.tools import VIEWS_PATH
26
+
27
+ ROUTINES_PATH = VIEWS_PATH.parent / "routines.json"
28
+
29
+ CADENCES = ("daily", "weekly", "monthly")
30
+ CHANNELS = ("in-app",) # email/slack gate on owner approval (outbound send)
31
+ ALERT_MODES = ("xmr", "threshold")
32
+ XMR_MONTHS = 18 # complete months fed to the individuals chart
33
+
34
+
35
+ def load():
36
+ if ROUTINES_PATH.exists():
37
+ return json.loads(ROUTINES_PATH.read_text(encoding="utf-8"))
38
+ return {"reports": {}, "alerts": {}}
39
+
40
+
41
+ def _save(d):
42
+ ROUTINES_PATH.parent.mkdir(parents=True, exist_ok=True)
43
+ ROUTINES_PATH.write_text(json.dumps(d, indent=1), encoding="utf-8")
44
+
45
+
46
+ # ------------------------------------------------------------------ scheduled reports
47
+
48
+ def schedule_report(view, cadence, channel="in-app"):
49
+ import harness.views as V
50
+ if view not in V.views() and view not in V.dashboards():
51
+ raise SEM.ModelError(f"unknown view/dashboard {view!r} β€” save_view/compose_dashboard first")
52
+ if cadence not in CADENCES:
53
+ raise SEM.ModelError(f"cadence must be one of {CADENCES}")
54
+ if channel not in CHANNELS:
55
+ raise SEM.ModelError(f"channel {channel!r} not available yet β€” v0 delivers in-app "
56
+ "(email/Slack ship with the delivery gate); use channel='in-app'")
57
+ d = load()
58
+ d["reports"][view] = {"cadence": cadence, "channel": channel,
59
+ "created_at": time.strftime("%Y-%m-%d %H:%M")}
60
+ _save(d)
61
+ return {"scheduled": view, "cadence": cadence, "channel": channel,
62
+ "note": "runs in-app (Workflows > Routines); external delivery ships with OM-5 email approval"}
63
+
64
+
65
+ def delete_report(view):
66
+ d = load()
67
+ d["reports"].pop(view, None)
68
+ _save(d)
69
+
70
+
71
+ _CADENCE_DAYS = {"daily": 1, "weekly": 7, "monthly": 30}
72
+
73
+
74
+ def mark_viewed(view):
75
+ """Called when the scheduled view/dashboard is actually opened β€” resets its due-ness."""
76
+ d = load()
77
+ r = d.get("reports", {}).get(view)
78
+ if r is not None:
79
+ r["last_viewed"] = time.strftime("%Y-%m-%d %H:%M")
80
+ _save(d)
81
+
82
+
83
+ def due_reports():
84
+ """Scheduled reports whose cadence has elapsed since last viewed (v0 in-app delivery via
85
+ the Routines page). Never-viewed ages from created_at."""
86
+ from datetime import datetime
87
+ out = []
88
+ for name, r in load().get("reports", {}).items():
89
+ days = _CADENCE_DAYS.get(r.get("cadence"), 7)
90
+ anchor = r.get("last_viewed") or r.get("created_at") or ""
91
+ try:
92
+ age = (datetime.now() - datetime.strptime(anchor, "%Y-%m-%d %H:%M")).days
93
+ except Exception:
94
+ age = days
95
+ if age >= days:
96
+ out.append({"name": name, "cadence": r.get("cadence"),
97
+ "last_viewed": r.get("last_viewed"), "age_days": age})
98
+ return out
99
+
100
+
101
+ # ------------------------------------------------------------------ insight routines (OM-5)
102
+ # A saved PROMPT on a cadence: the Analyst runs it in the background and the narrative (plus its
103
+ # governed artifacts) lands in-app on the Routines page. The narrative is a deliberate
104
+ # point-in-time SNAPSHOT (it narrates a moment); its charts still carry re-runnable queries.
105
+
106
+ def schedule_insight(name, prompt, cadence):
107
+ if cadence not in CADENCES:
108
+ raise SEM.ModelError(f"cadence must be one of {CADENCES}")
109
+ if not (prompt or "").strip():
110
+ raise SEM.ModelError("an insight needs a prompt β€” what should the Analyst look at?")
111
+ d = load()
112
+ d.setdefault("insights", {})[name] = {"prompt": prompt.strip(), "cadence": cadence,
113
+ "created_at": time.strftime("%Y-%m-%d %H:%M")}
114
+ _save(d)
115
+ return {"insight": name, "cadence": cadence,
116
+ "note": "runs in the background when due; the narrative lands on "
117
+ "Workflows > Routines"}
118
+
119
+
120
+ def delete_insight(name):
121
+ d = load()
122
+ (d.get("insights") or {}).pop(name, None)
123
+ _save(d)
124
+
125
+
126
+ def due_insights():
127
+ from datetime import datetime
128
+ out = []
129
+ for name, i in (load().get("insights") or {}).items():
130
+ days = _CADENCE_DAYS.get(i.get("cadence"), 7)
131
+ anchor = i.get("last_run_at") or ""
132
+ try:
133
+ age = (datetime.now() - datetime.strptime(anchor, "%Y-%m-%d %H:%M")).days
134
+ except Exception:
135
+ age = days # never run -> due now
136
+ if age >= days:
137
+ out.append(name)
138
+ return out
139
+
140
+
141
+ def run_insight(name):
142
+ """Execute one insight through the Analyst and persist the narrative + artifacts."""
143
+ d = load()
144
+ ins = (d.get("insights") or {}).get(name)
145
+ if not ins:
146
+ raise SEM.ModelError(f"unknown insight {name!r}")
147
+ import harness.analyst as A
148
+ r = A.ask(ins["prompt"])
149
+ now = time.strftime("%Y-%m-%d %H:%M")
150
+ ins["last_run_at"] = now
151
+ ins["last"] = {"at": now, "answer": r["answer"], "artifacts": r.get("artifacts") or [],
152
+ "tokens": (r.get("usage") or {}).get("prompt_tokens", 0)
153
+ + (r.get("usage") or {}).get("completion_tokens", 0)}
154
+ _save(d)
155
+ return ins["last"]
156
+
157
+
158
+ # ------------------------------------------------------------------ KPI alerts
159
+
160
+ def create_alert(metric, mode="xmr", team_id=None, threshold=None, direction=None):
161
+ m = SEM.metrics().get(metric)
162
+ if not m:
163
+ raise SEM.ModelError(f"unknown metric {metric!r} (use list_topics)")
164
+ if mode not in ALERT_MODES:
165
+ raise SEM.ModelError(f"mode must be one of {ALERT_MODES}")
166
+ if mode == "threshold":
167
+ if threshold is None or direction not in ("above", "below"):
168
+ raise SEM.ModelError("threshold mode needs threshold=<number> and "
169
+ "direction='above'|'below'")
170
+ d = load()
171
+ key = metric + (f"_team{team_id}" if team_id else "")
172
+ d["alerts"][key] = {"metric": metric, "mode": mode, "team_id": team_id,
173
+ "threshold": threshold, "direction": direction,
174
+ "created_at": time.strftime("%Y-%m-%d %H:%M")}
175
+ _save(d)
176
+ return {"alert": key, **evaluate_alert(d["alerts"][key])}
177
+
178
+
179
+ def delete_alert(key):
180
+ d = load()
181
+ d["alerts"].pop(key, None)
182
+ _save(d)
183
+
184
+
185
+ def xmr(series):
186
+ """Wheeler XmR individuals limits + the three detection rules β€” the model-level primitive
187
+ (generalized from modules/warehouse._xmr; no percentage clamps). series=[(label, value)]."""
188
+ vals = [v for _, v in series]
189
+ if len(vals) < 8:
190
+ return {"center": None, "unpl": None, "lnpl": None,
191
+ "points": [{"label": l, "value": v, "signal": ""} for l, v in series]}
192
+ center = sum(vals) / len(vals)
193
+ mrs = [abs(vals[i] - vals[i - 1]) for i in range(1, len(vals))]
194
+ mr_bar = sum(mrs) / len(mrs) if mrs else 0.0
195
+ unpl = center + 2.66 * mr_bar
196
+ lnpl = center - 2.66 * mr_bar
197
+ pts = []
198
+ for i, (l, v) in enumerate(series):
199
+ sig = ""
200
+ if v > unpl or v < lnpl:
201
+ sig = "outside limits"
202
+ if not sig and i >= 7 and all((x - center) > 0 for _, x in series[i - 7:i + 1]):
203
+ sig = "sustained shift (8 above center)"
204
+ if not sig and i >= 7 and all((x - center) < 0 for _, x in series[i - 7:i + 1]):
205
+ sig = "sustained shift (8 below center)"
206
+ if not sig and i >= 3:
207
+ last4 = [x for _, x in series[i - 3:i + 1]]
208
+ near = sum(1 for x in last4
209
+ if abs(x - center) > 0.5 * ((unpl - center) if x >= center
210
+ else (center - lnpl)))
211
+ if near >= 3:
212
+ sig = "moderate shift emerging"
213
+ pts.append({"label": l, "value": v, "signal": sig})
214
+ return {"center": center, "unpl": unpl, "lnpl": lnpl, "points": pts}
215
+
216
+
217
+ def _month_window(n=XMR_MONTHS):
218
+ """Last n COMPLETE months (the in-progress month would false-signal every alert)."""
219
+ first_cur = date.today().replace(day=1)
220
+ y, mo = first_cur.year, first_cur.month
221
+ for _ in range(n):
222
+ mo -= 1
223
+ if mo == 0:
224
+ y, mo = y - 1, 12
225
+ start = date(y, mo, 1)
226
+ return start.isoformat(), (first_cur - timedelta(days=1)).isoformat()
227
+
228
+
229
+ def evaluate_alert(a):
230
+ """Live evaluation: monthly series of the metric (complete months only) -> mode verdict."""
231
+ metric = a["metric"]
232
+ topic = SEM.metrics()[metric]["topic"]
233
+ date_from, date_to = _month_window()
234
+ res = SEM.store_query(topic, [metric], grain="month", date_from=date_from, date_to=date_to,
235
+ team_id=a.get("team_id"))
236
+ series = [(r["period"][:7], r.get(metric) or 0) for r in res["rows"]]
237
+ out = {"metric": metric, "mode": a["mode"], "team_id": a.get("team_id"),
238
+ "window": f"{date_from}..{date_to}", "series": series}
239
+ if not series:
240
+ return {**out, "status": "no data", "fired": False}
241
+ last_label, last_val = series[-1]
242
+ if a["mode"] == "threshold":
243
+ thr, direc = a.get("threshold"), a.get("direction")
244
+ fired = (last_val > thr) if direc == "above" else (last_val < thr)
245
+ return {**out, "last": {"label": last_label, "value": last_val},
246
+ "threshold": thr, "direction": direc,
247
+ "status": (f"FIRED β€” {last_label} is {direc} {thr:,.0f}" if fired else "OK"),
248
+ "fired": bool(fired)}
249
+ lim = xmr(series)
250
+ sig = lim["points"][-1]["signal"] if lim["points"] else ""
251
+ return {**out, "last": {"label": last_label, "value": last_val},
252
+ "center": lim["center"], "unpl": lim["unpl"], "lnpl": lim["lnpl"],
253
+ "points": lim["points"],
254
+ "status": (f"SIGNAL β€” {sig} ({last_label})" if sig else "OK"),
255
+ "fired": bool(sig)}
256
+
257
+
258
+ def evaluate_all():
259
+ return {k: evaluate_alert(a) for k, a in load().get("alerts", {}).items()}
platform/harness/runtime.py ADDED
@@ -0,0 +1,248 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """harness/runtime.py β€” X7: per-REQUEST tenant resolution for the shared stateless API (EXIT-4a).
2
+
3
+ THE COST ARGUMENT, in one rule: *no per-tenant state may be resident in the app process.* Today
4
+ each tenant carries its own interpreter, its own never-evicted pool, its own 9 prewarmed bundles
5
+ and its own DuckDB β€” EXIT-0 measured a **~0.6 GB commit floor per tenant**, duplicated exactly
6
+ (tenant B came within 2% of tenant A; nothing is shared). The target is ~30–80 MB. That is only
7
+ reachable if the API resolves the tenant from the REQUEST and holds nothing tenant-shaped between
8
+ requests except a bounded cache it is willing to throw away.
9
+
10
+ WHAT THIS REPLACES. `ui/cache.py`'s `st.cache_resource` singleton β€” one process, one tenant,
11
+ caches that are never evicted by design. That is correct for Streamlit and fatal for a shared
12
+ process. The Streamlit side keeps its singleton untouched this wave (two processes, two caches,
13
+ accepted for the strangler period; `ui/cache.py` dies with app.py at EXIT-6).
14
+
15
+ THREE RULES THIS FILE OBEYS, each of which is a way the old shape leaked:
16
+ 1. **No module-level per-tenant globals.** Every tenant-shaped value hangs off a TenantRuntime
17
+ reached through `get_runtime(slug)`; nothing is stashed at import time. A module global is
18
+ exactly what makes "which tenant is this process serving?" unanswerable.
19
+ 2. **LRU-BOUNDED.** `_MAX_RUNTIMES` runtimes live at once; the least recently used is evicted.
20
+ An unbounded registry cache is a memory leak with a tenant-count-shaped growth curve β€” the
21
+ thing this whole program exists to remove.
22
+ 3. **The store handle is tenant-bound**, so a write cannot land in another tenant's namespace
23
+ even if a caller passes the wrong key downstream (EXIT-4b proof #3).
24
+
25
+ ⚠ THE DUCKDB CAVEAT, STATED NOT HIDDEN. `harness.datastore` holds ONE process-wide connection
26
+ (the fix for the 2026-07-16 "connection error unless I refresh" bug), so one PROCESS can serve
27
+ one analytical store at a time. This module therefore hands out `datastore_path` β€” the file a
28
+ tenant's analytical reads belong to β€” and does NOT switch the global underneath a live request.
29
+ Until the singleton is per-tenant (EXIT-5), an API process answering measure/Analyst queries must
30
+ be pinned to one tenant's file, and `assert_datastore_matches()` below is what turns a
31
+ mis-pinned deployment into a loud failure instead of a silent cross-tenant read.
32
+ """
33
+ from __future__ import annotations
34
+
35
+ import os
36
+ import threading
37
+ from collections import OrderedDict
38
+ from dataclasses import dataclass, field
39
+
40
+ import core.store as store
41
+ from harness import datastore as _ds
42
+ from harness import tenants as _tenants
43
+
44
+ #: How many tenant runtimes stay resident. Small on purpose: a runtime is cheap to rebuild (a
45
+ #: Tenant literal + a couple of handles) and expensive to leak. Raise only with a measurement.
46
+ _MAX_RUNTIMES = int(os.environ.get("AIOS_MAX_RUNTIMES") or 32)
47
+
48
+ #: slug -> builder. The literal registry `harness/tenants.py` describes; per its own docstring
49
+ #: this becomes "a tenants store (DB / HF dataset)" β€” X4's control DB is where it goes, and
50
+ #: `register()` below is the seam that makes the swap a one-liner rather than a refactor.
51
+ _BUILDERS = {"royal-imports": _tenants.royal_imports}
52
+
53
+ # X8's isolation fixture, admitted ONLY on request. A stub tenant that is always present is a
54
+ # slug an attacker can mint a session against in production; gated, it exists exactly where it is
55
+ # needed (the TestClient isolation proofs) and nowhere else. Read at import so the flag cannot be
56
+ # flipped by a request.
57
+ if os.environ.get("AIOS_ENABLE_QA_TENANT") == "1":
58
+ _BUILDERS["qa-b"] = _tenants.qa_tenant_b
59
+
60
+ _CACHE: "OrderedDict[str, TenantRuntime]" = OrderedDict()
61
+ _LOCK = threading.RLock()
62
+
63
+
64
+ @dataclass
65
+ class TenantRuntime:
66
+ """Everything a request needs to serve ONE tenant, and nothing about any other.
67
+
68
+ `store_namespace` is the prefix every product-data key is written under. Tenant #0 keeps the
69
+ EMPTY prefix so its existing keys (`users`, `customer_lists`, `customer_docs`, …) are found
70
+ exactly where they already live β€” a namespacing change that renamed tenant #0's keys would be
71
+ a silent data loss dressed as a refactor. Every OTHER tenant is prefixed, which is what makes
72
+ proof #3 hold.
73
+ """
74
+ key: str
75
+ tenant: object # harness.base.Tenant
76
+ store_namespace: str = ""
77
+ pool_cache: dict = field(default_factory=dict)
78
+ #: EXIT-5 β€” the API adapter's memos for `core.measure_resolve` (the Streamlit adapter keeps
79
+ #: its own in st.session_state). Bounded by the module's own clear-past-cap rule; keyed on
80
+ #: (stamp, scope, pool identity, question) so nothing user-shaped lives in them.
81
+ measure_memo: dict = field(default_factory=dict)
82
+ mset_memo: dict = field(default_factory=dict)
83
+ #: Wave 2026-08-02 (C-TS): the time-series channel's own memo β€” separate from
84
+ #: measure_memo so column churn and series churn cannot evict each other's answers.
85
+ series_memo: dict = field(default_factory=dict)
86
+ _lock: threading.RLock = field(default_factory=threading.RLock, repr=False)
87
+
88
+ @property
89
+ def name(self):
90
+ return getattr(self.tenant, "name", self.key)
91
+
92
+ @property
93
+ def datastore_path(self):
94
+ """The DuckDB file this tenant's analytical reads belong to (file-per-tenant IS the
95
+ isolation model β€” DuckDB has no RLS, so the OS boundary is the only one that holds)."""
96
+ return _ds.path_for(self.key)
97
+
98
+ # ---------------------------------------------------------------- the store handle
99
+ #: Wave 18 (C1-TENANT, R2): a tenant that OWNS its dataset repo carries its bound Store
100
+ #: instance here (namespace stays EMPTY β€” a whole repo needs no prefix). None = the shared
101
+ #: default repo through the MODULE functions, which is what keeps the gates' fake-store
102
+ #: installation effective (the fake patches module attributes, and royal-imports + the
103
+ #: prefix tenants resolve through exactly those).
104
+ store_handle: object = None
105
+
106
+ def _store(self):
107
+ return self.store_handle if self.store_handle is not None else store
108
+
109
+ def store_key(self, name):
110
+ """Namespace a product-data key for this tenant. The ONE place the prefix is applied."""
111
+ return f"{self.store_namespace}{name}" if self.store_namespace else str(name)
112
+
113
+ def get(self, name, fresh=False):
114
+ return self._store().get(self.store_key(name), fresh=fresh)
115
+
116
+ def put(self, name, data):
117
+ return self._store().put(self.store_key(name), data)
118
+
119
+ def update(self, name, fn, flush="sync"):
120
+ return self._store().update(self.store_key(name), fn, flush=flush)
121
+
122
+ def exists(self, name):
123
+ return self._store().exists(self.store_key(name))
124
+
125
+ def available(self):
126
+ return self._store().available()
127
+
128
+ def assert_datastore_matches(self):
129
+ """RAISE if the process-wide DuckDB file is not this tenant's.
130
+
131
+ The alternative is the failure mode that matters: an analytical read served from another
132
+ tenant's file returns real, plausible, wrong rows β€” no exception, no telemetry, and the
133
+ number reconciles against the wrong book. A loud failure is strictly better, and this is
134
+ the assertion EXIT-5 deletes when the connection becomes per-tenant.
135
+ """
136
+ want, have = self.datastore_path.resolve(), _ds.DB_PATH.resolve()
137
+ if want != have:
138
+ raise RuntimeError(
139
+ f"tenant {self.key!r} expects the analytical store at {want}, but this process "
140
+ f"has {have} open. Refusing the read β€” a cross-tenant answer would reconcile "
141
+ f"against the wrong book. (Pin the process with AIOS_DUCKDB_PATH, or call "
142
+ f"harness.datastore.use_path before serving this tenant.)")
143
+
144
+
145
+ def register(key, builder):
146
+ """Add a tenant to the registry (a stub connector is a legitimate builder β€” see X8's qa-b).
147
+ Invalidates any cached runtime for the slug so a re-registration cannot be shadowed."""
148
+ _BUILDERS[str(key)] = builder
149
+ invalidate(key)
150
+
151
+
152
+ def known_tenants():
153
+ keys = set(_BUILDERS)
154
+ try:
155
+ keys |= set(_tenant_records())
156
+ except Exception:
157
+ pass
158
+ return sorted(keys)
159
+
160
+
161
+ #: Wave 18 (C1-TENANT) β€” the CONTROL-PLANE bucket: tenants provisioned at runtime rather than
162
+ #: compiled in. Lives in the DEFAULT store (tenant #0's repo) deliberately: it answers "which
163
+ #: tenants exist", which is a platform fact, not any one tenant's data.
164
+ TENANTS_KEY = "tenants"
165
+
166
+
167
+ def _tenant_records():
168
+ """{slug: record} from the control-plane bucket. {} on any failure β€” an unreachable bucket
169
+ must degrade to "only the compiled-in tenants exist", never to an exception at login."""
170
+ try:
171
+ recs = store.get(TENANTS_KEY) or {}
172
+ return {str(k).strip().lower(): v for k, v in recs.items() if isinstance(v, dict)}
173
+ except Exception:
174
+ return {}
175
+
176
+
177
+ def _builder_from_record(key, rec):
178
+ """A Tenant for a bucket-provisioned client: no sources yet (connectors arrive via the
179
+ Keychain flow), config carries the platform fields the request path reads."""
180
+ def _build():
181
+ from harness.base import Tenant
182
+ return Tenant(key=key, name=str(rec.get("name") or key),
183
+ config={"modules": rec.get("modules", []),
184
+ "store_repo": rec.get("store_repo") or None})
185
+ return _build
186
+
187
+
188
+ def get_runtime(tenant_key):
189
+ """The runtime for `tenant_key`, built on first use and LRU-cached.
190
+
191
+ Raises KeyError for an unknown slug β€” the caller turns that into a 401 (fail-closed: an
192
+ unknown tenant is not a 404, because confirming which slugs exist answers a question the
193
+ request was not entitled to ask).
194
+ """
195
+ key = str(tenant_key or "").strip().lower()
196
+ if not key:
197
+ raise KeyError("no tenant")
198
+ with _LOCK:
199
+ rt = _CACHE.get(key)
200
+ if rt is not None:
201
+ _CACHE.move_to_end(key) # LRU: touch on read
202
+ return rt
203
+ builder = _BUILDERS.get(key)
204
+ # Wave 18 (C1-TENANT): a slug the compiled registry does not know may be a PROVISIONED
205
+ # tenant β€” consult the control-plane bucket before failing closed. A suspended record is
206
+ # the same KeyError as an unknown one: "which tenants exist" is not this caller's question.
207
+ if builder is None:
208
+ rec = _tenant_records().get(key)
209
+ if not rec or rec.get("status", "active") != "active":
210
+ raise KeyError(key)
211
+ builder = _builder_from_record(key, rec)
212
+ # Built OUTSIDE the lock: a builder may do real work (a connector handshake), and holding the
213
+ # registry lock through it would serialise every other tenant's requests behind it.
214
+ tenant = builder()
215
+ with _LOCK:
216
+ existing = _CACHE.get(key)
217
+ if existing is not None: # another thread won the race β€” keep ONE runtime
218
+ _CACHE.move_to_end(key)
219
+ return existing
220
+ # R2: a tenant with its OWN repo gets a bound Store instance and an EMPTY prefix (the
221
+ # repo boundary is the namespace); everyone else keeps the t/<slug>/ prefix in the
222
+ # shared repo. Tenant #0 keeps both defaults β€” empty prefix, shared repo.
223
+ repo = (getattr(tenant, "config", {}) or {}).get("store_repo")
224
+ rt = TenantRuntime(
225
+ key=key, tenant=tenant,
226
+ store_namespace=("" if (key == "royal-imports" or repo) else f"t/{key}/"),
227
+ store_handle=(store.for_repo(repo) if repo else None))
228
+ _CACHE[key] = rt
229
+ while len(_CACHE) > _MAX_RUNTIMES:
230
+ _CACHE.popitem(last=False) # evict least recently used
231
+ return rt
232
+
233
+
234
+ def invalidate(tenant_key=None):
235
+ """Drop one tenant's runtime, or all of them. Explicit, per X7 β€” a cache with no documented
236
+ way to clear it becomes a restart."""
237
+ with _LOCK:
238
+ if tenant_key is None:
239
+ _CACHE.clear()
240
+ else:
241
+ _CACHE.pop(str(tenant_key or "").strip().lower(), None)
242
+
243
+
244
+ def resident():
245
+ """The tenant slugs currently holding a runtime β€” LRU order, oldest first. For the EXIT-4
246
+ measurement and for asserting the bound actually binds."""
247
+ with _LOCK:
248
+ return list(_CACHE)
platform/harness/semantic.py ADDED
@@ -0,0 +1,936 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """harness/semantic.py β€” the semantic seam (OM-0a, 2026-07-11).
2
+
3
+ The ONE governed registry of topics + metrics (platform/model/*.yml) and the resolver that
4
+ turns a metric key into a number through the SAME proven data layer the validated modules use
5
+ (core/odoo domain builders + aggregate helpers) β€” so scope parity is by construction, and every
6
+ metric carries its definition, ai_context, and an independent validate contract.
7
+
8
+ Consumers (the point of the seam): the app's pages, the metric dictionary (OM-0c), the saved-
9
+ dashboard viewer (OM-3; the Explore page was retired 2026-07-23), the AI Analyst's tools (OM-4:
10
+ list_topics / describe_topic / run_semantic_query), and the MCP server (OM-6). Plan:
11
+ .claude/wiki/research/omni-adoption.md (Part IV + the productization addendum).
12
+
13
+ Design constraints honored (from /odoo-api):
14
+ - read_group dot-path GROUPBY fails β†’ scalar aggregates here use sum_field/distinct_count;
15
+ grouped queries (OM-3+) must group by direct m2o fields only.
16
+ - Scope is never hand-rolled: topics BIND to a named domain builder (sale_line_domain, ...);
17
+ the YAML `scope:` block is the declarative statement of what that builder bakes in.
18
+ - READ-ONLY on Odoo, always.
19
+ """
20
+ import ast
21
+ import functools
22
+ import operator
23
+ from pathlib import Path
24
+
25
+ import yaml
26
+
27
+ import core.odoo as O
28
+
29
+ MODEL_DIR = Path(__file__).resolve().parents[1] / "model"
30
+
31
+ # Topic scope binds to a NAMED, reviewed domain builder β€” never a YAML-assembled domain (one
32
+ # source of truth for scope until OM-2 makes topics executable against the store).
33
+ def _order_domain(*a, **kw):
34
+ # lazy: order scope currently lives in modules/sales.py; OM-2 lifts it into core/the store
35
+ import modules.sales as S
36
+ return S.order_domain(*a, **kw)
37
+
38
+
39
+ def _customer_move_domain(move_type):
40
+ """A posted customer document domain, matching `modules/returns.py`'s `_mdom` exactly.
41
+
42
+ ⚠ COMPANY-LEVEL, and it REFUSES a team rather than ignoring one. Credit notes all carry
43
+ team_id = 1, so a BU filter on the document is meaningless β€” silently dropping the argument
44
+ would hand a business-unit user the company's number under their own name, which is the
45
+ widening sin in a different currency.
46
+ """
47
+ def build(date_from=None, date_to=None, team_id=None, **kw):
48
+ if team_id:
49
+ raise ModelError(
50
+ f"customer {move_type} documents are COMPANY-LEVEL β€” credit notes and invoices "
51
+ f"carry no trustworthy business-unit tag, so a team-scoped total cannot be "
52
+ f"produced. Refusing rather than returning the company number.")
53
+ dom = [("move_type", "=", move_type), ("state", "=", "posted")]
54
+ if date_from:
55
+ dom.append(("invoice_date", ">=", date_from))
56
+ if date_to:
57
+ dom.append(("invoice_date", "<=", date_to))
58
+ return dom
59
+ return build
60
+
61
+
62
+ _BUILDERS = {
63
+ "sale_line_domain": O.sale_line_domain,
64
+ "sale_order_domain": _order_domain,
65
+ "credit_note_domain": _customer_move_domain("out_refund"),
66
+ "customer_invoice_domain": _customer_move_domain("out_invoice"),
67
+ }
68
+
69
+
70
+ class ModelError(Exception):
71
+ """A model-file problem (unknown key, bad reference, unparseable expr)."""
72
+
73
+
74
+ # ---------------------------------------------------------------- loading
75
+
76
+ @functools.lru_cache(maxsize=1)
77
+ def _tenant():
78
+ f = MODEL_DIR / "tenant.yml"
79
+ return yaml.safe_load(f.read_text(encoding="utf-8")) if f.exists() else {"params": {}}
80
+
81
+
82
+ def _sql_value(v):
83
+ """Render a tenant param into SQL safely (params are versioned config, not user input β€”
84
+ but escape anyway): ints as-is, lists comma-joined, strings single-quoted + escaped."""
85
+ if isinstance(v, bool):
86
+ raise ModelError("boolean tenant params not supported in scope_sql")
87
+ if isinstance(v, (int, float)):
88
+ return str(v)
89
+ if isinstance(v, (list, tuple)):
90
+ return ", ".join(_sql_value(x) for x in v)
91
+ return "'" + str(v).replace("'", "''") + "'"
92
+
93
+
94
+ def render_scope(sql):
95
+ """Substitute {param} placeholders in a store scope/filter SQL fragment from tenant.yml."""
96
+ params = _tenant().get("params") or {}
97
+ out = sql
98
+ for k, v in params.items():
99
+ out = out.replace("{" + k + "}", _sql_value(v))
100
+ if "{" in out and "}" in out:
101
+ import re as _re
102
+ missing = _re.findall(r"\{([a-z_]+)\}", out)
103
+ if missing:
104
+ raise ModelError(f"scope_sql references unknown tenant params: {missing}")
105
+ return out
106
+
107
+
108
+ @functools.lru_cache(maxsize=1)
109
+ def _model():
110
+ topics, metrics = {}, {}
111
+ for f in sorted((MODEL_DIR / "topics").glob("*.yml")):
112
+ d = yaml.safe_load(f.read_text(encoding="utf-8"))
113
+ if not d or "key" not in d:
114
+ raise ModelError(f"topic file {f.name} lacks a 'key'")
115
+ if d.get("domain_builder") and d["domain_builder"] not in _BUILDERS:
116
+ raise ModelError(f"topic {d['key']}: unknown domain_builder {d.get('domain_builder')!r}")
117
+ # a topic may be STORE-ONLY (no live-path builder yet, e.g. gl_lines) β€” metric() raises
118
+ # a clear error if a live resolution is attempted on it
119
+ topics[d["key"]] = d
120
+ for f in sorted((MODEL_DIR / "metrics").glob("*.yml")):
121
+ d = yaml.safe_load(f.read_text(encoding="utf-8"))
122
+ topic = d.get("topic")
123
+ if topic not in topics:
124
+ raise ModelError(f"metrics file {f.name}: unknown topic {topic!r}")
125
+ for m in d.get("metrics", []):
126
+ if "key" not in m:
127
+ raise ModelError(f"metrics file {f.name}: a metric lacks a 'key'")
128
+ if m["key"] in metrics:
129
+ raise ModelError(f"duplicate metric key {m['key']!r}")
130
+ m.setdefault("topic", topic) # a metric may name its own topic (rare)
131
+ if m["topic"] not in topics:
132
+ raise ModelError(f"metric {m['key']!r}: unknown topic {m['topic']!r}")
133
+ metrics[m["key"]] = m
134
+ return {"topics": topics, "metrics": metrics}
135
+
136
+
137
+ def reload_model():
138
+ _model.cache_clear()
139
+ _tenant.cache_clear()
140
+ return _model()
141
+
142
+
143
+ def topics():
144
+ return dict(_model()["topics"])
145
+
146
+
147
+ def metrics():
148
+ return dict(_model()["metrics"])
149
+
150
+
151
+ def describe(key):
152
+ """Full metadata for a metric (the dictionary page / AI describe_topic feed)."""
153
+ m = _model()["metrics"].get(key)
154
+ if not m:
155
+ raise ModelError(f"unknown metric {key!r}")
156
+ return {**m, "topic_def": _model()["topics"][m["topic"]]}
157
+
158
+
159
+ # ---------------------------------------------------------------- resolving
160
+
161
+ def _domain(topic_def, date_from, date_to, team_id):
162
+ b = topic_def.get("domain_builder")
163
+ if not b:
164
+ raise ModelError(f"topic {topic_def['key']!r} is store-only β€” use store_query() "
165
+ f"(no live-path domain builder bound yet)")
166
+ return _BUILDERS[b](date_from, date_to, team_id=team_id)
167
+
168
+
169
+ _OPS = {ast.Add: operator.add, ast.Sub: operator.sub,
170
+ ast.Mult: operator.mul, ast.Div: operator.truediv}
171
+
172
+
173
+ def _eval_expr(expr, resolve):
174
+ """Safely evaluate a derived-metric expression: metric keys, numbers, + - * / and parens."""
175
+ def ev(node):
176
+ if isinstance(node, ast.Expression):
177
+ return ev(node.body)
178
+ if isinstance(node, ast.BinOp) and type(node.op) in _OPS:
179
+ return _OPS[type(node.op)](ev(node.left), ev(node.right))
180
+ if isinstance(node, ast.UnaryOp) and isinstance(node.op, ast.USub):
181
+ return -ev(node.operand)
182
+ if isinstance(node, ast.Num): # py<3.8 compat name; Constant below
183
+ return node.n
184
+ if isinstance(node, ast.Constant) and isinstance(node.value, (int, float)):
185
+ return node.value
186
+ if isinstance(node, ast.Name):
187
+ return resolve(node.id)
188
+ raise ModelError(f"disallowed token in derived expr: {ast.dump(node)}")
189
+ try:
190
+ return ev(ast.parse(expr, mode="eval"))
191
+ except SyntaxError as e:
192
+ raise ModelError(f"unparseable derived expr {expr!r}: {e}")
193
+
194
+
195
+ def metric(key, date_from=None, date_to=None, team_id=None, _seen=None):
196
+ """Resolve a registered metric to a number for a window (+optional BU). The only public
197
+ compute path β€” every surface that shows this metric goes through here."""
198
+ _seen = _seen or set()
199
+ if key in _seen:
200
+ raise ModelError(f"circular metric reference at {key!r}")
201
+ _seen = _seen | {key}
202
+ m = _model()["metrics"].get(key)
203
+ if not m:
204
+ raise ModelError(f"unknown metric {key!r}")
205
+ agg = m.get("agg")
206
+ if agg == "ratio":
207
+ num = metric(m["numerator"], date_from, date_to, team_id, _seen)
208
+ den = metric(m["denominator"], date_from, date_to, team_id, _seen)
209
+ return (num / den) if den else 0.0
210
+ if agg == "derived":
211
+ return _eval_expr(m["expr"], lambda k: metric(k, date_from, date_to, team_id, _seen))
212
+ t = _model()["topics"][m["topic"]]
213
+ dom = _domain(t, date_from, date_to, team_id)
214
+ if agg == "sum":
215
+ # ⚠ `negate` applies HERE too, not only in `_measure_sql`. It existed for GL income on a
216
+ # STORE-ONLY topic, so the live path never met it β€” the first metric that is both negated
217
+ # and parity-checked (`returns`) would otherwise compare a negative live number against a
218
+ # positive store one, and the "parity failure" would look like a data problem.
219
+ got = O.sum_field(t["entity"], dom, m["field"])
220
+ return -got if m.get("negate") else got
221
+ if agg == "count_distinct":
222
+ return O.distinct_count(t["entity"], dom, m["field"])
223
+ if agg == "count":
224
+ return O.get_odoo().search_count(t["entity"], dom)
225
+ raise ModelError(f"metric {key!r}: unknown agg {agg!r}")
226
+
227
+
228
+ # ---------------------------------------------------------------- validate contracts
229
+
230
+ def _v_order_level_revenue(date_from, date_to, team_id):
231
+ """Independent check: line-level Ξ£ price_subtotal == order-level Ξ£ amount_untaxed."""
232
+ import modules.sales as S
233
+ line = metric("revenue", date_from, date_to, team_id)
234
+ order = O.sum_field("sale.order", S.order_domain(date_from, date_to, team_id), "amount_untaxed")
235
+ return line, order
236
+
237
+
238
+ def _v_order_count(date_from, date_to, team_id):
239
+ """Exact reconciliation: order-level count == distinct line-parent orders + line-LESS orders.
240
+ (Confirmed orders with zero lines exist in the data β€” 2 found at first proof, 2026-07-11;
241
+ counting them explicitly makes the identity exact instead of hiding them in a tolerance,
242
+ and surfaces them as a data-health signal.)"""
243
+ import modules.sales as S
244
+ order_level = metric("orders", date_from, date_to, team_id)
245
+ dom = S.order_domain(date_from, date_to, team_id)
246
+ from_lines = O.distinct_count("sale.order.line",
247
+ O.sale_line_domain(date_from, date_to, team_id), "order_id")
248
+ empty = O.get_odoo().search_count("sale.order", dom + [("order_line", "=", False)])
249
+ return order_level, from_lines + empty
250
+
251
+
252
+ def _v_gl_opex(date_from, date_to, team_id):
253
+ """Store-path opex == the live Odoo aggregate with the Expenses-module domain (posted,
254
+ expense-type accounts), to the cent. team_id N/A (company-level)."""
255
+ res = store_query("gl_lines", ["opex"], date_from=date_from, date_to=date_to)
256
+ ours = res["rows"][0]["opex"] if res["rows"] else 0
257
+ dom = [("move_id.state", "=", "posted"),
258
+ ("account_id.account_type", "in", ["expense", "expense_depreciation"])]
259
+ if date_from:
260
+ dom.append(("date", ">=", date_from))
261
+ if date_to:
262
+ dom.append(("date", "<=", date_to))
263
+ live = O.sum_field("account.move.line", dom, "balance")
264
+ return ours, live
265
+
266
+
267
+ def _v_ar_outstanding(date_from, date_to, team_id):
268
+ """Store-path AR outstanding == live Odoo aggregate (posted out_invoice/out_refund residuals).
269
+ No date window β€” outstanding is an as-of-now balance."""
270
+ res = store_query("receivables", ["ar_outstanding"])
271
+ ours = res["rows"][0]["ar_outstanding"] if res["rows"] else 0
272
+ live = O.sum_field("account.move",
273
+ [("state", "=", "posted"), ("move_type", "in", ["out_invoice", "out_refund"])],
274
+ "amount_residual_signed")
275
+ return ours, live
276
+
277
+
278
+ def _v_returns_sign(date_from, date_to, team_id):
279
+ """Independent check on the NEGATION: `returns` must equal the ABSOLUTE untaxed total of the
280
+ same posted credit notes, read straight from Odoo without going through the metric.
281
+
282
+ A lost negation makes returns negative; a doubled one makes it negative again. Both read as
283
+ "no returns" rather than as an error, and neither is visible to any self-consistency check β€”
284
+ the store and the live path would agree with each other perfectly while both being wrong.
285
+ """
286
+ ours = metric("returns", date_from, date_to, team_id)
287
+ raw = O.sum_field("account.move",
288
+ _BUILDERS["credit_note_domain"](date_from, date_to, team_id),
289
+ "amount_untaxed_signed")
290
+ return ours, abs(raw)
291
+
292
+
293
+ def _v_line_vs_move_grain(date_from, date_to, team_id):
294
+ """Independent check on the invoice-LINE topic: the store's netted product-line total must
295
+ equal the same figure read straight from LIVE Odoo.
296
+
297
+ Two things this catches that self-consistency cannot. (1) A LOST CREDIT-NOTE NEGATION β€”
298
+ `price_subtotal` is stored POSITIVE on a refund line, so a dropped subtraction inflates
299
+ revenue while every internal check still agrees. (2) A WRONG `display_type` FILTER β€” the
300
+ invoice tables carry 'cogs' and 'payment_term' rows alongside 'product' ones, and including
301
+ them changes the total silently rather than erroring.
302
+
303
+ Company-level (team_id None) uses the strongest possible reference: the MOVE-grain
304
+ `amount_untaxed_signed`, a different table entirely. Per-BU there is no move-grain analogue
305
+ (account.move carries no business unit β€” every invoice sits on team 1), so the reference is
306
+ the live line-level aggregate reached through the sale-order dot path: still an independent
307
+ engine and query path from the DuckDB store, which is what the contract requires.
308
+ """
309
+ ours = (store_query("invoice_lines", ["invoiced_line_sales"],
310
+ date_from=date_from, date_to=date_to, team_id=team_id)
311
+ .get("rows") or [{}])[0].get("invoiced_line_sales") or 0.0
312
+ if team_id is None:
313
+ raw = O.sum_field("account.move",
314
+ [("state", "=", "posted"),
315
+ ("move_type", "in", ["out_invoice", "out_refund"]),
316
+ ("invoice_date", ">=", date_from), ("invoice_date", "<=", date_to)],
317
+ "amount_untaxed_signed")
318
+ return ours, raw
319
+ dom = [("parent_state", "=", "posted"), ("display_type", "=", "product"),
320
+ ("date", ">=", date_from), ("date", "<=", date_to),
321
+ ("sale_line_ids.order_id.team_id", "=", team_id)]
322
+ gross = O.sum_field("account.move.line", dom + [("move_type", "=", "out_invoice")],
323
+ "price_subtotal")
324
+ refunds = O.sum_field("account.move.line", dom + [("move_type", "=", "out_refund")],
325
+ "price_subtotal")
326
+ return ours, (gross or 0) - (refunds or 0)
327
+
328
+
329
+ _VALIDATORS = {
330
+ "returns_sign": _v_returns_sign,
331
+ "line_vs_move_grain": _v_line_vs_move_grain,
332
+ "order_level_revenue": _v_order_level_revenue,
333
+ "order_count": _v_order_count,
334
+ "gl_opex": _v_gl_opex,
335
+ "ar_outstanding": _v_ar_outstanding,
336
+ }
337
+
338
+
339
+ def validate_metric(key, date_from=None, date_to=None, team_id=None, tol=0.01):
340
+ """Run a metric's declared independent cross-check. Returns the standard check dict
341
+ (the validate.py convention: check/ok/gap)."""
342
+ m = _model()["metrics"].get(key)
343
+ if not m:
344
+ raise ModelError(f"unknown metric {key!r}")
345
+ contract = m.get("validate")
346
+ if not contract:
347
+ return {"check": f"semantic.{key}: no independent contract declared", "ok": True, "gap": 0.0,
348
+ "note": "declare one in model/metrics when an independent aggregate exists"}
349
+ fn = _VALIDATORS.get(contract["method"])
350
+ if not fn:
351
+ raise ModelError(f"metric {key!r}: unknown validate method {contract['method']!r}")
352
+ ours, independent = fn(date_from, date_to, team_id)
353
+ gap = abs((ours or 0) - (independent or 0))
354
+ return {"check": f"semantic.{key} == {contract['method']} ({date_from}..{date_to}, team={team_id})",
355
+ "ok": gap <= tol, "gap": round(gap, 4), "ours": ours, "independent": independent}
356
+
357
+
358
+ def validate(pre=None):
359
+ """Module-convention validate(): run every declared contract over YTD, all-BU + per-BU."""
360
+ import core.periods as P
361
+ yf, yt = P.ytd()
362
+ out = []
363
+ for key, m in _model()["metrics"].items():
364
+ if not m.get("validate"):
365
+ continue
366
+ out.append(validate_metric(key, yf, yt, None))
367
+ topic = _model()["topics"][m["topic"]]
368
+ if (topic.get("store") or {}).get("team_col"): # company-level topics have no BU runs
369
+ for tid in O.TEAM_IDS:
370
+ out.append(validate_metric(key, yf, yt, tid))
371
+ return out
372
+
373
+
374
+ # ---------------------------------------------------------------- store path (OM-2)
375
+ # The compiler behind run_semantic_query: registered metrics + whitelisted dims β†’ DuckDB SQL over
376
+ # the tenant store (harness/datastore.py). EVERY identifier comes from the versioned model files
377
+ # (trusted); every VALUE is parameterized β€” the small model only ever picks keys, so there is no
378
+ # injection surface. Live XML-RPC stays the validate() path (the store never validates itself).
379
+
380
+ def _store_con():
381
+ import harness.datastore as DS
382
+ if not DS.ready(): # a mid-backfill store returns PARTIAL totals silently
383
+ raise ModelError("the data cache is still warming up after a restart (a one-time first "
384
+ "sync) β€” the dashboards work now from live data; ask me again in a "
385
+ "moment and I'll have it")
386
+ return DS.ro_cursor() # cursor on the shared instance β€” no sync-writer contention
387
+
388
+
389
+ def _measure_sql(m, alias):
390
+ """SQL for a base measure. Supports FILTERED measures (`store_filter_sql` β€” the Omni
391
+ filtered-measure concept: sum(CASE WHEN … )) and `negate: true` (e.g. GL income, stored
392
+ credit-negative, reported positive)."""
393
+ agg, f = m.get("agg"), m.get("field")
394
+ flt = render_scope(m["store_filter_sql"]) if m.get("store_filter_sql") else None
395
+ if agg == "sum":
396
+ core = f"sum(CASE WHEN {flt} THEN {alias}.{f} ELSE 0 END)" if flt else f"sum({alias}.{f})"
397
+ elif agg == "count_distinct":
398
+ core = (f"count(DISTINCT CASE WHEN {flt} THEN {alias}.{f} END)" if flt
399
+ else f"count(DISTINCT {alias}.{f})")
400
+ elif agg == "count":
401
+ core = f"count(CASE WHEN {flt} THEN 1 END)" if flt else "count(*)"
402
+ else:
403
+ raise ModelError(f"metric {m['key']!r}: agg {agg!r} has no store expression")
404
+ return f"-({core})" if m.get("negate") else core
405
+
406
+
407
+ def _expand_measures(keys):
408
+ """Expand ratio/derived metrics into the base measures SQL must aggregate, keeping the
409
+ requested keys for post-compute. Returns (base_keys, requested_keys)."""
410
+ mts = _model()["metrics"]
411
+ base, seen = [], set()
412
+
413
+ def need(k):
414
+ if k in seen:
415
+ return
416
+ seen.add(k)
417
+ m = mts.get(k)
418
+ if not m:
419
+ raise ModelError(f"unknown metric {k!r}")
420
+ if m["agg"] == "ratio":
421
+ need(m["numerator"]); need(m["denominator"])
422
+ elif m["agg"] == "derived":
423
+ import ast as _ast
424
+ for node in _ast.walk(_ast.parse(m["expr"], mode="eval")):
425
+ if isinstance(node, _ast.Name):
426
+ need(node.id)
427
+ else:
428
+ if k not in base:
429
+ base.append(k)
430
+ for k in keys:
431
+ need(k)
432
+ return base, list(keys)
433
+
434
+
435
+ def _post_compute(row, key):
436
+ m = _model()["metrics"][key]
437
+ if m["agg"] == "ratio":
438
+ num, den = _post_compute(row, m["numerator"]), _post_compute(row, m["denominator"])
439
+ return (num / den) if den else 0.0
440
+ if m["agg"] == "derived":
441
+ return _eval_expr(m["expr"], lambda k: _post_compute(row, k))
442
+ return row.get(key) or 0.0
443
+
444
+
445
+ # A metric's declared `format` -> the grid field type the compiler and the cells both read.
446
+ # One mapping, so a new metric cannot arrive as an untyped column.
447
+ _FORMAT_TYPE = {"usd": "currency", "int": "int", "pct": "pct"}
448
+
449
+
450
+ def _clean_tree(tree, cols):
451
+ """Run a filter tree through the SAME validator the client's view state goes through.
452
+
453
+ `filter_sql`'s documented input contract is "feed me CLEANED trees" β€” it is faithful to the
454
+ TS engine rather than fail-closed on garbage, because rejecting garbage is the validator's
455
+ job. The UI path always cleans (view state is sanitised on the way into the store), but
456
+ store_query/store_rows are callable directly, and an uncleaned tree is where the two
457
+ diverge: `{"value": null}` is ACTIVE in TS (and throws on a text field) and INACTIVE here.
458
+
459
+ Normalising at the entry point makes that unreachable rather than merely unlikely, and
460
+ brings the depth / width / node-count caps to the server side too. `aios_grid` is a leaf
461
+ module with zero imports of its own, so this cannot cycle.
462
+ """
463
+ from aios_grid import (clean_filter_tree, COHORT_FIELD as _COHORT_FIELD,
464
+ MAX_FILTER_DEPTH, MAX_FILTER_NODES, MAX_FILTER_SIBLINGS)
465
+
466
+ # REFUSE an over-large tree rather than TRUNCATE it. clean_filter_tree caps siblings per
467
+ # level, total nodes and depth by DROPPING the excess β€” correct on the client, where the
468
+ # cap is a DoS guard on a per-row render loop and a dropped condition is the lesser evil.
469
+ # On the server it is not: dropping AND-conditions WIDENS the result, and store_rows then
470
+ # reports a row_count and scope-wide totals that look authoritative for a query the caller
471
+ # never asked for. That is precisely the silent [:N] that [[no-unverifiable-aggregates]]
472
+ # forbids. A caller that exceeds the caps gets an error, not a quietly different answer.
473
+ total = 0
474
+
475
+ def _walk(nodes, depth):
476
+ nonlocal total
477
+ nodes = list(nodes or [])
478
+ if len(nodes) > MAX_FILTER_SIBLINGS:
479
+ raise ModelError(f"filter tree has {len(nodes)} conditions at one level; the limit "
480
+ f"is {MAX_FILTER_SIBLINGS}. Refusing rather than dropping the "
481
+ f"excess, which would silently widen the result.")
482
+ for node in nodes:
483
+ if not isinstance(node, dict):
484
+ continue
485
+ total += 1
486
+ if isinstance(node.get('children'), list):
487
+ if depth >= MAX_FILTER_DEPTH:
488
+ raise ModelError(f"filter tree nests deeper than {MAX_FILTER_DEPTH} levels; "
489
+ f"refusing rather than dropping the deepest group.")
490
+ _walk(node['children'], depth + 1)
491
+
492
+ _walk(tree, 1)
493
+ if total > MAX_FILTER_NODES:
494
+ raise ModelError(f"filter tree has {total} nodes; the limit is {MAX_FILTER_NODES}. "
495
+ f"Refusing rather than truncating, which would silently widen it.")
496
+
497
+ # CG-8. A MEASURE condition ("Sales in the last 90 days > 5,000") is answered by
498
+ # harness/measure_filter.py, which resolves it to a SET OF CUSTOMER IDS. It has no meaning
499
+ # on this path, and both ways it could arrive here are silent:
500
+ # - on a topic without that column, `clean_filter_tree` DROPS it as unknown β€” widening the
501
+ # query while store_rows reports an authoritative row_count for something nobody asked
502
+ # for (the same class as the sibling/depth caps above);
503
+ # - on sales_lines at row grain, `revenue` IS a real column, so it would compile as a
504
+ # per-LINE predicate with the window silently discarded β€” a different question answered
505
+ # confidently, which is worse than dropping it.
506
+ # The discriminator is the `window`, not the column name: no column condition has one.
507
+ windowed = []
508
+ cohorts = []
509
+
510
+ def _find(nodes):
511
+ for node in nodes or []:
512
+ if not isinstance(node, dict):
513
+ continue
514
+ if isinstance(node.get('children'), list):
515
+ _find(node['children'])
516
+ elif node.get('window') is not None:
517
+ windowed.append(str(node.get('colId')))
518
+ elif node.get('colId') == _COHORT_FIELD:
519
+ cohorts.append(str(node.get('value')))
520
+
521
+ _find(tree)
522
+ # Owner item 5, and the same refusal for the same reason. A cohort leaf names a
523
+ # hand-curated set of CUSTOMERS held per user in the tenant store β€” this path knows nothing
524
+ # about it, and `clean_filter_tree` below would drop it as an unknown key, widening the
525
+ # query while store_rows still reported an authoritative row_count. If a caller ever needs
526
+ # it here, the fix is to pass the membership in, not to let it fall through.
527
+ if cohorts:
528
+ raise ModelError(
529
+ f"filter tree carries cohort-membership condition(s) {sorted(set(cohorts))} β€” "
530
+ f"cohort membership is host state, not a column of this topic. Refusing rather than "
531
+ f"dropping it, which would silently widen the result under an authoritative count.")
532
+ if windowed:
533
+ raise ModelError(
534
+ f"filter tree carries measure condition(s) over a date window {sorted(set(windowed))}"
535
+ f" β€” those are resolved to a set of ids by harness.measure_filter, not compiled into "
536
+ f"this query. Refusing rather than dropping or mis-compiling them, either of which "
537
+ f"would silently answer a different question under an authoritative count.")
538
+ return clean_filter_tree(tree, set(cols))
539
+
540
+
541
+ def _measure_row_sql(m, alias):
542
+ """A sum-metric's per-ROW value: the same expression `_measure_sql` aggregates, without the
543
+ aggregate wrapper. At line grain `revenue` IS `l.price_subtotal` for that line β€” deriving it
544
+ from the metric rather than hand-writing the column is what keeps ONE definition of the
545
+ number (the drift the semantic layer exists to prevent). Only `sum` has a row value; a
546
+ count/count_distinct metric is a property of a SET, not of a row."""
547
+ if m.get("agg") != "sum":
548
+ return None
549
+ f = m.get("field")
550
+ flt = render_scope(m["store_filter_sql"]) if m.get("store_filter_sql") else None
551
+ core = f"CASE WHEN {flt} THEN {alias}.{f} ELSE 0 END" if flt else f"{alias}.{f}"
552
+ return f"-({core})" if m.get("negate") else core
553
+
554
+
555
+ def store_columns(topic, include_measures=True, grain="aggregate"):
556
+ """The `{colId: {sql, type, aggregate}}` spec `harness.filter_sql` compiles against (CG-1).
557
+
558
+ `grain="row"` is the LINE-grain projection (CG-2): sum-metrics resolve to their own raw
559
+ field with `aggregate=False`, so a line table filters and sorts on real row values in WHERE
560
+ rather than needing HAVING. count-style metrics are dropped β€” they have no row value.
561
+
562
+ This is the seam between the grid's field contract and the semantic model: the grid speaks
563
+ field KEYS, the compiler needs SQL expressions and a field TYPE, and only the model may say
564
+ what a key resolves to. Nothing about a topic leaks into filter_sql itself.
565
+
566
+ <dim> -> the dim's NAME column (what the grid displays and what a user means when
567
+ they type "contains floral"), filtered PRE-aggregation in WHERE. Correct
568
+ even in a grouped query: every row of a group shares its group's name.
569
+ <dim>_id -> the raw id, for exact machine-keyed filtering.
570
+ date -> the topic's date column, so a grid can filter/sort it like any other field.
571
+ <metric> -> the aggregate expression, marked `aggregate` so the caller routes it to
572
+ HAVING rather than WHERE (see store_query β€” that path is not built yet).
573
+ """
574
+ t = _model()["topics"].get(topic)
575
+ if not t or "store" not in t:
576
+ raise ModelError(f"topic {topic!r} has no store binding")
577
+ s = t["store"]
578
+ cols = {}
579
+ for key, d in (s.get("dims") or {}).items():
580
+ cols[key] = {"sql": d.get("name_col") or d["col"], "type": "text", "aggregate": False}
581
+ # Only expose a separate id column when the dim actually HAS one. `city` is its own
582
+ # name (col == name_col), so a `city_id` would be a text column typed int β€” filtering
583
+ # it numerically would quietly compare 0 against 0 for every row.
584
+ if d.get("name_col") and d["name_col"] != d["col"]:
585
+ cols[f"{key}_id"] = {"sql": d["col"], "type": "int", "aggregate": False}
586
+ if s.get("date_col"):
587
+ cols["date"] = {"sql": s["date_col"], "type": "date", "aggregate": False}
588
+ if include_measures:
589
+ for k, m in _model()["metrics"].items():
590
+ if m.get("topic") != topic or m.get("agg") in ("ratio", "derived"):
591
+ continue # ratio/derived are post-computed, not SQL
592
+ # The field TYPE comes from the metric's declared format, never a guess: `units` is
593
+ # format:int, and typing it currency would render 791,960 units as dollars.
594
+ ftype = _FORMAT_TYPE.get(m.get("format"), "currency")
595
+ if grain == "row":
596
+ row_sql = _measure_row_sql(m, s["alias"])
597
+ if row_sql:
598
+ cols[k] = {"sql": row_sql, "type": ftype, "aggregate": False}
599
+ else:
600
+ cols[k] = {"sql": _measure_sql(m, s["alias"]),
601
+ "type": ftype, "aggregate": True}
602
+ return cols
603
+
604
+
605
+ def store_query(topic, measures, group_by=None, grain=None, date_from=None, date_to=None,
606
+ team_id=None, filters=None, sort=None, limit=1000, exclude_services=False,
607
+ filter_tree=None, filter_conj="and", today=None):
608
+ """The semantic query over the tenant store β€” the engine behind the Analyst's
609
+ run_semantic_query tool and the saved-view re-runner. All keys whitelisted against the model."""
610
+ t = _model()["topics"].get(topic)
611
+ if not t or "store" not in t:
612
+ raise ModelError(f"topic {topic!r} has no store binding")
613
+ s = t["store"]
614
+ alias = s["alias"]
615
+ dims = s.get("dims") or {}
616
+ limit = max(1, min(int(limit or 1000), 5000))
617
+
618
+ base_keys, requested = _expand_measures(list(measures or []))
619
+ if not base_keys:
620
+ raise ModelError("at least one measure required")
621
+ # Cross-topic base metrics (e.g. aov = revenue Γ· orders, where orders lives on sales_orders):
622
+ # in a SCALAR query they resolve via a nested scalar query on their HOME topic (correct grain β€”
623
+ # never count(*) on the wrong table); in a GROUPED query they are refused (split the request).
624
+ foreign = [k for k in base_keys if _model()["metrics"][k]["topic"] != topic]
625
+ if foreign and (group_by or grain):
626
+ raise ModelError(f"metrics {foreign} belong to another topic β€” cross-topic measures are "
627
+ f"scalar-only; run them against their own topic when grouping")
628
+ base_keys = [k for k in base_keys if k not in foreign]
629
+ foreign_vals = {}
630
+ for k in foreign:
631
+ sub = store_query(_model()["metrics"][k]["topic"], [k], date_from=date_from,
632
+ date_to=date_to, team_id=team_id)
633
+ foreign_vals[k] = sub["rows"][0][k] if sub["rows"] else 0
634
+ if not base_keys: # purely foreign scalar request
635
+ return {"topic": topic, "rows": [foreign_vals], "row_count": 1, "sql": "(nested)",
636
+ "measures": requested, "group_by": [], "grain": None}
637
+
638
+ select, group_cols, params = [], [], []
639
+ gb = [g for g in (group_by or []) if g]
640
+ for g in gb:
641
+ if g not in dims:
642
+ raise ModelError(f"group_by {g!r} not a dim of {topic!r} (allowed: {list(dims)})")
643
+ d = dims[g]
644
+ select.append(f"{d['col']} AS {g}_id" if d.get("name_col") else f"{d['col']} AS {g}")
645
+ group_cols.append(d["col"])
646
+ if d.get("name_col"):
647
+ select.append(f"max({d['name_col']}) AS {g}")
648
+ if grain:
649
+ if grain not in ("month", "week", "day"):
650
+ raise ModelError("grain must be month|week|day")
651
+ select.insert(0, f"date_trunc('{grain}', CAST({s['date_col']} AS TIMESTAMP)) AS period")
652
+ group_cols.insert(0, f"date_trunc('{grain}', CAST({s['date_col']} AS TIMESTAMP))")
653
+ for k in base_keys:
654
+ select.append(f"{_measure_sql(_model()['metrics'][k], alias)} AS {k}")
655
+
656
+ where = [render_scope(s["scope_sql"].strip())]
657
+ # TYPE-CORRECT date bounds (the 2025-07-01 lesson): columns hold ISO strings in TWO shapes β€”
658
+ # datetimes ('YYYY-MM-DD HH:MM:SS', sale date_order) and bare dates ('YYYY-MM-DD', GL date).
659
+ # Lexical string comparison EXCLUDES the window's first day for bare dates ('2025-07-01' <
660
+ # '2025-07-01 00:00:00'), which silently dropped a whole day of GL ($49,467). Cast both sides.
661
+ if date_from:
662
+ where.append(f"CAST({s['date_col']} AS TIMESTAMP) >= ?")
663
+ params.append(f"{date_from} 00:00:00")
664
+ if date_to:
665
+ where.append(f"CAST({s['date_col']} AS TIMESTAMP) <= ?")
666
+ params.append(f"{date_to} 23:59:59")
667
+ if team_id:
668
+ if not s.get("team_col"):
669
+ raise ModelError(f"topic {topic!r} is company-level β€” it has no business-unit filter")
670
+ where.append(f"{s['team_col']} = ?"); params.append(int(team_id))
671
+ if exclude_services and s.get("service_filter"):
672
+ where.append(render_scope(s["service_filter"]))
673
+ for dim_key, vals in (filters or {}).items():
674
+ if dim_key not in dims:
675
+ raise ModelError(f"filter dim {dim_key!r} not allowed (use: {list(dims)})")
676
+ # id dims filter by int; TEXT dims (e.g. city) filter by the value itself β€” either way
677
+ # every VALUE stays parameterized (the whitelist covers identifiers only)
678
+ def _coerce(v):
679
+ try:
680
+ return int(v)
681
+ except (TypeError, ValueError):
682
+ return str(v)
683
+ ids = [_coerce(v) for v in (vals if isinstance(vals, (list, tuple)) else [vals])]
684
+ where.append(f"{dims[dim_key]['col']} IN ({','.join('?' for _ in ids)})")
685
+ params.extend(ids)
686
+
687
+ # The grid's filter TREE (CG-1). `filters` above stays the Analyst's flat dim=IN shape;
688
+ # this is the nested, 11-operator contract the table UI emits. Compiled by
689
+ # harness/filter_sql, which is held in lock-step with the TS engine and the validator by
690
+ # aios-web/verify_filter_engine.py. Appended AFTER the dim filters so params stay in
691
+ # positional order with `where`.
692
+ if filter_tree:
693
+ from harness import filter_sql as _fs
694
+ cols = store_columns(topic)
695
+ pred = _fs.compile_filter_tree(_clean_tree(filter_tree, cols), filter_conj, cols,
696
+ today=today)
697
+ if pred is not None:
698
+ if pred.uses_aggregate:
699
+ bad = sorted(c for c in pred.columns_used if cols[c].get("aggregate"))
700
+ raise ModelError(
701
+ f"filter_tree references measure(s) {bad} β€” a measure filter has to land in "
702
+ f"HAVING, and that path is deliberately NOT built: CG-1 ships the WHERE path "
703
+ f"because its consumer (CG-2, the line-grain sales table) does not aggregate. "
704
+ f"Filter on dims, or aggregate first and filter the result.")
705
+ where.append(pred.sql)
706
+ params.extend(pred.params)
707
+
708
+ sql = (f"SELECT {', '.join(select)} FROM {s['table']} {alias} {s.get('join','')} "
709
+ f"WHERE {' AND '.join(where)}")
710
+ if group_cols:
711
+ sql += f" GROUP BY {', '.join(group_cols)}"
712
+ if sort:
713
+ key = sort.lstrip("-")
714
+ if key not in base_keys + gb + (["period"] if grain else []):
715
+ raise ModelError(f"sort {sort!r} must reference a selected measure/dim")
716
+ sql += f" ORDER BY {key} {'DESC' if sort.startswith('-') else 'ASC'}"
717
+ elif grain:
718
+ sql += " ORDER BY period"
719
+ sql += f" LIMIT {limit}"
720
+
721
+ con = _store_con()
722
+ try:
723
+ cur = con.execute(sql, params)
724
+ cols = [d[0] for d in cur.description]
725
+ rows = [dict(zip(cols, r)) for r in cur.fetchall()]
726
+ finally:
727
+ con.close()
728
+ for row in rows: # ratio/derived post-compute per group
729
+ row.update(foreign_vals) # scalar cross-topic components
730
+ for k in requested:
731
+ if _model()["metrics"][k]["agg"] in ("ratio", "derived"):
732
+ row[k] = _post_compute(row, k)
733
+ if "period" in row and row["period"] is not None:
734
+ row["period"] = str(row["period"])[:10]
735
+ return {"topic": topic, "rows": rows, "row_count": len(rows), "sql": sql,
736
+ "measures": requested, "group_by": gb, "grain": grain}
737
+
738
+
739
+ def store_rows(topic, date_from=None, date_to=None, team_id=None, filter_tree=None,
740
+ filter_conj="and", search=None, sorts=None, limit=200, offset=0,
741
+ aggregates=None, exclude_services=False, id_sql=None, member_ids=None,
742
+ today=None):
743
+ """ROW-grain fetch β€” the line-grain counterpart to store_query's aggregate path (CG-2).
744
+
745
+ store_query answers "what is the number"; this answers "which rows". At line grain the
746
+ payload is the binding constraint (254,837 sale_order_lines against 1,550 customers β€” the
747
+ whole book would be ~96 MB at the customer table's measured 377 B/row), so rows come back
748
+ WINDOWED.
749
+
750
+ A window is only honest if the counts and totals are not windowed with it
751
+ ([[no-unverifiable-aggregates]]). So this returns THREE independent numbers, each from its
752
+ own query over the FULL scope, never from `len(rows)`:
753
+
754
+ row_count rows matching the filter across the whole scope -> the N of "N of M"
755
+ total_count rows in the scope with no filter at all -> the M
756
+ aggregates the topic's OWN metric definitions, summed over the whole FILTERED scope
757
+
758
+ That is the difference between a windowed fetch and a silent `[:N]` cap: the user is told
759
+ how many rows exist and shown totals for all of them, while receiving only the page they
760
+ can see.
761
+
762
+ Filters/sorts/search compile through harness.filter_sql, so a line table means EXACTLY what
763
+ the client engine means at customer grain β€” that equivalence is what verify_filter_engine.py
764
+ exists to hold.
765
+
766
+ Dates are returned already truncated to the 10-char ISO shape the filter compares against,
767
+ so what is displayed and what is filtered are the same string. Numerics come back RAW; the
768
+ payload layer rounds them (aios_grid._round), and the compiler mirrors that rounding.
769
+ """
770
+ t = _model()["topics"].get(topic)
771
+ if not t or "store" not in t:
772
+ raise ModelError(f"topic {topic!r} has no store binding")
773
+ from harness import filter_sql as _fs
774
+ s = t["store"]
775
+ alias = s["alias"]
776
+ cols = store_columns(topic, grain="row")
777
+ id_sql = id_sql or f"{alias}.id"
778
+ limit = max(1, min(int(limit or 200), 5000))
779
+ offset = max(0, int(offset or 0))
780
+
781
+ # --- the scope every one of the three queries shares -------------------------------
782
+ scope, scope_params = [render_scope(s["scope_sql"].strip())], []
783
+ if date_from:
784
+ scope.append(f"CAST({s['date_col']} AS TIMESTAMP) >= ?")
785
+ scope_params.append(f"{date_from} 00:00:00")
786
+ if date_to:
787
+ scope.append(f"CAST({s['date_col']} AS TIMESTAMP) <= ?")
788
+ scope_params.append(f"{date_to} 23:59:59")
789
+ if team_id:
790
+ if not s.get("team_col"):
791
+ raise ModelError(f"topic {topic!r} is company-level β€” it has no business-unit filter")
792
+ scope.append(f"{s['team_col']} = ?")
793
+ scope_params.append(int(team_id))
794
+ if exclude_services and s.get("service_filter"):
795
+ scope.append(render_scope(s["service_filter"]))
796
+
797
+ # --- the user's narrowing, on top of the scope --------------------------------------
798
+ narrow, narrow_params = [], []
799
+ pred = _fs.compile_filter_tree(_clean_tree(filter_tree or [], cols), filter_conj, cols,
800
+ today=today,
801
+ member_ids=member_ids or None, id_sql=id_sql)
802
+ if pred is not None:
803
+ if pred.uses_aggregate: # cannot happen at grain="row"; guard anyway
804
+ raise ModelError(f"row-grain filter referenced an aggregate: {sorted(pred.columns_used)}")
805
+ narrow.append(pred.sql)
806
+ narrow_params.extend(pred.params)
807
+ spred = _fs.compile_search(search, cols)
808
+ if spred is not None:
809
+ narrow.append(spred.sql)
810
+ narrow_params.extend(spred.params)
811
+
812
+ frm = f"{s['table']} {alias} {s.get('join', '')}"
813
+ scope_where = " AND ".join(scope)
814
+ all_where = " AND ".join(scope + narrow)
815
+ con = _store_con()
816
+ try:
817
+ # 1. the WINDOW of rows
818
+ select = [f"{id_sql} AS _rid"] + [
819
+ (f"SUBSTR(CAST({c['sql']} AS VARCHAR), 1, 10) AS {k}" if c["type"] == "date"
820
+ else f"{c['sql']} AS {k}")
821
+ for k, c in cols.items()]
822
+ order = _fs.compile_order_by(sorts, cols, tiebreak_sql=id_sql) or f"{id_sql} ASC"
823
+ sql = (f"SELECT {', '.join(select)} FROM {frm} WHERE {all_where} "
824
+ f"ORDER BY {order} LIMIT {limit} OFFSET {offset}")
825
+ cur = con.execute(sql, scope_params + narrow_params)
826
+ names = [d[0] for d in cur.description]
827
+ rows = [dict(zip(names, r)) for r in cur.fetchall()]
828
+
829
+ # 2. the two counts β€” over the WHOLE scope, never len(rows)
830
+ count_sql = f"SELECT count(*) FROM {frm} WHERE {all_where}"
831
+ row_count = con.execute(count_sql, scope_params + narrow_params).fetchone()[0]
832
+ total_count = con.execute(f"SELECT count(*) FROM {frm} WHERE {scope_where}",
833
+ scope_params).fetchone()[0]
834
+
835
+ # 3. the aggregates β€” the topic's OWN metric definitions over the whole FILTERED scope
836
+ mts = _model()["metrics"]
837
+ keys = [k for k in (aggregates if aggregates is not None
838
+ else [k for k, m in mts.items()
839
+ if m.get("topic") == topic and m.get("agg") == "sum"])]
840
+ aggs = {}
841
+ if keys:
842
+ for k in keys:
843
+ if k not in mts or mts[k].get("topic") != topic:
844
+ raise ModelError(f"aggregate {k!r} is not a metric of topic {topic!r}")
845
+ agg_sql = ("SELECT " + ", ".join(f"{_measure_sql(mts[k], alias)} AS {k}" for k in keys)
846
+ + f" FROM {frm} WHERE {all_where}")
847
+ cur = con.execute(agg_sql, scope_params + narrow_params)
848
+ aggs = dict(zip([d[0] for d in cur.description], cur.fetchone()))
849
+ finally:
850
+ con.close()
851
+
852
+ return {
853
+ "topic": topic,
854
+ "rows": rows,
855
+ "columns": {k: {"type": c["type"]} for k, c in cols.items()},
856
+ "window": {"offset": offset, "limit": limit, "returned": len(rows)},
857
+ "row_count": row_count, # N β€” matches the filter, across the WHOLE scope
858
+ "total_count": total_count, # M β€” the unfiltered scope
859
+ "aggregates": aggs, # over the whole FILTERED scope, not the window
860
+ "sql": sql,
861
+ "count_sql": count_sql,
862
+ }
863
+
864
+
865
+ def store_field_values(topic, dim, search=None, limit=25):
866
+ """Resolve real filterable values for a dim (the Analyst's get_field_values tool β€” fixes
867
+ 'NYC Florist' vs the actual record name BEFORE querying)."""
868
+ t = _model()["topics"].get(topic)
869
+ if not t or "store" not in t:
870
+ raise ModelError(f"topic {topic!r} has no store binding")
871
+ d = (t["store"].get("dims") or {}).get(dim)
872
+ if not d:
873
+ raise ModelError(f"unknown dim {dim!r} for topic {topic!r}")
874
+ if not d.get("name_col"):
875
+ raise ModelError(f"dim {dim!r} has no name column (filter by id)")
876
+ s = t["store"]
877
+ sql = (f"SELECT DISTINCT {d['col']} AS id, {d['name_col']} AS name "
878
+ f"FROM {s['table']} {s['alias']} {s.get('join','')} "
879
+ f"WHERE {render_scope(s['scope_sql'].strip())}")
880
+ params = []
881
+ if search:
882
+ sql += f" AND {d['name_col']} ILIKE ?"
883
+ params.append(f"%{search}%")
884
+ sql += f" ORDER BY name LIMIT {max(1, min(int(limit), 100))}"
885
+ con = _store_con()
886
+ try:
887
+ return [{"id": r[0], "name": r[1]} for r in con.execute(sql, params).fetchall()]
888
+ finally:
889
+ con.close()
890
+
891
+
892
+ def store_parity(date_from=None, date_to=None):
893
+ """THE OM-2 gate: the store path must equal the live path to the cent, per metric per scope."""
894
+ import core.periods as P
895
+ if not date_from:
896
+ date_from, date_to = P.ytd()
897
+ out = []
898
+ for key in ("revenue", "margin", "units", "orders", "customers", "returns", "invoiced"):
899
+ m = _model()["metrics"][key]
900
+ # ⚠ A COMPANY-LEVEL topic is checked at company scope ONLY. `store_query` RAISES when a
901
+ # team is asked of a topic with no team_col, and the live builder refuses too β€” so
902
+ # looping the BUs here would fail the gate for a topic that is correct. Parity for a
903
+ # company-level number means company-level parity; pretending otherwise would either
904
+ # crash or, worse, compare two numbers that silently ignored the team.
905
+ _store = (_model()["topics"].get(m["topic"]) or {}).get("store") or {}
906
+ scopes = (None, *O.TEAM_IDS) if _store.get("team_col") else (None,)
907
+ for tid in scopes:
908
+ live = metric(key, date_from, date_to, tid)
909
+ res = store_query(m["topic"], [key], date_from=date_from, date_to=date_to, team_id=tid)
910
+ ours = res["rows"][0][key] if res["rows"] else 0
911
+ gap = abs((ours or 0) - (live or 0))
912
+ out.append({"check": f"store.{key} == live.{key} (team={tid})",
913
+ "ok": gap <= 0.01, "gap": round(gap, 4), "store": ours, "live": live})
914
+ # The agent dim (2026-07-17): store agent-filtered revenue must equal live revenue over the
915
+ # agent's FIRST-agent book β€” a fully independent path (live partner read + line-domain sum)
916
+ # against the store's res_partner.agent_id attribution. Largest book = the sharpest check.
917
+ recs = O.search_read("res.partner", [("active", "in", [True, False]),
918
+ ("agent_ids", "!=", False)], ["agent_ids"], limit=100000)
919
+ books = {}
920
+ for r in recs:
921
+ a = (r.get("agent_ids") or [None])[0]
922
+ if a:
923
+ books.setdefault(a, []).append(r["id"])
924
+ if books:
925
+ top = max(books, key=lambda k: len(books[k]))
926
+ live = O.sum_field("sale.order.line",
927
+ O.sale_line_domain(date_from, date_to, partner_ids=books[top]),
928
+ "price_subtotal")
929
+ res = store_query("sales_lines", ["revenue"], date_from=date_from, date_to=date_to,
930
+ filters={"agent": [top]})
931
+ ours = res["rows"][0]["revenue"] if res["rows"] else 0
932
+ gap = abs((ours or 0) - (live or 0))
933
+ out.append({"check": f"store.revenue[agent={top}] == live book sum (first-agent, "
934
+ f"{len(books[top])} customers)",
935
+ "ok": gap <= 0.01, "gap": round(gap, 4), "store": ours, "live": live})
936
+ return out
platform/harness/skills/__init__.py ADDED
@@ -0,0 +1 @@
 
 
1
+ """Portable skills β€” written once against the canonical model, run on any compatible connector."""
platform/harness/skills/sales.py ADDED
@@ -0,0 +1,168 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Sales skill β€” PORTABLE. The de-Odoo'd replacement for modules/sales.py: every metric is expressed
2
+ against the canonical model and runs on any connector with the SALES capability (INVOICING for
3
+ returns). No model names, no `amount_untaxed`, no Odoo β€” the connector translates.
4
+
5
+ `validate(tenant)` parity-checks each metric against the current Odoo module, so a port only replaces
6
+ the original once it ties out to the cent (same discipline as the modules' own validate()).
7
+
8
+ Date windows come from core.periods (calendar math is source-agnostic).
9
+ """
10
+ from __future__ import annotations
11
+ import sys
12
+ from pathlib import Path
13
+ sys.path.insert(0, str(Path(__file__).resolve().parents[2])) # platform/
14
+ import core.periods as P
15
+ from harness.canonical import Capability, Entity, Query, Filter
16
+ from harness.base import Tenant
17
+
18
+ REQUIRES = (Capability.SALES,)
19
+
20
+
21
+ # ------------------------------------------------------------------ tenant scope -> canonical filters
22
+ def _scope(tenant: Tenant, business_unit: str | None):
23
+ """The client's scope CONFIG (was hard-coded Odoo constants) as canonical filters β€” valid on both
24
+ SALES_ORDER and SALES_ORDER_LINE (status/business_unit/customer exist on both)."""
25
+ cfg = tenant.config
26
+ fs = []
27
+ if cfg.get('confirmed_statuses'):
28
+ fs.append(Filter('status', 'in', list(cfg['confirmed_statuses'])))
29
+ bus = cfg.get('business_units') or {}
30
+ if business_unit and business_unit in bus:
31
+ fs.append(Filter('business_unit', 'in', [bus[business_unit]]))
32
+ elif bus:
33
+ fs.append(Filter('business_unit', 'in', list(bus.values())))
34
+ if cfg.get('excluded_customer_ids'):
35
+ fs.append(Filter('customer', 'not_in', list(cfg['excluded_customer_ids'])))
36
+ return tuple(fs)
37
+
38
+
39
+ def _sum(conn, entity, measure, df, dt, filters):
40
+ return sum(r[measure] for r in conn.aggregate(
41
+ Query(entity, (measure,), 'sum', date_from=df, date_to=dt, filters=filters)))
42
+
43
+
44
+ def _count(conn, entity, df, dt, filters):
45
+ return sum(r['count'] for r in conn.aggregate(
46
+ Query(entity, (), 'count', date_from=df, date_to=dt, filters=filters)))
47
+
48
+
49
+ # ------------------------------------------------------------------ metrics (portable)
50
+ def revenue(tenant, date_from, date_to, business_unit=None):
51
+ return _sum(tenant.source(Capability.SALES), Entity.SALES_ORDER, 'amount',
52
+ date_from, date_to, _scope(tenant, business_unit))
53
+
54
+
55
+ def headline(tenant, business_unit=None, t=None):
56
+ conn = tenant.source(Capability.SALES)
57
+ t = t or P.today()
58
+ yf, yt = P.ytd(t)
59
+ lf, lt = P.ytd_last_year(t)
60
+ sc = _scope(tenant, business_unit)
61
+ rev = _sum(conn, Entity.SALES_ORDER, 'amount', yf, yt, sc)
62
+ rev_ly = _sum(conn, Entity.SALES_ORDER, 'amount', lf, lt, sc)
63
+ orders = _count(conn, Entity.SALES_ORDER, yf, yt, sc)
64
+ custs = conn.distinct_count(Query(Entity.SALES_ORDER, date_from=yf, date_to=yt, filters=sc), 'customer')
65
+ return {'ytd_revenue': rev, 'ytd_revenue_ly': rev_ly, 'yoy_pct': P.yoy_pct(rev, rev_ly),
66
+ 'ytd_orders': orders, 'ytd_customers': custs, 'aov': (rev / orders) if orders else 0.0}
67
+
68
+
69
+ def monthly_trend(tenant, n=13, business_unit=None, t=None):
70
+ conn = tenant.source(Capability.SALES)
71
+ t = t or P.today()
72
+ first = P.month_starts(n, t)[0][1]
73
+ span_from = f"{int(first[:4]) - 1:04d}{first[4:]}" # a year before the first shown month
74
+ q = Query(Entity.SALES_ORDER, ('amount',), 'sum', grain='month',
75
+ date_from=span_from, date_to=t.isoformat(), filters=_scope(tenant, business_unit))
76
+ m = {r['period']: r['amount'] for r in conn.aggregate(q)}
77
+ rows = []
78
+ for ym, _s, _e in P.month_starts(n, t):
79
+ y, mo = int(ym[:4]) - 1, int(ym[5:7])
80
+ this, last = m.get(ym, 0.0), m.get(f'{y:04d}-{mo:02d}', 0.0)
81
+ rows.append({'month': ym, 'revenue': this, 'revenue_ly': last, 'yoy_pct': P.yoy_pct(this, last)})
82
+ return rows
83
+
84
+
85
+ def top_customers(tenant, limit=25, business_unit=None, t=None):
86
+ conn = tenant.source(Capability.SALES)
87
+ yf, yt = P.ytd(t or P.today())
88
+ q = Query(Entity.SALES_ORDER, ('amount',), 'sum', group_by=('customer',),
89
+ date_from=yf, date_to=yt, filters=_scope(tenant, business_unit))
90
+ rows = [{'pid': r['customer']['id'], 'customer': r['customer']['name'],
91
+ 'revenue': r['amount'], 'orders': r['count']}
92
+ for r in conn.aggregate(q) if r.get('customer')]
93
+ return sorted(rows, key=lambda x: -x['revenue'])[:limit]
94
+
95
+
96
+ def top_skus(tenant, limit=25, business_unit=None, t=None):
97
+ conn = tenant.source(Capability.SALES)
98
+ yf, yt = P.ytd(t or P.today())
99
+ q = Query(Entity.SALES_ORDER_LINE, ('amount', 'qty', 'margin'), 'sum', group_by=('product',),
100
+ date_from=yf, date_to=yt, filters=_scope(tenant, business_unit))
101
+ rows = []
102
+ for r in conn.aggregate(q):
103
+ if not r.get('product'):
104
+ continue
105
+ lines = r['count'] or 0
106
+ rows.append({'product': r['product']['name'], 'pid': r['product']['id'], 'revenue': r['amount'],
107
+ 'qty': r['qty'], 'margin': r['margin'],
108
+ 'profit_per_order': (r['margin'] / lines) if lines else 0.0})
109
+ return sorted(rows, key=lambda x: -x['revenue'])[:limit]
110
+
111
+
112
+ def by_category(tenant, limit=20, business_unit=None, t=None):
113
+ conn = tenant.source(Capability.SALES)
114
+ yf, yt = P.ytd(t or P.today())
115
+ q = Query(Entity.SALES_ORDER_LINE, ('amount',), 'sum', group_by=('category',),
116
+ date_from=yf, date_to=yt, filters=_scope(tenant, business_unit))
117
+ rows = [{'category': r['category'], 'revenue': r['amount']} for r in conn.aggregate(q)]
118
+ return sorted(rows, key=lambda x: -x['revenue'])[:limit]
119
+
120
+
121
+ def returns_monthly(tenant, n=13, t=None):
122
+ """Returns (credit notes) per month β€” needs INVOICING, consolidated (no BU split)."""
123
+ conn = tenant.source(Capability.INVOICING)
124
+ t = t or P.today()
125
+ first = P.month_starts(n, t)[0][1]
126
+ span_from = f"{int(first[:4]) - 1:04d}{first[4:]}"
127
+ ex = tenant.config.get('excluded_customer_ids') or []
128
+ filters = [Filter('doc_type', 'eq', 'out_refund'), Filter('status', 'eq', 'posted')]
129
+ if ex:
130
+ filters.append(Filter('customer', 'not_in', list(ex)))
131
+ q = Query(Entity.INVOICE, ('amount',), 'sum', grain='month',
132
+ date_from=span_from, date_to=t.isoformat(), filters=tuple(filters))
133
+ m = {r['period']: -r['amount'] for r in conn.aggregate(q)} # out_refund amount is negative -> flip
134
+ rows = []
135
+ for ym, _s, _e in P.month_starts(n, t):
136
+ y, mo = int(ym[:4]) - 1, int(ym[5:7])
137
+ this, last = m.get(ym, 0.0), m.get(f'{y:04d}-{mo:02d}', 0.0)
138
+ rows.append({'month': ym, 'revenue': this, 'revenue_ly': last, 'yoy_pct': P.yoy_pct(this, last)})
139
+ return rows
140
+
141
+
142
+ # ------------------------------------------------------------------ parity validation (port gate)
143
+ def validate(tenant, t=None):
144
+ """Reconcile the ported skill to the current Odoo module β€” a port ships only when it ties out."""
145
+ import modules.sales as M
146
+ t = t or P.today()
147
+ yf, yt = P.ytd(t)
148
+ checks = []
149
+
150
+ def chk(name, a, b, tol=1.0):
151
+ checks.append({'check': name, 'a': round(a, 2), 'b': round(b, 2),
152
+ 'gap': round(a - b, 2), 'ok': abs(a - b) <= tol})
153
+
154
+ h, mh = headline(tenant), M.headline()
155
+ chk('headline YTD revenue == module', h['ytd_revenue'], mh['ytd_revenue'])
156
+ chk('headline YTD orders == module', h['ytd_orders'], mh['ytd_orders'])
157
+ chk('headline YTD customers == module', h['ytd_customers'], mh['ytd_customers'])
158
+ chk('Ξ£ top_customers == module', sum(r['revenue'] for r in top_customers(tenant, 10**9)),
159
+ sum(r['revenue'] for r in M.top_customers(limit=10**9)))
160
+ chk('Ξ£ by_category == module', sum(r['revenue'] for r in by_category(tenant, 10**9)),
161
+ sum(r['revenue'] for r in M.by_category(limit=10**9)))
162
+ chk('Ξ£ top_skus revenue == module', sum(r['revenue'] for r in top_skus(tenant, 10**9)),
163
+ sum(r['revenue'] for r in M.top_skus(limit=10**9)))
164
+ chk('monthly_trend Ξ£ this-yr == module', sum(r['revenue'] for r in monthly_trend(tenant)),
165
+ sum(r['revenue'] for r in M.monthly_trend()))
166
+ chk('returns_monthly Ξ£ this-yr == module', sum(r['revenue'] for r in returns_monthly(tenant)),
167
+ sum(r['revenue'] for r in M.returns_monthly()))
168
+ return checks
platform/harness/tables.py ADDED
@@ -0,0 +1,129 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """harness/tables.py β€” a topic-backed TABLE for the grid (CG-2).
2
+
3
+ The grid's first table (customers) is built from `modules.customer_data.pool()`: one row per
4
+ customer, the whole book shipped as component args, filtered client-side. That works at 1,550
5
+ rows and does not work at line grain β€” `sales_lines` is 201,558 rows / ~89 MB.
6
+
7
+ This module builds the SECOND table: rows come from `semantic.store_rows`, so the fetch is
8
+ WINDOWED and the filtering is SQL. Its whole job is to hand the grid a payload in the same
9
+ host-neutral shape the React component already speaks, plus the one thing a windowed table must
10
+ carry that a whole-book table never needed:
11
+
12
+ counts = {matched, total, shown, windowed}
13
+
14
+ `matched` and `total` come from their own queries over the FULL scope; `shown` is the size of
15
+ this window. The component renders "showing SHOWN of MATCHED (filtered from TOTAL)" from those
16
+ numbers and NEVER from `rows.length` β€” which is the difference between an honest window and a
17
+ silent `[:N]` wearing a total ([[no-unverifiable-aggregates]]).
18
+
19
+ The field CONTRACT is derived from the topic, not hand-written: labels and types come from the
20
+ model's dims and metrics, so a new metric cannot arrive as an untyped, unlabelled column and the
21
+ grid can never disagree with the semantic layer about what a column means.
22
+ """
23
+ from harness import semantic as _sem
24
+
25
+ #: Topic-backed tables the grid can open. `key` is the storage key its saved views live under β€”
26
+ #: separate from the customer table's, so the two tables' views can never collide.
27
+ TOPIC_TABLES = {
28
+ "sales_lines": {
29
+ "label": "Sales lines",
30
+ "storage_key": "sales_lines_table_workspace",
31
+ # the columns worth defaulting to visible, in order; everything else stays available
32
+ "default_visible": ["date", "order_partner", "product", "category", "state", "agent",
33
+ "team", "revenue", "units", "margin"],
34
+ # a line-grain table is unusable sorted by nothing in particular
35
+ "default_sorts": [{"colId": "date", "dir": "desc"}],
36
+ },
37
+ }
38
+
39
+ #: how many rows one window carries. Small enough that the payload stays a few hundred KB at
40
+ #: ~440 B/row, large enough to scroll. The honest counts make the window visible rather than
41
+ #: silent, so this is a performance knob and not a correctness one.
42
+ PAGE_ROWS = 200
43
+
44
+
45
+ def _labels(topic):
46
+ """colId -> human label, taken from the MODEL (dim labels, metric labels)."""
47
+ t = _sem.topics()[topic]
48
+ out = {}
49
+ for key, d in ((t.get("store") or {}).get("dims") or {}).items():
50
+ out[key] = d.get("label") or key.replace("_", " ").title()
51
+ out[f"{key}_id"] = f"{out[key]} id"
52
+ out["date"] = "Date"
53
+ for k, m in _sem.metrics().items():
54
+ if m.get("topic") == topic:
55
+ out[k] = m.get("label") or k
56
+ return out
57
+
58
+
59
+ def table_fields(topic):
60
+ """The grid field contract for a topic, derived from the semantic model.
61
+
62
+ Every column is `source='odoo'` β€” read-only. A line is a fact from the ERP; unlike the
63
+ customer table there is no overlay stratum here (notes/tags hang off a CUSTOMER, not off an
64
+ individual order line), so the two-strata split simply does not arise.
65
+ """
66
+ cfg = TOPIC_TABLES[topic]
67
+ cols = _sem.store_columns(topic, grain="row")
68
+ labels = _labels(topic)
69
+ order = [k for k in cfg["default_visible"] if k in cols]
70
+ order += [k for k in cols if k not in order]
71
+ return [{
72
+ "key": k,
73
+ "label": labels.get(k, k.replace("_", " ").title()),
74
+ "type": cols[k]["type"],
75
+ "source": "odoo",
76
+ "default": k in cfg["default_visible"],
77
+ } for k in order]
78
+
79
+
80
+ def table_payload(topic, config=None, offset=0, limit=None, team_id=None,
81
+ date_from=None, date_to=None):
82
+ """Fetch ONE window of a topic-backed table plus its scope-wide counts and totals.
83
+
84
+ `config` is a grid ViewConfig (filters / filterConj / sorts / search) β€” the same shape the
85
+ customer table persists, so a saved view means the same thing on either table.
86
+
87
+ Returns the component payload: {fields, rows, counts, aggregates, storageKey}.
88
+ """
89
+ if topic not in TOPIC_TABLES:
90
+ raise _sem.ModelError(f"no topic-backed table registered for {topic!r}")
91
+ cfg = TOPIC_TABLES[topic]
92
+ config = config or {}
93
+ limit = PAGE_ROWS if limit is None else limit
94
+
95
+ got = _sem.store_rows(
96
+ topic,
97
+ filter_tree=config.get("filters") or [],
98
+ filter_conj=config.get("filterConj") or "and",
99
+ search=config.get("search") or None,
100
+ sorts=config.get("sorts") or cfg["default_sorts"],
101
+ member_ids=config.get("memberPids") or None,
102
+ limit=limit, offset=offset,
103
+ team_id=team_id, date_from=date_from, date_to=date_to,
104
+ )
105
+
106
+ rows = []
107
+ for i, r in enumerate(got["rows"]):
108
+ # `pid` is the grid's row identity contract (selection, detail, overlay all key on it).
109
+ # At line grain that identity is the order LINE id, which store_rows returns as _rid.
110
+ row = {k: v for k, v in r.items() if k != "_rid"}
111
+ row["pid"] = r["_rid"]
112
+ rows.append(row)
113
+
114
+ return {
115
+ "fields": table_fields(topic),
116
+ "rows": rows,
117
+ # THE honest window. `matched`/`total` are scope-wide queries; `shown` is this window.
118
+ # A caller must never recompute `matched` from len(rows).
119
+ "counts": {
120
+ "shown": len(rows),
121
+ "matched": got["row_count"],
122
+ "total": got["total_count"],
123
+ "windowed": True,
124
+ },
125
+ "aggregates": got["aggregates"], # over the whole FILTERED scope, not the window
126
+ "storageKey": cfg["storage_key"],
127
+ "label": cfg["label"],
128
+ "window": got["window"],
129
+ }