fsanyoto commited on
Commit
092334a
Β·
verified Β·
1 Parent(s): 662dc97

Deploy AIOS web (React glide grid + FastAPI slice)

Browse files
This view is limited to 50 files because it contains too many changes. Β  See raw diff
Files changed (50) hide show
  1. Dockerfile +62 -0
  2. README.md +13 -10
  3. api/aios_session.py +124 -0
  4. api/deps.py +129 -0
  5. api/main.py +240 -0
  6. api/pages.py +362 -0
  7. api/pages_sales.py +270 -0
  8. api/routes_admin.py +325 -0
  9. api/routes_auth.py +143 -0
  10. api/routes_customers.py +208 -0
  11. api/routes_grid.py +155 -0
  12. api/routes_nav.py +34 -0
  13. api/routes_pages.py +65 -0
  14. requirements.txt +12 -0
  15. royalimports_os/aios_grid.py +1221 -0
  16. royalimports_os/core/auth_google.py +91 -0
  17. royalimports_os/core/context.py +59 -0
  18. royalimports_os/core/grid_events.py +1070 -0
  19. royalimports_os/core/links.py +59 -0
  20. royalimports_os/core/management_pl.py +161 -0
  21. royalimports_os/core/odoo.py +201 -0
  22. royalimports_os/core/periods.py +155 -0
  23. royalimports_os/core/perms.py +164 -0
  24. royalimports_os/core/registry.py +137 -0
  25. royalimports_os/core/store.py +307 -0
  26. royalimports_os/core/store_backend.py +85 -0
  27. royalimports_os/core/store_pg.py +249 -0
  28. royalimports_os/core/table_store.py +281 -0
  29. royalimports_os/core/user_tables.py +179 -0
  30. royalimports_os/core/users.py +238 -0
  31. royalimports_os/evals/analyst_golden.yml +104 -0
  32. royalimports_os/harness/__init__.py +3 -0
  33. royalimports_os/harness/analyst.py +265 -0
  34. royalimports_os/harness/base.py +123 -0
  35. royalimports_os/harness/canonical.py +101 -0
  36. royalimports_os/harness/chatlog.py +102 -0
  37. royalimports_os/harness/connectors/__init__.py +1 -0
  38. royalimports_os/harness/connectors/odoo.py +147 -0
  39. royalimports_os/harness/connectors/quickbooks.py +37 -0
  40. royalimports_os/harness/datastore.py +750 -0
  41. royalimports_os/harness/evals.py +134 -0
  42. royalimports_os/harness/filter_sql.py +654 -0
  43. royalimports_os/harness/measure_filter.py +666 -0
  44. royalimports_os/harness/routines.py +259 -0
  45. royalimports_os/harness/runtime.py +186 -0
  46. royalimports_os/harness/semantic.py +936 -0
  47. royalimports_os/harness/skills/__init__.py +1 -0
  48. royalimports_os/harness/skills/sales.py +168 -0
  49. royalimports_os/harness/tables.py +129 -0
  50. royalimports_os/harness/telemetry.py +94 -0
Dockerfile ADDED
@@ -0,0 +1,62 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # AIOS web β€” the React shell + FastAPI over the royalimports_os data layer. THE CANONICAL IMAGE:
2
+ # this is the one that actually runs, and the one a host migration re-points (C1e).
3
+ #
4
+ # β›” WHY A SECOND DOCKERFILE EXISTS (ops/Dockerfile.api), stated here so nobody "tidies" the wrong
5
+ # one away. It is NOT a duplicated service β€” it is the SAME service built from a DIFFERENT BUILD
6
+ # CONTEXT:
7
+ # * HF Spaces: the context is the Space repo root, which `deploy_web.py` uploads FLATTENED β€”
8
+ # `api/`, `web/`, `royalimports_os/`, `requirements.txt` all at top level. Hence the paths here.
9
+ # * ops/compose.yml: the context is the local repo root, where the same files live at
10
+ # `aios-web/api/`, `aios-web/web/`, `royalimports_os/`. Different prefixes, same content.
11
+ # β‡’ THE REAL FIX, and it is a follow-up rather than something to do the day of a first deploy:
12
+ # make `deploy_web.py` upload with the REPO layout so both contexts are identical, then this file
13
+ # serves both and `ops/Dockerfile.api` is deleted. Until then `ops/verify_portability.py` gates
14
+ # the two against each other on every property that matters (same app object, same launcher
15
+ # contract, non-root, healthchecked, no baked secrets), so they cannot drift silently.
16
+ #
17
+ # Two stages: node builds the React bundle, python serves it + the JSON API. Login is the app's own
18
+ # branded page over a signed session cookie (wave 1 removed the HTTP Basic gate).
19
+
20
+ # ---- stage 1: build the React bundle ----
21
+ FROM node:22-slim AS web
22
+ WORKDIR /web
23
+ # Resolve deps FRESH from the public registry (the local lockfile pinned a mirror-only lodash
24
+ # 4.18.1 that isn't on npmjs.org; a fresh install gets the public 4.17.x that glide accepts).
25
+ COPY web/package.json ./
26
+ RUN npm install --no-audit --no-fund --no-package-lock
27
+ COPY web/ ./
28
+ RUN rm -f package-lock.json && npm run build # -> /web/dist
29
+
30
+ # ---- stage 2: python API serving the bundle ----
31
+ FROM python:3.12-slim
32
+ WORKDIR /app
33
+ COPY requirements.txt ./
34
+ RUN pip install --no-cache-dir -r requirements.txt
35
+ COPY api/ ./api/
36
+ COPY royalimports_os/ ./royalimports_os/ # the reused data layer (modules/core/harness)
37
+ COPY --from=web /web/dist ./web/dist
38
+
39
+ # Run unprivileged β€” the C2 security baseline, and matched to ops/Dockerfile.api so the two images
40
+ # do not differ on a property that would only surface after a migration. uid 1000 is what HF
41
+ # Spaces expects.
42
+ RUN useradd --create-home --uid 1000 user && chown -R user:user /app
43
+ USER user
44
+
45
+ ENV RI_DIR=/app/royalimports_os \
46
+ PORT=7860 \
47
+ PYTHONUNBUFFERED=1
48
+ # β›” AIOS_INSECURE_SSL is deliberately NOT set. It disables TLS verification for every outbound
49
+ # call in the process and exists for one developer laptop. ops/verify_portability.py fails if any
50
+ # deploy config sets it.
51
+ EXPOSE 7860
52
+
53
+ # Liveness via stdlib rather than curl β€” it keeps the image free of an apt layer that exists only
54
+ # for the healthcheck.
55
+ HEALTHCHECK --interval=30s --timeout=5s --start-period=60s --retries=3 \
56
+ CMD python -c "import os,urllib.request,sys; sys.exit(0 if urllib.request.urlopen(f'http://127.0.0.1:{os.environ.get(\"PORT\",\"7860\")}/api/health',timeout=4).status==200 else 1)"
57
+
58
+ # ⚠ THE PORT COMES FROM THE ENVIRONMENT, ALWAYS. The application itself reads PORT nowhere β€” it is
59
+ # a launcher argument β€” which is what makes moving hosts a config change instead of a code change.
60
+ # Secrets (Odoo creds, APP_PASSWORD, AIOS_SESSION_SECRET, APP_BASE_URL) arrive as env at RUN time
61
+ # and are never baked; the repo-root .dockerignore drops **/.env from the build context.
62
+ CMD ["sh", "-c", "uvicorn api.main:app --host 0.0.0.0 --port ${PORT}"]
README.md CHANGED
@@ -1,10 +1,13 @@
1
- ---
2
- title: Loopable
3
- emoji: πŸš€
4
- colorFrom: red
5
- colorTo: gray
6
- sdk: docker
7
- pinned: false
8
- ---
9
-
10
- Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
 
 
 
 
1
+ ---
2
+ title: Loopable
3
+ emoji: πŸ—‚
4
+ colorFrom: indigo
5
+ colorTo: gray
6
+ sdk: docker
7
+ app_port: 7860
8
+ pinned: false
9
+ ---
10
+
11
+ Loopable β€” the AIOS platform shell: a React front end (Airtable-grade grid + dashboard pages)
12
+ over a FastAPI service reusing the royalimports_os data layer. Login-gated: a branded login page over an HMAC-signed session cookie, with per-user
13
+ business-unit scope and module grants. Separate from the Streamlit app.
api/aios_session.py ADDED
@@ -0,0 +1,124 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """aios_session.py β€” X3: the signed, STATELESS session cookie (EXIT-3a, 2026-07-30).
2
+
3
+ aios_session = base64url(json {t,u,e,i,x}) . base64url(HMAC-SHA256)
4
+
5
+ `t` tenant slug Β· `u` username Β· `e` the user's revocation epoch Β· `i` idle deadline Β·
6
+ `x` absolute expiry. Signed with `AIOS_SESSION_SECRET` using stdlib `hmac` β€” no new dependency,
7
+ and nothing to store server-side, which is the whole point: EXIT-4 requires a process that holds
8
+ no per-tenant state, and a session TABLE is per-tenant state. D2's Postgres sessions-MIRROR (an
9
+ audit view, not the source of truth) arrives with C-2 and is deliberately deferred.
10
+
11
+ WHY STATELESS STILL REVOKES. A cookie nobody can delete sounds like a cookie nobody can revoke.
12
+ The answer is the `epoch` integer on the user record (`core.users.epoch` / `bump_epoch`): the
13
+ cookie carries the epoch it was minted under, every verification re-reads the account's current
14
+ epoch, and a password change or a deactivation bumps it β€” so every outstanding cookie for that
15
+ user dies at once, without a session table.
16
+
17
+ ⚠ NO FALLBACK SECRET, EVER. With `AIOS_SESSION_SECRET` unset this module mints a RANDOM
18
+ per-process key: sessions then die on restart, which is an inconvenience. A hardcoded default
19
+ would instead let anyone who has read this repo forge a cookie for any user of any tenant β€” an
20
+ inconvenience is the correct trade against a forgery key. `EPHEMERAL_SECRET` records which
21
+ happened so the app can say so at startup and a gate can assert it.
22
+ """
23
+ import base64
24
+ import hmac
25
+ import json
26
+ import os
27
+ import secrets
28
+ import time
29
+ from hashlib import sha256
30
+
31
+ COOKIE_NAME = "aios_session"
32
+ IDLE_SECONDS = 8 * 60 * 60 # D2: reissued on every authenticated request
33
+ ABSOLUTE_SECONDS = 30 * 24 * 60 * 60 # D2: a hard ceiling no amount of activity extends
34
+
35
+ _env_secret = os.environ.get("AIOS_SESSION_SECRET") or ""
36
+ EPHEMERAL_SECRET = not _env_secret
37
+ _SECRET = _env_secret.encode("utf-8") if _env_secret else secrets.token_bytes(32)
38
+
39
+
40
+ def _b64(raw: bytes) -> str:
41
+ return base64.urlsafe_b64encode(raw).decode("ascii").rstrip("=")
42
+
43
+
44
+ def _unb64(txt: str) -> bytes:
45
+ return base64.urlsafe_b64decode(txt + "=" * (-len(txt) % 4))
46
+
47
+
48
+ def _sign(payload: bytes) -> str:
49
+ return _b64(hmac.new(_SECRET, payload, sha256).digest())
50
+
51
+
52
+ def mint(tenant, username, epoch, now=None, absolute_expiry=None):
53
+ """A fresh cookie value. `absolute_expiry` carries over on reissue so refreshing the idle
54
+ window can never extend the 30-day ceiling β€” that would make the ceiling unreachable."""
55
+ now = int(now if now is not None else time.time())
56
+ claims = {"t": str(tenant), "u": str(username), "e": int(epoch),
57
+ "i": now + IDLE_SECONDS,
58
+ "x": int(absolute_expiry if absolute_expiry is not None
59
+ else now + ABSOLUTE_SECONDS)}
60
+ payload = json.dumps(claims, separators=(",", ":"), sort_keys=True).encode("utf-8")
61
+ return f"{_b64(payload)}.{_sign(payload)}", claims
62
+
63
+
64
+ def read(value, now=None):
65
+ """Parse + verify a cookie value β†’ claims dict, or None.
66
+
67
+ None for EVERY failure mode β€” bad shape, bad signature, expired, unparseable. The caller
68
+ answers 401 and must not learn which: an error that distinguishes "signature wrong" from
69
+ "expired" tells a forger whether their key is right.
70
+ """
71
+ if not value or not isinstance(value, str) or value.count(".") != 1:
72
+ return None
73
+ body, sig = value.split(".", 1)
74
+ try:
75
+ payload = _unb64(body)
76
+ except Exception:
77
+ return None
78
+ # compare_digest on the SIGNATURE, always β€” a byte-wise early exit is a timing oracle.
79
+ if not hmac.compare_digest(sig, _sign(payload)):
80
+ return None
81
+ try:
82
+ claims = json.loads(payload.decode("utf-8"))
83
+ except Exception:
84
+ return None
85
+ if not isinstance(claims, dict):
86
+ return None
87
+ if not isinstance(claims.get("u"), str) or not isinstance(claims.get("t"), str):
88
+ return None
89
+ if not claims["u"] or not claims["t"]:
90
+ return None
91
+ try:
92
+ idle, absolute, epoch = int(claims["i"]), int(claims["x"]), int(claims["e"])
93
+ except (KeyError, TypeError, ValueError):
94
+ return None
95
+ now = int(now if now is not None else time.time())
96
+ if now >= idle or now >= absolute:
97
+ return None
98
+ claims["e"], claims["i"], claims["x"] = epoch, idle, absolute
99
+ return claims
100
+
101
+
102
+ def is_secure_request(request):
103
+ """Secure flag ON unless this is plain-HTTP localhost. Set unconditionally in production;
104
+ unset on a local http:// dev origin, where a Secure cookie is simply never sent back and
105
+ login would appear to succeed and then not work."""
106
+ host = (request.headers.get("host") or "").split(":")[0].lower()
107
+ proto = (request.headers.get("x-forwarded-proto")
108
+ or request.url.scheme or "http").split(",")[0].strip().lower()
109
+ if proto == "https":
110
+ return True
111
+ return host not in ("localhost", "127.0.0.1", "[::1]", "::1")
112
+
113
+
114
+ def set_cookie(response, request, value):
115
+ response.set_cookie(
116
+ COOKIE_NAME, value, httponly=True, samesite="lax",
117
+ secure=is_secure_request(request), max_age=ABSOLUTE_SECONDS, path="/")
118
+
119
+
120
+ def clear_cookie(response, request):
121
+ # Same attributes as when it was set: a browser matches on path/secure/samesite when
122
+ # deleting, and a mismatched delete silently leaves the cookie in place.
123
+ response.delete_cookie(COOKIE_NAME, path="/", httponly=True, samesite="lax",
124
+ secure=is_secure_request(request))
api/deps.py ADDED
@@ -0,0 +1,129 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """deps.py β€” the request-scoped session + tenant resolution every v1 route depends on.
2
+
3
+ ONE place answers "who is asking, for which tenant, and what may they see?", because a
4
+ permission check that exists in four routes has four chances to be forgotten. The routes take
5
+ `session: Session = Depends(require_session)` and receive an object that has ALREADY failed
6
+ closed if anything was wrong.
7
+
8
+ FAIL-CLOSED, spelled out (X2): no cookie β†’ **401**. Cookie present but unverifiable, expired,
9
+ epoch-revoked, or naming an unknown tenant β†’ **401**. Authenticated but not granted the surface β†’
10
+ **403**. Never an empty 200: an empty list is a legitimate answer meaning "no rows", and using it
11
+ to mean "you are not allowed" is how a permission bug becomes invisible.
12
+ """
13
+ import os
14
+ import sys
15
+ from dataclasses import dataclass
16
+ from pathlib import Path
17
+
18
+ from fastapi import Depends, HTTPException, Request, Response
19
+
20
+ import aios_session
21
+
22
+ _RI = Path(os.environ.get("RI_DIR") or Path(__file__).resolve().parents[2] / "royalimports_os")
23
+ if str(_RI) not in sys.path:
24
+ sys.path.insert(0, str(_RI))
25
+
26
+ import core.perms as perms # noqa: E402
27
+ import core.users as users # noqa: E402
28
+ from harness import runtime # noqa: E402
29
+
30
+
31
+ def err(status, code, message):
32
+ """The one error shape (X2): non-2xx JSON `{"error": {"code", "message"}}`."""
33
+ return HTTPException(status_code=status, detail={"error": {"code": code, "message": message}})
34
+
35
+
36
+ @dataclass
37
+ class Session:
38
+ """A verified request identity. `user` is the PUBLIC record (never a hash or a salt)."""
39
+ tenant: str
40
+ user: dict
41
+ claims: dict
42
+ runtime: object
43
+
44
+ @property
45
+ def uname(self):
46
+ return self.user.get("username", "")
47
+
48
+ @property
49
+ def admin(self):
50
+ return perms.is_admin(self.user)
51
+
52
+ def require(self, module_key):
53
+ """403 unless this session may open `module_key`. Returns nothing β€” it is a gate."""
54
+ if not perms.may_open(self.user, module_key):
55
+ raise err(403, "forbidden", f"your account does not have access to {module_key}")
56
+
57
+
58
+ def _user_for(claims):
59
+ """The account named by a verified cookie, or None β€” with the epoch check that makes a
60
+ stateless cookie revocable.
61
+
62
+ β›” THE RULE THIS FUNCTION MUST OBEY: **it has to accept exactly the identities `users.verify`
63
+ issues.** Any divergence produces the worst failure shape there is β€” a login that returns 200
64
+ with a cookie and then 401s every request after it. This function got that wrong once (the
65
+ emergency-master branch below ran only when NO record existed, while `verify` grants the master
66
+ identity even when one does), so it is now written as a deliberate mirror of `verify`'s
67
+ structure: try the record, else the master.
68
+
69
+ TWO CASES, in `verify`'s own order:
70
+ * THE RECORD, if it is readable and satisfies everything a session adds on top of a login:
71
+ the account is active, and its CURRENT epoch equals the cookie's β€” otherwise the session
72
+ is revoked (a password change or a deactivation bumped it).
73
+ * THE EMERGENCY MASTER β€” username 'admin' with APP_PASSWORD configured β€” exactly as
74
+ `verify`'s last branch grants it, whether or not a record exists. This is what stops a
75
+ store outage, a self-deactivation or a bumped epoch locking the owner out of the product.
76
+ ⚠ AND IT MEANS AN `admin` SESSION IS NOT EPOCH-REVOCABLE while APP_PASSWORD is set. That
77
+ is not a weakening: whoever holds APP_PASSWORD can simply log in again, so bumping admin's
78
+ epoch never revoked them in the first place. Rotating APP_PASSWORD is how you revoke it.
79
+ Every OTHER account is fully epoch-revocable, which is asserted in `verify_api.py`.
80
+ * anything else β†’ None. An unknown username is NOT admitted just because its signature was
81
+ valid: a signature proves the cookie is ours, not that the account still exists.
82
+ """
83
+ uname = (claims.get("u") or "").strip().lower()
84
+ if not uname:
85
+ return None
86
+ try:
87
+ reg = users.registry() or {}
88
+ except Exception:
89
+ reg = {}
90
+ rec = reg.get(uname)
91
+ if (rec and rec.get("active", True)
92
+ and int(rec.get("epoch") or 0) == int(claims.get("e") or 0)):
93
+ return users._public(uname, rec)
94
+ if uname == "admin" and os.environ.get("APP_PASSWORD", ""):
95
+ return {"username": "admin", "name": "Administrator", "role": "admin",
96
+ "bus": "all", "modules": "all", "epoch": int(claims.get("e") or 0)}
97
+ return None
98
+
99
+
100
+ def require_session(request: Request, response: Response) -> Session:
101
+ """Verify the cookie, resolve the tenant, and REISSUE the cookie (the 8h idle window is
102
+ refreshed on use; the 30-day absolute expiry is carried over, never extended)."""
103
+ raw = request.cookies.get(aios_session.COOKIE_NAME)
104
+ if not raw:
105
+ raise err(401, "no_session", "sign in to continue")
106
+ claims = aios_session.read(raw)
107
+ if not claims:
108
+ raise err(401, "invalid_session", "your session has expired β€” sign in again")
109
+ user = _user_for(claims)
110
+ if not user:
111
+ raise err(401, "invalid_session", "your session has expired β€” sign in again")
112
+ try:
113
+ rt = runtime.get_runtime(claims["t"])
114
+ except KeyError:
115
+ # An unknown tenant slug is a 401, not a 404: the cookie is not a valid identity for this
116
+ # deployment, and confirming which slugs exist would answer a question we were not asked.
117
+ raise err(401, "invalid_session", "your session has expired β€” sign in again")
118
+ fresh, _ = aios_session.mint(claims["t"], user["username"], int(user.get("epoch") or 0),
119
+ absolute_expiry=claims["x"])
120
+ aios_session.set_cookie(response, request, fresh)
121
+ return Session(tenant=claims["t"], user=user, claims=claims, runtime=rt)
122
+
123
+
124
+ def module_gate(module_key):
125
+ """A dependency that 401s without a session and 403s without the grant for `module_key`."""
126
+ def _dep(session: Session = Depends(require_session)) -> Session:
127
+ session.require(module_key)
128
+ return session
129
+ return _dep
api/main.py ADDED
@@ -0,0 +1,240 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """AIOS web API β€” the ONE shared, stateless process the Streamlit exit is aimed at.
2
+
3
+ It REUSES the `royalimports_os/` data layer verbatim (nothing re-implemented): the canonical,
4
+ reconciled Odoo model stays server-side and the browser only ever sees derived JSON. Serves the
5
+ JSON API under `/api/*` and the built React bundle (`aios-web/web/dist`) at `/`.
6
+
7
+ WHAT CHANGED IN EXIT WAVE 1 (2026-07-30) β€” three things, all of them load-bearing:
8
+
9
+ * **Real sessions replace HTTP Basic.** The whole app used to sit behind one shared
10
+ `APP_PASSWORD` with the browser's native Basic prompt, which means every caller was the same
11
+ anonymous principal and no route could scope anything. Now a signed stateless cookie carries
12
+ a real `core/users` identity (X3), and every v1 route resolves BU + own-book scope from the
13
+ user RECORD. `/api/health` stays unauthenticated (liveness only).
14
+ * **The overlay fork is deleted.** `aios-web/api/data/overlay.json` was a second writable home
15
+ for user-owned fields the Streamlit app keeps in the tenant store. ONE store now (C1c).
16
+ * **The SSL shim is env-gated.** It used to run at import, unconditionally.
17
+
18
+ ⚠ TENANT RESOLUTION IS PER REQUEST (X7). Nothing tenant-shaped is held at module level; the
19
+ session's tenant claim resolves through `harness.runtime.get_runtime`, which is LRU-bounded. This
20
+ is the rule the ~30–80 MB/tenant target depends on β€” EXIT-0 measured the alternative at a ~0.6 GB
21
+ commit FLOOR per tenant, duplicated within 2% per tenant, nothing shared.
22
+ """
23
+ import os
24
+ import sys
25
+ from pathlib import Path
26
+
27
+ # --- the local Odoo SSL quirk, now BEHIND A GATE -------------------------------------------------
28
+ # The Windows trust store reports the (valid) Odoo cert as expired, so local runs need an
29
+ # unverified default context; the HF Space and the container are unaffected. This used to run
30
+ # unconditionally at import β€” i.e. the shipped container disabled TLS verification for every
31
+ # outbound HTTPS call it ever made, to Odoo and to everything else, forever. That is a
32
+ # man-in-the-middle away from being someone else's data. It is now opt-in, must be set
33
+ # deliberately, and is never on by default.
34
+ # β›” NEVER set AIOS_INSECURE_SSL in a deployed environment. It exists for one developer laptop.
35
+ if os.environ.get("AIOS_INSECURE_SSL") == "1":
36
+ import ssl
37
+ ssl._create_default_https_context = ssl._create_unverified_context # noqa: S323
38
+
39
+ # --- reuse the tenant #0 data layer verbatim. RI_DIR overrides the location in the container
40
+ # (where the layout differs from the local sibling-dir default). ---
41
+ _HERE = Path(__file__).resolve()
42
+ _RI = Path(os.environ.get("RI_DIR") or (_HERE.parents[2] / "royalimports_os"))
43
+ for _p in (str(_RI), str(_HERE.parent)):
44
+ if _p not in sys.path:
45
+ sys.path.insert(0, _p)
46
+
47
+ from dotenv import load_dotenv # noqa: E402
48
+ load_dotenv(_RI / ".env") # Odoo creds + APP_PASSWORD, git-ignored, never committed/printed
49
+
50
+ from fastapi import Body, Depends, FastAPI, Request # noqa: E402
51
+ from fastapi.responses import JSONResponse # noqa: E402
52
+ from fastapi.staticfiles import StaticFiles # noqa: E402
53
+ from starlette.exceptions import HTTPException as StarletteHTTPException # noqa: E402
54
+
55
+ import aios_session # noqa: E402
56
+ import routes_admin # noqa: E402
57
+ import routes_auth # noqa: E402
58
+ import routes_customers # noqa: E402
59
+ import routes_grid # noqa: E402
60
+ import routes_nav # noqa: E402
61
+ import routes_pages # noqa: E402
62
+ from core import grid_events # noqa: E402
63
+ from deps import Session, module_gate # noqa: E402
64
+
65
+ _WEB_DIST = _HERE.parents[1] / "web" / "dist"
66
+
67
+ # --- THE CANONICAL FIELD CONTRACT: a startup assertion, and the referee gate's subject ----------
68
+ # Every field tags its semantic TYPE and its SOURCE. `source='odoo'` is READ-ONLY (Odoo is never
69
+ # written); `source='overlay'` is the editable stratum that lives OUTSIDE Odoo. Loaded from the
70
+ # ONE canonical file shared with the embedded host (`royalimports_os/aios_grid_fields.json`), so
71
+ # embed == standalone by construction.
72
+ #
73
+ # TWO REASONS THIS IS HERE and not folded into the routes:
74
+ # 1. It FAILS LOUDLY at import when the file is missing β€” which means the deploy or the RI_DIR
75
+ # layout is broken, and discovering that from a 500 on the first customer request instead of
76
+ # at startup costs a debugging session. (The pre-wave main.py had this guard; the EXIT-2a
77
+ # rewrite dropped it and this restores it.)
78
+ # 2. `aios-web/verify_fields_contract.py` β€” the cross-side REFEREE β€” reads `FIELDS` and
79
+ # `_PASSTHROUGH_KEYS` from this module to prove the canonical file, `aios_grid.py` and this
80
+ # API have not drifted. The rewrite removed them and the referee went red; retargeting the
81
+ # gate would have been the wrong repair ([[gate-can-report-green-on-nothing]]: retarget, do
82
+ # not delete β€” but only when the subject genuinely moved. Here it should not have moved).
83
+ #
84
+ # ⚠ THE ROUTES SERVE A SUPERSET OF THIS. `routes_customers._payload` derives its field list from
85
+ # `aios_grid.fields_from_workspace(ws)`, which is `FIELDS` PLUS the session user's own custom_ and
86
+ # measure_ columns β€” the same list the Streamlit host renders. That is the point: the standalone
87
+ # shell now sees the user's own columns instead of the bare base contract. `FIELDS` is the
88
+ # canonical FLOOR, asserted below to be exactly what `aios_grid` starts from.
89
+ import json as _json_contract # noqa: E402
90
+ _FIELDS_PATH = _RI / "aios_grid_fields.json"
91
+ if not _FIELDS_PATH.is_file():
92
+ raise FileNotFoundError(
93
+ f"AIOS web API: canonical field contract missing at {_FIELDS_PATH}. Set RI_DIR to the "
94
+ "royalimports_os root (it also carries the data layer this API imports)."
95
+ )
96
+ _contract = _json_contract.loads(_FIELDS_PATH.read_text(encoding="utf-8"))
97
+ FIELDS = _contract["fields"] if isinstance(_contract, dict) else _contract
98
+ # text/status/date pass through untouched; every OTHER odoo field is numeric -> rounded. Derived
99
+ # from field TYPE (not a hand-kept key list) so a new text/date field can never be wrongly rounded.
100
+ _PASSTHROUGH_KEYS = {f["key"] for f in FIELDS
101
+ if f["source"] == "odoo" and f["type"] in ("text", "status", "date")}
102
+
103
+ app = FastAPI(title="AIOS web API")
104
+
105
+
106
+ @app.exception_handler(StarletteHTTPException)
107
+ def _error_shape(request: Request, exc: StarletteHTTPException):
108
+ """ONE error shape for every non-2xx (X2): `{"error": {"code", "message"}}`.
109
+
110
+ `deps.err` already raises detail in that shape; anything FastAPI raises on its own (a 404, a
111
+ 422 from a malformed path param) is wrapped here so a client never has to branch on two
112
+ different error bodies.
113
+ """
114
+ detail = exc.detail
115
+ if isinstance(detail, dict) and "error" in detail:
116
+ body = detail
117
+ else:
118
+ body = {"error": {"code": f"http_{exc.status_code}", "message": str(detail)}}
119
+ return JSONResponse(body, status_code=exc.status_code,
120
+ headers=getattr(exc, "headers", None))
121
+
122
+
123
+ @app.exception_handler(grid_events.StoreUnavailable)
124
+ def _store_unavailable(request: Request, exc: grid_events.StoreUnavailable):
125
+ """THE SAFETY NET for "the store is down" β†’ 503, from anywhere.
126
+
127
+ ⚠ WHY THIS IS APP-LEVEL AND NOT A `try` PER ROUTE. It was a try per route first, and a
128
+ `StoreUnavailable` raised while BUILDING THE PAYLOAD β€” before the route reached its own
129
+ try/except β€” surfaced as a 500. A store outage is a normal operational state and every route
130
+ here touches the store at least twice (the workspace read, then the write), so "remember to
131
+ wrap it" is a rule that gets forgotten once and then reports the wrong thing. One handler
132
+ means a store outage can only ever be a 503, whichever call raised it.
133
+
134
+ The seam raises this only when the caller passed no `fallback_ws` β€” i.e. exactly on this
135
+ adapter, which has no durable session dict to degrade into. A 200 over a write that
136
+ evaporated is the failure the whole rule exists to prevent.
137
+ """
138
+ return JSONResponse(
139
+ {"error": {"code": "store_unavailable",
140
+ "message": "the tenant store is unavailable β€” no change was saved"}},
141
+ status_code=503)
142
+
143
+
144
+ @app.get("/api/health")
145
+ def health():
146
+ """Unauthenticated LIVENESS only β€” it must answer before anyone can sign in, so it may not
147
+ reveal anything about the deployment beyond "the process is up". No version, no tenant list,
148
+ no config: a health endpoint is the one URL every scanner finds first."""
149
+ return {"ok": True}
150
+
151
+
152
+ app.include_router(routes_auth.router)
153
+ app.include_router(routes_nav.router)
154
+ app.include_router(routes_customers.router)
155
+ app.include_router(routes_grid.router)
156
+ # EXIT wave 2: the Y1 page-data envelope (one route for every ported dashboard) and Y4's user
157
+ # administration. `routes_pages` imports `pages`, which lazily imports each `pages_*` builder β€” so
158
+ # a new page is a builder module plus one registry line, and nothing here changes.
159
+ app.include_router(routes_pages.router)
160
+ app.include_router(routes_admin.router)
161
+
162
+
163
+ # --- DEPRECATED ALIASES (removed when S2's shell flips; kept so the current bundle keeps working)
164
+ # They are the v1 handlers with the v1 session requirement β€” NOT the old unauthenticated Basic
165
+ # behavior. An alias that kept the old auth would be a bypass of everything above it.
166
+ _GATE = module_gate(routes_customers.MODULE)
167
+
168
+
169
+ @app.get("/api/customers", deprecated=True)
170
+ def _customers_alias(session: Session = Depends(_GATE)):
171
+ return routes_customers._payload(session)
172
+
173
+
174
+ @app.patch("/api/customers/{pid}", deprecated=True)
175
+ def _patch_alias(pid: int, body: dict = Body(default=None),
176
+ session: Session = Depends(_GATE)):
177
+ return routes_customers.patch_customer(pid, body, session)
178
+
179
+
180
+ def _assert_contract_floor():
181
+ """The canonical FIELDS must be exactly what `aios_grid` starts an empty workspace from.
182
+
183
+ This is what stops `FIELDS` becoming a constant that exists only to satisfy a gate. If the
184
+ canonical JSON and `aios_grid.fields_from_workspace({})` ever disagree, the standalone API and
185
+ the embedded host are serving two different schemas and the grid's own contract has forked β€”
186
+ the exact drift `verify_fields_contract.py` was written to catch, now also caught at startup
187
+ on whatever machine is actually running.
188
+ """
189
+ import aios_grid
190
+ base = [f for f in aios_grid.fields_from_workspace({}) if not f.get("custom")]
191
+ if [f["key"] for f in base] != [f["key"] for f in FIELDS]:
192
+ raise RuntimeError(
193
+ "AIOS web API: the canonical field contract and aios_grid.fields_from_workspace({}) "
194
+ "disagree on the base field set β€” embed and standalone would serve different schemas. "
195
+ "Run aios-web/verify_fields_contract.py.")
196
+
197
+
198
+ def _startup_notes():
199
+ """Say the two things an operator must know, once, at import β€” never in a response body."""
200
+ if aios_session.EPHEMERAL_SECRET:
201
+ print("[aios-api] AIOS_SESSION_SECRET is not set β€” signing with a random per-process "
202
+ "key. Sessions will not survive a restart and will not work across workers. "
203
+ "Set it in production.")
204
+ if os.environ.get("AIOS_INSECURE_SSL") == "1":
205
+ print("[aios-api] AIOS_INSECURE_SSL=1 β€” TLS verification is DISABLED for outbound "
206
+ "requests. Local development only; never in a deployed environment.")
207
+
208
+
209
+ _assert_contract_floor()
210
+ _startup_notes()
211
+
212
+
213
+ # β›” THE STATIC MOUNT IS NOW UNAUTHENTICATED, and that is a change this wave made on purpose.
214
+ # Before EXIT-3a, `BasicAuth` middleware gated the WHOLE app including this mount. A branded login
215
+ # page cannot live behind a password prompt, so the shell's own assets must be public β€” which they
216
+ # are: `index.html`, the JS/CSS bundle and the favicon reveal nothing.
217
+ #
218
+ # ⚠ WHAT THAT SILENTLY DE-GATED, caught in review rather than in production. `web/dist/` also
219
+ # carries `sample_customers.json`, a DEV FIXTURE copied from `web/public/`. Its 8 customers are
220
+ # synthetic, but the `agent` column holds REAL EMPLOYEE NAMES, and it went from Basic-gated to
221
+ # publicly fetchable in this commit. Nothing needs it: both the API bridge and `useCustomerData`
222
+ # deleted their sample fallback on purpose ("NOTHING HERE FALLS BACK TO sample_customers.json"),
223
+ # and it survives only because it sits in `web/public/`. So it is refused here β€” 404, the same
224
+ # answer as any other path that is not part of the app.
225
+ #
226
+ # The right long-term fix is deleting it from `web/public/` (S2's lane β€” flagged in the mailbox);
227
+ # this guard is what makes the API safe regardless of what the bundle happens to contain.
228
+ _DEV_FIXTURES = {"sample_customers.json"}
229
+
230
+
231
+ class _AppStatic(StaticFiles):
232
+ async def get_response(self, path, scope):
233
+ if Path(path).name in _DEV_FIXTURES:
234
+ raise StarletteHTTPException(status_code=404, detail="Not Found")
235
+ return await super().get_response(path, scope)
236
+
237
+
238
+ # static bundle LAST so /api/* wins; html=True serves index.html at / plus the built assets
239
+ if _WEB_DIST.is_dir():
240
+ app.mount("/", _AppStatic(directory=str(_WEB_DIST), html=True), name="web")
api/pages.py ADDED
@@ -0,0 +1,362 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """pages.py β€” Y1: the page-data envelope, and the ONE place its rules live (W2-2).
2
+
3
+ THE POINT OF THIS FILE IS WAVE 3. Collections and Procurement must land by adding a BUILDER, not
4
+ by inventing a shape β€” so every rule the envelope has (the closed `fmt` enum, the drill grammar,
5
+ the no-silent-caps contract, how BU scope is resolved) lives here as a constructor a builder calls,
6
+ never as a convention a builder is expected to remember. A rule that exists only in a docstring is
7
+ a rule the second page gets wrong.
8
+
9
+ WHAT A BUILDER OWES, and nothing more:
10
+ metrics(team_id, granularity, today) -> a dict of plain data, SCOPE-SHAPED ONLY
11
+ blocks(metrics, team_id, granularity) -> [block, ...] built with the constructors below
12
+ TITLE / SUBTITLE / MODULE / CONTROLS
13
+ `build_envelope` does the rest: it resolves the BU, caches the metrics on the SCOPE key, assembles
14
+ the per-request half, and returns the envelope.
15
+
16
+ β›” THE CACHE LINE, WHICH IS WHERE WAVE 1 SHIPPED A CROSS-USER LEAK. What may be cached on a
17
+ `(page, team_id, doc_mode, day)` key is the BUILDER'S METRICS β€” the answer to a question two users
18
+ with the same scope are asking identically. What may NEVER be cached there is anything with a
19
+ per-USER stratum, and on this envelope that is `controls` (its `bu.options` come from
20
+ `perms.allowed_bu_labels(user)`, i.e. from the user RECORD). Wave 1's defect was exactly this shape
21
+ one layer down: a whole payload cached on a scope key while carrying per-user fields and overlay
22
+ cells. It survived 129 green checks because every fixture user had a distinct scope pair. So the
23
+ split is structural here, not remembered: `metrics()` cannot see the session at all β€” it takes
24
+ `team_id`, and `verify_api.py` asserts the cached object holds nothing user-shaped.
25
+
26
+ ⚠ THE CLIENT FORMATS. The server never sends a pre-formatted number, so one formatter change fixes
27
+ every page β€” and `fmt` is a CLOSED enum for the same reason. Two vocabularies exist and they are
28
+ not interchangeable: `FMTS` for kpi/table/validation, and viz `FieldType` for a chart block's
29
+ `fields[]` (those objects are fed straight into `chartData`). `money` vs `currency` is the trap.
30
+ """
31
+ import time
32
+
33
+ # `deps` FIRST, deliberately: it is what puts `royalimports_os/` on `sys.path`, so importing it
34
+ # ahead of any `core.*` makes this module importable in whatever order a caller reaches it.
35
+ from deps import Session, err, perms # noqa: I001
36
+
37
+ import core.context as ctxlib # noqa: E402
38
+
39
+ # ── the two vocabularies ─────────────────────────────────────────────────────────────────────────
40
+ #: The Y1 client-format enum for kpi / table / validation values. CLOSED on purpose: a page that
41
+ #: needs a new one adds it HERE and to the client's `fmt` at the same time, which is the property
42
+ #: that keeps one formatter authoritative. A table column with NO `fmt` renders verbatim β€” that is
43
+ #: how a text column is expressed without opening the enum.
44
+ FMTS = ("money", "num", "pct", "int", "date")
45
+
46
+ #: The viz `FieldType` values a chart block's `fields[]` may use β€” a DIFFERENT vocabulary, because
47
+ #: `chartData(spec, rows, fieldByKey)` consumes those objects directly. `currency`, not `money`.
48
+ VIZ_TYPES = ("text", "status", "currency", "int", "pct", "date")
49
+
50
+ #: `chartData`'s own display cap (`MAX_BUCKETS`). A payload must fit it at the SOURCE: the engine
51
+ #: truncates past 12 and, on a date axis, sorts ASCENDING first β€” so a 13-month series loses the
52
+ #: CURRENT month, the one anybody actually looks at. Mirrored here so a builder can size its own
53
+ #: series without importing anything from the client tree.
54
+ MAX_BUCKETS = 12
55
+
56
+ #: Drill kinds the client is expected to understand. `decomp` names a WINDOW + a scope; the others
57
+ #: name an ENTITY. ⚠ THIS WAVE THEY ARE DECLARATIONS β€” there is no resolver route (Β§5 Y1 amendment
58
+ #: 4). Each descriptor is COMPLETE, so wave 3 adds the resolver with no shape change, and Y6's
59
+ #: panel must say it is thin rather than look finished.
60
+ DRILL_KINDS = ("decomp", "customer", "sku", "rep", "bu")
61
+
62
+
63
+ def _fmt_ok(fmt, allowed, where):
64
+ if fmt is not None and fmt not in allowed:
65
+ # A wrong `fmt` renders β€” wrongly β€” and nothing complains. Refusing it at construction is
66
+ # the only moment anybody finds out, so this raises rather than coercing.
67
+ raise ValueError(f"{where}: fmt {fmt!r} is not one of {allowed}")
68
+ return fmt
69
+
70
+
71
+ # ── block constructors ───────────────────────────────────────────────────────────────────────────
72
+ def kpi(key, label, value, fmt="money", delta=None, delta_fmt=None, delta_label=None,
73
+ delta_dir=None, note=None, drill=None):
74
+ """One KPI card. `delta` is a NUMBER or None β€” never a string.
75
+
76
+ THE ZERO-REVENUE RULE, ported from `app.py:2846`: a period with no orders yet must not show an
77
+ alarming βˆ’100%. It sends NO number (`delta=None`) plus a `delta_label` and `delta_dir='off'`,
78
+ so no formatter is involved and the card states the situation instead of computing a ratio
79
+ against nothing.
80
+
81
+ THE BIG-RATIO RULE IS THE CLIENT'S: a tiny LY base makes `+14,975%` read as noise, and
82
+ Streamlit rewrites it per-card as `+150Γ— LY`. That belongs in ONE formatter (`fmt.pct` renders
83
+ |v| β‰₯ 999 as a multiple), so the server just sends the number.
84
+ """
85
+ out = {"key": key, "label": label, "value": value,
86
+ "fmt": _fmt_ok(fmt, FMTS, f"kpi {key}")}
87
+ if delta is not None:
88
+ out["delta"] = delta
89
+ out["delta_fmt"] = _fmt_ok(delta_fmt or "pct", FMTS, f"kpi {key} delta")
90
+ if delta_label:
91
+ out["delta_label"] = delta_label
92
+ if delta_dir:
93
+ out["delta_dir"] = delta_dir # up | down | flat | off β€” a TONE hint, not the sign
94
+ if note:
95
+ out["note"] = note
96
+ if drill:
97
+ out["drill"] = drill
98
+ return out
99
+
100
+
101
+ def kpis(key, items):
102
+ return {"type": "kpis", "key": key, "items": list(items)}
103
+
104
+
105
+ def section(key, label, note=None):
106
+ out = {"type": "section", "key": key, "label": label}
107
+ if note:
108
+ out["note"] = note
109
+ return out
110
+
111
+
112
+ def column(key, label, fmt=None, align=None):
113
+ """A table column. `fmt` omitted β‡’ render verbatim (that is how text columns work)."""
114
+ out = {"key": key, "label": label}
115
+ if fmt:
116
+ out["fmt"] = _fmt_ok(fmt, FMTS, f"column {key}")
117
+ if align:
118
+ out["align"] = align
119
+ elif fmt in ("money", "num", "pct", "int"):
120
+ out["align"] = "right" # numbers align right by default, everywhere
121
+ return out
122
+
123
+
124
+ def table(key, columns, rows, total=None, drill=None, download=None, empty=None):
125
+ """A table block. `shown`/`total` ALWAYS ship β€” `shown == total` still sends both.
126
+
127
+ Owner rule [[no-unverifiable-aggregates]]: a silent `[:N]` cap is a defect. `total` defaults to
128
+ `len(rows)` so a builder that forgot to truncate cannot accidentally claim it did not.
129
+ """
130
+ rows = list(rows)
131
+ out = {"type": "table", "key": key, "columns": list(columns), "rows": rows,
132
+ "shown": len(rows), "total": int(total if total is not None else len(rows))}
133
+ if drill:
134
+ out["drill"] = drill
135
+ if download:
136
+ out["download"] = download
137
+ if empty:
138
+ out["empty"] = empty
139
+ return out
140
+
141
+
142
+ def chart(key, spec, series, fields, rows, x_order=None, drill=None, total=None):
143
+ """A chart block: ONE x, N SERIES.
144
+
145
+ β›” WHY `series[]` AND NOT `ChartSpec.splitBy`. The viz engine has no series dimension:
146
+ `chartData()` buckets on `x` only and `DashboardView` draws one flat `buckets[]`. `splitBy` is
147
+ persisted, validated and offered in the chart editor but never reaches the arithmetic or the
148
+ renderer, so a `splitBy` chart draws identically to one without it. A YoY comparison is
149
+ therefore N `chartData` calls over the SAME wide rows, one per series `y` β€” which also means
150
+ nothing about `chartData` changes and `verify_charts.py`'s verdict stays put, which W2-5's
151
+ pure-move proof requires.
152
+
153
+ β›” WHY `x_order`. The engine sorts buckets by VALUE unless the x field is date-family β€” a trend
154
+ would render as a revenue-ordered sawtooth. And date-family is not an escape: it buckets to
155
+ `YYYY-MM`, so a weekly axis would collapse into months. So the SERVER declares the chronology
156
+ and the client re-orders the buckets it got back.
157
+ """
158
+ for f in fields:
159
+ if f.get("type") not in VIZ_TYPES:
160
+ raise ValueError(f"chart {key}: field {f.get('key')!r} type {f.get('type')!r} is not a "
161
+ f"viz FieldType {VIZ_TYPES} β€” `money` is the kpi/table vocabulary")
162
+ rows = list(rows)
163
+ out = {"type": "chart", "key": key, "spec": dict(spec), "series": list(series),
164
+ "fields": list(fields), "rows": rows,
165
+ "shown": len(rows), "total": int(total if total is not None else len(rows))}
166
+ if x_order is not None:
167
+ out["x_order"] = list(x_order)
168
+ if drill:
169
+ out["drill"] = drill
170
+ return out
171
+
172
+
173
+ def validation(checks, note=None):
174
+ """The reconciliation panel, from a module's `validate()` output.
175
+
176
+ `actual` is the module's `a` β€” the number the page shows. `expected` is its `b` β€” the
177
+ INDEPENDENT aggregate it must tie to. The signed gap rides in `note` rather than being
178
+ recomputed client-side, because the module's own tolerance decided `ok` and a second
179
+ subtraction in the client could disagree with it.
180
+ """
181
+ out = {"type": "validation", "key": "validation",
182
+ "checks": [{"name": c.get("check") or c.get("name") or "",
183
+ "ok": bool(c.get("ok")),
184
+ "actual": c.get("a"), "expected": c.get("b"),
185
+ "fmt": "money" if isinstance(c.get("a"), float) else "num",
186
+ "note": f"gap {c.get('gap')}"} for c in (checks or [])]}
187
+ if note:
188
+ out["note"] = note
189
+ return out
190
+
191
+
192
+ # ── drill descriptors: ONE grammar, three placements ─────────────────────────────────────────────
193
+ def decomp_drill(window_row, scope_type="period", scope_value=None, label=None):
194
+ """A WINDOW + scope descriptor. Mirrors `ui/drawers._decomp_desc` key-for-key, so the eventual
195
+ resolver can serve both front-ends from one function β€” and accepts both row shapes the sales
196
+ module produces (`date_from`/`date_to` on the scorecard, `start`/`end` on a trend row)."""
197
+ return {"kind": "decomp", "label": label,
198
+ "scope_type": scope_type, "scope_value": scope_value,
199
+ "date_from": window_row.get("date_from") or window_row.get("start"),
200
+ "date_to": window_row.get("date_to") or window_row.get("end"),
201
+ "cmp_from": window_row.get("cmp_from"), "cmp_to": window_row.get("cmp_to")}
202
+
203
+
204
+ def entity_drill(kind, id_key, label_key=None):
205
+ """Block-level: every row of this block opens the same kind of entity panel."""
206
+ if kind not in DRILL_KINDS:
207
+ raise ValueError(f"drill kind {kind!r} is not one of {DRILL_KINDS}")
208
+ out = {"kind": kind, "id_key": id_key}
209
+ if label_key:
210
+ out["label_key"] = label_key
211
+ return out
212
+
213
+
214
+ def row_drill(row_key="_drill"):
215
+ """Block-level: each row carries its OWN descriptor under `row_key`."""
216
+ return {"kind": "decomp", "row_key": row_key}
217
+
218
+
219
+ def truncate(rows, limit):
220
+ """`(rows[:limit], total)` β€” the ONLY sanctioned way to cap a list. Returning the total beside
221
+ the slice is what makes "showing N of M" possible; a bare `[:N]` is the defect."""
222
+ rows = list(rows)
223
+ return rows[:limit], len(rows)
224
+
225
+
226
+ # ── BU scope: the one line that could fail open ──────────────────────────────────────────────────
227
+ def resolve_bu(session: Session, bu):
228
+ """The effective `team_id` for this request. `bu` NARROWS; it never authorises.
229
+
230
+ β›” THE RULE, AND WHY IT IS SHAPED THIS WAY. `perms.scope_team_id` derives scope from the user
231
+ RECORD precisely because a UI selector defaults to 'All' β†’ consolidated, and mirroring that
232
+ default on an API with no selector would BE the leak. So a query parameter cannot widen:
233
+
234
+ * absent / 'all' β†’ `scope_team_id(user)`, which is None ONLY when the user may see every BU
235
+ and is the PINNED team_id otherwise. A Royal-only user asking for 'all' gets 6, and 'all'
236
+ for them honestly means "everything you may see".
237
+ * an explicit id β†’ must be in `allowed_team_ids(user)`, else **403**. Never silently coerced
238
+ to their own BU: a coerced parameter hides a client bug behind data that looks right.
239
+ * anything else β†’ 400.
240
+ """
241
+ raw = "" if bu is None else str(bu).strip().lower()
242
+ if raw in ("", "all"):
243
+ return perms.scope_team_id(session.user)
244
+ try:
245
+ want = int(raw)
246
+ except (TypeError, ValueError):
247
+ raise err(400, "bad_bu", "bu must be 'all' or a business-unit id")
248
+ if want not in perms.allowed_team_ids(session.user):
249
+ raise err(403, "bu_forbidden",
250
+ "your account does not have access to that business unit")
251
+ return want
252
+
253
+
254
+ def bu_control(session: Session, team_id):
255
+ """The BU picker, built from the user's OWN permitted labels β€” which is exactly why `controls`
256
+ is assembled per request and never cached on a scope key."""
257
+ labels = perms.allowed_bu_labels(session.user)
258
+ options = []
259
+ for lab in labels:
260
+ if lab == "All":
261
+ options.append({"value": "all", "label": "All"})
262
+ continue
263
+ tid = ctxlib.BRANDS.get(lab)
264
+ if tid is not None:
265
+ options.append({"value": tid, "label": lab})
266
+ return {"key": "bu", "kind": "bu", "label": "Business Unit",
267
+ "value": "all" if team_id is None else team_id, "options": options}
268
+
269
+
270
+ def choice_control(key, label, value, options):
271
+ return {"key": key, "kind": "choice", "label": label, "value": value,
272
+ "options": [{"value": v, "label": l} for v, l in options]}
273
+
274
+
275
+ def bu_label(team_id):
276
+ """'All Business Units' / 'Fisch' / 'Royal' β€” for prose, never for a permission decision."""
277
+ if team_id is None:
278
+ return "All Business Units"
279
+ for lab, tid in ctxlib.BRANDS.items():
280
+ if tid == team_id:
281
+ return lab
282
+ return str(team_id)
283
+
284
+
285
+ # ── the registry + the assembler ─────────────────────────────────────────────────────────────────
286
+ #: page key -> builder module. Wave 3 adds `'ar'` and `'procurement'` HERE and nowhere else β€” the
287
+ #: route, the grant wall, the BU resolution, the caching and the envelope are already written.
288
+ #: ⚠ The key must be the REGISTRY key, because it is what `session.require()` gates on.
289
+ _BUILDERS = {}
290
+
291
+
292
+ def register(key, builder):
293
+ _BUILDERS[str(key)] = builder
294
+
295
+
296
+ def builder_for(key):
297
+ """The builder for `key`, or None. A missing key is a 404 at the route β€” NOT a 403: an
298
+ unported page is a fact about our roadmap, not about this user's grants."""
299
+ if not _BUILDERS:
300
+ import pages_sales # noqa: F401 (self-registers on import)
301
+ return _BUILDERS.get(str(key or "").strip().lower())
302
+
303
+
304
+ def known_pages():
305
+ builder_for("") # force the lazy import
306
+ return sorted(_BUILDERS)
307
+
308
+
309
+ #: How long a scope's metrics stay warm. Matches `routes_customers._CACHE_TTL` β€” the underlying
310
+ #: pull is the same order of cost and a second TTL would be a second thing to reason about.
311
+ _CACHE_TTL = 900
312
+ #: The metrics cache is bounded for the reason every cache in this process is: an entry per
313
+ #: (page Γ— scope Γ— day) that is never evicted is a memory leak with a tenant-shaped growth curve.
314
+ _CACHE_MAX = 24
315
+
316
+
317
+ def _cached_metrics(session: Session, page_key, team_id, granularity, builder):
318
+ """The SCOPE-shaped half, cached on the runtime (per tenant, LRU-bounded).
319
+
320
+ The key carries the DAY: every metric here is relative to "today" (YTD, WTD, the trailing 12
321
+ months), so an entry that outlives midnight serves yesterday's question with today's label.
322
+ """
323
+ rt = session.runtime
324
+ today = time.strftime("%Y-%m-%d")
325
+ key = ("page", page_key, team_id, granularity, today)
326
+ hit = rt.pool_cache.get(key)
327
+ now = time.time()
328
+ if hit and now - hit[0] < _CACHE_TTL:
329
+ return hit[1]
330
+ metrics = builder.metrics(team_id=team_id, granularity=granularity)
331
+ rt.pool_cache[key] = (now, metrics)
332
+ # A `while`, not a one-shot slice: evicting a fixed batch leaves the map above its own bound
333
+ # whenever it is more than that batch over, which is a bound that does not bind.
334
+ # ⚠ This dict is SHARED with `routes_customers._pool_rows`'s pool entries (they key on
335
+ # `("pool", …)`, these on `("page", …)`), so either side's eviction can drop the other's entry.
336
+ # Harmless β€” a miss is a rebuild β€” and deliberately one bounded map per tenant rather than two.
337
+ while len(rt.pool_cache) > _CACHE_MAX:
338
+ rt.pool_cache.pop(min(rt.pool_cache, key=lambda k: rt.pool_cache[k][0]), None)
339
+ return metrics
340
+
341
+
342
+ def build_envelope(session: Session, page_key, bu=None, period=None):
343
+ """The Y1 envelope for `page_key`, scoped to this session. Raises the contracted errors."""
344
+ builder = builder_for(page_key)
345
+ if builder is None:
346
+ raise err(404, "unknown_page", f"no page data is published for {page_key!r}")
347
+ # The GRANT WALL, on the builder's registry key. 403, never an empty 200 β€” an empty page reads
348
+ # as "you have no data" and is how a permission bug hides in plain sight.
349
+ session.require(builder.MODULE)
350
+ team_id = resolve_bu(session, bu)
351
+ granularity = builder.resolve_granularity(period)
352
+ metrics = _cached_metrics(session, page_key, team_id, granularity, builder)
353
+ return {
354
+ "key": page_key,
355
+ "title": builder.TITLE,
356
+ "subtitle": builder.subtitle(team_id),
357
+ # UTC with an explicit Z. A naive local timestamp from a container whose TZ nobody set is
358
+ # indistinguishable from a wrong one.
359
+ "as_of": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime(metrics["pulled_at"])),
360
+ "controls": [bu_control(session, team_id)] + builder.controls(granularity),
361
+ "blocks": builder.blocks(metrics, team_id, granularity),
362
+ }
api/pages_sales.py ADDED
@@ -0,0 +1,270 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """pages_sales.py β€” the Sales page builder (W2-2). The FIRST Y1 builder, and the template.
2
+
3
+ It reuses `royalimports_os/modules/sales.py` VERBATIM β€” the reconciled, validated data layer the
4
+ Streamlit page already reads β€” so there is exactly one definition of every number and the API is a
5
+ projection, not a second implementation. Nothing here computes revenue.
6
+
7
+ WHAT A BUILDER IS, in three functions:
8
+ metrics(team_id, granularity) the SCOPE-shaped data. Cached by `pages._cached_metrics`.
9
+ β›” It cannot see the session, by design β€” that is what makes
10
+ "nothing user-shaped is cached" a structural property rather
11
+ than a rule somebody has to remember.
12
+ blocks(metrics, team_id, …) the ordered Y1 blocks, built with `pages`' constructors.
13
+ controls(granularity) the page's own toolbar (the BU picker is added by `pages`).
14
+
15
+ β›” THE ONE LEAK THIS FILE HAD TO CLOSE. `sales.headline(team_id=…)` returns a `by_team` map that
16
+ **always covers BOTH business units** β€” its own docstring says "by_team always shows both" β€” so a
17
+ Royal-only request's headline carries Fisch revenue. `metrics()` therefore STRIPS `by_team` when a
18
+ BU is pinned, at the source, before it is cached: the other BU's numbers never enter the cache entry
19
+ for a scoped user, so a later edit that emits the block unconditionally renders nothing instead of
20
+ leaking. Fail-closed by construction, not by review. Asserted in `verify_api.py`.
21
+
22
+ ⚠ `by_rep` IS NOT THE STREAMLIT PAGE'S "By agent". `sales.by_rep` groups by Odoo `user_id` (the
23
+ salesperson on the order); the Streamlit page shows `customers.by_dimension('agent')` (the
24
+ CUSTOMER's assigned agent β€” a different number, see [[invoice-line-agent-commission]]). The split
25
+ doc's brief names `by_rep`, so `by_rep` is what ships; the divergence is flagged in the S1 mailbox
26
+ for the owner rather than resolved in code.
27
+ """
28
+ import sys as _sys
29
+ import time
30
+
31
+ import core.odoo as O
32
+ import core.periods as P
33
+ import modules.sales as sales
34
+
35
+ import pages
36
+
37
+ MODULE = "sales" # the REGISTRY key β€” what the grant wall gates on
38
+ TITLE = "Sales"
39
+
40
+ _GRANULARITIES = (("monthly", "Monthly"), ("weekly", "Weekly"))
41
+
42
+
43
+ def resolve_granularity(period):
44
+ """`?period=` β†’ the trend granularity.
45
+
46
+ An unrecognised value is COERCED to the default rather than refused, and that asymmetry with
47
+ `bu` is deliberate: `bu` is a SCOPE question, where a coerced parameter would silently show the
48
+ wrong book, so it 403s. `period` is a DISPLAY question, and the envelope echoes the resolved
49
+ value back in `controls[].value`, so a coerced parameter cannot look accepted.
50
+ """
51
+ return "weekly" if str(period or "").strip().lower() in ("weekly", "week") else "monthly"
52
+
53
+
54
+ def controls(granularity):
55
+ return [pages.choice_control("period", "Granularity", granularity, _GRANULARITIES)]
56
+
57
+
58
+ def subtitle(team_id):
59
+ return ("Confirmed orders only (Odoo state sale/done), house accounts excluded. Floral "
60
+ "wholesale is seasonal, so every comparison is against the SAME period last year.")
61
+
62
+
63
+ # ── the scope-shaped half ────────────────────────────────────────────────────────────────────────
64
+ def metrics(team_id=None, granularity="monthly"):
65
+ """Everything the blocks need, for ONE scope. Pure data β€” no session, no user, no formatting."""
66
+ # Pin the recognition basis explicitly. It is thread-local and defaults to 'order', but a
67
+ # worker thread reused from a container that once served 'invoice' would otherwise inherit a
68
+ # different definition of revenue mid-page, and nothing in the payload would say so.
69
+ O.set_doc_mode("order")
70
+ t = P.today()
71
+
72
+ head = sales.headline(t=t, team_id=team_id)
73
+ if team_id is not None:
74
+ # β›” See the module docstring: `by_team` always covers both BUs. Strip it BEFORE caching.
75
+ head = dict(head, by_team={})
76
+
77
+ if granularity == "weekly":
78
+ trend = sales.weekly_trend(n_weeks=pages.MAX_BUCKETS, t=t, team_id=team_id)
79
+ else:
80
+ trend = sales.monthly_trend(n=pages.MAX_BUCKETS, t=t, team_id=team_id)
81
+
82
+ return {
83
+ "pulled_at": time.time(),
84
+ "today": P._d(t),
85
+ "scorecard": sales.period_scorecard(t=t, team_id=team_id),
86
+ "headline": head,
87
+ "trend": trend,
88
+ # limit=1e9 so the TOTAL is knowable: the module slices after aggregating, so asking for
89
+ # everything costs the same read and lets the block report "showing 25 of 812" instead of
90
+ # capping silently. A bare `[:N]` with no total is the defect [[no-unverifiable-aggregates]]
91
+ # exists to name.
92
+ "by_rep": sales.by_rep(t=t, limit=10 ** 9, team_id=team_id),
93
+ "top_customers": sales.top_customers(t=t, limit=10 ** 9, team_id=team_id),
94
+ "validation": sales.validate(t=t, team_id=team_id),
95
+ }
96
+
97
+
98
+ # ── the blocks ───────────────────────────────────────────────────────────────────────────────────
99
+ _LIST_LIMIT = 25
100
+
101
+
102
+ def _delta_dir(v):
103
+ if v is None:
104
+ return None
105
+ return "up" if v > 0 else "down" if v < 0 else "flat"
106
+
107
+
108
+ def _scorecard_kpis(scorecard):
109
+ """One card per period, every one of them drillable to its exact window."""
110
+ items = []
111
+ for r in scorecard:
112
+ drill = pages.decomp_drill(r, "period", None, f"Sales Β· {r['label']}")
113
+ if not r["revenue"]:
114
+ # No orders yet in this window. Sending a βˆ’100% here is the alarming lie
115
+ # `app.py:2846` guards against; send no number at all.
116
+ items.append(pages.kpi(r["key"], r["label"], 0.0, "money",
117
+ delta_label="no orders yet", delta_dir="off", drill=drill))
118
+ continue
119
+ items.append(pages.kpi(r["key"], r["label"], r["revenue"], "money",
120
+ delta=r["yoy_pct"], delta_fmt="pct",
121
+ delta_label="vs LY" if r["yoy_pct"] is not None else None,
122
+ delta_dir=_delta_dir(r["yoy_pct"]), drill=drill))
123
+ return pages.kpis("scorecard", items)
124
+
125
+
126
+ def _ytd_context_kpis(scorecard, head):
127
+ """The three YTD numbers the Streamlit page puts in a caption (`app.py:2858`). As KPI cards
128
+ rather than prose, so each one carries the drill the caption could not."""
129
+ ytd = next((r for r in scorecard if r["key"] == "ytd"), scorecard[-1] if scorecard else {})
130
+ drill = pages.decomp_drill(ytd, "period", None, "Sales Β· Year to date")
131
+ return pages.kpis("ytd_context", [
132
+ pages.kpi("ytd_orders", "Orders (YTD)", head.get("ytd_orders") or 0, "int", drill=drill),
133
+ pages.kpi("ytd_aov", "Average order value", head.get("aov") or 0.0, "money", drill=drill),
134
+ pages.kpi("ytd_customers", "Active customers (YTD)", head.get("ytd_customers") or 0,
135
+ "int", drill=drill),
136
+ ])
137
+
138
+
139
+ def _trend_block(trend, granularity, today):
140
+ """The YoY trend: ONE x, TWO series, chronology declared by the server.
141
+
142
+ See `pages.chart` for why this is `series[]` + `x_order[]` rather than `ChartSpec.splitBy` and
143
+ a date-typed axis. `spec.y` is set to the FIRST series so a naive single-series
144
+ `chartData(spec, …)` call still draws the primary bars instead of a "choose a number field"
145
+ problem card.
146
+ """
147
+ label_key = "week" if granularity == "weekly" else "month"
148
+ rows, x_order = [], []
149
+ for r in trend:
150
+ label = str(r.get(label_key) or "")
151
+ rows.append({
152
+ "period": label,
153
+ "revenue": r.get("revenue") or 0.0,
154
+ "revenue_ly": r.get("revenue_ly") or 0.0,
155
+ # The in-progress period compares a PARTIAL window against a full one, so the client
156
+ # fades it and drops its YoY (`app.py:2867-2875`). One rule for both granularities:
157
+ # the window that CONTAINS today. ISO strings, so string order is date order.
158
+ "partial": bool(r.get("start") and r.get("end")
159
+ and str(r["start"]) <= today <= str(r["end"])),
160
+ "_drill": pages.decomp_drill(r, "period", None, f"Sales Β· {label}"),
161
+ })
162
+ x_order.append(label)
163
+ unit = "Week of" if granularity == "weekly" else "Month"
164
+ spec = {"id": "sales-trend", "kind": "bar", "x": "period", "y": "revenue", "agg": "sum",
165
+ "title": "Revenue trend", "palette": "brand",
166
+ "axis": {"x": {"label": unit}, "y": {"format": "currency"}}}
167
+ fields = [{"key": "period", "label": unit, "type": "text", "source": "odoo"},
168
+ {"key": "revenue", "label": "Revenue", "type": "currency", "source": "odoo"},
169
+ {"key": "revenue_ly", "label": "Revenue LY", "type": "currency", "source": "odoo"}]
170
+ series = [{"y": "revenue", "label": "This year"},
171
+ {"y": "revenue_ly", "label": "Last year"}]
172
+ return pages.chart("trend", spec, series, fields, rows, x_order=x_order,
173
+ drill=pages.row_drill())
174
+
175
+
176
+ def _by_bu_block(head, scorecard):
177
+ """YTD per Business Unit β€” CONSOLIDATED VIEWS ONLY.
178
+
179
+ Omitted entirely when a BU is pinned, matching `app.py:2898`'s `if tid is None`. `metrics()` has
180
+ already emptied `by_team` in that case, so this cannot accidentally render the other BU even if
181
+ the guard here were removed β€” belt and braces on the one number that must never cross.
182
+ """
183
+ by_team = head.get("by_team") or {}
184
+ if not by_team:
185
+ return None
186
+ ytd = next((r for r in scorecard if r["key"] == "ytd"), scorecard[-1] if scorecard else {})
187
+ rows = []
188
+ for tid in O.TEAM_IDS:
189
+ name = O.TEAM_NAMES.get(tid)
190
+ e = by_team.get(name)
191
+ if not e:
192
+ continue
193
+ rows.append({
194
+ "bu": name, "team_id": tid,
195
+ "revenue": e.get("ytd") or 0.0, "revenue_ly": e.get("ytd_ly") or 0.0,
196
+ "yoy_pct": P.yoy_pct(e.get("ytd") or 0.0, e.get("ytd_ly") or 0.0),
197
+ "_drill": pages.decomp_drill(ytd, "brand", tid, f"Sales Β· {name} (YTD)"),
198
+ })
199
+ return pages.table(
200
+ "by_bu",
201
+ [pages.column("bu", "Business Unit"),
202
+ pages.column("revenue", "YTD $", "money"),
203
+ pages.column("revenue_ly", "LY $", "money"),
204
+ pages.column("yoy_pct", "YoY", "pct")],
205
+ rows, drill=pages.row_drill(), download={"filename": "sales_by_business_unit"},
206
+ empty="No confirmed orders in either Business Unit this year.")
207
+
208
+
209
+ def _by_rep_block(by_rep):
210
+ rows, total = pages.truncate(by_rep, _LIST_LIMIT)
211
+ return pages.table(
212
+ "by_rep",
213
+ [pages.column("rep", "Salesperson"),
214
+ pages.column("revenue", "YTD $", "money"),
215
+ pages.column("orders", "Orders", "int")],
216
+ rows, total=total, drill=pages.entity_drill("rep", "uid", "rep"),
217
+ download={"filename": "sales_by_rep"},
218
+ empty="No confirmed orders are attributed to a salesperson in this scope.")
219
+
220
+
221
+ def _top_customers_block(top_customers):
222
+ rows, total = pages.truncate(top_customers, _LIST_LIMIT)
223
+ return pages.table(
224
+ "top_customers",
225
+ [pages.column("customer", "Customer"),
226
+ pages.column("revenue", "YTD $", "money"),
227
+ pages.column("orders", "Orders", "int")],
228
+ rows, total=total, drill=pages.entity_drill("customer", "pid", "customer"),
229
+ download={"filename": "top_customers"},
230
+ empty="No confirmed orders in this scope yet.")
231
+
232
+
233
+ def blocks(metrics, team_id=None, granularity="monthly"):
234
+ """The ORDERED blocks, mirroring `app.py:2833-2961`'s reading order.
235
+
236
+ ⚠ The BU table is present only on a consolidated view, so the client must not assume a fixed
237
+ block count or index β€” it renders `blocks` in order and skips a type it does not know.
238
+ """
239
+ sc = metrics["scorecard"]
240
+ head = metrics["headline"]
241
+ out = [
242
+ _scorecard_kpis(sc),
243
+ _ytd_context_kpis(sc, head),
244
+ pages.section("trend_h", "Revenue trend",
245
+ "This period versus the same period last year. Click any bar to decompose "
246
+ "that period by customer and SKU. The in-progress period is marked partial "
247
+ "and carries no YoY."),
248
+ _trend_block(metrics["trend"], granularity, metrics["today"]),
249
+ ]
250
+ bu = _by_bu_block(head, sc)
251
+ if bu is not None:
252
+ out.append(pages.section("by_bu_h", "By Business Unit",
253
+ "YTD revenue per Business Unit versus last year."))
254
+ out.append(bu)
255
+ out.append(pages.section("by_rep_h", "By salesperson",
256
+ "YTD revenue per Odoo salesperson on the order."))
257
+ out.append(_by_rep_block(metrics["by_rep"]))
258
+ out.append(pages.section("top_customers_h", "Top customers",
259
+ "Largest customers by YTD revenue."))
260
+ out.append(_top_customers_block(metrics["top_customers"]))
261
+ out.append(pages.validation(
262
+ metrics["validation"],
263
+ "Every headline number reconciles to an independent Odoo aggregate, and every "
264
+ "decomposition sums back to it."))
265
+ return out
266
+
267
+
268
+ #: Self-registers on import, so adding a page is one file plus one import β€” `pages.builder_for`
269
+ #: does the import and never needs a per-page branch.
270
+ pages.register("sales", _sys.modules[__name__])
api/routes_admin.py ADDED
@@ -0,0 +1,325 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """routes_admin.py β€” Y4: user administration + the session's own settings (W2-3).
2
+
3
+ These routes are the standalone shell's replacement for `app.py`'s `users_dialog` / `settings_dialog`
4
+ modals, over the SAME `core/users` account store β€” so both front-ends administer one set of
5
+ accounts and the eventual OIDC migration (D-3) swaps the CREDENTIAL check, not the user model.
6
+
7
+ ADMIN-ONLY, FAIL-CLOSED. `admin_gate` is a role check (`perms.is_admin`), mirroring the Streamlit
8
+ dialog's `if not is_admin()`. A default record has `role: 'user'`, so an unreadable or partial record
9
+ is denied rather than admitted. `verify_api.py` proves it by having a viewer TRY every route.
10
+
11
+ β›” THIS IS THE FIRST WRITER OF `modules` THAT HAS EVER EXISTED. `core.users.set_access(modules=…)`
12
+ has no caller anywhere in the shipped app β€” module grants are only settable by editing the store
13
+ JSON out of band. That matters because the record format carries TWO documented fail-OPENS, both
14
+ asserted in `verify_api.py` section A:
15
+
16
+ * `modules: []` is FALSY, so `perms.allowed_modules` reads it as 'all' = UNRESTRICTED. An admin
17
+ clearing every checkbox to lock an account down would grant it everything.
18
+ * `bus: [<unknown id>]` falls through `allowed_bus_labels`'s "no recognisable label" branch to
19
+ `['All','Fisch','Royal']` β€” so one typo'd BU id is FULL cross-BU access, in the model whose
20
+ whole point is strict isolation.
21
+
22
+ Y4 says do not "fix" `modules: []` in this wave, and that is right: the READER is mirrored in
23
+ `ui/session.py` and `core/perms.py` and diverging one of them mid-wave breaks lock-step. But the
24
+ WRITER is new, and it can simply refuse to create either footgun. So both are 400s here and both
25
+ read semantics are untouched β€” the seam is write-strict / read-unchanged.
26
+
27
+ ⚠ A STORE OUTAGE IS A 503, NEVER AN EMPTY LIST. `users.registry()` swallows a failed read into `{}`
28
+ one level down, and serving that as `{"users": []}` would tell an administrator their tenant has no
29
+ accounts. Same rule as the write path: an empty 200 is never how this API says "something is wrong".
30
+ """
31
+ import re
32
+
33
+ from fastapi import APIRouter, Body, Depends, Response
34
+
35
+ import core.registry as registry
36
+ import core.store as store
37
+
38
+ from deps import Session, err, perms, require_session, users
39
+ from routes_auth import _public_user
40
+
41
+ router = APIRouter(prefix="/api/v1")
42
+
43
+ #: A username is a STORE KEY (`users.json`) and also the key the table workspace is filed under
44
+ #: (`data[username]`), so it is constrained rather than trusted: lowercase, no separators, no
45
+ #: whitespace, nothing that could traverse or collide once it becomes part of a path or a filename.
46
+ _UNAME_RE = re.compile(r"^[a-z0-9][a-z0-9._-]{1,31}$")
47
+ #: Passwords are PBKDF2-200k, which is only as strong as what it is given. The Streamlit dialog
48
+ #: enforces nothing; this does, and the two are allowed to differ because only one of them is
49
+ #: reachable from the internet.
50
+ _MIN_PW = 8
51
+ _ROLES = ("user", "admin")
52
+
53
+
54
+ def admin_gate(session: Session = Depends(require_session)) -> Session:
55
+ """401 without a session, 403 without the admin role. Role, not a module grant β€” mirroring
56
+ `app.py`'s `users_dialog`, whose only wall is `is_admin()`."""
57
+ if not perms.is_admin(session.user):
58
+ raise err(403, "forbidden", "administrators only")
59
+ return session
60
+
61
+
62
+ def _require_store():
63
+ """A 503 the moment the store cannot serve, so no route below can report emptiness as truth."""
64
+ try:
65
+ ok = store.available()
66
+ except Exception:
67
+ ok = False
68
+ if not ok:
69
+ raise err(503, "store_unavailable",
70
+ "the tenant store is unavailable β€” accounts cannot be read or changed")
71
+
72
+
73
+ def _registry():
74
+ _require_store()
75
+ try:
76
+ reg = users.registry() or {}
77
+ except Exception:
78
+ raise err(503, "store_unavailable", "the tenant store is unavailable")
79
+ return reg
80
+
81
+
82
+ def _view(uname, rec):
83
+ """What an administrator may see about an account. NEVER `salt` or `hash` β€” this function is
84
+ the only projection these routes use, so there is one place that can leak and it does not."""
85
+ return {"username": uname,
86
+ "name": rec.get("name") or uname,
87
+ "role": rec.get("role", "user"),
88
+ "bus": rec.get("bus", "all"),
89
+ "bu_labels": perms.allowed_bu_labels(rec),
90
+ "modules": rec.get("modules", "all"),
91
+ "agent": rec.get("agent") or None,
92
+ "email": rec.get("email") or None,
93
+ "active": bool(rec.get("active", True)),
94
+ # The session-revocation handle. An admin needs it: it is the only visible evidence
95
+ # that "sign them out everywhere" actually happened.
96
+ "epoch": int(rec.get("epoch") or 0)}
97
+
98
+
99
+ # ── request validation: every rule below is fail-closed ──────────────────────────────────────────
100
+ def _clean_username(v):
101
+ uname = str(v or "").strip().lower()
102
+ if not _UNAME_RE.match(uname):
103
+ raise err(400, "bad_username",
104
+ "a username is 2-32 characters: lowercase letters, digits, dot, dash or "
105
+ "underscore, starting with a letter or digit")
106
+ return uname
107
+
108
+
109
+ def _clean_password(v):
110
+ pw = str(v or "")
111
+ if len(pw) < _MIN_PW:
112
+ raise err(400, "weak_password", f"a password must be at least {_MIN_PW} characters")
113
+ return pw
114
+
115
+
116
+ def _clean_role(v):
117
+ role = str(v or "").strip().lower()
118
+ if role not in _ROLES:
119
+ raise err(400, "bad_role", f"role must be one of {list(_ROLES)}")
120
+ return role
121
+
122
+
123
+ def _clean_bus(v):
124
+ """'all', or a non-empty list of KNOWN business-unit ids.
125
+
126
+ Refusing an unknown id is the whole point: `allowed_bus_labels` treats a list with no
127
+ recognisable label as full access, so `bus: [7]` would be a cross-BU grant created by a typo.
128
+ """
129
+ if isinstance(v, str):
130
+ if v.strip().lower() == "all":
131
+ return "all"
132
+ raise err(400, "bad_bus", "bus must be 'all' or a list of business-unit ids")
133
+ if not isinstance(v, (list, tuple)) or not v:
134
+ raise err(400, "bad_bus",
135
+ "bus must be 'all' or a NON-EMPTY list of business-unit ids (an empty list "
136
+ "would read as unrestricted)")
137
+ out = []
138
+ for b in v:
139
+ try:
140
+ b = int(b)
141
+ except (TypeError, ValueError):
142
+ raise err(400, "bad_bus", "a business-unit id must be a number")
143
+ if b not in users.BU_LABELS:
144
+ raise err(400, "bad_bus",
145
+ f"{b} is not a known business unit {sorted(users.BU_LABELS)} β€” an "
146
+ "unrecognised id would widen access to every BU")
147
+ out.append(b)
148
+ return sorted(set(out))
149
+
150
+
151
+ def _clean_modules(v):
152
+ """'all', or a non-empty list of KNOWN module keys.
153
+
154
+ `[]` is refused because it READS as unrestricted (see the module docstring). An unknown key is
155
+ refused because `may_open` is fail-closed on it: an admin who typed `sale` for `sales` would
156
+ silently lock the account out of the page they meant to grant.
157
+ """
158
+ if isinstance(v, str):
159
+ if v.strip().lower() == "all":
160
+ return "all"
161
+ raise err(400, "bad_modules", "modules must be 'all' or a list of module keys")
162
+ if not isinstance(v, (list, tuple)) or not v:
163
+ raise err(400, "bad_modules",
164
+ "modules must be 'all' or a NON-EMPTY list of module keys β€” an empty list "
165
+ "reads as UNRESTRICTED, which is the opposite of locking an account down")
166
+ known = set(registry.BY_KEY) | set(perms._LEGACY_KEYS)
167
+ out, bad = [], []
168
+ for k in v:
169
+ k = str(k or "").strip()
170
+ (out if k in known else bad).append(k)
171
+ if bad:
172
+ raise err(400, "bad_modules", f"unknown module keys: {sorted(bad)}")
173
+ return sorted(set(out))
174
+
175
+
176
+ # ── the routes ───────────────────────────────────────────────────────────────────────────────────
177
+ @router.get("/admin/users")
178
+ def list_users(session: Session = Depends(admin_gate)):
179
+ reg = _registry()
180
+ return {"users": [_view(u, reg[u]) for u in sorted(reg)]}
181
+
182
+
183
+ @router.post("/admin/users", status_code=201)
184
+ def create_user(body: dict = Body(default=None), session: Session = Depends(admin_gate)):
185
+ """Create an account. **409 if the username already exists β€” `PATCH` is the update path.**
186
+
187
+ β›” WHY 409 AND NOT AN UPSERT. `core.users.create_user` writes a fresh `_record()`, and a fresh
188
+ record has NO `epoch` β€” so overwriting an existing account used to drop its epoch back to 0 and
189
+ RESURRECT every cookie minted before its last password change. `core/users.py` now
190
+ preserves-and-bumps on overwrite (which also closes it for `app.py`'s dialog, where "Add /
191
+ update a user" calls the same function), but this route still refuses: an upsert that silently
192
+ replaces an account's password, role and BU access because someone reused a username is not an
193
+ update anybody asked for.
194
+ """
195
+ body = body or {}
196
+ _require_store()
197
+ uname = _clean_username(body.get("username"))
198
+ pw = _clean_password(body.get("password"))
199
+ role = _clean_role(body.get("role") or "user")
200
+ bus = _clean_bus(body.get("bus") if body.get("bus") is not None else "all")
201
+ modules = _clean_modules(body.get("modules") if body.get("modules") is not None else "all")
202
+ name = str(body.get("name") or "").strip() or uname
203
+ agent = str(body.get("agent") or "").strip() or None
204
+ email = str(body.get("email") or "").strip() or None
205
+
206
+ if uname in _registry():
207
+ raise err(409, "user_exists", f"{uname} already exists β€” PATCH it to change it")
208
+ users.create_user(uname, pw, name, role=role, bus=bus, modules=modules,
209
+ agent=agent, email=email)
210
+ reg = _registry()
211
+ if uname not in reg:
212
+ # The store accepted the write and does not have it. Reporting 201 here would be the
213
+ # "200 over a write that evaporated" failure the whole store-outage rule exists to prevent.
214
+ raise err(503, "store_unavailable", "the account was not saved β€” try again")
215
+ return {"user": _view(uname, reg[uname])}
216
+
217
+
218
+ @router.patch("/admin/users/{username}")
219
+ def update_user(username: str, body: dict = Body(default=None),
220
+ session: Session = Depends(admin_gate)):
221
+ """Change access on an existing account. Only the keys PRESENT in the body change.
222
+
223
+ ⚠ NO EPOCH BUMP, and that is not an omission. `deps._user_for` re-reads the record on every
224
+ request, so a narrowed role / BU / module grant applies to the target's LIVE session on its very
225
+ next call β€” bumping the epoch would only add a forced re-login on top. `active: false` is the
226
+ exception and it bumps, inside `core.users.set_active`, because a disabled account must lose its
227
+ session rather than keep working until the cookie expires.
228
+ """
229
+ body = body if isinstance(body, dict) else {}
230
+ reg = _registry()
231
+ uname = str(username or "").strip().lower()
232
+ if uname not in reg:
233
+ raise err(404, "no_such_user", f"no account named {uname!r}")
234
+ if not body:
235
+ raise err(400, "empty_patch", "no fields to update")
236
+
237
+ unknown = sorted(set(body) - {"name", "role", "bus", "modules", "active", "agent", "email"})
238
+ if unknown:
239
+ # A silently-ignored field is how a UI ends up believing it saved something it did not.
240
+ raise err(400, "unknown_fields", f"cannot update: {unknown}")
241
+
242
+ is_self = uname == session.uname
243
+ role = _clean_role(body["role"]) if "role" in body else None
244
+ active = bool(body["active"]) if "active" in body else None
245
+ # THE ONE-CLICK LOCKOUT GUARD. APP_PASSWORD remains the real backstop for 'admin', so this is
246
+ # not the thing that keeps the product reachable β€” it just stops an administrator removing
247
+ # their own access with a toggle and having to go find the master password.
248
+ if is_self and role is not None and role != "admin":
249
+ raise err(400, "self_demote",
250
+ "you cannot remove your own administrator role β€” ask another admin")
251
+ if is_self and active is False:
252
+ raise err(400, "self_deactivate", "you cannot deactivate your own account")
253
+
254
+ kw = {}
255
+ if role is not None:
256
+ kw["role"] = role
257
+ if "name" in body:
258
+ kw["name"] = str(body["name"] or "").strip() or uname
259
+ if "bus" in body:
260
+ kw["bus"] = _clean_bus(body["bus"])
261
+ if "modules" in body:
262
+ kw["modules"] = _clean_modules(body["modules"])
263
+ if "agent" in body:
264
+ kw["agent"] = str(body["agent"] or "").strip() # '' clears the link
265
+ if "email" in body:
266
+ kw["email"] = str(body["email"] or "").strip()
267
+ if kw:
268
+ users.set_access(uname, **kw)
269
+ if active is not None and active != bool(reg[uname].get("active", True)):
270
+ users.set_active(uname, active) # bumps the epoch: kills live sessions
271
+
272
+ fresh = _registry()
273
+ return {"user": _view(uname, fresh[uname])}
274
+
275
+
276
+ @router.post("/admin/users/{username}/password", status_code=204)
277
+ def set_password(username: str, body: dict = Body(default=None),
278
+ session: Session = Depends(admin_gate)):
279
+ """Rotate a password. **Revokes every outstanding session for that account** β€” the epoch bump
280
+ happens inside the same read-modify-write as the hash (`core.users.set_password`), so the two
281
+ can never disagree.
282
+
283
+ 204 with no body: there is nothing to say that the caller did not already know, and echoing the
284
+ account back would invite a client to diff a response for a password change.
285
+ """
286
+ reg = _registry()
287
+ uname = str(username or "").strip().lower()
288
+ if uname not in reg:
289
+ raise err(404, "no_such_user", f"no account named {uname!r}")
290
+ pw = _clean_password((body or {}).get("password"))
291
+ before = int(reg[uname].get("epoch") or 0)
292
+ users.set_password(uname, pw)
293
+ after = int((_registry().get(uname) or {}).get("epoch") or 0)
294
+ if after <= before:
295
+ # The whole value of this route is the revocation. If the epoch did not move, the sessions
296
+ # were not revoked, and answering 204 would say they were.
297
+ raise err(503, "store_unavailable",
298
+ "the password change did not persist β€” outstanding sessions were NOT revoked")
299
+ return Response(status_code=204)
300
+
301
+
302
+ @router.get("/settings")
303
+ def settings(session: Session = Depends(require_session)):
304
+ """The session's OWN settings β€” any authenticated user, not admin-only.
305
+
306
+ `user` comes from `routes_auth._public_user`, deliberately reusing the ONE definition of what a
307
+ client may know about itself (never a hash, never the epoch). `scope` restates it as the two
308
+ things the Settings UI shows, and `tenant` names only THIS tenant β€” enumerating the others
309
+ would answer a question about our customer list.
310
+ """
311
+ modules = perms.allowed_modules(session.user)
312
+ return {
313
+ "user": _public_user(session.user),
314
+ "scope": {
315
+ "bus": perms.allowed_bu_labels(session.user),
316
+ "team_id": perms.scope_team_id(session.user),
317
+ "agent": perms.scope_agent(session.user),
318
+ "modules": sorted(modules) if modules is not None else "all",
319
+ },
320
+ "tenant": {"key": session.tenant, "name": getattr(session.runtime, "name",
321
+ session.tenant)},
322
+ # CHROME ONLY. Every /admin route re-checks the role server-side; this exists so the client
323
+ # does not paint a "Manage users" button that 403s.
324
+ "admin": perms.is_admin(session.user),
325
+ }
api/routes_auth.py ADDED
@@ -0,0 +1,143 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """routes_auth.py β€” X2's auth leg: login / logout / me (EXIT-3a).
2
+
3
+ Password auth against the SAME `core/users` accounts the Streamlit `gate()` uses, so both
4
+ front-ends share one identity today. The amended D-3 (our own OIDC client against Authentik, with
5
+ a branded login page) lands when D-1/D-2 unblock (owner blocker B-4); this is the interim leg it
6
+ replaces, and it is deliberately the same account store so the migration is a swap of the
7
+ CREDENTIAL check, not of the user model.
8
+ """
9
+ import hmac
10
+ import os
11
+ import time
12
+ from collections import OrderedDict
13
+
14
+ from fastapi import APIRouter, Body, Depends, Request, Response
15
+
16
+ import aios_session
17
+ from deps import Session, err, require_session, users, perms
18
+
19
+ router = APIRouter(prefix="/api/v1/auth")
20
+
21
+ #: THE SERVER OWNS BACKOFF (X5): the login page shows an inline error and never paces itself, so
22
+ #: the pacing has to be here or it does not exist. Bounded so it cannot grow into a memory leak.
23
+ #: ⚠ HONEST LIMIT: per-PROCESS. Behind several workers an attacker gets this budget per worker,
24
+ #: and it resets on deploy. It raises the cost of online guessing against PBKDF2-200k; it is not
25
+ #: a substitute for the real thing, which arrives with Authentik (D-1) owning rate limits centrally.
26
+ #: ⚠ KEYED BY (TENANT, USERNAME), not by username. Two tenants can each have an 'admin' or a
27
+ #: 'boss', and a shared bucket would mean failed logins against tenant B lock the same name out of
28
+ #: tenant A β€” a cross-tenant coupling in the one wave whose point is that tenants cannot reach
29
+ #: each other. It is also a (weak) existence oracle: a 429 for a name you never tried here would
30
+ #: tell you somebody else did.
31
+ _FAILS = OrderedDict()
32
+ _MAX_TRACKED = 2048
33
+ _LOCKOUT_AFTER = 8
34
+ _LOCKOUT_SECONDS = 60
35
+
36
+
37
+ def _throttled(key, now=None):
38
+ now = now or time.time()
39
+ hit = _FAILS.get(key)
40
+ if not hit:
41
+ return 0
42
+ count, last = hit
43
+ if count < _LOCKOUT_AFTER:
44
+ return 0
45
+ remaining = int(_LOCKOUT_SECONDS - (now - last))
46
+ return remaining if remaining > 0 else 0
47
+
48
+
49
+ def _note_failure(key, now=None):
50
+ now = now or time.time()
51
+ count, last = _FAILS.get(key, (0, 0.0))
52
+ # A window that has fully elapsed starts the count again β€” a user who mistypes twice today
53
+ # and twice next week is not an attacker.
54
+ if now - last > _LOCKOUT_SECONDS:
55
+ count = 0
56
+ _FAILS[key] = (count + 1, now)
57
+ _FAILS.move_to_end(key)
58
+ while len(_FAILS) > _MAX_TRACKED:
59
+ _FAILS.popitem(last=False)
60
+
61
+
62
+ def _clear_failures(key):
63
+ _FAILS.pop(key, None)
64
+
65
+
66
+ def _public_user(user):
67
+ """What the client is allowed to know about itself. Never a hash, a salt, or the epoch β€”
68
+ the epoch is a server-side revocation handle and a client has no use for it."""
69
+ return {"username": user.get("username", ""), "name": user.get("name", ""),
70
+ "role": user.get("role", "user"),
71
+ "bus": perms.allowed_bu_labels(user),
72
+ "team_id": perms.scope_team_id(user),
73
+ "agent": perms.scope_agent(user),
74
+ "modules": sorted(perms.allowed_modules(user) or []) or "all",
75
+ "landing": perms.landing_page(user)}
76
+
77
+
78
+ @router.post("/login")
79
+ def login(request: Request, response: Response, body: dict = Body(default=None)):
80
+ body = body or {}
81
+ uname = str(body.get("username") or "").strip().lower()
82
+ pw = str(body.get("password") or "")
83
+ tenant = str(body.get("tenant") or "royal-imports").strip().lower()
84
+
85
+ # The tenant is validated BEFORE the credential check so an unknown slug can never mint a
86
+ # cookie β€” but it answers with the SAME 401 as a bad password, because "that tenant does not
87
+ # exist" is a fact about our customer list.
88
+ from harness import runtime
89
+ try:
90
+ runtime.get_runtime(tenant)
91
+ except KeyError:
92
+ raise err(401, "invalid_credentials", "that username and password do not match")
93
+
94
+ throttle_key = (tenant, uname)
95
+ wait = _throttled(throttle_key)
96
+ if wait:
97
+ raise err(429, "too_many_attempts",
98
+ f"too many failed attempts β€” try again in {wait} seconds")
99
+
100
+ user = users.verify(uname, pw) if (uname and pw) else None
101
+ if not user:
102
+ # CONSTANT-TIME-ISH 401. `users.verify` returns immediately for an unknown username (no
103
+ # record, no PBKDF2) and burns ~200k iterations for a known one, so the response time
104
+ # answers "does this account exist?" to anyone with a stopwatch. Burning one equivalent
105
+ # hash on the failure path removes that oracle. It costs nothing on the happy path.
106
+ try:
107
+ users._hash(pw or "x", "00" * 16)
108
+ except Exception:
109
+ pass
110
+ _note_failure(throttle_key)
111
+ raise err(401, "invalid_credentials", "that username and password do not match")
112
+
113
+ _clear_failures(throttle_key)
114
+ value, _claims = aios_session.mint(tenant, user["username"], int(user.get("epoch") or 0))
115
+ aios_session.set_cookie(response, request, value)
116
+ return {"user": _public_user(user)}
117
+
118
+
119
+ @router.post("/logout", status_code=204)
120
+ def logout(request: Request):
121
+ """204 and the cookie is cleared. Deliberately NOT session-gated: logging out must work from
122
+ an already-invalid session, or a user holding a broken cookie has no way to get rid of it.
123
+
124
+ ⚠ THE COOKIE IS CLEARED ON THE RETURNED RESPONSE, not on an injected `response` param. When a
125
+ handler RETURNS a Response object, FastAPI ships that object β€” anything written to the
126
+ injected `response` is silently dropped. The first version of this route took `response:
127
+ Response`, called `clear_cookie` on it, then returned a fresh `Response(204)`: the 204 was
128
+ correct, no `Set-Cookie` was ever sent, and the session stayed alive after a "successful"
129
+ logout. Caught by asserting `/auth/me` is 401 AFTER the logout rather than trusting the 204.
130
+
131
+ ⚠ This clears the BROWSER's copy only β€” the signed value stays cryptographically valid until
132
+ it expires, which is the honest cost of a stateless session. "Sign me out everywhere" is
133
+ `core.users.bump_epoch` (a password change already does it); D2's Postgres session mirror
134
+ makes per-device revocation possible and arrives with C-2.
135
+ """
136
+ out = Response(status_code=204)
137
+ aios_session.clear_cookie(out, request)
138
+ return out
139
+
140
+
141
+ @router.get("/me")
142
+ def me(session: Session = Depends(require_session)):
143
+ return {"user": _public_user(session.user), "tenant": session.tenant}
api/routes_customers.py ADDED
@@ -0,0 +1,208 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """routes_customers.py β€” X2's read + write of the customer table, BU-SCOPED (EXIT-3b / EXIT-2a).
2
+
3
+ Two things happen here that did not happen in the pre-wave `main.py`:
4
+
5
+ 1. **ROWS ARE SCOPED TO THE SESSION.** The old `/api/customers` served `cl.pool()` β€” the whole
6
+ book, to anyone holding the shared APP_PASSWORD. Now the pool is built with
7
+ `(team_id, agent_name)` derived from the USER RECORD, so a Royal-only user never receives a
8
+ Fisch row and an agent-linked login never receives another rep's book. The scope is applied
9
+ at the QUERY, not as a post-filter, so there is no moment at which the other BU's rows exist
10
+ in this response.
11
+
12
+ 2. **THE OVERLAY FORK IS GONE.** `aios-web/api/data/overlay.json` was a SECOND writable home
13
+ for the same user-owned fields the Streamlit app keeps in the tenant store β€” two truths, and
14
+ whichever process you asked last was right. Reads and writes now both go through
15
+ `modules.customer_data`'s table-workspace functions: ONE store (C1c, ARCHITECTURE Β§1a rule 3).
16
+
17
+ ⚠ ONE STORE IS NOT YET ONE CACHE (strangler-period, booked honestly). `core.store.get()` is
18
+ cache-first per PROCESS, so a write from the Streamlit container is invisible to a running API
19
+ container until its cache is refreshed, and vice versa. Deleting the fork removes the second
20
+ SOURCE OF TRUTH; it does not make the two runtimes coherent. The fix is X4/Postgres (task C-4,
21
+ owner-blocked on B-3), and no test here may claim read-your-writes ACROSS runtimes β€” a TestClient
22
+ proof is single-process and would report green on exactly the thing that is still broken.
23
+ """
24
+ import time
25
+
26
+ from fastapi import APIRouter, Body, Depends
27
+
28
+ from deps import Session, err, module_gate, perms
29
+
30
+ router = APIRouter(prefix="/api/v1")
31
+
32
+ #: The surface these routes serve. Both legs are gated on it, so a user without the grant gets a
33
+ #: 403 rather than an empty table that looks like "you have no customers".
34
+ MODULE = "customer_data"
35
+
36
+ _CACHE_TTL = 900 # the pool build is slow (Odoo + reconciliation); 15 min, as before
37
+
38
+
39
+ def _pool_rows(session: Session):
40
+ """The reconciled Odoo pool for this session's SCOPE β€” the slow part, and the only part that
41
+ may be shared between users.
42
+
43
+ β›” WHAT MAY BE CACHED HERE, AND WHY THE LINE IS EXACTLY HERE. The cache key is
44
+ `(team_id, agent)` and the cached value is the raw pool: Odoo-source columns only. That is
45
+ genuinely scope-shaped β€” two users with the same BU and the same book are asking the same
46
+ question, and `cl.pool()` is expensive (an Odoo pull plus reconciliation).
47
+
48
+ The FULL PAYLOAD is NOT cacheable on this key, and caching it here was a real defect I shipped
49
+ and then removed. `fields` comes from `fields_from_workspace(ws)` β€” that user's own `custom_`
50
+ and `measure_` columns β€” and every overlay cell comes from `ws['overlays']`; the table
51
+ workspace is read as `data[username]` and written as `patch_table_overlay(uname, …)`, i.e.
52
+ PER USER. So two Royal-only users with no agent link share `(6, None)` and the second one
53
+ would have been served the FIRST one's private notes and private columns. The scope key was
54
+ right for the pool and wrong for everything wrapped around it.
55
+
56
+ It survived a green 129-check battery because every fixture user had a DISTINCT
57
+ `(team_id, agent)` pair, so no two of them ever collided on the key β€” the test set could not
58
+ express the bug. `verify_api.py` now carries a same-scope second user for exactly this.
59
+ """
60
+ import modules.customer_data as cl
61
+
62
+ rt = session.runtime
63
+ team_id = perms.scope_team_id(session.user)
64
+ agent = perms.scope_agent(session.user)
65
+ key = ("pool", team_id, agent)
66
+ hit = rt.pool_cache.get(key)
67
+ now = time.time()
68
+ if hit and now - hit[0] < _CACHE_TTL:
69
+ return hit[1]
70
+ # ⚠ THE SCOPE GOES INTO THE BUILDER. `pool(agent_name, team_id)` is the same reconciled
71
+ # builder the Streamlit page uses β€” passing the scope here is what makes the isolation a
72
+ # property of the QUERY instead of a filter someone can forget to apply downstream.
73
+ rows_src = cl.pool(agent, team_id)
74
+ rt.pool_cache[key] = (now, rows_src)
75
+ # Bounded: a scope cache that only ever grows is a memory leak in a shared process.
76
+ if len(rt.pool_cache) > 16:
77
+ for stale in sorted(rt.pool_cache, key=lambda k: rt.pool_cache[k][0])[:8]:
78
+ rt.pool_cache.pop(stale, None)
79
+ return rows_src
80
+
81
+
82
+ def _payload(session: Session):
83
+ """`{fields, rows, today, pulled_at}` β€” X2 fixes this at the EXACT shape the pre-wave
84
+ `/api/customers` served, which `verify_fields_contract.py` referees.
85
+
86
+ The per-USER half (this user's field list, this user's overlay cells) is assembled fresh on
87
+ every request from one workspace read. That read is a single store `get`, which is cheap next
88
+ to the pool build β€” and it is the only arrangement in which nothing user-shaped can outlive
89
+ the request that asked for it. See `_pool_rows` for what the cache holds and why.
90
+ """
91
+ import aios_grid
92
+ from core import grid_events
93
+
94
+ rows_src = _pool_rows(session)
95
+
96
+ # ONE workspace read serves both the field list and the overlay values (doing it per stratum
97
+ # would triple the store reads per request).
98
+ ws = grid_events.table_workspace(_ctx_for(session, []), allowed_pids=None)
99
+ fields = aios_grid.fields_from_workspace(ws)
100
+ odoo_keys = [f["key"] for f in fields if f.get("source") == "odoo"]
101
+ overlay_keys = [f["key"] for f in fields if f.get("source") == "overlay"]
102
+ passthrough = {f["key"] for f in fields
103
+ if f.get("source") == "odoo" and f.get("type") in ("text", "status", "date")}
104
+ overlays = ws.get("overlays") or {}
105
+
106
+ # ⚠ `rows_src` IS THE SHARED CACHED LIST β€” read it, never mutate it. Every row below is a NEW
107
+ # dict; writing an overlay value onto `r` instead of `row` would put one user's private cell
108
+ # into the copy the next same-scope user is served, which is the leak this split exists to
109
+ # prevent, re-introduced one line lower down.
110
+ rows = []
111
+ for r in rows_src:
112
+ pid = r.get("pid")
113
+ row = {"pid": pid, "_created": r.get("_created") or ""}
114
+ for k in odoo_keys:
115
+ v = r.get(k)
116
+ row[k] = v if k in passthrough else _round(v)
117
+ stored = overlays.get(str(pid)) or {}
118
+ for k in overlay_keys:
119
+ row[k] = stored.get(k, "")
120
+ rows.append(row)
121
+
122
+ return {"fields": fields, "rows": rows,
123
+ # `today` rides the payload because every relative date condition must resolve against
124
+ # the TENANT's day, never the browser's β€” a client that falls back to its own clock
125
+ # disagrees with the server for everyone west of it.
126
+ "today": time.strftime("%Y-%m-%d"),
127
+ "pulled_at": time.strftime("%Y-%m-%d %H:%M")}
128
+
129
+
130
+ def _round(v):
131
+ # bool is a subclass of int β€” guard it so True/False never becomes 1/0. Mirrors
132
+ # royalimports_os/aios_grid.py._round exactly (embed == standalone on boolean-valued fields).
133
+ return round(v) if isinstance(v, (int, float)) and not isinstance(v, bool) else v
134
+
135
+
136
+ def _ctx_for(session: Session, pids):
137
+ """An EventCtx for the READ path β€” no fallback workspace, so a store outage is a 503 rather
138
+ than a phantom in-memory workspace an API request cannot persist."""
139
+ from core import grid_events
140
+ return grid_events.EventCtx(
141
+ uname=session.uname, allowed_pids=frozenset(pids or ()), fields=[],
142
+ admin=session.admin, fallback_ws=None, seen_ids={})
143
+
144
+
145
+ def allowed_pids(session: Session):
146
+ """The pids this session may touch β€” the POOL's own ids, so the write wall and the read scope
147
+ can never disagree.
148
+
149
+ Reads `_pool_rows` rather than `_payload`: the wall only needs identities, and going through
150
+ the full payload would pay for a workspace read and a row assembly on every write.
151
+ """
152
+ return frozenset(r["pid"] for r in _pool_rows(session) if r.get("pid") is not None)
153
+
154
+
155
+ @router.get("/customers")
156
+ def customers(session: Session = Depends(module_gate(MODULE))):
157
+ return _payload(session)
158
+
159
+
160
+ @router.patch("/customers/{pid}")
161
+ def patch_customer(pid: int, body: dict = Body(default=None),
162
+ session: Session = Depends(module_gate(MODULE))):
163
+ """Write the EDITABLE overlay stratum only β€” Odoo stays read-only, forever.
164
+
165
+ Routed through `core.grid_events.handle_one` as an `overlay_patch` event rather than writing
166
+ the store directly: that handler is where the per-key `permissions.edit` wall, the pid wall
167
+ and the truncation rules live, and a second implementation of those would be a second set of
168
+ them to keep in step. The response reports what was ACCEPTED, which is not always what was
169
+ asked for.
170
+ """
171
+ from core import grid_events
172
+
173
+ updates = dict(body or {})
174
+ if not updates:
175
+ raise err(400, "empty_patch", "no fields to update")
176
+ pool = allowed_pids(session)
177
+ if pid not in pool:
178
+ # 403, not 404: the pid may well exist β€” it is simply not in this session's book, and
179
+ # saying "no such customer" would confirm the opposite to anyone who guessed right.
180
+ raise err(403, "out_of_scope", "that customer is not in your book")
181
+ payload = _payload(session)
182
+ ctx = grid_events.EventCtx(
183
+ uname=session.uname, allowed_pids=pool, fields=payload["fields"],
184
+ admin=session.admin, fallback_ws=None, seen_ids={})
185
+ try:
186
+ grid_events.handle_one(
187
+ {"id": f"patch:{pid}:{time.time_ns()}", "type": "overlay_patch",
188
+ "pid": pid, "updates": updates}, ctx)
189
+ except grid_events.StoreUnavailable:
190
+ raise err(503, "store_unavailable",
191
+ "the tenant store is unavailable β€” your change was not saved")
192
+
193
+ # What actually landed, read back from the store rather than echoed from the request: a
194
+ # refused key or a truncated value must not be reported as accepted.
195
+ stored = (grid_events.table_workspace(_ctx_for(session, pool), allowed_pids=None)
196
+ .get("overlays") or {}).get(str(pid)) or {}
197
+ accepted = {k: stored.get(k) for k in updates if k in stored}
198
+ refused = sorted(k for k in updates if k not in accepted or stored.get(k) != str(updates[k]))
199
+ # No cache to patch: the overlay stratum is re-read from the store on every `_payload`, so
200
+ # read-your-writes within this runtime is a property of the design rather than of a
201
+ # write-through step somebody has to remember. (It was a write-through step while the whole
202
+ # payload was cached on a scope key β€” the arrangement that leaked one user's notes to
203
+ # another. See `_pool_rows`.) Cross-RUNTIME coherence is still not claimed: the module
204
+ # docstring says why, and Postgres is the fix.
205
+ out = {"ok": True, "pid": pid, "updates": accepted}
206
+ if refused:
207
+ out["refused"] = refused
208
+ return out
api/routes_grid.py ADDED
@@ -0,0 +1,155 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """routes_grid.py β€” X2's write seam over HTTP: the SECOND adapter on `core.grid_events` (EXIT-1c).
2
+
3
+ `POST /api/v1/grid/events` takes the component's event objects VERBATIM β€” the same objects the
4
+ Streamlit host receives through the component value slot, unchanged β€” and runs them through the
5
+ same `core.grid_events.handle_events` the Streamlit adapter runs. That is the whole point of
6
+ EXIT-1a: one implementation of every permission wall, two transports. A route that re-validated
7
+ anything here would be a second wall to keep in step, and the two would drift on the first change.
8
+
9
+ DEDUP IS PER REQUEST, and that is a deliberate limit, not an oversight. The Streamlit adapter's
10
+ `seen_ids` lives in `st.session_state` β€” genuinely per user session β€” because the client resends
11
+ its recent 24-event window on every emit inside one page session. A stateless API has no such
12
+ dict, and inventing a per-user server-side one would be exactly the resident per-tenant state
13
+ EXIT-4a exists to remove. So: ids are deduped WITHIN a request body (the resend window's whole
14
+ purpose β€” a batch that repeats an id processes it once), and a genuinely replayed request is
15
+ handled by the operations being idempotent. The one event where a replay is observable is
16
+ `add_to_list` (it would add the same pids to the same cohort twice β€” a set union, so the
17
+ membership is unchanged, but the toast count repeats). Noted rather than papered over; a
18
+ server-side idempotency key belongs with D2's session mirror (C-2).
19
+
20
+ STORE DOWN = 503. `fallback_ws` is None on this adapter, so `core.grid_events` raises
21
+ `StoreUnavailable` rather than writing to an in-memory workspace no API request could ever read
22
+ back. A 200 over a write that evaporated is the failure this rule exists to prevent.
23
+ """
24
+ import time
25
+
26
+ from fastapi import APIRouter, Body, Depends
27
+
28
+ from deps import Session, err, module_gate
29
+
30
+ router = APIRouter(prefix="/api/v1")
31
+
32
+ MODULE = "customer_data"
33
+
34
+ #: The client's own resend window (`app.py`'s `[-24:]`). A body larger than this is not a
35
+ #: legitimate client β€” refuse it rather than doing 500 store writes on one request.
36
+ _MAX_EVENTS = 24
37
+
38
+
39
+ def _ctx(session: Session, fields, pids, **kw):
40
+ from core import grid_events
41
+ return grid_events.EventCtx(
42
+ uname=session.uname, allowed_pids=pids, fields=fields, admin=session.admin,
43
+ fallback_ws=None, seen_ids={}, **kw)
44
+
45
+
46
+ #: The two surfaces the ONE grid serves. `cohort` is the same table over hand-curated SETS rather
47
+ #: than over the whole scoped pool β€” `app.py:page_cohort` calls the same `_table_grid` with
48
+ #: `cohort_mode=True, scope_key='cohort'`.
49
+ _SCOPES = ("customer", "cohort")
50
+
51
+
52
+ def _scope_or_400(raw):
53
+ """β›” REFUSE AN UNKNOWN SCOPE, never default it. A typo silently served as `customer` would
54
+ hand a user the whole book on a page they opened to see one cohort β€” the widening direction,
55
+ which is the one that must fail closed."""
56
+ scope = (raw or "customer").strip().lower()
57
+ if scope not in _SCOPES:
58
+ raise err(400, "bad_scope",
59
+ f"scope must be one of {', '.join(_SCOPES)} β€” refusing to guess")
60
+ return scope
61
+
62
+
63
+ @router.get("/workspace")
64
+ def workspace(scope: str = "customer",
65
+ session: Session = Depends(module_gate(MODULE))):
66
+ """The durable table workspace for this session: views, fields, overlays, folders.
67
+
68
+ ⚠ WHY THIS EXISTS (S1↔S2, 2026-07-30 β€” an X2 AMENDMENT, see the split doc). X2 fixes
69
+ `/customers` at the exact shape `verify_fields_contract.py` referees, so the workspace cannot
70
+ ride along in it without forking the thing that gate keeps single-sourced. Without a
71
+ workspace route the standalone shell's write path would be WRITE-ONLY: events persist
72
+ server-side and nothing reads them back on reload, so a saved view looks lost to the user
73
+ even though the store has it. This route closes that read-back gap at its own URL.
74
+
75
+ `allowed_pids` is passed so a SHARED view's `memberPids` are re-scoped to THIS reader β€” the
76
+ wave-9 leak rule. Omitting it would hand a Fisch-scoped user a member list built by a
77
+ full-access user.
78
+ """
79
+ from core import grid_events
80
+ from routes_customers import allowed_pids
81
+
82
+ scope = _scope_or_400(scope)
83
+ try:
84
+ pids = allowed_pids(session)
85
+ ws = grid_events.table_workspace(_ctx(session, [], pids), allowed_pids=pids)
86
+ except grid_events.StoreUnavailable:
87
+ raise err(503, "store_unavailable", "the tenant store is unavailable")
88
+
89
+ # β›” THE SURFACE STAMP β€” a MIRROR of `app.py:5830-5836`, and it must stay one. These four keys
90
+ # are the ONLY thing that tells the client which surface it is drawing: `CustomerGrid.tsx:685`
91
+ # reads `payload.workspace.cohortMode` and nothing else. Until this route could express them,
92
+ # the standalone shell had no way to render Cohort at all β€” the write path already accepted
93
+ # `scopeKey` (below), so the read path was the whole gap.
94
+ #
95
+ # ⚠ Deliberately NOT `hideViews`. The Streamlit Cohort page does not set it either: `cohortMode`
96
+ # is what swaps the Views panel for the cohort list, so setting both would be two switches for
97
+ # one behaviour and they would drift.
98
+ ws = dict(ws) # never stamp the object the store handed back
99
+ ws["scopeKey"] = scope
100
+ if scope == "cohort":
101
+ ws["cohortMode"] = True
102
+ # The create pane offers "this cohort table only" (default) vs "all customer tables".
103
+ ws["scopeChoice"] = True
104
+ return {"workspace": ws}
105
+
106
+
107
+ @router.post("/grid/events")
108
+ def grid_events_route(body: dict = Body(default=None),
109
+ session: Session = Depends(module_gate(MODULE))):
110
+ """`{events: [<component event objects, verbatim>]}` β†’ `{results, doc?, toast?}`."""
111
+ from core import grid_events
112
+ from routes_customers import _payload, allowed_pids
113
+
114
+ events = (body or {}).get("events")
115
+ if events is None and isinstance(body, dict) and body.get("type"):
116
+ events = [body] # a single event object, the legacy shape
117
+ if not isinstance(events, list):
118
+ raise err(400, "bad_events", "expected {events: [...]}")
119
+ if len(events) > _MAX_EVENTS:
120
+ raise err(400, "too_many_events",
121
+ f"at most {_MAX_EVENTS} events per request (the client's resend window)")
122
+
123
+ payload = _payload(session)
124
+ pids = frozenset(r["pid"] for r in payload["rows"] if r.get("pid") is not None)
125
+ # Validated with the SAME predicate as the read route. An unrecognised scopeKey used to be
126
+ # passed through verbatim, and `core.grid_events` only ever compares it to 'cohort' β€” so a
127
+ # typo degraded silently to customer-scope behaviour on a WRITE. Read and write must agree on
128
+ # what a scope is, or the surface you read is not the surface you wrote.
129
+ ctx = _ctx(session, payload["fields"], pids,
130
+ scope_key=_scope_or_400((body or {}).get("scopeKey")))
131
+
132
+ # Per-event results so the client can tell which of a batch landed β€” the component's own
133
+ # bridge has no response channel at all, so this is strictly more than the embed gets.
134
+ results = []
135
+ try:
136
+ for one in events:
137
+ eid = str(one.get("id") or "") if isinstance(one, dict) else ""
138
+ rerender = grid_events.handle_one(one, ctx)
139
+ results.append({"id": eid, "rerender": bool(rerender)})
140
+ except grid_events.StoreUnavailable:
141
+ raise err(503, "store_unavailable",
142
+ "the tenant store is unavailable β€” none of your changes were saved")
143
+
144
+ out = {"results": results, "rerender": any(r["rerender"] for r in results)}
145
+ if ctx.out.doc is not None:
146
+ out["doc"] = ctx.out.doc
147
+ if ctx.out.toast is not None:
148
+ out["toast"] = ctx.out.toast
149
+ # ⚠ NOTHING IS INVALIDATED HERE, on purpose. The runtime cache holds ONLY the scope-shaped
150
+ # Odoo pool (see `routes_customers._pool_rows`), and no event on this route can change an
151
+ # Odoo column β€” Odoo is read-only. Everything an event DOES change (overlays, fields, views,
152
+ # folders, cohorts) is re-read from the store on the next request. An earlier version cleared
153
+ # `pool_cache` after an `overlay_patch`, which threw away an expensive Odoo pull to refresh
154
+ # data that was never in it.
155
+ return out
api/routes_nav.py ADDED
@@ -0,0 +1,34 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """routes_nav.py β€” X2's `GET /api/v1/nav`: the registry, filtered by what this session may open.
2
+
3
+ SERVER-FILTERED, not client-filtered. The client renders what it is given and never decides who
4
+ may see what β€” a nav that hides a link the API would still serve is a UI courtesy, not a
5
+ permission. `core.perms.nav_pages` is the single predicate (shared with `may_open`'s page gate),
6
+ so the nav and the 403 can never disagree about a grant.
7
+
8
+ The full rule set β€” archived is invisible to everyone, `group_only` rows are excluded so no
9
+ `parent` reference can dangle, `nav: False` surfaces ship as `chrome: 'utility'`, and the
10
+ `prefs.json` Library preference is deliberately NOT applied β€” is documented on
11
+ `core.perms.nav_pages`, which is where it belongs: one place, both callers.
12
+ """
13
+ from fastapi import APIRouter, Depends
14
+
15
+ from deps import Session, perms, require_session
16
+
17
+ router = APIRouter(prefix="/api/v1")
18
+
19
+
20
+ @router.get("/nav")
21
+ def nav(session: Session = Depends(require_session)):
22
+ """`{pages: [{key,label,source?,chrome}], landing}`.
23
+
24
+ Note what this does NOT do: it never returns an empty `pages` list as a way of saying "you
25
+ are not allowed". A session that may open nothing at all is a misconfigured account, and it
26
+ gets an explicit 403 β€” an empty 200 is indistinguishable from "the registry is empty" and is
27
+ how a permission bug hides in plain sight (X2's never-an-empty-200 rule).
28
+ """
29
+ pages = perms.nav_pages(session.user)
30
+ if not pages:
31
+ from deps import err
32
+ raise err(403, "no_surfaces",
33
+ "your account has no dashboards assigned β€” ask an administrator")
34
+ return {"pages": pages, "landing": perms.landing_page(session.user)}
api/routes_pages.py ADDED
@@ -0,0 +1,65 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """routes_pages.py β€” X2/Y1's read of a whole dashboard page: `GET /api/v1/pages/{key}`.
2
+
3
+ ONE route serves every ported page. That is the wave-3 promise made mechanical: Collections and
4
+ Procurement arrive as a builder registered in `pages.py`, and they inherit this route's session
5
+ wall, grant wall, BU resolution, caching and error shape without a line being written here.
6
+
7
+ WHY THE GATE IS INSIDE THE HANDLER and not `Depends(module_gate(...))`. `module_gate` closes over a
8
+ FIXED module key at import; the key here is a PATH PARAMETER, so the check has to happen once the
9
+ key is known. It is the same predicate (`Session.require` β†’ `perms.may_open`), reached from
10
+ `pages.build_envelope` so a future builder cannot be added without it.
11
+
12
+ FAIL-CLOSED, and the order matters:
13
+ no cookie -> 401 (require_session)
14
+ unknown / unported page key -> 404 (a fact about our roadmap, not about this user)
15
+ no grant for the page -> 403 (never an empty 200 β€” an empty page reads as "no data")
16
+ a BU the account may not see -> 403 (never coerced to their own BU)
17
+ a malformed bu -> 400
18
+ """
19
+ from fastapi import APIRouter, Depends, Query
20
+
21
+ import pages
22
+ # β›” IMPORTED HERE AT MODULE LEVEL ON PURPOSE, even though `pages.builder_for` also imports it
23
+ # lazily. `deploy_web.py` ships a hand-maintained file set, and W2-11 adds a `check_include()`-style
24
+ # IMPORT WALK to catch what that set misses β€” a walk that only follows module-level imports would
25
+ # never reach a builder registered from inside a function body, and a missing module CRASHED the
26
+ # live Space on 2026-07-04. One visible import makes the builder reachable by static analysis; the
27
+ # lazy import in `pages.py` stays as the fallback that keeps the registry self-assembling.
28
+ # (No circularity: `pages` is fully loaded by the line above before `pages_sales` imports it back.)
29
+ import pages_sales # noqa: F401
30
+ from deps import Session, require_session
31
+
32
+ router = APIRouter(prefix="/api/v1")
33
+
34
+
35
+ @router.get("/pages/{key}")
36
+ def page_data(key: str,
37
+ bu: str = Query(default=None,
38
+ description="'all' or a business-unit id. NARROWS the session's own "
39
+ "scope; it can never widen it."),
40
+ period: str = Query(default=None,
41
+ description="Page-specific display parameter. The resolved value "
42
+ "is echoed in controls[].value."),
43
+ session: Session = Depends(require_session)):
44
+ """The Y1 envelope for one page, scoped to this session's BU + own-book grant."""
45
+ return pages.build_envelope(session, key, bu=bu, period=period)
46
+
47
+
48
+ @router.get("/pages")
49
+ def page_index(session: Session = Depends(require_session)):
50
+ """Which pages have published data, filtered to what this session may open.
51
+
52
+ The shell needs this to know whether a nav entry leads to a ported page or to the Streamlit
53
+ host during the strangler period β€” and it is `may_open`-filtered for the same reason
54
+ `/api/v1/nav` is: the client renders what it is given and never decides who may see what.
55
+
56
+ ⚠ AN EMPTY LIST IS A LEGITIMATE 200 HERE, and that is not a hole in the never-an-empty-200 rule.
57
+ `/api/v1/nav` 403s on empty because a session that may open NOTHING is a misconfigured account.
58
+ This route answers "which of your pages have been PORTED yet", and during the strangler period
59
+ "none of them" is the ordinary, truthful answer for a user granted only surfaces that still
60
+ live in the Streamlit host. Returning 403 would say they lack access to pages they can see.
61
+ """
62
+ from deps import perms
63
+ keys = [k for k in pages.known_pages()
64
+ if perms.may_open(session.user, (pages.builder_for(k).MODULE))]
65
+ return {"pages": keys}
requirements.txt ADDED
@@ -0,0 +1,12 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # AIOS web API β€” the FastAPI backend + the royalimports_os data-layer deps it reuses.
2
+ # (streamlit is included only because the shared modules/core may import it at load; it is never
3
+ # run here. No altair/plotly/openpyxl/reportlab β€” those are Streamlit-UI/export only.)
4
+ fastapi>=0.139
5
+ uvicorn[standard]>=0.30
6
+ python-dotenv>=1.0
7
+ pandas>=2.0
8
+ requests>=2.28
9
+ huggingface_hub>=0.20
10
+ duckdb>=1.0
11
+ pyyaml>=6.0
12
+ streamlit==1.58.0
royalimports_os/aios_grid.py ADDED
@@ -0,0 +1,1221 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """aios_grid β€” embed the AIOS React/glide Airtable-style grid inside Streamlit.
2
+
3
+ This is REUSABLE MODULE INFRASTRUCTURE: the same self-contained grid that runs as the
4
+ standalone aios-web app is inlined into a single HTML file and hosted inside a Streamlit
5
+ component. The React bundle reads its data from `window.__AIOS_GRID__` (an object
6
+ `{fields, rows}`) that we inject into the page <head> before the app's module script runs
7
+ β€” so there is NO backend and NO /api call in the Streamlit container; the browser only ever
8
+ sees the derived JSON we hand it.
9
+
10
+ Usage (from any page):
11
+ import aios_grid
12
+ aios_grid.render(aios_grid.rows_from_pool(pool_rows), aios_grid.FIELDS)
13
+
14
+ Design notes:
15
+ * ZERO app-internal imports (no `modules.*` / `core.*`). FIELDS is a literal and
16
+ `rows_from_pool` takes already-built rows β€” so this helper is tenant/module-agnostic and
17
+ sidesteps deploy_hf.py's import guard entirely.
18
+ * The built HTML ships as `aios_grid_embed.html` in THIS directory (added to
19
+ deploy_hf.py INCLUDE). Build it with `npm run build:embed` in aios-web/web, then copy
20
+ dist-embed/index.html -> royalimports_os/aios_grid_embed.html (see build_embed.py).
21
+ For LOCAL dev before that copy, we fall back to reading dist-embed/index.html directly.
22
+ * The preferred host is a Streamlit Components v1 bridge. It sends data/view/schema args
23
+ into the React app and returns guarded events for persistence in the tenant store. The old
24
+ injected-HTML path remains a read/local-write fallback when component assets are absent.
25
+ """
26
+ import json
27
+ import re
28
+ from pathlib import Path
29
+
30
+ _HERE = Path(__file__).resolve().parent
31
+
32
+ # Where the inlined single-file build lives. FIRST match wins:
33
+ # 1. the shipped copy in this dir (what deploy_hf.py uploads to the Space)
34
+ # 2. the raw build output in the sibling aios-web tree (local dev, pre-copy)
35
+ _EMBED_CANDIDATES = [
36
+ _HERE / "aios_grid_embed.html",
37
+ _HERE.parent / "aios-web" / "web" / "dist-embed" / "index.html",
38
+ ]
39
+ _COMPONENT_CANDIDATES = [
40
+ _HERE / "aios_grid_component",
41
+ _HERE.parent / "aios-web" / "web" / "dist-embed",
42
+ ]
43
+ _DECLARED_COMPONENTS = {}
44
+
45
+ # --- the FIELD CONTRACT β€” loaded from the CANONICAL source `aios_grid_fields.json` in THIS
46
+ # directory (the SINGLE source of truth, shared with aios-web/api/main.py). source='odoo' is
47
+ # READ-ONLY; source='overlay' is the editable stratum (notes/tags) that lives OUTSIDE Odoo.
48
+ #
49
+ # Why a sibling JSON and NOT an import: aios_grid.py's contract is ZERO app-internal imports so
50
+ # it stays tenant/module-agnostic and sidesteps deploy_hf.py's import guard. A JSON next to the
51
+ # module preserves that exactly β€” no import, no guard interaction β€” while still single-sourcing
52
+ # the values (a build-time copy from another tree would reintroduce the drift we're removing).
53
+ # The file ships to the Space via deploy_hf.py INCLUDE. Edit the JSON, then run
54
+ # aios-web/verify_fields_contract.py. ---
55
+ _FIELDS_PATH = _HERE / "aios_grid_fields.json"
56
+
57
+
58
+ def _load_fields():
59
+ if not _FIELDS_PATH.is_file():
60
+ raise FileNotFoundError(
61
+ f"aios_grid: canonical field contract missing at {_FIELDS_PATH}. It is the single "
62
+ "source of truth for the grid schema and MUST ship (deploy_hf.py INCLUDE lists it)."
63
+ )
64
+ doc = json.loads(_FIELDS_PATH.read_text(encoding="utf-8"))
65
+ fields = doc.get("fields") if isinstance(doc, dict) else doc
66
+ if not isinstance(fields, list) or not fields:
67
+ raise ValueError(f"aios_grid: {_FIELDS_PATH} has no 'fields' list.")
68
+ return fields
69
+
70
+
71
+ FIELDS = _load_fields()
72
+
73
+ # text/date fields pass through untouched; every OTHER odoo (numeric) field is rounded β€”
74
+ # mirrors aios-web/api/main.py _payload() exactly so embed == standalone byte-for-byte.
75
+
76
+
77
+ def _round(v):
78
+ return round(v) if isinstance(v, (int, float)) and not isinstance(v, bool) else v
79
+
80
+
81
+ # Types a USER may create from the column menu (owner item 7, 2026-07-26). Mirrors
82
+ # customer-grid/types.ts CREATABLE_TYPES; verify_fields_contract.py holds the two in step.
83
+ # select β€” a single-select with its own `options` (the "Status" a user wants to add; distinct
84
+ # from the Odoo-sourced `status` lifecycle, which is not user-defined)
85
+ # user β€” an assignee, whose choices come from the HOST's real user list, never from here
86
+ #: `multiselect` (wave-2 item 5, 2026-07-27): the Airtable-style MULTI select β€” declared
87
+ #: options like `select`, but the cell holds a comma-joined SET and the row belongs to every
88
+ #: member of it (the `multi` grouping/cell contract the Cohorts column established).
89
+ #: Wave-5 item 11 (2026-07-27): `checkbox` (cell = bool; the overlay stores '1' or '') Β·
90
+ #: `phone` / `email` / `url` (text-family with per-type rendering) Β· `rating` (top-level
91
+ #: `max`, SVG stars client-side β€” never emoji) Β· `created_time` (READ-ONLY, renders the row's
92
+ #: `_created`) Β· `formula` (READ-ONLY, client-computed from the row's other cells).
93
+ CUSTOM_FIELD_TYPES = {"text", "select", "multiselect", "user", "int", "currency", "pct", "date",
94
+ "checkbox", "phone", "email", "url", "rating", "created_time", "formula"}
95
+ #: User-created types whose CELLS are read-only: their values are computed (formula β€” client
96
+ #: side, any error degrades to BLANK) or system-owned (created_time = the row's `_created`).
97
+ #: Emitted with the cohort column's read-only mechanism β€” `source: 'odoo'` + `derived` β€” so
98
+ #: the client never offers an editor and the host never accepts a cell write for them.
99
+ READONLY_CUSTOM_TYPES = {"created_time", "formula"}
100
+ MAX_FIELD_OPTIONS = 50
101
+ MAX_FORMULA_LEN = 500
102
+ #: `rating` bounds. Airtable caps at 10; below 2 a rating is a checkbox.
103
+ RATING_MAX_DEFAULT, RATING_MAX_MIN, RATING_MAX_MAX = 5, 2, 10
104
+
105
+ #: Key prefix of a FORMULA-MEASURE column (owner item 7, 2026-07-27): a user-created field that
106
+ #: IS a measure over a window β€” `Sales Β· the last 90 days` as a column. Mirrors the client's
107
+ #: `measure_` keys in CustomerGrid.createField. Distinct from `custom_` because the two strata
108
+ #: could not be more different: `custom_` is the EDITABLE overlay (user-typed values), while a
109
+ #: measure field is READ-ONLY and its values are computed by the host per render.
110
+ MEASURE_FIELD_PREFIX = "measure_"
111
+ #: The numeric types a measure can render as (semantic._FORMAT_TYPE's range).
112
+ MEASURE_FIELD_TYPES = {"currency", "int", "pct"}
113
+
114
+
115
+ def _clean_options(raw):
116
+ """Choices for a `select`: strings, trimmed, de-duplicated case-insensitively, capped.
117
+ Mirrors types.ts parseOptions β€” a choice list that means one thing in the picker and
118
+ another in the filter dropdown is a column with two vocabularies."""
119
+ out, seen = [], set()
120
+ for v in list(raw or [])[:MAX_FIELD_OPTIONS * 2]:
121
+ if not isinstance(v, (str, int, float)) or isinstance(v, bool):
122
+ continue
123
+ s = str(v).strip()[:120]
124
+ if not s or s.lower() in seen:
125
+ continue
126
+ seen.add(s.lower())
127
+ out.append(s)
128
+ return out[:MAX_FIELD_OPTIONS]
129
+
130
+
131
+ def _clean_rating_max(raw):
132
+ """A rating's star count, bounded. Anything unusable is the default, not a refusal β€” the
133
+ field still holds its 1..max integers either way."""
134
+ try:
135
+ return max(RATING_MAX_MIN, min(int(raw), RATING_MAX_MAX))
136
+ except (TypeError, ValueError):
137
+ return RATING_MAX_DEFAULT
138
+
139
+
140
+ #: The field types a number-style display format may apply to. `formula` is here because its
141
+ #: RESULT is a number the client renders; `pct` already renders in points and takes decimals.
142
+ _NUMBER_FORMAT_TYPES = {"int", "currency", "pct", "formula"}
143
+
144
+
145
+ def _clean_format(raw, ftype):
146
+ """Per-type DISPLAY format (wave-5 item 10), fail-closed: unknown keys are DROPPED, wrong
147
+ types return None (the property is simply absent). Rendering-only β€” a format can change how
148
+ a value reads, never what it is, which is why this needs no parity gate of its own."""
149
+ if not isinstance(raw, dict):
150
+ return None
151
+ out = {}
152
+ if ftype in _NUMBER_FORMAT_TYPES:
153
+ if isinstance(raw.get("thousands"), bool):
154
+ out["thousands"] = raw["thousands"]
155
+ if raw.get("decimals") is not None:
156
+ try:
157
+ d = int(raw["decimals"])
158
+ except (TypeError, ValueError):
159
+ d = None
160
+ if d is not None and 0 <= d <= 4:
161
+ out["decimals"] = d
162
+ if isinstance(raw.get("abbrev"), bool):
163
+ out["abbrev"] = raw["abbrev"]
164
+ elif ftype in ("date", "created_time"):
165
+ if isinstance(raw.get("time"), bool):
166
+ out["time"] = raw["time"]
167
+ if raw.get("tz") in ("local", "utc"):
168
+ out["tz"] = raw["tz"]
169
+ return out or None
170
+
171
+
172
+ def _clean_permissions(raw):
173
+ """`{edit: 'everyone' | 'creator' | 'admins'}` or None (wave-5 item 1). WHO may set it is
174
+ the host handler's business (creator/admin, enforced fail-closed there); this validates
175
+ only the shape, like every other property here."""
176
+ if isinstance(raw, dict) and raw.get("edit") in ("everyone", "creator", "admins"):
177
+ return {"edit": raw["edit"]}
178
+ return None
179
+
180
+
181
+ _FORMULA_CHARS = re.compile(r"^[\w\s{}()+\-*/.,<>=!]*$")
182
+ _FORMULA_REF = re.compile(r"\{([^{}]*)\}")
183
+
184
+
185
+ def _clean_formula(raw, valid_keys=None):
186
+ """STRUCTURAL passthrough for a formula field's expression (wave-5 item 9).
187
+
188
+ Meaning is NOT checked here: the CLIENT engine owns the grammar (arithmetic over `{field}`
189
+ refs, ABS/ROUND/MIN/MAX/IF), and any evaluation error degrades to a BLANK cell β€” never a
190
+ wrong number. That is the `_clean_window` split one stratum up, and deliberately NOT a
191
+ Python mirror of the grammar: a second engine is the filter_sql-class drift risk.
192
+ Structure IS checked β€” charset, length, balanced parens, well-formed non-empty `{refs}` β€”
193
+ and at WRITE time (`valid_keys` given) every ref must name a field this table has, fail
194
+ closed. At READ time refs are left alone: a referenced field deleted later must blank the
195
+ CELLS, not vaporise the column.
196
+ """
197
+ if not isinstance(raw, str):
198
+ return None
199
+ s = raw.strip()
200
+ if not s or len(s) > MAX_FORMULA_LEN or not _FORMULA_CHARS.match(s):
201
+ return None
202
+ depth = 0
203
+ for ch in s:
204
+ if ch == "(":
205
+ depth += 1
206
+ elif ch == ")":
207
+ depth -= 1
208
+ if depth < 0:
209
+ return None
210
+ if depth:
211
+ return None
212
+ refs = _FORMULA_REF.findall(s)
213
+ leftover = _FORMULA_REF.sub("", s)
214
+ if "{" in leftover or "}" in leftover: # unbalanced / nested braces
215
+ return None
216
+ if any(not r.strip() for r in refs): # a `{}` ref names nothing
217
+ return None
218
+ if valid_keys is not None and any(r not in valid_keys for r in refs):
219
+ return None
220
+ return s
221
+
222
+
223
+ def _field_extras(saved, ftype):
224
+ """createdBy / permissions / format / scope β€” the validated passthrough the created strata
225
+ share (wave 5). `createdBy` is only ever WRITTEN host-side (the handler stamps it); here it
226
+ is carried so the client can gate its menus and the handler can enforce against it.
227
+ `scope` (wave-6 item 9): 'cohort' marks a field created as cohort-specific β€” the handler
228
+ stamps it at create (cohort page only) and preserves it like createdBy; carried here so the
229
+ client can label the field, filtered OUT of other pages by fields_from_workspace."""
230
+ out = {}
231
+ who = saved.get("createdBy")
232
+ if isinstance(who, str) and who.strip():
233
+ out["createdBy"] = who.strip()[:80]
234
+ perms = _clean_permissions(saved.get("permissions"))
235
+ if perms:
236
+ out["permissions"] = perms
237
+ fmt = _clean_format(saved.get("format"), ftype)
238
+ if fmt:
239
+ out["format"] = fmt
240
+ if saved.get("scope") == "cohort":
241
+ out["scope"] = "cohort"
242
+ return out
243
+
244
+
245
+ #: The DERIVED column listing the cohorts a customer is in (owner, 2026-07-27).
246
+ #:
247
+ #: NOT in `aios_grid_fields.json`, deliberately. That contract is per-TABLE and shared with the
248
+ #: standalone API and the dev sample; a cohort is per-USER, so the column exists exactly when the
249
+ #: caller has cohorts β€” the same condition under which the `__cohort__` FILTER field is offered.
250
+ #: Putting it in the canonical contract would mean an always-present column that is empty for
251
+ #: everyone else, plus three consumers to keep in step for a value none of them can produce.
252
+ COHORT_COLUMN = 'cohorts'
253
+
254
+
255
+ def cohort_field(label='Cohorts'):
256
+ """The derived cohort column's descriptor.
257
+
258
+ `source: 'odoo'` is doing ONE job here and it is not provenance: the client keys editability
259
+ off `source == 'overlay'`, and `_cl_handle_grid_event` accepts cell writes only for overlay
260
+ keys. So 'odoo' is what makes this column READ-ONLY at both ends. `derived: True` is what
261
+ stops the column menu calling it a "source field" on that basis.
262
+
263
+ ⚠ `filterable: False`, and the replacement is the `Where [Cohort] […]` CONDITION, not another
264
+ column. Text ops over a joined string would ALMOST work and disagree at the edges β€” `contains
265
+ "VIP"` also matches a cohort called "VIP club" β€” and a filter that is nearly right is worse
266
+ than one that is absent.
267
+ """
268
+ return {
269
+ 'key': COHORT_COLUMN, 'label': label, 'type': 'text', 'source': 'odoo',
270
+ 'default': False, 'filterable': False, 'derived': True, 'multi': True,
271
+ 'note': 'The cohorts this customer belongs to, newest membership last. A cohort is a '
272
+ 'LIST, so grouping by this column puts a customer under EVERY cohort they are '
273
+ 'in β€” the group counts therefore add up to more than the record count, which '
274
+ 'stays the number of distinct customers. Read-only: a cohort changes only on '
275
+ 'the Cohort page. To FILTER by one, use the Cohort condition in Filter.',
276
+ }
277
+
278
+
279
+ def cohort_cells(cohorts, allowed_pids=None):
280
+ """`{pid: {'cohorts': 'Q3 calls, Lost'}}` from `[{id,name,pids}, ...]`.
281
+
282
+ Built per render from the caller's OWN cohorts and handed to `rows_from_pool` as `derived`,
283
+ never merged into the cached pool rows β€” those are shared across users, and stamping one
284
+ user's cohorts onto them would leak the membership to everybody else on the next render.
285
+ """
286
+ cells = {}
287
+ for c in cohorts or []:
288
+ # ⚠ The COMMA is the separator the client splits on to group a customer into EVERY
289
+ # cohort they are in, so it cannot also occur inside a name. Cohort names are free text
290
+ # ("Q3 calls, west" is a name somebody will type), so a comma is replaced here rather
291
+ # than left to break the split silently β€” one group called "Q3 calls" and another called
292
+ # "west" would be two lists that do not exist. The cost is cosmetic and confined to the
293
+ # cell; the Cohort page still shows the name the user typed.
294
+ name = str(c.get('name') or c.get('id') or '').replace(',', ' ').strip()
295
+ for pid in c.get('pids') or ():
296
+ if allowed_pids is not None and pid not in allowed_pids:
297
+ continue
298
+ cells.setdefault(pid, []).append(name)
299
+ return {pid: {COHORT_COLUMN: ', '.join(names)} for pid, names in cells.items()}
300
+
301
+
302
+ def clean_measure_field(raw, offered):
303
+ """Validate one UNTRUSTED formula-measure field (owner item 7) against the caller's OFFER.
304
+
305
+ `offered` is `{measure key: {label, type}}` from `measure_filter.measure_fields(team_id)` β€”
306
+ the same admission the condition builder uses, so a field can only name a measure this
307
+ caller could also filter by. Returns the canonical stored shape, or None (fail closed).
308
+
309
+ `source:'odoo'` + `derived:True` is the cohort column's read-only mechanism, reused:
310
+ the client keys editability off `source == 'overlay'` and the host accepts cell writes only
311
+ for overlay keys, so a measure column cannot be typed into at either end. `filterable:False`
312
+ because the REPLACEMENT is the measure CONDITION with the same measure and window β€” the
313
+ governed, gate-proved path (CG-8/CG-12) β€” not text ops over a derived cell.
314
+ """
315
+ if not isinstance(raw, dict):
316
+ return None
317
+ key = str(raw.get("key") or "")
318
+ if not key.startswith(MEASURE_FIELD_PREFIX) or len(key) > 80:
319
+ return None
320
+ spec = raw.get("measure")
321
+ if not isinstance(spec, dict):
322
+ return None
323
+ m = (offered or {}).get(spec.get("key"))
324
+ if not m:
325
+ return None # not admitted for this caller -> fail closed
326
+ window = _clean_window(spec.get("window"))
327
+ if window is None:
328
+ return None # a measure column with no period is not a column
329
+ mtype = m.get("type") if m.get("type") in MEASURE_FIELD_TYPES else "currency"
330
+ return {
331
+ "key": key,
332
+ "label": str(raw.get("label") or m.get("label") or "Measure")[:120],
333
+ "type": mtype,
334
+ "source": "odoo",
335
+ "default": True,
336
+ "custom": True,
337
+ "derived": True,
338
+ "filterable": False,
339
+ "agg": "sum" if mtype in ("currency", "int") else None,
340
+ "note": str(raw.get("note") or "")[:2000],
341
+ "measure": {"key": str(spec.get("key"))[:80], "window": window},
342
+ }
343
+
344
+
345
+ def measure_fields_of(fields):
346
+ """The formula-measure columns among `fields` β€” the ones whose values the host must compute
347
+ per render (see `rows_from_pool`'s `derived`)."""
348
+ return [f for f in fields or [] if isinstance(f.get("measure"), dict)]
349
+
350
+
351
+ def fields_from_workspace(workspace=None, cohorts=False, scope_key=None):
352
+ """Overlay persisted notes/custom fields onto the immutable source-field contract.
353
+
354
+ `cohorts=True` appends the derived cohort column β€” see `cohort_field`. Appended LAST and
355
+ `default: False`, so it never displaces a column somebody already reads; the Fields menu is
356
+ where you turn it on.
357
+
358
+ Three saved strata pass through: notes on base fields, `custom_` overlay fields (editable),
359
+ and `measure_` formula-measure fields (read-only, host-computed β€” owner item 7). A measure
360
+ field was validated against the caller's measure OFFER when it was written
361
+ (`clean_measure_field`); here only its SHAPE is re-checked, because this module has zero
362
+ app-internal imports and cannot know the offer. A measure that has since become
363
+ unanswerable (a BU scope on a company-level measure) degrades to a BLANK column at value
364
+ time, never to an error.
365
+
366
+ `scope_key` (wave-6 item 9) names the PAGE doing the asking ('customer' / 'cohort'). A
367
+ saved def carrying `scope` is emitted only when it matches β€” so a cohort-specific field
368
+ never appears on the Customer table. FAIL-CLOSED: a caller that passes no scope_key sees
369
+ only unscoped (global) fields; base contract fields are never scoped.
370
+ """
371
+ saved = dict((workspace or {}).get("fields") or {})
372
+ out = []
373
+ base_keys = {field["key"] for field in FIELDS}
374
+ for base in FIELDS:
375
+ meta = saved.get(base["key"]) or {}
376
+ field = dict(base)
377
+ if isinstance(meta.get("note"), str):
378
+ field["note"] = meta["note"][:2000]
379
+ # Wave-5 item 10: a saved DISPLAY format on any base field (a preset included) β€” how a
380
+ # number or date READS, per user. Rendering-only, so this is the whole acceptance.
381
+ fmt = _clean_format(meta.get("format"), base.get("type"))
382
+ if fmt:
383
+ field["format"] = fmt
384
+ # PRESET measure fields (wave-2 item 8): a preset+measure base may take a saved
385
+ # window/label override. Wave 6 deleted every preset member (the owner's
386
+ # no-buildable-presets rule) so this branch is currently MEMBERLESS β€” kept as the
387
+ # measure_ path's twin for any future preset-carrying contract, and because deleting
388
+ # it would silently change what a re-added preset means.
389
+ if base.get("preset") and isinstance(base.get("measure"), dict):
390
+ saved_measure = meta.get("measure") if isinstance(meta.get("measure"), dict) else {}
391
+ window = _clean_window(saved_measure.get("window"))
392
+ if window is not None:
393
+ field["measure"] = {"key": base["measure"]["key"], "window": window}
394
+ if isinstance(meta.get("label"), str) and meta["label"].strip():
395
+ field["label"] = meta["label"][:120]
396
+ out.append(field)
397
+ for key, field in saved.items():
398
+ if key in base_keys or not isinstance(field, dict):
399
+ continue
400
+ if field.get("scope") and field.get("scope") != scope_key:
401
+ continue # a cohort-specific field on another page (wave-6 item 9)
402
+ if (str(key).startswith(MEASURE_FIELD_PREFIX)
403
+ and isinstance(field.get("measure"), dict)):
404
+ window = _clean_window(field["measure"].get("window"))
405
+ mkey = str(field["measure"].get("key") or "")
406
+ if window is None or not mkey:
407
+ continue
408
+ mtype = field.get("type") if field.get("type") in MEASURE_FIELD_TYPES else "currency"
409
+ out.append({
410
+ "key": str(key)[:80],
411
+ "label": str(field.get("label") or "Measure")[:120],
412
+ "type": mtype,
413
+ "source": "odoo",
414
+ "default": bool(field.get("default", True)),
415
+ "custom": True,
416
+ "derived": True,
417
+ "filterable": False,
418
+ "agg": "sum" if mtype in ("currency", "int") else None,
419
+ "note": str(field.get("note") or "")[:2000],
420
+ "measure": {"key": mkey[:80], "window": window},
421
+ **_field_extras(field, mtype),
422
+ })
423
+ continue
424
+ if not str(key).startswith("custom_"):
425
+ continue
426
+ ftype = field.get("type")
427
+ if ftype not in CUSTOM_FIELD_TYPES:
428
+ continue
429
+ if ftype in READONLY_CUSTOM_TYPES:
430
+ # Wave-5 items 9/11: the read-only user-created pair. Emitted with the cohort
431
+ # column's mechanism (source 'odoo' + derived) so the client never offers an
432
+ # editor and the host's overlay-write guard excludes them by construction.
433
+ # FILTERABLE since wave 6 (owner item 6): their values live client-side
434
+ # (formula computes over the row, created_time renders `_created`), this
435
+ # table's counts are client-mode, and the windowed count path never sees
436
+ # these tables β€” so the client engine answers them soundly.
437
+ entry = {
438
+ "key": str(key)[:80],
439
+ "label": str(field.get("label") or "Untitled")[:120],
440
+ "type": ftype,
441
+ "source": "odoo",
442
+ "derived": True,
443
+ "filterable": True,
444
+ "default": bool(field.get("default", True)),
445
+ "custom": True,
446
+ "note": str(field.get("note") or "")[:2000],
447
+ **_field_extras(field, ftype),
448
+ }
449
+ if ftype == "formula":
450
+ formula = _clean_formula(field.get("formula"))
451
+ if formula is None:
452
+ continue # a formula field without a formula is nothing
453
+ entry["formula"] = formula
454
+ out.append(entry)
455
+ continue
456
+ if field.get("source") != "overlay":
457
+ continue
458
+ options = (_clean_options(field.get("options"))
459
+ if ftype in ("select", "multiselect") else [])
460
+ if ftype in ("select", "multiselect") and not options:
461
+ # A select with no surviving choices can never hold a value. Dropping the COLUMN
462
+ # would lose the user's data; degrading it to text keeps every stored value
463
+ # readable and lets them re-add choices.
464
+ ftype = "text"
465
+ out.append({
466
+ "key": str(key)[:80],
467
+ "label": str(field.get("label") or "Untitled")[:120],
468
+ "type": ftype,
469
+ "source": "overlay",
470
+ "default": bool(field.get("default", True)),
471
+ "custom": True,
472
+ "agg": field.get("agg") if field.get("agg") in {"sum"} else None,
473
+ "note": str(field.get("note") or "")[:2000],
474
+ **({"options": options} if ftype in ("select", "multiselect") else {}),
475
+ # comma-joined SET semantics (the Cohorts column's contract): the row belongs to
476
+ # every member, groups count it under each, the toolbar count stays distinct.
477
+ **({"multi": True} if ftype == "multiselect" else {}),
478
+ **({"max": _clean_rating_max(field.get("max"))} if ftype == "rating" else {}),
479
+ **_field_extras(field, ftype),
480
+ })
481
+ if cohorts:
482
+ out.append(cohort_field())
483
+ return out
484
+
485
+
486
+ def rows_from_pool(pool_rows, fields=None, overlays=None, derived=None):
487
+ """Map customer_data.pool() dicts -> the API row shape the grid expects:
488
+ pid + each Odoo field (numeric fields rounded, text/date passed through) + the
489
+ persisted external overlay. Mirrors the standalone API payload contract.
490
+
491
+ `derived` is `{pid: {key: value}}` for columns the HOST computes per render rather than
492
+ reads off the pool row β€” today just the cohort column. A separate argument from `overlays`
493
+ on purpose: `overlays` is the PERSISTED user stratum, and putting a value there that is
494
+ never written back would make the dict mean two things.
495
+ """
496
+ fields = fields or FIELDS
497
+ odoo_fields = [field for field in fields if field["source"] == "odoo"]
498
+ overlay_fields = [field for field in fields if field["source"] == "overlay"]
499
+ derived_keys = [field["key"] for field in fields if field.get("derived")]
500
+ overlays = overlays or {}
501
+ derived = derived or {}
502
+ out = []
503
+ for r in pool_rows:
504
+ pid = r.get("pid")
505
+ # `_created` (wave-5 item 11) rides every row like `pid` does β€” the datum the
506
+ # `created_time` field type renders, regardless of that field's own key. Not a Field:
507
+ # it has no column of its own until a user creates one. `lat`/`lon` (wave-7 W11) ride
508
+ # the same way: the Map VIEW's data, nullable, deliberately not a column.
509
+ row = {"pid": pid, "_created": r.get("_created") or "",
510
+ "lat": r.get("lat"), "lon": r.get("lon")}
511
+ for field in odoo_fields:
512
+ k = field["key"]
513
+ if field.get("derived"):
514
+ continue # not on the pool row β€” filled from `derived` below
515
+ v = r.get(k)
516
+ row[k] = v if field["type"] in {"text", "status", "date"} else _round(v)
517
+ saved = overlays.get(str(pid), {}) or {}
518
+ for field in overlay_fields:
519
+ row[field["key"]] = saved.get(field["key"], "")
520
+ got = derived.get(pid) or {}
521
+ for k in derived_keys:
522
+ # '' not None: a customer in no cohort has an EMPTY cohort list, and `is empty` on a
523
+ # text column is the question somebody will ask of it.
524
+ row[k] = got.get(k, "")
525
+ out.append(row)
526
+ return out
527
+
528
+
529
+ # --- the FILTER-TREE contract (mirrors customer-grid/types.ts) ---------------
530
+ # Ops the Airtable-parity condition builder can emit. isEmpty/isNotEmpty are
531
+ # VALUE-FREE (they legitimately carry no value and must never be dropped for it).
532
+ FILTER_OPS = {'contains', 'doesNotContain', 'eq', 'neq', 'isEmpty', 'isNotEmpty',
533
+ 'gt', 'gte', 'lt', 'lte', 'between', 'within'}
534
+ #: The RESERVED pseudo-column of a cohort-membership leaf (owner item 5, 2026-07-26). It is not
535
+ #: a Field and never will be β€” a cohort is a hand-curated SET, so making it a column would mean
536
+ #: a cell per row per cohort. Mirrors customer-grid/types.ts COHORT_FIELD.
537
+ COHORT_FIELD = '__cohort__'
538
+ #: Ops a cohort leaf may carry (owner, 2026-07-27): set operators over a SET of cohorts.
539
+ #: Anything else is dropped. Deliberately DISJOINT from FILTER_OPS β€” see types.ts COHORT_OPS:
540
+ #: a set op reaching a column leaf would fall through the client engine's switch to "no
541
+ #: narrowing", so keeping the vocabularies apart makes the existing fail-closed drop do the work.
542
+ COHORT_OPS = {'anyOf', 'allOf', 'noneOf'}
543
+ #: The single-cohort ops this leaf shipped with, kept as PERMANENT aliases and REWRITTEN here:
544
+ #: `is part of [one]` is `is any of [that one]`, so a saved view keeps answering and upgrades the
545
+ #: next time it is written. Mirrors types.ts COHORT_OP_ALIASES.
546
+ COHORT_OP_ALIASES = {'eq': 'anyOf', 'neq': 'noneOf'}
547
+ #: How many cohorts one condition may name. Mirrors types.ts MAX_COHORT_IDS.
548
+ MAX_COHORT_IDS = 20
549
+
550
+
551
+ def parse_cohort_ids(value):
552
+ """The cohorts a leaf names, parsed out of `value`. Mirrors types.ts `cohortIds()`.
553
+
554
+ Comma-separated in one string because `FilterRule.value` is what all four layers persist and
555
+ round-trip, and a one-element list is byte-identical to what the single-cohort leaf already
556
+ stored β€” so every shipped view parses with no migration. Safe because a cohort id is built
557
+ from `[a-z0-9_]` only (modules/cohort.new_id), so a comma cannot occur inside one.
558
+ """
559
+ out = []
560
+ for raw in ('' if value is None else str(value)).split(','):
561
+ cid = raw.strip()[:120]
562
+ if not cid or cid in out:
563
+ continue
564
+ out.append(cid)
565
+ if len(out) >= MAX_COHORT_IDS:
566
+ break
567
+ return out
568
+ # Airtable allows 3 nesting levels (root conditions -> group -> group), then grays
569
+ # the button out. MAX_FILTER_DEPTH in types.ts must stay in lock-step with this.
570
+ MAX_FILTER_DEPTH = 3
571
+ MAX_FILTER_NODES = 100 # total nodes across the whole tree
572
+ MAX_FILTER_SIBLINGS = 50 # per level
573
+
574
+
575
+ #: Shape of a measure condition's date window. aios_grid has ZERO app-internal imports by
576
+ #: design, so it does NOT know the window VOCABULARY β€” `harness/windows.py` owns that, mirrored
577
+ #: in `customer-grid/windows.ts`, and a third copy here is exactly the drift those two already
578
+ #: need a gate to prevent. This validates SHAPE only.
579
+ WINDOW_MAX_N = 3650
580
+
581
+
582
+ def _clean_window(raw):
583
+ """Structural passthrough for a measure condition's `{kind, n?, from?, to?}` window.
584
+
585
+ Meaning is NOT checked here: an unrecognised `kind` survives this function and is REFUSED by
586
+ `harness.measure_filter.resolve_rule`, the layer that owns the vocabulary. Splitting it this
587
+ way keeps the grid module reusable and keeps one definition of what "last quarter" means.
588
+ """
589
+ if not isinstance(raw, dict):
590
+ return None
591
+ kind = raw.get('kind')
592
+ if not isinstance(kind, str) or not kind or len(kind) > 40:
593
+ return None
594
+ out = {'kind': kind}
595
+ if raw.get('n') is not None:
596
+ try:
597
+ out['n'] = max(1, min(int(raw['n']), WINDOW_MAX_N))
598
+ except (TypeError, ValueError):
599
+ return None
600
+ for side in ('from', 'to'):
601
+ if raw.get(side) not in (None, ''):
602
+ out[side] = str(raw[side])[:32]
603
+ return out
604
+
605
+
606
+ def _clean_rhs(raw, valid_keys):
607
+ """CG-9 β€” validate `{kind, colId, window?}`, the "compare against another attribute" side.
608
+
609
+ SHAPE and KEY only: `colId` must be something this table has (the caller widens `valid_keys`
610
+ with the measure keys, exactly as it does for the left side), and a measure rhs must carry a
611
+ window. What the window MEANS is `harness/windows.py`'s business, same split as `_clean_window`.
612
+ """
613
+ if not isinstance(raw, dict):
614
+ return None
615
+ kind = raw.get('kind')
616
+ if kind not in ('field', 'measure', 'stat'):
617
+ return None
618
+ if kind == 'stat':
619
+ # A STATISTIC carries no column: the population is the comparand. Shape only β€” which
620
+ # statistics exist is `harness/measure_filter.STATS`'s business, and an unrecognised one
621
+ # is REFUSED there rather than guessed at, exactly like an unrecognised window kind.
622
+ stat = raw.get('stat')
623
+ if not isinstance(stat, str) or not stat or len(stat) > 24:
624
+ return None
625
+ return {'kind': 'stat', 'stat': stat}
626
+ col = raw.get('colId')
627
+ if col not in valid_keys:
628
+ return None
629
+ out = {'kind': kind, 'colId': col}
630
+ if kind == 'measure':
631
+ window = _clean_window(raw.get('window'))
632
+ if window is None:
633
+ return None # a measure comparand with no period is not a question
634
+ out['window'] = window
635
+ return out
636
+
637
+
638
+ #: View DISPLAY MODES beside the grid (wave-6 item 10; 'map' wave-7 W11; 'dashboard' wave-8
639
+ #: I19). Mirrors customer-grid/types.ts DISPLAY_MODES; 'grid' is what an absent/unknown
640
+ #: display means, so it is never stored.
641
+ #: ⚠ 'dashboard' is RETAINED FOREVER (wave-9 I10, contract C2). The owner renamed the mode to
642
+ #: "Chart" (a Dashboard MODULE is coming and the two would collide), but this set is the
643
+ #: gatekeeper for a STORED value: `_clean_display` DROPS an unknown mode, so removing
644
+ #: 'dashboard' here would silently downgrade every already-saved chart view to grid β€” and live
645
+ #: views are sitting in mode:'dashboard' right now (wave 8's own close-out records one). The
646
+ #: rename is therefore a stored-value MIGRATION, not a constant rename: accept 'dashboard' on
647
+ #: READ forever, only ever WRITE 'chart'.
648
+ DISPLAY_MODES = {'grid', 'list', 'calendar', 'kanban', 'map', 'dashboard', 'chart'}
649
+
650
+ #: Old wire value -> the value we store today. Applied AFTER the membership test so an unknown
651
+ #: mode is still rejected rather than accidentally aliased.
652
+ _LEGACY_MODES = {'dashboard': 'chart'}
653
+
654
+ #: Chart kinds a chart-mode view may hold (wave-8 I19, contract C2). Mirrors the client's
655
+ #: union. Deliberately small: the owner asked to "start with simple charts" and expand, and a
656
+ #: kind the client cannot draw is worse than one that does not exist yet.
657
+ CHART_KINDS = {'bar', 'line', 'area', 'donut', 'kpi'}
658
+ CHART_AGGS = {'sum', 'avg', 'count', 'min', 'max'}
659
+ MAX_CHARTS = 12 #: per view β€” a dashboard, not an unbounded render loop
660
+ MAX_CHART_TITLE = 60
661
+
662
+ #: Wave-9 I11 (contract C2) β€” chart customisation, host-validated.
663
+ #:
664
+ #: `palette` names a colour JOB, never a colour. A browser must not be able to post a raw hex:
665
+ #: the four names below map to the four jobs a palette can do (identity / magnitude / polarity)
666
+ #: and resolve to brand ramps client-side, so a tenant restyle cannot be defeated by a stored
667
+ #: literal. STATUS colours (good/warning/serious/critical) are deliberately NOT selectable β€”
668
+ #: they are reserved signal, and reusing them as "series 4" is how a chart starts lying.
669
+ CHART_PALETTES = {'brand', 'categorical', 'sequential', 'diverging'}
670
+ CHART_FORMATS = {'auto', 'number', 'currency', 'percent', 'compact'}
671
+ MAX_AXIS_LABEL = 40
672
+ #: `size` is the I10 drag. Width is in GRID COLUMNS (a 12-column board), height in px.
673
+ CHART_W_RANGE = (1, 12)
674
+ CHART_H_RANGE = (120, 800)
675
+
676
+
677
+ def _clean_chart(raw, valid_keys):
678
+ """One dashboard chart, fail-closed. Returns None if the chart cannot be drawn.
679
+
680
+ A chart's `y` is optional (absent = count of rows, which is what "how many customers per
681
+ state" means). `x` is NOT: a chart with no category axis has nothing to plot against, and
682
+ silently keeping it would put an empty card on the dashboard with no way to tell why.
683
+ """
684
+ if not isinstance(raw, dict):
685
+ return None
686
+ kind = raw.get('kind')
687
+ if kind not in CHART_KINDS:
688
+ return None
689
+ x = raw.get('x')
690
+ if x not in valid_keys:
691
+ return None # dead category ref -> the chart goes, not the board
692
+ cid = raw.get('id')
693
+ if not isinstance(cid, str) or not cid.strip():
694
+ return None # the client owns chart ids; an unidentified card
695
+ # cannot be edited or removed, so it must not persist
696
+ out = {'id': cid.strip()[:64], 'kind': kind, 'x': x,
697
+ 'agg': raw.get('agg') if raw.get('agg') in CHART_AGGS else 'sum'}
698
+ if raw.get('y') in valid_keys:
699
+ out['y'] = raw['y']
700
+ else:
701
+ # no measurable column -> the only honest aggregation left is "how many rows"
702
+ out['agg'] = 'count'
703
+ title = raw.get('title')
704
+ if isinstance(title, str) and title.strip():
705
+ out['title'] = title.strip()[:MAX_CHART_TITLE]
706
+
707
+ # ── wave-9 I11 (contract C2): customisation ────────────────────────────────────────────
708
+ # `splitBy` is the field whose values become the SERIES. It is deliberately not called
709
+ # `colorBy`: that name already means two other things here (`config.colorBy` = row
710
+ # colouring, `display.colorField` = map pin colour) and a third sense would be unreadable.
711
+ if raw.get('splitBy') in valid_keys and raw['splitBy'] != x:
712
+ out['splitBy'] = raw['splitBy']
713
+ # Stacking is only a question once there are series to stack, and only for the two kinds
714
+ # that can express it. Anywhere else it is dropped rather than stored as a lie the client
715
+ # would have to re-decide.
716
+ if out.get('splitBy') and kind in ('bar', 'area') and raw.get('stacked') is True:
717
+ out['stacked'] = True
718
+ if raw.get('palette') in CHART_PALETTES:
719
+ out['palette'] = raw['palette']
720
+
721
+ axis = raw.get('axis')
722
+ if isinstance(axis, dict):
723
+ # β›” ONE y-scale, always. There is no second-axis key here and there must never be:
724
+ # two y-scales on one frame can manufacture any correlation you like by rescaling, and
725
+ # the honest alternatives are two charts, small multiples, or indexing to a common base.
726
+ # Ruled explicitly in contract C2 against the "Tableau versatility" brief.
727
+ clean_axis = {}
728
+ for side in ('x', 'y'):
729
+ spec = axis.get(side)
730
+ if not isinstance(spec, dict):
731
+ continue
732
+ one = {}
733
+ lab = spec.get('label')
734
+ if isinstance(lab, str) and lab.strip():
735
+ one['label'] = lab.strip()[:MAX_AXIS_LABEL]
736
+ if spec.get('format') in CHART_FORMATS:
737
+ one['format'] = spec['format']
738
+ if one:
739
+ clean_axis[side] = one
740
+ if clean_axis:
741
+ out['axis'] = clean_axis
742
+
743
+ size = raw.get('size')
744
+ if isinstance(size, dict):
745
+ one = {}
746
+ for key, (lo, hi) in (('w', CHART_W_RANGE), ('h', CHART_H_RANGE)):
747
+ try:
748
+ one[key] = max(lo, min(hi, int(size[key])))
749
+ except (KeyError, TypeError, ValueError):
750
+ pass # a partial size is fine: the client defaults the missing axis
751
+ if one:
752
+ out['size'] = one
753
+ return out
754
+
755
+
756
+ def _clean_display(raw, valid_keys):
757
+ """Structural passthrough for a view's `config.display` (wave-6 item 10), fail-closed.
758
+
759
+ `{mode, dateField?, stackField?, titleField?, colorField?, sizeField?, charts?}` β€” mode
760
+ must be a known non-grid mode (grid is the absent default, so storing it would be a second
761
+ way to say nothing); every field ref must name a field this table has (a ref to a deleted
762
+ field is DROPPED and the client falls back to its per-mode default); unknown keys are
763
+ dropped. What each mode MEANS β€” calendar wants a date-family field, kanban a select-family
764
+ stack, map a single-select to colour by and a numeric to size by β€” is the client's
765
+ business: it is the only layer that renders them, and a wrong-typed ref degrades to that
766
+ surface's default rather than to an error (the `_clean_window` split).
767
+
768
+ Wave-8 (contract C2) adds the map encodings (`colorField` I3, `sizeField` I5) and
769
+ dashboard `charts` (I19). A chart whose x/y names a deleted field is dropped INDIVIDUALLY β€”
770
+ never the whole array, because losing one column should not cost the user a dashboard they
771
+ spent time building.
772
+ """
773
+ if not isinstance(raw, dict):
774
+ return None
775
+ mode = raw.get('mode')
776
+ if mode not in DISPLAY_MODES or mode == 'grid':
777
+ return None
778
+ # Wave-9 I10 (C2): normalise the legacy wire value AFTER the membership test, so an unknown
779
+ # mode is still rejected rather than accidentally aliased into a real one. Every already
780
+ # saved 'dashboard' view reads back as 'chart' from here on; nothing writes 'dashboard'.
781
+ mode = _LEGACY_MODES.get(mode, mode)
782
+ out = {'mode': mode}
783
+ for ref in ('dateField', 'stackField', 'titleField', 'colorField', 'sizeField'):
784
+ if raw.get(ref) in valid_keys:
785
+ out[ref] = raw[ref]
786
+ charts = raw.get('charts')
787
+ if isinstance(charts, list):
788
+ clean = [c for c in (_clean_chart(x, valid_keys) for x in charts[:MAX_CHARTS]) if c]
789
+ # de-dupe by id: two cards sharing an id are one card as far as the client's keyed
790
+ # render is concerned, and the second would silently shadow the first
791
+ seen, uniq = set(), []
792
+ for c in clean:
793
+ if c['id'] in seen:
794
+ continue
795
+ seen.add(c['id'])
796
+ uniq.append(c)
797
+ if uniq:
798
+ out['charts'] = uniq
799
+ return out
800
+
801
+
802
+ #: FOLDERS over the saved views / cohorts sidebars (wave-8 I11, contract C4).
803
+ #:
804
+ #: ⚠ Folder membership is stored as a SIDE MAP (`itemFolders`), not as a `folderId` ON each
805
+ #: view or cohort β€” a deliberate amendment to C4's first wording, recorded in the split doc.
806
+ #: Two reasons. (1) A cohort lives in a DIFFERENT store (`customer_cohorts`, keyed by cohort
807
+ #: id) and adding a `folders` key beside those ids would collide with a cohort whose generated
808
+ #: id happened to be 'folders'. (2) Folder placement is a per-user ORGANISING act, not part of
809
+ #: what a view IS: keeping it out of the view config means duplicating or exporting a view does
810
+ #: not drag a folder reference along with it. One map, one home, both surfaces.
811
+ FOLDER_SURFACES = {"views", "cohorts"}
812
+ MAX_FOLDERS = 60
813
+ MAX_FOLDER_NAME = 80
814
+
815
+ #: Wave-9 I15 (contract C5) β€” a user-chosen folder icon, as {shape, tone}.
816
+ #:
817
+ #: Both halves are WHITELISTS, never free values: `shape` names geometry the client already
818
+ #: draws (one source, `iconShapes.ts`, read by both painters) and `tone` names a palette token,
819
+ #: not a colour β€” so a browser cannot post a hex and defeat a tenant restyle, and a shape the
820
+ #: client cannot render can never reach the store.
821
+ #: ⚠ MIRRORS CLIENT'S `iconShapes.ts` ENUMERATION EXACTLY (C5: CLIENT enumerates, HOST mirrors β€”
822
+ #: posted in the split doc 2026-07-29, HOST adopted it the same day, replacing a provisional
823
+ #: 12-shape guess of mine that contained shapes the client cannot draw). Do not extend this set
824
+ #: without the matching client geometry: an unknown shape falls back to the default folder mark,
825
+ #: which is also I14's "existing folders get the folder icon" for every pre-wave-9 folder.
826
+ FOLDER_ICON_SHAPES = {"folder", "star", "flag", "tag", "bookmark", "box", "circle", "square"}
827
+ #: Tones are the C1 pastels β€” FILLS ONLY, never text (the standing palette rule). 'grey' is the
828
+ #: default, and is CLIENT's key name: not 'neutral', which is what HOST first guessed.
829
+ FOLDER_ICON_TONES = {"blue", "green", "yellow", "red", "grey"}
830
+ FOLDER_ICON_DEFAULT_TONE = "grey"
831
+
832
+
833
+ #: Wave-9 I17 (contract C4) β€” who may EDIT a saved view.
834
+ #:
835
+ #: ⚠ READ THIS BEFORE BUILDING ON IT. Views are stored PER USER today
836
+ #: (`core/table_store.TableStore.workspace` reads `store.get(table_key)[username]`), so one
837
+ #: user's views are invisible to every other user and "collaborative" has nothing to act on
838
+ #: yet. This validator is therefore CORRECT-BUT-INERT plumbing: it makes the setting durable
839
+ #: and fail-closed now, so that when shared views land the permission does not need a data
840
+ #: migration and no saved view is retro-restricted. It does NOT make anything shared, and
841
+ #: nothing in the app currently reads it to grant or deny cross-user access.
842
+ #: Recorded as the C4 amendment in .claude/wiki/research/grid-wave9-split.md.
843
+ VIEW_EDIT_MODES = {'personal', 'collaborative', 'users'}
844
+ MAX_VIEW_USERS = 50
845
+
846
+
847
+ def clean_view_permissions(raw, default, known_users=None):
848
+ """{edit, users?} β€” fail-closed on both halves.
849
+
850
+ `default` is supplied by the CALLER because it splits by path, and that split is a
851
+ permission rule rather than a formatting one: absent on a view that already exists means a
852
+ pre-wave-9 view and must stay 'collaborative' (retro-restricting somebody's saved view is a
853
+ silent takeaway), while absent on CREATE must be 'personal' (a new view must never be
854
+ anyone-can-edit purely by omission).
855
+
856
+ `known_users` (when given) is the real account list: an unknown name is DROPPED, and an
857
+ 'users' grant left with nobody in it collapses to 'personal' rather than to everyone.
858
+ """
859
+ mode = (raw or {}).get('edit') if isinstance(raw, dict) else None
860
+ if mode not in VIEW_EDIT_MODES:
861
+ mode = default if default in VIEW_EDIT_MODES else 'personal'
862
+ if mode != 'users':
863
+ return {'edit': mode}
864
+ names, seen = [], set()
865
+ for u in list((raw or {}).get('users') or [])[:MAX_VIEW_USERS]:
866
+ u = str(u or '').strip()
867
+ if not u or u.lower() in seen:
868
+ continue
869
+ if known_users is not None and u not in known_users:
870
+ continue # fail-closed: a name we cannot resolve grants nothing
871
+ seen.add(u.lower())
872
+ names.append(u)
873
+ if not names:
874
+ return {'edit': 'personal'} # an empty grant is NOT "everyone"
875
+ return {'edit': 'users', 'users': names}
876
+
877
+
878
+ def clean_folder_icon(raw):
879
+ """{shape, tone} or None. Fail-closed on both halves, independently.
880
+
881
+ A folder with a valid shape but a junk tone keeps the shape and defaults the tone rather
882
+ than losing the icon entirely β€” losing a user's pick because one half was wrong is the kind
883
+ of silent data loss the folder events already avoid elsewhere.
884
+ """
885
+ if not isinstance(raw, dict):
886
+ return None
887
+ shape = raw.get("shape")
888
+ if shape not in FOLDER_ICON_SHAPES:
889
+ return None
890
+ tone = raw.get("tone")
891
+ return {"shape": shape,
892
+ "tone": tone if tone in FOLDER_ICON_TONES else FOLDER_ICON_DEFAULT_TONE}
893
+
894
+
895
+ def clean_folders(raw):
896
+ """Validate the per-surface folder lists, fail-closed. {surface: [{id, name, order}]}."""
897
+ out = {}
898
+ for surface in FOLDER_SURFACES:
899
+ items, seen = [], set()
900
+ for f in list((raw or {}).get(surface) or [])[:MAX_FOLDERS]:
901
+ if not isinstance(f, dict):
902
+ continue
903
+ fid = str(f.get("id") or "").strip()[:80]
904
+ name = str(f.get("name") or "").strip()[:MAX_FOLDER_NAME]
905
+ if not fid or not name or fid in seen:
906
+ continue # an unidentified or unnamed folder cannot be shown or edited
907
+ seen.add(fid)
908
+ try:
909
+ order = int(f.get("order", len(items)))
910
+ except (TypeError, ValueError):
911
+ order = len(items)
912
+ row = {"id": fid, "name": name, "order": order}
913
+ icon = clean_folder_icon(f.get("icon")) # wave-9 I15 (C5); absent = default mark
914
+ if icon:
915
+ row["icon"] = icon
916
+ items.append(row)
917
+ items.sort(key=lambda x: x["order"])
918
+ for i, f in enumerate(items):
919
+ f["order"] = i # re-index so `order` is always dense and total
920
+ if items:
921
+ out[surface] = items
922
+ return out
923
+
924
+
925
+ def clean_item_folders(raw, folders, valid_ids):
926
+ """{surface: {itemId: folderId}} β€” dropping any placement whose ITEM or FOLDER is gone.
927
+
928
+ This is what makes a deleted folder's contents fall back to the root rather than vanish:
929
+ nothing stores "this item is in no folder", so an unresolvable placement simply disappears
930
+ and the item renders at the top level. Same for an item that was deleted elsewhere β€” its
931
+ stale placement can never resurrect it, because the sidebars render ITEMS and consult this
932
+ map, never the other way round.
933
+ """
934
+ out = {}
935
+ for surface in FOLDER_SURFACES:
936
+ fids = {f["id"] for f in (folders or {}).get(surface, [])}
937
+ ok = {}
938
+ for item_id, fid in ((raw or {}).get(surface) or {}).items():
939
+ if not isinstance(item_id, str) or not isinstance(fid, str):
940
+ continue
941
+ if item_id in (valid_ids or {}).get(surface, ()) and fid in fids:
942
+ ok[item_id[:120]] = fid
943
+ if ok:
944
+ out[surface] = ok
945
+ return out
946
+
947
+
948
+ def clean_filter_tree(raw, valid_keys, depth=1, budget=None, cohort_ids=None):
949
+ """Recursively validate an UNTRUSTED filter tree (conditions + nested groups).
950
+
951
+ Returns a clean tree of leaf conditions ({colId, op, value, value2}) and
952
+ groups ({conj, children}). Module-agnostic on purpose: any module embedding
953
+ the grid validates its own view state through this one function.
954
+
955
+ Fail-closed PER NODE: anything unrecognised is DROPPED rather than raised β€”
956
+ the same contract the rest of the view sanitiser follows, so one bad rule can
957
+ never cost a user their whole saved view. Depth, per-level width and total
958
+ node count are all capped: the tree is re-evaluated for every row on every
959
+ render, so an unbounded structure would be a persistent client-side DoS.
960
+ Empty groups are dropped (they carry no meaning once persisted).
961
+
962
+ `cohort_ids` is the set of cohorts the CALLER may see. A cohort leaf naming anything else is
963
+ DROPPED here rather than left for the engine β€” a deleted cohort would otherwise leave a
964
+ condition that can only match nothing, so `List is not [deleted]` would show an empty table
965
+ forever with no way to tell why. `None` means this host has no cohorts, and then every
966
+ cohort leaf is dropped: fail-closed, like every other unknown key.
967
+ """
968
+ if budget is None:
969
+ budget = [MAX_FILTER_NODES]
970
+ out = []
971
+ for node in list(raw or [])[:MAX_FILTER_SIBLINGS]:
972
+ if budget[0] <= 0:
973
+ break
974
+ if not isinstance(node, dict):
975
+ continue
976
+ if isinstance(node.get('children'), list): # a condition GROUP
977
+ if depth >= MAX_FILTER_DEPTH:
978
+ continue # too deep -> drop
979
+ budget[0] -= 1
980
+ children = clean_filter_tree(node['children'], valid_keys,
981
+ depth + 1, budget, cohort_ids)
982
+ if children:
983
+ out.append({'conj': 'or' if node.get('conj') == 'or' else 'and',
984
+ 'children': children})
985
+ continue
986
+ if node.get('colId') == COHORT_FIELD: # a cohort-membership leaf
987
+ op = COHORT_OP_ALIASES.get(node.get('op'), node.get('op'))
988
+ named = parse_cohort_ids(node.get('value'))
989
+ # ALL of them, or the leaf goes. A set that quietly lost a member asks a DIFFERENT
990
+ # question, and for `noneOf` a strictly wider one: `is none of [A, B]` degrading to
991
+ # `is none of [A]` would show every row in B under a count nobody would doubt. This
992
+ # is the same all-or-nothing the single-cohort leaf already had, extended to a set.
993
+ if op in COHORT_OPS and named and all(c in (cohort_ids or ()) for c in named):
994
+ budget[0] -= 1
995
+ out.append({'colId': COHORT_FIELD, 'op': op,
996
+ 'value': ','.join(named), 'value2': ''})
997
+ continue
998
+ if node.get('colId') in valid_keys and node.get('op') in FILTER_OPS:
999
+ budget[0] -= 1
1000
+ # `or ''` would be wrong here: it maps every FALSY value to '', and '' is the
1001
+ # signal for "inactive". A numeric 0 (or 0.0, or False) is a real value the client
1002
+ # treats as active β€” `0 === ""` is false in TS β€” so `revenue = 0` would silently
1003
+ # stop filtering and show every row instead of the zero-revenue ones.
1004
+ val, val2 = node.get('value'), node.get('value2')
1005
+ leaf = {'colId': node['colId'], 'op': node['op'],
1006
+ 'value': ('' if val is None else str(val))[:500],
1007
+ 'value2': ('' if val2 is None else str(val2))[:500]}
1008
+ # CG-8. A MEASURE condition ("Sales, in the last 90 days, > 5,000") carries two
1009
+ # extra members: a stable client-generated `id`, which is how the server's answer
1010
+ # finds its way back to the condition that asked (positional matching silently
1011
+ # re-associates every answer the moment a user deletes a condition), and the
1012
+ # `window`. Emitted ONLY when the input has them β€” a column condition's cleaned
1013
+ # shape is unchanged, so every persisted view deserialises byte-identically and
1014
+ # `clean_filter_tree` stays idempotent (verify_filter_engine.py asserts that by
1015
+ # exact structural comparison).
1016
+ rid = node.get('id')
1017
+ if rid not in (None, ''):
1018
+ leaf['id'] = str(rid)[:64]
1019
+ window = _clean_window(node.get('window'))
1020
+ if window is not None:
1021
+ leaf['window'] = window
1022
+ # Owner items 3 + 4, carried under the SAME rule as CG-8's `id`/`window`: emitted
1023
+ # only when the input has them, so a plain column condition's cleaned shape is
1024
+ # byte-identical to what it always was and `clean_filter_tree` stays idempotent
1025
+ # (verify_filter_engine.py asserts that by exact structural comparison). Drop the
1026
+ # carry-through and the next autosave silently strips a date condition back to a
1027
+ # bare comparison against an empty value β€” i.e. back to INACTIVE.
1028
+ date_window = _clean_window(node.get('dateWindow'))
1029
+ if date_window is not None:
1030
+ leaf['dateWindow'] = date_window
1031
+ mode = node.get('dateMode')
1032
+ # SHAPE only. An unrecognised mode survives here and is refused by
1033
+ # `windows.resolve_anchor`, which returns None and makes the condition match
1034
+ # NOTHING β€” the same split as `_clean_window`, and the reason this module can stay
1035
+ # free of the date vocabulary it would otherwise have to keep in step.
1036
+ if isinstance(mode, str) and 0 < len(mode) <= 40:
1037
+ leaf['dateMode'] = mode
1038
+ rhs = _clean_rhs(node.get('rhs'), valid_keys)
1039
+ if rhs is not None:
1040
+ leaf['rhs'] = rhs
1041
+ out.append(leaf)
1042
+ return out
1043
+
1044
+
1045
+ def _default_view_config(fields):
1046
+ shown = [field["key"] for field in fields
1047
+ if field.get("default") is not False or field["source"] == "overlay"]
1048
+ hidden = [field["key"] for field in fields if field["key"] not in shown]
1049
+ return {
1050
+ # `filters` is the ROOT of the filter tree: leaf conditions and/or nested
1051
+ # condition groups ({conj, children}); `filterConj` joins the root level.
1052
+ "filters": [], "filterConj": "and",
1053
+ "sorts": [], "groupBy": None, "colorBy": None,
1054
+ "rowHeightMode": "short", "order": shown + hidden, "visible": shown,
1055
+ "widths": {}, "memberPids": [],
1056
+ }
1057
+
1058
+
1059
+ def views_from_defs(defs, saved_views, fields):
1060
+ """Convert legacy list formulas into the shared serializable SavedView contract."""
1061
+ base = _default_view_config(fields)
1062
+ views = [{
1063
+ "id": "all-customers", "name": "All customers", "kind": "system",
1064
+ "locked": True, "config": dict(base),
1065
+ }]
1066
+ op_map = {">=": "gte", ">": "gt", "<=": "lte", "<": "lt", "=": "eq",
1067
+ "contains": "contains"}
1068
+ for name, definition in (defs or {}).items():
1069
+ filters = []
1070
+ for rule in definition.get("rules") or []:
1071
+ if rule.get("field") not in {field["key"] for field in fields}:
1072
+ continue
1073
+ filters.append({
1074
+ "colId": rule["field"],
1075
+ "op": op_map.get(rule.get("op"), "eq"),
1076
+ "value": str(rule.get("value") if rule.get("value") is not None else ""),
1077
+ })
1078
+ sort = str(definition.get("sort") or "")
1079
+ sorts = ([{"colId": sort.lstrip("-"),
1080
+ "dir": "desc" if sort.startswith("-") else "asc"}]
1081
+ if sort.lstrip("-") in {field["key"] for field in fields} else [])
1082
+ views.append({
1083
+ "id": "list:" + str(name),
1084
+ "name": str(name),
1085
+ "kind": "list",
1086
+ "note": str(definition.get("note") or ""),
1087
+ "config": {
1088
+ **base, "filters": filters, "sorts": sorts,
1089
+ "memberPids": [int(pid) for pid in definition.get("members") or []
1090
+ if isinstance(pid, int) or str(pid).isdigit()],
1091
+ },
1092
+ })
1093
+ index = {view["id"]: i for i, view in enumerate(views)}
1094
+ for view_id, saved in (saved_views or {}).items():
1095
+ if not isinstance(saved, dict) or not isinstance(saved.get("config"), dict):
1096
+ continue
1097
+ clean = dict(saved)
1098
+ clean["id"] = str(view_id)
1099
+ if view_id in index:
1100
+ views[index[view_id]] = clean
1101
+ else:
1102
+ views.append(clean)
1103
+ return views
1104
+
1105
+
1106
+ def embed_html_path():
1107
+ """The first existing candidate path for the inlined single-file build, or None."""
1108
+ for p in _EMBED_CANDIDATES:
1109
+ if p.is_file():
1110
+ return p
1111
+ return None
1112
+
1113
+
1114
+ def scope_counts(shown, matched, total):
1115
+ """The honest 'N of M' a SERVER-WINDOWED table must carry (CG-2).
1116
+
1117
+ `matched` and `total` MUST come from their own queries over the whole scope. Never pass
1118
+ `len(rows)` as `matched` β€” that is the silent [:N] this exists to prevent: the page would
1119
+ report the window size as though it were the result size.
1120
+
1121
+ Refuses the shapes that could only be a mistake, because a wrong count here is invisible on
1122
+ screen (it looks like a smaller dataset, not like an error).
1123
+ """
1124
+ shown, matched, total = int(shown), int(matched), int(total)
1125
+ if matched > total:
1126
+ raise ValueError(f"matched ({matched}) exceeds total ({total}) β€” a filter cannot match "
1127
+ f"more rows than the scope holds")
1128
+ if shown > matched:
1129
+ raise ValueError(f"shown ({shown}) exceeds matched ({matched}) β€” the window cannot hold "
1130
+ f"more rows than the filter matched")
1131
+ return {"shown": shown, "matched": matched, "total": total, "windowed": True}
1132
+
1133
+
1134
+ def build_html(rows, fields, *, workspace=None, path=None, counts=None):
1135
+ """PURE: read the inlined single-file build and inject the data handshake.
1136
+
1137
+ Returns a self-contained HTML string with
1138
+ <script>window.__AIOS_GRID__ = {"fields": ..., "rows": ...};</script>
1139
+ inserted immediately after the opening <head> tag β€” a CLASSIC (non-module) script, so it
1140
+ executes during parse BEFORE the app's deferred `type="module"` script reads the global.
1141
+ Raises FileNotFoundError if no build is present (callers should try/except and fall back).
1142
+ """
1143
+ p = Path(path) if path else embed_html_path()
1144
+ if p is None or not Path(p).is_file():
1145
+ raise FileNotFoundError(
1146
+ "aios_grid: no embed build found. Run `npm run build:embed` in aios-web/web and "
1147
+ "copy dist-embed/index.html -> royalimports_os/aios_grid_embed.html "
1148
+ "(python build_embed.py). Looked in: "
1149
+ + " ; ".join(str(c) for c in _EMBED_CANDIDATES))
1150
+ html = Path(p).read_text(encoding="utf-8")
1151
+ # ensure_ascii=False keeps names readable; default=str is a safety net for stray
1152
+ # date/Decimal; the </ escape prevents a value containing "</script>" from breaking out.
1153
+ data = {"fields": fields, "rows": rows, "workspace": workspace or {}}
1154
+ # Only a WINDOWED table carries counts. Omitting the key leaves the client counting the rows
1155
+ # it holds, which is correct when it holds all of them.
1156
+ if counts:
1157
+ data["counts"] = counts
1158
+ payload = json.dumps(data, ensure_ascii=False, default=str).replace("</", "<\\/")
1159
+ tag = "<script>window.__AIOS_GRID__ = " + payload + ";</script>"
1160
+ new_html, n = re.subn(r"(<head\b[^>]*>)", lambda m: m.group(1) + tag, html, count=1)
1161
+ if n == 0: # no <head> (shouldn't happen) β€” still run it first
1162
+ new_html = tag + html
1163
+ return new_html
1164
+
1165
+
1166
+ def component_dir():
1167
+ for path in _COMPONENT_CANDIDATES:
1168
+ if (path / "index.html").is_file():
1169
+ return path
1170
+ return None
1171
+
1172
+
1173
+ def render(rows, fields, *, workspace=None, height=680, key="aios_grid",
1174
+ user_options=None, measures=None, measure_sets=None, today=None, viewer=None):
1175
+ """Render the Airtable-style grid and return its latest guarded host event.
1176
+
1177
+ The preferred custom-component path forwards `key` and supports two-way persistence.
1178
+ If only the legacy HTML build exists, it renders safely with no return event.
1179
+
1180
+ `user_options` are the assignable people for `user`-typed overlay columns. They come from
1181
+ the HOST (the tenant's real accounts) rather than from anything the grid could invent, so
1182
+ an assignee is always someone who can actually log in and see the row.
1183
+
1184
+ `measures` / `measure_sets` are CG-8. A measure is FILTERABLE BUT NOT DISPLAYABLE: "Sales in
1185
+ the last 90 days" is a question about a window, not a column of this table, and rendering it
1186
+ as a column would silently pick a window for the user. The HOST resolves each condition to a
1187
+ set of row ids (`{ruleId: [pid, ...]}`) and the client treats that set as a membership test β€”
1188
+ which is why this module can offer measure filtering while keeping ZERO app-internal imports:
1189
+ it never computes a measure, it only carries one.
1190
+
1191
+ `today` is the TENANT'S current date as an ISO string, and every relative date condition
1192
+ ("is within the past month", "is before one month ago") resolves against it. It comes from
1193
+ the host rather than from `new Date()` in the browser for the same three reasons
1194
+ harness/windows.py takes it as a parameter: a viewer a day ahead of the server would resolve
1195
+ a different month than the server summed, a window must not shift between the count and the
1196
+ rows, and the tenant's day boundary is the tenant's. Omit it and those conditions match
1197
+ NOTHING with a visible marker β€” never everything.
1198
+ """
1199
+ import streamlit.components.v1 as _c
1200
+ directory = component_dir()
1201
+ payload = {"fields": fields, "rows": rows, "workspace": workspace or {},
1202
+ "userOptions": [str(u)[:120] for u in (user_options or [])][:500],
1203
+ "measures": list(measures or []),
1204
+ "today": str(today) if today else None,
1205
+ # Wave-5 item 1: who is looking, host-computed β€” what gates the client's
1206
+ # "Edit field permissions" menu entry. The HOST-side enforcement is the wall;
1207
+ # this is the courtesy that keeps the menu honest.
1208
+ "viewer": ({"name": str(viewer.get("name") or "")[:80],
1209
+ "isAdmin": bool(viewer.get("isAdmin"))}
1210
+ if isinstance(viewer, dict) else None),
1211
+ "measureSets": {str(k): list(v) for k, v in (measure_sets or {}).items()}}
1212
+ if directory is not None:
1213
+ cache_key = str(directory)
1214
+ component = _DECLARED_COMPONENTS.get(cache_key)
1215
+ if component is None:
1216
+ component = _c.declare_component("aios_customer_grid", path=cache_key)
1217
+ _DECLARED_COMPONENTS[cache_key] = component
1218
+ return component(payload=payload, height=height, key=key, default=None)
1219
+ html = build_html(rows, fields, workspace=workspace)
1220
+ _c.html(html, height=height, scrolling=False)
1221
+ return None
royalimports_os/core/auth_google.py ADDED
@@ -0,0 +1,91 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Sign in with Google β€” TEMPLATE ONLY (wave-7 W10, 2026-07-28). Nothing here runs in
2
+ production yet: every flow function raises until the wiring checklist in
3
+ docs/GOOGLE_LOGIN_TODO.md is completed and GOOGLE_LOGIN_ENABLED=1 is set.
4
+
5
+ DESIGN (recorded now so the full build has a contract to meet):
6
+ - OAuth 2.0 authorization-code flow with PKCE against Google Identity
7
+ (https://accounts.google.com/.well-known/openid-configuration). No SDK dependency β€”
8
+ three HTTPS calls (authorize redirect, token exchange, JWKS fetch) keep the surface
9
+ auditable and the requirements.txt unchanged until we commit.
10
+ - Google is an IDENTITY, not an ACCOUNT SOURCE. A Google sign-in maps to an EXISTING
11
+ core/users.py account via its `email` attribute (set_access(..., email=...)); an
12
+ unknown email FAILS CLOSED with "no account for this Google identity" β€” Google login
13
+ never creates users, so the per-BU/per-module grant model stays the only door.
14
+ - The password gate stays byte-for-byte intact beside it. APP_PASSWORD remains the
15
+ bootstrap master; removing password auth is a separate, owner-approved step.
16
+
17
+ ENV (placeholders β€” see .env.example additions in docs/GOOGLE_LOGIN_TODO.md):
18
+ GOOGLE_OAUTH_CLIENT_ID OAuth client id from Google Cloud console
19
+ GOOGLE_OAUTH_CLIENT_SECRET its secret (Space secret / .env β€” never committed)
20
+ GOOGLE_OAUTH_REDIRECT_URI e.g. https://royal-imports-cfo-os.hf.space/ (must be
21
+ registered VERBATIM in the console; localhost:8501 for dev)
22
+ GOOGLE_LOGIN_ENABLED '1' arms the flow; anything else keeps this a template
23
+ """
24
+ from __future__ import annotations
25
+
26
+ import os
27
+
28
+ AUTH_ENDPOINT = "https://accounts.google.com/o/oauth2/v2/auth"
29
+ TOKEN_ENDPOINT = "https://oauth2.googleapis.com/token"
30
+ JWKS_URI = "https://www.googleapis.com/oauth2/v3/certs"
31
+ SCOPES = "openid email profile"
32
+
33
+
34
+ def configured() -> bool:
35
+ """True when the console credentials exist β€” the LOGIN BUTTON renders iff this is True.
36
+ (Separate from enabled(): a configured-but-disabled deploy shows nothing either, so a
37
+ half-finished setup never paints a dead button.)"""
38
+ return bool(os.environ.get("GOOGLE_OAUTH_CLIENT_ID")) and enabled()
39
+
40
+
41
+ def enabled() -> bool:
42
+ return os.environ.get("GOOGLE_LOGIN_ENABLED", "") == "1"
43
+
44
+
45
+ def auth_url(state: str, code_challenge: str) -> str:
46
+ """The Google consent-screen URL for the authorization-code + PKCE flow.
47
+
48
+ `state` must be an unguessable per-session token the callback VERIFIES (CSRF);
49
+ `code_challenge` is BASE64URL(SHA256(code_verifier)) with the verifier held in the
50
+ session. Template guard: raises until the TODO checklist lands.
51
+ """
52
+ raise NotImplementedError(
53
+ "Google login is a template β€” complete docs/GOOGLE_LOGIN_TODO.md, then implement "
54
+ "auth_url() (urlencode client_id, redirect_uri, response_type=code, scope, state, "
55
+ "code_challenge, code_challenge_method=S256 onto AUTH_ENDPOINT)."
56
+ )
57
+
58
+
59
+ def exchange_code(code: str, code_verifier: str) -> dict:
60
+ """POST the authorization code to TOKEN_ENDPOINT β†’ {'id_token', 'access_token', ...}.
61
+ Template guard: raises until implemented (requests.post with client_id/secret,
62
+ redirect_uri, grant_type=authorization_code, code, code_verifier)."""
63
+ raise NotImplementedError(
64
+ "Google login is a template β€” implement exchange_code() per docs/GOOGLE_LOGIN_TODO.md."
65
+ )
66
+
67
+
68
+ def verify_id_token(id_token: str) -> dict:
69
+ """Validate the JWT against Google's JWKS (signature, iss, aud=client_id, exp) and
70
+ return its claims ({'email', 'email_verified', 'name', ...}). MUST reject
71
+ email_verified=False. Template guard: raises until implemented."""
72
+ raise NotImplementedError(
73
+ "Google login is a template β€” implement verify_id_token() per docs/GOOGLE_LOGIN_TODO.md."
74
+ )
75
+
76
+
77
+ def account_for_claims(claims: dict):
78
+ """Map verified Google claims β†’ the core/users.py account, FAIL-CLOSED.
79
+
80
+ The one function already real: the mapping rule is the design's heart and testable
81
+ today. Returns the users.registry() record whose `email` equals the verified claim
82
+ (case-insensitive), only if the account is active; None otherwise β€” never a new user.
83
+ """
84
+ email = str(claims.get("email") or "").strip().lower()
85
+ if not email or claims.get("email_verified") is not True:
86
+ return None
87
+ from core import users
88
+ for username, u in (users.registry() or {}).items():
89
+ if str(u.get("email") or "").strip().lower() == email and u.get("active", True):
90
+ return users._public(username, u)
91
+ return None
royalimports_os/core/context.py ADDED
@@ -0,0 +1,59 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Request context β€” the single object that carries the GLOBAL filters through a page render.
2
+
3
+ Instead of threading date_from / date_to / team_id / ... through every function, a page builds
4
+ one `Ctx` from the sidebar controls and passes it (or its primitives) down. Modules read only
5
+ what they need. Add a new global filter once here and every module can opt into it.
6
+
7
+ The DBA (doing-business-as) brand filter lives here as `team_id`:
8
+ None -> All / consolidated (Fisch + Royal)
9
+ 5 -> Fisch
10
+ 6 -> Royal
11
+ HQ / company-level modules (cash flow, balance sheet, AR/AP, consolidated P&L) ignore brand and
12
+ call `ctx.consolidated()`; operational modules (sales, customers, products, margins) honour it.
13
+ """
14
+ from dataclasses import dataclass, replace
15
+ import core.odoo as O
16
+ import core.periods as P
17
+
18
+ # label -> team_id used by the sidebar Brand (DBA) selector
19
+ BRANDS = {'All': None, 'Fisch': 5, 'Royal': 6}
20
+
21
+
22
+ @dataclass(frozen=True)
23
+ class Ctx:
24
+ date_from: str
25
+ date_to: str
26
+ period_label: str = 'Year to date'
27
+ team_id: int | None = None # None = All DBAs (consolidated); 5 = Fisch; 6 = Royal
28
+ doc_mode: str = 'order' # reserved: 'order' vs 'invoice' recognition basis
29
+
30
+ @property
31
+ def brand(self) -> str:
32
+ return O.TEAM_NAMES.get(self.team_id, 'All') if self.team_id else 'All'
33
+
34
+ @property
35
+ def teams(self) -> list:
36
+ """Team ids in scope (single brand, or both when consolidated)."""
37
+ return [self.team_id] if self.team_id else list(O.TEAM_IDS)
38
+
39
+ @property
40
+ def window(self) -> tuple:
41
+ return self.date_from, self.date_to
42
+
43
+ def consolidated(self) -> 'Ctx':
44
+ """All-DBA copy β€” HQ modules use this so the brand filter never scopes company numbers."""
45
+ return replace(self, team_id=None)
46
+
47
+ def with_period(self, date_from, date_to, label=None) -> 'Ctx':
48
+ return replace(self, date_from=date_from, date_to=date_to,
49
+ period_label=label or self.period_label)
50
+
51
+
52
+ def from_ui(period_label, period_window, brand_label='All', doc_mode='order') -> Ctx:
53
+ df, dt = period_window
54
+ return Ctx(df, dt, period_label, BRANDS.get(brand_label), doc_mode)
55
+
56
+
57
+ def default(t=None) -> Ctx:
58
+ df, dt = P.ytd(t)
59
+ return Ctx(df, dt, 'Year to date', None)
royalimports_os/core/grid_events.py ADDED
@@ -0,0 +1,1070 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """core/grid_events.py β€” THE WRITE SEAM, lifted out of Streamlit (EXIT-1a, X1, 2026-07-30).
2
+
3
+ This is the grid's event handler: the code that takes an UNTRUSTED event object from the
4
+ browser component and decides what may be persisted. It was `app.py:_cl_handle_grid_event` /
5
+ `_cl_handle_one` and moved here as a PURE MOVE β€” no behavior change inside a move (MA principle
6
+ 5). Every body below is the app.py body verbatim; the ONLY mechanical rewrites are the
7
+ Streamlit couplings the contract names:
8
+
9
+ st.session_state['_cl_grid_seen_ids'] -> ctx.seen_ids (adapter-owned dedup state)
10
+ st.session_state['_cl_doc_payload'] -> ctx.out.doc (EventResult.doc)
11
+ st.session_state['_cohort_toast'] -> ctx.out.toast (EventResult.toast)
12
+ st.session_state['cl_table_workspace'] -> ctx.fallback_ws (store-unavailable fallback)
13
+ is_admin() (the bare call inside doc_delete, and _cl_table_workspace's admin=None default)
14
+ -> ctx.admin
15
+
16
+ There is NO `import streamlit` in this file and there must never be one β€” `verify_seam.py`
17
+ enforces that with a `sys.meta_path` block, which is the only proof that survives refactoring.
18
+
19
+ TWO ADAPTERS, ONE HANDLER:
20
+ * the Streamlit adapter β€” app.py keeps `_cl_handle_grid_event(event, uname, allowed_pids, …)`
21
+ with its old name and signature, builds an EventCtx whose `seen_ids` and `fallback_ws` are
22
+ session-state-backed dicts, and applies the Result to session state.
23
+ * the HTTP adapter β€” `aios-web/api` builds an EventCtx with `fallback_ws=None`. A None
24
+ fallback with the store unavailable is a **503**, never a silent in-memory workspace: an
25
+ API request has no durable session dict to fall back to, so pretending to persist would
26
+ lose the write with a 200 on it.
27
+
28
+ ⚠ LAYERING, stated honestly. `core/` is not supposed to import `modules/` (ARCHITECTURE §1's
29
+ one-directional rule). This handler genuinely straddles: it is the WRITE half of the customer
30
+ table and its persistence verbs live in `modules.customer_data` / `modules.cohort`. The
31
+ contracted path for the strangler is `core/grid_events.py` (both adapters import it from here),
32
+ so the import stays and the violation is RECORDED rather than hidden. It is the one file in
33
+ `core/` that points up; when EXIT-5 ports the surfaces, the persistence verbs are what move
34
+ down, and this note is the marker for that work.
35
+ """
36
+ import datetime as dt
37
+ from dataclasses import dataclass, field
38
+ from typing import MutableMapping, Optional
39
+
40
+ import core.store as store
41
+ import core.table_store as _tstore # wave-9 I17: the shared-view authorisation predicates
42
+ import core.users as users
43
+ import harness.telemetry as _tel # _tel.error() = THE error sink (all caught errors log)
44
+ import modules.cohort as cohort_mod
45
+ import modules.customer_data as cl_mod
46
+
47
+ #: C5 (owner item 12): customer documents. The cap is enforced HOST-side on the DECODED byte
48
+ #: count β€” the client checks the same number first, but a client check only spares the user a
49
+ #: pointless upload; this is what bounds what the tenant's dataset can be made to hold.
50
+ #: MOVED from app.py:1410–1412 with the handler (app.py re-imports these during the strangler).
51
+ _DOCS_KEY = 'customer_docs'
52
+ _DOC_MAX_BYTES = 5 * 1024 * 1024
53
+ _DOC_MAX_PER_ROW = 50
54
+
55
+ #: The props of a field def the CLIENT actually draws β€” the wave-6 no-blip comparison set.
56
+ #: Host-only stamps (createdBy, permissions) are deliberately absent: the client renders
57
+ #: nothing from its own copy of those, and a permissions refusal is answered explicitly.
58
+ _FIELD_ECHO_PROPS = ('label', 'type', 'note', 'options', 'max', 'formula', 'agg',
59
+ 'scope', 'format', 'measure')
60
+
61
+
62
+ class StoreUnavailable(RuntimeError):
63
+ """The store is down AND the caller offered no durable fallback (HTTP adapter β†’ 503).
64
+
65
+ The Streamlit adapter never sees this: it passes a session-state-backed `fallback_ws`, so
66
+ its store-down path degrades to the same captioned in-memory workspace it always had. An
67
+ API caller has no such dict, and the honest answer to "persist this" with nowhere to
68
+ persist it is an error β€” not a 200 over a write that evaporates.
69
+ """
70
+
71
+
72
+ @dataclass
73
+ class EventResult:
74
+ """What the HANDLER needs to tell its adapter β€” the three things it cannot do itself.
75
+
76
+ `rerender` is the old return value with its old meaning: "this page must re-render from
77
+ server state", NOT "handled" (see `handle_one`'s docstring β€” the distinction is the
78
+ difference between a snappy grid and a sluggish one).
79
+ """
80
+ rerender: bool = False
81
+ doc: Optional[dict] = None # doc_fetch's answer β€” the bridge has no response channel
82
+ toast: Optional[str] = None # add_to_list's confirmation
83
+
84
+
85
+ @dataclass
86
+ class EventCtx:
87
+ """Everything the handler is allowed to know, and nothing about how it is being served.
88
+
89
+ ⚠ `allowed_pids` is a SECURITY parameter: it is the pool this caller may touch at all, and
90
+ every row-scoped event (overlay_patch, doc_*, cohort membership, add_to_list, memberPids)
91
+ is intersected with it. `admin` and `uname` are likewise the HOST's statement of identity β€”
92
+ never anything the browser said about itself.
93
+
94
+ `seen_ids` is ADAPTER-OWNED dedup state (the event log resends its recent window on every
95
+ emit, so consumed ids must stay consumed): Streamlit passes a session-state dict, HTTP
96
+ passes the caller's per-session mapping.
97
+
98
+ `fallback_ws` is the store-unavailable workspace home, the OUTER dict keyed by username
99
+ (exactly what `st.session_state.setdefault('cl_table_workspace', {})` is). HTTP passes
100
+ None, and None + store down raises StoreUnavailable (β†’ 503).
101
+
102
+ `out` is the sink for the two parked answers, so `handle_one` can keep returning a bare
103
+ bool exactly as it did in app.py while doc/toast still escape. (Additive amendment to X1's
104
+ field list, 2026-07-30 β€” defaulted; recorded in the S1 mailbox.)
105
+ """
106
+ uname: str
107
+ allowed_pids: frozenset
108
+ fields: list
109
+ measure_keys: frozenset = frozenset()
110
+ resolved_ids: frozenset = frozenset()
111
+ cohort_ids: frozenset = frozenset()
112
+ measure_offer: tuple = ()
113
+ admin: bool = False
114
+ scope_key: str = 'customer'
115
+ seen_ids: Optional[MutableMapping] = None
116
+ fallback_ws: Optional[MutableMapping] = None
117
+ out: EventResult = field(default_factory=EventResult)
118
+
119
+ def __post_init__(self):
120
+ # A caller that forgets `seen_ids` gets per-call dedup rather than a crash β€” the SET is
121
+ # what makes re-delivery idempotent, and its absence must not be the thing that decides
122
+ # whether an event is processed at all.
123
+ if self.seen_ids is None:
124
+ self.seen_ids = {}
125
+
126
+
127
+ # ------------------------------------------------------------------ the workspace family (X1 amd.)
128
+ def table_workspace(ctx, allowed_pids=None):
129
+ """Durable Airtable-style view/schema/overlay state, with an honest session fallback.
130
+
131
+ Wave-9 I17: the returned `views` are this user's OWN views MERGED with every view SHARED
132
+ with them. Own views win a same-id collision β€” a personal view is the user's own object and
133
+ must never be shadowed by somebody else's share.
134
+
135
+ ⚠ `allowed_pids` is a SECURITY parameter, not a convenience. A shared view carries
136
+ `memberPids`, which the writer's scope validated on the way IN β€” but the READER may have a
137
+ narrower scope. Handing a Fisch-scoped user a view whose member list was built by a
138
+ full-access user would leak the other BU's customers straight through the strict-isolation
139
+ rule. So membership is re-filtered against the READER on the way OUT. Pass it wherever the
140
+ result reaches a renderer or an event handler; omitting it drops memberPids entirely rather
141
+ than trusting the stored list (fail-closed: a smaller view, never a wider one).
142
+
143
+ MOVED from app.py:5323 (`_cl_table_workspace`). `admin` was a parameter defaulting to a live
144
+ `is_admin()` call; it now always comes from `ctx.admin`, which is identical at the one call
145
+ site because app.py computes `admin = bool(is_admin())` once per render. `allowed_pids`
146
+ stays an EXPLICIT argument because the folder leg deliberately omits it.
147
+ """
148
+ uname = ctx.uname
149
+ if store.available():
150
+ ws = cl_mod.table_workspace(uname)
151
+ try:
152
+ shared = cl_mod.shared_table_views(uname, bool(ctx.admin))
153
+ except Exception as _se:
154
+ _tel.error('sharedviews:read', _se)
155
+ shared = {}
156
+ if shared:
157
+ merged = dict(shared)
158
+ merged.update(ws.get('views') or {}) # own views win the id collision
159
+ ws['views'] = {vid: scope_view(v, uname, allowed_pids)
160
+ for vid, v in merged.items()}
161
+ return ws
162
+ # Store down. Streamlit degrades to its captioned session workspace exactly as before; an
163
+ # API caller has no durable dict and must be told, not quietly served memory.
164
+ if ctx.fallback_ws is None:
165
+ raise StoreUnavailable('table workspace unavailable: no store and no session fallback')
166
+ return ctx.fallback_ws.setdefault(
167
+ uname, {'views': {}, 'fields': {}, 'overlays': {}})
168
+
169
+
170
+ def scope_view(view, uname, allowed_pids):
171
+ """Re-scope a view to the READER. A no-op for a view the reader created.
172
+
173
+ Two things travel inside a shared view that are SCOPE, not presentation:
174
+ 1. `memberPids` β€” a customer list. Re-filtered against the reader's permitted pool.
175
+ 2. cohort conditions β€” a cohort id is a permission handle owned by ONE user. For a reader
176
+ who does not own it the leaf is UNANSWERABLE, and partial evaluation of an unanswerable
177
+ leaf WIDENS `is none of` ([[cg-cohort-set-conditions]]) β€” i.e. it would show MORE rows,
178
+ not fewer. So the whole filter tree is dropped for a foreign reader rather than
179
+ evaluated with a hole in it. A visibly unfiltered shared view is recoverable; a
180
+ silently widened one is a leak.
181
+ """
182
+ if not isinstance(view, dict) or view.get('createdBy') == uname:
183
+ return view
184
+ out = dict(view)
185
+ cfg = dict(out.get('config') or {})
186
+ mp = cfg.get('memberPids')
187
+ if isinstance(mp, list):
188
+ cfg['memberPids'] = ([p for p in mp if p in (allowed_pids or ())]
189
+ if allowed_pids is not None else [])
190
+ if tree_has_cohort(cfg.get('filters')):
191
+ cfg['filters'] = []
192
+ cfg['_scopedOut'] = 'cohort' # the client says so rather than pretending
193
+ out['config'] = cfg
194
+ return out
195
+
196
+
197
+ def tree_has_cohort(node):
198
+ """True when a filter tree references the Cohorts column anywhere (any depth)."""
199
+ import aios_grid as _agc
200
+ col = _agc.COHORT_COLUMN
201
+ if isinstance(node, list):
202
+ return any(tree_has_cohort(n) for n in node)
203
+ if not isinstance(node, dict):
204
+ return False
205
+ if node.get('colId') == col or node.get('field') == col:
206
+ return True
207
+ return tree_has_cohort(node.get('conditions') or node.get('children'))
208
+
209
+
210
+ def measure_leaves(nodes):
211
+ """Every leaf of a filter tree that carries a date WINDOW β€” i.e. every measure condition.
212
+
213
+ Keyed off the window rather than the column name on purpose: a window is the one thing no
214
+ column condition has, so this cannot mistake `revenue_ytd > 5000` for a measure.
215
+ """
216
+ out = []
217
+ for node in nodes or []:
218
+ if not isinstance(node, dict):
219
+ continue
220
+ if isinstance(node.get('children'), list):
221
+ out.extend(measure_leaves(node['children']))
222
+ elif node.get('window') is not None:
223
+ out.append(node)
224
+ return out
225
+
226
+
227
+ # ------------------------------------------------------------------ private handler helpers
228
+ def _field_echo_equal(accepted, raw):
229
+ """Does the ACCEPTED def match what the client already renders? (wave-6 item 3)
230
+
231
+ The no-blip law: a field_upsert whose acceptance changed NOTHING the client draws needs no
232
+ re-render β€” the client is the ORIGIN of the change, its optimistic copy is already right,
233
+ and the rerun it would buy costs a full payload re-serialisation plus an iframe remount
234
+ (the "blip" the owner named). STRICT per-prop equality after collapsing the empty forms
235
+ ('' / [] / {} / absent all render identically): any normalisation a validator applied is a
236
+ real difference the client must be SHOWN. Fail-closed β€” unsure means not equal means rerun.
237
+ """
238
+ empty = ('', [], {})
239
+ for k in _FIELD_ECHO_PROPS:
240
+ a, b = accepted.get(k), raw.get(k) if isinstance(raw, dict) else None
241
+ if k == 'scope':
242
+ # 'global' and absent are the SAME rendering (an unscoped field) β€” the event may
243
+ # say 'global' explicitly while the stored def says it by omission.
244
+ a = None if a == 'global' else a
245
+ b = None if b == 'global' else b
246
+ if (None if a in empty else a) != (None if b in empty else b):
247
+ return False
248
+ return True
249
+
250
+
251
+ def _known_usernames():
252
+ """Real account names, for C4's fail-closed `users` grant. None = "could not resolve".
253
+
254
+ None means DO NOT FILTER, deliberately. Returning an empty set on a transient registry
255
+ error would silently collapse a user's stored 'specific users' grant to 'personal' on the
256
+ very next autosave β€” destroying a setting because a read blipped. The grant is inert today
257
+ anyway (views are per-user; nothing reads it to allow cross-user access), so the honest
258
+ trade is: never invent access, never destroy a stored intent either.
259
+ """
260
+ try:
261
+ reg = set(users.registry() or {})
262
+ except Exception as _ue:
263
+ _tel.error('viewperms:users', _ue)
264
+ return None
265
+ # ⚠ EMPTY IS "UNRESOLVED", NOT "NOBODY EXISTS". registry() returns {} rather than raising
266
+ # when the user store cannot be read (a store blip logs `store:get:users` and hands back an
267
+ # empty dict) β€” and a working app ALWAYS has at least the bootstrap admin. Treating {} as a
268
+ # real answer would filter every name out of a stored grant and collapse it to 'personal'
269
+ # on the next autosave. Caught by testing this in BARE mode with no store token.
270
+ return reg or None
271
+
272
+
273
+ # ------------------------------------------------------------------ THE SEAM
274
+ def handle_events(events, ctx):
275
+ """The event LOG entry point β†’ one EventResult. The `[-24:]` window lives here.
276
+
277
+ ⚠ THE VALUE IS AN EVENT LOG `{'events': [...]}`, NOT ONE EVENT (2026-07-27). A component
278
+ has exactly one value slot, and two quick emits share it: creating a field emits
279
+ `field_upsert` and the insertColumn autosave emits `view_upsert` 420ms later β€” with the
280
+ server still inside the first rerun, the second write clobbered the slot before any run
281
+ read it, and the field creation VANISHED with no error anywhere (measured live; the same
282
+ race sat latent under overlay-field creation, masked by localStorage). The client now
283
+ sends its recent window of events; each is processed exactly once, keyed by its id. The
284
+ single-event shape is still accepted β€” it is what an older client in a stale tab sends.
285
+
286
+ Accepts everything the component value can be: the log dict, a bare list of events (what
287
+ the HTTP body's `events` array unwraps to), or one legacy single-event object.
288
+
289
+ `EventResult.rerender` is True when ANY processed event demands a re-render from server
290
+ state β€” the same OR-fold `_cl_handle_grid_event` did.
291
+ """
292
+ if isinstance(events, dict) and isinstance(events.get('events'), list):
293
+ seq = events['events'][-24:]
294
+ elif isinstance(events, list):
295
+ seq = events[-24:]
296
+ else:
297
+ seq = [events]
298
+ rerun = False
299
+ for one in seq:
300
+ rerun = handle_one(one, ctx) or rerun
301
+ ctx.out.rerender = rerun
302
+ return ctx.out
303
+
304
+
305
+ def handle_one(event, ctx):
306
+ """Validate ONE untrusted component event.
307
+
308
+ Odoo-source fields are never accepted as cell updates. View and schema objects are
309
+ user-scoped; row patches additionally require the pid to be in the currently permitted
310
+ Customer List pool.
311
+
312
+ RETURN VALUE = "this page must re-render from server state", NOT "handled".
313
+ The distinction is the difference between a snappy grid and a sluggish one.
314
+ Persisting is not by itself a reason to rerun: Streamlit has ALREADY re-run the
315
+ script once simply to deliver the component value, and the client is the ORIGIN
316
+ of the change, so its own state is correct before the server hears about it. An
317
+ extra st.rerun() therefore buys nothing and costs a second full page run.
318
+ Measured on the 1,550-row book (2026-07-25): one filter tweak produced TWO
319
+ page_customer_data runs ~2.8s apart, each re-serialising the whole ~576 KB row
320
+ payload β€” and the handler on the second run is a guaranteed no-op, because the
321
+ event-id dedupe two lines below has already consumed that id.
322
+
323
+ THE WAVE-6 NO-BLIP LAW (item 3, 2026-07-27) β€” True ONLY when the next render
324
+ genuinely differs from what the client already draws:
325
+ field_upsert β€” False when the accepted def ECHOES the client's copy
326
+ (_field_echo_equal); True on any validator mutation, any
327
+ permissions refusal, and ALWAYS when a measure window changed
328
+ (values must be re-resolved server-side).
329
+ overlay_patch β€” False when every requested update was accepted verbatim;
330
+ True the moment anything was refused or truncated.
331
+ field_delete β€” False: the client removed the column optimistically and the
332
+ store scrub is invisible to the next render.
333
+ field_duplicate β€” True only when host-held VALUES must flow back (custom_ cell
334
+ copies, measure_ recomputes); formula/created_time clones
335
+ compute client-side.
336
+ view_delete β€” False (the deleted view simply stops being served).
337
+ view_upsert β€” unchanged: False, except an unresolved measure condition.
338
+ cohort_* / add_to_list β€” True: the lists payload is server truth, and these are
339
+ not the hot path.
340
+
341
+ The id dedupe is a SET (not a last-id slot): the event log resends the whole recent
342
+ window on every emit, so every id must stay consumed, not just the latest one.
343
+ """
344
+ # The ctx fields the moved body reads as plain locals β€” bound once here so every branch
345
+ # below is textually the app.py body (which is what makes this move reviewable).
346
+ uname = ctx.uname
347
+ allowed_pids = ctx.allowed_pids
348
+ fields = ctx.fields
349
+ measure_keys = ctx.measure_keys
350
+ resolved_ids = ctx.resolved_ids
351
+ cohort_ids = ctx.cohort_ids
352
+ measure_offer = ctx.measure_offer
353
+ admin = ctx.admin
354
+ scope_key = ctx.scope_key
355
+
356
+ if not isinstance(event, dict):
357
+ return False
358
+ event_id = str(event.get('id') or '')[:180]
359
+ seen = ctx.seen_ids
360
+ if not event_id or event_id in seen:
361
+ return False
362
+ seen[event_id] = True
363
+ if len(seen) > 400: # a session-lifetime set, kept bounded
364
+ for stale in list(seen)[:200]:
365
+ seen.pop(stale, None)
366
+ kind = event.get('type')
367
+ field_by_key = {f['key']: f for f in fields}
368
+ valid_keys = set(field_by_key)
369
+ overlay_keys = {f['key'] for f in fields if f.get('source') == 'overlay'}
370
+ ws = table_workspace(ctx, allowed_pids=allowed_pids)
371
+
372
+ def _session_ready():
373
+ ws.setdefault('views', {})
374
+ ws.setdefault('fields', {})
375
+ ws.setdefault('overlays', {})
376
+
377
+ if kind == 'view_upsert':
378
+ raw = event.get('view')
379
+ if not isinstance(raw, dict) or not isinstance(raw.get('config'), dict):
380
+ return False
381
+ view_id = str(raw.get('id') or '').strip()[:120]
382
+ name = str(raw.get('name') or '').strip()[:120]
383
+ if not view_id or not name:
384
+ return False
385
+ cfg = raw['config']
386
+ # The filter TREE (leaf conditions and/or nested condition groups) is
387
+ # validated by the module-agnostic grid contract helper β€” imported lazily
388
+ # here for the same reason the page does it: a missing embed build must
389
+ # never break page import.
390
+ import aios_grid as _ag
391
+ # A MEASURE key is valid in a FILTER and nowhere else (CG-8): it may not be ordered,
392
+ # shown, grouped or coloured by, because it is not a column of this table. Hence the
393
+ # widened key set here and the plain `valid_keys` everywhere below.
394
+ # `cohort_ids` is a PERMISSION list, not a convenience: a cohort rule naming anything
395
+ # else is dropped here. Leaving it in would persist a condition the engine can only
396
+ # answer with "nothing", so `Cohort is not [a list I deleted]` would show an empty table
397
+ # forever with nothing on screen explaining why.
398
+ filters = _ag.clean_filter_tree(cfg.get('filters'),
399
+ valid_keys | set(measure_keys),
400
+ cohort_ids=set(cohort_ids))
401
+ sorts = []
402
+ for rule in list(cfg.get('sorts') or [])[:20]:
403
+ if (isinstance(rule, dict) and rule.get('colId') in valid_keys
404
+ and rule.get('dir') in ('asc', 'desc')):
405
+ sorts.append({'colId': rule['colId'], 'dir': rule['dir']})
406
+ order = list(dict.fromkeys(
407
+ key for key in list(cfg.get('order') or []) if key in valid_keys))
408
+ order += [field['key'] for field in fields if field['key'] not in order]
409
+ visible = list(dict.fromkeys(
410
+ key for key in list(cfg.get('visible') or []) if key in valid_keys))
411
+ widths = {}
412
+ raw_widths = cfg.get('widths') if isinstance(cfg.get('widths'), dict) else {}
413
+ for key, value in raw_widths.items():
414
+ if key in valid_keys:
415
+ try:
416
+ widths[key] = max(60, min(600, round(float(value))))
417
+ except (TypeError, ValueError):
418
+ pass
419
+ config = {
420
+ 'filters': filters,
421
+ # conjunction joining the ROOT-level conditions; anything but an
422
+ # explicit 'or' means 'and' (legacy views had an implicit AND).
423
+ 'filterConj': 'or' if cfg.get('filterConj') == 'or' else 'and',
424
+ 'sorts': sorts,
425
+ 'groupBy': cfg.get('groupBy') if cfg.get('groupBy') in valid_keys else None,
426
+ 'colorBy': cfg.get('colorBy') if cfg.get('colorBy') in valid_keys else None,
427
+ 'rowHeightMode': (cfg.get('rowHeightMode')
428
+ if cfg.get('rowHeightMode') in ('short', 'medium', 'tall')
429
+ else 'short'),
430
+ 'order': order,
431
+ 'visible': visible,
432
+ 'widths': widths,
433
+ 'memberPids': [pid for pid in list(cfg.get('memberPids') or [])
434
+ if isinstance(pid, int) and pid in allowed_pids],
435
+ }
436
+ # Wave-6 item 10: a non-grid display mode rides the view config, validated
437
+ # structurally (mode whitelist, refs must be this table's fields).
438
+ display = _ag._clean_display(cfg.get('display'), valid_keys)
439
+ if display:
440
+ config['display'] = display
441
+ # Wave-6 item 11 (pin): how many leading columns stay frozen while scrolling
442
+ # sideways. Clamped 1..8 β€” 0 would unfreeze the identity column the whole
443
+ # surface pivots on; absent = the legacy single frozen column.
444
+ if cfg.get('frozenCount') is not None:
445
+ try:
446
+ config['frozenCount'] = max(1, min(8, int(cfg['frozenCount'])))
447
+ except (TypeError, ValueError):
448
+ pass
449
+ # ── wave-9 C3 (lock) + C4 (permissions) ────────────────────────────────────────────
450
+ # The PRIOR stored view decides both: whether this is a create or an update (C4's
451
+ # default splits on exactly that) and who is allowed to change the lock (C3).
452
+ prior = (ws.get('views') or {}).get(view_id) or {}
453
+ # ⚠ THE SHARED WALL (wave-9 I17). `ws` is already visibility-filtered, so a view the
454
+ # caller may not SEE reads as absent β€” which would look like a CREATE and let anybody
455
+ # overwrite somebody else's shared view by guessing its id. Ask the store for the raw
456
+ # record and refuse before deciding anything else. Fail-closed on both branches.
457
+ _raw_shared = cl_mod.shared_table_view(view_id) if store.available() else None
458
+ if _raw_shared is not None and not prior:
459
+ return False # exists, not visible to this caller: refuse
460
+ if _raw_shared is not None and not _tstore._may_edit(_raw_shared, uname, admin):
461
+ return False # visible but not editable by this caller
462
+ is_new = not prior
463
+ # C4: the creator is HOST-STAMPED and never read from the browser β€” the same rule as
464
+ # the field-level createdBy. An existing view keeps whoever made it.
465
+ creator = prior.get('createdBy') or uname
466
+ # Only the creator or an admin may change the PERMISSIONS themselves. A collaborator who
467
+ # could rewrite them could grant themselves sole ownership of someone else's view, or
468
+ # quietly widen a users-scoped view to everyone β€” privilege escalation by edit.
469
+ if not is_new and not _tstore._may_administer(prior, uname, admin):
470
+ raw = dict(raw)
471
+ raw['permissions'] = prior.get('permissions')
472
+ raw['locked'] = prior.get('locked')
473
+ # ⚠ A SYSTEM VIEW IS PER-USER STATE AND MUST NEVER BE SHARED. Caught in the wave-9
474
+ # integration pass, before deploy: `all-customers` is every user's DEFAULT view and it
475
+ # carries their own column widths, visible fields and row height. C4's read-default
476
+ # ('collaborative' when permissions are absent, so nobody's saved view is
477
+ # retro-restricted) is right for a view somebody AUTHORED β€” but applied to the system
478
+ # view it routed `all-customers` into the shared bucket, and the first user to resize a
479
+ # column would have published their layout to everyone. Verified in the store:
480
+ # shared_table_views() was returning ['all-customers'].
481
+ # System views are pinned personal, so they stay in each user's own workspace exactly as
482
+ # before this wave. `kind` is host-validated two lines below; read it here from `raw` and
483
+ # treat the stored kind as authoritative for an existing view.
484
+ _kind = (prior.get('kind') if not is_new
485
+ else (raw.get('kind') if raw.get('kind') in ('system', 'list', 'custom')
486
+ else 'custom'))
487
+ if _kind == 'system' or view_id == 'all-customers':
488
+ perms = {'edit': 'personal'}
489
+ else:
490
+ perms = _ag.clean_view_permissions(
491
+ raw.get('permissions'),
492
+ default='personal' if is_new else 'collaborative',
493
+ known_users=_known_usernames())
494
+ # C3: only the creator or an admin may SET or CLEAR the lock; anyone else keeps whatever
495
+ # is already stored. Enforced here rather than hidden in the client. This composes with
496
+ # C4 and does not override it: C4 governs who may edit the CONTENTS, C3 who may freeze
497
+ # the display MODE.
498
+ # ⚠ Inert in practice TODAY β€” a user can only ever write their own per-user workspace,
499
+ # so `uname == creator` always holds. Correct plumbing for shared views, not a live gate.
500
+ may_lock = admin or uname == creator
501
+ locked = bool(raw.get('locked')) if may_lock else bool(prior.get('locked'))
502
+ # C3: the LOCK's actual job β€” a locked view's display mode is frozen, so a locked Kanban
503
+ # stays Kanban. This half IS live per-user and is the part the owner asked for. Filters,
504
+ # sorts and columns stay editable by design.
505
+ if locked and not is_new:
506
+ was = (prior.get('config') or {}).get('display')
507
+ now = config.get('display')
508
+ if (was or {}).get('mode') != (now or {}).get('mode'):
509
+ if was:
510
+ config['display'] = was
511
+ else:
512
+ config.pop('display', None)
513
+ view = {'id': view_id, 'name': name,
514
+ 'kind': raw.get('kind') if raw.get('kind') in ('system', 'list', 'custom')
515
+ else 'custom',
516
+ 'locked': locked,
517
+ 'createdBy': creator,
518
+ 'permissions': perms,
519
+ 'note': str(raw.get('note') or '')[:2000],
520
+ 'config': config}
521
+ if store.available():
522
+ cl_mod.save_table_view(uname, view)
523
+ else:
524
+ _session_ready()
525
+ ws['views'][view_id] = view
526
+ # PERSISTED, but deliberately NO rerun β€” see the return-value contract in
527
+ # this function's docstring. This is the hot path: every filter, sort,
528
+ # group, colour, column-resize and column-toggle autosaves through here.
529
+ #
530
+ # CG-8 is the ONE exception, and it is self-limiting. A measure condition is answered
531
+ # server-side, so if this view carries one whose answer was NOT in the payload we just
532
+ # shipped, the client is showing a pending marker with nothing coming. Ask for exactly
533
+ # one more run β€” and only then, so the common case (a column filter) still costs zero.
534
+ # If the top-of-run read of the component's own value already covered it, this is False
535
+ # and nothing extra happens; if that read is ever stale, this is what stops the
536
+ # condition hanging unresolved.
537
+ #
538
+ # ⚠ COMPLETE rules only (wave-6 item 3, found by the live roundtrip on this rev):
539
+ # picking a measure FIELD mints a valueless rule whose autosave used to land here as
540
+ # "unresolved" and trigger a rerun that could resolve NOTHING β€” `rule_complete` refuses
541
+ # half-asked questions β€” so the only effect was a remount that closed the builder
542
+ # popover mid-build (the wave-5 "gate flake" runs, now explained). A rule still being
543
+ # typed stays client-side PENDING by design; the run that matters fires when its VALUE
544
+ # arrives and the question becomes answerable.
545
+ from harness import measure_filter as _mfc
546
+ unresolved = {str(r.get('id') or '') for r in measure_leaves(filters)
547
+ if _mfc.rule_complete(r)} - set(resolved_ids)
548
+ return bool(unresolved - {''})
549
+
550
+ if kind == 'view_delete':
551
+ view_id = str(event.get('viewId') or '')[:120]
552
+ if not view_id or view_id == 'all-customers':
553
+ return False
554
+ # Wave-9 I17: deleting a SHARED view is narrower than editing it β€” creator or admin
555
+ # only. A collaborator can change what a shared view shows; they cannot destroy
556
+ # everybody else's copy of it, because there is only one copy.
557
+ if store.available():
558
+ _raw_shared = cl_mod.shared_table_view(view_id)
559
+ if _raw_shared is not None and not _tstore._may_administer(
560
+ _raw_shared, uname, admin):
561
+ return False
562
+ cl_mod.delete_table_view(uname, view_id)
563
+ else:
564
+ _session_ready()
565
+ ws['views'].pop(view_id, None)
566
+ # Wave-6 item 3: the client removed the view optimistically; a deleted view simply
567
+ # stops being served, so the next render differs in nothing the client still shows.
568
+ return False
569
+
570
+ if kind == 'field_upsert':
571
+ raw = event.get('field')
572
+ if not isinstance(raw, dict):
573
+ return False
574
+ key = str(raw.get('key') or '')[:80]
575
+ import aios_grid as _ag2
576
+ if key.startswith(_ag2.MEASURE_FIELD_PREFIX):
577
+ # A FORMULA-MEASURE column (owner item 7). Validated against THIS caller's measure
578
+ # offer β€” the same admission the condition builder uses β€” so a field can only name a
579
+ # measure this user could also filter by. Fail closed on anything else.
580
+ field = _ag2.clean_measure_field(raw, {m['key']: m for m in measure_offer})
581
+ if field is None:
582
+ return False
583
+ fmt = _ag2._clean_format(raw.get('format'), field.get('type'))
584
+ if fmt:
585
+ field['format'] = fmt
586
+ elif key in field_by_key and not key.startswith('custom_'):
587
+ base = field_by_key[key]
588
+ field = {**base, 'note': str(raw.get('note') or '')[:2000]}
589
+ # Wave-5 item 10: a DISPLAY format on any base field (presets included) β€” how a
590
+ # number/date reads, per user. Rendering-only; validated fail-closed per type.
591
+ fmt = _ag2._clean_format(raw.get('format'), base.get('type'))
592
+ if fmt:
593
+ field['format'] = fmt
594
+ else:
595
+ field.pop('format', None) # cleared -> back to the type's default render
596
+ # PRESET measure fields (wave-2 item 8): the browser may change the PERIOD (and the
597
+ # auto-renamed label) of revenue_ytd / revenue_ly / orders_24m β€” nothing else. The
598
+ # whitelist IS the base contract: only a field that shipped with preset+measure can
599
+ # take a window here, and the measure KEY is pinned to the base's (fail closed).
600
+ if (base.get('preset') and isinstance(base.get('measure'), dict)
601
+ and isinstance(raw.get('measure'), dict)):
602
+ window = _ag2._clean_window(raw['measure'].get('window'))
603
+ if window is not None:
604
+ field['measure'] = {'key': base['measure']['key'], 'window': window}
605
+ if isinstance(raw.get('label'), str) and raw['label'].strip():
606
+ field['label'] = str(raw['label'])[:120]
607
+ elif key.startswith('custom_') and raw.get('type') in _ag2.CUSTOM_FIELD_TYPES:
608
+ ftype = raw['type']
609
+ readonly = ftype in _ag2.READONLY_CUSTOM_TYPES
610
+ # The editable family must SAY it is overlay (the client's own contract); the
611
+ # read-only pair (formula / created_time) is stored under the cohort column's
612
+ # mechanism instead, so no cell write can ever be accepted for it.
613
+ if not readonly and raw.get('source') != 'overlay':
614
+ return False
615
+ field = {'key': key, 'label': str(raw.get('label') or 'Untitled')[:120],
616
+ 'type': ftype, 'source': 'odoo' if readonly else 'overlay',
617
+ 'default': True, 'custom': True,
618
+ 'note': str(raw.get('note') or '')[:2000]}
619
+ if readonly:
620
+ field['derived'] = True
621
+ # FILTERABLE since wave 6 (owner item 6): formula/created_time values live
622
+ # client-side, and this table's counts are client-mode β€” the client engine
623
+ # answers conditions on them soundly. Mirrors fields_from_workspace.
624
+ field['filterable'] = True
625
+ if raw.get('agg') == 'sum' and not readonly:
626
+ field['agg'] = 'sum'
627
+ # A `select`/`multiselect` carries its own vocabulary; a `user` does NOT β€” its
628
+ # choices are the tenant's accounts, resolved fresh on every render, so persisting
629
+ # a snapshot here would go stale the moment somebody joins or leaves.
630
+ if ftype in ('select', 'multiselect'):
631
+ field['options'] = _ag2._clean_options(raw.get('options'))
632
+ if not field['options']:
633
+ return False # a select with no choices can never hold a value
634
+ if ftype == 'rating':
635
+ field['max'] = _ag2._clean_rating_max(raw.get('max'))
636
+ if ftype == 'formula':
637
+ # WRITE-time is where refs must exist (fail closed); read-time leaves them
638
+ # alone so a later-deleted ref blanks the CELLS, not the column.
639
+ formula = _ag2._clean_formula(raw.get('formula'), valid_keys)
640
+ if formula is None:
641
+ return False
642
+ field['formula'] = formula
643
+ fmt = _ag2._clean_format(raw.get('format'), ftype)
644
+ if fmt:
645
+ field['format'] = fmt
646
+ else:
647
+ return False
648
+ # Wave-5 item 1 β€” createdBy + permissions on the CREATED strata (custom_/measure_).
649
+ # `createdBy` is stamped HOST-SIDE at create and preserved on every later write: the
650
+ # client never supplies it, so a browser cannot claim someone else's field. A
651
+ # permissions CHANGE is accepted only from the creator or an admin; a legacy field
652
+ # with no createdBy is admin-only (fail closed, both directions). The table workspace
653
+ # is per-user today, so this wall becomes load-bearing when sharing arrives β€” but the
654
+ # admin path and the legacy path are enforceable (and NC'd) right now.
655
+ needs_values = False
656
+ refused_perms = False
657
+ if key.startswith('custom_') or key.startswith(_ag2.MEASURE_FIELD_PREFIX):
658
+ prior = ws.get('fields', {}).get(key)
659
+ prior = prior if isinstance(prior, dict) else None
660
+ if prior is None:
661
+ field['createdBy'] = uname
662
+ # Wave-6 item 9: scope is chosen at CREATE, on the cohort page only β€”
663
+ # 'cohort' marks the field cohort-specific; 'global' or absent stays
664
+ # unscoped, so an old bundle that sends no scope keeps today's behavior.
665
+ if scope_key == 'cohort' and raw.get('scope') == 'cohort':
666
+ field['scope'] = 'cohort'
667
+ else:
668
+ if isinstance(prior.get('createdBy'), str) and prior['createdBy'].strip():
669
+ field['createdBy'] = prior['createdBy']
670
+ # Scope is PRESERVED like createdBy: the client copy never moves an
671
+ # existing field between scopes.
672
+ if prior.get('scope') == 'cohort':
673
+ field['scope'] = 'cohort'
674
+ want = _ag2._clean_permissions(raw.get('permissions'))
675
+ have = _ag2._clean_permissions((prior or {}).get('permissions'))
676
+ owner = field.get('createdBy')
677
+ if want is not None and (admin or (owner and owner == uname)):
678
+ field['permissions'] = want
679
+ else:
680
+ if have is not None:
681
+ field['permissions'] = have
682
+ # Asked for a permissions change and did not get it: the client's
683
+ # optimistic copy is a lie that must be repainted (wave-6 item 3).
684
+ refused_perms = want is not None and want != have
685
+ # A NEW measure column, or an existing one whose window moved: the VALUES
686
+ # are host-computed, so the next render genuinely differs.
687
+ if key.startswith(_ag2.MEASURE_FIELD_PREFIX):
688
+ needs_values = prior is None or prior.get('measure') != field.get('measure')
689
+ if store.available():
690
+ cl_mod.save_table_field(uname, field)
691
+ else:
692
+ _session_ready()
693
+ if not field.get('custom') and not str(field.get('note') or '').strip():
694
+ ws['fields'].pop(key, None)
695
+ else:
696
+ ws['fields'][key] = field
697
+ if needs_values or refused_perms:
698
+ return True
699
+ # THE NO-BLIP RETURN (wave-6 item 3): an accepted def that echoes what the client
700
+ # already renders needs no rerun β€” see _field_echo_equal and the docstring's law.
701
+ return not _field_echo_equal(field, raw)
702
+
703
+ if kind in ('cohort_add', 'cohort_remove', 'cohort_rename', 'cohort_delete'):
704
+ # Cohort membership/lifecycle edits from the grid's cohort panel (wave-2 item 2 β€”
705
+ # replaces the retired Manage popover, so rename/delete keep a surface). Fail-closed:
706
+ # only the caller's OWN cohorts (all_for), and member pids are intersected with the
707
+ # caller's permitted pool β€” the same guard add_to_list rides.
708
+ if not store.available():
709
+ return False
710
+ cohort_id = str(event.get('cohortId') or '').strip()[:120]
711
+ if not cohort_id or cohort_id not in cohort_mod.all_for(uname):
712
+ return False
713
+ try:
714
+ if kind == 'cohort_rename':
715
+ name = str(event.get('name') or '').strip()[:120]
716
+ if not name:
717
+ return False
718
+ cohort_mod.rename(uname, cohort_id, name)
719
+ elif kind == 'cohort_delete':
720
+ cohort_mod.delete(uname, cohort_id)
721
+ else:
722
+ raw_pids = event.get('pids')
723
+ if not isinstance(raw_pids, list):
724
+ return False
725
+ pids = [p for p in raw_pids if isinstance(p, int) and p in allowed_pids]
726
+ if not pids:
727
+ return False
728
+ if kind == 'cohort_add':
729
+ cohort_mod.add_members(uname, cohort_id, pids)
730
+ else:
731
+ cohort_mod.remove_members(uname, cohort_id, pids)
732
+ except ValueError as e:
733
+ _tel.error('cohort:panel-edit', e)
734
+ return False
735
+ # True: the lists payload (names, membership, counts) must reflect this on the next
736
+ # render β€” the panel is showing server truth, not local optimism.
737
+ return True
738
+
739
+ if kind in ('doc_add', 'doc_delete', 'doc_fetch'):
740
+ # DOCUMENTS on a customer row (owner item 12, contract C5). Metadata in the store key
741
+ # `customer_docs`; the BYTES live at their own path in the tenant dataset, never in a
742
+ # JSON blob β€” see core.store.upload_bytes for why that distinction is load-bearing.
743
+ import base64 as _b64c
744
+ import re as _re_mod
745
+ if not store.available():
746
+ return False
747
+ pid = event.get('pid')
748
+ doc_id = str(event.get('docId') or '').strip()[:80]
749
+ # The pid wall is the SAME one every other row-scoped event rides: a caller may only
750
+ # touch customers inside their own permitted pool. Without it, documents would be the
751
+ # one surface where a scoped agent could read another book's rows.
752
+ if not isinstance(pid, int) or pid not in allowed_pids or not doc_id:
753
+ return False
754
+ meta_all = store.get(_DOCS_KEY) or {}
755
+ row = list((meta_all.get(str(pid)) or []))
756
+
757
+ if kind == 'doc_add':
758
+ raw_b64 = event.get('data_b64')
759
+ name = str(event.get('name') or 'document')[:200]
760
+ if not isinstance(raw_b64, str) or not raw_b64:
761
+ return False
762
+ try:
763
+ blob = _b64c.b64decode(raw_b64, validate=True)
764
+ except Exception:
765
+ return False # not base64 -> not a file we will store
766
+ # Re-check the DECODED size host-side. The client checks it too, but a client check
767
+ # is a courtesy: this is the number that bounds what the tenant's repo can be made
768
+ # to hold by anyone who can reach the endpoint.
769
+ if not blob or len(blob) > _DOC_MAX_BYTES:
770
+ return False
771
+ if len(row) >= _DOC_MAX_PER_ROW or any(d.get('id') == doc_id for d in row):
772
+ return False
773
+ safe = _re_mod.sub(r'[^A-Za-z0-9._-]', '_', name)[:80] or 'file'
774
+ path = f'docs/{pid}/{doc_id}_{safe}'
775
+ try:
776
+ store.upload_bytes(path, blob, message=f'doc {doc_id} for customer {pid}')
777
+ except Exception as _ue:
778
+ _tel.error('docs:upload', _ue)
779
+ return False
780
+ row.append({'id': doc_id, 'name': name,
781
+ 'mime': str(event.get('mime') or 'application/octet-stream')[:120],
782
+ 'size': len(blob), 'by': uname,
783
+ 'ts': dt.datetime.now().strftime('%Y-%m-%dT%H:%M:%S'),
784
+ 'path': path})
785
+ elif kind == 'doc_delete':
786
+ hit = next((d for d in row if d.get('id') == doc_id), None)
787
+ if not hit:
788
+ return False
789
+ # THE WALL: the uploader or an admin. Enforced here regardless of what the client
790
+ # chose to render β€” `canDelete` in the payload only hides a button.
791
+ if hit.get('by') != uname and not admin:
792
+ return False
793
+ store.delete_path(hit.get('path') or '')
794
+ row = [d for d in row if d.get('id') != doc_id]
795
+ else: # doc_fetch
796
+ hit = next((d for d in row if d.get('id') == doc_id), None)
797
+ if not hit:
798
+ return False
799
+ blob = store.download_bytes(hit.get('path') or '')
800
+ if blob is None:
801
+ return False
802
+ # The bridge is a fire-and-forget event LOG with no response channel, so the answer
803
+ # is PARKED on the Result and rides the next render's payload (C5).
804
+ ctx.out.doc = {
805
+ 'pid': pid, 'docId': doc_id, 'name': hit.get('name') or 'document',
806
+ 'mime': hit.get('mime') or 'application/octet-stream',
807
+ 'data_b64': _b64c.b64encode(blob).decode('ascii')}
808
+ return True # re-render to deliver it
809
+
810
+ def _write(cur):
811
+ cur[str(pid)] = row
812
+ return cur
813
+ store.update(_DOCS_KEY, _write, flush='async')
814
+ return True
815
+
816
+ if kind in ('folder_create', 'folder_rename', 'folder_delete', 'folder_duplicate',
817
+ 'item_move'):
818
+ # FOLDERS over the Views / Cohorts sidebars (owner item 11, contract C4). All five
819
+ # events touch ONE home β€” the table workspace's `folders` + `itemFolders` β€” so a cohort
820
+ # can be filed without the cohort STORE learning about folders at all.
821
+ import aios_grid as _agf
822
+ if not store.available():
823
+ return False
824
+ surface = str(event.get('surface') or '')
825
+ if surface not in _agf.FOLDER_SURFACES:
826
+ return False
827
+ # The ids this user may actually file: their own views, their own cohorts. Fail-closed β€”
828
+ # an item_move naming somebody else's cohort is dropped by clean_item_folders anyway,
829
+ # but refusing here means we never write it in the first place.
830
+ def _own_ids(sfc, cur_ws):
831
+ if sfc == 'views':
832
+ return set((cur_ws.get('views') or {}))
833
+ return set(cohort_mod.all_for(uname))
834
+
835
+ def _mutate(cur_ws):
836
+ folders = _agf.clean_folders(cur_ws.get('folders'))
837
+ placed = dict(cur_ws.get('itemFolders') or {})
838
+ lst = list(folders.get(surface) or [])
839
+ # ⚠ the wire field is `folderId` on EVERY folder event (CLIENT's types.ts is the
840
+ # built contract; the split doc's first draft said `id`). Converged here rather
841
+ # than asking the client to re-emit β€” see the C4 reconciliation note in the doc.
842
+ fid = str(event.get('folderId') or '').strip()[:80]
843
+ if kind == 'folder_create':
844
+ name = str(event.get('name') or '').strip()[:_agf.MAX_FOLDER_NAME]
845
+ if not fid or not name or len(lst) >= _agf.MAX_FOLDERS:
846
+ return None
847
+ if any(f['id'] == fid for f in lst):
848
+ return None
849
+ row = {'id': fid, 'name': name, 'order': len(lst)}
850
+ icon = _agf.clean_folder_icon(event.get('icon')) # wave-9 I15 (C5)
851
+ if icon:
852
+ row['icon'] = icon
853
+ lst.append(row)
854
+ elif kind == 'folder_rename':
855
+ name = str(event.get('name') or '').strip()[:_agf.MAX_FOLDER_NAME]
856
+ hit = next((f for f in lst if f['id'] == fid), None)
857
+ if not hit or not name:
858
+ return None
859
+ hit['name'] = name
860
+ # wave-9 I15 (C5): the rename pane is also where the icon is changed. An event
861
+ # that carries NO icon key leaves the existing one alone (a rename must not
862
+ # silently strip a chosen icon); an explicit null clears it back to default.
863
+ if 'icon' in event:
864
+ icon = _agf.clean_folder_icon(event.get('icon'))
865
+ if icon:
866
+ hit['icon'] = icon
867
+ else:
868
+ hit.pop('icon', None)
869
+ elif kind == 'folder_delete':
870
+ if not any(f['id'] == fid for f in lst):
871
+ return None
872
+ lst = [f for f in lst if f['id'] != fid]
873
+ # CONTENTS MOVE TO ROOT, never delete. Dropping the placements is exactly that:
874
+ # an item with no placement renders at the top level.
875
+ placed[surface] = {k: v for k, v in (placed.get(surface) or {}).items()
876
+ if v != fid}
877
+ elif kind == 'folder_duplicate':
878
+ new_id = str(event.get('newId') or '').strip()[:80]
879
+ src = next((f for f in lst if f['id'] == fid), None)
880
+ if not src or not new_id or len(lst) >= _agf.MAX_FOLDERS:
881
+ return None
882
+ if any(f['id'] == new_id for f in lst):
883
+ return None
884
+ lst.append({'id': new_id, 'name': f"{src['name']} copy"[:_agf.MAX_FOLDER_NAME],
885
+ 'order': len(lst)})
886
+ # ⚠ The folder's CONTENTS are duplicated by the client emitting the ordinary
887
+ # per-item duplicate events it already owns (view_upsert with a fresh id, the
888
+ # cohort copy path) and then item_move-ing them here. Copying views/cohorts
889
+ # inside this handler would be a SECOND implementation of "duplicate an item",
890
+ # and the two would drift the first time either one changed.
891
+ else: # item_move
892
+ item_id = str(event.get('itemId') or '').strip()[:120]
893
+ target = event.get('folderId')
894
+ if not item_id or item_id not in _own_ids(surface, cur_ws):
895
+ return None
896
+ cur = dict(placed.get(surface) or {})
897
+ if target is None:
898
+ cur.pop(item_id, None) # back to the root
899
+ else:
900
+ tid = str(target)[:80]
901
+ if not any(f['id'] == tid for f in lst):
902
+ return None
903
+ cur[item_id] = tid
904
+ placed[surface] = cur
905
+ folders[surface] = lst
906
+ clean = _agf.clean_folders(folders)
907
+ return clean, _agf.clean_item_folders(
908
+ placed, clean,
909
+ {sfc: _own_ids(sfc, cur_ws) for sfc in _agf.FOLDER_SURFACES})
910
+
911
+ # ⚠ NO allowed_pids ON PURPOSE β€” this is app.py's bare `_cl_table_workspace(uname)`
912
+ # call at the old :6268, preserved exactly. Omitting the pool drops memberPids from
913
+ # any foreign shared view (fail-closed), which is harmless here because the folder
914
+ # legs read only view IDS and the folder maps. Passing the pool would be a behavior
915
+ # change inside a pure move.
916
+ _res = _mutate(table_workspace(ctx))
917
+ if _res is None:
918
+ return False # refused: nothing is written
919
+ cl_mod.save_table_folders(uname, _res[0], _res[1])
920
+ # True: the sidebars render from server state (which folder holds what), not from local
921
+ # optimism β€” the same contract the cohort panel's membership edits ride.
922
+ return True
923
+
924
+ if kind == 'field_delete':
925
+ # Delete a USER-CREATED column outright (owner gap closed 2026-07-27). The prefix IS
926
+ # the permission: only the created strata (`custom_` overlay fields, `measure_` formula
927
+ # columns) are deletable β€” a base field's key fails the prefix test no matter what the
928
+ # browser claims, so the table's contract cannot be deleted from the client. The store
929
+ # scrubs the field's stored cell values with it (delete_table_field); views naming the
930
+ # key self-heal on their next autosave, the rule every stale colId already rides.
931
+ key = str(event.get('key') or '')[:80]
932
+ import aios_grid as _ag3
933
+ if not (key.startswith('custom_') or key.startswith(_ag3.MEASURE_FIELD_PREFIX)):
934
+ return False
935
+ if store.available():
936
+ cl_mod.delete_table_field(uname, key)
937
+ else:
938
+ _session_ready()
939
+ ws['fields'].pop(key, None)
940
+ for _row in ws['overlays'].values():
941
+ if isinstance(_row, dict):
942
+ _row.pop(key, None)
943
+ # Wave-6 item 3: the client dropped the column optimistically; the store scrub is
944
+ # invisible to the next render (the values leave WITH the column).
945
+ return False
946
+
947
+ if kind == 'field_duplicate':
948
+ # Wave-5 item 1. The DESTINATION key is client-generated (the established pattern β€”
949
+ # the client must know it synchronously to place the clone right of the source), and
950
+ # both keys must sit in the SAME created stratum: base odoo fields offer no Duplicate,
951
+ # because duplicating the source-of-truth into an editable copy would need a
952
+ # value-snapshot mechanism nobody asked for. Overlay VALUES are copied only for
953
+ # `custom_` sources (a measure clone recomputes; a formula clone re-evaluates).
954
+ # The duplicator becomes the clone's creator β€” it is a NEW field they made.
955
+ src_key = str(event.get('sourceKey') or '')[:80]
956
+ new_key = str(event.get('key') or '')[:80]
957
+ import aios_grid as _ag5
958
+ prefix = ('custom_' if src_key.startswith('custom_')
959
+ else _ag5.MEASURE_FIELD_PREFIX
960
+ if src_key.startswith(_ag5.MEASURE_FIELD_PREFIX) else None)
961
+ if (prefix is None or not new_key.startswith(prefix) or new_key == src_key
962
+ or new_key in field_by_key):
963
+ return False
964
+ src = ws.get('fields', {}).get(src_key)
965
+ if not isinstance(src, dict):
966
+ return False
967
+ clone = dict(src)
968
+ clone['key'] = new_key
969
+ label = str(event.get('label') or '').strip()[:120]
970
+ clone['label'] = label or (str(src.get('label') or 'Untitled')[:110] + ' copy')
971
+ clone['createdBy'] = uname
972
+ # Wave-6 item 9: the clone inherits the source's scope via dict(src); on the cohort
973
+ # page the duplicate surface may explicitly place it instead (same create-time rule
974
+ # as field_upsert β€” a page that is not the cohort page cannot mint a cohort field).
975
+ if scope_key == 'cohort' and event.get('scope') in ('cohort', 'global'):
976
+ if event['scope'] == 'cohort':
977
+ clone['scope'] = 'cohort'
978
+ else:
979
+ clone.pop('scope', None)
980
+ if store.available():
981
+ cl_mod.duplicate_table_field(uname, src_key, new_key, clone)
982
+ else:
983
+ _session_ready()
984
+ ws['fields'][new_key] = clone
985
+ if src_key.startswith('custom_'):
986
+ for _row in ws['overlays'].values():
987
+ if isinstance(_row, dict) and src_key in _row:
988
+ _row[new_key] = _row[src_key]
989
+ # Wave-6 item 3: True only when host-held VALUES must flow back β€” copied overlay
990
+ # cells (custom_ sources) or host-computed measure values. formula/created_time
991
+ # clones compute client-side, so the optimistic insert is already complete.
992
+ return src.get('type') not in ('formula', 'created_time')
993
+
994
+ if kind == 'add_to_list':
995
+ # "Add to list" (owner item 6): a view's current matches become fixed cohort members.
996
+ #
997
+ # ⚠ `pids` arrives from the BROWSER. It is intersected with `allowed_pids` β€” the pool this
998
+ # user may actually see β€” before anything is stored. The client computing the set is a
999
+ # correctness decision (it is the only engine that already knows the answer, and asking a
1000
+ # second engine the same question is the drift verify_filter_engine.py exists to prevent);
1001
+ # it is NOT a trust decision.
1002
+ raw_pids = event.get('pids')
1003
+ if not isinstance(raw_pids, list):
1004
+ return False
1005
+ pids = [p for p in raw_pids if isinstance(p, int) and p in allowed_pids]
1006
+ if not pids:
1007
+ return False
1008
+ cohort_id = str(event.get('cohortId') or '').strip()[:120]
1009
+ name = str(event.get('name') or '').strip()[:120]
1010
+ if not store.available():
1011
+ return False
1012
+ try:
1013
+ if not cohort_id:
1014
+ if not name:
1015
+ return False
1016
+ cohort_id = cohort_mod.create(uname, name, pids)
1017
+ else:
1018
+ if cohort_id not in cohort_mod.all_for(uname):
1019
+ return False # not this user's cohort β€” fail closed, do not create
1020
+ cohort_mod.add_members(uname, cohort_id, pids)
1021
+ except ValueError as e:
1022
+ _tel.error('cohort:add_to_list', e)
1023
+ return False
1024
+ # True: the Cohort page's list and counts must reflect this, and the toast below only
1025
+ # renders on a fresh run.
1026
+ ctx.out.toast = (
1027
+ f"Added {len(pids):,} customer{'' if len(pids) == 1 else 's'} to "
1028
+ f"{name or 'the list'}.")
1029
+ return True
1030
+
1031
+ if kind == 'overlay_patch':
1032
+ pid = event.get('pid')
1033
+ if not isinstance(pid, int) or pid not in allowed_pids:
1034
+ return False
1035
+ updates = {}
1036
+ refused = False
1037
+ raw_updates = event.get('updates') if isinstance(event.get('updates'), dict) else {}
1038
+ for key, value in raw_updates.items():
1039
+ if key not in overlay_keys or not isinstance(value, (str, int, float)):
1040
+ refused = True
1041
+ continue
1042
+ # Wave-5 item 1: `permissions.edit` is enforced HERE, per key β€” the client hiding
1043
+ # its editor is courtesy, this is the wall. Fail-closed: a restricted field with
1044
+ # no readable createdBy admits only admins.
1045
+ perms = (field_by_key.get(key) or {}).get('permissions') or {}
1046
+ edit = perms.get('edit')
1047
+ if edit == 'admins' and not admin:
1048
+ refused = True
1049
+ continue
1050
+ if edit == 'creator' and not (
1051
+ admin or (field_by_key.get(key) or {}).get('createdBy') == uname):
1052
+ refused = True
1053
+ continue
1054
+ clean = str(value)[:10000]
1055
+ if isinstance(value, str) and clean != value:
1056
+ # Truncated: the stored truth is shorter than the client's copy. (An
1057
+ # int/float arriving as its string form renders identically β€” not a refusal.)
1058
+ refused = True
1059
+ updates[key] = clean
1060
+ if updates:
1061
+ if store.available():
1062
+ cl_mod.patch_table_overlay(uname, pid, updates)
1063
+ else:
1064
+ _session_ready()
1065
+ ws['overlays'].setdefault(str(pid), {}).update(updates)
1066
+ # Wave-6 item 3: the client's cell already shows the typed value β€” rerun ONLY when
1067
+ # some of what it shows was refused or clamped and must be repainted to the truth.
1068
+ return refused
1069
+
1070
+ return False
royalimports_os/core/links.py ADDED
@@ -0,0 +1,59 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Deep links into the app β€” ?page=<key>&entity=<kind>:<id>&src=<source>.
2
+
3
+ The app (app.py) consumes these params once per session right after login and routes to the page,
4
+ opening the entity drawer when an entity is present. Digest/alert/mention emails build their links
5
+ here so every emailed number lands the reader on the exact row it came from. `src` tags where the
6
+ click came from (digest / alert / mention) β€” the app logs it as the click-through read receipt.
7
+ """
8
+ import os
9
+ import sys
10
+ from urllib.parse import quote
11
+
12
+ #: β›” THE ONE HOST LITERAL LEFT IN THE RUNTIME PATH, and it is a MIGRATION HAZARD β€” named here
13
+ #: rather than buried in a default argument, because the day we change hosts this is what silently
14
+ #: keeps mailing customers a link to the DEAD one. Found by `ops/verify_portability.py` on its
15
+ #: first run (2026-07-30).
16
+ #:
17
+ #: Why it is still here instead of raising: `deeplink()` has exactly one family of callers
18
+ #: (`modules/digest.py`, 7 sites, all building email hrefs), `DIGEST_ENABLED` defaults to ON, and
19
+ #: no deploy sets `APP_BASE_URL` today β€” so refusing to guess would break a LIVE feature to fix a
20
+ #: latent one. Instead: both deploy scripts now push `APP_BASE_URL`, which makes the fallback
21
+ #: unreachable in a deployed environment, and reaching it prints an operator warning.
22
+ #:
23
+ #: β‡’ EXIT CONDITION for this constant: once a deploy has set `APP_BASE_URL` and it is confirmed
24
+ #: present in the running environment, delete the fallback and let `deeplink()` raise instead. It
25
+ #: is declared in the portability gate's RESIDUALS so the gate reports it honestly rather than
26
+ #: reporting green over it.
27
+ _LEGACY_HF_FALLBACK = 'https://royal-imports-cfo-os.hf.space'
28
+
29
+ BASE_URL = (os.environ.get('APP_BASE_URL') or _LEGACY_HF_FALLBACK).rstrip('/')
30
+
31
+ #: Warned LAZILY and ONCE β€” at the first link actually built, not at import. An import-time print
32
+ #: fires in every gate, every subprocess and every unit test that so much as touches `core`, which
33
+ #: is how a real warning becomes noise nobody reads.
34
+ _WARNED = [False]
35
+
36
+
37
+ def _warn_once_if_defaulted():
38
+ if _WARNED[0] or os.environ.get('APP_BASE_URL'):
39
+ return
40
+ _WARNED[0] = True
41
+ print(f"[links] APP_BASE_URL is not set - emailed deep links will point at "
42
+ f"{_LEGACY_HF_FALLBACK}, which is correct ONLY while that is where the app lives. "
43
+ f"Set APP_BASE_URL in the deployed environment.", file=sys.stderr)
44
+
45
+ # entity kinds the URL scheme supports (must stay in sync with app.py's _desc_from_param)
46
+ LINK_KINDS = ('customer', 'sku', 'agent', 'account')
47
+
48
+
49
+ def deeplink(page=None, kind=None, ident=None, src=None):
50
+ """An absolute app URL. deeplink('customer_data', 'customer', 1234, src='digest')."""
51
+ _warn_once_if_defaulted()
52
+ parts = []
53
+ if page:
54
+ parts.append(f"page={quote(str(page), safe='')}")
55
+ if kind in LINK_KINDS and ident not in (None, ''):
56
+ parts.append(f"entity={quote(f'{kind}:{ident}', safe=':')}")
57
+ if src:
58
+ parts.append(f"src={quote(str(src), safe='')}")
59
+ return BASE_URL + ('/?' + '&'.join(parts) if parts else '/')
royalimports_os/core/management_pl.py ADDED
@@ -0,0 +1,161 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """core/management_pl.py β€” reproduce the operating model's MANAGEMENT P&Ls
2
+ (FFS-M / RI-M / GD-M / BU-M) monthly from RAW ODOO, Sep-2024 cutover onward. READ-ONLY.
3
+
4
+ Decoded from v42 (see 17. Application development/financial_analysis/MODEL_LOGIC_MAP.md Β§10):
5
+
6
+ -M[entity][account] = clean_entity[account] # live from Odoo (PROVEN to the cent)
7
+ + hq_overlay[entity][account] # HQ cost allocated to the entity
8
+ + adj_overrides[entity][account] # tiny, transition-month only
9
+
10
+ clean_entity = account.analytic.line grouped by general_account_id (GL row) x account_id
11
+ (analytic = ENTITY column). Analytic ids: Fisch=1 Royal=2 GiftwareDeals=3
12
+ HQ=4 Internal=5. Sections by account_type. This is build_income_statements.py
13
+ (verified exact). Proven live: Oct-2025 acct 51000 -> Fisch 327,531.85 etc.
14
+
15
+ hq_overlay / adj_overrides come from data/mgmt_overlay.json (extracted read-only from v42 by
16
+ build_mgmt_overlay.py). The HQ allocation is HARDCODED in the model (PNL (HQ-*) tabs are literal
17
+ values, not formulas), so storing it is faithful to how the model is actually maintained.
18
+
19
+ Consolidation: BU-M = FFS-M + RI-M + GD-M (the Internal analytic is eliminated; HQ is allocated out).
20
+
21
+ Pure: data in -> plain dicts out. No Streamlit, no writes.
22
+ """
23
+ from __future__ import annotations
24
+ import json
25
+ import os
26
+ from functools import lru_cache
27
+
28
+ from core import odoo as O
29
+
30
+ # ---- entity <-> Odoo analytic account ids (confirmed live) --------------------------------------
31
+ ENTITY_ANALYTIC = {'FFS': 1, 'RI': 2, 'GD': 3, 'HQ': 4, 'Internal': 5}
32
+ ANALYTIC_ENTITY = {v: k for k, v in ENTITY_ANALYTIC.items()}
33
+ BUS = ['FFS', 'RI', 'GD'] # the business units that get a -M statement
34
+ ENTITY_LABEL = {'FFS': 'Fisch (FFS)', 'RI': 'Royal (RI)', 'GD': 'Giftware Deals (GD)', 'BU': 'Consolidated (BU)'}
35
+
36
+ # P&L sections by Odoo account_type (mirrors build_income_statements.py)
37
+ INCOME_TYPES = {'income'}
38
+ OTHER_INCOME_TYPES = {'income_other'}
39
+ COGS_TYPES = {'expense_direct_cost'}
40
+ EXPENSE_TYPES = {'expense'}
41
+ DEPREC_TYPES = {'expense_depreciation'}
42
+ PL_TYPES = INCOME_TYPES | OTHER_INCOME_TYPES | COGS_TYPES | EXPENSE_TYPES | DEPREC_TYPES
43
+
44
+ _OVERLAY_PATH = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))),
45
+ 'data', 'mgmt_overlay.json')
46
+
47
+
48
+ def month_bounds(year: int, month: int) -> tuple[str, str]:
49
+ import calendar
50
+ last = calendar.monthrange(year, month)[1]
51
+ return f'{year}-{month:02d}-01', f'{year}-{month:02d}-{last:02d}'
52
+
53
+
54
+ @lru_cache(maxsize=1)
55
+ def account_master() -> dict:
56
+ """{gl_account_id: {'code','name','type'}} for every account."""
57
+ rows = O.search_read('account.account', [], ['id', 'code', 'name', 'account_type'])
58
+ return {r['id']: {'code': r.get('code') or '', 'name': r.get('name') or '',
59
+ 'type': r.get('account_type')} for r in rows}
60
+
61
+
62
+ def _display(acc_type: str, raw: float) -> float:
63
+ """Income shown raw; costs/expenses/depreciation flipped positive (model convention)."""
64
+ if acc_type in INCOME_TYPES or acc_type in OTHER_INCOME_TYPES:
65
+ return raw
66
+ return -raw
67
+
68
+
69
+ @lru_cache(maxsize=64)
70
+ def clean_entities(year: int, month: int) -> dict:
71
+ """The clean per-entity P&L straight from Odoo analytic lines, for one month.
72
+
73
+ Returns {entity_key: {account_code: display_amount}} for FFS/RI/GD/HQ/Internal, restricted to
74
+ P&L account types. This is the LIVE, proven-exact foundation (= build_income_statements.py).
75
+ """
76
+ start, end = month_bounds(year, month)
77
+ rows = O.read_group('account.analytic.line',
78
+ domain=[('date', '>=', start), ('date', '<=', end)],
79
+ fields=['amount:sum'], groupby=['general_account_id', 'account_id'], lazy=False)
80
+ am = account_master()
81
+ out = {k: {} for k in ENTITY_ANALYTIC}
82
+ for r in rows:
83
+ gen = r.get('general_account_id'); ana = r.get('account_id')
84
+ if not gen or not ana:
85
+ continue
86
+ gid, aid = gen[0], ana[0]
87
+ ent = ANALYTIC_ENTITY.get(aid)
88
+ acc = am.get(gid)
89
+ if ent is None or not acc or acc['type'] not in PL_TYPES:
90
+ continue
91
+ amt = _display(acc['type'], r.get('amount') or 0.0)
92
+ if abs(amt) < 0.005:
93
+ continue
94
+ out[ent][acc['code']] = out[ent].get(acc['code'], 0.0) + amt
95
+ return out
96
+
97
+
98
+ @lru_cache(maxsize=1)
99
+ def _overlay() -> dict:
100
+ """The stored HQ-allocation + transition-reconciliation overlay (build_mgmt_overlay.py).
101
+ Shape: {'YYYY-MM': {'FFS': {acct: amt}, ...}}. These are financial figures, so they live in the
102
+ PRIVATE HF Dataset store on the Space (never in the public Space repo); locally they load from the
103
+ committed JSON. File first, so local runs need no token. Empty -> -M = clean entity only."""
104
+ try:
105
+ with open(_OVERLAY_PATH, encoding='utf-8') as f:
106
+ return json.load(f)
107
+ except (FileNotFoundError, json.JSONDecodeError):
108
+ pass
109
+ try:
110
+ from core import store
111
+ return store.get('mgmt_overlay') or {}
112
+ except Exception:
113
+ return {}
114
+
115
+
116
+ def management_pl(year: int, month: int, entity: str) -> dict:
117
+ """The management P&L for one entity-month: {account_code: amount} = clean + overlay.
118
+
119
+ entity in {'FFS','RI','GD'}; 'BU' returns the consolidation (sum of the three).
120
+ """
121
+ if entity == 'BU':
122
+ agg: dict = {}
123
+ for bu in BUS:
124
+ for code, amt in management_pl(year, month, bu).items():
125
+ agg[code] = agg.get(code, 0.0) + amt
126
+ return agg
127
+ clean = dict(clean_entities(year, month).get(entity, {}))
128
+ ov = _overlay().get(f'{year}-{month:02d}', {}).get(entity, {})
129
+ for code, amt in ov.items():
130
+ clean[code] = clean.get(code, 0.0) + amt
131
+ return clean
132
+
133
+
134
+ # ---- section roll-ups (for the statement view) --------------------------------------------------
135
+ def _bucket(code: str) -> str | None:
136
+ acc = next((a for a in account_master().values() if a['code'] == code), None)
137
+ if not acc:
138
+ return None
139
+ t = acc['type']
140
+ if t in INCOME_TYPES: return 'income'
141
+ if t in OTHER_INCOME_TYPES: return 'other_income'
142
+ if t in COGS_TYPES: return 'cogs'
143
+ if t in EXPENSE_TYPES: return 'expense'
144
+ if t in DEPREC_TYPES: return 'deprec'
145
+ return None
146
+
147
+
148
+ def statement(year: int, month: int, entity: str) -> dict:
149
+ """Section subtotals + net for one entity-month (income / cogs / gross / opex / deprec / net)."""
150
+ lines = management_pl(year, month, entity)
151
+ sec = {'income': 0.0, 'other_income': 0.0, 'cogs': 0.0, 'expense': 0.0, 'deprec': 0.0}
152
+ for code, amt in lines.items():
153
+ b = _bucket(code)
154
+ if b:
155
+ sec[b] += amt
156
+ income = sec['income'] + sec['other_income']
157
+ gross = income - sec['cogs']
158
+ net = gross - sec['expense'] - sec['deprec']
159
+ return {'income': income, 'cogs': sec['cogs'], 'gross_profit': gross,
160
+ 'operating_expense': sec['expense'], 'depreciation': sec['deprec'], 'net_profit': net,
161
+ 'lines': lines}
royalimports_os/core/odoo.py ADDED
@@ -0,0 +1,201 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """CFO-OS shared Odoo data layer (READ-ONLY).
2
+
3
+ Reuses the existing write-guarded client (ffs_dashboard/odoo_client.py) so there is
4
+ a single source of truth for the connection. Adds:
5
+ - cached singleton client
6
+ - read_group / search_read passthroughs with light cleanup
7
+ - business constants (teams, excluded partners)
8
+ - the "sales universe" base domain (confirmed orders, RI+FFS only)
9
+
10
+ NOTHING here writes to Odoo β€” the underlying client hard-blocks writes.
11
+ """
12
+ import os
13
+ import sys
14
+ import queue as _queue
15
+ import threading
16
+ from concurrent.futures import ThreadPoolExecutor
17
+ from functools import lru_cache
18
+ from pathlib import Path
19
+
20
+ # Self-contained read-only client lives at the app root (royalimports_os/odoo_client.py) with its
21
+ # own .env β€” so this whole folder can be moved/hosted anywhere without external paths.
22
+ _ROOT = Path(__file__).resolve().parents[1] # royalimports_os/
23
+ if str(_ROOT) not in sys.path:
24
+ sys.path.insert(0, str(_ROOT))
25
+ from odoo_client import OdooClient # noqa: E402 (read-only, write-guarded)
26
+
27
+ # ---- business constants ----
28
+ TEAM_IDS = [5, 6]
29
+ TEAM_NAMES = {5: 'Fisch', 6: 'Royal'}
30
+ # Partners to exclude from analytics (not part of RI/FFS). Sourced from env so no
31
+ # customer name is ever hardcoded in the (potentially public) repo; set as a Space secret.
32
+ EXCLUDE_PARTNER_NAMES = {n.strip() for n in os.environ.get('EXCLUDE_PARTNER_NAMES', '').split(',') if n.strip()}
33
+
34
+ # Confirmed sale.order.line scope for RI+FFS (the analytics "sales universe").
35
+ SALE_LINE_BASE = [
36
+ ('state', 'in', ['sale', 'done']),
37
+ ('order_id.team_id', 'in', TEAM_IDS),
38
+ ('product_id', '!=', False),
39
+ ]
40
+ # Posted customer invoices/credit notes (AR + revenue recognised).
41
+ INVOICE_BASE = [
42
+ ('move_type', 'in', ['out_invoice', 'out_refund']),
43
+ ('state', '=', 'posted'),
44
+ ]
45
+
46
+
47
+ @lru_cache(maxsize=1)
48
+ def _global_client():
49
+ """The single read-only client used for normal (main-thread) calls."""
50
+ return OdooClient()
51
+
52
+
53
+ # ---- concurrent read pool -------------------------------------------------------------------
54
+ # Each Odoo XML-RPC call is a round-trip; a heavy drawer fires a dozen of them and they add up to
55
+ # many seconds when run one-by-one. `parallel()` runs independent READ tasks across a small pool of
56
+ # extra connections, with each task transparently using its own connection via a thread-local β€” so
57
+ # the existing read_group/search_read/sum_field helpers need no change. READ-ONLY only.
58
+ _POOL_MAX = 8
59
+ _tlocal = threading.local()
60
+ _pool_q = _queue.Queue()
61
+ _pool_n = 0
62
+ _pool_lock = threading.Lock()
63
+
64
+
65
+ def _checkout():
66
+ global _pool_n
67
+ try:
68
+ return _pool_q.get_nowait()
69
+ except _queue.Empty:
70
+ with _pool_lock:
71
+ grow = _pool_n < _POOL_MAX
72
+ if grow:
73
+ _pool_n += 1
74
+ return OdooClient() if grow else _pool_q.get() # else block until one frees
75
+
76
+
77
+ def _checkin(c):
78
+ _pool_q.put(c)
79
+
80
+
81
+ def get_odoo():
82
+ """The active read-only client. Inside a parallel() task this is that task's pooled connection;
83
+ otherwise it's the shared singleton. Transparent to every read_group/search_read/... caller."""
84
+ c = getattr(_tlocal, 'client', None)
85
+ return c if c is not None else _global_client()
86
+
87
+
88
+ def set_doc_mode(mode):
89
+ """Recognition basis for THIS thread's domain builds: 'order' (all confirmed orders) or 'invoice'
90
+ (fully-invoiced orders only). Cached bundles call this before pulling; parallel() propagates it to
91
+ each worker. Thread-local, so concurrent user sessions never race on each other's basis."""
92
+ _tlocal.doc_mode = 'invoice' if mode == 'invoice' else 'order'
93
+
94
+
95
+ def doc_mode():
96
+ return getattr(_tlocal, 'doc_mode', 'order')
97
+
98
+
99
+ def parallel(tasks):
100
+ """Run independent zero-arg READ callables concurrently across the connection pool; returns
101
+ results in order. Inside each task, get_odoo() (hence read_group/search_read/sum_field/
102
+ search_count) uses that task's own connection. Never use for writes."""
103
+ tasks = list(tasks)
104
+ if not tasks:
105
+ return []
106
+ if len(tasks) == 1:
107
+ return [tasks[0]()]
108
+
109
+ caller_mode = doc_mode() # propagate the submitter's basis to each worker
110
+ def _run(fn):
111
+ c = _checkout()
112
+ _tlocal.client = c
113
+ _tlocal.doc_mode = caller_mode
114
+ try:
115
+ return fn()
116
+ finally:
117
+ _tlocal.client = None
118
+ _tlocal.doc_mode = 'order'
119
+ _checkin(c)
120
+ with ThreadPoolExecutor(max_workers=min(len(tasks), _POOL_MAX)) as ex:
121
+ return list(ex.map(_run, tasks))
122
+
123
+
124
+ def parallel_map(task_dict):
125
+ """parallel() over a {name: callable} mapping -> {name: result}."""
126
+ keys = list(task_dict)
127
+ return dict(zip(keys, parallel([task_dict[k] for k in keys])))
128
+
129
+
130
+ def m2o_id(v):
131
+ return v[0] if isinstance(v, list) and v else None
132
+
133
+
134
+ def m2o_name(v):
135
+ return v[1] if isinstance(v, list) and len(v) > 1 else ''
136
+
137
+
138
+ @lru_cache(maxsize=1)
139
+ def excluded_partner_ids():
140
+ """Resolve the excluded partner names (from EXCLUDE_PARTNER_NAMES) to ids, for fast
141
+ domain filtering. Cached for the process."""
142
+ o = get_odoo()
143
+ if not EXCLUDE_PARTNER_NAMES:
144
+ return tuple()
145
+ rows = o.search_read('res.partner',
146
+ [('name', 'in', list(EXCLUDE_PARTNER_NAMES))], ['id'])
147
+ return tuple(r['id'] for r in rows)
148
+
149
+
150
+ def sale_line_domain(date_from=None, date_to=None, team_id=None, extra=None, partner_ids=None):
151
+ """Build a sale.order.line domain in the RI+FFS confirmed-order scope, excluding
152
+ the configured house accounts, with optional date window / single team / extra clauses.
153
+ partner_ids (a collection, possibly empty) restricts to those customers β€” the carrier for
154
+ the Customer module's Agent filter; None = no partner restriction (an empty set matches none)."""
155
+ dom = list(SALE_LINE_BASE)
156
+ if team_id is not None:
157
+ dom = [d for d in dom if not (isinstance(d, tuple) and d[0] == 'order_id.team_id')]
158
+ dom.append(('order_id.team_id', '=', team_id))
159
+ if date_from:
160
+ dom.append(('order_id.date_order', '>=', f'{date_from} 00:00:00'))
161
+ if date_to:
162
+ dom.append(('order_id.date_order', '<=', f'{date_to} 23:59:59'))
163
+ ex = excluded_partner_ids()
164
+ if ex:
165
+ dom.append(('order_partner_id', 'not in', list(ex)))
166
+ if partner_ids is not None:
167
+ dom.append(('order_partner_id', 'in', list(partner_ids)))
168
+ if doc_mode() == 'invoice':
169
+ dom.append(('order_id.invoice_status', '=', 'invoiced'))
170
+ if extra:
171
+ dom.extend(extra)
172
+ return dom
173
+
174
+
175
+ def read_group(model, domain, fields, groupby, **kw):
176
+ return get_odoo().read_group(model, domain=domain, fields=fields, groupby=groupby, **kw)
177
+
178
+
179
+ def search_read(model, domain=None, fields=None, **kw):
180
+ return get_odoo().search_read(model, domain=domain or [], fields=fields or [], **kw)
181
+
182
+
183
+ def sum_field(model, domain, field):
184
+ """One-line aggregate sum of `field` over a domain (uses read_group, no row fetch).
185
+
186
+ Robust to the empty match set: a groupby=[] read_group over zero rows makes Odoo return a
187
+ None aggregate, which the server's XML-RPC layer cannot marshal ("cannot marshal None").
188
+ An empty set simply means the sum is 0.0, so we treat that specific Fault as zero."""
189
+ try:
190
+ g = get_odoo().read_group(model, domain=domain, fields=[f'{field}:sum'], groupby=[], lazy=False)
191
+ except Exception as e: # noqa: BLE001 β€” narrow check below
192
+ if 'cannot marshal None' in str(e):
193
+ return 0.0
194
+ raise
195
+ return (g[0].get(field) or 0.0) if g else 0.0
196
+
197
+
198
+ def distinct_count(model, domain, field):
199
+ """Distinct count of a (m2o) field over a domain via grouped read_group."""
200
+ g = get_odoo().read_group(model, domain=domain, fields=[field], groupby=[field], lazy=False)
201
+ return sum(1 for r in g if r.get(field))
royalimports_os/core/periods.py ADDED
@@ -0,0 +1,155 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Period helpers β€” floral wholesale is highly seasonal, so YoY / same-period
2
+ comparisons are first-class. Everything returns ISO date strings (yyyy-mm-dd).
3
+
4
+ `today` is injectable for testing/reproducibility.
5
+ """
6
+ import datetime as dt
7
+
8
+
9
+ def _d(x):
10
+ return x.isoformat()
11
+
12
+
13
+ def today():
14
+ return dt.date.today()
15
+
16
+
17
+ def ytd(t=None):
18
+ """(start, end) for year-to-date through `t`."""
19
+ t = t or today()
20
+ return _d(dt.date(t.year, 1, 1)), _d(t)
21
+
22
+
23
+ def ytd_last_year(t=None):
24
+ """Same calendar window one year earlier (Jan 1 LY β†’ same month/day LY).
25
+ Clamps Feb 29 β†’ Feb 28 in non-leap years."""
26
+ t = t or today()
27
+ try:
28
+ end = dt.date(t.year - 1, t.month, t.day)
29
+ except ValueError:
30
+ end = dt.date(t.year - 1, t.month, t.day - 1)
31
+ return _d(dt.date(t.year - 1, 1, 1)), end if isinstance(end, str) else _d(end)
32
+
33
+
34
+ def ltm(t=None):
35
+ """Trailing 12 months ending `t`."""
36
+ t = t or today()
37
+ return _d(t - dt.timedelta(days=365)), _d(t)
38
+
39
+
40
+ def prior_ltm(t=None):
41
+ t = t or today()
42
+ return _d(t - dt.timedelta(days=730)), _d(t - dt.timedelta(days=365))
43
+
44
+
45
+ def mtd(t=None):
46
+ t = t or today()
47
+ return _d(dt.date(t.year, t.month, 1)), _d(t)
48
+
49
+
50
+ def wtd(t=None):
51
+ """(Monday of this ISO week, t) β€” week-to-date."""
52
+ t = t or today()
53
+ return _d(t - dt.timedelta(days=t.weekday())), _d(t)
54
+
55
+
56
+ def qtd(t=None):
57
+ """(first day of this calendar quarter, t) β€” quarter-to-date."""
58
+ t = t or today()
59
+ q_start_month = ((t.month - 1) // 3) * 3 + 1
60
+ return _d(dt.date(t.year, q_start_month, 1)), _d(t)
61
+
62
+
63
+ def quarter_range(t=None):
64
+ """(first day, last day) of the calendar quarter containing t."""
65
+ t = t or today()
66
+ qs = ((t.month - 1) // 3) * 3 + 1
67
+ first = dt.date(t.year, qs, 1)
68
+ nxt = dt.date(t.year + (1 if qs + 3 > 12 else 0), ((qs + 2) % 12) + 1, 1)
69
+ return _d(first), _d(nxt - dt.timedelta(days=1))
70
+
71
+
72
+ def shift_year(date_from, date_to, weeks=False):
73
+ """Same window one year earlier. For week-aligned comparisons (weeks=True) shift by exactly 52
74
+ weeks (364 days) so the weekday lines up; otherwise shift by calendar year (clamping Feb 29)."""
75
+ f = dt.date.fromisoformat(date_from)
76
+ t = dt.date.fromisoformat(date_to)
77
+ if weeks:
78
+ return _d(f - dt.timedelta(days=364)), _d(t - dt.timedelta(days=364))
79
+
80
+ def back(d):
81
+ try:
82
+ return dt.date(d.year - 1, d.month, d.day)
83
+ except ValueError:
84
+ return dt.date(d.year - 1, d.month, d.day - 1)
85
+ return _d(back(f)), _d(back(t))
86
+
87
+
88
+ def last_n_days(n, t=None):
89
+ t = t or today()
90
+ return _d(t - dt.timedelta(days=n)), _d(t)
91
+
92
+
93
+ def year_range(year):
94
+ return f'{year}-01-01', f'{year}-12-31'
95
+
96
+
97
+ def month_range(year, month):
98
+ """(first day, last day) of a given calendar month, ISO strings."""
99
+ first = dt.date(year, month, 1)
100
+ nxt = dt.date(year + (1 if month == 12 else 0), 1 if month == 12 else month + 1, 1)
101
+ return _d(first), _d(nxt - dt.timedelta(days=1))
102
+
103
+
104
+ def period_options(t=None, n_months=15):
105
+ """Ordered {label: (date_from, date_to)} for the close/reconciliation period picker."""
106
+ t = t or today()
107
+ opts = {}
108
+ opts['This month (to date)'] = (_d(dt.date(t.year, t.month, 1)), _d(t))
109
+ lm_year, lm_month = (t.year, t.month - 1) if t.month > 1 else (t.year - 1, 12)
110
+ lf, lt = month_range(lm_year, lm_month)
111
+ opts[f'Last month ({lf[:7]})'] = (lf, lt)
112
+ opts['Quarter to date'] = (_d(dt.date(t.year, ((t.month - 1) // 3) * 3 + 1, 1)), _d(t))
113
+ opts['Year to date'] = (_d(dt.date(t.year, 1, 1)), _d(t))
114
+ for label, mf, mt in reversed(month_starts(n_months, t)):
115
+ if label == f'{t.year}-{t.month:02d}' or label == lf[:7]:
116
+ continue
117
+ opts[label] = (mf, mt)
118
+ return opts
119
+
120
+
121
+ def month_starts(n_back=13, t=None):
122
+ """List of (yyyy-mm, start, end) for the last n_back months including current."""
123
+ t = t or today()
124
+ out = []
125
+ y, m = t.year, t.month
126
+ for _ in range(n_back):
127
+ start = dt.date(y, m, 1)
128
+ end = (dt.date(y + (m // 12), (m % 12) + 1, 1) - dt.timedelta(days=1))
129
+ out.append((f'{y}-{m:02d}', _d(start), _d(end)))
130
+ m -= 1
131
+ if m == 0:
132
+ m = 12
133
+ y -= 1
134
+ return list(reversed(out))
135
+
136
+
137
+ def week_starts(n_back=12, t=None):
138
+ """List of (label, start, end) for the last n_back ISO weeks (Mon–Sun) including the current
139
+ (partial) week. label = the Monday as 'MM-DD' (compact, chart-friendly); end is clamped to t
140
+ for the current week so weekly totals never include future days."""
141
+ t = t or today()
142
+ this_mon = t - dt.timedelta(days=t.weekday())
143
+ out = []
144
+ for i in range(n_back):
145
+ mon = this_mon - dt.timedelta(weeks=i)
146
+ sun = mon + dt.timedelta(days=6)
147
+ end = min(sun, t)
148
+ out.append((f'{mon.month:02d}-{mon.day:02d}', _d(mon), _d(end)))
149
+ return list(reversed(out))
150
+
151
+
152
+ def yoy_pct(this_v, last_v):
153
+ if last_v:
154
+ return (this_v - last_v) / last_v * 100.0
155
+ return None
royalimports_os/core/perms.py ADDED
@@ -0,0 +1,164 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """core/perms.py β€” the permission + scope predicates as PURE functions of a user record.
2
+
3
+ EXIT-3b (2026-07-30). `ui/session.py` holds the same predicates keyed off `st.session_state`;
4
+ these take the user record explicitly, so the API can answer "what may this session see?" without
5
+ a Streamlit session. ONE implementation of each rule, two callers β€” and `verify_api.py` asserts
6
+ lock-step against `ui/session.py`'s source (the [[date-window-vocabulary]] discipline: a mirrored
7
+ vocabulary that is not gated has already drifted).
8
+
9
+ ⚠ THE TRAP THIS FILE EXISTS TO AVOID. `ui/session.brand_team()` reads the sidebar's BU selector
10
+ and defaults to `'All'` β†’ `team_id=None` β†’ CONSOLIDATED. Mirroring it literally on the API would
11
+ be a fail-OPEN: an API session has no selector, so a Royal-only user would be handed Fisch rows β€”
12
+ the exact leak strict isolation exists to prevent. The scope must be derived from the USER RECORD
13
+ (`bus`), never from a UI default. `scope_team_id()` below is that derivation.
14
+
15
+ Fail-closed everywhere: an unknown module key is denied, an unreadable `bus` narrows rather than
16
+ widens, and a user with no grant sees nothing rather than everything.
17
+ """
18
+ import core.context as ctxlib
19
+ import core.registry as registry
20
+ import core.users as users
21
+
22
+ # Old module keys migrate ON READ β€” grants, Library prefs and ?page= deep links keep working
23
+ # without ever editing a stored user record. MUST stay identical to ui/session._LEGACY_KEYS
24
+ # (gated in verify_api.py): 'myday' shipped before the Customer List rename, 'customer_list'
25
+ # before the 2026-07-26 rename to 'customer_data', 'map' joined in wave 8 when the Map PAGE
26
+ # became a VIEW of the customer table. A chain is fine only if every hop resolves to the CURRENT
27
+ # key, so 'myday' points at the END of the chain, not at the middle.
28
+ # ⚠ 'customers' is deliberately NOT here. It is a real registry key β€” the parent of
29
+ # customer_data + cohort β€” and may_open() gives a parent grant to its children. Mapping it away
30
+ # would silently narrow every "customers" grant to one child.
31
+ _LEGACY_KEYS = {'myday': 'customer_data', 'customer_list': 'customer_data',
32
+ 'map': 'customer_data', 'library': 'settings'}
33
+
34
+
35
+ def is_admin(user):
36
+ return (user or {}).get('role') == 'admin'
37
+
38
+
39
+ def allowed_modules(user):
40
+ """Registry keys this user may open β€” None means ALL (admins / unrestricted users)."""
41
+ m = (user or {}).get('modules', 'all')
42
+ if not m or m == 'all':
43
+ return None
44
+ return {_LEGACY_KEYS.get(k, k) for k in m}
45
+
46
+
47
+ def may_open(user, key):
48
+ """True iff `user` may OPEN module `key`. A PARENT grant covers its sub-modules (granting
49
+ 'customers' grants 'customer_data'). Fail-closed β€” an unknown key stays denied."""
50
+ am = allowed_modules(user)
51
+ if am is None or key in am:
52
+ return True
53
+ parent = registry.BY_KEY.get(key, {}).get('parent')
54
+ return parent in am if parent else False
55
+
56
+
57
+ # ------------------------------------------------------------------ BU (business unit) scope
58
+ def allowed_bu_labels(user):
59
+ """The BU labels this user may see β€” `core.users.allowed_bus_labels`, which is already a
60
+ pure function of the record. Named here so callers have one import for the whole rule set."""
61
+ return users.allowed_bus_labels(user)
62
+
63
+
64
+ def allowed_team_ids(user):
65
+ """The concrete Odoo team_ids this user may see, as a frozenset. `All` expands to every BU."""
66
+ labels = allowed_bu_labels(user)
67
+ if 'All' in labels:
68
+ return frozenset(t for t in ctxlib.BRANDS.values() if t is not None)
69
+ return frozenset(ctxlib.BRANDS[l] for l in labels
70
+ if l in ctxlib.BRANDS and ctxlib.BRANDS[l] is not None)
71
+
72
+
73
+ def scope_team_id(user):
74
+ """The `team_id` an API request must query with β€” the ONE number that enforces BU isolation.
75
+
76
+ None means CONSOLIDATED, and it is returned ONLY when the user may genuinely see every BU
77
+ (`bus='all'`, or a multi-BU grant, both of which put 'All' in the label list). A user
78
+ permitted exactly one BU gets that BU's id PINNED, never None.
79
+
80
+ ⚠ This is the opposite default from `ui/session.brand_team()`, which falls back to 'All'
81
+ because a UI selector has to start somewhere. Here there is no selector to read, so the
82
+ fallback would BE the leak. A single-BU user's rows are pinned at the query, not filtered
83
+ afterwards, so there is no window in which the other BU's rows exist in the response.
84
+ """
85
+ labels = allowed_bu_labels(user)
86
+ if 'All' in labels:
87
+ return None
88
+ ids = sorted(allowed_team_ids(user))
89
+ return ids[0] if len(ids) == 1 else None
90
+
91
+
92
+ def scope_agent(user):
93
+ """The sales agent whose OWN BOOK this session is confined to, or None for the whole book.
94
+
95
+ ⚠ Mirrors `page_customer_data` (app.py), which reads `u.get('agent')` with NO admin
96
+ exemption β€” deliberately different from `page_agent`'s `None if is_admin() else …`. The
97
+ customer TABLE is the surface `/api/v1/customers` serves, so its rule is the one that
98
+ travels: an account linked to an agent sees that agent's book, admin or not. Getting this
99
+ wrong in the permissive direction would hand a restricted sales-agent login the whole book.
100
+ """
101
+ return (user or {}).get('agent') or None
102
+
103
+
104
+ # ------------------------------------------------------------------ nav projection
105
+ def nav_pages(user):
106
+ """The nav this user may see: registry order, `may_open`-filtered, archived EXCLUDED.
107
+
108
+ Shape (X2's `{key,label,source?,parent?}` plus ONE additive field, `chrome`):
109
+ [{key, label, source?, chrome: 'main'|'utility'}, …]
110
+
111
+ Four rules, each of which has bitten before:
112
+
113
+ * ARCHIVED IS INVISIBLE TO EVERYONE ([[module-archive-semantics]]) β€” not "hidden unless
114
+ admin". 18 of the 28 registry rows are archived and none may appear here.
115
+
116
+ * `group_only` ROWS ARE EXCLUDED ENTIRELY, and therefore **no `parent` is ever emitted**.
117
+ Wave-9 I8 flipped the host nav to a FLAT list: children became top-level and the folder
118
+ key was dropped, because a `group_only` key names a FAMILY and has no `PAGE_FUNCS` entry
119
+ β€” rendering a row for it is a white screen one click away (app.py:8182-8187 says exactly
120
+ this). `customers` is the only such row. Emitting `parent: 'customers'` on its children
121
+ while the parent itself is filtered out would hand the client a DANGLING reference to a
122
+ row that is not in the payload, which is worse than a flat list β€” so the flat list is
123
+ what ships, matching the surface a user actually sees today.
124
+
125
+ * `nav: False` IS NOT `archived`. `settings`, `analyst` and `dictionary` are LIVE surfaces
126
+ the host renders outside the module list (Analyst is the unrestricted landing page,
127
+ Settings sits in the chrome). Dropping them would leave the shell unable to reach two of
128
+ the nine live surfaces, so they ship flagged `chrome: 'utility'` and the client places
129
+ them where the host does. `chrome: 'main'` is the module list.
130
+
131
+ * NOT filtered by `ui/session.user_enabled_modules()`. That is a PREFERENCE read from
132
+ `royalimports_os/data/store/prefs.json` β€” a local file on one box. Consuming it here
133
+ would put per-user FILESYSTEM state inside a process EXIT-4 requires to be stateless,
134
+ and the same user's nav would differ per container. It is a Library setting, not a
135
+ permission; the permission gate is `may_open`, and that is what this applies.
136
+ """
137
+ out = []
138
+ for m in registry.REGISTRY:
139
+ if m.get('archived') or m.get('group_only'):
140
+ continue
141
+ if not may_open(user, m['key']):
142
+ continue
143
+ row = {'key': m['key'], 'label': m['label'],
144
+ 'chrome': 'main' if m.get('nav', True) else 'utility'}
145
+ if m.get('source'):
146
+ row['source'] = m['source']
147
+ out.append(row)
148
+ return out
149
+
150
+
151
+ def landing_page(user):
152
+ """Where a restricted user lands: the first MAIN-chrome page in their grant. None for
153
+ unrestricted users (the client lands on the Analyst, as the host does).
154
+
155
+ Mirrors `ui/session.first_allowed_module` minus its archived last-resort leg β€” that leg
156
+ exists so a misconfigured Streamlit login is not bricked; an API caller gets an explicit
157
+ answer instead, and serving an archived key would contradict the rule above.
158
+ """
159
+ if allowed_modules(user) is None:
160
+ return None
161
+ for row in nav_pages(user):
162
+ if row['chrome'] == 'main':
163
+ return row['key']
164
+ return None
royalimports_os/core/registry.py ADDED
@@ -0,0 +1,137 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Module registry β€” the single source of truth for the dashboards.
2
+
3
+ `app.py` builds the sidebar nav from this, and decides whether the global Brand (DBA) filter
4
+ applies to a page. `validate.py` runs each module's validate() from this. Adding a module =
5
+ add one row here + a page function in app.py + a module in modules/.
6
+
7
+ Fields:
8
+ key stable id (matches the modules/<key>.py file and the app page_<key> function)
9
+ label sidebar/title text (no emojis)
10
+ brand True -> DBA-filterable (honours the Fisch/Royal selector)
11
+ False -> consolidated/company-level (ignores the brand selector)
12
+ hq True -> HQ / company-level (cash flow, balance sheet, AR/AP, consolidated)
13
+ validate True -> module exposes validate() and is included in validate.py
14
+ parent set -> this row is a SUB-MODULE of <parent key>: it leaves the Workflows dropdown and
15
+ renders as a sub-nav button when its parent (or a sibling) is the active module; a
16
+ parent grant covers its sub-modules (owner nav redesign 2026-07-23)
17
+ archived True -> RETIRED from the UI for EVERYONE (owner 2026-07-27, supersedes the
18
+ 2026-07-23 per-user-restore semantics): no nav entry, no Settings>Modules checkbox,
19
+ prewarm and validate.py skip it. Code and data layer stay in the repo but are NOT
20
+ maintained β€” spend no effort on archived modules until the owner un-archives.
21
+ 2026-07-23 pass: backorders, pricecomp, o2c, bookings, spend, expenses.
22
+ 2026-07-27 pass (keep = Sales + Customers family + Agents + Procurement +
23
+ Collections + Analyst/plumbing): products, assortment, financial, management,
24
+ pricing, inventory, vendor, warehouse, returns, outreach, complexity, health.
25
+ note one-line scope note shown in the UI / docs
26
+ """
27
+ # WAVE-9 I8 β€” `'source'`: which connected system a module's data comes from.
28
+ #
29
+ # The owner's model: every nav entry is "a piece of database from a source", so the nav shows the
30
+ # source beside the name ("Sales Β· Odoo"). It is a REGISTRY FACT rather than a string in the nav
31
+ # code precisely so the nav needs no edit when a second connector lands β€” a user-created blank
32
+ # table simply carries a different source, and a module with no `source` (Metric Dictionary,
33
+ # Settings, the Analyst) renders no badge because it is not a database at all.
34
+ # See [[connector-onboarding]]: the plug-in seam is datastore.py, and Odoo has no delegated
35
+ # OAuth, so the "+ New" connect flow is the credential-form shape, not an OAuth redirect.
36
+ REGISTRY = [
37
+ {'key': 'sales', 'label': 'Sales', 'brand': True, 'hq': False, 'validate': True, 'source': 'Odoo',
38
+ 'note': 'Revenue, YoY, seasonality, reps, customers, SKUs β€” by BU.'},
39
+ # ARCHIVED AS A PAGE (owner item 14, wave 8): "archive the current form of the Customer
40
+ # dashboard completely, i want to redesign it, exactly with the backend we have". So the
41
+ # PAGE goes and the DATA LAYER stays β€” modules/customers.py still powers the Customer table's
42
+ # metrics and still validates against Odoo, which is what the redesign will be built on.
43
+ #
44
+ # ⚠ NOT 'archived': True. Archived means invisible to everyone, and this row is the parent of
45
+ # customer_data + cohort β€” archiving it would take the whole family out of the nav and strand
46
+ # its children. 'group_only' says exactly what is true: this key names a FOLDER, never a
47
+ # destination. It has no PAGE_FUNCS entry, the nav never renders it as a leaf, and a stale
48
+ # deep link to it redirects to the family's first visible member.
49
+ {'key': 'customers', 'label': 'Customers', 'brand': True, 'hq': False, 'validate': True,
50
+ 'group_only': True,
51
+ 'note': 'Folder for the customer surfaces (Customer table, Cohort). The old Customers '
52
+ 'dashboard was retired for redesign (owner item 14, 2026-07-28); its metric layer '
53
+ '(modules/customers.py) still backs the table and still validates against Odoo.'},
54
+ # LABEL renamed 'Customer List' -> 'Data' (owner 2026-07-25) -> 'Customer' (owner item 5,
55
+ # 2026-07-27); KEY renamed 'customer_list' -> 'customer_data' (owner 2026-07-26). The key
56
+ # rename is safe ONLY because _LEGACY_KEYS (ui/session.py) migrates grants, Library prefs
57
+ # and ?page= deep links ON READ β€” stored user records are never edited, exactly as
58
+ # myday->customer_list was done. The STORE keys are deliberately NOT renamed:
59
+ # 'customer_table_workspace' and 'customer_lists' hold real user data, and renaming them
60
+ # would orphan every saved view. Label-only changes need none of that machinery.
61
+ {'key': 'customer_data', 'label': 'Customer', 'brand': True, 'hq': False, 'validate': False, 'source': 'Odoo', 'parent': 'customers',
62
+ 'note': 'Build, filter and save customer lists: the whole scoped book with per-customer '
63
+ 'metrics; saved lists are visible, editable formulas (Call list and Win-back ship '
64
+ 'as templates) plus hand-picked members. Sub-module of Customers (owner nav redesign 2026-07-23; replaced My Day).'},
65
+ # Cohort (owner item 6, 2026-07-26): the SAME table as Data over a FIXED set. The
66
+ # difference is membership semantics, not layout β€” a saved view re-populates as the
67
+ # data moves, a cohort only changes when a person edits it. validate:False because
68
+ # there is no Odoo aggregate to reconcile a hand-picked list against; its metrics are
69
+ # the customers module's, already validated there.
70
+ {'key': 'cohort', 'label': 'Cohort', 'brand': True, 'hq': False, 'validate': False, 'source': 'Odoo', 'parent': 'customers',
71
+ 'note': 'Hand-curated, unchanging customer lists. Same grid as Data (filter, sort, '
72
+ 'group, custom columns) but the SET is fixed: it changes only when someone '
73
+ 'adds or removes a member. Populated from a view via ... > Add to list.'},
74
+ {'key': 'agent', 'label': 'Agents', 'brand': True, 'hq': False, 'validate': True, 'source': 'Odoo',
75
+ 'note': 'Per-agent book: sales (custom dates), returns, top SKUs, full customer list incl. inactive. Returns consolidated.'},
76
+ {'key': 'products', 'label': 'SKU', 'brand': True, 'hq': False, 'validate': True, 'archived': True,
77
+ 'note': 'SKU movers, zombies, velocity, coverage, drawers β€” by BU (sales-derived).'},
78
+ {'key': 'assortment', 'label': 'Assortment', 'brand': True, 'hq': False, 'validate': True, 'archived': True,
79
+ 'note': 'Facet-level performance (category/color/occasion/collection/season) + season readiness β€” by BU.'},
80
+ {'key': 'financial', 'label': 'Financial', 'brand': True, 'hq': False, 'validate': True, 'archived': True,
81
+ 'note': 'Gross margin by BU/category/SKU. Cash-conversion cycle is HQ-consolidated.'},
82
+ {'key': 'pricing', 'label': 'Pricing', 'brand': True, 'hq': False, 'validate': True, 'archived': True,
83
+ 'note': 'Per-SKU margin/markup/LTM sales + allocated net P&L (channel-rate cost-to-SKU) for pricing decisions.'},
84
+ {'key': 'management', 'label': 'Management P&L', 'brand': False, 'hq': True, 'validate': True, 'archived': True,
85
+ 'nav': False, # rendered INSIDE the Financial module (view toggle) β€” archived WITH financial
86
+ 'note': 'Operating-model management income statements (FFS/RI/GD/BU-M) reproduced from raw Odoo, Sep-2024+.'},
87
+ {'key': 'inventory', 'label': 'Inventory', 'brand': False, 'hq': False, 'validate': True, 'archived': True,
88
+ 'note': 'On-hand stock is physically consolidated (shared warehouse); not DBA-split.'},
89
+ {'key': 'procurement', 'label': 'Procurement', 'brand': False, 'hq': False, 'validate': True, 'source': 'Odoo',
90
+ 'note': 'Biweekly buy list: lead-time-cover trigger, 8-month demand baseline + YTD trend, by supplier.'},
91
+ {'key': 'vendor', 'label': 'Vendors', 'brand': False, 'hq': False, 'validate': False, 'archived': True,
92
+ 'note': 'Vendor master data + the SKUs each supplies; multiple-vendor / cheapest-price view.'},
93
+ {'key': 'warehouse', 'label': 'Warehouse', 'brand': False, 'hq': False, 'validate': True, 'archived': True,
94
+ 'note': 'Movement & efficiency: on-time ship, Pick-Pack-Ship cycle times, throughput, backlog age, shrinkage. Physical ops (consolidated).'},
95
+ {'key': 'ar', 'label': 'Collections', 'brand': False, 'hq': True, 'validate': True,
96
+ 'source': 'Odoo',
97
+ 'note': 'Receivables are company-level (consolidated).'},
98
+ {'key': 'returns', 'label': 'Returns', 'brand': False, 'hq': True, 'validate': True, 'archived': True,
99
+ 'note': 'Credit-note lens: refund concentration by SKU (quality) and customer (behavior). Company-level.'},
100
+ {'key': 'outreach', 'label': 'Outreach', 'brand': True, 'hq': False, 'validate': False, 'archived': True,
101
+ 'note': 'Campaign emails to customer segments: templates, suppression, send log, revenue attribution. Sending is admin-gated behind SAFE_MODE.'},
102
+ {'key': 'backorders', 'label': 'Backorders', 'brand': True, 'hq': False, 'validate': True, 'archived': True,
103
+ 'note': 'Confirmed-undelivered order lines aged vs promise date, valued, with a supply-aware next action per row (ship / expedite / call). Wholesale scope.'},
104
+ {'key': 'pricecomp', 'label': 'Price Compliance', 'brand': True, 'hq': False, 'validate': True, 'archived': True,
105
+ 'note': 'Selling below the customer pricelist tier (LTM, per customer x SKU): the pocket-price floor worklist with annualized leak $. Sub-30% ratios usually mean a stale or pack-basis RULE β€” fix the rule, not the rep.'},
106
+ {'key': 'o2c', 'label': 'Cash Timing', 'brand': True, 'hq': False, 'validate': True, 'archived': True,
107
+ 'note': 'Order-to-cash stage decomposition (order-to-ship / ship-to-invoice / invoice-to-paid, each with its owner) + the terms-gap rollup: contractual vs actual days per payment term with the free-credit $ it strands.'},
108
+ {'key': 'bookings', 'label': 'Order Book', 'brand': True, 'hq': False, 'validate': True, 'archived': True,
109
+ 'note': 'Pre-season booking coverage: cumulative booked $ for Aug-Dec delivery vs last year same-week; category TY-vs-LY-as-of-date. The container-program decision, months ahead.'},
110
+ {'key': 'spend', 'label': 'Spend & Payables', 'brand': False, 'hq': True, 'validate': True, 'archived': True,
111
+ 'note': 'Vendor-bill side: payment-terms capture (early-pay cash surrendered), duplicate-bill review, the opex spend cube (fragmentation, consolidate+rebid) and freight recovery. Company-level.'},
112
+ {'key': 'complexity', 'label': 'SKU Complexity', 'brand': False, 'hq': False, 'validate': True, 'archived': True,
113
+ 'note': 'BCG tail rationalization: every SKU re-costed by ACTIVITY (lines, picks, returns) + 25%/yr carrying; KILL candidates rest on hard math (GM minus carrying), REVIEW on the pooled activity estimate. All-channel.'},
114
+ {'key': 'expenses', 'label': 'Expenses', 'brand': False, 'hq': True, 'validate': True, 'archived': True,
115
+ 'note': 'Operating expense from the GL (all expense-type accounts; COGS excluded): trend, operating leverage (opex % of revenue), the YoY cost bridge, XmR control-limit spike watch list, fixed/variable split and a drill-to-ledger category directory. Company-level.'},
116
+ {'key': 'health', 'label': 'Data Health', 'brand': False, 'hq': True, 'validate': True, 'archived': True,
117
+ 'note': 'Close/reconciliation scan; period-filtered, mostly company-level. ARCHITECTURE '
118
+ '(owner 2026-07-23): every discrepancy / potential-error marker lives here β€” '
119
+ 'procurement mapping gaps, count-trust (unverified on-hand counts), untracked-on-'
120
+ 'order β€” so operational workflows stay clean for doing the work.'},
121
+ {'key': 'dictionary', 'label': 'Metric Dictionary', 'brand': False, 'hq': True, 'validate': True,
122
+ 'nav': False, # sidebar sentinel button (like Feedback) β€” not a Dashboards/Workflows entry
123
+ 'note': 'PAGE RETIRED 2026-07-23 (owner: broken/not customer-facing) β€” row kept ONLY so validate.py keeps running the semantic-layer contracts (the Analyst grounding proof). No nav, no page.'},
124
+ {'key': 'settings', 'label': 'Settings', 'brand': False, 'hq': True, 'validate': False,
125
+ 'nav': False, # sidebar sentinel (account group) β€” owner IA 2026-07-12
126
+ 'note': 'User scope settings: the Business Unit toggle (strict isolation) and the Data basis '
127
+ 'toggle (Orders vs Invoiced) moved here from the sidebar.'},
128
+ {'key': 'analyst', 'label': 'AIOS Analyst', 'brand': False, 'hq': True, 'validate': False,
129
+ 'nav': False, # sidebar sentinel button; eval gate = harness/evals.py (run pre-ship, NOT in validate.py β€” live LLM cost)
130
+ 'note': 'Ask the business a question in plain language: a small AI model calls governed tools over the semantic layer β€” answers carry their query trace and drill links. AI-generated output (Art. 50 labeled).'},
131
+ ]
132
+
133
+ BY_KEY = {m['key']: m for m in REGISTRY}
134
+
135
+
136
+ def brand_filterable(key) -> bool:
137
+ return bool(BY_KEY.get(key, {}).get('brand'))
royalimports_os/core/store.py ADDED
@@ -0,0 +1,307 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Tiny JSON persistence on a PRIVATE Hugging Face Dataset repo β€” the platform's ONLY writable store
2
+ (Odoo stays strictly read-only). Holds the user registry, customer notes and per-user tags.
3
+
4
+ Auth uses HF_TOKEN (a Space secret / .env value with write access to the royal-imports org). The
5
+ repo is created private on first write. Reads use a short in-process cache that writes refresh, so
6
+ within a running instance reads are consistent with writes; a fresh instance re-fetches the latest.
7
+ """
8
+ import io
9
+ import os
10
+ import json
11
+ import threading
12
+ import time as _time
13
+ from pathlib import Path
14
+
15
+ try:
16
+ from dotenv import load_dotenv
17
+ load_dotenv(Path(__file__).resolve().parents[1] / '.env')
18
+ except Exception:
19
+ pass
20
+
21
+ from huggingface_hub import HfApi, hf_hub_download
22
+ from huggingface_hub.utils import EntryNotFoundError, RepositoryNotFoundError
23
+
24
+ REPO = os.environ.get('OS_DATA_REPO', 'royal-imports/cfo-os-data')
25
+ _LOCK = threading.RLock()
26
+ _CACHE = {}
27
+ # --- async flush state (wave-7 W3, 2026-07-28) --------------------------------------------
28
+ # One typed character in a Notes cell used to cost THREE synchronous network calls inside the
29
+ # component round-trip (strict re-download + create_repo + full re-upload = one hub commit per
30
+ # edit β€” measured seconds each, and a commit-budget hazard). update(..., flush='async') now
31
+ # applies the mutation to the in-process cache synchronously (read-your-writes: every reader
32
+ # already consults _CACHE first) and a background worker coalesces uploads. The restart-wipe
33
+ # protection is UNCHANGED where it matters: the FIRST update of a key in a fresh process still
34
+ # strict-reads (a transient read error aborts the write, never merges into {}); after that the
35
+ # cache we last wrote IS the latest known state, so re-downloading it per keystroke bought
36
+ # nothing but latency. Known, accepted: with a second concurrent writer (local QA against the
37
+ # shared store) the stale window widens from ~one round-trip to ~the flush gap.
38
+ _OWNED = {} # name -> True once _CACHE[name] reflects our own last accepted write
39
+ _DIRTY = {} # name -> cache is ahead of the hub
40
+ _FLUSHERS = {} # name -> live worker thread
41
+ _REPO_OK = [False] # create_repo(exist_ok=True) confirmed once per process
42
+ _FLUSH_DELAY = 2.0 # coalesce a burst of edits into one commit
43
+ _FLUSH_MIN_GAP = 20.0 # floor between commits of one key (the 256-commits/hr budget)
44
+ _LAST_FLUSH = {} # name -> monotonic time of the last completed upload
45
+
46
+
47
+ def _token():
48
+ return os.environ.get('HF_TOKEN') or None
49
+
50
+
51
+ def available():
52
+ """True when a token is configured β€” callers degrade gracefully (read-only UI) without one."""
53
+ return bool(_token())
54
+
55
+
56
+ def _download(name):
57
+ """Parse {name}.json into a dict. Returns {} ONLY when the file is genuinely absent. Any
58
+ other failure (network, auth, rate-limit, corrupt/partial JSON) RAISES β€” a transient read
59
+ error must never be mistaken for an empty store, or a caller could overwrite real data with
60
+ an empty dict (this is what wiped the user registry on restart)."""
61
+ try:
62
+ path = hf_hub_download(REPO, f'{name}.json', repo_type='dataset', token=_token(),
63
+ force_download=True)
64
+ with open(path, encoding='utf-8') as f:
65
+ return json.load(f)
66
+ except (EntryNotFoundError, RepositoryNotFoundError, FileNotFoundError):
67
+ return {}
68
+
69
+
70
+ def get(name, fresh=False):
71
+ """Load {name}.json as a dict. LENIENT (for display reads): on a transient fetch error, hand
72
+ back the last known-good cached value if we have one, else {}. Callers that MUTATE the store
73
+ must go through update()/_read_strict, which RAISE on such errors so they never clobber real
74
+ data with a phantom-empty read."""
75
+ with _LOCK:
76
+ if not fresh and name in _CACHE:
77
+ return json.loads(json.dumps(_CACHE[name])) # hand back a copy
78
+ try:
79
+ data = _download(name)
80
+ except Exception as e:
81
+ try: # lazy import β€” core stays layer-clean
82
+ import harness.telemetry as _tel # noqa: PLC0415
83
+ _tel.error(f'store:get:{name}', e,
84
+ fallback='cached' if name in _CACHE else 'empty')
85
+ except Exception:
86
+ pass
87
+ if name in _CACHE:
88
+ return json.loads(json.dumps(_CACHE[name]))
89
+ return {}
90
+ _CACHE[name] = data
91
+ return json.loads(json.dumps(data))
92
+
93
+
94
+ def _read_strict(name):
95
+ """Like get(fresh=True) but RAISES on any transient read error ({} only for a genuinely
96
+ absent file). Read-modify-write callers use this so a failed read ABORTS the write instead
97
+ of merging changes into an empty dict and overwriting the store."""
98
+ with _LOCK:
99
+ data = _download(name)
100
+ _CACHE[name] = data
101
+ return json.loads(json.dumps(data))
102
+
103
+
104
+ def exists(name):
105
+ """True if {name}.json is confirmed present, False if confirmed absent, and True on ANY
106
+ uncertainty β€” so a caller that would seed or overwrite the store never clobbers a registry
107
+ we merely failed to reach."""
108
+ try:
109
+ files = HfApi(token=_token()).list_repo_files(REPO, repo_type='dataset')
110
+ return f'{name}.json' in files
111
+ except RepositoryNotFoundError:
112
+ return False
113
+ except Exception:
114
+ return True
115
+
116
+
117
+ def _upload(name, data):
118
+ """The raw hub write. create_repo is confirmed ONCE per process β€” it was a full extra API
119
+ round-trip on every put(), pure latency after the first call ever succeeded."""
120
+ api = HfApi(token=_token())
121
+ if not _REPO_OK[0]:
122
+ api.create_repo(REPO, repo_type='dataset', private=True, exist_ok=True)
123
+ _REPO_OK[0] = True
124
+ buf = io.BytesIO(json.dumps(data, ensure_ascii=False, indent=2).encode('utf-8'))
125
+ api.upload_file(path_or_fileobj=buf, path_in_repo=f'{name}.json', repo_id=REPO,
126
+ repo_type='dataset', commit_message=f'update {name}')
127
+
128
+
129
+ def upload_bytes(path_in_repo, data, message=None):
130
+ """Persist RAW BYTES at an arbitrary path in the tenant dataset (wave-8 C5, documents).
131
+
132
+ The rest of this module is JSON-per-key; a customer's attachment is neither JSON nor
133
+ per-key, so it gets its own path rather than being base64'd into somebody's blob. That
134
+ matters for a reason beyond tidiness: the workspace blob is READ ON EVERY PAGE RENDER, and
135
+ a few megabytes of base64 inside it would be paid on every keystroke by every user of that
136
+ table, not just by whoever opened the document.
137
+
138
+ Bytes are never cached in-process: they are large, cold, and fetched one at a time on an
139
+ explicit user action.
140
+ """
141
+ if not available():
142
+ raise RuntimeError('No HF_TOKEN configured β€” persistence is unavailable.')
143
+ with _LOCK:
144
+ api = HfApi(token=_token())
145
+ if not _REPO_OK[0]:
146
+ api.create_repo(REPO, repo_type='dataset', private=True, exist_ok=True)
147
+ _REPO_OK[0] = True
148
+ api.upload_file(path_or_fileobj=io.BytesIO(data), path_in_repo=path_in_repo,
149
+ repo_id=REPO, repo_type='dataset',
150
+ commit_message=message or f'add {path_in_repo}')
151
+
152
+
153
+ def download_bytes(path_in_repo):
154
+ """Raw bytes back, or None when absent/unreachable.
155
+
156
+ None rather than an exception: a missing attachment must degrade to "that file is gone",
157
+ never to a broken page β€” the same contract every other read here follows.
158
+ """
159
+ if not available():
160
+ return None
161
+ try:
162
+ local = hf_hub_download(REPO, path_in_repo, repo_type='dataset', token=_token())
163
+ return Path(local).read_bytes()
164
+ except (EntryNotFoundError, RepositoryNotFoundError):
165
+ return None
166
+ except Exception:
167
+ return None
168
+
169
+
170
+ def delete_path(path_in_repo):
171
+ """Remove a stored file. Absent is SUCCESS β€” deleting what is already gone is the goal."""
172
+ if not available():
173
+ return False
174
+ try:
175
+ HfApi(token=_token()).delete_file(path_in_repo=path_in_repo, repo_id=REPO,
176
+ repo_type='dataset',
177
+ commit_message=f'delete {path_in_repo}')
178
+ return True
179
+ except (EntryNotFoundError, RepositoryNotFoundError):
180
+ return True
181
+ except Exception:
182
+ return False
183
+
184
+
185
+ def put(name, data):
186
+ """Persist {name}.json (creates the private repo on first write). Updates the in-process cache."""
187
+ if not available():
188
+ raise RuntimeError('No HF_TOKEN configured β€” persistence is unavailable.')
189
+ with _LOCK:
190
+ _upload(name, data)
191
+ _CACHE[name] = json.loads(json.dumps(data))
192
+ _OWNED[name] = True
193
+ _LAST_FLUSH[name] = _time.monotonic()
194
+ return _CACHE[name]
195
+
196
+
197
+ def _flush_worker(name):
198
+ """Coalescing uploader for one key. Runs while dirty; retries with backoff on failure β€”
199
+ the cache stays the truth and the dirty mark survives, so nothing is ever lost silently."""
200
+ backoff = 2.0
201
+ while True:
202
+ _time.sleep(_FLUSH_DELAY)
203
+ with _LOCK:
204
+ gap = _time.monotonic() - _LAST_FLUSH.get(name, 0.0)
205
+ if gap < _FLUSH_MIN_GAP:
206
+ wait = _FLUSH_MIN_GAP - gap
207
+ elif not _DIRTY.get(name):
208
+ _FLUSHERS.pop(name, None)
209
+ return
210
+ else:
211
+ _DIRTY[name] = False
212
+ snapshot = json.loads(json.dumps(_CACHE.get(name, {})))
213
+ wait = None
214
+ if wait is not None:
215
+ _time.sleep(wait)
216
+ continue
217
+ try:
218
+ _upload(name, snapshot)
219
+ with _LOCK:
220
+ _LAST_FLUSH[name] = _time.monotonic()
221
+ backoff = 2.0
222
+ except Exception as e:
223
+ with _LOCK:
224
+ _DIRTY[name] = True # keep it queued; cache is still the truth
225
+ try:
226
+ import harness.telemetry as _tel # noqa: PLC0415
227
+ _tel.error(f'store:flush:{name}', e)
228
+ except Exception:
229
+ pass
230
+ _time.sleep(backoff)
231
+ backoff = min(backoff * 2, 60.0)
232
+
233
+
234
+ def _schedule_flush(name):
235
+ """Mark dirty + ensure exactly one live worker for the key. Caller holds _LOCK."""
236
+ _DIRTY[name] = True
237
+ t = _FLUSHERS.get(name)
238
+ if t is not None and t.is_alive():
239
+ return
240
+ t = threading.Thread(target=_flush_worker, args=(name,), daemon=True,
241
+ name=f'store-flush-{name}')
242
+ _FLUSHERS[name] = t
243
+ t.start()
244
+
245
+
246
+ def flush(name=None, timeout=30.0):
247
+ """Block until pending async writes for `name` (or ALL keys) reach the hub β€” QA gates and
248
+ shutdown hooks use this; the app itself never needs to."""
249
+ deadline = _time.monotonic() + timeout
250
+ names = [name] if name else None
251
+ while _time.monotonic() < deadline:
252
+ with _LOCK:
253
+ pending = {k for k, v in _DIRTY.items() if v} | {
254
+ k for k, t in _FLUSHERS.items() if t.is_alive()}
255
+ if names is not None:
256
+ pending &= set(names)
257
+ if not pending:
258
+ return True
259
+ _time.sleep(0.2)
260
+ return False
261
+
262
+
263
+ def _flush_now_at_exit():
264
+ """Process-exit safety net: push any dirty keys synchronously, bypassing the coalescing
265
+ gap β€” a Space restart must not eat the last seconds of the async window."""
266
+ with _LOCK:
267
+ dirty = [k for k, v in _DIRTY.items() if v]
268
+ snaps = {k: json.loads(json.dumps(_CACHE.get(k, {}))) for k in dirty}
269
+ for k in dirty:
270
+ _DIRTY[k] = False
271
+ for k, d in snaps.items():
272
+ try:
273
+ _upload(k, d)
274
+ except Exception:
275
+ pass # exiting anyway β€” nothing left to reschedule
276
+
277
+
278
+ import atexit # noqa: E402 (registered after the helpers it needs)
279
+ atexit.register(_flush_now_at_exit)
280
+
281
+
282
+ def update(name, fn, flush='sync'):
283
+ """Read-modify-write. STRICT first read (raises on a transient error rather than starting
284
+ from an empty dict β€” a network blip during 'add user' can never wipe the accounts), apply
285
+ fn(data)->data, persist.
286
+
287
+ flush='sync' (default) = exactly the historical behavior: fresh strict read + blocking
288
+ upload. flush='async' (the table-workspace hot path, wave-7 W3) = the mutation applies to
289
+ the in-process cache and returns immediately; a coalescing worker commits to the hub. The
290
+ strict read still guards the FIRST async update of a key per process; after we own the
291
+ cache, it IS the latest known state and re-downloading it per edit was pure latency."""
292
+ with _LOCK:
293
+ if flush == 'async' and _OWNED.get(name) and name in _CACHE:
294
+ data = json.loads(json.dumps(_CACHE[name]))
295
+ else:
296
+ data = _read_strict(name)
297
+ result = fn(data)
298
+ data = result if result is not None else data
299
+ if flush == 'async':
300
+ if not available():
301
+ raise RuntimeError('No HF_TOKEN configured β€” persistence is unavailable.')
302
+ _CACHE[name] = json.loads(json.dumps(data))
303
+ _OWNED[name] = True
304
+ _schedule_flush(name)
305
+ return data
306
+ put(name, data)
307
+ return data
royalimports_os/core/store_backend.py ADDED
@@ -0,0 +1,85 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """core/store_backend.py β€” which store is active: `STORE_BACKEND=hf|pg` (X4 / EXIT-2b).
2
+
3
+ import core.store_backend as store # new code
4
+ store.get('users') # goes to whichever backend is selected
5
+
6
+ β›” WHAT THIS DOES **NOT** DO, stated first because assuming otherwise would be a silent data
7
+ split. Flipping `STORE_BACKEND=pg` does NOT redirect the ~40 existing callers that say
8
+ `import core.store as store` β€” they are bound to the HF module directly. Rewiring them is task
9
+ **C-4** (the HF→Postgres cutover with its dual-read window), and C-4 cannot start until the
10
+ database exists (owner blocker **B-3**). Until then this selector is the SEAM, exercised by new
11
+ code and by the gate, and the default is `hf`, which is exactly today's behaviour.
12
+
13
+ Doing it any other way would be worse. The tempting shortcut β€” have `core/store.py` itself
14
+ delegate β€” would mean every existing caller silently changes backend the moment an env var is set
15
+ in some shell, with no dual-read window and no way to compare the two stores' contents first.
16
+ "One store" is the goal; getting there through a flag nobody audited would produce two.
17
+
18
+ THE DEFAULT IS `hf` AND STAYS `hf` until C-4 says otherwise. An unrecognised value is a
19
+ configuration error and RAISES rather than falling back β€” a typo'd backend name that quietly
20
+ served the old store would be discovered by data going missing.
21
+ """
22
+ import os
23
+
24
+ import core.store as _hf
25
+
26
+ _NAMES = ('available', 'get', 'exists', 'put', 'update', 'upload_bytes', 'download_bytes',
27
+ 'delete_path', 'flush')
28
+
29
+
30
+ def name():
31
+ """The selected backend name, validated. `hf` unless explicitly told otherwise."""
32
+ raw = (os.environ.get('STORE_BACKEND') or 'hf').strip().lower()
33
+ if raw not in ('hf', 'pg'):
34
+ raise RuntimeError(
35
+ f"STORE_BACKEND={raw!r} is not a backend. Use 'hf' (the HF Dataset store, the "
36
+ f"default) or 'pg' (Postgres, needs DATABASE_URL). Refusing to guess: a typo that "
37
+ f"silently served the other store is how data ends up in two places.")
38
+ return raw
39
+
40
+
41
+ def active():
42
+ """The backend MODULE. Resolved per call, so a test can flip the env var without a reimport."""
43
+ if name() == 'pg':
44
+ import core.store_pg as _pg
45
+ return _pg
46
+ return _hf
47
+
48
+
49
+ # --- the nine delegates. Written out rather than generated with setattr so that reading this file
50
+ # tells you the whole interface, and so a new store operation cannot be added to core/store.py
51
+ # without something here failing to match (verify_store_pg.py compares the signatures).
52
+ def available():
53
+ return active().available()
54
+
55
+
56
+ def get(name_, fresh=False):
57
+ return active().get(name_, fresh=fresh)
58
+
59
+
60
+ def exists(name_):
61
+ return active().exists(name_)
62
+
63
+
64
+ def put(name_, data):
65
+ return active().put(name_, data)
66
+
67
+
68
+ def update(name_, fn, flush='sync'):
69
+ return active().update(name_, fn, flush=flush)
70
+
71
+
72
+ def upload_bytes(path_in_repo, data, message=None):
73
+ return active().upload_bytes(path_in_repo, data, message=message)
74
+
75
+
76
+ def download_bytes(path_in_repo):
77
+ return active().download_bytes(path_in_repo)
78
+
79
+
80
+ def delete_path(path_in_repo):
81
+ return active().delete_path(path_in_repo)
82
+
83
+
84
+ def flush(name_=None, timeout=30.0):
85
+ return active().flush(name_, timeout=timeout)
royalimports_os/core/store_pg.py ADDED
@@ -0,0 +1,249 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """core/store_pg.py β€” `core/store.py`'s interface, backed by Postgres (X4 / EXIT-2b, 2026-07-30).
2
+
3
+ THE INTERFACE IS THE CONTRACT, and it is `core/store.py`'s nine functions:
4
+
5
+ available() Β· get(name, fresh=False) Β· exists(name) Β· put(name, data)
6
+ update(name, fn, flush='sync') Β· upload_bytes(path, data, message=None)
7
+ download_bytes(path) Β· delete_path(path) Β· flush(name=None, timeout=30.0)
8
+
9
+ Same names, same signatures, same return types, same failure semantics β€” so switching backends is
10
+ an env var (`STORE_BACKEND=hf|pg`) and not a rewrite of every caller. `core/store_backend.py`
11
+ does the selection; nothing above it needs to know which store it is talking to.
12
+
13
+ β›” NOT VERIFIED AGAINST A REAL SERVER β€” owner blocker **B-3** (choose + provision managed
14
+ Postgres, US-region, co-located with Ashburn). `verify_store_pg.py` is written and it SKIPS LOUDLY
15
+ without `DATABASE_URL`; it is not reported as passing, and this module is not the default backend.
16
+ The schema is `harness/pg/schema.sql`. Sequenced around the blocker, never stubbed past it.
17
+
18
+ WHAT POSTGRES FIXES, precisely (C1c β€” these are the reasons, not a preference):
19
+ * **`get()` is cache-first on the HF store**, so a write from another process is invisible to a
20
+ running app until restart. Observed live in wave 11. Here a read is a SELECT: cross-process
21
+ coherence is the default rather than a thing to remember.
22
+ * **`update()` was read-modify-write with no concurrency control** β€” two writers raced and the
23
+ loser vanished silently. Here it runs inside ONE transaction with `SELECT … FOR UPDATE`, so
24
+ the second writer waits and then sees the first writer's value.
25
+ * **A write was an HTTP commit** against a 256-commits/hour repo budget, which is why the hot
26
+ path needed a coalescing async flusher at all. A Postgres write is a write; `flush='async'`
27
+ stays in the signature and becomes a no-op, because there is nothing to coalesce.
28
+
29
+ ⚠ ONE DELIBERATE DIFFERENCE FROM THE HF BACKEND, and it is a fix. `store.get()` swallows transient
30
+ read failures and hands back a cached-or-empty dict, because a display read must not break a page.
31
+ That leniency is exactly what wiped the user registry once (an empty read merged into a write), and
32
+ `_read_strict` exists to opt out of it. Postgres has no equivalent "the network blipped" state
33
+ worth hiding: a connection failure RAISES from `get()` here. Callers that must not break on a read
34
+ should catch it β€” silently returning `{}` from a database that is simply unreachable is how an
35
+ empty dict gets written back over real data.
36
+ """
37
+ import json
38
+ import os
39
+ import threading
40
+
41
+ _URL_ENV = 'DATABASE_URL'
42
+ _LOCK = threading.RLock()
43
+ _POOL = {'pool': None, 'url': None, 'schema': None}
44
+
45
+
46
+ def _url():
47
+ return os.environ.get(_URL_ENV) or None
48
+
49
+
50
+ def _schema_for(tenant_slug=None):
51
+ """`t_<slug>` with `-` normalised to `_` (a hyphen is not legal in an unquoted identifier).
52
+
53
+ Mirrors `control.provision_tenant_schema` exactly β€” the two MUST agree or a write lands in a
54
+ schema the DDL never created. `AIOS_TENANT` names the tenant this process serves; it defaults
55
+ to tenant #0 so the default behaviour is unchanged, the same rule `harness.datastore.path_for`
56
+ follows for the DuckDB filename.
57
+ """
58
+ slug = (tenant_slug or os.environ.get('AIOS_TENANT') or 'royal-imports').strip().lower()
59
+ return 't_' + slug.replace('-', '_')
60
+
61
+
62
+ def _pool():
63
+ """The psycopg connection pool, opened once per process.
64
+
65
+ psycopg is imported LAZILY and only here: `STORE_BACKEND=hf` must not require the dependency
66
+ to be installed at all, which is what keeps this file safe to commit before B-3 unblocks.
67
+ """
68
+ with _LOCK:
69
+ url = _url()
70
+ if not url:
71
+ raise RuntimeError(
72
+ f'{_URL_ENV} is not set β€” the Postgres store backend has nothing to connect to.')
73
+ if _POOL['pool'] is not None and _POOL['url'] == url:
74
+ return _POOL['pool']
75
+ try:
76
+ from psycopg_pool import ConnectionPool
77
+ except ImportError as e: # pragma: no cover - depends on B-3
78
+ raise RuntimeError(
79
+ 'STORE_BACKEND=pg needs psycopg[binary,pool]. It is deliberately NOT in '
80
+ 'requirements.txt until the database exists (owner blocker B-3), so that the '
81
+ 'default hf backend installs nothing it does not use.') from e
82
+ if _POOL['pool'] is not None:
83
+ try:
84
+ _POOL['pool'].close()
85
+ except Exception:
86
+ pass
87
+ # min_size=0 so a process that never touches the store opens no connection at all β€” the
88
+ # shared API serves plenty of requests (health, static, a cached payload) that never read.
89
+ _POOL['pool'] = ConnectionPool(url, min_size=0, max_size=8, open=True, timeout=10.0)
90
+ _POOL['url'] = url
91
+ return _POOL['pool']
92
+
93
+
94
+ def available():
95
+ """True when a connection can actually be MADE, not merely when a URL is configured.
96
+
97
+ `core.store.available()` only checks for a token, which is all it can cheaply do. Here the
98
+ honest answer needs a round-trip, so the result is memoised per process: callers use this to
99
+ decide whether to degrade the UI, and a `SELECT 1` per render would be absurd.
100
+ """
101
+ if not _url():
102
+ return False
103
+ if _POOL.get('ok'):
104
+ return True
105
+ try:
106
+ with _pool().connection() as con:
107
+ con.execute('SELECT 1')
108
+ _POOL['ok'] = True
109
+ return True
110
+ except Exception:
111
+ return False
112
+
113
+
114
+ def _table(kind, tenant_slug=None):
115
+ """`"t_slug"."store_kv"` β€” the qualified table, safely quoted."""
116
+ from psycopg import sql
117
+ return sql.SQL('{}.{}').format(sql.Identifier(_schema_for(tenant_slug)),
118
+ sql.Identifier(kind))
119
+
120
+
121
+ def _q(template, kind, tenant_slug=None):
122
+ """Compose `template` with `{tbl}` = the QUALIFIED table and `{bare}` = the table name alone.
123
+
124
+ ⚠ THE TWO ARE NOT INTERCHANGEABLE, and getting it wrong is a runtime syntax error nothing
125
+ local would catch (no database here β€” owner blocker B-3). Inside `ON CONFLICT … DO UPDATE`,
126
+ Postgres refers to the conflicting row by the table's ALIAS, which defaults to the bare
127
+ table name: `store_kv.rev` is valid, `t_royal_imports.store_kv.rev` is not. So the FROM
128
+ position takes `{tbl}` and the DO-UPDATE position takes `{bare}`.
129
+
130
+ Identifiers go through `psycopg.sql.Identifier`, so the schema name β€” derived from a tenant
131
+ slug β€” cannot be an injection vector even though it is interpolated. The slug is ALSO
132
+ regex-constrained at its source (`control.tenants`), which is the belt to this brace.
133
+ """
134
+ from psycopg import sql
135
+ return sql.SQL(template).format(tbl=_table(kind, tenant_slug),
136
+ bare=sql.Identifier(kind))
137
+
138
+
139
+ def get(name, fresh=False, tenant_slug=None):
140
+ """The value stored under `name`, or `{}` when the key is genuinely absent.
141
+
142
+ `fresh` is accepted and IGNORED: it exists in the HF signature to bypass a process cache, and
143
+ there is no cache here β€” every read is a SELECT. Keeping the parameter means callers do not
144
+ branch on the backend.
145
+ """
146
+ with _pool().connection() as con:
147
+ row = con.execute(
148
+ _q('SELECT value FROM {tbl} WHERE key = %s', 'store_kv', tenant_slug),
149
+ (str(name),)).fetchone()
150
+ return dict(row[0]) if row and isinstance(row[0], dict) else (row[0] if row else {})
151
+
152
+
153
+ def exists(name, tenant_slug=None):
154
+ """True/False β€” and unlike the HF backend this is NEVER "True on uncertainty".
155
+
156
+ `core.store.exists` returns True on any error so a caller that would seed the store cannot
157
+ clobber a registry it merely failed to reach. That guard exists because a FILE HOST cannot
158
+ distinguish "absent" from "unreachable". Postgres can: an absent row is a fact, and an
159
+ unreachable server raises instead of answering.
160
+ """
161
+ with _pool().connection() as con:
162
+ row = con.execute(
163
+ _q('SELECT 1 FROM {tbl} WHERE key = %s', 'store_kv', tenant_slug),
164
+ (str(name),)).fetchone()
165
+ return row is not None
166
+
167
+
168
+ def put(name, data, tenant_slug=None):
169
+ with _pool().connection() as con:
170
+ con.execute(
171
+ _q('INSERT INTO {tbl} (key, value) VALUES (%s, %s::jsonb) '
172
+ 'ON CONFLICT (key) DO UPDATE SET value = EXCLUDED.value, '
173
+ 'updated_at = now(), rev = {bare}.rev + 1', 'store_kv', tenant_slug),
174
+ (str(name), json.dumps(data)))
175
+ return data
176
+
177
+
178
+ def update(name, fn, flush='sync', tenant_slug=None):
179
+ """Read-modify-write inside ONE transaction, with the row LOCKED for the duration.
180
+
181
+ This is the operation the HF backend could not make safe. There, `update` read over HTTP,
182
+ applied `fn`, and uploaded β€” so two concurrent writers both read the old value and the second
183
+ upload silently discarded the first writer's change. `SELECT … FOR UPDATE` makes the second
184
+ writer WAIT and then apply `fn` to the first writer's result, which is what "read-modify-write"
185
+ was always supposed to mean.
186
+
187
+ `flush` is accepted and ignored (see the module docstring): there is no commit budget to
188
+ coalesce against, so 'async' has nothing to defer.
189
+ """
190
+ with _pool().connection() as con:
191
+ with con.transaction():
192
+ row = con.execute(
193
+ _q('SELECT value FROM {tbl} WHERE key = %s FOR UPDATE',
194
+ 'store_kv', tenant_slug),
195
+ (str(name),)).fetchone()
196
+ data = dict(row[0]) if row and isinstance(row[0], dict) else (row[0] if row else {})
197
+ result = fn(data)
198
+ data = result if result is not None else data
199
+ con.execute(
200
+ _q('INSERT INTO {tbl} (key, value) VALUES (%s, %s::jsonb) '
201
+ 'ON CONFLICT (key) DO UPDATE SET value = EXCLUDED.value, '
202
+ 'updated_at = now(), rev = {bare}.rev + 1', 'store_kv', tenant_slug),
203
+ (str(name), json.dumps(data)))
204
+ return data
205
+
206
+
207
+ def upload_bytes(path_in_repo, data, message=None, tenant_slug=None):
208
+ with _pool().connection() as con:
209
+ con.execute(
210
+ _q('INSERT INTO {tbl} (path, bytes, message) VALUES (%s, %s, %s) '
211
+ 'ON CONFLICT (path) DO UPDATE SET bytes = EXCLUDED.bytes, '
212
+ 'message = EXCLUDED.message, updated_at = now()', 'store_blobs', tenant_slug),
213
+ (str(path_in_repo), bytes(data), message))
214
+
215
+
216
+ def download_bytes(path_in_repo, tenant_slug=None):
217
+ """Raw bytes, or None when absent β€” the same contract as the HF backend: a missing attachment
218
+ degrades to "that file is gone", never to a broken page."""
219
+ with _pool().connection() as con:
220
+ row = con.execute(
221
+ _q('SELECT bytes FROM {tbl} WHERE path = %s', 'store_blobs', tenant_slug),
222
+ (str(path_in_repo),)).fetchone()
223
+ return bytes(row[0]) if row else None
224
+
225
+
226
+ def delete_path(path_in_repo, tenant_slug=None):
227
+ """Absent is SUCCESS β€” deleting what is already gone is the goal (HF backend's contract)."""
228
+ with _pool().connection() as con:
229
+ con.execute(_q('DELETE FROM {tbl} WHERE path = %s', 'store_blobs', tenant_slug),
230
+ (str(path_in_repo),))
231
+ return True
232
+
233
+
234
+ def flush(name=None, timeout=30.0):
235
+ """A no-op that returns True: every write above is already committed when it returns. Kept in
236
+ the interface because QA gates and shutdown hooks call it and must not branch on the backend."""
237
+ return True
238
+
239
+
240
+ def close():
241
+ """Release the pool (tests, and a process that is shutting down cleanly)."""
242
+ with _LOCK:
243
+ if _POOL['pool'] is not None:
244
+ try:
245
+ _POOL['pool'].close()
246
+ finally:
247
+ _POOL['pool'] = None
248
+ _POOL['url'] = None
249
+ _POOL.pop('ok', None)
royalimports_os/core/table_store.py ADDED
@@ -0,0 +1,281 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """The generic per-user TABLE WORKSPACE store β€” the persistence half of the table-page factory.
2
+
3
+ One durable store key holds one table OBJECT's per-user Airtable-style state:
4
+
5
+ {username: {'views': {view_id: SavedView},
6
+ 'fields': {field_key: Field}, # notes + custom_ + measure_ strata
7
+ 'overlays': {str(pid): {field_key: value}}}}
8
+
9
+ `make(table_key)` returns the six operations a table page's host loop needs, closed over that
10
+ key. The Customer table's ops (`modules/customer_data.py`, key 'customer_table_workspace') are
11
+ these exact functions β€” the logic MOVED here 2026-07-27 so that duplicating the Customer table
12
+ pattern to a new object is a registry row + a config, not a copy of the store plumbing
13
+ (owner directive: the table-page factory).
14
+
15
+ A LIST (membership/formula semantics) is deliberately a different store from a VIEW
16
+ (presentation/query state) β€” see modules/customer_data.py's customer_lists key.
17
+ """
18
+ import core.store as store
19
+
20
+ #: Wave-9 I17 β€” the SHARED bucket. Views whose permissions make them visible to anyone but
21
+ #: their creator live here instead of in a personal workspace, under a key that cannot collide
22
+ #: with a username (usernames come from core/users.py and are never dunder-wrapped; `is_shared`
23
+ #: guards it anyway). ONE HOME PER VIEW, never both: a view moved to personal is REMOVED from
24
+ #: here, and a view shared is removed from its creator's workspace. Two homes would mean two
25
+ #: divergent copies the moment either was edited.
26
+ SHARED_KEY = '__shared__'
27
+
28
+
29
+ def _may_see(view, viewer, is_admin=False):
30
+ """Visibility for ONE shared view, fail-closed.
31
+
32
+ 'collaborative' = everyone who can already open the module (the caller has gated that).
33
+ 'users' = the named users, plus the creator, plus admins β€” an admin who could not
34
+ see a view could not administer it either.
35
+ Anything unrecognised returns False rather than defaulting open: an unreadable permission
36
+ must never widen access ([[aios-permissioning]] β€” no fail-open defaults).
37
+ """
38
+ if not isinstance(view, dict):
39
+ return False
40
+ if view.get('createdBy') == viewer or is_admin:
41
+ return True
42
+ perms = view.get('permissions') or {}
43
+ edit = perms.get('edit')
44
+ if edit == 'collaborative':
45
+ return True
46
+ if edit == 'users':
47
+ return viewer in set(perms.get('users') or ())
48
+ return False # 'personal', absent, or junk
49
+
50
+
51
+ def _may_edit(view, viewer, is_admin=False):
52
+ """Who may WRITE a shared view. Same set as visibility today β€” the owner's item asks 'who
53
+ can edit' and lists who 'can have access', i.e. seeing and editing are one grant. Kept as a
54
+ separate function so they can diverge (a future read-only share) without hunting callers."""
55
+ return _may_see(view, viewer, is_admin)
56
+
57
+
58
+ def _may_administer(view, viewer, is_admin=False):
59
+ """Who may change a view's PERMISSIONS, or delete it: the creator or an admin ONLY.
60
+
61
+ Deliberately narrower than _may_edit. If a collaborator could rewrite `permissions` they
62
+ could grant themselves sole ownership of somebody else's view, or quietly widen a
63
+ users-scoped view to everyone β€” the classic privilege-escalation-by-edit hole.
64
+ """
65
+ if not isinstance(view, dict):
66
+ return False
67
+ return bool(is_admin) or view.get('createdBy') == viewer
68
+
69
+
70
+ def is_shared(view):
71
+ """A view belongs in the shared bucket when its permissions reach beyond its creator."""
72
+ return ((view or {}).get('permissions') or {}).get('edit') in ('collaborative', 'users')
73
+
74
+
75
+ class TableStore:
76
+ """The six store operations for one table object's workspace, closed over its store key."""
77
+
78
+ def __init__(self, table_key):
79
+ self.table_key = table_key
80
+
81
+ # ---------------------------------------------------------------- read
82
+ def workspace(self, username):
83
+ """One user's durable workspace: always the full three-strata shape."""
84
+ try:
85
+ ws = (store.get(self.table_key) or {}).get(username, {}) or {}
86
+ except Exception:
87
+ ws = {}
88
+ return {
89
+ 'views': dict(ws.get('views') or {}),
90
+ 'fields': dict(ws.get('fields') or {}),
91
+ 'overlays': dict(ws.get('overlays') or {}),
92
+ # wave-8 I11 (C4): folders over the saved views / cohorts sidebars. A FOURTH
93
+ # stratum rather than a key on each item β€” see aios_grid.clean_folders for why
94
+ # (a cohort lives in another store, and filing is an organising act, not part of
95
+ # what a view is). Absent for every workspace saved before this wave, which is
96
+ # exactly "no folders yet".
97
+ 'folders': dict(ws.get('folders') or {}),
98
+ 'itemFolders': dict(ws.get('itemFolders') or {}),
99
+ }
100
+
101
+ # ---------------------------------------------------------------- write
102
+ def _update(self, username, change):
103
+ def _up(data):
104
+ ws = data.setdefault(username, {})
105
+ ws.setdefault('views', {})
106
+ ws.setdefault('fields', {})
107
+ ws.setdefault('overlays', {})
108
+ ws.setdefault('folders', {})
109
+ ws.setdefault('itemFolders', {})
110
+ change(ws)
111
+ return data
112
+ # flush='async' (wave-7 W3): this is THE hot path β€” every autosaved filter tweak,
113
+ # column note and typed overlay cell lands here inside the component round-trip, and
114
+ # the historical synchronous hub commit cost seconds per edit. The mutation applies to
115
+ # the in-process cache (read-your-writes for every subsequent render); the hub write
116
+ # coalesces in the background. Registry/auth writes elsewhere stay flush='sync'.
117
+ store.update(self.table_key, _up, flush='async')
118
+
119
+ def save_folders(self, username, folders, item_folders):
120
+ """Replace the folder stratum wholesale (wave-8 I11).
121
+
122
+ Wholesale rather than per-folder because the caller has ALREADY validated the complete
123
+ picture through aios_grid.clean_folders / clean_item_folders, and those two are
124
+ interdependent: a placement is only legal while its folder exists, so committing them
125
+ separately would leave a window where a reader sees an item filed into a folder that is
126
+ not there yet. One write, one consistent state.
127
+ """
128
+ def _set(ws):
129
+ ws['folders'] = dict(folders or {})
130
+ ws['itemFolders'] = dict(item_folders or {})
131
+ self._update(username, _set)
132
+
133
+ def shared_views(self, viewer, is_admin=False):
134
+ """Every SHARED view this viewer may see, by id (wave-9 I17).
135
+
136
+ Read-only and independent of the viewer's own workspace: the caller merges. Returns
137
+ only what `_may_see` allows, so a caller cannot accidentally render somebody else's
138
+ personal view by forgetting to filter.
139
+ """
140
+ try:
141
+ bucket = ((store.get(self.table_key) or {}).get(SHARED_KEY) or {}).get('views') or {}
142
+ except Exception:
143
+ return {}
144
+ return {vid: dict(v) for vid, v in bucket.items()
145
+ if _may_see(v, viewer, is_admin)}
146
+
147
+ def shared_view(self, view_id):
148
+ """One shared view RAW β€” no visibility filter. For authorisation decisions only: a
149
+ caller must know a view exists and who owns it before it can decide whether the actor
150
+ may touch it. Never hand the result to a renderer without checking `_may_see`."""
151
+ try:
152
+ return ((store.get(self.table_key) or {}).get(SHARED_KEY) or {}
153
+ ).get('views', {}).get(str(view_id))
154
+ except Exception:
155
+ return None
156
+
157
+ def save_view(self, username, view, shared=None):
158
+ """Upsert a SavedView into its ONE home β€” personal workspace or the shared bucket.
159
+
160
+ `shared` defaults to reading the view's own permissions (`is_shared`). Whichever home
161
+ it lands in, the view is REMOVED from the other, so a view can never exist as two
162
+ copies that diverge on the next edit.
163
+
164
+ ⚠ AUTHORISATION IS THE CALLER'S JOB and must happen BEFORE this is called β€” this layer
165
+ moves data and does not know who is asking. `_cl_handle_one` is the wall.
166
+ """
167
+ view_id = str((view or {}).get('id') or '').strip()
168
+ if not view_id:
169
+ raise ValueError('view id is required')
170
+ if username == SHARED_KEY:
171
+ raise ValueError('reserved username')
172
+ to_shared = is_shared(view) if shared is None else bool(shared)
173
+ payload = dict(view)
174
+
175
+ def _up(data):
176
+ if to_shared:
177
+ bucket = data.setdefault(SHARED_KEY, {})
178
+ bucket.setdefault('views', {})[view_id] = payload
179
+ # it may have lived in the creator's workspace before being shared
180
+ owner = data.get(payload.get('createdBy') or username) or {}
181
+ (owner.get('views') or {}).pop(view_id, None)
182
+ else:
183
+ ws = data.setdefault(username, {})
184
+ ws.setdefault('views', {})[view_id] = payload
185
+ (data.get(SHARED_KEY, {}).get('views') or {}).pop(view_id, None)
186
+ return data
187
+
188
+ store.update(self.table_key, _up, flush='async')
189
+
190
+ def delete_view(self, username, view_id):
191
+ """Delete a custom/list view override. The system all-rows view is guarded by caller.
192
+
193
+ Removes from BOTH homes: the caller has already authorised the delete, and leaving a
194
+ stale copy in the other bucket would resurrect the view on the next read.
195
+ """
196
+ vid = str(view_id)
197
+
198
+ def _up(data):
199
+ (data.get(username, {}).get('views') or {}).pop(vid, None)
200
+ (data.get(SHARED_KEY, {}).get('views') or {}).pop(vid, None)
201
+ return data
202
+
203
+ store.update(self.table_key, _up, flush='async')
204
+
205
+ def save_field(self, username, field):
206
+ """Persist a column note or a user-created (custom_/measure_) field definition."""
207
+ key = str((field or {}).get('key') or '').strip()
208
+ if not key:
209
+ raise ValueError('field key is required')
210
+
211
+ def _save(ws):
212
+ # A cleared note on an immutable source field returns to the canonical schema
213
+ # instead of leaving a meaningless override row. Custom fields remain even with
214
+ # an empty note β€” and so does a PRESET field carrying a measure-window override
215
+ # (wave-2 item 8) or a DISPLAY-format override (wave-5 item 10): the note may be
216
+ # empty but the period/format is real user state.
217
+ if (not field.get('custom') and not str(field.get('note') or '').strip()
218
+ and not isinstance(field.get('measure'), dict)
219
+ and not isinstance(field.get('format'), dict)):
220
+ ws['fields'].pop(key, None)
221
+ else:
222
+ ws['fields'][key] = dict(field)
223
+
224
+ self._update(username, _save)
225
+
226
+ def duplicate_field(self, username, source_key, new_key, field):
227
+ """Clone a user-created field in ONE store transaction (wave-5 item 1): the new
228
+ definition plus β€” for `custom_` overlay sources only β€” every stored cell value under
229
+ the source key. One transaction, because a def without its values (or values without a
230
+ def) is exactly the orphan state delete_field exists to prevent, in reverse.
231
+ The caller validated both keys (same created stratum) and stamped the clone's
232
+ createdBy; this layer only moves data."""
233
+ source_key = str(source_key or '').strip()
234
+ new_key = str(new_key or '').strip()
235
+ if not source_key or not new_key or source_key == new_key:
236
+ raise ValueError('duplicate_field needs two distinct keys')
237
+
238
+ def _dup(ws):
239
+ ws['fields'][new_key] = dict(field)
240
+ if source_key.startswith('custom_'):
241
+ for row in ws['overlays'].values():
242
+ if isinstance(row, dict) and source_key in row:
243
+ row[new_key] = row[source_key]
244
+
245
+ self._update(username, _dup)
246
+
247
+ def delete_field(self, username, key):
248
+ """Delete a USER-CREATED field definition outright (owner gap closed 2026-07-27).
249
+
250
+ Only the created strata ever reach here (`custom_` overlay fields, `measure_` formula
251
+ columns β€” the caller enforces the prefix). The stored overlay VALUES for the key are
252
+ scrubbed with it: a deleted column's cells must not linger as orphan data that would
253
+ silently resurface if the key were ever reused. Views referencing the key self-heal on
254
+ their next autosave (an unknown colId is dropped) β€” the rule every stale key rides.
255
+ """
256
+ key = str(key or '').strip()
257
+ if not key:
258
+ return
259
+
260
+ def _drop(ws):
261
+ ws['fields'].pop(key, None)
262
+ for row in ws['overlays'].values():
263
+ if isinstance(row, dict):
264
+ row.pop(key, None)
265
+
266
+ self._update(username, _drop)
267
+
268
+ def patch_overlay(self, username, pid, updates):
269
+ """Patch only the external editable stratum; never writes to the source system."""
270
+ clean = dict(updates or {})
271
+ if not clean:
272
+ return
273
+
274
+ def _patch(ws):
275
+ ws['overlays'].setdefault(str(int(pid)), {}).update(clean)
276
+
277
+ self._update(username, _patch)
278
+
279
+
280
+ def make(table_key):
281
+ return TableStore(table_key)
royalimports_os/core/user_tables.py ADDED
@@ -0,0 +1,179 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """USER-CREATED TABLES (wave-9 I8 / contract C6) β€” databases that are not a connector.
2
+
3
+ The owner's "+ New" flow has two branches: connect a SOURCE (Odoo today) or *"create a completely
4
+ new database blank, where this time the user can actually insert row that they want."* The first
5
+ branch is the connector seam and lives in `harness/datastore.py`; THIS module is the second.
6
+
7
+ WHY A SEPARATE STORE AND NOT A REGISTRY ROW. `core/registry.py` is a static Python list read at
8
+ import time β€” it is the catalogue of modules the PRODUCT ships, and a tenant cannot append to it
9
+ at runtime without editing code. A user-created table is tenant data, so it lives in the tenant
10
+ store and is MERGED into the nav beside the registry rows. That also keeps the archived/allowed
11
+ machinery honest: a user table is never "archived", it is deleted, and it is never in
12
+ `allowed_modules` because it is not a module.
13
+
14
+ SHAPE (one store key, `user_tables`):
15
+
16
+ {table_key: {'key', 'label', 'source', 'createdBy', 'created', 'fields': [...], 'rows': {...}}}
17
+
18
+ `source` is deliberately NOT 'Odoo' β€” the nav badge is generated from it, so a blank table reads
19
+ "Blank" and can never be mistaken for connected data. `rows` is `{row_id: {field_key: value}}`,
20
+ which is the same overlay shape `TableStore` already uses, so the grid's write path needs no new
21
+ storage concept.
22
+
23
+ ⚠ ROW WRITES ARE ONLY EVER LEGAL HERE. An Odoo-sourced table is READ-ONLY at the source and
24
+ `_cl_handle_one`'s row_add / row_delete refuse unless `is_user_table(key)` β€” a user must not be
25
+ able to invent a customer in Odoo by typing into a grid.
26
+ """
27
+ import datetime as _dt
28
+ import re
29
+
30
+ import core.store as store
31
+
32
+ STORE_KEY = 'user_tables'
33
+ MAX_TABLES = 40
34
+ MAX_LABEL = 60
35
+ MAX_ROWS = 5000
36
+ #: the badge the nav paints for a table with no connector behind it
37
+ BLANK_SOURCE = 'Blank'
38
+ #: every user table's key is prefixed, so it can never collide with a registry module key
39
+ KEY_PREFIX = 'ut_'
40
+
41
+
42
+ def _slug(label):
43
+ s = re.sub(r'[^a-z0-9]+', '_', str(label or '').strip().lower()).strip('_')
44
+ return (s or 'table')[:40]
45
+
46
+
47
+ def all_tables():
48
+ """{key: definition} β€” every user-created table in this tenant."""
49
+ try:
50
+ return dict(store.get(STORE_KEY) or {})
51
+ except Exception:
52
+ return {}
53
+
54
+
55
+ def get(table_key):
56
+ return all_tables().get(str(table_key))
57
+
58
+
59
+ def is_user_table(table_key):
60
+ """THE permission predicate for row writes. Cheap and total: a key that is not in this store
61
+ is not a user table, so it cannot accept invented rows."""
62
+ return str(table_key or '').startswith(KEY_PREFIX) and get(table_key) is not None
63
+
64
+
65
+ def create(label, username, fields=None):
66
+ """Create a blank table. Returns its key, or None if refused.
67
+
68
+ Refuses rather than raises on the ordinary conditions (no name, duplicate, cap reached) β€”
69
+ the caller is a UI form and a refusal is a message, not an exception.
70
+ """
71
+ label = str(label or '').strip()[:MAX_LABEL]
72
+ if not label:
73
+ return None
74
+ existing = all_tables()
75
+ if len(existing) >= MAX_TABLES:
76
+ return None
77
+ key = KEY_PREFIX + _slug(label)
78
+ if key in existing: # a second table of the same name gets a suffix
79
+ for n in range(2, 50):
80
+ if f'{key}_{n}' not in existing:
81
+ key = f'{key}_{n}'
82
+ break
83
+ else:
84
+ return None
85
+ # A blank table still needs ONE column to be a table at all: a row with no fields cannot be
86
+ # displayed, edited or identified. 'name' is the identity column, the same role `customer`
87
+ # plays in the Customer table.
88
+ defn = {
89
+ 'key': key,
90
+ 'label': label,
91
+ 'source': BLANK_SOURCE,
92
+ 'createdBy': username,
93
+ 'created': _dt.datetime.now().strftime('%Y-%m-%dT%H:%M:%S'),
94
+ 'fields': list(fields or [{'key': 'name', 'label': 'Name', 'type': 'text',
95
+ 'source': 'overlay', 'default': True}]),
96
+ 'rows': {},
97
+ }
98
+
99
+ def _add(cur):
100
+ cur[key] = defn
101
+ return cur
102
+
103
+ store.update(STORE_KEY, _add, flush='sync') # a creation is not a hot path; commit it
104
+ return key
105
+
106
+
107
+ def delete(table_key):
108
+ def _drop(cur):
109
+ cur.pop(str(table_key), None)
110
+ return cur
111
+ store.update(STORE_KEY, _drop, flush='sync')
112
+
113
+
114
+ def add_row(table_key, values=None, username=None):
115
+ """Append one row. Returns the new row id, or None if refused."""
116
+ if not is_user_table(table_key):
117
+ return None # ⚠ never on a connector-backed table
118
+ defn = get(table_key) or {}
119
+ rows = defn.get('rows') or {}
120
+ if len(rows) >= MAX_ROWS:
121
+ return None
122
+ rid = str(max((int(r) for r in rows if str(r).isdigit()), default=0) + 1)
123
+ valid = {f['key'] for f in (defn.get('fields') or [])}
124
+ clean = {k: v for k, v in (values or {}).items() if k in valid}
125
+
126
+ def _add(cur):
127
+ t = cur.get(str(table_key))
128
+ if t is not None:
129
+ t.setdefault('rows', {})[rid] = clean
130
+ return cur
131
+
132
+ store.update(STORE_KEY, _add, flush='async')
133
+ return rid
134
+
135
+
136
+ def delete_row(table_key, row_id):
137
+ if not is_user_table(table_key):
138
+ return False
139
+
140
+ def _drop(cur):
141
+ t = cur.get(str(table_key))
142
+ if t is not None:
143
+ (t.get('rows') or {}).pop(str(row_id), None)
144
+ return cur
145
+
146
+ store.update(STORE_KEY, _drop, flush='async')
147
+ return True
148
+
149
+
150
+ def may_open(table_key, viewer, is_admin=False):
151
+ """FAIL-CLOSED visibility: the creator, or an admin. Nothing else.
152
+
153
+ A user table cannot be gated by `allowed_modules` β€” a module grant written last month cannot
154
+ describe a table created this morning β€” so it needs its own rule, and the safe rule is the
155
+ narrow one. Tables are stored tenant-wide under one key, so without this ANY user (including
156
+ a BU-scoped sales agent) could open a table somebody else created just by knowing its key.
157
+ Sharing a user table with named colleagues is a follow-on, and it should reuse the shared-
158
+ VIEW vocabulary rather than inventing a second one.
159
+ """
160
+ t = get(table_key)
161
+ if not t:
162
+ return False
163
+ return bool(is_admin) or t.get('createdBy') == viewer
164
+
165
+
166
+ def nav_entries(viewer=None, is_admin=False):
167
+ """Registry-SHAPED dicts so the nav can render user tables beside real modules with no
168
+ special-casing: the same keys the flat nav reads (`key`, `label`, `source`).
169
+
170
+ Filtered by `may_open`, so the nav cannot offer a row the dispatcher would refuse.
171
+ """
172
+ out = []
173
+ for key, t in sorted(all_tables().items(), key=lambda kv: (kv[1].get('label') or '').lower()):
174
+ if viewer is not None and not may_open(key, viewer, is_admin):
175
+ continue
176
+ out.append({'key': key, 'label': t.get('label') or key,
177
+ 'source': t.get('source') or BLANK_SOURCE,
178
+ 'user_table': True, 'nav': True, 'validate': False})
179
+ return out
royalimports_os/core/users.py ADDED
@@ -0,0 +1,238 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Per-user accounts for the platform, persisted in the HF Dataset store (users.json).
2
+
3
+ Passwords are salted + PBKDF2-HMAC-SHA256 (200k iterations) β€” never stored or logged in plaintext.
4
+ A bootstrap 'admin' account is seeded from APP_PASSWORD so the owner can always log in and create
5
+ users; APP_PASSWORD also works as an emergency master for 'admin' if the registry is unreachable.
6
+
7
+ Each account carries BU access ('all' or a list of team-ids [5=Fisch, 6=Royal]) which drives
8
+ allowed_bus() β€” the basis for per-Business-Unit permissioning (a Royal-only user never sees Fisch).
9
+ """
10
+ import os
11
+ import hmac
12
+ import hashlib
13
+ import secrets
14
+
15
+ import core.store as store
16
+
17
+ BU_LABELS = {5: 'Fisch', 6: 'Royal'}
18
+ _ITER = 200_000
19
+
20
+
21
+ def _hash(pw, salt):
22
+ return hashlib.pbkdf2_hmac('sha256', str(pw).encode('utf-8'), bytes.fromhex(salt), _ITER).hex()
23
+
24
+
25
+ def _record(pw, name, role, bus, active=True, modules='all', agent=None, email=None):
26
+ salt = secrets.token_hex(16)
27
+ return {'salt': salt, 'hash': _hash(pw, salt), 'name': name, 'role': role,
28
+ 'bus': bus, 'active': active, 'modules': modules,
29
+ 'agent': agent or None, 'email': email or None}
30
+
31
+
32
+ def registry():
33
+ return store.get('users')
34
+
35
+
36
+ def ensure_bootstrap():
37
+ """Seed an 'admin' account from APP_PASSWORD ONLY on a truly fresh store (no users file yet).
38
+ Idempotent; no-op if the store is unavailable (the app then falls back to the master-password
39
+ path in verify()).
40
+
41
+ Critically, this NEVER overwrites an existing registry: it seeds only when store.exists('users')
42
+ is definitively False. A transient read failure at startup used to return {} and make this
43
+ re-seed just {admin} over the real accounts β€” that is the bug that wiped users on restart."""
44
+ if not store.available():
45
+ return
46
+ if store.exists('users'): # present, or uncertain -> never clobber
47
+ return
48
+ try:
49
+ reg = store.get('users', fresh=True)
50
+ except Exception:
51
+ return
52
+ if reg:
53
+ return
54
+ master = os.environ.get('APP_PASSWORD', '')
55
+ if not master:
56
+ return
57
+ try:
58
+ store.put('users', {'admin': _record(master, 'Administrator', 'admin', 'all')})
59
+ except Exception:
60
+ pass
61
+
62
+
63
+ def _public(username, u):
64
+ return {'username': username, 'name': u.get('name', username),
65
+ 'role': u.get('role', 'user'), 'bus': u.get('bus', 'all'),
66
+ 'modules': u.get('modules', 'all'),
67
+ 'agent': u.get('agent'), 'email': u.get('email'),
68
+ 'epoch': int(u.get('epoch') or 0)}
69
+
70
+
71
+ # ------------------------------------------------------------------ session revocation (X3)
72
+ # The API's session cookie is SIGNED AND STATELESS: there is no server-side session table to
73
+ # delete from, so "log this user out everywhere" needs a number that lives with the account. The
74
+ # cookie carries the epoch it was minted under; bumping the account's epoch makes every
75
+ # outstanding cookie for that user fail verification on its next use. Absent == 0, so every
76
+ # record written before this wave is valid without a migration.
77
+ def epoch(username):
78
+ """The current session epoch for `username`. None when there is no such account.
79
+
80
+ None is NOT 0. 0 is "this account exists and has never been revoked"; None is "no record" β€”
81
+ which the session verifier must treat as a reason to refuse, not as a default to compare
82
+ against. (The APP_PASSWORD emergency-master admin has no record at all; the verifier handles
83
+ that case explicitly rather than inventing an epoch for it here.)
84
+ """
85
+ username = (username or '').strip().lower()
86
+ try:
87
+ u = (store.get('users') or {}).get(username)
88
+ except Exception:
89
+ return None
90
+ return int((u or {}).get('epoch') or 0) if u else None
91
+
92
+
93
+ def bump_epoch(username):
94
+ """Revoke every outstanding API session for this account."""
95
+ username = (username or '').strip().lower()
96
+
97
+ def _set(reg):
98
+ u = reg.get(username)
99
+ if u:
100
+ u['epoch'] = int(u.get('epoch') or 0) + 1
101
+ return reg
102
+ store.update('users', _set)
103
+
104
+
105
+ def verify(username, pw):
106
+ """Return a public user dict on success, else None. APP_PASSWORD is an emergency master for the
107
+ 'admin' login even if the store is unreachable, so the owner is never locked out."""
108
+ username = (username or '').strip().lower()
109
+ if not username or not pw:
110
+ return None
111
+ master = os.environ.get('APP_PASSWORD', '')
112
+ try:
113
+ # read fresh so accounts created moments ago (UI or out-of-band) are recognised at once
114
+ reg = store.get('users', fresh=True)
115
+ except Exception:
116
+ reg = {}
117
+ u = reg.get(username)
118
+ if u and u.get('active', True) and hmac.compare_digest(_hash(pw, u['salt']), u['hash']):
119
+ return _public(username, u)
120
+ # emergency master: admin + APP_PASSWORD always works (covers first run / store outage)
121
+ if username == 'admin' and master and hmac.compare_digest(str(pw), master):
122
+ # ⚠ CARRY THE RECORD'S CURRENT EPOCH when there is a record to read, so the session cookie
123
+ # the API mints from this dict AGREES with the stored account.
124
+ #
125
+ # This is not what stops the emergency lockout β€” `deps._user_for`'s master fallback does
126
+ # that, and a negative control confirmed the lockout is gone with or without this line.
127
+ # What it fixes is subtler and is a SCOPE question: a cookie whose epoch disagrees with the
128
+ # record falls through to that master fallback, which hands back a SYNTHETIC identity
129
+ # (`bus: 'all'`, `modules: 'all'`). An admin whose record narrows either field would
130
+ # therefore be silently WIDENED to consolidated, all-module access for the life of that
131
+ # session. Matching the epoch means the record branch wins and the account's real scope
132
+ # applies, leaving the master fallback as the true last resort it is meant to be.
133
+ # 0 when the store is unreachable, which is the case this branch was written for.
134
+ return {'username': 'admin', 'name': 'Administrator', 'role': 'admin', 'bus': 'all',
135
+ 'modules': 'all', 'epoch': int((reg.get('admin') or {}).get('epoch') or 0)}
136
+ return None
137
+
138
+
139
+ def create_user(username, pw, name, role='user', bus='all', modules='all',
140
+ agent=None, email=None):
141
+ """Create β€” or, from app.py's dialog, OVERWRITE β€” an account.
142
+
143
+ β›” X3: OVERWRITING AN ACCOUNT MUST NOT RESURRECT ITS OLD SESSIONS. `_record()` builds a fresh
144
+ record with no `epoch` key, i.e. absent == 0. So re-saving an existing username used to reset
145
+ the epoch to 0, and every cookie minted before that account's last password rotation started
146
+ verifying again β€” a silent un-revocation. `app.py`'s "Add / update a user" calls this function
147
+ for BOTH add and update, so the hole was reachable from the shipped UI.
148
+
149
+ Epoch revocation is only ever as strong as the NARROWEST write path that touches the record, so
150
+ the carry-and-bump lives here rather than in each caller: an overwrite is at least as
151
+ session-invalidating as a password change, and it usually IS one.
152
+ """
153
+ username = (username or '').strip().lower()
154
+ if not username or not pw:
155
+ raise ValueError('username and password are required')
156
+
157
+ def _add(reg):
158
+ prior = reg.get(username) or {}
159
+ rec = _record(pw, name or username, role, bus, modules=modules,
160
+ agent=agent, email=email)
161
+ if prior:
162
+ rec['epoch'] = int(prior.get('epoch') or 0) + 1
163
+ reg[username] = rec
164
+ return reg
165
+ store.update('users', _add)
166
+
167
+
168
+ def set_password(username, pw):
169
+ username = (username or '').strip().lower()
170
+
171
+ def _set(reg):
172
+ u = reg.get(username)
173
+ if u:
174
+ u['salt'] = secrets.token_hex(16)
175
+ u['hash'] = _hash(pw, u['salt'])
176
+ # X3: a password change revokes every outstanding API session for the account. Bumped
177
+ # INSIDE the same read-modify-write as the hash so the two can never disagree β€” a
178
+ # separate update() could rotate the password and leave old cookies live if the second
179
+ # write failed.
180
+ u['epoch'] = int(u.get('epoch') or 0) + 1
181
+ return reg
182
+ store.update('users', _set)
183
+
184
+
185
+ def set_active(username, active):
186
+ username = (username or '').strip().lower()
187
+
188
+ def _set(reg):
189
+ if username in reg:
190
+ reg[username]['active'] = bool(active)
191
+ # X3: DEACTIVATION must kill live sessions, not just future logins β€” otherwise a
192
+ # disabled account keeps working until its cookie expires. Bumped on reactivation too:
193
+ # cheap, and it means a re-enabled account never resurrects a stale cookie.
194
+ reg[username]['epoch'] = int(reg[username].get('epoch') or 0) + 1
195
+ return reg
196
+ store.update('users', _set)
197
+
198
+
199
+ def set_access(username, role=None, bus=None, modules=None, agent=None, email=None,
200
+ name=None):
201
+ """Update access fields. agent/email: pass '' to clear, None to leave unchanged β€”
202
+ the user↔agent link scopes the Customer List page / digests to that agent's book.
203
+
204
+ `name` follows the same None-means-unchanged idiom. It is here because it had no setter at all:
205
+ a display name could previously only be changed by re-creating the record through
206
+ `create_user`, i.e. by also resetting the password (and, before the fix above, the session
207
+ epoch). Y4's `PATCH {name?}` needs the narrow write, not the destructive one."""
208
+ username = (username or '').strip().lower()
209
+
210
+ def _set(reg):
211
+ u = reg.get(username)
212
+ if u:
213
+ if name is not None:
214
+ u['name'] = name
215
+ if role is not None:
216
+ u['role'] = role
217
+ if bus is not None:
218
+ u['bus'] = bus
219
+ if modules is not None:
220
+ u['modules'] = modules
221
+ if agent is not None:
222
+ u['agent'] = agent or None
223
+ if email is not None:
224
+ u['email'] = email or None
225
+ return reg
226
+ store.update('users', _set)
227
+
228
+
229
+ def allowed_bus_labels(user):
230
+ """BU labels this user may select. 'all' -> All+Fisch+Royal; a single BU -> just that BU (no
231
+ 'All', so the other BU is never reachable); multiple -> All + each."""
232
+ bus = (user or {}).get('bus', 'all')
233
+ if bus == 'all':
234
+ return ['All', 'Fisch', 'Royal']
235
+ labels = [BU_LABELS[b] for b in bus if b in BU_LABELS]
236
+ if not labels:
237
+ return ['All', 'Fisch', 'Royal']
238
+ return (['All'] + labels) if len(labels) > 1 else labels
royalimports_os/evals/analyst_golden.yml ADDED
@@ -0,0 +1,104 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Analyst golden eval set v1 (OM-4 gate, 2026-07-11; truths REFRESHED 2026-07-17 after the
2
+ # Jul-15 Odoo mass-recompute + hard-delete reconciliation β€” closed windows DO drift when docs
3
+ # are edited/deleted; refresh truths from the parity-proven layer, never loosen tolerances).
4
+ # Original header: truths computed from the PARITY-PROVEN
5
+ # semantic layer over CLOSED windows (H1 2026 = 2026-01-01..2026-06-30) so live data drift cannot
6
+ # break them. Checks are DETERMINISTIC (number within tolerance + required substrings + tool-use
7
+ # assertions) β€” the validate() culture applied to AI answers. Run: harness/evals.py.
8
+ # expect.value tolerance: abs or rel (fraction). expect.contains: ALL substrings (case-insensitive).
9
+ # expect.tools: tool names that MUST appear in the trace (the answer must come FROM the tools).
10
+ # expect.not_tools: tool names that must NOT appear (the over-refusal guard β€” an answerable ask
11
+ # must never end in report_gap; a gap ask must never end in a forced wrong query).
12
+ items:
13
+ - id: rev_h1
14
+ ask: "What was our wholesale revenue for the first half of 2026 (Jan 1 through Jun 30)?"
15
+ expect: {value: 3501508.33, rel: 0.001, tools: [run_semantic_query]}
16
+ - id: rev_h1_fisch
17
+ ask: "Fisch revenue only, H1 2026 (Jan 1 - Jun 30)?"
18
+ expect: {value: 2494244.70, rel: 0.001, tools: [run_semantic_query]}
19
+ - id: rev_h1_royal
20
+ ask: "And Royal's revenue for the same H1 2026 window?"
21
+ expect: {value: 1007263.63, rel: 0.001, tools: [run_semantic_query]}
22
+ - id: rev_june
23
+ ask: "Revenue for June 2026?"
24
+ expect: {value: 535046.06, rel: 0.001, tools: [run_semantic_query]}
25
+ - id: margin_h1
26
+ ask: "Gross margin dollars for H1 2026 (Jan-Jun), wholesale?"
27
+ expect: {value: 2067509.45, rel: 0.001, tools: [run_semantic_query]}
28
+ - id: margin_pct_h1
29
+ ask: "What gross margin percentage did we run in H1 2026?"
30
+ expect: {value: 0.59, abs: 0.005, allow_pct_form: true, tools: [run_semantic_query]}
31
+ - id: orders_h1
32
+ ask: "How many confirmed wholesale orders did we take in H1 2026 (Jan 1 - Jun 30)?"
33
+ expect: {value: 4989, abs: 1, tools: [run_semantic_query]}
34
+ - id: customers_h1
35
+ ask: "How many distinct customers bought from us in H1 2026?"
36
+ expect: {value: 1037, abs: 1, tools: [run_semantic_query]}
37
+ - id: top_customer
38
+ ask: "Who was our biggest customer by revenue in H1 2026?"
39
+ expect: {contains: ["Poppy Flowers"], tools: [run_semantic_query], not_tools: [report_gap]}
40
+ - id: opex_h1
41
+ ask: "Total operating expense on the GL for H1 2026 (Jan 1 - Jun 30)?"
42
+ expect: {value: 4683428.81, rel: 0.001, tools: [run_semantic_query]}
43
+ - id: top_expense
44
+ ask: "What is our single largest expense account in H1 2026?"
45
+ expect: {contains: ["FBA"], tools: [run_semantic_query]}
46
+ - id: chart_request
47
+ ask: "Make me a line chart of monthly wholesale revenue for H1 2026."
48
+ expect: {artifact: chart, tools: [run_semantic_query, make_chart]}
49
+ - id: bu_guard
50
+ ask: "Show one single revenue line combining Fisch and Royal together per month for H1 2026."
51
+ expect: {tools: [run_semantic_query]} # combined-total ok; the guard is it must NOT error
52
+ - id: name_resolution
53
+ ask: "How much revenue did the customer 'flower' β€” I don't remember the exact name, something with FLOWERS in it β€” bring in H1 2026? Pick the top match."
54
+ expect: {tools: [get_field_values, run_semantic_query]}
55
+ - id: map_request
56
+ ask: "Put my top 20 customers by H1 2026 revenue on a map."
57
+ expect: {artifact: chart, tools: [run_semantic_query, make_chart]}
58
+ # --- the gap loop + location/YoY competence (added 2026-07-16 with the debug fixes) ---
59
+ - id: city_filter
60
+ ask: "Which customer in Brooklyn brought us the most revenue in H1 2026 (Jan 1 - Jun 30)?"
61
+ expect: {value: 48785.00, rel: 0.001, contains: ["J KAY"],
62
+ tools: [get_field_values, run_semantic_query], not_tools: [report_gap]}
63
+ - id: yoy_decliners
64
+ ask: "Which Fisch customer declined the most in revenue in H1 2026 (Jan 1 - Jun 30) versus the same window of 2025? Give the dollar decline."
65
+ expect: {value: 16451.25, rel: 0.005, contains: ["JUNE"],
66
+ tools: [run_semantic_query, transform_result], not_tools: [report_gap]}
67
+ # --- the agent dimension (added 2026-07-17: res.partner.agent_ids m2m sync β€” the #1 logged
68
+ # gap, closed). agent_book is VERBATIM the old gap_refusal ask: yesterday's honest refusal
69
+ # must become today's answer.
70
+ - id: agent_book
71
+ ask: "Who are the top customers handled by our salesperson Naomi in H1 2026?"
72
+ expect: {contains: ["Poppy Flowers"], tools: [get_field_values, run_semantic_query],
73
+ not_tools: [report_gap]}
74
+ - id: agent_revenue
75
+ ask: "How much revenue did agent Naomi's book of customers bring in H1 2026 (Jan 1 - Jun 30)?"
76
+ expect: {value: 872869.98, rel: 0.005, tools: [run_semantic_query], not_tools: [report_gap]}
77
+ # --- the TWO agent sources (added 2026-07-28). Odoo holds a customer-master book AND a
78
+ # per-invoice-line commission agent; they name different people, and the commission table also
79
+ # carries internal SALESPEOPLE who are not agents. These three lock the distinction: the
80
+ # unallocated bucket must be answerable at all, and Anna/Shantal must never be called agents.
81
+ - id: agent_unallocated
82
+ ask: "Using the invoice-line basis, how much of Fisch's 2025 sales was NOT allocated to an agent?"
83
+ expect: {value: 1447579.59, rel: 0.01, tools: [run_semantic_query], not_tools: [report_gap]}
84
+ - id: agent_not_salesperson
85
+ ask: "Who are our agents on the Fisch invoice lines in 2025, and what did each generate?"
86
+ expect: {contains: ["Pasternak"], not_contains: ["Anna", "Shantal"],
87
+ tools: [run_semantic_query], not_tools: [report_gap]}
88
+ - id: agent_allocated_2025
89
+ ask: "On the invoice-line basis, how much of Fisch's 2025 sales was allocated to a real agent?"
90
+ expect: {value: 2995664.75, rel: 0.01, tools: [run_semantic_query], not_tools: [report_gap]}
91
+ - id: gap_refusal
92
+ ask: "How many units of our best-selling SKU do we physically have in stock right now?"
93
+ expect: {tools: [report_gap]} # stock-on-hand has NO topic in the store β€” must self-report, never guess
94
+ # --- the analytics transform library (added 2026-07-18: 60-op Tableau-class set). Truths from
95
+ # the parity-proven semantic layer over the CLOSED H1 2026 window.
96
+ - id: abc_a_tier
97
+ ask: "Using an ABC / Pareto analysis of H1 2026 revenue (Jan 1 - Jun 30), how many customers fall in the A tier β€” the vital few that make up the first 80% of revenue?"
98
+ expect: {value: 311, abs: 6, tools: [run_semantic_query, transform_result], not_tools: [report_gap]}
99
+ - id: ytd_customers
100
+ ask: "How many distinct customers had we sold to cumulatively from Jan 1 through the end of March 2026?"
101
+ expect: {value: 790, abs: 1, tools: [run_semantic_query], not_tools: [report_gap]}
102
+ - id: concentration_top5
103
+ ask: "What percent of our H1 2026 revenue (Jan 1 - Jun 30) is accounted for by our top 5 customers?"
104
+ expect: {value: 7.84, abs: 0.5, tools: [run_semantic_query], not_tools: [report_gap]}
royalimports_os/harness/__init__.py ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ """Harness β€” the multi-source financial-dashboarding platform (connectors + canonical model + skills)."""
2
+ from .canonical import Capability, Entity, Query, Filter
3
+ from .base import Connector, Tenant, Skill, register_skill, skills_for
royalimports_os/harness/analyst.py ADDED
@@ -0,0 +1,265 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """harness/analyst.py β€” the AIOS Analyst loop (OM-4, 2026-07-11).
2
+
3
+ A CHEAP small model (via OpenRouter, OpenAI-compatible) drives the compounding tool registry
4
+ (harness/tools.py) to answer questions, build charts, and spawn dashboards over the semantic
5
+ layer. The model NEVER writes SQL β€” it picks registry keys, guided by the in-product SKILL
6
+ recipes (model/skills/*.skill.yml) injected into its system prompt. Plan: omni-adoption addendum.
7
+
8
+ Architecture (mirrors the Omni coordinator pattern):
9
+ ask(question) -> bounded tool loop: chat -> tool_calls -> tools.dispatch() -> results -> chat …
10
+ until a plain answer or MAX_ITERS. chat_fn is INJECTABLE: live = OpenRouter POST; tests = a
11
+ scripted double β€” the loop mechanics are provable offline (no key, no tokens).
12
+
13
+ Correctness posture: bounded iterations (never endless); tool errors go BACK to the model as
14
+ readable envelopes (it self-corrects); the final answer carries the tool trace (auditability +
15
+ the drill guarantee); exhausted-budget is reported AS exhausted, never as success (loop rules).
16
+ Cost posture: DEFAULT_MODEL is a cheap tool-calling model; override per-tenant/env; prompt = the
17
+ skills + topic summary (small, cacheable), NOT raw data.
18
+ """
19
+ import json
20
+ import os
21
+ from pathlib import Path
22
+
23
+ import harness.tools as T
24
+
25
+ # Provider LADDER (owner 2026-07-12: "use Groq and Cerebras"): all OpenAI-compatible chat
26
+ # endpoints; tried in order, failing over on payment/rate/server errors β€” an exhausted or
27
+ # rate-limited provider degrades to the next instead of taking the Analyst down (the
28
+ # OpenRouter-402 incident). Pin one with ANALYST_PROVIDER=groq|cerebras|openrouter; override
29
+ # its model with ANALYST_MODEL. Every model here must re-pass the eval gate (Β§4b).
30
+ PROVIDERS = [
31
+ {"name": "groq", "env": "GROQ_API_KEY",
32
+ "url": "https://api.groq.com/openai/v1/chat/completions",
33
+ "model": "llama-3.3-70b-versatile"},
34
+ {"name": "cerebras", "env": "CEREBRAS_API_KEY",
35
+ "url": "https://api.cerebras.ai/v1/chat/completions",
36
+ "model": "gpt-oss-120b"}, # this account's tool-capable model (from /v1/models)
37
+ {"name": "openrouter", "env": "OPENROUTER_API_KEY",
38
+ "url": "https://openrouter.ai/api/v1/chat/completions",
39
+ "model": "openai/gpt-4o-mini"},
40
+ ]
41
+ DEFAULT_MODEL = None # None = the selected provider's default model
42
+ MAX_ITERS = 8
43
+
44
+
45
+ def available_providers():
46
+ pin = os.environ.get("ANALYST_PROVIDER")
47
+ ps = [p for p in PROVIDERS if os.environ.get(p["env"])]
48
+ if pin:
49
+ ps = [p for p in ps if p["name"] == pin] or ps
50
+ return ps
51
+
52
+
53
+ def llm_configured():
54
+ return bool(available_providers())
55
+ SKILLS_DIR = Path(__file__).resolve().parents[1] / "model" / "skills"
56
+
57
+
58
+ def _skills_text():
59
+ parts = []
60
+ for f in sorted(SKILLS_DIR.glob("*.skill.yml")):
61
+ parts.append(f.read_text(encoding="utf-8"))
62
+ return "\n\n".join(parts) or "(no skill files installed)"
63
+
64
+
65
+ def system_prompt():
66
+ """The small model's standing instructions: the platform rules + the skill recipes.
67
+ Stable content (cache-friendly); per-question content goes in the user turn."""
68
+ topics = T.list_topics()
69
+ topic_summary = "\n".join(
70
+ f"- {t['topic']}: {t['label']} | metrics: {', '.join(t['metrics'])} | dims: {', '.join(t['dims'])}"
71
+ for t in topics)
72
+ return f"""You are the AIOS Analyst for a wholesale business. You answer data questions by
73
+ calling TOOLS over a governed semantic layer. THE RULES (non-negotiable):
74
+ 1. You NEVER write SQL or invent field names β€” only registry keys from the tools' schemas.
75
+ 2. Start unknown asks with list_topics/describe_topic. Resolve typed names (customers, products)
76
+ with get_field_values BEFORE filtering. Business units: Fisch=5, Royal=6 β€” never mix them in
77
+ one series unless explicitly comparing.
78
+ 3. Every number you present must come from a tool result and keep its result_id (drillable).
79
+ If a tool errors, read the error and correct your call β€” do not make numbers up, ever.
80
+ 4. Artifact actions (save_view, compose_dashboard, schedule_report, create_alert) need the
81
+ user's explicit confirmation first.
82
+ 5. Answer plainly and lead with the finding. State the window and scope you used β€” READ the
83
+ window from the tool result's 'window' field, never guess it: a query without dates covers
84
+ ALL recorded history (there is no hidden default window). Time-bound asks ("this year",
85
+ "past 4 weeks") must pass explicit date_from/date_to.
86
+ 6. A single-number question gets THE SINGLE NUMBER first (the total for the asked window/scope),
87
+ THEN any breakdown. Never answer a "what was X" question with only components: no monthly
88
+ lists without the total, no per-BU splits when the consolidated figure was asked. If the ask
89
+ is consolidated, query WITHOUT group_by (or sum before answering).
90
+ 7. NEVER state an aggregate you did not obtain from a tool result. If you want to mention a
91
+ total across the rows you fetched, either run the scalar query for it or ADD UP those exact
92
+ rows β€” a round "approximately" figure is a fabrication and forbidden. When in doubt, present
93
+ the rows without a total. Joining or diffing two result sets BY HAND (e.g. this year vs last
94
+ year per customer) is equally forbidden β€” that is transform_result's job ({{"op":"yoy"}} adds
95
+ <m>_ly, <m>_delta, <m>_yoy_pct per row); if no transform can compute it, say you cannot.
96
+ 8. THE GAP PROTOCOL. If β€” and only if β€” after checking the schema (list_topics/describe_topic)
97
+ no registered dimension, metric, transform or chart kind can answer, do NOT guess and do NOT
98
+ force a wrong query: call report_gap(kind, missing, question, workaround) ONCE, then answer
99
+ in one honest sentence what is missing and offer the nearest ask that IS answerable. Before
100
+ concluding a gap, check the TRANSFORMS: year-over-year / decline / growth comparisons (yoy),
101
+ rankings (top_n/sort/head), shares, running totals and distributions are ALL answerable via
102
+ transform_result β€” a derived-analytics question is never a gap. Both directions are
103
+ failures: refusing without report_gap, and calling report_gap for something the tools
104
+ support.
105
+ 9. NEVER emit markdown images, data: URIs, or links to charts β€” you cannot draw. Charts exist
106
+ ONLY through make_chart / make_table; the platform renders them below your answer. Refer to
107
+ a chart as "the chart below". Derived analytics (top-N, YoY, shares, running totals, moving
108
+ averages, ranks, bins) exist ONLY through transform_result β€” never arithmetic in your head.
109
+
110
+ CHART PICKER (the form follows the comparison β€” pick by what the question compares):
111
+ - trend over time -> line (few periods -> bar); seasonal/two-dim intensity -> heatmap
112
+ - this year vs last year by period -> transform_result [{{"op":"yoy"}}] then yoy_bars
113
+ - ranking / "top N" -> transform_result top_n then ranked_bar (sorted, labeled)
114
+ - part-to-whole -> pie|donut ONLY when <=6 slices (top_n first); over time -> stacked_bar;
115
+ shares of a shifting total -> stacked_pct; hierarchy of shares -> treemap
116
+ - what drove the change / contribution -> waterfall (signed steps; platform adds the Total)
117
+ - concentration ("do the top X carry the book?") -> pareto
118
+ - correlation of two measures -> scatter; + a third measure -> bubble (size)
119
+ - distribution of a measure -> histogram (or transform bin -> bar)
120
+ - stage conversion -> funnel; actual vs target -> bullet (y=actual, y2=target)
121
+ - level + rate together (e.g. GM$ + GM%) -> combo (y=bars left, y2=line right)
122
+ - same small chart per segment -> facet on line|bar|area|scatter
123
+ - exact figures, many columns, lists -> make_table (totals row included)
124
+ - customer geography -> map. When two forms tie, pick the simpler one.
125
+
126
+ DATASETS:
127
+ {topic_summary}
128
+
129
+ YOUR SKILL RECIPES (follow these plans; adjust the [bracketed] parameters):
130
+ {_skills_text()}"""
131
+
132
+
133
+ _DEAD_STATUS = {401, 402, 403} # payment/auth: skip the provider immediately, no retry
134
+ _RETRY_STATUS = {429, 500, 502, 503} # rate/server: retry with backoff before failing over
135
+ _MAX_ANSWER_TOKENS = 1500 # answers are short; also keeps low-credit providers usable
136
+ _COOLDOWN = {} # provider name -> unix ts until which we skip it (in-proc)
137
+
138
+
139
+ def _live_chat(messages, tools, model, tool_choice="auto"):
140
+ """POST down the provider ladder with LATENCY-FIRST failover (a chat user is waiting):
141
+ a rate-limited provider is SKIPPED IMMEDIATELY in favor of the next one β€” we only sleep
142
+ when the ENTIRE ladder is limited (then short ladder-level backoffs). A LONG Retry-After
143
+ (daily quota, e.g. Groq TPD) puts the provider in an in-process COOLDOWN; 401/402/403 skip
144
+ the provider outright."""
145
+ import time as _time
146
+
147
+ import requests
148
+ provs = available_providers()
149
+ if not provs:
150
+ raise RuntimeError("no LLM provider configured β€” set GROQ_API_KEY / CEREBRAS_API_KEY / "
151
+ "OPENROUTER_API_KEY in royalimports_os/.env")
152
+ last_err = None
153
+ # Two patience profiles: CHAT (default) fails over fast β€” a user is waiting; BATCH
154
+ # (ANALYST_PATIENT=1, set by the eval gate / insight runner) waits out per-minute rate
155
+ # windows across more ladder rounds instead of exhausting the ladder in seconds.
156
+ patient = os.environ.get("ANALYST_PATIENT") == "1"
157
+ round_sleeps = (5, 10, 20, 30, 45, 60) if patient else (2, 6, 12)
158
+ for rnd in range(len(round_sleeps)): # ladder rounds
159
+ saw_retryable = False
160
+ for p in provs:
161
+ if _COOLDOWN.get(p["name"], 0) > _time.time():
162
+ last_err = RuntimeError(f"{p['name']}: cooling down (quota)")
163
+ continue
164
+ try:
165
+ r = requests.post(p["url"], timeout=120,
166
+ headers={"Authorization": f"Bearer {os.environ[p['env']]}",
167
+ "HTTP-Referer": "https://aios.local",
168
+ "X-Title": "AIOS Analyst"},
169
+ json={"model": (model or os.environ.get("ANALYST_MODEL")
170
+ or p["model"]),
171
+ "messages": messages, "tools": tools,
172
+ "tool_choice": tool_choice, "temperature": 0.1,
173
+ "max_tokens": _MAX_ANSWER_TOKENS})
174
+ if r.status_code in _DEAD_STATUS:
175
+ last_err = RuntimeError(f"{p['name']}: HTTP {r.status_code}")
176
+ _COOLDOWN[p["name"]] = _time.time() + 300 # don't re-poke dead auth/payment
177
+ continue
178
+ if r.status_code in _RETRY_STATUS:
179
+ last_err = RuntimeError(f"{p['name']}: HTTP {r.status_code}")
180
+ try:
181
+ wait = float(r.headers.get("retry-after") or 0)
182
+ except ValueError:
183
+ wait = 0
184
+ if wait > 60: # daily/long quota β€” cool down
185
+ _COOLDOWN[p["name"]] = _time.time() + wait
186
+ else:
187
+ saw_retryable = True # NO sleep β€” next provider NOW
188
+ continue
189
+ r.raise_for_status()
190
+ usage = r.json().get("usage", {})
191
+ return r.json()["choices"][0]["message"], usage
192
+ except requests.RequestException as e:
193
+ last_err = e
194
+ continue # network blip β€” next provider
195
+ if not saw_retryable:
196
+ break # nothing retryable left anywhere
197
+ _time.sleep(round_sleeps[rnd]) # whole ladder limited β€” pause a round
198
+ raise RuntimeError(f"all LLM providers failed (last: {last_err})")
199
+
200
+
201
+ def ask(question, history=None, chat_fn=None, model=DEFAULT_MODEL, max_iters=MAX_ITERS,
202
+ telemetry_kind="analyst"):
203
+ """Run one Analyst turn. Returns {answer, artifacts, tool_trace, iterations, exhausted,
204
+ usage}. chat_fn(messages, tools, model) -> (assistant_message_dict, usage_dict)."""
205
+ chat = chat_fn or _live_chat
206
+ tools = T.openai_tools()
207
+ messages = [{"role": "system", "content": system_prompt()}]
208
+ messages += list(history or [])
209
+ # The model has no clock: anchor relative windows ("last week", "this month") to today.
210
+ # Lives in the USER turn so the system prompt stays byte-stable (prompt-cache friendly).
211
+ import datetime as _dt
212
+ messages.append({"role": "user",
213
+ "content": f"(today is {_dt.date.today().isoformat()})\n{question}"})
214
+ trace, artifacts, usage_total = [], [], {"prompt_tokens": 0, "completion_tokens": 0}
215
+
216
+ for i in range(max_iters):
217
+ msg, usage = chat(messages, tools, model)
218
+ for k in usage_total:
219
+ usage_total[k] += usage.get(k, 0) or 0
220
+ tool_calls = msg.get("tool_calls") or []
221
+ # Free-tier models INTERMITTENTLY answer a data question without calling any tool
222
+ # (~1 in 4 observed on gpt-oss-120b β€” the "no chart" defect). Deterministic fix: if
223
+ # the VERY FIRST turn used no tools, redo it once with tool_choice='required'. Meta
224
+ # questions just get a harmless schema lookup before answering.
225
+ if i == 0 and not tool_calls and chat is _live_chat:
226
+ msg, usage = _live_chat(messages, tools, model, tool_choice="required")
227
+ for k in usage_total:
228
+ usage_total[k] += usage.get(k, 0) or 0
229
+ tool_calls = msg.get("tool_calls") or []
230
+ messages.append({"role": "assistant", "content": msg.get("content") or "",
231
+ **({"tool_calls": tool_calls} if tool_calls else {})})
232
+ if not tool_calls:
233
+ answer = (msg.get("content") or "").strip()
234
+ if not answer:
235
+ # A model occasionally returns NO content and NO tool calls (observed live
236
+ # 2026-07-16: the reply bubble was simply empty and the next turn hallucinated).
237
+ # Never surface silence β€” report the failure honestly instead.
238
+ answer = ("(the model returned an empty answer β€” nothing was computed; "
239
+ "please rephrase or ask again)")
240
+ import harness.telemetry as _tel
241
+ _tel.log(telemetry_kind, question=question[:200],
242
+ model=model or (available_providers() or [{}])[0].get("name", "auto"),
243
+ iterations=i + 1,
244
+ tools=[t["tool"] for t in trace],
245
+ tokens=usage_total["prompt_tokens"] + usage_total["completion_tokens"])
246
+ return {"answer": answer, "artifacts": artifacts,
247
+ "tool_trace": trace, "iterations": i + 1, "exhausted": False,
248
+ "usage": usage_total}
249
+ for tc in tool_calls:
250
+ name = tc["function"]["name"]
251
+ args = tc["function"].get("arguments") or "{}"
252
+ res = T.dispatch(name, args)
253
+ trace.append({"tool": name, "args": args if isinstance(args, str) else json.dumps(args),
254
+ "ok": res["ok"], "error": res.get("error")})
255
+ if res["ok"] and isinstance(res["data"], dict) and (
256
+ "chart" in res["data"] or "kpi" in res["data"] or "table" in res["data"]
257
+ or "dashboard" in res["data"]):
258
+ artifacts.append(res["data"])
259
+ messages.append({"role": "tool", "tool_call_id": tc.get("id", name),
260
+ "content": json.dumps(res, default=str)[:8000]})
261
+ # budget exhausted β€” report honestly (never as success)
262
+ return {"answer": "(stopped: tool-call budget exhausted before a final answer β€” "
263
+ "narrow the question or raise max_iters)",
264
+ "artifacts": artifacts, "tool_trace": trace, "iterations": max_iters,
265
+ "exhausted": True, "usage": usage_total}
royalimports_os/harness/base.py ADDED
@@ -0,0 +1,123 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Harness core β€” the platform contract that turns 'a dashboard' into 'a product'.
2
+
3
+ Three concepts:
4
+ Connector a source adapter (Odoo / QuickBooks / Amazon / …). Declares its capabilities and
5
+ answers canonical Queries. Adding a source = subclass Connector.
6
+ Tenant one client (Royal Imports, …). Bundles the connectors they've hooked up + their scope
7
+ config (business units, excluded accounts, channel map, branding). Multi-tenant lives here.
8
+ Skill a portable dashboard/metric written ONCE against the canonical model; it declares the
9
+ capabilities it needs so a tenant only ever sees skills its sources can serve.
10
+
11
+ The UI (app.py) becomes a thin renderer over (tenant β†’ available skills). Nothing above the
12
+ connector layer knows what Odoo is.
13
+ """
14
+ from __future__ import annotations
15
+ from abc import ABC, abstractmethod
16
+ from dataclasses import dataclass, field
17
+ from typing import Callable
18
+ from .canonical import Capability, Query, Entity
19
+
20
+
21
+ class Connector(ABC):
22
+ """A data source. Implement `capabilities()` + `aggregate()` (and optionally `records()`), map the
23
+ canonical model to the source, and the source is usable by every compatible skill."""
24
+ key: str = 'connector'
25
+ name: str = 'Connector'
26
+
27
+ @abstractmethod
28
+ def capabilities(self) -> set[Capability]:
29
+ """The canonical capabilities this source can serve for the given account."""
30
+
31
+ @abstractmethod
32
+ def aggregate(self, q: Query) -> list[dict]:
33
+ """Answer an aggregate Query β†’ list of canonical dicts (keys = measures/group_by, + 'count')."""
34
+
35
+ def records(self, q: Query) -> list[dict]:
36
+ """Optional row-level fetch (canonical dicts). Default: not supported."""
37
+ raise NotImplementedError(f'{self.key}: row-level records() not implemented')
38
+
39
+ def distinct_count(self, q: Query, fieldname: str) -> int:
40
+ """Number of distinct values of a canonical field over the query's scope (e.g. active
41
+ customers). Default: derive from an aggregate group_by; connectors may override efficiently."""
42
+ from dataclasses import replace
43
+ rows = self.aggregate(replace(q, measures=(), aggregate='count', group_by=(fieldname,)))
44
+ return sum(1 for r in rows if r.get(fieldname) not in (None, False))
45
+
46
+ def supports(self, cap: Capability) -> bool:
47
+ return cap in self.capabilities()
48
+
49
+ def entities(self) -> set[Entity]:
50
+ """Canonical entities this connector can materialize (advertised by subclasses)."""
51
+ return set()
52
+
53
+
54
+ @dataclass
55
+ class Tenant:
56
+ """One client of the platform. `sources` maps a connector key β†’ Connector instance. `config` holds
57
+ the client's scope semantics (which is what used to be hard-coded in core/odoo.py):
58
+ business_units {label: source_scope_value} e.g. {'Fisch': 5, 'Royal': 6}
59
+ confirmed_statuses [...] which order states count as 'real'
60
+ excluded_customers [ids] house / inter-co accounts to drop
61
+ channels {label: match} e.g. {'Amazon': <partner>} (marketplace split)
62
+ branding {...} palette, name, logo
63
+ A skill reads tenant.config for scope and asks tenant.source(capability) for the right connector."""
64
+ key: str
65
+ name: str
66
+ sources: dict[str, Connector] = field(default_factory=dict)
67
+ config: dict = field(default_factory=dict)
68
+
69
+ def add(self, conn: Connector) -> 'Tenant':
70
+ self.sources[conn.key] = conn
71
+ return self
72
+
73
+ def capabilities(self) -> set[Capability]:
74
+ caps: set[Capability] = set()
75
+ for c in self.sources.values():
76
+ caps |= c.capabilities()
77
+ return caps
78
+
79
+ def source(self, cap: Capability, prefer: str | None = None) -> Connector:
80
+ """The connector that serves `cap` (a tenant may hook several sources; `prefer` picks one,
81
+ else first that supports it). Raises if no connected source provides the capability."""
82
+ if prefer and prefer in self.sources and self.sources[prefer].supports(cap):
83
+ return self.sources[prefer]
84
+ for c in self.sources.values():
85
+ if c.supports(cap):
86
+ return c
87
+ raise LookupError(f"tenant {self.key!r} has no source for capability {cap.value!r}")
88
+
89
+ def has(self, *caps: Capability) -> bool:
90
+ mine = self.capabilities()
91
+ return all(c in mine for c in caps)
92
+
93
+
94
+ # --------------------------------------------------------------------------- skill registry
95
+ @dataclass(frozen=True)
96
+ class Skill:
97
+ """A portable dashboard. `requires` are the capabilities it needs; the harness only offers it to a
98
+ tenant whose connected sources cover them. `render` is the UI entrypoint (given tenant + context)."""
99
+ key: str
100
+ label: str
101
+ requires: tuple[Capability, ...]
102
+ render: Callable | None = None
103
+ hq: bool = False # consolidated / company-level (ignores the business-unit filter)
104
+
105
+ def available_for(self, tenant: Tenant) -> bool:
106
+ return tenant.has(*self.requires)
107
+
108
+
109
+ _SKILLS: dict[str, Skill] = {}
110
+
111
+
112
+ def register_skill(skill: Skill) -> Skill:
113
+ _SKILLS[skill.key] = skill
114
+ return skill
115
+
116
+
117
+ def skills_for(tenant: Tenant) -> list[Skill]:
118
+ """The skills a tenant can actually run, given its connected sources β€” this is the dynamic nav."""
119
+ return [s for s in _SKILLS.values() if s.available_for(tenant)]
120
+
121
+
122
+ def all_skills() -> list[Skill]:
123
+ return list(_SKILLS.values())
royalimports_os/harness/canonical.py ADDED
@@ -0,0 +1,101 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Canonical financial data model β€” the SHARED vocabulary of the harness.
2
+
3
+ Skills speak ONLY in these terms; connectors (Odoo / QuickBooks / Amazon / …) translate them to
4
+ their own source. This is the seam that decouples the dashboards from any single system: add a new
5
+ source by writing a connector that maps its API onto these entities/fields β€” every skill that needs
6
+ only the capabilities that source provides then works, unchanged.
7
+
8
+ Design rules:
9
+ - Entities and canonical field names are source-agnostic (no `amount_untaxed`, no `sale.order`).
10
+ - A skill builds a `Query` (what business question), a connector answers it (how, per source).
11
+ - Capabilities let a skill declare what it needs and a tenant show only the dashboards its
12
+ connected sources can actually serve.
13
+ """
14
+ from __future__ import annotations
15
+ from dataclasses import dataclass, field
16
+ from enum import Enum
17
+ from typing import Any
18
+
19
+
20
+ class Capability(str, Enum):
21
+ """What a connector can serve. Skills require a set of these; a tenant's nav is the union of
22
+ what its connectors provide. (A pure-accounting source like QuickBooks has LEDGER+INVOICING+
23
+ CUSTOMERS but not SKU-level SALES velocity; Amazon has SALES but no LEDGER; Odoo has all.)"""
24
+ SALES = 'sales' # orders / order lines (SKU-level revenue, units)
25
+ INVOICING = 'invoicing' # customer invoices + credit notes (AR, returns)
26
+ LEDGER = 'ledger' # GL accounts + journal lines (P&L, balance sheet)
27
+ PAYABLES = 'payables' # vendor bills / AP
28
+ INVENTORY = 'inventory' # on-hand stock levels + valuation
29
+ PRODUCTS = 'products' # product master (SKU, category, cost)
30
+ CUSTOMERS = 'customers' # customer master (name, geo, assigned agent)
31
+
32
+
33
+ class Entity(str, Enum):
34
+ """Canonical business objects. Each connector declares which it can materialize."""
35
+ SALES_ORDER = 'sales_order'
36
+ SALES_ORDER_LINE = 'sales_order_line'
37
+ INVOICE = 'invoice' # customer invoice / credit note
38
+ JOURNAL_LINE = 'journal_line' # GL posting
39
+ GL_ACCOUNT = 'gl_account'
40
+ VENDOR_BILL = 'vendor_bill'
41
+ STOCK_LEVEL = 'stock_level'
42
+ PRODUCT = 'product'
43
+ CUSTOMER = 'customer'
44
+
45
+
46
+ # Canonical field names per entity β€” the ONLY field identifiers a skill may use. Connectors own the
47
+ # mapping to their native fields. `date` is the entity's primary time axis (used by Query windows).
48
+ FIELDS: dict[Entity, set[str]] = {
49
+ Entity.SALES_ORDER: {'date', 'amount', 'customer', 'salesperson', 'agent', 'status', 'business_unit', 'channel'},
50
+ Entity.SALES_ORDER_LINE: {'date', 'amount', 'qty', 'margin', 'product', 'category', 'customer', 'status', 'business_unit'},
51
+ Entity.INVOICE: {'date', 'amount', 'customer', 'doc_type', 'status', 'salesperson', 'business_unit'},
52
+ Entity.JOURNAL_LINE: {'date', 'amount', 'account', 'account_type', 'business_unit', 'status'},
53
+ Entity.GL_ACCOUNT: {'code', 'name', 'type'},
54
+ Entity.VENDOR_BILL: {'date', 'amount', 'vendor', 'status'},
55
+ Entity.STOCK_LEVEL: {'product', 'qty', 'value', 'location'},
56
+ Entity.PRODUCT: {'id', 'code', 'name', 'category', 'cost', 'active'},
57
+ Entity.CUSTOMER: {'id', 'name', 'city', 'state', 'agent', 'active'},
58
+ }
59
+
60
+ # Allowed filter/aggregate vocabulary (connectors translate these; they never see source operators).
61
+ OPS = frozenset({'eq', 'ne', 'in', 'not_in', 'gte', 'lte', 'gt', 'lt', 'like'})
62
+ AGGS = frozenset({'sum', 'count', 'avg', 'min', 'max'})
63
+ GRAINS = frozenset({'day', 'week', 'month', 'quarter', 'year'}) # time-bucket for a trend
64
+
65
+
66
+ @dataclass(frozen=True)
67
+ class Filter:
68
+ """A canonical predicate: (canonical_field, op, value). op ∈ OPS."""
69
+ fieldname: str
70
+ op: str
71
+ value: Any
72
+
73
+ def __post_init__(self):
74
+ if self.op not in OPS:
75
+ raise ValueError(f'unknown op {self.op!r}; allowed {sorted(OPS)}')
76
+
77
+
78
+ @dataclass(frozen=True)
79
+ class Query:
80
+ """A source-agnostic analytical request. `measures` are canonical fields aggregated by `aggregate`,
81
+ optionally grouped by canonical `group_by` fields, filtered by `filters`, over [date_from, date_to]
82
+ on the entity's `date` axis. A connector returns a list of plain dicts keyed by the canonical
83
+ measure/group names (+ 'count'). Skills post-process those dicts β€” never source rows."""
84
+ entity: Entity
85
+ measures: tuple[str, ...] = ()
86
+ aggregate: str = 'sum'
87
+ group_by: tuple[str, ...] = ()
88
+ filters: tuple[Filter, ...] = ()
89
+ date_from: str | None = None
90
+ date_to: str | None = None
91
+ grain: str | None = None # bucket the date axis into a trend; result rows carry 'period' (e.g. '2026-05')
92
+
93
+ def __post_init__(self):
94
+ if self.aggregate not in AGGS:
95
+ raise ValueError(f'unknown aggregate {self.aggregate!r}')
96
+ if self.grain is not None and self.grain not in GRAINS:
97
+ raise ValueError(f'unknown grain {self.grain!r}; allowed {sorted(GRAINS)}')
98
+ valid = FIELDS.get(self.entity, set())
99
+ for f in list(self.measures) + list(self.group_by) + [x.fieldname for x in self.filters]:
100
+ if f not in valid:
101
+ raise ValueError(f'{f!r} is not a canonical field of {self.entity.value} (have {sorted(valid)})')
royalimports_os/harness/chatlog.py ADDED
@@ -0,0 +1,102 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """harness/chatlog.py β€” durable Analyst conversations (2026-07-16).
2
+
3
+ Every Analyst conversation is a logged SESSION that survives reruns, page navigation and Space
4
+ restarts. Persistence rides the platform's only writable store (core/store.py β†’ the private HF
5
+ dataset), the SAME mechanism as users/notes β€” Odoo stays read-only.
6
+
7
+ Shape (store key 'analyst_sessions'): { username: [ session, ... ] } (newest first), each
8
+ session = {id, title, started_at, updated_at, ui:[[role,content,artifacts],...],
9
+ llm:[{role,content},...]}.
10
+ ui = the redisplay transcript; llm = the trimmed model-context transcript.
11
+
12
+ Write posture (per the design review): the LIVE turn stays in st.session_state for a snappy chat;
13
+ we persist in a BACKGROUND daemon thread (core/store.update is a full dataset commit β€” never block
14
+ the UI on it), and bound growth (MAX_SESSIONS per user, MAX_TURNS per session) so the JSON stays
15
+ small. A failed/absent store degrades to in-memory only (chat still works this session).
16
+ """
17
+ import threading
18
+ import time
19
+ import uuid
20
+
21
+ import core.store as store
22
+
23
+ KEY = "analyst_sessions"
24
+ MAX_SESSIONS = 40 # keep the last N conversations per user (bounds the JSON)
25
+ MAX_TURNS = 60 # ui/llm entries kept per conversation
26
+ TITLE_LEN = 60
27
+
28
+
29
+ def new_id():
30
+ return uuid.uuid4().hex[:12]
31
+
32
+
33
+ def new_session(sid=None):
34
+ now = time.strftime("%Y-%m-%d %H:%M:%S")
35
+ return {"id": sid or new_id(), "title": "", "started_at": now, "updated_at": now,
36
+ "ui": [], "llm": []}
37
+
38
+
39
+ def _safe(obj):
40
+ """JSON round-trip a value (chart/kpi artifacts) so only serializable content is stored;
41
+ anything that won't serialize is dropped rather than breaking the whole write."""
42
+ import json
43
+ try:
44
+ return json.loads(json.dumps(obj, default=str))
45
+ except Exception:
46
+ return None
47
+
48
+
49
+ def load_sessions(username):
50
+ """The user's conversations, newest first (lenient read β€” never raises on a store blip)."""
51
+ if not store.available():
52
+ return []
53
+ try:
54
+ all_s = store.get(KEY) or {}
55
+ except Exception:
56
+ return []
57
+ return list(all_s.get(username, []))
58
+
59
+
60
+ def touch(session, question):
61
+ """Stamp updated_at and derive a title from the first user question."""
62
+ session["updated_at"] = time.strftime("%Y-%m-%d %H:%M:%S")
63
+ if not session.get("title") and question:
64
+ t = " ".join(str(question).split())
65
+ session["title"] = t[:TITLE_LEN] + ("…" if len(t) > TITLE_LEN else "")
66
+ # bound in-memory transcript length (keep the most recent turns)
67
+ if len(session.get("ui", [])) > MAX_TURNS:
68
+ session["ui"] = session["ui"][-MAX_TURNS:]
69
+ if len(session.get("llm", [])) > MAX_TURNS:
70
+ session["llm"] = session["llm"][-MAX_TURNS:]
71
+ return session
72
+
73
+
74
+ def upsert(sessions, session):
75
+ """Return the sessions list with `session` moved to the front (deduped by id), capped."""
76
+ out = [s for s in (sessions or []) if s.get("id") != session.get("id")]
77
+ out.insert(0, session)
78
+ return out[:MAX_SESSIONS]
79
+
80
+
81
+ def _persist(username, session):
82
+ # sanitize artifacts for storage (charts are plain dicts; drop any non-serializable stragglers)
83
+ stored = dict(session)
84
+ stored["ui"] = [[role, content, [a for a in (_safe(x) for x in (arts or [])) if a is not None]]
85
+ for (role, content, arts) in session.get("ui", [])]
86
+
87
+ def _fn(data):
88
+ data = data or {}
89
+ data[username] = upsert(data.get(username, []), stored)
90
+ return data
91
+ try:
92
+ store.update(KEY, _fn)
93
+ except Exception:
94
+ pass # store blip / no token β†’ in-memory only this run
95
+
96
+
97
+ def save_async(username, session):
98
+ """Persist in a daemon thread so the chat never waits on a dataset commit."""
99
+ if not store.available():
100
+ return
101
+ threading.Thread(target=_persist, args=(username, dict(session)),
102
+ daemon=True, name="analyst-chatlog").start()
royalimports_os/harness/connectors/__init__.py ADDED
@@ -0,0 +1 @@
 
 
1
+ """Source connectors β€” one module per system (odoo, quickbooks, amazon, ...)."""
royalimports_os/harness/connectors/odoo.py ADDED
@@ -0,0 +1,147 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Odoo connector β€” the reference implementation.
2
+
3
+ ALL Odoo specifics (model names, field names, XML-RPC quirks, the read_group dot-path gotchas) live
4
+ here and nowhere else. It maps the canonical model onto the existing, battle-tested `core.odoo` data
5
+ layer (read-only, write-guarded) so nothing is re-plumbed β€” the harness just gives it a source-agnostic
6
+ face. Every other connector (QuickBooks, Amazon, …) is a sibling of this file.
7
+ """
8
+ from __future__ import annotations
9
+ import sys
10
+ from dataclasses import replace
11
+ from functools import lru_cache
12
+ from pathlib import Path
13
+ sys.path.insert(0, str(Path(__file__).resolve().parents[2])) # royalimports_os/
14
+ import core.odoo as O
15
+ from harness.canonical import Capability, Entity, Query, Filter
16
+
17
+
18
+ @lru_cache(maxsize=1)
19
+ def _product_category_map():
20
+ """product_id -> main category name (Odoo-side enrichment; reuses the existing resolver)."""
21
+ import modules.sales as _sales
22
+ return _sales._product_cat()
23
+
24
+
25
+ def _replace_group(q: Query, new_group_by):
26
+ return replace(q, group_by=new_group_by, grain=None)
27
+
28
+ _OPS = {'eq': '=', 'ne': '!=', 'in': 'in', 'not_in': 'not in',
29
+ 'gte': '>=', 'lte': '<=', 'gt': '>', 'lt': '<', 'like': 'ilike'}
30
+
31
+ # Canonical entity -> Odoo model + (canonical field -> Odoo field) + its time axis.
32
+ _MAP = {
33
+ Entity.SALES_ORDER: {
34
+ 'model': 'sale.order', 'date': 'date_order', 'dt': True,
35
+ 'f': {'amount': 'amount_untaxed', 'customer': 'partner_id', 'salesperson': 'user_id',
36
+ 'status': 'state', 'business_unit': 'team_id'}},
37
+ Entity.SALES_ORDER_LINE: {
38
+ 'model': 'sale.order.line', 'date': 'order_id.date_order', 'dt': True,
39
+ 'f': {'amount': 'price_subtotal', 'qty': 'product_uom_qty', 'margin': 'margin',
40
+ 'product': 'product_id', 'customer': 'order_partner_id',
41
+ 'status': 'order_id.state', 'business_unit': 'order_id.team_id'}},
42
+ Entity.INVOICE: {
43
+ 'model': 'account.move', 'date': 'invoice_date', 'dt': False,
44
+ 'f': {'amount': 'amount_untaxed_signed', 'customer': 'partner_id', 'doc_type': 'move_type',
45
+ 'status': 'state', 'salesperson': 'invoice_user_id', 'business_unit': 'team_id'}},
46
+ Entity.STOCK_LEVEL: {
47
+ 'model': 'stock.quant', 'date': None, 'dt': False,
48
+ 'f': {'product': 'product_id', 'qty': 'quantity', 'location': 'location_id'}},
49
+ Entity.PRODUCT: {
50
+ 'model': 'product.product', 'date': None, 'dt': False,
51
+ 'f': {'id': 'id', 'code': 'default_code', 'name': 'name', 'category': 'categ_id',
52
+ 'cost': 'standard_price', 'active': 'active'}},
53
+ Entity.CUSTOMER: {
54
+ 'model': 'res.partner', 'date': None, 'dt': False,
55
+ 'f': {'id': 'id', 'name': 'name', 'city': 'city', 'state': 'state_id',
56
+ 'agent': 'agent_ids', 'active': 'active'}},
57
+ }
58
+
59
+
60
+ class OdooConnector:
61
+ key = 'odoo'
62
+ name = 'Odoo ERP'
63
+
64
+ def __init__(self, doc_mode: str = 'order'):
65
+ self._doc_mode = doc_mode
66
+
67
+ def capabilities(self) -> set[Capability]:
68
+ return {Capability.SALES, Capability.INVOICING, Capability.LEDGER, Capability.PAYABLES,
69
+ Capability.INVENTORY, Capability.PRODUCTS, Capability.CUSTOMERS}
70
+
71
+ def entities(self) -> set[Entity]:
72
+ return set(_MAP)
73
+
74
+ def supports(self, cap: Capability) -> bool:
75
+ return cap in self.capabilities()
76
+
77
+ # ------------------------------------------------------------------ query translation
78
+ def _domain(self, q: Query, meta):
79
+ dom = []
80
+ if q.date_from and meta['date']:
81
+ dom.append((meta['date'], '>=', q.date_from + (' 00:00:00' if meta['dt'] else '')))
82
+ if q.date_to and meta['date']:
83
+ dom.append((meta['date'], '<=', q.date_to + (' 23:59:59' if meta['dt'] else '')))
84
+ for flt in q.filters:
85
+ dom.append((meta['f'].get(flt.fieldname, flt.fieldname), _OPS[flt.op], flt.value))
86
+ return dom
87
+
88
+ def aggregate(self, q: Query) -> list[dict]:
89
+ meta = _MAP[q.entity]
90
+ O.set_doc_mode(self._doc_mode)
91
+ # 'category' isn't a native line field (dot-path groupby fails) β€” enrich from the product master.
92
+ if 'category' in q.group_by:
93
+ return self._by_category(q, meta)
94
+ dom = self._domain(q, meta)
95
+ measures = [f"{meta['f'].get(m, m)}:{q.aggregate}" for m in q.measures]
96
+ groupby = [meta['f'].get(g, g) for g in q.group_by]
97
+ if q.grain and meta['date']: # time-bucket -> Odoo's date:grain groupby
98
+ groupby.append(f"{meta['date']}:{q.grain}")
99
+ rows = O.read_group(meta['model'], dom, measures, groupby, lazy=False)
100
+ return [self._canon_row(r, q, meta) for r in rows]
101
+
102
+ def _canon_row(self, r, q, meta):
103
+ rec = {}
104
+ for m in q.measures:
105
+ rec[m] = r.get(meta['f'].get(m, m)) or 0.0
106
+ for g in q.group_by:
107
+ v = r.get(meta['f'].get(g, g))
108
+ rec[g] = ({'id': O.m2o_id(v), 'name': O.m2o_name(v)} if isinstance(v, (list, tuple)) else v)
109
+ if q.grain and meta['date']:
110
+ rng = (r.get('__range') or {}).get(f"{meta['date']}:{q.grain}") or {}
111
+ rec['period'] = (rng.get('from') or '')[:7]
112
+ rec['count'] = r.get('__count') or 0
113
+ return rec
114
+
115
+ def _by_category(self, q, meta):
116
+ """Group SALES_ORDER_LINE by main product category by grouping on product then folding
117
+ product->category in-connector (source-specific enrichment lives here, not in the skill)."""
118
+ catmap = _product_category_map()
119
+ sub = _replace_group(q, tuple(g for g in q.group_by if g != 'category') + ('product',))
120
+ agg = {}
121
+ for r in self.aggregate(sub):
122
+ prod = r.get('product') or {}
123
+ cat = catmap.get(prod.get('id') if isinstance(prod, dict) else prod, '(uncategorized)')
124
+ e = agg.setdefault(cat, {'category': cat, 'count': 0})
125
+ for m in q.measures:
126
+ e[m] = e.get(m, 0.0) + (r.get(m) or 0.0)
127
+ e['count'] += r.get('count') or 0
128
+ return sorted(agg.values(), key=lambda x: -(x.get(q.measures[0]) if q.measures else 0))
129
+
130
+ def distinct_count(self, q: Query, fieldname: str) -> int:
131
+ meta = _MAP[q.entity]
132
+ O.set_doc_mode(self._doc_mode)
133
+ return O.distinct_count(meta['model'], self._domain(q, meta), meta['f'].get(fieldname, fieldname))
134
+
135
+ def records(self, q: Query) -> list[dict]:
136
+ meta = _MAP[q.entity]
137
+ O.set_doc_mode(self._doc_mode)
138
+ fields = [meta['f'].get(m, m) for m in (q.measures or tuple(meta['f']))]
139
+ rows = O.search_read(meta['model'], self._domain(q, meta), fields)
140
+ inv = {v: k for k, v in meta['f'].items()}
141
+ return [{inv.get(k, k): (O.m2o_name(v) if isinstance(v, (list, tuple)) else v)
142
+ for k, v in r.items()} for r in rows]
143
+
144
+ # convenience for tenant setup: resolve names β†’ ids for scope config
145
+ @staticmethod
146
+ def excluded_customer_ids() -> list[int]:
147
+ return list(O.excluded_partner_ids())
royalimports_os/harness/connectors/quickbooks.py ADDED
@@ -0,0 +1,37 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """QuickBooks Online connector β€” STUB (scaffolding for the next real integration).
2
+
3
+ It exists to prove the harness is source-agnostic: it declares the capabilities QBO actually has
4
+ (accounting + AR + customers, but NOT SKU-level sales velocity or warehouse inventory), so the
5
+ platform will automatically OFFER ledger/AR/customer skills on a QuickBooks-only tenant and HIDE the
6
+ SKU/inventory skills β€” no per-skill if-statements anywhere. Fill in aggregate() against the QBO
7
+ Reports/Query API (OAuth2) to make it live.
8
+ """
9
+ from __future__ import annotations
10
+ from harness.canonical import Capability, Entity, Query
11
+
12
+
13
+ class QuickBooksConnector:
14
+ key = 'quickbooks'
15
+ name = 'QuickBooks Online'
16
+
17
+ def __init__(self, realm_id: str | None = None, token: str | None = None):
18
+ self.realm_id = realm_id
19
+ self._token = token # OAuth2 access token (never logged)
20
+
21
+ def capabilities(self) -> set[Capability]:
22
+ # QBO gives you the books + receivables + the customer list; it does not model SKU-level
23
+ # order lines or physical stock the way an ERP/marketplace does.
24
+ return {Capability.LEDGER, Capability.INVOICING, Capability.PAYABLES, Capability.CUSTOMERS}
25
+
26
+ def entities(self) -> set[Entity]:
27
+ return {Entity.INVOICE, Entity.JOURNAL_LINE, Entity.GL_ACCOUNT, Entity.VENDOR_BILL, Entity.CUSTOMER}
28
+
29
+ def supports(self, cap: Capability) -> bool:
30
+ return cap in self.capabilities()
31
+
32
+ def aggregate(self, q: Query) -> list[dict]:
33
+ raise NotImplementedError(
34
+ "QuickBooksConnector.aggregate: TODO β€” translate the canonical Query to the QBO "
35
+ "Query/Reports API (GET /v3/company/{realm}/query) and map rows back to canonical dicts. "
36
+ "Entity map: INVOICE->Invoice/CreditMemo, JOURNAL_LINE->JournalEntry lines, "
37
+ "GL_ACCOUNT->Account, VENDOR_BILL->Bill, CUSTOMER->Customer.")
royalimports_os/harness/datastore.py ADDED
@@ -0,0 +1,750 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """harness/store.py β€” the tenant data store (OM-1, 2026-07-11).
2
+
3
+ Incremental Odoo β†’ DuckDB mirror: the local analytical store that saved views/dashboards (OM-3)
4
+ and the AI Analyst's tools (OM-4) query at warehouse speed, instead of hammering live XML-RPC
5
+ per question.
6
+ Plan: .claude/wiki/research/omni-adoption.md Part IV (OM-1) + [[odoo-open-source]] BUILD-NOW #1.
7
+
8
+ Design rules:
9
+ - The store is a RAW mirror (unscoped). Business scope (wholesale teams, GIFTWARE exclusion,
10
+ confirmed-only) is applied by the SEMANTIC layer at query time (OM-2) β€” one source of truth.
11
+ - Sync is CHECKPOINTED + BOUNDED + RESUMABLE (loop-library discipline): backfill paginates by
12
+ id; live mode advances a write_date cursor; every run is bounded by max_batches and safe to
13
+ kill/re-run (upserts are idempotent).
14
+ - validate() compares the store against LIVE Odoo (raw fidelity: counts + monthly sums to the
15
+ cent) β€” the store never validates itself. Odoo hard-deletes (unlink) don't move write_date;
16
+ the count checks are the drift detector for those.
17
+ - Client data: the .duckdb file lives under data/store/ (git-ignored). READ-ONLY on Odoo.
18
+ """
19
+ import datetime as dt
20
+ import json
21
+ import os
22
+ import sys
23
+ import time
24
+ from pathlib import Path
25
+
26
+ import duckdb
27
+
28
+ import core.odoo as O
29
+
30
+ _STORE_DIR = Path(__file__).resolve().parents[1] / "data" / "store"
31
+
32
+ # ⚠ MUTABLE ON PURPOSE, and read LATE by every function below (Python resolves a module global at
33
+ # CALL time), which is how `provision_tenant.py` already points a fresh instance at its own file.
34
+ # EXIT-4a (X7) formalises that: `AIOS_DUCKDB_PATH` lets a container name the file without an edit,
35
+ # `path_for()` owns the per-tenant naming convention, and `use_path()` is the SAFE way to switch β€”
36
+ # see its docstring for why a bare assignment is not.
37
+ # The DEFAULT is unchanged: <app root>/data/store/royal.duckdb.
38
+ DB_PATH = Path(os.environ.get("AIOS_DUCKDB_PATH") or (_STORE_DIR / "royal.duckdb"))
39
+
40
+
41
+ def path_for(tenant_key):
42
+ """The canonical DuckDB file for a tenant slug. `royal-imports` keeps the historical
43
+ `royal.duckdb` name so tenant #0's existing 175 MB file is not orphaned by a rename.
44
+
45
+ File-per-tenant IS the isolation model (C1c): DuckDB has no row-level security, so the only
46
+ boundary that holds is the operating system's β€” one file, one tenant.
47
+ """
48
+ key = str(tenant_key or "").strip().lower()
49
+ if key in ("", "royal-imports"):
50
+ return DB_PATH if key == "" else Path(
51
+ os.environ.get("AIOS_DUCKDB_PATH") or (_STORE_DIR / "royal.duckdb"))
52
+ safe = "".join(c if (c.isalnum() or c in "-_") else "_" for c in key)[:60]
53
+ return _STORE_DIR / f"{safe}.duckdb"
54
+
55
+
56
+ def use_path(path):
57
+ """Repoint the store at `path`, CLOSING the process singleton first.
58
+
59
+ ⚠ A bare `datastore.DB_PATH = …` is not enough and is the more dangerous half of this
60
+ operation. `_instance()` caches ONE open connection for the life of the process, so after a
61
+ plain reassignment every read keeps being served from the PREVIOUSLY opened file β€” a silent
62
+ cross-tenant read, returning real rows that belong to somebody else. Closing the connection
63
+ and clearing the readiness memo (which is also per-file) is what makes the switch honest.
64
+
65
+ ⚠ AND CLOSING THE CONNECTION IS STILL NOT ENOUGH ON ITS OWN, because `ro_con()` caches a
66
+ cursor in a `threading.local`: this function can only clear the CALLING thread's, and
67
+ `ro_con`'s liveness probe (`SELECT 1`) SUCCEEDS on a cursor whose underlying connection was
68
+ closed out from under it in some cases β€” so another thread would keep answering from the old
69
+ file with no error to notice. The generation counter below is what closes that: `ro_con`
70
+ compares the generation its cursor was opened under and reopens when it has moved, which is
71
+ a check no individual thread can forget to do.
72
+
73
+ Returns the new path. Callers: `provision_tenant.py`, and any single-tenant worker process.
74
+ NOT a per-request operation β€” one process serves one analytical store at a time, which is why
75
+ `harness.runtime` hands out the PATH rather than switching the global underneath a request.
76
+ """
77
+ global DB_PATH
78
+ with _INSTANCE_LOCK:
79
+ con = _INSTANCE.get("con")
80
+ if con is not None:
81
+ try:
82
+ con.close()
83
+ except Exception:
84
+ pass
85
+ _INSTANCE["con"] = None
86
+ _RO_TLS.__dict__.pop("cur", None)
87
+ _READY["ok"] = False
88
+ _GENERATION[0] += 1 # every other thread's cached cursor is now stale
89
+ DB_PATH = Path(path)
90
+ return DB_PATH
91
+
92
+ # Entity specs: Odoo model β†’ store table. m2o fields land as <name>_id + <name>_name.
93
+ # 'archivable' entities are pulled with active in [True, False] (the re-SKU/merge rule).
94
+ ENTITIES = {
95
+ "sale_order": {
96
+ "model": "sale.order", "archivable": False,
97
+ "fields": ["name", "date_order", "partner_id", "team_id", "user_id", "state",
98
+ "amount_untaxed", "invoice_status", "write_date"],
99
+ },
100
+ "sale_order_line": {
101
+ "model": "sale.order.line", "archivable": False,
102
+ "fields": ["order_id", "order_partner_id", "product_id", "product_uom_qty",
103
+ "price_subtotal", "margin", "purchase_price", "write_date"],
104
+ },
105
+ "res_partner": {
106
+ "model": "res.partner", "archivable": True,
107
+ # `agent` (bool, labelled "Creditor/Agent") is THE discriminator between a real sales
108
+ # AGENT and an internal SALESPERSON who merely carries commission lines β€” the commission
109
+ # table itself cannot tell them apart. `salesman_as_agent` ("convert salesman into agent")
110
+ # marks agents who came from staff; the owner ruling on whether those count as agents is
111
+ # OUTSTANDING, so both flags are synced and neither reading is baked in.
112
+ "fields": ["name", "team_id", "city", "state_id", "country_id", "active",
113
+ "agent", "salesman_as_agent", "write_date"],
114
+ "backfill_fields": ["agent", "salesman_as_agent"], # added 2026-07-28
115
+ # m2m fields land as <base>_id (the FIRST linked id β€” the Customers-module convention:
116
+ # a customer's assigned agent is agent_ids[0], ~one per customer, MECE) + <field> as a
117
+ # JSON list of ALL ids. Declared explicitly: a 2-element m2m read would otherwise be
118
+ # indistinguishable from an m2o (id, name) pair in _flatten.
119
+ "m2m": ["agent_ids"],
120
+ },
121
+ "product_product": {
122
+ "model": "product.product", "archivable": True,
123
+ "fields": ["default_code", "name", "categ_id", "type", "standard_price", "active",
124
+ "write_date"],
125
+ },
126
+ "account_move": {
127
+ "model": "account.move", "archivable": False,
128
+ "fields": ["name", "move_type", "state", "invoice_date", "invoice_date_due",
129
+ "partner_id", "amount_untaxed_signed", "amount_residual_signed",
130
+ "payment_state", "write_date"],
131
+ },
132
+ "account_move_line": {
133
+ "model": "account.move.line", "archivable": False,
134
+ # display_type/move_type/parent_state/price_subtotal/product_id added 2026-07-28 so the
135
+ # invoice-LINE grain is queryable (topic `invoice_lines`).
136
+ # ⚠ display_type values here are 'product', 'cogs', 'payment_term', 'line_note',
137
+ # 'line_section'. Commission rows attach to 'cogs' lines as well as 'product' ones
138
+ # (96,936 of 147,160 in 2025). Filtering to 'product' is a GRAIN guard: measured
139
+ # 2026-07-29, omitting it does not move revenue (non-product lines carry
140
+ # price_subtotal = 0) but it inflates commission ROW COUNTS ~80% and would corrupt any
141
+ # count-of-lines measure.
142
+ "fields": ["move_id", "account_id", "partner_id", "date", "debit", "credit",
143
+ "balance", "display_type", "move_type", "parent_state", "price_subtotal",
144
+ "product_id", "write_date"],
145
+ "backfill_fields": ["display_type", "move_type", "parent_state", "price_subtotal",
146
+ "product_id"], # added 2026-07-28
147
+ # the BU bridge: account.move carries NO business unit (every invoice sits on team 1 β€”
148
+ # see [[invoice-bu-attribution]]), so Fisch/Royal on the invoice basis is only reachable
149
+ # through the originating sale order. sale_line_id (the FIRST linked sale line) is
150
+ # lossless here: ZERO 2025 invoice lines span more than one team (measured).
151
+ "m2m": ["sale_line_ids"],
152
+ },
153
+ "account_invoice_line_agent": {
154
+ # The OCA sale-commission module: one row per (invoice line Γ— agent). This is the SECOND
155
+ # agent source β€” per-line commission attribution β€” and it names a different population
156
+ # than res_partner.agent_ids (the customer-master book). See topic `invoice_lines`.
157
+ # ⚠ TWO similarly-named FKs, do not confuse them:
158
+ # object_id -> account_move_line (THE grain; join on this)
159
+ # invoice_id -> account_move (the parent document)
160
+ "model": "account.invoice.line.agent", "archivable": False,
161
+ "fields": ["agent_id", "commission_id", "amount", "invoice_id", "object_id",
162
+ "invoice_date", "settled", "write_date"],
163
+ },
164
+ "account_account": {
165
+ # NOTE: no `active` field on account.account in this Odoo version (learned 2026-07-11 β€”
166
+ # a ('active','in',...) domain 500s); treat as non-archivable.
167
+ "model": "account.account", "archivable": False,
168
+ "fields": ["code", "name", "account_type", "write_date"],
169
+ },
170
+ }
171
+
172
+ BATCH = 2000
173
+
174
+ # Fields that are genuinely BOOLEAN. This list is load-bearing in TWO places: _cols types the
175
+ # column, and _flatten must NOT collapse a real False into NULL. Odoo returns False both for
176
+ # "empty" and for "boolean false", so a bool field missing from here silently becomes NULL β€” and
177
+ # for res_partner.agent that would erase the ONE flag separating an AGENT from a SALESPERSON
178
+ # (see [[invoice-line-agent-commission]]): every row would read "not an agent" indistinguishably
179
+ # from "unknown".
180
+ BOOL_FIELDS = ("active", "agent", "salesman_as_agent", "settled", "commission_free")
181
+
182
+
183
+ import threading as _thr
184
+
185
+ _RO_TLS = _thr.local()
186
+ _READY = {"ok": False}
187
+ _INSTANCE = {"con": None}
188
+ _INSTANCE_LOCK = _thr.Lock()
189
+ #: Bumped by `use_path()` whenever the store FILE changes. Every per-thread cursor is stamped with
190
+ #: the generation it was opened under, so a thread that never called `use_path` still cannot keep
191
+ #: reading the previous tenant's file β€” see `ro_con`. A plain "is the cursor alive?" probe cannot
192
+ #: answer that question, which is why this counter exists rather than more probing.
193
+ _GENERATION = [0]
194
+
195
+
196
+ def mark_ready():
197
+ """In-process signal that the store is fully synced (every entity live). The sync writer calls
198
+ this after an all-live pass so a reader never has to (re)discover usability by touching the
199
+ file. Belt-and-suspenders alongside the shared-instance model below."""
200
+ _READY["ok"] = True
201
+
202
+
203
+ def _instance():
204
+ """THE one process-wide read-write DuckDB connection. Readers take .cursor() off it and the
205
+ sync writer uses it too β€” DuckDB serves concurrent reads + a single writer from ONE instance
206
+ via MVCC, so a read NEVER contends with a sync pass for the file lock.
207
+
208
+ This is the fix for the 'connection error unless I refresh' bug (2026-07-16): an independent
209
+ read-only `duckdb.connect()` FAILS while the writer holds the lock ('Can't open a connection
210
+ to same database file with a different configuration than existing connections'); a cursor on
211
+ the shared instance does not. Openers must ensure the file is fully present first β€” ensure_seed
212
+ writes atomically (os.replace) and every read path guards on DB_PATH.exists() β€” so the instance
213
+ never binds a partial or absent file. (Trade-off: the app now holds the write lock for its whole
214
+ life, so a SEPARATE process β€” e.g. mcp_server.py β€” cannot open the same store file concurrently;
215
+ in production only the app runs, and that was already true during any sync pass.)"""
216
+ con = _INSTANCE["con"]
217
+ if con is not None:
218
+ return con
219
+ with _INSTANCE_LOCK:
220
+ if _INSTANCE["con"] is None:
221
+ DB_PATH.parent.mkdir(parents=True, exist_ok=True)
222
+ _INSTANCE["con"] = duckdb.connect(str(DB_PATH))
223
+ return _INSTANCE["con"]
224
+
225
+
226
+ def ro_cursor():
227
+ """A read cursor on the shared instance β€” the contention-free way to read the store."""
228
+ return _instance().cursor()
229
+
230
+
231
+ def ready():
232
+ """True once EVERY entity has completed its backfill (phase 'live'). Gate all store READS on
233
+ this: a mid-backfill store would return PARTIAL totals silently β€” worse than an error. Positive
234
+ results cache for the process (backfill never regresses). Reads via a shared-instance cursor,
235
+ so it is never mis-reported as 'not ready' just because a background sync pass is mid-flight."""
236
+ if _READY["ok"]:
237
+ return True
238
+ if not DB_PATH.exists():
239
+ return False
240
+ try:
241
+ cur = ro_cursor()
242
+ try:
243
+ phases = {r[0]: r[1] for r in
244
+ cur.execute("SELECT entity, phase FROM _sync_state").fetchall()}
245
+ finally:
246
+ cur.close()
247
+ except Exception:
248
+ return False
249
+ if all(phases.get(k) == "live" for k in ENTITIES):
250
+ _READY["ok"] = True
251
+ return True
252
+ return False
253
+
254
+
255
+ def ro_con():
256
+ """Per-thread cached read CURSOR on the shared instance for page/module store reads (the
257
+ open cost dominates repeated small queries — the Expenses retrofit measured 7.7s→1.0s). No
258
+ lock contention with the sync writer; a cursor that has gone bad is reopened once. REFUSES
259
+ until ready() β€” callers either fall back to live reads (modules) or relay a readable message
260
+ (Analyst).
261
+
262
+ ⚠ THE GENERATION CHECK IS A CORRECTNESS GUARD, NOT AN OPTIMISATION (EXIT-4a). The liveness
263
+ probe below asks "does this cursor still work?", which is a different question from "is this
264
+ cursor still pointing at the file we are supposed to be reading?". After `use_path()` switches
265
+ tenants, a cursor cached in ANOTHER thread's `threading.local` can still answer `SELECT 1` β€”
266
+ and would then serve that thread real, plausible rows from the PREVIOUS tenant's file, with no
267
+ exception anywhere. `use_path` bumps the generation; a cursor opened under an older one is
268
+ discarded here, which is the only place every thread is guaranteed to pass through."""
269
+ cur = getattr(_RO_TLS, "cur", None)
270
+ if cur is not None and getattr(_RO_TLS, "gen", None) != _GENERATION[0]:
271
+ try:
272
+ cur.close()
273
+ except Exception:
274
+ pass
275
+ cur, _RO_TLS.cur = None, None # the store moved underneath this thread
276
+ if cur is not None:
277
+ try:
278
+ cur.execute("SELECT 1") # cheap liveness probe
279
+ return cur
280
+ except Exception:
281
+ _RO_TLS.cur = None # stale β†’ reopen below
282
+ if not ready():
283
+ raise RuntimeError("the tenant data store is completing its FIRST sync on this "
284
+ "deployment β€” dashboards work meanwhile (live reads); retry "
285
+ "store queries in a few minutes")
286
+ cur = ro_cursor()
287
+ _RO_TLS.cur = cur
288
+ _RO_TLS.gen = _GENERATION[0] # stamp WHICH file this cursor belongs to
289
+ return cur
290
+
291
+
292
+ SEED_DATASET = os.environ.get("STORE_SEED_DATASET", "royal-imports/cfo-os-data")
293
+
294
+
295
+ def ensure_seed():
296
+ """Fresh/ephemeral deployment (HF Space disks reset on every rebuild): hydrate the store
297
+ from the PRIVATE dataset's seed snapshot β€” Analyst-ready in ~a minute instead of a 20-40 min
298
+ XML-RPC backfill; the auto-sync then closes the gap via write_date cursors. Also restores
299
+ the pilot views/routines when absent. Fail-quiet: no seed/token β†’ normal backfill."""
300
+ import shutil
301
+ if DB_PATH.exists():
302
+ return False
303
+ tok = os.environ.get("HF_TOKEN")
304
+ if not tok:
305
+ return False
306
+ try:
307
+ from huggingface_hub import hf_hub_download
308
+ DB_PATH.parent.mkdir(parents=True, exist_ok=True)
309
+ p = hf_hub_download(SEED_DATASET, "store_seed/royal.duckdb",
310
+ repo_type="dataset", token=tok)
311
+ # Atomic install: copy to a temp path then os.replace β€” a reader that guards on
312
+ # DB_PATH.exists() must never bind a half-written file into the shared instance.
313
+ tmp = DB_PATH.with_name(DB_PATH.name + ".tmp")
314
+ shutil.copyfile(p, tmp)
315
+ os.replace(tmp, DB_PATH)
316
+ for fn in ("views.json", "routines.json"):
317
+ tgt = DB_PATH.parent / fn
318
+ if not tgt.exists():
319
+ try:
320
+ q = hf_hub_download(SEED_DATASET, f"store_seed/{fn}",
321
+ repo_type="dataset", token=tok)
322
+ shutil.copyfile(q, tgt)
323
+ except Exception:
324
+ pass
325
+ return True
326
+ except Exception:
327
+ return False
328
+
329
+
330
+ def connect():
331
+ """A cursor on the shared instance for the SYNC WRITER (and status/validate). Returned as a
332
+ cursor so callers' con.close() frees the cursor without closing the process-wide instance;
333
+ one writer cursor is active at a time (the sync thread is sequential), readers use others."""
334
+ cur = _instance().cursor()
335
+ cur.execute("""CREATE TABLE IF NOT EXISTS _sync_state (
336
+ entity VARCHAR PRIMARY KEY, phase VARCHAR, last_id BIGINT,
337
+ cursor_wd VARCHAR, rows BIGINT, updated_at VARCHAR)""")
338
+ return cur
339
+
340
+
341
+ def _flatten(rec, fields, m2m=()):
342
+ out = {"id": rec["id"]}
343
+ for f in m2m:
344
+ v = rec.get(f) or []
345
+ base = f.removesuffix("_ids")
346
+ out[f"{base}_id"] = v[0] if v else None
347
+ out[f] = json.dumps(list(v)) if v else None
348
+ for f in fields:
349
+ v = rec.get(f)
350
+ if isinstance(v, (list, tuple)) and len(v) == 2 and isinstance(v[0], int):
351
+ out[f"{f.removesuffix('_id')}_id"] = v[0]
352
+ out[f"{f.removesuffix('_id')}_name"] = str(v[1])
353
+ elif isinstance(v, (list, tuple)):
354
+ out[f] = json.dumps(v)
355
+ elif v is False and f not in BOOL_FIELDS: # Odoo False = NULL for non-bool fields
356
+ out[f] = None
357
+ else:
358
+ out[f] = v
359
+ return out
360
+
361
+
362
+ def _cols(spec):
363
+ cols = ["id BIGINT PRIMARY KEY"]
364
+ for f in spec["fields"]:
365
+ base = f.removesuffix("_id")
366
+ if f.endswith("_id"):
367
+ cols += [f"{base}_id BIGINT", f"{base}_name VARCHAR"]
368
+ elif f in BOOL_FIELDS:
369
+ cols.append(f"{f} BOOLEAN")
370
+ elif f in ("amount_untaxed", "amount_untaxed_signed", "amount_residual_signed",
371
+ "price_subtotal", "margin", "purchase_price", "product_uom_qty",
372
+ "standard_price", "debit", "credit", "balance", "amount"):
373
+ cols.append(f"{f} DOUBLE")
374
+ else:
375
+ cols.append(f"{f} VARCHAR")
376
+ for f in spec.get("m2m") or []:
377
+ base = f.removesuffix("_ids")
378
+ cols += [f"{base}_id BIGINT", f"{f} VARCHAR"]
379
+ return cols
380
+
381
+
382
+ def _ensure_table(con, key, spec):
383
+ con.execute(f"CREATE TABLE IF NOT EXISTS {key} ({', '.join(_cols(spec))})")
384
+
385
+
386
+ def _ensure_columns(con, key, spec):
387
+ """Schema evolution: existing stores/seeds predate columns a newer spec introduces β€”
388
+ CREATE TABLE IF NOT EXISTS won't add them. Returns the newly added column names."""
389
+ have = {r[1] for r in con.execute(f"PRAGMA table_info('{key}')").fetchall()}
390
+ added = []
391
+ for c in _cols(spec):
392
+ name = c.split()[0]
393
+ if name not in have:
394
+ con.execute(f"ALTER TABLE {key} ADD COLUMN {c}")
395
+ added.append(name)
396
+ return added
397
+
398
+
399
+ def _field_cols(f):
400
+ """The store column(s) one spec field materialises into (m2o fields become _id + _name)."""
401
+ if f.endswith("_id"):
402
+ base = f.removesuffix("_id")
403
+ return [f"{base}_id", f"{base}_name"]
404
+ return [f]
405
+
406
+
407
+ def _backfill_columns(con, key, spec, log=print):
408
+ """Scalar/m2o schema evolution β€” the twin of _backfill_m2m, and for the same reason.
409
+
410
+ _ensure_columns ALTERs a new column in, but existing rows NEVER re-sync (WE moved the schema;
411
+ their write_date didn't move), so a newly added field stays NULL forever β€” silently, and NULL
412
+ is indistinguishable from a legitimately empty value. That is how a store starts answering
413
+ 'no rows match' instead of erroring. Re-pull just the affected fields for every row,
414
+ paginated by id, marked durably so a crash retries rather than declaring completion.
415
+
416
+ The fields are DECLARED in the spec (`backfill_fields`), never inferred from "which columns
417
+ did _ensure_columns just add" β€” that inference is wrong on the second run, when the ALTER has
418
+ already happened and the list comes back empty while the data is still missing. Declaring it
419
+ makes the migration re-entrant and reviewable. Leave the entry in place after it completes;
420
+ the durable marker, not the absence of the declaration, is what stops it re-running.
421
+ """
422
+ fields = [f for f in (spec.get("backfill_fields") or []) if f != "write_date"]
423
+ if not fields:
424
+ return
425
+ marker = f"{key}.cols.{','.join(sorted(fields))}"
426
+ st = con.execute("SELECT phase, last_id, rows FROM _sync_state WHERE entity=?",
427
+ [marker]).fetchone()
428
+ if st and st[0] == "done":
429
+ return
430
+ # RESUMABLE, not restart-from-zero: these passes run to ~1M rows, and a migration that loses
431
+ # an hour of work to one dropped connection gets skipped by the next person under time
432
+ # pressure. Progress is checkpointed every page; 'done' is written only at the end, so an
433
+ # interrupted run resumes at the last committed id instead of re-pulling or, worse, declaring
434
+ # completion it never reached.
435
+ last, n = (st[1] or 0, st[2] or 0) if st else (0, 0)
436
+ if last:
437
+ log(f" {key}: column backfill resuming at id {last:,} ({n:,} rows already done)")
438
+ cols = [c for f in fields for c in _field_cols(f)]
439
+ setter = ", ".join(f"{c}=?" for c in cols)
440
+ while True:
441
+ recs = O.search_read(spec["model"], _base_domain(spec) + [("id", ">", last)],
442
+ fields, limit=5000, order="id asc")
443
+ if not recs:
444
+ break
445
+ rows = [_flatten(r, fields) for r in recs]
446
+ con.executemany(f"UPDATE {key} SET {setter} WHERE id=?",
447
+ [[r.get(c) for c in cols] + [r["id"]] for r in rows])
448
+ last, n = recs[-1]["id"], n + len(recs)
449
+ con.execute("INSERT OR REPLACE INTO _sync_state VALUES (?,?,?,?,?,?)",
450
+ [marker, "running", last, None, n, time.strftime("%Y-%m-%d %H:%M:%S")])
451
+ log(f" {key}: column backfill {'+'.join(fields)} -> id {last:,} ({n:,} rows)")
452
+ con.execute("INSERT OR REPLACE INTO _sync_state VALUES (?,?,?,?,?,?)",
453
+ [marker, "done", last, None, n, time.strftime("%Y-%m-%d %H:%M:%S")])
454
+ log(f" {key}: column backfill complete -> {n:,} rows")
455
+
456
+
457
+ def _backfill_m2m(con, key, spec, log=print):
458
+ """One-time targeted backfill after a schema migration: existing rows never re-sync (their
459
+ write_date didn't move when WE added a column), so pull every record where the m2m field is
460
+ SET and update in place. Rows with the field empty keep NULL β€” correct and free. Completion
461
+ is a durable _sync_state marker ('<key>.m2m.<field>') β€” NOT 'column just added': a crash
462
+ between the ALTER and this backfill (seen live: the SSL-failed first run) must retry."""
463
+ for f in spec.get("m2m") or []:
464
+ marker = f"{key}.m2m.{f}"
465
+ if con.execute("SELECT 1 FROM _sync_state WHERE entity=?", [marker]).fetchone():
466
+ continue
467
+ base = f.removesuffix("_ids")
468
+ # PAGINATED by id. A single limit=100000 call silently TRUNCATED here: account_move_line
469
+ # has 228,067 rows with sale_line_ids set, so 56% of the BU bridge would have gone
470
+ # missing with no error β€” a partial backfill that reads as a complete one (the
471
+ # [[no-unverifiable-aggregates]] no-silent-caps rule).
472
+ rows, last, page = [], 0, 20000
473
+ while True:
474
+ recs = O.search_read(spec["model"],
475
+ _base_domain(spec) + [(f, "!=", False), ("id", ">", last)],
476
+ [f], limit=page, order="id asc")
477
+ if not recs:
478
+ break
479
+ rows += [(r[f][0], json.dumps(list(r[f])), r["id"]) for r in recs if r.get(f)]
480
+ last = recs[-1]["id"]
481
+ log(f" {key}: m2m {f} scanned to id {last:,} ({len(rows):,} rows)")
482
+ if rows:
483
+ con.executemany(f"UPDATE {key} SET {base}_id=?, {f}=? WHERE id=?", rows)
484
+ con.execute("INSERT OR REPLACE INTO _sync_state VALUES (?,?,?,?,?,?)",
485
+ [marker, "done", 0, None, len(rows), time.strftime("%Y-%m-%d %H:%M:%S")])
486
+ log(f" {key}: m2m backfill {f} -> {len(rows):,} rows updated")
487
+
488
+
489
+ def _upsert(con, key, spec, recs):
490
+ if not recs:
491
+ return
492
+ rows = [_flatten(r, spec["fields"], spec.get("m2m") or ()) for r in recs]
493
+ colnames = [c.split()[0] for c in _cols(spec)]
494
+ ids = [r["id"] for r in rows]
495
+ con.execute(f"DELETE FROM {key} WHERE id IN ({','.join(map(str, ids))})")
496
+ con.executemany(
497
+ f"INSERT INTO {key} ({', '.join(colnames)}) VALUES ({', '.join('?' for _ in colnames)})",
498
+ [[r.get(c) for c in colnames] for r in rows])
499
+
500
+
501
+ def _state(con, key):
502
+ row = con.execute("SELECT phase, last_id, cursor_wd, rows FROM _sync_state WHERE entity=?",
503
+ [key]).fetchone()
504
+ return {"phase": row[0], "last_id": row[1], "cursor_wd": row[2], "rows": row[3]} if row else None
505
+
506
+
507
+ def _save_state(con, key, phase, last_id, cursor_wd):
508
+ n = con.execute(f"SELECT count(*) FROM {key}").fetchone()[0]
509
+ con.execute("""INSERT OR REPLACE INTO _sync_state VALUES (?,?,?,?,?,?)""",
510
+ [key, phase, last_id, cursor_wd, n, time.strftime("%Y-%m-%d %H:%M:%S")])
511
+
512
+
513
+ def _base_domain(spec):
514
+ return [("active", "in", [True, False])] if spec["archivable"] else []
515
+
516
+
517
+ def sync_entity(key, max_batches=200, batch=BATCH, log=print):
518
+ """One bounded, resumable sync pass for an entity. Backfill (by id) β†’ live (by write_date).
519
+ Returns (phase, pulled_rows)."""
520
+ spec = ENTITIES[key]
521
+ con = connect()
522
+ _ensure_table(con, key, spec)
523
+ _ensure_columns(con, key, spec)
524
+ _backfill_columns(con, key, spec, log=log)
525
+ _backfill_m2m(con, key, spec, log=log)
526
+ st = _state(con, key) or {"phase": "backfill", "last_id": 0, "cursor_wd": None, "rows": 0}
527
+ fields = spec["fields"] + list(spec.get("m2m") or [])
528
+ pulled = 0
529
+ try:
530
+ if st["phase"] == "backfill":
531
+ last_id = st["last_id"] or 0
532
+ for i in range(max_batches):
533
+ dom = _base_domain(spec) + [("id", ">", last_id)]
534
+ recs = O.search_read(spec["model"], dom, ["id"] + fields,
535
+ limit=batch, order="id asc")
536
+ if not recs:
537
+ # backfill complete β†’ switch to live mode anchored at max write_date seen
538
+ wd = con.execute(f"SELECT max(write_date) FROM {key}").fetchone()[0]
539
+ _save_state(con, key, "live", last_id, wd or "1970-01-01 00:00:00")
540
+ log(f" {key}: BACKFILL COMPLETE ({st['rows'] + pulled:,} rows) -> live mode")
541
+ return "live", pulled
542
+ _upsert(con, key, spec, recs)
543
+ last_id = recs[-1]["id"]
544
+ pulled += len(recs)
545
+ _save_state(con, key, "backfill", last_id, None)
546
+ if (i + 1) % 10 == 0:
547
+ log(f" {key}: backfill …{pulled:,} rows (id≀{last_id})")
548
+ log(f" {key}: backfill PAUSED at {pulled:,} rows this run (bounded; resumes)")
549
+ return "backfill", pulled
550
+ # live mode: write_date >= cursor (idempotent overlap; upsert dedupes). MASS-STAMP
551
+ # FALLBACK (found live 2026-07-17): a batch job can stamp >batch rows with ONE
552
+ # write_date second (an Odoo recompute stamped ~88k account_move rows '2026-07-15
553
+ # 16:35:03'; stored values carry microseconds, XML-RPC strings don't) β€” then the
554
+ # write_date cursor can NEVER advance and every pass re-pulls the same first page
555
+ # forever. When a full batch leaves the cursor unchanged we switch to an ID-WALK over
556
+ # write_date >= cursor (order id asc, id > tie), checkpointed as 'cursor|<id>' in
557
+ # cursor_wd so a killed walk resumes; on completion the cursor jumps one second past
558
+ # the stamp (safe: the walk covered every row in that second, and the stamp is in the
559
+ # past β€” guarded by a 2-minute recency check).
560
+ raw = st["cursor_wd"] or "1970-01-01 00:00:00"
561
+ cursor, _pipe, _tie = raw.partition("|")
562
+ walking, tie_id = bool(_pipe), int(_tie or 0)
563
+ walk_max = cursor
564
+ for _ in range(max_batches):
565
+ if walking:
566
+ dom = _base_domain(spec) + [("write_date", ">=", cursor), ("id", ">", tie_id)]
567
+ recs = O.search_read(spec["model"], dom, ["id"] + fields,
568
+ limit=batch, order="id asc")
569
+ if not recs: # walk complete β†’ advance past the stamp
570
+ try:
571
+ wm = dt.datetime.strptime(walk_max, "%Y-%m-%d %H:%M:%S")
572
+ if wm < dt.datetime.utcnow() - dt.timedelta(minutes=2):
573
+ cursor = (wm + dt.timedelta(seconds=1)).strftime("%Y-%m-%d %H:%M:%S")
574
+ else:
575
+ cursor = walk_max
576
+ except ValueError:
577
+ cursor = walk_max
578
+ walking = False
579
+ log(f" {key}: id-walk complete -> cursor {cursor}")
580
+ break
581
+ _upsert(con, key, spec, recs)
582
+ pulled += len(recs)
583
+ tie_id = recs[-1]["id"]
584
+ walk_max = max(walk_max, max(str(r["write_date"]) for r in recs))
585
+ _save_state(con, key, "live", st["last_id"], f"{cursor}|{tie_id}")
586
+ continue
587
+ dom = _base_domain(spec) + [("write_date", ">=", cursor)]
588
+ recs = O.search_read(spec["model"], dom, ["id"] + fields,
589
+ limit=batch, order="write_date asc, id asc")
590
+ if not recs or (len(recs) == 1 and str(recs[0].get("write_date")) == cursor
591
+ and pulled == 0 and _already(con, key, recs[0])):
592
+ break
593
+ new_cursor = str(recs[-1]["write_date"])
594
+ if new_cursor == cursor and len(recs) == batch:
595
+ # full batch stuck on one second: enter the id-walk from id 0 (the write_date
596
+ # ordering shuffles ids within the stamp second, so the batch just pulled is
597
+ # NOT an id prefix β€” restart coverage by id; upserts make the overlap free)
598
+ walking, tie_id, walk_max = True, 0, cursor
599
+ _save_state(con, key, "live", st["last_id"], f"{cursor}|0")
600
+ log(f" {key}: mass-stamp second at {cursor} -> switching to id-walk")
601
+ continue
602
+ _upsert(con, key, spec, recs)
603
+ pulled += len(recs)
604
+ if new_cursor == cursor and len(recs) < batch:
605
+ cursor = new_cursor
606
+ break
607
+ cursor = new_cursor
608
+ # Checkpoint the cursor EVERY batch: a big delta (88k account_move rows after the
609
+ # 2026-07-15 recompute) means a killed pass otherwise re-pulls everything β€”
610
+ # upserts are idempotent, but the wasted RPC is real (learned 2026-07-17).
611
+ _save_state(con, key, "live", st["last_id"], cursor)
612
+ _save_state(con, key, "live", st["last_id"],
613
+ f"{cursor}|{tie_id}" if walking else cursor)
614
+ if pulled:
615
+ log(f" {key}: live sync +{pulled:,} rows (cursor {cursor}"
616
+ + (f" walking id>{tie_id}" if walking else "") + ")")
617
+ return "live", pulled
618
+ finally:
619
+ con.close()
620
+
621
+
622
+ def _already(con, key, rec):
623
+ return bool(con.execute(f"SELECT 1 FROM {key} WHERE id=?", [rec["id"]]).fetchone())
624
+
625
+
626
+ def sync_all(entities=None, max_batches=200, log=print):
627
+ out = {}
628
+ for key in (entities or list(ENTITIES)):
629
+ t0 = time.time()
630
+ phase, pulled = sync_entity(key, max_batches=max_batches, log=log)
631
+ out[key] = {"phase": phase, "pulled": pulled, "secs": round(time.time() - t0, 1)}
632
+ # A full pass that lands every entity in live mode latches process readiness in-process β€” so
633
+ # readers never race the writer's lock to (re)discover the store is usable.
634
+ if entities is None and out and all(v["phase"] == "live" for v in out.values()):
635
+ mark_ready()
636
+ return out
637
+
638
+
639
+ def reconcile_deletes(entities=None, log=print):
640
+ """Remove store rows hard-deleted in Odoo. unlink() doesn't move write_date, so the cursor
641
+ sync can never see a deletion β€” the count parity DETECTS the drift (by design); this repairs
642
+ it. Pulls the full live id set per entity in bounded id-ascending pages (id-only reads are
643
+ cheap) and deletes store ids not present live. Run from sync_runner / maintenance, not the
644
+ app's frequent passes."""
645
+ con = connect()
646
+ try:
647
+ out = {}
648
+ for key in (entities or list(ENTITIES)):
649
+ spec = ENTITIES[key]
650
+ live_ids, last = set(), 0
651
+ while True:
652
+ recs = O.search_read(spec["model"], _base_domain(spec) + [("id", ">", last)],
653
+ ["id"], limit=50000, order="id asc")
654
+ if not recs:
655
+ break
656
+ live_ids.update(r["id"] for r in recs)
657
+ last = recs[-1]["id"]
658
+ store_ids = {r[0] for r in con.execute(f"SELECT id FROM {key}").fetchall()}
659
+ dead = store_ids - live_ids
660
+ if dead:
661
+ con.execute("CREATE TEMP TABLE IF NOT EXISTS _dead(id BIGINT)")
662
+ con.execute("DELETE FROM _dead")
663
+ con.executemany("INSERT INTO _dead VALUES (?)", [[i] for i in dead])
664
+ con.execute(f"DELETE FROM {key} WHERE id IN (SELECT id FROM _dead)")
665
+ con.execute("DELETE FROM _dead")
666
+ n = con.execute(f"SELECT count(*) FROM {key}").fetchone()[0]
667
+ con.execute("UPDATE _sync_state SET rows=? WHERE entity=?", [n, key])
668
+ out[key] = len(dead)
669
+ log(f" {key}: reconciled {len(dead):,} hard-deleted rows")
670
+ return out
671
+ finally:
672
+ con.close()
673
+
674
+
675
+ def status():
676
+ con = connect()
677
+ try:
678
+ return {r[0]: {"phase": r[1], "cursor": r[3], "rows": r[4], "updated": r[5]}
679
+ for r in con.execute("SELECT * FROM _sync_state").fetchall()}
680
+ finally:
681
+ con.close()
682
+
683
+
684
+ # ------------------------------------------------------------------ parity (store vs LIVE Odoo)
685
+
686
+ def _check(name, ours, live, tol=0.01):
687
+ gap = abs((ours or 0) - (live or 0))
688
+ return {"check": name, "ok": gap <= tol, "gap": round(gap, 4), "ours": ours, "live": live}
689
+
690
+
691
+ def validate(pre=None, months=("2026-01-01", "2026-07-01")):
692
+ """Raw-fidelity parity: the store must mirror live Odoo β€” row counts per synced entity +
693
+ unscoped monetary sums over a window, to the cent. Only checks entities in LIVE phase."""
694
+ con = connect()
695
+ out = []
696
+ try:
697
+ live_state = status()
698
+ d0, d1 = months
699
+ for key, spec in ENTITIES.items():
700
+ if live_state.get(key, {}).get("phase") != "live":
701
+ continue
702
+ ours = con.execute(f"SELECT count(*) FROM {key}").fetchone()[0]
703
+ live = O.get_odoo().search_count(spec["model"], _base_domain(spec))
704
+ out.append(_check(f"store.{key} row count == live search_count", ours, live, tol=0))
705
+ if live_state.get("sale_order_line", {}).get("phase") == "live":
706
+ ours = con.execute(
707
+ "SELECT coalesce(sum(l.price_subtotal),0) FROM sale_order_line l "
708
+ "JOIN sale_order o ON o.id = l.order_id "
709
+ "WHERE o.date_order >= ? AND o.date_order < ?", [d0, d1]).fetchone()[0]
710
+ live = O.sum_field("sale.order.line",
711
+ [("order_id.date_order", ">=", f"{d0} 00:00:00"),
712
+ ("order_id.date_order", "<", f"{d1} 00:00:00")],
713
+ "price_subtotal")
714
+ out.append(_check(f"store Ξ£ line revenue [{d0}..{d1}) == live (unscoped)", ours, live))
715
+ if live_state.get("sale_order", {}).get("phase") == "live":
716
+ ours = con.execute(
717
+ "SELECT coalesce(sum(amount_untaxed),0) FROM sale_order "
718
+ "WHERE date_order >= ? AND date_order < ?", [d0, d1]).fetchone()[0]
719
+ live = O.sum_field("sale.order",
720
+ [("date_order", ">=", f"{d0} 00:00:00"),
721
+ ("date_order", "<", f"{d1} 00:00:00")], "amount_untaxed")
722
+ out.append(_check(f"store Ξ£ order amount_untaxed [{d0}..{d1}) == live (unscoped)",
723
+ ours, live))
724
+ finally:
725
+ con.close()
726
+ return out
727
+
728
+
729
+ if __name__ == "__main__":
730
+ args = sys.argv[1:]
731
+ if args and args[0] == "status":
732
+ for k, v in status().items():
733
+ print(f" {k:18s} {v['phase']:9s} rows={v['rows']:>9,} cursor={v['cursor']} ({v['updated']})")
734
+ elif args and args[0] == "reconcile":
735
+ ents = args[1].split(",") if len(args) > 1 else None
736
+ for k, n in reconcile_deletes(ents).items():
737
+ print(f" {k:18s} -{n:,} hard-deleted rows")
738
+ elif args and args[0] == "validate":
739
+ ok = True
740
+ for c in validate():
741
+ mark = "OK " if c["ok"] else "XX "
742
+ ok &= c["ok"]
743
+ print(f" {mark}{c['check'][:80]} gap={c['gap']}")
744
+ print("PARITY:", "GREEN" if ok else "RED")
745
+ sys.exit(0 if ok else 1)
746
+ else:
747
+ ents = args[0].split(",") if args else None
748
+ mb = int(args[1]) if len(args) > 1 else 200
749
+ for k, v in sync_all(ents, max_batches=mb).items():
750
+ print(f" {k:18s} {v['phase']:9s} +{v['pulled']:,} rows in {v['secs']}s")
royalimports_os/harness/evals.py ADDED
@@ -0,0 +1,134 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """harness/evals.py β€” the Analyst eval gate (OM-4, 2026-07-11).
2
+
3
+ Runs the golden set (evals/analyst_golden.yml β€” truths from the parity-proven semantic layer over
4
+ CLOSED windows) against the Analyst and scores DETERMINISTICALLY: expected number found in the
5
+ answer (tolerance-aware, handles $/commas/% forms), required substrings, required tools in the
6
+ trace, required artifacts. No LLM judge in v1 β€” the validate() culture applied to AI answers.
7
+
8
+ Gate rule (from the plan): run before any deploy that touches model/ or the Analyst; ship only on
9
+ pass-rate >= the bar (start 80%, tighten as the model/skills improve).
10
+
11
+ Usage:
12
+ python -X utf8 -c "import ssl; ssl._create_default_https_context=ssl._create_unverified_context;
13
+ import harness.evals as E; E.run()" # live (needs OPENROUTER_API_KEY)
14
+ E.run(chat_fn=scripted) # offline harness test
15
+ """
16
+ import re
17
+ from pathlib import Path
18
+
19
+ import yaml
20
+
21
+ GOLDEN = Path(__file__).resolve().parents[1] / "evals" / "analyst_golden.yml"
22
+
23
+
24
+ def _numbers(text):
25
+ """Every number in the answer, normalized: $1,234.56 -> 1234.56; '59%'/'59.0 %' -> 0.59 too."""
26
+ out = []
27
+ for m in re.finditer(r"\$?([\d,]+(?:\.\d+)?)\s*(%?)", text):
28
+ try:
29
+ v = float(m.group(1).replace(",", ""))
30
+ except ValueError:
31
+ continue
32
+ out.append(v)
33
+ if m.group(2): # percent form: also offer the fraction
34
+ out.append(v / 100.0)
35
+ return out
36
+
37
+
38
+ def _norm_ws(s):
39
+ """Collapse ALL unicode whitespace to single spaces. Models emit U+202F (narrow no-break
40
+ space) and friends inside names β€” 'Poppy\\u202fFlowers' failed a plain substring check on
41
+ two different models (the 'top_customer flake', diagnosed 2026-07-12)."""
42
+ return re.sub(r"\s+", " ", s)
43
+
44
+
45
+ def _check(item, result):
46
+ exp = item.get("expect") or {}
47
+ answer = _norm_ws(result.get("answer") or "")
48
+ reasons = []
49
+ if result.get("exhausted"):
50
+ reasons.append("tool budget exhausted")
51
+ if "value" in exp:
52
+ want = float(exp["value"])
53
+ tol = exp.get("abs", exp.get("rel", 0.001) * abs(want) or 0.01)
54
+ nums = _numbers(answer)
55
+ if exp.get("allow_pct_form"):
56
+ nums += [n for n in list(nums)]
57
+ if not any(abs(n - want) <= tol for n in nums):
58
+ reasons.append(f"expected {want:,.2f} (Β±{tol:,.2f}) not in answer")
59
+ for s in exp.get("contains", []):
60
+ if _norm_ws(s).lower() not in answer.lower():
61
+ reasons.append(f"missing substring {s!r}")
62
+ # WRONG-ANSWER guard (2026-07-28). `contains` can only prove an answer said the right thing;
63
+ # some failures are about saying a WRONG thing that a right answer never says β€” e.g. naming an
64
+ # internal salesperson as an agent ([[invoice-line-agent-commission]]). Those need a negative
65
+ # assertion or the eval passes on an answer that is confidently incorrect.
66
+ for s in exp.get("not_contains", []):
67
+ if _norm_ws(s).lower() in answer.lower():
68
+ reasons.append(f"forbidden substring {s!r} present")
69
+ used = [t["tool"] for t in result.get("tool_trace", [])]
70
+ for t in exp.get("tools", []):
71
+ if t not in used:
72
+ reasons.append(f"tool {t!r} not used (trace: {used})")
73
+ for t in exp.get("not_tools", []): # over-refusal guard: these must NOT appear
74
+ if t in used:
75
+ reasons.append(f"tool {t!r} used but forbidden here (trace: {used})")
76
+ if exp.get("artifact") == "chart":
77
+ if not any("chart" in a for a in result.get("artifacts", [])):
78
+ reasons.append("no chart artifact produced")
79
+ return (not reasons), reasons
80
+
81
+
82
+ def run(chat_fn=None, model=None, only=None, bar=0.8, verbose=True):
83
+ """Run the golden set. Returns {passed, total, pass_rate, gate_ok, results}."""
84
+ import harness.analyst as A
85
+ items = yaml.safe_load(GOLDEN.read_text(encoding="utf-8"))["items"]
86
+ if only:
87
+ items = [i for i in items if i["id"] in only]
88
+ results, passed = [], 0
89
+ import os as _os
90
+ import time as _time
91
+ _os.environ["ANALYST_PATIENT"] = "1" # batch mode: wait out rate windows, don't fail fast
92
+ infra_fails = 0
93
+ aborted = False
94
+ for item in items:
95
+ _time.sleep(6) # pace the free-tier per-minute windows (Groq/Cerebras)
96
+ kw = {"chat_fn": chat_fn} if chat_fn else {}
97
+ if model:
98
+ kw["model"] = model
99
+ try:
100
+ res = A.ask(item["ask"], telemetry_kind="analyst_eval", **kw)
101
+ ok, reasons = _check(item, res)
102
+ infra_fails = 0
103
+ except Exception as e:
104
+ res, ok, reasons = {"answer": "", "tool_trace": []}, False, [f"RUN ERROR: {e}"]
105
+ # Provider exhaustion is NOT a model failure: two in a row means the ladder is out
106
+ # of quota β€” abort and report INCONCLUSIVE instead of burning the remaining items
107
+ # into a misleading FAIL (learned 2026-07-16: 14 wasted items, 4/18 "22%").
108
+ if "all LLM providers failed" in str(e):
109
+ infra_fails += 1
110
+ if infra_fails >= 2:
111
+ aborted = True
112
+ passed += ok
113
+ results.append({"id": item["id"], "ok": ok, "reasons": reasons,
114
+ "usage": res.get("usage"), "iterations": res.get("iterations")})
115
+ if verbose:
116
+ mark = "OK " if ok else "XX "
117
+ print(f" {mark}{item['id']:18s}" + ("" if ok else f" β€” {'; '.join(reasons)[:90]}"))
118
+ if aborted:
119
+ break
120
+ if aborted:
121
+ ran = [r for r in results if not str(r["reasons"])[:60].startswith("['RUN ERROR")]
122
+ if verbose:
123
+ print(f"\nEVAL ABORTED β€” LLM providers exhausted after {len(results)} item(s); "
124
+ f"{passed}/{len(ran) or 1} of the items that actually ran passed. "
125
+ "Gate is INCONCLUSIVE (not a model failure) β€” re-run when quota recovers.")
126
+ return {"passed": passed, "total": len(items), "pass_rate": None, "gate_ok": False,
127
+ "inconclusive": True, "results": results}
128
+ rate = passed / len(items) if items else 0.0
129
+ gate = rate >= bar
130
+ if verbose:
131
+ print(f"\nEVAL: {passed}/{len(items)} = {rate:.0%} GATE({bar:.0%}): "
132
+ f"{'PASS' if gate else 'FAIL'}")
133
+ return {"passed": passed, "total": len(items), "pass_rate": rate, "gate_ok": gate,
134
+ "results": results}
royalimports_os/harness/filter_sql.py ADDED
@@ -0,0 +1,654 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """harness/filter_sql.py β€” the grid's filter/sort engine, compiled to SQL (CG-1).
2
+
3
+ THE PORT. `customer-grid/useVisibleRows.ts` evaluates the Airtable-parity filter TREE and the
4
+ multi-level sort in TypeScript, per row, in the browser. That works at customer grain (1,550 rows,
5
+ 576 KB) and does NOT work at line grain, so a line-grain table has to filter server-side. This
6
+ module is the second implementation of that contract, emitting parameterized SQL instead of
7
+ booleans. `aios_grid.clean_filter_tree` (the validator) is the third. All three are held in
8
+ lock-step by `aios-web/verify_filter_engine.py`, which runs the SAME cases through the TS engine,
9
+ this compiler (executed on DuckDB), and the validator, and compares row sets.
10
+
11
+ Design: this file is PURE β€” no model, no store, no app imports. It takes a filter tree plus a
12
+ `columns` spec and returns SQL + params. `harness/semantic.store_columns` builds the spec from a
13
+ topic; nothing else here knows what a topic is.
14
+
15
+ columns = {colId: {"sql": <SQL expression>, "type": <field type>, "aggregate": bool}}
16
+
17
+ `sql` is a trusted IDENTIFIER/expression from the semantic model (never user input); every VALUE
18
+ is a bound parameter. That split is the whole injection story.
19
+
20
+ --------------------------------------------------------------------------------------------
21
+ TRI-STATE, AND WHY IT BECOMES COMPILE-TIME PRUNING
22
+ --------------------------------------------------------------------------------------------
23
+ `evalNode` in TS returns true | false | null, where null means "this node is INACTIVE and its
24
+ parent must IGNORE it". Get that wrong and a half-typed condition returns true, makes its `or`
25
+ group true, and shows every record.
26
+
27
+ Activeness reads ONLY the rule (op, value, value2, is-the-column-known) β€” never the row. So in
28
+ SQL it is not a third truth value at all: it is decided while COMPILING. An inactive node emits
29
+ no SQL, a group with no active children emits no SQL, and a root with nothing active emits NO
30
+ PREDICATE AT ALL (`compile_filter_tree` returns None) β€” which is exactly `matchFilterTree`
31
+ returning true for every row.
32
+
33
+ --------------------------------------------------------------------------------------------
34
+ THE FOUR PLACES A NAIVE PORT SILENTLY DIVERGES
35
+ --------------------------------------------------------------------------------------------
36
+ JavaScript coerces null to 0 / "" before comparing; SQL propagates NULL and drops the row.
37
+
38
+ 1. `toNum(null)` is 0, so a NULL revenue PASSES `>= 0`. SQL: NULL >= 0 -> NULL -> dropped.
39
+ 2. `String(null)` is "", so a NULL state PASSES `is not FL`. SQL: NULL <> 'fl' -> dropped.
40
+ 3. `"" <= "2026-01-01"` is true, so a NULL date PASSES `is before`. SQL: dropped.
41
+ 4. `contains` is a LITERAL substring test. `LIKE '%v%'` treats % and _ as wildcards, so a
42
+ value containing either matches the wrong rows. Hence `strpos`, not LIKE.
43
+
44
+ Every column is therefore read through `_value_sql`, which COALESCEs to the same zero-value
45
+ JavaScript would have coerced to. The one thing that must NOT be coalesced away is the
46
+ blank test itself β€” `isBlank` looks at the RAW value, and numeric 0 is NOT blank.
47
+
48
+ --------------------------------------------------------------------------------------------
49
+ AND THE FIFTH: `default: return true` IS ACTIVE, NOT INACTIVE
50
+ --------------------------------------------------------------------------------------------
51
+ `matchFilter`'s per-type switch ends in `default: return true`, reachable whenever an operator
52
+ does not apply to the field type β€” `contains` on a currency/date field, or `gt`/`between` on a
53
+ text field. A persisted tree can hold such a pair. That rule is ACTIVE and evaluates TRUE for
54
+ every row, which is NOT the same as inactive: under `or` it makes the whole group true and shows
55
+ everything. We emit `1=1` for those, and the gate asserts the difference is observable.
56
+
57
+ --------------------------------------------------------------------------------------------
58
+ INPUT CONTRACT
59
+ --------------------------------------------------------------------------------------------
60
+ Feed this CLEANED trees (`aios_grid.clean_filter_tree`). It is deliberately faithful to the TS
61
+ engine rather than fail-closed on garbage β€” an unknown OP compiles to `1=1` exactly as TS's
62
+ `default:` branch returns true β€” because the validator's job is to reject garbage and this
63
+ module's job is to mean the same thing as the client. The gate proves the composition.
64
+
65
+ Ported ahead of their consumer (CG-2) and gated at the same fidelity as the tree:
66
+ `compile_search` (`matchSearch`) and `member_ids` (the hand-picked-members union in
67
+ `useVisibleRows`).
68
+ """
69
+ import math
70
+ import re
71
+ from collections import namedtuple
72
+
73
+ # Mirrors customer-grid/types.ts FilterOp. verify_filter_engine.py asserts this set is
74
+ # byte-identical to aios_grid.FILTER_OPS β€” three copies of a vocabulary need a drift gate.
75
+ FILTER_OPS = frozenset({
76
+ 'contains', 'doesNotContain', 'eq', 'neq', 'isEmpty', 'isNotEmpty',
77
+ 'gt', 'gte', 'lt', 'lte', 'between', 'within',
78
+ })
79
+ #: The RESERVED pseudo-column of a cohort-membership leaf. Mirrors types.ts COHORT_FIELD.
80
+ COHORT_FIELD = '__cohort__'
81
+ #: Set operators over a SET of cohorts. Mirrors types.ts COHORT_OPS; DISJOINT from FILTER_OPS.
82
+ COHORT_OPS = frozenset({'anyOf', 'allOf', 'noneOf'})
83
+ #: The single-cohort ops the leaf shipped with, kept as permanent aliases. types.ts mirrors this.
84
+ COHORT_OP_ALIASES = {'eq': 'anyOf', 'neq': 'noneOf'}
85
+ #: Mirrors types.ts MAX_COHORT_IDS.
86
+ MAX_COHORT_IDS = 20
87
+
88
+
89
+ def parse_cohort_ids(value):
90
+ """The cohorts a leaf names. Mirrors types.ts `cohortIds()` and aios_grid.parse_cohort_ids."""
91
+ out = []
92
+ for raw in ('' if value is None else str(value)).split(','):
93
+ cid = raw.strip()[:120]
94
+ if not cid or cid in out:
95
+ continue
96
+ out.append(cid)
97
+ if len(out) >= MAX_COHORT_IDS:
98
+ break
99
+ return out
100
+ # types.ts VALUE_FREE_OPS: no value control is rendered, and they must never be judged
101
+ # "inactive because the value is blank" β€” that would turn them into no-ops.
102
+ VALUE_FREE_OPS = frozenset({'isEmpty', 'isNotEmpty'})
103
+ # types.ts isNumericType()
104
+ NUMERIC_TYPES = frozenset({'currency', 'int', 'pct'})
105
+ TEXTUAL_TYPES = frozenset({'text', 'status'})
106
+
107
+ SQL_TRUE = '1=1'
108
+
109
+ _NUM_CMP = {'gt': '>', 'gte': '>=', 'lt': '<', 'lte': '<=', 'eq': '=', 'neq': '<>'}
110
+ _DATE_CMP = dict(_NUM_CMP)
111
+
112
+ #: sql = the predicate (already parenthesised); params = bound values, positional;
113
+ #: uses_aggregate = the predicate references an aggregate column, so it belongs in HAVING
114
+ #: rather than WHERE; columns_used = colIds actually referenced in the emitted SQL (an
115
+ #: always-true `1=1` branch references nothing, and must not force HAVING routing).
116
+ Compiled = namedtuple('Compiled', 'sql params uses_aggregate columns_used')
117
+
118
+
119
+ # --------------------------------------------------------------------------- value coercion
120
+
121
+ # JS `Number(string)` trims exactly this set (WhiteSpace + LineTerminator). Python's float() and
122
+ # str.strip() use a DIFFERENT set -- notably U+0085 NEL, which Python treats as whitespace and JS
123
+ # does not -- so the set is spelled out rather than left to strip().
124
+ _JS_WS = "".join(chr(c) for c in (9, 10, 11, 12, 13, 32, 160, 5760, 8232, 8233,
125
+ 8239, 8287, 12288, 65279)) + \
126
+ "".join(chr(c) for c in range(8192, 8203))
127
+ # ASCII-ONLY on purpose: `\d` in a Python str pattern matches every Unicode decimal digit, so
128
+ # `\d` here would parse the fullwidth "250" that JS rejects -- reintroducing the exact bug
129
+ # this function exists to avoid.
130
+ _JS_DEC = re.compile(r"^[+-]?(?:[0-9]+\.?[0-9]*|\.[0-9]+)(?:[eE][+-]?[0-9]+)?$")
131
+ _JS_INF = re.compile(r"^[+-]?Infinity$")
132
+ _JS_RADIX = ((re.compile(r"^0[xX][0-9a-fA-F]+$"), 16),
133
+ (re.compile(r"^0[oO][0-7]+$"), 8),
134
+ (re.compile(r"^0[bB][01]+$"), 2))
135
+
136
+
137
+ def to_num(v):
138
+ """Port of `toNum`: JS `Number(v)`, then `Number.isFinite` or 0.
139
+
140
+ NOT `float(v)`. Python's float() is MORE permissive than JS's Number() in ways that make SQL
141
+ narrow where the client shows everything -- the worse direction, because at line grain that
142
+ is a silent under-fetch sitting beside an honest-looking row_count:
143
+
144
+ "1_000" float() -> 1000.0 (PEP-515 underscores) Number() -> NaN -> 0
145
+ "250" float() -> 250.0 (any Unicode Nd) Number() -> NaN -> 0
146
+ "\\u0085" + "12" float() -> 12.0 (NEL is Python whitespace) Number() -> NaN -> 0
147
+
148
+ and LESS permissive in one family, which errs the other way:
149
+
150
+ "0x10" float() -> raises -> 0 Number() -> 16 (also 0b, 0o)
151
+
152
+ A tree carrying any of these is reachable: the React builder renders <input type="number">,
153
+ but persisted / seeded / imported view JSON goes through `clean_filter_tree`, which passes
154
+ the value string through untouched by design (it validates structure, not numeric syntax).
155
+
156
+ Agreed by construction on the rest: ""/None -> 0, "abc"/"12,000" -> 0, " 250 " -> 250,
157
+ ".5"/"5."/"+5", "1e400" -> Infinity -> 0, "-0" -> 0, and float64 truncation of
158
+ "9007199254740993" -> 9007199254740992.
159
+ """
160
+ if isinstance(v, bool):
161
+ return 1.0 if v else 0.0 # JS Number(true) === 1
162
+ if isinstance(v, (int, float)):
163
+ n = float(v)
164
+ return n if math.isfinite(n) else 0.0
165
+ if v is None:
166
+ return 0.0 # JS Number(null) === 0
167
+ s = str(v).strip(_JS_WS)
168
+ if s == "":
169
+ return 0.0 # JS Number("") === 0
170
+ if _JS_INF.match(s):
171
+ return 0.0 # Infinity is not finite -> 0
172
+ for pattern, base in _JS_RADIX:
173
+ if pattern.match(s):
174
+ return float(int(s[2:], base))
175
+ if not _JS_DEC.match(s):
176
+ return 0.0 # JS NaN -> 0
177
+ n = float(s)
178
+ return n if math.isfinite(n) else 0.0
179
+
180
+
181
+ def _value_sql(spec):
182
+ """The ONE expression for a column, used by BOTH the filter and the sort.
183
+
184
+ Two expressions would let two rows tie in the grid and not tie in SQL (dates truncated for
185
+ comparison but ordered by the raw timestamp is the obvious way to get that wrong), so the
186
+ next sort key would break the tie in one engine and not the other.
187
+
188
+ numeric: `round_even(x, 0)` reproduces `aios_grid._round`, which is Python's round() =
189
+ BANKER'S rounding. DuckDB's plain ROUND() rounds half AWAY from zero, so ROUND(1234.5)=1235
190
+ while the grid shows 1234 β€” `eq 1234` would then match on screen and miss in SQL. The grid
191
+ displays rounded values; filters must agree with what is on screen.
192
+ date: truncate to the 10-char ISO shape the payload carries, so lexical compare is
193
+ chronological and a timestamp column cannot smuggle ' 00:00:00' into the comparison.
194
+ text: lowercased, matching `String(raw ?? "").toLowerCase()`.
195
+ """
196
+ e = spec['sql']
197
+ t = spec['type']
198
+ if t in NUMERIC_TYPES:
199
+ return f"COALESCE(round_even(TRY_CAST({e} AS DOUBLE), 0), 0)"
200
+ if t == 'date':
201
+ return f"COALESCE(SUBSTR(CAST({e} AS VARCHAR), 1, 10), '')"
202
+ return f"LOWER(COALESCE(CAST({e} AS VARCHAR), ''))"
203
+
204
+
205
+ def _blank_sql(spec):
206
+ """Port of `isBlank`: null/undefined/"" are blank β€” numeric 0 is NOT.
207
+
208
+ Reads the RAW column, never `_value_sql` (which has already coalesced blanks into 0/"",
209
+ the very distinction this test exists to make).
210
+ """
211
+ e = spec['sql']
212
+ if spec['type'] in NUMERIC_TYPES:
213
+ # 0 stays non-blank: CAST(0 AS VARCHAR) is '0'. The '' arm covers a text-typed column
214
+ # holding "", which `_round` passes through unchanged into the payload.
215
+ return f"(({e}) IS NULL OR CAST({e} AS VARCHAR) = '')"
216
+ if spec['type'] == 'date':
217
+ return f"(COALESCE(SUBSTR(CAST({e} AS VARCHAR), 1, 10), '') = '')"
218
+ return f"(COALESCE(CAST({e} AS VARCHAR), '') = '')"
219
+
220
+
221
+ # --------------------------------------------------------------------------- activeness
222
+
223
+ def is_rule_active(rule, columns):
224
+ """Port of `isRuleActive` + `evalNode`'s two leaf guards. Row-independent by construction.
225
+
226
+ Unknown column -> inactive (evalNode returns null before matchFilter is ever called).
227
+ Value-free ops are always active. `between` needs BOTH bounds.
228
+
229
+ ⚠ The four VALUE-FREE ANCHOR MODES (`is before [today]`) carry no value, and the historical
230
+ "blank value = inactive" rule would IGNORE them β€” widening the result under a count nobody
231
+ would doubt. Same shape as the isEmpty/isNotEmpty no-op bug, arriving by a different door.
232
+ """
233
+ if not isinstance(rule, dict):
234
+ return False
235
+ col = rule.get('colId')
236
+ if col == COHORT_FIELD: # a cohort leaf has no column to look up
237
+ # Driven by the SET it names, so a value of "," is INACTIVE rather than active-and-
238
+ # unanswerable. Mirrors the cohort branch at the top of types.ts isRuleActive.
239
+ return bool(parse_cohort_ids(rule.get('value')))
240
+ if col not in columns:
241
+ return False
242
+ op = rule.get('op')
243
+ if op in VALUE_FREE_OPS:
244
+ return True
245
+ if op == 'within':
246
+ return _window_bounds(rule.get('dateWindow'), None, probe=True)
247
+ if rule.get('rhs') is not None:
248
+ rhs = rule['rhs']
249
+ # An rhs naming a column this table does not have is IGNORED, by the same rule as an
250
+ # unknown left column two lines above β€” for EITHER kind. Ignoring one side and hiding
251
+ # every row for the other would make the comparison asymmetric in a way nobody could
252
+ # predict. Doing it here rather than in _compile_leaf keeps activeness the single place
253
+ # that decides, which is what makes the compiled branch below provably unreachable.
254
+ if not isinstance(rhs, dict) or rhs.get('colId') not in columns:
255
+ return False
256
+ return _rhs_complete(rhs)
257
+ if rule.get('dateMode') in _ANCHOR_VALUE_FREE:
258
+ return True
259
+ value = _as_str(rule.get('value'))
260
+ if op == 'between':
261
+ v2 = rule.get('value2')
262
+ return value != '' and v2 is not None and _as_str(v2) != ''
263
+ return value != ''
264
+
265
+
266
+ def _rhs_complete(rhs):
267
+ """Port of types.ts isRhsComplete. An incomplete rhs makes the rule INACTIVE, exactly as a
268
+ half-typed value does."""
269
+ if not isinstance(rhs, dict) or not rhs.get('colId'):
270
+ return False
271
+ if rhs.get('kind') == 'measure':
272
+ return _windows().normalize(rhs.get('window')) is not None
273
+ return rhs.get('kind') == 'field'
274
+
275
+
276
+ def _windows():
277
+ """`harness.windows` imported lazily, so this module stays importable on its own.
278
+
279
+ filter_sql is otherwise PURE (no model/store/app imports) and that property is what lets
280
+ aios_grid embed the grid anywhere. windows.py is the one exception and it is a deliberate
281
+ one: the date vocabulary is a two-ENGINE contract, and a third copy of "what the past month
282
+ means" is precisely the drift verify_windows.py exists to prevent. It imports nothing but
283
+ datetime, so purity is preserved in substance.
284
+ """
285
+ from harness import windows as _w
286
+ return _w
287
+
288
+
289
+ #: Mirrors windows.ANCHOR_VALUE_FREE, resolved lazily on first use (see _windows).
290
+ class _AnchorValueFree:
291
+ def __contains__(self, mode):
292
+ return mode in _windows().ANCHOR_VALUE_FREE
293
+
294
+
295
+ _ANCHOR_VALUE_FREE = _AnchorValueFree()
296
+
297
+
298
+ def _window_bounds(spec, today, probe=False):
299
+ """Resolve a date-condition RANGE, or refuse.
300
+
301
+ `probe=True` answers only "is this a resolvable window?" without needing `today` β€” which is
302
+ what activeness has to know while compiling, before any date is in hand.
303
+
304
+ ⚠ A missing `today` on the real path RAISES rather than widening. The client's answer to the
305
+ same gap is to match nothing; the server's must be to refuse, because a server that quietly
306
+ drops a condition reports an authoritative row_count for a query nobody asked for. Same rule
307
+ as the sibling/depth caps: truncate on the client, refuse on the server.
308
+ """
309
+ w = _windows()
310
+ if probe:
311
+ return w.normalize(spec) is not None
312
+ if today is None:
313
+ raise ValueError(
314
+ 'a relative-date condition needs `today` β€” compile_filter_tree(today=...). '
315
+ 'Refusing rather than dropping the condition, which would widen the result while '
316
+ 'the count beside it still looked authoritative.')
317
+ return w.resolve(spec, today)
318
+
319
+
320
+ def _as_str(v):
321
+ """The builder stores values as raw input strings; be tolerant of a number that survived
322
+ an older payload. `str(500) != ''` is active and `to_num` reads it back, matching JS where
323
+ `500 === ""` is false."""
324
+ return '' if v is None else str(v)
325
+
326
+
327
+ # --------------------------------------------------------------------------- leaf
328
+
329
+ def _compile_leaf(rule, columns, state):
330
+ """Port of `matchFilter`. Returns (sql, params) or None when the rule is INACTIVE."""
331
+ if not is_rule_active(rule, columns):
332
+ return None
333
+ col_id = rule['colId']
334
+ op = rule.get('op')
335
+ value = _as_str(rule.get('value'))
336
+
337
+ # --- owner item 5: a COHORT leaf, before any column lookup -------------------------------
338
+ # `__cohort__` is not a column. Handled first for the same reason evalNode handles it first:
339
+ # falling through to the column path would make it INACTIVE, and inactive means every row.
340
+ if col_id == COHORT_FIELD:
341
+ return _compile_cohort(rule, state)
342
+
343
+ spec = columns[col_id]
344
+
345
+ def used(sql, params=()):
346
+ state['used'].add(col_id)
347
+ if spec.get('aggregate'):
348
+ state['aggregate'] = True
349
+ return (sql, list(params))
350
+
351
+ # --- value-free ops, BEFORE any value-based short-circuit (the no-op bug) ---------------
352
+ if op == 'isEmpty':
353
+ return used(_blank_sql(spec))
354
+ if op == 'isNotEmpty':
355
+ return used(f"NOT {_blank_sql(spec)}")
356
+
357
+ v = _value_sql(spec)
358
+ t = spec['type']
359
+
360
+ # --- CG-9: the comparand is another COLUMN, coerced through the LEFT field's type --------
361
+ # `_value_sql` on a spec whose `sql` is the RIGHT column and whose `type` is the LEFT one β€”
362
+ # exactly what TS does (`toNum(other)` for a numeric LHS, `String(other ?? "")` for a date,
363
+ # `.toLowerCase()` for text). Reading the right column through its OWN type would compare a
364
+ # rounded number against an unrounded one and disagree on the ties.
365
+ rhs_v = None
366
+ if rule.get('rhs') is not None:
367
+ # No `rspec is None` branch: `is_rule_active` already rejected an rhs naming a column
368
+ # this table does not have, so it cannot be missing here. A defensive fallback would be
369
+ # unreachable, and the negative control showed exactly that β€” deleting it changed
370
+ # nothing, which is how dead code hides behind a careful-looking line.
371
+ rspec = columns[rule['rhs']['colId']]
372
+ rhs_v = _value_sql({'sql': rspec['sql'], 'type': t})
373
+ state['used'].add(rule['rhs']['colId'])
374
+ if rspec.get('aggregate'):
375
+ state['aggregate'] = True
376
+
377
+ if t in NUMERIC_TYPES:
378
+ if rhs_v is not None:
379
+ return used(f"({v} {_NUM_CMP[op]} {rhs_v})") if op in _NUM_CMP else (SQL_TRUE, [])
380
+ if op == 'between':
381
+ a, b = to_num(value), to_num(rule.get('value2'))
382
+ return used(f"({v} >= ? AND {v} <= ?)", [min(a, b), max(a, b)])
383
+ if op in _NUM_CMP:
384
+ return used(f"({v} {_NUM_CMP[op]} ?)", [to_num(value)])
385
+ return (SQL_TRUE, []) # contains/doesNotContain on a number -> default: true
386
+
387
+ if t == 'date':
388
+ if op == 'within':
389
+ bounds = _window_bounds(rule.get('dateWindow'), state.get('today'))
390
+ if bounds is None:
391
+ return ('1=0', []) # unreachable: activeness probed it. Never widen.
392
+ lo, hi = bounds
393
+ # `{v} <> ''` is NOT redundant. `_value_sql` COALESCEs a NULL date to '', and
394
+ # '' <= any upper bound is true β€” so an open-ended window would sweep in every
395
+ # never-ordered customer. TS guards it with `if (a === "") return false`.
396
+ parts, params = [f"{v} <> ''"], []
397
+ if lo is not None:
398
+ parts.append(f"{v} >= ?")
399
+ params.append(lo)
400
+ if hi is not None:
401
+ parts.append(f"{v} <= ?")
402
+ params.append(hi)
403
+ return used('(' + ' AND '.join(parts) + ')', params)
404
+ if rhs_v is not None:
405
+ return used(f"({v} {_DATE_CMP[op]} {rhs_v})") if op in _DATE_CMP else (SQL_TRUE, [])
406
+ if op == 'between':
407
+ v2 = _as_str(rule.get('value2'))
408
+ lo, hi = (value, v2) if value < v2 else (v2, value)
409
+ return used(f"({v} >= ? AND {v} <= ?)", [lo, hi])
410
+ if op in _DATE_CMP:
411
+ anchor = _resolve_anchor(rule.get('dateMode'), value, state.get('today'))
412
+ if anchor is None:
413
+ return ('1=0', []) # unresolvable anchor -> matches nothing (TS: false)
414
+ return used(f"({v} {_DATE_CMP[op]} ?)", [anchor])
415
+ return (SQL_TRUE, []) # contains/doesNotContain on a date -> default: true
416
+
417
+ # text / status β€” case-insensitive. strpos, not LIKE: `includes` is a LITERAL substring
418
+ # test, so a value holding % or _ must not act as a wildcard.
419
+ #
420
+ # BOTH SIDES fold in SQL (`LOWER(?)`, never Python's str.lower()). Python and JS agree on
421
+ # case folding; DuckDB does NOT β€” LOWER('ISTANBUL' with a dotted capital I) drops the dot
422
+ # while both languages keep it as i + combining dot. Folding the needle in Python and the
423
+ # haystack in DuckDB therefore produced two different strings, and a value could fail to
424
+ # match ITSELF. One folder for both operands is the only version that cannot do that.
425
+ if rhs_v is not None:
426
+ # Both operands are already folded by `_value_sql` (text branch), so no LOWER(?) here.
427
+ if op == 'contains':
428
+ return used(f"(strpos({v}, {rhs_v}) > 0)")
429
+ if op == 'doesNotContain':
430
+ return used(f"(strpos({v}, {rhs_v}) = 0)")
431
+ if op == 'eq':
432
+ return used(f"({v} = {rhs_v})")
433
+ if op == 'neq':
434
+ return used(f"({v} <> {rhs_v})")
435
+ return (SQL_TRUE, [])
436
+ if op == 'contains':
437
+ return used(f"(strpos({v}, LOWER(?)) > 0)", [value])
438
+ if op == 'doesNotContain':
439
+ return used(f"(strpos({v}, LOWER(?)) = 0)", [value])
440
+ if op == 'eq':
441
+ return used(f"({v} = LOWER(?))", [value])
442
+ if op == 'neq':
443
+ return used(f"({v} <> LOWER(?))", [value])
444
+ return (SQL_TRUE, []) # gt/gte/lt/lte/between/within on text -> default: true
445
+
446
+
447
+ def _resolve_anchor(mode, value, today):
448
+ """A date condition's anchor -> one ISO date, or None. `today` may be absent ONLY for the
449
+ `exact` mode, which does not use it.
450
+
451
+ The sentinel is deliberate and load-bearing for BACK-COMPAT: every date filter that existed
452
+ before anchors is `exact`, and every caller that compiles one (store_query, store_rows)
453
+ passes no `today`. Requiring it unconditionally would turn a working filter into an
454
+ exception. A RELATIVE anchor still refuses β€” the server never widens.
455
+ """
456
+ mode = mode or 'exact'
457
+ if today is None:
458
+ if mode != 'exact':
459
+ raise ValueError(
460
+ f"the date anchor {mode!r} is relative and needs `today` β€” "
461
+ f"compile_filter_tree(today=...). Refusing rather than dropping the condition, "
462
+ f"which would widen the result under an authoritative-looking count.")
463
+ # `exact` ignores `today` entirely (verify_windows resolves it across 11 probe dates and
464
+ # gets the same answer every time), so any valid date serves as the unused argument.
465
+ today = '1970-01-01'
466
+ return _windows().resolve_anchor(mode, value, today)
467
+
468
+
469
+ def _compile_cohort(rule, state):
470
+ """Port of evalNode's cohort branch: membership in a SET of hand-curated sets of row ids.
471
+
472
+ Owner 2026-07-27: the leaf names one OR MORE cohorts and asks `anyOf` / `allOf` / `noneOf`.
473
+ A single id is a one-element set, so the shipped `is part of` compiles down this same path.
474
+
475
+ States, mirroring TS exactly:
476
+ every set present -> the union (anyOf / noneOf) or the intersection (allOf)
477
+ a set present but EMPTY -> that member contributes 1=0, because nobody is in an empty
478
+ cohort. So `allOf` collapses to 1=0 and `noneOf` to 1=1 β€”
479
+ which is what `set.has(pid)` on an empty Set does in TS.
480
+ ANY set ABSENT -> 1=0 for ALL THREE ops. The condition is unanswerable, and
481
+ unanswerable matches nothing: answering "everything" to
482
+ `is none of [a cohort we cannot see]` is the widening sin
483
+ wearing a plausible face, and answering it for the members we
484
+ DO have would silently ask a different question.
485
+
486
+ COALESCE(... , FALSE) because a NULL row id makes `id IN (...)` NULL, and `NOT NULL` is NULL
487
+ β€” SQL would drop the row where TS keeps it.
488
+ """
489
+ sets = state.get('cohort_sets')
490
+ if sets is None:
491
+ raise ValueError(
492
+ 'a cohort condition needs cohort_sets β€” compile_filter_tree(cohort_sets=...). '
493
+ 'Refusing rather than dropping it: a dropped condition widens the result while the '
494
+ 'count beside it still looks authoritative.')
495
+ id_sql = state.get('id_sql')
496
+ if not id_sql:
497
+ raise ValueError('a cohort condition needs id_sql (the row-identity column)')
498
+ op = COHORT_OP_ALIASES.get(rule.get('op'), rule.get('op'))
499
+ named = parse_cohort_ids(rule.get('value'))
500
+ members = [sets.get(cid) for cid in named]
501
+ if not named or any(m is None for m in members):
502
+ return ('1=0', [])
503
+
504
+ parts, params = [], []
505
+ for ids in members:
506
+ ids = [int(x) for x in ids]
507
+ if not ids:
508
+ parts.append('1=0')
509
+ continue
510
+ parts.append(f"COALESCE({id_sql} IN ({','.join('?' for _ in ids)}), FALSE)")
511
+ params.extend(ids)
512
+ if op == 'allOf':
513
+ return ('(' + ' AND '.join(parts) + ')', params)
514
+ union = '(' + ' OR '.join(parts) + ')'
515
+ return ((f"(NOT {union})" if op == 'noneOf' else union), params)
516
+
517
+
518
+ # --------------------------------------------------------------------------- tree
519
+
520
+ def _compile_node(node, columns, state):
521
+ """Port of `evalNode`. Returns (sql, params), or None when the node is INACTIVE.
522
+
523
+ A group combines only its ACTIVE children with its OWN conjunction; a group with no active
524
+ children is itself inactive and disappears. That is what stops a half-built group from
525
+ either zeroing the grid (`and`) or disabling it (`or`).
526
+ """
527
+ if not isinstance(node, dict):
528
+ return None
529
+ if isinstance(node.get('children'), list):
530
+ parts, params = [], []
531
+ for child in node['children']:
532
+ got = _compile_node(child, columns, state)
533
+ if got is None:
534
+ continue # inactive -> ignored entirely
535
+ parts.append(got[0])
536
+ params.extend(got[1])
537
+ if not parts:
538
+ return None
539
+ joiner = ' OR ' if node.get('conj') == 'or' else ' AND '
540
+ return ('(' + joiner.join(parts) + ')', params)
541
+ return _compile_leaf(node, columns, state)
542
+
543
+
544
+ def compile_filter_tree(nodes, conj='and', columns=None, member_ids=None, id_sql=None,
545
+ today=None, cohort_sets=None):
546
+ """Port of `matchFilterTree` (+ the `memberPids` union in `useVisibleRows`).
547
+
548
+ Returns a `Compiled`, or **None when nothing is active** β€” meaning no predicate at all, not
549
+ a false one. `matchFilterTree` returns true when the tree is inactive, so emitting anything
550
+ here would narrow rows the client would have shown.
551
+
552
+ `member_ids` reproduces `members.has(r.pid) || matchFilterTree(...)`. Note the union lives
553
+ INSIDE `if (filters.length)` and is OR'd with a tree that is true for everything when
554
+ inactive β€” so an inactive tree must emit NOTHING, never `pid IN (...)`, which would narrow
555
+ to just the hand-picked members. Same family of bug as the tri-state one, inverted.
556
+
557
+ `today` is the TENANT'S today, required by any relative date condition (`is within …`,
558
+ `is before [one month ago]`) and by nothing else β€” an `exact` date needs none, which is what
559
+ keeps every pre-anchor caller working unchanged. `cohort_sets` is `{cohortId: [pid, ...]}`
560
+ for cohort-membership leaves. Both REFUSE when needed and missing; neither defaults.
561
+ """
562
+ state = {'aggregate': False, 'used': set(),
563
+ 'today': today, 'cohort_sets': cohort_sets, 'id_sql': id_sql}
564
+ got = _compile_node({'conj': conj, 'children': list(nodes or [])}, columns or {}, state)
565
+ if got is None:
566
+ return None
567
+ sql, params = got
568
+ if member_ids:
569
+ if not id_sql:
570
+ raise ValueError('member_ids requires id_sql (the row-identity column)')
571
+ ids = [int(x) for x in member_ids]
572
+ sql = f"({sql} OR {id_sql} IN ({','.join('?' for _ in ids)}))"
573
+ params = list(params) + ids
574
+ return Compiled(sql, list(params), state['aggregate'], frozenset(state['used']))
575
+
576
+
577
+ def compile_search(q, columns):
578
+ """Port of `matchSearch`: the narrowing search box.
579
+
580
+ True if ANY human-readable (text/status) column contains `q`, case-insensitively. NUMERIC
581
+ columns are skipped on purpose so a query like "10" does not match revenue or order counts.
582
+ Dates are skipped too β€” `matchSearch` tests only text and status.
583
+
584
+ The caller ANDs this with the filter predicate, matching the TS pipeline order
585
+ (filter -> search). It is deliberately OUTSIDE the `member_ids` union: in `useVisibleRows`
586
+ the union applies to the filter step only, so a hand-picked member is still searchable away.
587
+ """
588
+ # `.strip(_JS_WS)`, not `.strip()`: the client trims with JS String.trim(), whose whitespace
589
+ # set differs from Python's in BOTH directions (Python strips U+0085 NEL and U+001C..1F,
590
+ # JS does not; JS strips U+FEFF, Python does not). A query of just those characters is
591
+ # blank to one engine and a real search to the other. Folding is left to SQL β€” see the
592
+ # LOWER(?) note in _compile_leaf.
593
+ q = (q or '').strip(_JS_WS)
594
+ if not q:
595
+ return None
596
+ parts, params, used = [], [], set()
597
+ aggregate = False
598
+ for col_id, spec in (columns or {}).items():
599
+ if spec['type'] not in TEXTUAL_TYPES:
600
+ continue
601
+ parts.append(f"(strpos({_value_sql(spec)}, LOWER(?)) > 0)")
602
+ params.append(q)
603
+ used.add(col_id)
604
+ aggregate = aggregate or bool(spec.get('aggregate'))
605
+ if not parts:
606
+ # No searchable column: `matchSearch` returns false for every row, so the search
607
+ # narrows to nothing. That is a real predicate, not an absent one.
608
+ return Compiled('1=0', [], False, frozenset())
609
+ return Compiled('(' + ' OR '.join(parts) + ')', params, aggregate, frozenset(used))
610
+
611
+
612
+ def compile_order_by(sorts, columns, tiebreak_sql=None):
613
+ """Port of `makeComparator`. Returns an ORDER BY body, or None when nothing sorts.
614
+
615
+ `tiebreak_sql` (the row-identity column) is appended as a final ASC key and is REQUIRED for
616
+ any windowed fetch. `Array.prototype.sort` is stable, so TS leaves tied rows in their
617
+ incoming order; SQL guarantees NOTHING for ties, so LIMIT/OFFSET paging over a non-total
618
+ order can repeat a row on one page and drop it from another. Appending row identity makes
619
+ the order total. It is a STRENGTHENING of the TS guarantee, not a divergence: the payload a
620
+ windowed fetch produces is itself in this order, so the client never re-sorts it differently.
621
+
622
+ BLANKS SORT LAST IN BOTH DIRECTIONS (Airtable's behavior, and the reason descending
623
+ "Last order" leads with the most recent rather than with never-ordered customers). Each key
624
+ contributes TWO terms: an always-ASC blank flag, then the value in the requested direction.
625
+ Two blanks tie on both terms, so the NEXT key decides β€” matching the comparator's
626
+ `if (aBlank && bBlank) continue`.
627
+
628
+ An unknown column is skipped: in TS its values are `undefined` on both sides, so both are
629
+ blank and the key is a no-op tiebreak.
630
+
631
+ Direction: `s.dir === "asc" ? c : -c` β€” only the exact string "asc" ascends, anything else
632
+ descends. Faithfully reproduced; do not "fix" it to default ASC.
633
+
634
+ Residual divergence, deliberate: TS uses `localeCompare`, we use LOWER() + DuckDB's binary
635
+ collation. They agree on case-insensitive ordering of ASCII and on ISO dates (unaffected by
636
+ LOWER), and differ on locale tie-breaks between case-variants of the same string ("a" vs
637
+ "A" sort adjacent under localeCompare, tie under LOWER) and on accent ordering.
638
+ """
639
+ terms = []
640
+ for s in (sorts or []):
641
+ if not isinstance(s, dict):
642
+ continue
643
+ spec = (columns or {}).get(s.get('colId'))
644
+ if spec is None:
645
+ continue
646
+ terms.append(f"CASE WHEN {_blank_sql(spec)} THEN 1 ELSE 0 END ASC")
647
+ terms.append(f"{_value_sql(spec)} {'ASC' if s.get('dir') == 'asc' else 'DESC'}")
648
+ # Append the tiebreak even when NO sort key survived (all unknown, or none given). A
649
+ # windowed fetch needs a total order regardless of what the user sorted by β€” an unsorted
650
+ # LIMIT/OFFSET is exactly as unstable as a tied one. Callers wanting pure client parity
651
+ # pass no tiebreak_sql and still get None.
652
+ if tiebreak_sql:
653
+ terms.append(f"{tiebreak_sql} ASC")
654
+ return ', '.join(terms) or None
royalimports_os/harness/measure_filter.py ADDED
@@ -0,0 +1,666 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """harness/measure_filter.py β€” answering `Where [Sales] [in the last 90 days] [>] [5,000]` (CG-8).
2
+
3
+ GATED 2026-07-26 by `royalimports_os/verify_measure_filter.py` β€” 119 assertions against the real
4
+ store (every resolved id set compared to a per-customer aggregate summed in Python from raw rows)
5
+ plus a synthetic store for the cases live data cannot produce. Negative control: 24 faithful
6
+ mutations, 24 caught. The gate found two things the reasoning below asserted and the code did not
7
+ do, both now fixed and both re-caught by the control:
8
+
9
+ - a rule with NO window resolved to ALL TIME. The error message four lines below says it
10
+ refuses to do exactly that; the line above it did it anyway.
11
+ - `sum()` over an all-NULL group is NULL, and `NULL > x` and `NULL <= x` are BOTH false, so
12
+ such a customer fell out of the two halves of the same question. Now COALESCEd to 0.
13
+
14
+ THE PROBLEM. `revenue_ytd` is a pre-computed column in `customer_data.pool()`. "Sales in the last
15
+ 90 days" is a DIFFERENT aggregate over a DIFFERENT window, and no column holds it. So a measure
16
+ condition cannot be answered by looking at the row in hand β€” it has to be a query.
17
+
18
+ THE SHAPE. Resolve each measure condition server-side to a **set of customer ids**, and let the
19
+ existing client engine treat that set as a membership test. Per condition, one query:
20
+
21
+ SELECT rp_id FROM <topic scope> WHERE <window> GROUP BY rp_id
22
+ HAVING <the metric's OWN aggregate expression> <op> ?
23
+
24
+ Three properties that fall out, none incidental:
25
+
26
+ * **No second definition of a number.** The aggregate comes from `semantic._measure_sql` β€” the
27
+ same expression the Analyst and every metric already use. This cannot disagree with "revenue"
28
+ elsewhere because it IS that computation.
29
+ * **The displayed columns are untouched.** `revenue_ytd` still comes from `pool()`. We ADD a
30
+ predicate; we do not re-source the table. So this needs no `pool()`β†’store migration, and does
31
+ not inherit that migration's parity problem.
32
+ * **Counts stay honest.** The client still holds every row, so the table stays whole-book and
33
+ `countLabel` keeps counting what it actually has (CG-3 `whole-book` mode).
34
+
35
+ ⚠ WHAT MAY BE OFFERED, AND WHY IT IS AN ALLOWLIST
36
+ `store_parity()` validates exactly five metrics against live Odoo. Every OTHER sum-metric on the
37
+ topic would get an expression with no independent check behind it β€” self-consistent with the rest
38
+ of the app and still wrong if the expression is wrong. Against "robust for EVERY numerically
39
+ driven object", that gap is load-bearing, so admission is gated on HAVING A CONTRACT and spelled
40
+ as a list. An allowlist fails CLOSED when a metric is added; iterating the topic's metrics would
41
+ fail open, silently offering an unvalidated number in the one place a number looks authoritative.
42
+
43
+ ⚠ AN UNRESOLVABLE CONDITION RAISES. It never returns "everything" or "nothing": both are answers,
44
+ and a wrong answer here is invisible β€” the grid just shows a different set of customers under a
45
+ count that looks authoritative. `ModelError` is the only safe failure.
46
+ """
47
+ from harness import semantic as _sem
48
+ from harness import windows as _win
49
+
50
+ #: metric key -> how it reads in the condition builder. ONLY metrics with an independent
51
+ #: validation contract (semantic.store_parity / a metric `validate:` block) appear here.
52
+ #: `orders` lives on sales_orders and `customers` is a property of a SET, not of a customer β€”
53
+ #: neither is a per-customer measure, so neither is offered.
54
+ #: Each entry carries its TOPIC, because admission is no longer single-topic. `orders` lives on
55
+ #: sales_orders (order-header grain) while the other three live on sales_lines β€” and the topic
56
+ #: decides the FROM clause, the scope SQL and which dim identifies a customer. Letting the caller
57
+ #: pass the topic (the old shape) meant a caller could pass a wrong one; deriving it from the
58
+ #: measure means there is nothing to get wrong.
59
+ ADMITTED = {
60
+ 'revenue': {'label': 'Sales', 'topic': 'sales_lines'},
61
+ 'margin': {'label': 'Gross Margin $', 'topic': 'sales_lines'},
62
+ 'units': {'label': 'Units', 'topic': 'sales_lines'},
63
+ # `orders` is admitted on the SAME rule as the other three and not by exception:
64
+ # `semantic.store_parity()` checks it store-vs-live per BU, and the metric carries an
65
+ # `order_count` validate contract. It is the replacement for the retired `Orders 24M`
66
+ # column β€” same question, with the period as a parameter instead of frozen at 24 months.
67
+ 'orders': {'label': 'Orders', 'topic': 'sales_orders'},
68
+ # ⚠ The first COMPOSITE measure (2026-07-27, owner's follow-up): a RATIO, not an aggregate.
69
+ # `aov` is `revenue` (line grain, sales_lines) over `orders` (header grain, sales_orders), so
70
+ # it has no single query and `_measure_sql` REFUSES it β€” which is the fail-closed shape that
71
+ # makes a mis-wired ratio raise instead of quietly computing something else. It is resolved
72
+ # by computing each BASE per customer and combining them through `semantic._post_compute`,
73
+ # the same function every other surface uses, so AOV here cannot disagree with AOV anywhere.
74
+ # `topic` names where the metric is DECLARED; the bases each resolve on their own.
75
+ 'aov': {'label': 'Avg order $', 'topic': 'sales_lines'},
76
+ # ⚠ `input_scale` SETTLES the question this measure was withheld over. The metric stays a
77
+ # FRACTION (margin/revenue, 0-1) β€” it is the semantic model's number and every other surface
78
+ # reads it β€” while the USER'S TYPED VALUE is divided by 100 on the way in, so
79
+ # `Gross Margin % > 20` compares against 0.20. The factor moves exactly once, at the input
80
+ # boundary, and never touches the metric. Scaling a COMPUTED comparand (a statistic, or the
81
+ # other side of a measure-vs-measure) would divide by 100 twice, so it is applied to the
82
+ # literal alone and the gate proves that by construction.
83
+ 'margin_pct': {'label': 'Gross Margin %', 'topic': 'sales_lines', 'input_scale': 100.0},
84
+ # ⚠ COMPANY-LEVEL (2026-07-27, owner's ask). Credit notes carry team_id = 1 for everything,
85
+ # so a business-unit total is not a thing that exists β€” `_query_parts` refuses one and the
86
+ # live domain builder refuses one. `measure_fields(team_id)` therefore stops OFFERING these
87
+ # to a BU-scoped caller rather than letting them build a condition that can only error: a
88
+ # dead entry in the picker is worse than an absent one, and hiding what cannot be answered
89
+ # is what `filterable: false` already does for columns.
90
+ 'returns': {'label': 'Returns $', 'topic': 'credit_notes'},
91
+ 'returns_pct': {'label': 'Return %', 'topic': 'credit_notes', 'input_scale': 100.0},
92
+ }
93
+
94
+ #: The metrics `semantic.store_parity()` actually exercises against live Odoo. ADMITTED must be
95
+ #: a SUBSET of this β€” that is the whole admission rule ("does this measure carry an independent
96
+ #: validation contract"), and stating it as data lets the gate assert it instead of trusting the
97
+ #: comment above. Adding a measure here without adding it to store_parity's loop fails the gate.
98
+ PARITY_CHECKED = frozenset({'revenue', 'margin', 'units', 'orders', 'customers',
99
+ 'returns', 'invoiced'})
100
+
101
+ #: numeric comparison ops, mirroring filter_sql._NUM_CMP. `between` is expressed as two
102
+ #: conditions by the caller rather than a third operator shape.
103
+ OPS = {'gt': '>', 'gte': '>=', 'lt': '<', 'lte': '<=', 'eq': '=', 'neq': '<>'}
104
+
105
+ #: CG-9 β€” the ops allowed when the right-hand side is ANOTHER MEASURE. `eq`/`neq` are withheld,
106
+ #: and not out of caution: both sides are sums of floats the user never sees, so `=` would match
107
+ #: nobody and `β‰ ` would match everybody, and both would LOOK like they worked. CG-8 could offer
108
+ #: `=` honestly because the other side was a number a person typed. Mirrors types.ts
109
+ #: MEASURE_PAIR_OPS; the gate asserts the two lists are the same.
110
+ PAIR_OPS = ('gte', 'gt', 'lte', 'lt')
111
+
112
+ #: The statistics a condition may compare against β€” "who is in the top quartile" (owner,
113
+ #: 2026-07-27). Each is computed over the CALLER'S POOL for the same measure and window.
114
+ #: The labels are the sentence the user reads; the client mirrors them.
115
+ STATS = {
116
+ 'avg': 'the average',
117
+ 'median': 'the median',
118
+ 'p10': 'the bottom 10%',
119
+ 'p25': 'the bottom quartile',
120
+ 'p75': 'the top quartile',
121
+ 'p90': 'the top 10%',
122
+ }
123
+ # ⚠ The owner's OWN words, and short enough to fit: "the 75th percentile (top quartile)" clipped
124
+ # to "the 75th perce" in the value select, and a truncated parenthetical reads as a rendering
125
+ # bug. `> the top quartile` is also how a person says it out loud, so the shorter label is the
126
+ # better one twice over. Seen in a live screenshot of a run whose every assertion was green.
127
+
128
+ #: ⚠ THE POPULATION IS THE CALLER'S POOL, INCLUDING ZEROS. A customer who bought nothing has a
129
+ #: measure of 0, not of nothing β€” that is the convention the zero-group already establishes
130
+ #: everywhere else, and "the median customer" means the median of the customers you can see.
131
+ #: Two consequences worth knowing before reading a result:
132
+ #: Β· it is a PERMISSION boundary as well as a definition β€” a scoped user's median is theirs,
133
+ #: and a statistic over the whole book would leak the shape of data they cannot open;
134
+ #: Β· on a measure most customers score 0 on, the 75th percentile IS 0, so "top quartile"
135
+ #: degenerates to "anyone with any activity at all". True, and surprising.
136
+
137
+
138
+ def percentile(values, q):
139
+ """The `q` quantile of `values` by LINEAR INTERPOLATION between closest ranks β€” the R-7
140
+ convention, which is what Excel's PERCENTILE.INC computes. `q` may also be a STATS key.
141
+
142
+ Pinned to one convention on purpose: nearest-rank and interpolation disagree on any even
143
+ population (the median of [1,2,3,4] is 2.5 here and 2 or 3 under nearest-rank), and the gate
144
+ checks it against hand-computed values rather than against `statistics.median` β€” a reference
145
+ borrowed from a library only proves we match that library.
146
+
147
+ Returns None for an empty population: there is no median of nobody, and 0 would be a number
148
+ the caller could act on.
149
+ """
150
+ if isinstance(q, str):
151
+ if q == 'avg':
152
+ xs = list(values)
153
+ return (sum(xs) / len(xs)) if xs else None
154
+ if q not in STATS:
155
+ raise _sem.ModelError(
156
+ f"{q!r} is not a statistic. Offered: {sorted(STATS)}")
157
+ q = 0.5 if q == 'median' else int(q[1:]) / 100.0
158
+ xs = sorted(float(v) for v in values)
159
+ if not xs:
160
+ return None
161
+ if len(xs) == 1:
162
+ return xs[0]
163
+ pos = (len(xs) - 1) * float(q)
164
+ lo = int(pos)
165
+ hi = min(lo + 1, len(xs) - 1)
166
+ return xs[lo] + (xs[hi] - xs[lo]) * (pos - lo)
167
+
168
+
169
+ #: The same six comparisons as OPS, as Python callables β€” used wherever the comparison happens in
170
+ #: Python rather than in a HAVING (a composite measure, or a two-measure pair).
171
+ _CMP = {'gt': lambda a, b: a > b, 'gte': lambda a, b: a >= b,
172
+ 'lt': lambda a, b: a < b, 'lte': lambda a, b: a <= b,
173
+ 'eq': lambda a, b: a == b, 'neq': lambda a, b: a != b}
174
+
175
+
176
+ def is_company_level(key):
177
+ """True when this measure cannot be scoped to a business unit.
178
+
179
+ Checked over every BASE the measure decomposes into, not just the declared topic: `Return %`
180
+ names credit_notes but also reads customer_invoices, and one un-scopable half is enough to
181
+ make the whole ratio un-scopable.
182
+ """
183
+ bases, _ = _sem._expand_measures([key])
184
+ tps = _sem.topics()
185
+ for b in bases:
186
+ store = (tps.get(_sem.metrics()[b].get('topic')) or {}).get('store') or {}
187
+ if not store.get('team_col'):
188
+ return True
189
+ return False
190
+
191
+
192
+ def measure_fields(team_id=None):
193
+ """The measures the condition builder may offer, in the payload's field shape.
194
+
195
+ These are FILTERABLE BUT NOT DISPLAYABLE β€” a measure is a question about a window, not a
196
+ column of the table, and rendering it as a column would silently pick a window for the user.
197
+
198
+ ⚠ `team_id` NARROWS THE OFFER. A COMPANY-LEVEL measure (returns, and anything else built on
199
+ documents that carry no trustworthy BU tag) cannot be answered for one business unit β€”
200
+ `_query_parts` refuses, and refusing is right. But a picker entry whose every use ends in an
201
+ error is worse than no entry, so a BU-scoped caller is not offered them at all. Same
202
+ principle as `filterable: false` on a column: do not offer what cannot be answered.
203
+ """
204
+ mts = _sem.metrics()
205
+ out = []
206
+ for key, spec in ADMITTED.items():
207
+ m = mts.get(key)
208
+ if not m:
209
+ continue
210
+ if team_id and is_company_level(key):
211
+ continue
212
+ out.append({'key': key, 'label': spec['label'],
213
+ 'type': _sem._FORMAT_TYPE.get(m.get('format'), 'currency')})
214
+ return out
215
+
216
+
217
+ def topic_for(key):
218
+ """The topic a measure is answered on. Raises for anything not admitted."""
219
+ spec = ADMITTED.get(key)
220
+ if not spec:
221
+ raise _sem.ModelError(
222
+ f"{key!r} is not an admitted measure. Offered: {sorted(ADMITTED)} β€” a measure "
223
+ f"without an independent validation contract is not filterable (see this module's "
224
+ f"docstring).")
225
+ return spec['topic']
226
+
227
+
228
+ def _entity_sql(topic):
229
+ """(from-clause, the id column customers are grouped by, the store binding) for a topic."""
230
+ t = _sem.topics().get(topic)
231
+ if not t or 'store' not in t:
232
+ raise _sem.ModelError(f"topic {topic!r} has no store binding")
233
+ s = t['store']
234
+ dims = s.get('dims') or {}
235
+ # sales_lines calls it `order_partner` (the line's parent order's partner); sales_orders
236
+ # calls the same thing `partner`. One name per topic, both meaning "the customer this row
237
+ # belongs to" β€” which is the only dim this module ever groups by.
238
+ dim = dims.get('order_partner') or dims.get('partner')
239
+ if not dim:
240
+ raise _sem.ModelError(f"topic {topic!r} has no customer dim to group by")
241
+ return f"{s['table']} {s['alias']} {s.get('join', '')}", dim['col'], s
242
+
243
+
244
+ def is_composite(key):
245
+ """True for a measure with no single aggregate expression β€” a `ratio` or a `derived`.
246
+
247
+ These are answered by computing each BASE per customer and combining in Python; they can
248
+ never be a HAVING, which is why `_measure_sql` refusing them is a feature.
249
+ """
250
+ return _sem.metrics().get(key, {}).get('agg') in ('ratio', 'derived')
251
+
252
+
253
+ def _base_topic(key):
254
+ """The topic a BASE metric aggregates on β€” read from the METRIC, never from ADMITTED.
255
+
256
+ A composite decomposes into bases that need not themselves be offerable (and whose topics may
257
+ differ from each other: `aov`'s numerator is on sales_lines and its denominator on
258
+ sales_orders). Resolving the topic from the allowlist would make that undecidable.
259
+ """
260
+ m = _sem.metrics().get(key)
261
+ if not m:
262
+ raise _sem.ModelError(f"unknown metric {key!r}")
263
+ topic = m.get('topic')
264
+ if not topic:
265
+ raise _sem.ModelError(f"metric {key!r} declares no topic to aggregate on")
266
+ return topic
267
+
268
+
269
+ def _query_parts(key, window, today, team_id):
270
+ """The FROM / GROUP-BY id / aggregate / WHERE for one (measure, window), shared by the
271
+ one-sided and the two-sided resolver so there is one definition of the scope."""
272
+ topic = _base_topic(key)
273
+ # NO DEFAULT WINDOW. A rule that carries no window is not "all time" β€” it is a rule whose
274
+ # window we do not know, and answering it over all time WIDENS the condition while the count
275
+ # beside it still looks authoritative. `all_time` is a legitimate ANSWER the user can pick;
276
+ # it is not a legitimate GUESS. (This line shipped as `rule.get('window') or {...}` and the
277
+ # gate caught it contradicting the error message two lines below.)
278
+ bounds = _win.resolve(window, today)
279
+ if bounds is None:
280
+ raise _sem.ModelError(
281
+ f"unresolvable window {window!r} β€” refusing rather than defaulting to all "
282
+ f"time, which would widen the condition while the count still looked authoritative")
283
+ date_from, date_to = bounds
284
+
285
+ frm, id_col, s = _entity_sql(topic)
286
+ m = _sem.metrics()[key]
287
+ # COALESCE, because SQL's sum() over an all-NULL group is NULL, and `NULL > x` and
288
+ # `NULL <= x` are BOTH false: the customer would vanish from the two halves of the same
289
+ # question, so `margin <= 0` and `margin > 0` would not cover the book. The zero-group below
290
+ # cannot rescue them either β€” they DO have rows in the window, just no values. Python's sum
291
+ # of the same rows is 0.0, and 0 is what the grid would show, so 0 is the answer.
292
+ agg = f"COALESCE({_sem._measure_sql(m, s['alias'])}, 0)"
293
+
294
+ where, params = [_sem.render_scope(s['scope_sql'].strip())], []
295
+ if date_from:
296
+ where.append(f"CAST({s['date_col']} AS TIMESTAMP) >= ?")
297
+ params.append(f"{date_from} 00:00:00")
298
+ if date_to:
299
+ where.append(f"CAST({s['date_col']} AS TIMESTAMP) <= ?")
300
+ params.append(f"{date_to} 23:59:59")
301
+ if team_id:
302
+ if not s.get('team_col'):
303
+ raise _sem.ModelError(f"topic {topic!r} is company-level β€” it has no business-unit "
304
+ f"filter, so a BU-scoped caller cannot be answered safely")
305
+ where.append(f"{s['team_col']} = ?")
306
+ params.append(int(team_id))
307
+ return frm, id_col, agg, ' AND '.join(where), params
308
+
309
+
310
+ def _scaled(key, value):
311
+ """A TYPED value in the units the metric actually holds.
312
+
313
+ `Gross Margin % > 20` is typed in percentage points and the metric is a fraction, so the 20
314
+ becomes 0.20. Applied to a LITERAL only β€” a statistic and the other side of a
315
+ measure-vs-measure are already on the metric's scale, and dividing them too would apply the
316
+ factor twice for a result that still looks plausible.
317
+ """
318
+ scale = ADMITTED.get(key, {}).get('input_scale')
319
+ return (value / scale) if scale else value
320
+
321
+
322
+ def resolve_stat(rule, today, team_id=None, allowed_pids=None):
323
+ """`Where [Sales] [this year] [>] [the 75th percentile]` β€” the top quartile of the pool.
324
+
325
+ One totals query for the measure, the statistic taken over the CALLER'S POOL (see the STATS
326
+ note on why the population is the pool and what that costs), then a comparison per customer.
327
+ """
328
+ stat = (rule.get('rhs') or {}).get('stat')
329
+ if stat not in STATS:
330
+ raise _sem.ModelError(f"{stat!r} is not a statistic. Offered: {sorted(STATS)}")
331
+ key = rule.get('colId')
332
+ topic_for(key)
333
+ op = rule.get('op')
334
+ if op not in PAIR_OPS:
335
+ raise _sem.ModelError(
336
+ f"{op!r} cannot compare a measure with a statistic ({sorted(PAIR_OPS)}). Equality "
337
+ f"against a computed cut-off matches nobody except by coincidence.")
338
+ pool = _pool_of(allowed_pids)
339
+ if pool is None:
340
+ raise _sem.ModelError(
341
+ "a statistic needs a population to be computed over, and none was supplied β€” "
342
+ "refusing rather than taking it over whichever customers happen to have rows, "
343
+ "which would be a different question with an authoritative-looking answer")
344
+ con = _sem._store_con()
345
+ try:
346
+ totals = _totals(con, key, rule.get('window'), today, team_id)
347
+ finally:
348
+ con.close()
349
+ # ⚠ THE POPULATION IS THE CUSTOMERS WHO HAVE THIS MEASURE AT ALL β€” owner's decision,
350
+ # 2026-07-27: *"top quartile of customers who actually has returned something"*.
351
+ #
352
+ # It was the whole pool including zeros, which is the convention everywhere else and was
353
+ # wrong HERE. The pool carries lapsed customers, so ~30% of it sits at zero for any 12-month
354
+ # window β€” that dragged the 25th percentile to exactly 0.0, made `< the bottom quartile`
355
+ # match NOBODY, and would have been worse for Return %, where most customers never return
356
+ # anything and "the top quartile" would have degenerated to "anyone with a single refund".
357
+ #
358
+ # NON-ZERO, not has-rows: a zero Return % means nothing was returned, which is precisely the
359
+ # customer the owner is excluding. It also reads as one sentence β€” the question is asked
360
+ # among customers who have that measure β€” and it generalises: "the median Sales" means the
361
+ # median among customers who sold something.
362
+ #
363
+ # The ANSWER is restricted to the same population, so `< the bottom quartile` returns a
364
+ # quarter of the returners rather than every customer who never returned.
365
+ population = {pid: v for pid, v in totals.items() if pid in pool and v}
366
+ cut = percentile(list(population.values()), stat)
367
+ if cut is None: # nobody has this measure -> nobody qualifies
368
+ return set()
369
+ return {pid for pid, v in population.items() if _CMP[op](v, cut)}
370
+
371
+
372
+ def _pool_of(allowed_pids):
373
+ """The caller's visible pool as a set of plain ints, or None.
374
+
375
+ ⚠ The pool arrives from `pool()`/a cohort and may hold numpy ints or id strings; the store
376
+ returns BIGINTs. A set difference across two int types silently finds nothing in common,
377
+ which here would look like "no lapsed customers" rather than an error.
378
+ """
379
+ return None if allowed_pids is None else {int(p) for p in allowed_pids if p is not None}
380
+
381
+
382
+ def rule_complete(rule):
383
+ """Is this rule's right-hand side FULLY present β€” a question that has actually been asked?
384
+
385
+ THE GAP THIS CLOSES (2026-07-27, the cold-container race). The component's view state
386
+ passes through the host while the user β€” or the client's own initialisation β€” is mid-edit,
387
+ and a rule can transiently arrive with its window, statistic or value missing. `to_num(None)`
388
+ is 0 by the engines' shared contract, so resolution used to ANSWER that half-asked question
389
+ as `<op> 0`: the id landed in `measureSets`, the client read the rule as answered and
390
+ cleared its pending marker, and the grid stood behind a count for a question nobody asked.
391
+ Refusing instead leaves the id OUT of the payload, which the client renders as PENDING
392
+ ("Calculating…") β€” the honest state for a question still being typed. Exactly the class the
393
+ module docstring already names: an unresolvable condition never becomes an answer.
394
+
395
+ INCOMPLETE is distinct from MALFORMED, on purpose. An ABSENT window / stat / value is a
396
+ normal transient (the caller skips it silently β€” mid-edit is not a telemetry event); a
397
+ PRESENT-but-bogus one is a real defect and still RAISES downstream (`unresolvable window`,
398
+ `not a statistic`, `not admitted`). `value` presence is textual, not numeric: "0" is a
399
+ typed answer, while parseability stays `to_num`'s business so the two engines keep one
400
+ contract for garbage input.
401
+ """
402
+ if not isinstance(rule, dict) or not rule.get('op'):
403
+ return False
404
+ if rule.get('window') is None:
405
+ return False
406
+ rhs = rule.get('rhs')
407
+ if isinstance(rhs, dict) and rhs.get('kind') == 'measure':
408
+ return bool(rhs.get('colId')) and rhs.get('window') is not None
409
+ if isinstance(rhs, dict) and rhs.get('kind') == 'stat':
410
+ return bool(rhs.get('stat'))
411
+ v = rule.get('value')
412
+ return v is not None and str(v).strip() != ''
413
+
414
+
415
+ def resolve_rule(rule, today, team_id=None, allowed_pids=None):
416
+ """One measure condition -> the SET OF CUSTOMER IDS satisfying it.
417
+
418
+ `rule` = {colId: <measure key>, window: <window spec>, op: <numeric op>, value: <str|num>},
419
+ or β€” CG-9 β€” the same with `rhs: {kind:'measure', colId, window}` instead of a value, which
420
+ hands off to `resolve_pair`.
421
+
422
+ ⚠ Customers with NO rows in the window are absent from the GROUP BY, so a condition like
423
+ `revenue < 100` would miss them even though 0 < 100 is true. That is the single most
424
+ dangerous thing about this query shape: the missing rows are exactly the lapsed customers a
425
+ "who has gone quiet" filter is FOR. So the caller passes `allowed_pids` (the pool) and the
426
+ zero-group is reconstructed explicitly below rather than left to the SQL.
427
+ """
428
+ if not rule_complete(rule):
429
+ # An absent value would coerce to 0 and resolve as a DIFFERENT question under an
430
+ # authoritative-looking count β€” the same failure class as defaulting a missing window
431
+ # to all time, caught by the same principle: refuse, never guess.
432
+ raise _sem.ModelError(
433
+ 'incomplete measure condition β€” its window, statistic or value is not present; '
434
+ 'refusing to answer a question that has not been fully asked')
435
+ rhs = rule.get('rhs')
436
+ if isinstance(rhs, dict) and rhs.get('kind') == 'measure':
437
+ return resolve_pair(rule, today, team_id, allowed_pids)
438
+ if isinstance(rhs, dict) and rhs.get('kind') == 'stat':
439
+ return resolve_stat(rule, today, team_id, allowed_pids)
440
+ key = rule.get('colId')
441
+ topic_for(key) # raises for anything not admitted
442
+ op = rule.get('op')
443
+ if op not in OPS:
444
+ raise _sem.ModelError(f"{op!r} is not a numeric comparison ({sorted(OPS)})")
445
+
446
+ from harness import filter_sql as _fs
447
+ # ⚠ THE SCALE IS APPLIED HERE, TO THE TYPED VALUE, AND NOWHERE ELSE.
448
+ #
449
+ # Its position relative to `_zero_satisfies` turns out NOT to matter, and the negative
450
+ # control is what established that rather than reasoning: dividing by a POSITIVE constant
451
+ # cannot change how a number compares to zero, so the zero-group is identical either way.
452
+ # (An earlier version of this comment claimed the opposite. It was wrong.) What DOES matter
453
+ # is that a COMPUTED comparand never passes through here β€” see `_scaled`.
454
+ value = _scaled(key, _fs.to_num(rule.get('value')))
455
+
456
+ pool = _pool_of(allowed_pids)
457
+ if is_composite(key):
458
+ # A ratio has no HAVING to push the comparison into: compute the value per customer, then
459
+ # compare in Python β€” the same shape `resolve_pair` uses, and for the same reason.
460
+ con = _sem._store_con()
461
+ try:
462
+ totals = _totals(con, key, rule.get('window'), today, team_id)
463
+ finally:
464
+ con.close()
465
+ # No pool means we can only speak about customers a base has seen. WITH a pool, everyone
466
+ # else scores 0 through the ratio's guard β€” which is what makes the zero-group work for a
467
+ # ratio exactly as it does for a sum ("AOV <= 250" includes those who bought nothing).
468
+ universe = pool if pool is not None else set(totals)
469
+ return {pid for pid in universe if _CMP[op](totals.get(pid, 0.0), value)}
470
+
471
+ frm, id_col, agg, where_sql, params = _query_parts(
472
+ key, rule.get('window'), today, team_id)
473
+ sql = (f"SELECT {id_col} AS pid, {agg} AS v FROM {frm} WHERE {where_sql} "
474
+ f"GROUP BY {id_col} HAVING {agg} {OPS[op]} ?")
475
+
476
+ con = _sem._store_con()
477
+ try:
478
+ matched = {int(r[0]) for r in con.execute(sql, params + [value]).fetchall()
479
+ if r[0] is not None}
480
+ # The zero-group: every customer in scope with NO rows in the window aggregates to 0, and
481
+ # SQL cannot return a group that has no rows. Reconstruct it from the pool rather than
482
+ # pretending those customers do not exist.
483
+ if pool is not None:
484
+ present = {int(r[0]) for r in con.execute(
485
+ f"SELECT DISTINCT {id_col} FROM {frm} WHERE {where_sql}",
486
+ params).fetchall() if r[0] is not None}
487
+ absent = pool - present
488
+ if absent and _zero_satisfies(op, value):
489
+ matched |= absent
490
+ finally:
491
+ con.close()
492
+ if pool is not None:
493
+ matched &= pool
494
+ return matched
495
+
496
+
497
+ def _totals(con, key, window, today, team_id):
498
+ """`{pid: value}` for one (measure, window), COMPOSITES included.
499
+
500
+ For a plain aggregate this is resolve_rule's query minus the HAVING. For a ratio or a derived
501
+ metric it is one query per BASE β€” each on that base's own topic β€” combined per customer by
502
+ `semantic._post_compute`, the same function `store_query` uses for the un-grouped number. So
503
+ AOV per customer is the app's AOV, computed once and defined once.
504
+
505
+ ⚠ The pid keys come from DIFFERENT dims for a cross-topic ratio (`l.order_partner_id` and
506
+ `o.partner_id`). The union is taken rather than the numerator's keys, so a customer either
507
+ side has seen gets a number; the ratio's own `den == 0 -> 0.0` guard covers the rest.
508
+ """
509
+ if is_composite(key):
510
+ bases, _ = _sem._expand_measures([key])
511
+ parts = {b: _totals(con, b, window, today, team_id) for b in bases}
512
+ pids = set()
513
+ for got in parts.values():
514
+ pids |= set(got)
515
+ return {pid: _sem._post_compute({b: parts[b].get(pid, 0.0) for b in bases}, key)
516
+ for pid in pids}
517
+ frm, id_col, agg, where_sql, params = _query_parts(key, window, today, team_id)
518
+ sql = (f"SELECT {id_col} AS pid, {agg} AS v FROM {frm} WHERE {where_sql} GROUP BY {id_col}")
519
+ # `float(r[1])`, not `float(r[1] or 0.0)`: `_query_parts` already wraps the aggregate in
520
+ # COALESCE(..., 0), so a NULL cannot arrive here β€” a second guard would be unreachable, and
521
+ # the negative control proved it (deleting it changed nothing). If one ever DID arrive, a
522
+ # TypeError is the right outcome: it would mean the SQL-level guard had been removed, which
523
+ # is the bug worth hearing about rather than papering over.
524
+ return {int(r[0]): float(r[1])
525
+ for r in con.execute(sql, params).fetchall() if r[0] is not None}
526
+
527
+
528
+ def resolve_pair(rule, today, team_id=None, allowed_pids=None):
529
+ """CG-9 β€” `Where [Sales] [this year] [>] [Sales] [year to date, last year]`.
530
+
531
+ TWO QUERIES, compared in Python. The obvious alternative β€” one query with two conditional
532
+ aggregates and a HAVING comparing them β€” would have to reach inside `_sem._measure_sql` to
533
+ inject a per-side window into an expression that is already a `sum(CASE WHEN … )` for
534
+ filtered measures. Two queries keep ONE definition of each number (the metric's own
535
+ aggregate, unmodified), let the two sides sit on DIFFERENT TOPICS, and make the zero-group
536
+ fall out for free: absent from a side's result means 0 on that side.
537
+
538
+ ⚠ The zero-group is LOUDER here than it is for a one-sided rule. Every customer with no rows
539
+ in EITHER window compares 0 against 0, so `<=` and `>=` match all of them β€” correct, and it
540
+ will look wrong on a narrow window. That is why `eq`/`neq` are not offered at all
541
+ (PAIR_OPS): two float sums are never exactly equal except at zero, where they always are.
542
+ """
543
+ rhs = rule.get('rhs') or {}
544
+ lkey, rkey = rule.get('colId'), rhs.get('colId')
545
+ topic_for(lkey), topic_for(rkey) # raises for anything not admitted
546
+ op = rule.get('op')
547
+ if op not in PAIR_OPS:
548
+ raise _sem.ModelError(
549
+ f"{op!r} cannot compare two measures ({sorted(PAIR_OPS)}). Equality between two sums "
550
+ f"of floats matches nobody and inequality matches everybody; neither answers the "
551
+ f"question, and both look like they work.")
552
+ cmp = _CMP[op]
553
+
554
+ pool = _pool_of(allowed_pids)
555
+ con = _sem._store_con()
556
+ try:
557
+ left = _totals(con, lkey, rule.get('window'), today, team_id)
558
+ right = _totals(con, rkey, rhs.get('window'), today, team_id)
559
+ finally:
560
+ con.close()
561
+ # With no pool we can only speak about customers at least one side has seen β€” the zero-vs-
562
+ # zero group is unknowable without being told who exists. With a pool, it is every member.
563
+ universe = pool if pool is not None else (set(left) | set(right))
564
+ return {pid for pid in universe if cmp(left.get(pid, 0.0), right.get(pid, 0.0))}
565
+
566
+
567
+ def _zero_satisfies(op, value):
568
+ """Does a customer with NO activity in the window (measure = 0) satisfy this condition?"""
569
+ return _CMP[op](0.0, value)
570
+
571
+
572
+ def resolve_values(key, window, today, team_id=None, allowed_pids=None):
573
+ """`{pid: display value}` for one (measure, window) over the caller's pool β€” the COLUMN
574
+ version of `resolve_rule` (owner item 7, 2026-07-27: a formula-measure FIELD).
575
+
576
+ Same `_totals` the filter conditions and statistics use, so the column and the condition
577
+ cannot disagree. Three display decisions live here, not in the client:
578
+
579
+ * ZERO-FILLED over the pool. A customer with no rows in the window has a measure of 0 β€”
580
+ the zero-group convention everywhere else β€” and 0 is a real value the cell shows. Blank
581
+ is reserved for "could not compute" (the caller omits the column entirely on failure).
582
+ * `input_scale` applied OUTWARD. `margin_pct` is a fraction (0-1) internally; the user
583
+ types percentage points into the filter (input divides by 100), so the CELL shows points
584
+ too (value Γ— 100) β€” one boundary, both directions, and `yoy_pct` beside it already reads
585
+ in points.
586
+ * Rounded like every other numeric column: whole numbers for money/counts (the pool's own
587
+ `_round` convention), one decimal for a percent.
588
+
589
+ Raises for anything not admitted / an unresolvable window β€” the caller degrades that column
590
+ to blank rather than to a number nobody computed.
591
+ """
592
+ topic_for(key) # raises for anything not admitted
593
+ # The window must RESOLVE before any query runs β€” same refusal as `_query_parts`, checked
594
+ # here too because a composite's `_totals` path resolves per base and would otherwise raise
595
+ # deeper with a less useful message.
596
+ if _win.resolve(window, today) is None:
597
+ raise _sem.ModelError(
598
+ f"unresolvable window {window!r} for the {key!r} column β€” refusing rather than "
599
+ f"defaulting to all time")
600
+ pool = _pool_of(allowed_pids)
601
+ con = _sem._store_con()
602
+ try:
603
+ totals = _totals(con, key, window, today, team_id)
604
+ finally:
605
+ con.close()
606
+ scale = ADMITTED.get(key, {}).get('input_scale') or 1.0
607
+ mtype = _sem._FORMAT_TYPE.get(_sem.metrics().get(key, {}).get('format'), 'currency')
608
+ nd = 1 if mtype == 'pct' else None # money/counts whole, percents one decimal
609
+ universe = pool if pool is not None else set(totals)
610
+ # round(x, None) returns an int β€” exactly the pool's `_round` convention for money/counts.
611
+ return {int(p): round(float(totals.get(p, 0.0)) * scale, nd) for p in universe}
612
+
613
+
614
+ def resolve_tree(rules, today, team_id=None, allowed_pids=None):
615
+ """{rule_id: sorted pids} for every measure condition in a tree.
616
+
617
+ Keyed by the rule's client-generated `id` so the browser can map an answer back to the
618
+ condition that asked it β€” positional matching would silently re-associate every answer the
619
+ moment a user deleted a condition.
620
+ """
621
+ out = {}
622
+ for r in rules or []:
623
+ rid = str(r.get('id') or '')
624
+ if not rid:
625
+ raise _sem.ModelError('a measure condition needs a stable id to carry its answer back')
626
+ out[rid] = sorted(resolve_rule(r, today, team_id, allowed_pids))
627
+ return out
628
+
629
+
630
+ def signature(rule):
631
+ """The identity of the QUESTION a rule asks, ignoring which condition row asked it.
632
+
633
+ Two conditions with the same measure, window, operator and value have the same answer, so
634
+ they cost one query between them β€” and a saved view re-rendered unchanged costs none, since
635
+ the caller can memoise on this. `id` is deliberately absent: it names the asker, not the
636
+ question.
637
+
638
+ ⚠ The RHS is part of the question. Without it, `Sales[ytd] > Sales[last year]` and
639
+ `Sales[ytd] > Sales[ytd_last_year]` would share one memo entry β€” same measure, same op, same
640
+ (empty) value β€” and the second would be served the first one's answer. Silent, and wrong in
641
+ exactly the comparison this was built for.
642
+ """
643
+ w = _win.normalize(rule.get('window'))
644
+ rhs = rule.get('rhs')
645
+ rsig = None
646
+ if isinstance(rhs, dict):
647
+ rw = _win.normalize(rhs.get('window'))
648
+ # `stat` rides the same tuple: `> the median` and `> the 75th percentile` are different
649
+ # questions with the same measure, window, op and (empty) value.
650
+ rsig = (rhs.get('kind'), rhs.get('colId'), rhs.get('stat'),
651
+ None if rw is None else tuple(sorted(rw.items())))
652
+ return (rule.get('colId'), rule.get('op'), str(rule.get('value') or ''),
653
+ None if w is None else tuple(sorted(w.items())), rsig)
654
+
655
+
656
+ def collect(nodes):
657
+ """Every MEASURE leaf in a filter tree, flattened (groups contribute their children)."""
658
+ found = []
659
+ for n in nodes or []:
660
+ if not isinstance(n, dict):
661
+ continue
662
+ if isinstance(n.get('children'), list):
663
+ found.extend(collect(n['children']))
664
+ elif n.get('colId') in ADMITTED:
665
+ found.append(n)
666
+ return found
royalimports_os/harness/routines.py ADDED
@@ -0,0 +1,259 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """harness/routines.py β€” OM-5 v0: scheduled reports + KPI alerts over the semantic layer.
2
+
3
+ Two of the four productization user jobs (Scheduled Reporting, KPI monitoring). Definitions live
4
+ in data/store/routines.json (tenant data, git-ignored); evaluation happens ON READ against the
5
+ tenant store through the same governed queries as everything else β€” an alert is a metric key +
6
+ a mode, never a hand-rolled query.
7
+
8
+ v0 boundaries (deliberate):
9
+ - Channel is 'in-app' only. Email/Slack delivery is OUTBOUND and ships only with owner approval
10
+ (the Odoo mail client is whitelisted for Collections statements ONLY β€” nothing else sends).
11
+ - No background scheduler: reports record cadence + show due-state in-app; alerts evaluate live
12
+ whenever the Routines page (or a tool) asks. The enterprise scheduler lands with OM-5 delivery.
13
+
14
+ Alert modes:
15
+ - 'xmr' Wheeler individuals chart on complete monthly values (promoted from
16
+ modules/warehouse._xmr, generalized: no 0/100 clamps β€” metrics aren't percentages).
17
+ Three detection rules only (more = alarm fatigue, per the warehouse brief).
18
+ - 'threshold' last complete month above/below a fixed value.
19
+ """
20
+ import json
21
+ import time
22
+ from datetime import date, timedelta
23
+
24
+ import harness.semantic as SEM
25
+ from harness.tools import VIEWS_PATH
26
+
27
+ ROUTINES_PATH = VIEWS_PATH.parent / "routines.json"
28
+
29
+ CADENCES = ("daily", "weekly", "monthly")
30
+ CHANNELS = ("in-app",) # email/slack gate on owner approval (outbound send)
31
+ ALERT_MODES = ("xmr", "threshold")
32
+ XMR_MONTHS = 18 # complete months fed to the individuals chart
33
+
34
+
35
+ def load():
36
+ if ROUTINES_PATH.exists():
37
+ return json.loads(ROUTINES_PATH.read_text(encoding="utf-8"))
38
+ return {"reports": {}, "alerts": {}}
39
+
40
+
41
+ def _save(d):
42
+ ROUTINES_PATH.parent.mkdir(parents=True, exist_ok=True)
43
+ ROUTINES_PATH.write_text(json.dumps(d, indent=1), encoding="utf-8")
44
+
45
+
46
+ # ------------------------------------------------------------------ scheduled reports
47
+
48
+ def schedule_report(view, cadence, channel="in-app"):
49
+ import harness.views as V
50
+ if view not in V.views() and view not in V.dashboards():
51
+ raise SEM.ModelError(f"unknown view/dashboard {view!r} β€” save_view/compose_dashboard first")
52
+ if cadence not in CADENCES:
53
+ raise SEM.ModelError(f"cadence must be one of {CADENCES}")
54
+ if channel not in CHANNELS:
55
+ raise SEM.ModelError(f"channel {channel!r} not available yet β€” v0 delivers in-app "
56
+ "(email/Slack ship with the delivery gate); use channel='in-app'")
57
+ d = load()
58
+ d["reports"][view] = {"cadence": cadence, "channel": channel,
59
+ "created_at": time.strftime("%Y-%m-%d %H:%M")}
60
+ _save(d)
61
+ return {"scheduled": view, "cadence": cadence, "channel": channel,
62
+ "note": "runs in-app (Workflows > Routines); external delivery ships with OM-5 email approval"}
63
+
64
+
65
+ def delete_report(view):
66
+ d = load()
67
+ d["reports"].pop(view, None)
68
+ _save(d)
69
+
70
+
71
+ _CADENCE_DAYS = {"daily": 1, "weekly": 7, "monthly": 30}
72
+
73
+
74
+ def mark_viewed(view):
75
+ """Called when the scheduled view/dashboard is actually opened β€” resets its due-ness."""
76
+ d = load()
77
+ r = d.get("reports", {}).get(view)
78
+ if r is not None:
79
+ r["last_viewed"] = time.strftime("%Y-%m-%d %H:%M")
80
+ _save(d)
81
+
82
+
83
+ def due_reports():
84
+ """Scheduled reports whose cadence has elapsed since last viewed (v0 in-app delivery via
85
+ the Routines page). Never-viewed ages from created_at."""
86
+ from datetime import datetime
87
+ out = []
88
+ for name, r in load().get("reports", {}).items():
89
+ days = _CADENCE_DAYS.get(r.get("cadence"), 7)
90
+ anchor = r.get("last_viewed") or r.get("created_at") or ""
91
+ try:
92
+ age = (datetime.now() - datetime.strptime(anchor, "%Y-%m-%d %H:%M")).days
93
+ except Exception:
94
+ age = days
95
+ if age >= days:
96
+ out.append({"name": name, "cadence": r.get("cadence"),
97
+ "last_viewed": r.get("last_viewed"), "age_days": age})
98
+ return out
99
+
100
+
101
+ # ------------------------------------------------------------------ insight routines (OM-5)
102
+ # A saved PROMPT on a cadence: the Analyst runs it in the background and the narrative (plus its
103
+ # governed artifacts) lands in-app on the Routines page. The narrative is a deliberate
104
+ # point-in-time SNAPSHOT (it narrates a moment); its charts still carry re-runnable queries.
105
+
106
+ def schedule_insight(name, prompt, cadence):
107
+ if cadence not in CADENCES:
108
+ raise SEM.ModelError(f"cadence must be one of {CADENCES}")
109
+ if not (prompt or "").strip():
110
+ raise SEM.ModelError("an insight needs a prompt β€” what should the Analyst look at?")
111
+ d = load()
112
+ d.setdefault("insights", {})[name] = {"prompt": prompt.strip(), "cadence": cadence,
113
+ "created_at": time.strftime("%Y-%m-%d %H:%M")}
114
+ _save(d)
115
+ return {"insight": name, "cadence": cadence,
116
+ "note": "runs in the background when due; the narrative lands on "
117
+ "Workflows > Routines"}
118
+
119
+
120
+ def delete_insight(name):
121
+ d = load()
122
+ (d.get("insights") or {}).pop(name, None)
123
+ _save(d)
124
+
125
+
126
+ def due_insights():
127
+ from datetime import datetime
128
+ out = []
129
+ for name, i in (load().get("insights") or {}).items():
130
+ days = _CADENCE_DAYS.get(i.get("cadence"), 7)
131
+ anchor = i.get("last_run_at") or ""
132
+ try:
133
+ age = (datetime.now() - datetime.strptime(anchor, "%Y-%m-%d %H:%M")).days
134
+ except Exception:
135
+ age = days # never run -> due now
136
+ if age >= days:
137
+ out.append(name)
138
+ return out
139
+
140
+
141
+ def run_insight(name):
142
+ """Execute one insight through the Analyst and persist the narrative + artifacts."""
143
+ d = load()
144
+ ins = (d.get("insights") or {}).get(name)
145
+ if not ins:
146
+ raise SEM.ModelError(f"unknown insight {name!r}")
147
+ import harness.analyst as A
148
+ r = A.ask(ins["prompt"])
149
+ now = time.strftime("%Y-%m-%d %H:%M")
150
+ ins["last_run_at"] = now
151
+ ins["last"] = {"at": now, "answer": r["answer"], "artifacts": r.get("artifacts") or [],
152
+ "tokens": (r.get("usage") or {}).get("prompt_tokens", 0)
153
+ + (r.get("usage") or {}).get("completion_tokens", 0)}
154
+ _save(d)
155
+ return ins["last"]
156
+
157
+
158
+ # ------------------------------------------------------------------ KPI alerts
159
+
160
+ def create_alert(metric, mode="xmr", team_id=None, threshold=None, direction=None):
161
+ m = SEM.metrics().get(metric)
162
+ if not m:
163
+ raise SEM.ModelError(f"unknown metric {metric!r} (use list_topics)")
164
+ if mode not in ALERT_MODES:
165
+ raise SEM.ModelError(f"mode must be one of {ALERT_MODES}")
166
+ if mode == "threshold":
167
+ if threshold is None or direction not in ("above", "below"):
168
+ raise SEM.ModelError("threshold mode needs threshold=<number> and "
169
+ "direction='above'|'below'")
170
+ d = load()
171
+ key = metric + (f"_team{team_id}" if team_id else "")
172
+ d["alerts"][key] = {"metric": metric, "mode": mode, "team_id": team_id,
173
+ "threshold": threshold, "direction": direction,
174
+ "created_at": time.strftime("%Y-%m-%d %H:%M")}
175
+ _save(d)
176
+ return {"alert": key, **evaluate_alert(d["alerts"][key])}
177
+
178
+
179
+ def delete_alert(key):
180
+ d = load()
181
+ d["alerts"].pop(key, None)
182
+ _save(d)
183
+
184
+
185
+ def xmr(series):
186
+ """Wheeler XmR individuals limits + the three detection rules β€” the model-level primitive
187
+ (generalized from modules/warehouse._xmr; no percentage clamps). series=[(label, value)]."""
188
+ vals = [v for _, v in series]
189
+ if len(vals) < 8:
190
+ return {"center": None, "unpl": None, "lnpl": None,
191
+ "points": [{"label": l, "value": v, "signal": ""} for l, v in series]}
192
+ center = sum(vals) / len(vals)
193
+ mrs = [abs(vals[i] - vals[i - 1]) for i in range(1, len(vals))]
194
+ mr_bar = sum(mrs) / len(mrs) if mrs else 0.0
195
+ unpl = center + 2.66 * mr_bar
196
+ lnpl = center - 2.66 * mr_bar
197
+ pts = []
198
+ for i, (l, v) in enumerate(series):
199
+ sig = ""
200
+ if v > unpl or v < lnpl:
201
+ sig = "outside limits"
202
+ if not sig and i >= 7 and all((x - center) > 0 for _, x in series[i - 7:i + 1]):
203
+ sig = "sustained shift (8 above center)"
204
+ if not sig and i >= 7 and all((x - center) < 0 for _, x in series[i - 7:i + 1]):
205
+ sig = "sustained shift (8 below center)"
206
+ if not sig and i >= 3:
207
+ last4 = [x for _, x in series[i - 3:i + 1]]
208
+ near = sum(1 for x in last4
209
+ if abs(x - center) > 0.5 * ((unpl - center) if x >= center
210
+ else (center - lnpl)))
211
+ if near >= 3:
212
+ sig = "moderate shift emerging"
213
+ pts.append({"label": l, "value": v, "signal": sig})
214
+ return {"center": center, "unpl": unpl, "lnpl": lnpl, "points": pts}
215
+
216
+
217
+ def _month_window(n=XMR_MONTHS):
218
+ """Last n COMPLETE months (the in-progress month would false-signal every alert)."""
219
+ first_cur = date.today().replace(day=1)
220
+ y, mo = first_cur.year, first_cur.month
221
+ for _ in range(n):
222
+ mo -= 1
223
+ if mo == 0:
224
+ y, mo = y - 1, 12
225
+ start = date(y, mo, 1)
226
+ return start.isoformat(), (first_cur - timedelta(days=1)).isoformat()
227
+
228
+
229
+ def evaluate_alert(a):
230
+ """Live evaluation: monthly series of the metric (complete months only) -> mode verdict."""
231
+ metric = a["metric"]
232
+ topic = SEM.metrics()[metric]["topic"]
233
+ date_from, date_to = _month_window()
234
+ res = SEM.store_query(topic, [metric], grain="month", date_from=date_from, date_to=date_to,
235
+ team_id=a.get("team_id"))
236
+ series = [(r["period"][:7], r.get(metric) or 0) for r in res["rows"]]
237
+ out = {"metric": metric, "mode": a["mode"], "team_id": a.get("team_id"),
238
+ "window": f"{date_from}..{date_to}", "series": series}
239
+ if not series:
240
+ return {**out, "status": "no data", "fired": False}
241
+ last_label, last_val = series[-1]
242
+ if a["mode"] == "threshold":
243
+ thr, direc = a.get("threshold"), a.get("direction")
244
+ fired = (last_val > thr) if direc == "above" else (last_val < thr)
245
+ return {**out, "last": {"label": last_label, "value": last_val},
246
+ "threshold": thr, "direction": direc,
247
+ "status": (f"FIRED β€” {last_label} is {direc} {thr:,.0f}" if fired else "OK"),
248
+ "fired": bool(fired)}
249
+ lim = xmr(series)
250
+ sig = lim["points"][-1]["signal"] if lim["points"] else ""
251
+ return {**out, "last": {"label": last_label, "value": last_val},
252
+ "center": lim["center"], "unpl": lim["unpl"], "lnpl": lim["lnpl"],
253
+ "points": lim["points"],
254
+ "status": (f"SIGNAL β€” {sig} ({last_label})" if sig else "OK"),
255
+ "fired": bool(sig)}
256
+
257
+
258
+ def evaluate_all():
259
+ return {k: evaluate_alert(a) for k, a in load().get("alerts", {}).items()}
royalimports_os/harness/runtime.py ADDED
@@ -0,0 +1,186 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """harness/runtime.py β€” X7: per-REQUEST tenant resolution for the shared stateless API (EXIT-4a).
2
+
3
+ THE COST ARGUMENT, in one rule: *no per-tenant state may be resident in the app process.* Today
4
+ each tenant carries its own interpreter, its own never-evicted pool, its own 9 prewarmed bundles
5
+ and its own DuckDB β€” EXIT-0 measured a **~0.6 GB commit floor per tenant**, duplicated exactly
6
+ (tenant B came within 2% of tenant A; nothing is shared). The target is ~30–80 MB. That is only
7
+ reachable if the API resolves the tenant from the REQUEST and holds nothing tenant-shaped between
8
+ requests except a bounded cache it is willing to throw away.
9
+
10
+ WHAT THIS REPLACES. `ui/cache.py`'s `st.cache_resource` singleton β€” one process, one tenant,
11
+ caches that are never evicted by design. That is correct for Streamlit and fatal for a shared
12
+ process. The Streamlit side keeps its singleton untouched this wave (two processes, two caches,
13
+ accepted for the strangler period; `ui/cache.py` dies with app.py at EXIT-6).
14
+
15
+ THREE RULES THIS FILE OBEYS, each of which is a way the old shape leaked:
16
+ 1. **No module-level per-tenant globals.** Every tenant-shaped value hangs off a TenantRuntime
17
+ reached through `get_runtime(slug)`; nothing is stashed at import time. A module global is
18
+ exactly what makes "which tenant is this process serving?" unanswerable.
19
+ 2. **LRU-BOUNDED.** `_MAX_RUNTIMES` runtimes live at once; the least recently used is evicted.
20
+ An unbounded registry cache is a memory leak with a tenant-count-shaped growth curve β€” the
21
+ thing this whole program exists to remove.
22
+ 3. **The store handle is tenant-bound**, so a write cannot land in another tenant's namespace
23
+ even if a caller passes the wrong key downstream (EXIT-4b proof #3).
24
+
25
+ ⚠ THE DUCKDB CAVEAT, STATED NOT HIDDEN. `harness.datastore` holds ONE process-wide connection
26
+ (the fix for the 2026-07-16 "connection error unless I refresh" bug), so one PROCESS can serve
27
+ one analytical store at a time. This module therefore hands out `datastore_path` β€” the file a
28
+ tenant's analytical reads belong to β€” and does NOT switch the global underneath a live request.
29
+ Until the singleton is per-tenant (EXIT-5), an API process answering measure/Analyst queries must
30
+ be pinned to one tenant's file, and `assert_datastore_matches()` below is what turns a
31
+ mis-pinned deployment into a loud failure instead of a silent cross-tenant read.
32
+ """
33
+ from __future__ import annotations
34
+
35
+ import os
36
+ import threading
37
+ from collections import OrderedDict
38
+ from dataclasses import dataclass, field
39
+
40
+ import core.store as store
41
+ from harness import datastore as _ds
42
+ from harness import tenants as _tenants
43
+
44
+ #: How many tenant runtimes stay resident. Small on purpose: a runtime is cheap to rebuild (a
45
+ #: Tenant literal + a couple of handles) and expensive to leak. Raise only with a measurement.
46
+ _MAX_RUNTIMES = int(os.environ.get("AIOS_MAX_RUNTIMES") or 32)
47
+
48
+ #: slug -> builder. The literal registry `harness/tenants.py` describes; per its own docstring
49
+ #: this becomes "a tenants store (DB / HF dataset)" β€” X4's control DB is where it goes, and
50
+ #: `register()` below is the seam that makes the swap a one-liner rather than a refactor.
51
+ _BUILDERS = {"royal-imports": _tenants.royal_imports}
52
+
53
+ # X8's isolation fixture, admitted ONLY on request. A stub tenant that is always present is a
54
+ # slug an attacker can mint a session against in production; gated, it exists exactly where it is
55
+ # needed (the TestClient isolation proofs) and nowhere else. Read at import so the flag cannot be
56
+ # flipped by a request.
57
+ if os.environ.get("AIOS_ENABLE_QA_TENANT") == "1":
58
+ _BUILDERS["qa-b"] = _tenants.qa_tenant_b
59
+
60
+ _CACHE: "OrderedDict[str, TenantRuntime]" = OrderedDict()
61
+ _LOCK = threading.RLock()
62
+
63
+
64
+ @dataclass
65
+ class TenantRuntime:
66
+ """Everything a request needs to serve ONE tenant, and nothing about any other.
67
+
68
+ `store_namespace` is the prefix every product-data key is written under. Tenant #0 keeps the
69
+ EMPTY prefix so its existing keys (`users`, `customer_lists`, `customer_docs`, …) are found
70
+ exactly where they already live β€” a namespacing change that renamed tenant #0's keys would be
71
+ a silent data loss dressed as a refactor. Every OTHER tenant is prefixed, which is what makes
72
+ proof #3 hold.
73
+ """
74
+ key: str
75
+ tenant: object # harness.base.Tenant
76
+ store_namespace: str = ""
77
+ pool_cache: dict = field(default_factory=dict)
78
+ _lock: threading.RLock = field(default_factory=threading.RLock, repr=False)
79
+
80
+ @property
81
+ def name(self):
82
+ return getattr(self.tenant, "name", self.key)
83
+
84
+ @property
85
+ def datastore_path(self):
86
+ """The DuckDB file this tenant's analytical reads belong to (file-per-tenant IS the
87
+ isolation model β€” DuckDB has no RLS, so the OS boundary is the only one that holds)."""
88
+ return _ds.path_for(self.key)
89
+
90
+ # ---------------------------------------------------------------- the store handle
91
+ def store_key(self, name):
92
+ """Namespace a product-data key for this tenant. The ONE place the prefix is applied."""
93
+ return f"{self.store_namespace}{name}" if self.store_namespace else str(name)
94
+
95
+ def get(self, name, fresh=False):
96
+ return store.get(self.store_key(name), fresh=fresh)
97
+
98
+ def put(self, name, data):
99
+ return store.put(self.store_key(name), data)
100
+
101
+ def update(self, name, fn, flush="sync"):
102
+ return store.update(self.store_key(name), fn, flush=flush)
103
+
104
+ def exists(self, name):
105
+ return store.exists(self.store_key(name))
106
+
107
+ def available(self):
108
+ return store.available()
109
+
110
+ def assert_datastore_matches(self):
111
+ """RAISE if the process-wide DuckDB file is not this tenant's.
112
+
113
+ The alternative is the failure mode that matters: an analytical read served from another
114
+ tenant's file returns real, plausible, wrong rows β€” no exception, no telemetry, and the
115
+ number reconciles against the wrong book. A loud failure is strictly better, and this is
116
+ the assertion EXIT-5 deletes when the connection becomes per-tenant.
117
+ """
118
+ want, have = self.datastore_path.resolve(), _ds.DB_PATH.resolve()
119
+ if want != have:
120
+ raise RuntimeError(
121
+ f"tenant {self.key!r} expects the analytical store at {want}, but this process "
122
+ f"has {have} open. Refusing the read β€” a cross-tenant answer would reconcile "
123
+ f"against the wrong book. (Pin the process with AIOS_DUCKDB_PATH, or call "
124
+ f"harness.datastore.use_path before serving this tenant.)")
125
+
126
+
127
+ def register(key, builder):
128
+ """Add a tenant to the registry (a stub connector is a legitimate builder β€” see X8's qa-b).
129
+ Invalidates any cached runtime for the slug so a re-registration cannot be shadowed."""
130
+ _BUILDERS[str(key)] = builder
131
+ invalidate(key)
132
+
133
+
134
+ def known_tenants():
135
+ return sorted(_BUILDERS)
136
+
137
+
138
+ def get_runtime(tenant_key):
139
+ """The runtime for `tenant_key`, built on first use and LRU-cached.
140
+
141
+ Raises KeyError for an unknown slug β€” the caller turns that into a 401 (fail-closed: an
142
+ unknown tenant is not a 404, because confirming which slugs exist answers a question the
143
+ request was not entitled to ask).
144
+ """
145
+ key = str(tenant_key or "").strip().lower()
146
+ if not key:
147
+ raise KeyError("no tenant")
148
+ with _LOCK:
149
+ rt = _CACHE.get(key)
150
+ if rt is not None:
151
+ _CACHE.move_to_end(key) # LRU: touch on read
152
+ return rt
153
+ builder = _BUILDERS.get(key)
154
+ if builder is None:
155
+ raise KeyError(key)
156
+ # Built OUTSIDE the lock: a builder may do real work (a connector handshake), and holding the
157
+ # registry lock through it would serialise every other tenant's requests behind it.
158
+ tenant = builder()
159
+ with _LOCK:
160
+ existing = _CACHE.get(key)
161
+ if existing is not None: # another thread won the race β€” keep ONE runtime
162
+ _CACHE.move_to_end(key)
163
+ return existing
164
+ rt = TenantRuntime(key=key, tenant=tenant,
165
+ store_namespace="" if key == "royal-imports" else f"t/{key}/")
166
+ _CACHE[key] = rt
167
+ while len(_CACHE) > _MAX_RUNTIMES:
168
+ _CACHE.popitem(last=False) # evict least recently used
169
+ return rt
170
+
171
+
172
+ def invalidate(tenant_key=None):
173
+ """Drop one tenant's runtime, or all of them. Explicit, per X7 β€” a cache with no documented
174
+ way to clear it becomes a restart."""
175
+ with _LOCK:
176
+ if tenant_key is None:
177
+ _CACHE.clear()
178
+ else:
179
+ _CACHE.pop(str(tenant_key or "").strip().lower(), None)
180
+
181
+
182
+ def resident():
183
+ """The tenant slugs currently holding a runtime β€” LRU order, oldest first. For the EXIT-4
184
+ measurement and for asserting the bound actually binds."""
185
+ with _LOCK:
186
+ return list(_CACHE)
royalimports_os/harness/semantic.py ADDED
@@ -0,0 +1,936 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """harness/semantic.py β€” the semantic seam (OM-0a, 2026-07-11).
2
+
3
+ The ONE governed registry of topics + metrics (royalimports_os/model/*.yml) and the resolver that
4
+ turns a metric key into a number through the SAME proven data layer the validated modules use
5
+ (core/odoo domain builders + aggregate helpers) β€” so scope parity is by construction, and every
6
+ metric carries its definition, ai_context, and an independent validate contract.
7
+
8
+ Consumers (the point of the seam): the app's pages, the metric dictionary (OM-0c), the saved-
9
+ dashboard viewer (OM-3; the Explore page was retired 2026-07-23), the AI Analyst's tools (OM-4:
10
+ list_topics / describe_topic / run_semantic_query), and the MCP server (OM-6). Plan:
11
+ .claude/wiki/research/omni-adoption.md (Part IV + the productization addendum).
12
+
13
+ Design constraints honored (from /odoo-api):
14
+ - read_group dot-path GROUPBY fails β†’ scalar aggregates here use sum_field/distinct_count;
15
+ grouped queries (OM-3+) must group by direct m2o fields only.
16
+ - Scope is never hand-rolled: topics BIND to a named domain builder (sale_line_domain, ...);
17
+ the YAML `scope:` block is the declarative statement of what that builder bakes in.
18
+ - READ-ONLY on Odoo, always.
19
+ """
20
+ import ast
21
+ import functools
22
+ import operator
23
+ from pathlib import Path
24
+
25
+ import yaml
26
+
27
+ import core.odoo as O
28
+
29
+ MODEL_DIR = Path(__file__).resolve().parents[1] / "model"
30
+
31
+ # Topic scope binds to a NAMED, reviewed domain builder β€” never a YAML-assembled domain (one
32
+ # source of truth for scope until OM-2 makes topics executable against the store).
33
+ def _order_domain(*a, **kw):
34
+ # lazy: order scope currently lives in modules/sales.py; OM-2 lifts it into core/the store
35
+ import modules.sales as S
36
+ return S.order_domain(*a, **kw)
37
+
38
+
39
+ def _customer_move_domain(move_type):
40
+ """A posted customer document domain, matching `modules/returns.py`'s `_mdom` exactly.
41
+
42
+ ⚠ COMPANY-LEVEL, and it REFUSES a team rather than ignoring one. Credit notes all carry
43
+ team_id = 1, so a BU filter on the document is meaningless β€” silently dropping the argument
44
+ would hand a business-unit user the company's number under their own name, which is the
45
+ widening sin in a different currency.
46
+ """
47
+ def build(date_from=None, date_to=None, team_id=None, **kw):
48
+ if team_id:
49
+ raise ModelError(
50
+ f"customer {move_type} documents are COMPANY-LEVEL β€” credit notes and invoices "
51
+ f"carry no trustworthy business-unit tag, so a team-scoped total cannot be "
52
+ f"produced. Refusing rather than returning the company number.")
53
+ dom = [("move_type", "=", move_type), ("state", "=", "posted")]
54
+ if date_from:
55
+ dom.append(("invoice_date", ">=", date_from))
56
+ if date_to:
57
+ dom.append(("invoice_date", "<=", date_to))
58
+ return dom
59
+ return build
60
+
61
+
62
+ _BUILDERS = {
63
+ "sale_line_domain": O.sale_line_domain,
64
+ "sale_order_domain": _order_domain,
65
+ "credit_note_domain": _customer_move_domain("out_refund"),
66
+ "customer_invoice_domain": _customer_move_domain("out_invoice"),
67
+ }
68
+
69
+
70
+ class ModelError(Exception):
71
+ """A model-file problem (unknown key, bad reference, unparseable expr)."""
72
+
73
+
74
+ # ---------------------------------------------------------------- loading
75
+
76
+ @functools.lru_cache(maxsize=1)
77
+ def _tenant():
78
+ f = MODEL_DIR / "tenant.yml"
79
+ return yaml.safe_load(f.read_text(encoding="utf-8")) if f.exists() else {"params": {}}
80
+
81
+
82
+ def _sql_value(v):
83
+ """Render a tenant param into SQL safely (params are versioned config, not user input β€”
84
+ but escape anyway): ints as-is, lists comma-joined, strings single-quoted + escaped."""
85
+ if isinstance(v, bool):
86
+ raise ModelError("boolean tenant params not supported in scope_sql")
87
+ if isinstance(v, (int, float)):
88
+ return str(v)
89
+ if isinstance(v, (list, tuple)):
90
+ return ", ".join(_sql_value(x) for x in v)
91
+ return "'" + str(v).replace("'", "''") + "'"
92
+
93
+
94
+ def render_scope(sql):
95
+ """Substitute {param} placeholders in a store scope/filter SQL fragment from tenant.yml."""
96
+ params = _tenant().get("params") or {}
97
+ out = sql
98
+ for k, v in params.items():
99
+ out = out.replace("{" + k + "}", _sql_value(v))
100
+ if "{" in out and "}" in out:
101
+ import re as _re
102
+ missing = _re.findall(r"\{([a-z_]+)\}", out)
103
+ if missing:
104
+ raise ModelError(f"scope_sql references unknown tenant params: {missing}")
105
+ return out
106
+
107
+
108
+ @functools.lru_cache(maxsize=1)
109
+ def _model():
110
+ topics, metrics = {}, {}
111
+ for f in sorted((MODEL_DIR / "topics").glob("*.yml")):
112
+ d = yaml.safe_load(f.read_text(encoding="utf-8"))
113
+ if not d or "key" not in d:
114
+ raise ModelError(f"topic file {f.name} lacks a 'key'")
115
+ if d.get("domain_builder") and d["domain_builder"] not in _BUILDERS:
116
+ raise ModelError(f"topic {d['key']}: unknown domain_builder {d.get('domain_builder')!r}")
117
+ # a topic may be STORE-ONLY (no live-path builder yet, e.g. gl_lines) β€” metric() raises
118
+ # a clear error if a live resolution is attempted on it
119
+ topics[d["key"]] = d
120
+ for f in sorted((MODEL_DIR / "metrics").glob("*.yml")):
121
+ d = yaml.safe_load(f.read_text(encoding="utf-8"))
122
+ topic = d.get("topic")
123
+ if topic not in topics:
124
+ raise ModelError(f"metrics file {f.name}: unknown topic {topic!r}")
125
+ for m in d.get("metrics", []):
126
+ if "key" not in m:
127
+ raise ModelError(f"metrics file {f.name}: a metric lacks a 'key'")
128
+ if m["key"] in metrics:
129
+ raise ModelError(f"duplicate metric key {m['key']!r}")
130
+ m.setdefault("topic", topic) # a metric may name its own topic (rare)
131
+ if m["topic"] not in topics:
132
+ raise ModelError(f"metric {m['key']!r}: unknown topic {m['topic']!r}")
133
+ metrics[m["key"]] = m
134
+ return {"topics": topics, "metrics": metrics}
135
+
136
+
137
+ def reload_model():
138
+ _model.cache_clear()
139
+ _tenant.cache_clear()
140
+ return _model()
141
+
142
+
143
+ def topics():
144
+ return dict(_model()["topics"])
145
+
146
+
147
+ def metrics():
148
+ return dict(_model()["metrics"])
149
+
150
+
151
+ def describe(key):
152
+ """Full metadata for a metric (the dictionary page / AI describe_topic feed)."""
153
+ m = _model()["metrics"].get(key)
154
+ if not m:
155
+ raise ModelError(f"unknown metric {key!r}")
156
+ return {**m, "topic_def": _model()["topics"][m["topic"]]}
157
+
158
+
159
+ # ---------------------------------------------------------------- resolving
160
+
161
+ def _domain(topic_def, date_from, date_to, team_id):
162
+ b = topic_def.get("domain_builder")
163
+ if not b:
164
+ raise ModelError(f"topic {topic_def['key']!r} is store-only β€” use store_query() "
165
+ f"(no live-path domain builder bound yet)")
166
+ return _BUILDERS[b](date_from, date_to, team_id=team_id)
167
+
168
+
169
+ _OPS = {ast.Add: operator.add, ast.Sub: operator.sub,
170
+ ast.Mult: operator.mul, ast.Div: operator.truediv}
171
+
172
+
173
+ def _eval_expr(expr, resolve):
174
+ """Safely evaluate a derived-metric expression: metric keys, numbers, + - * / and parens."""
175
+ def ev(node):
176
+ if isinstance(node, ast.Expression):
177
+ return ev(node.body)
178
+ if isinstance(node, ast.BinOp) and type(node.op) in _OPS:
179
+ return _OPS[type(node.op)](ev(node.left), ev(node.right))
180
+ if isinstance(node, ast.UnaryOp) and isinstance(node.op, ast.USub):
181
+ return -ev(node.operand)
182
+ if isinstance(node, ast.Num): # py<3.8 compat name; Constant below
183
+ return node.n
184
+ if isinstance(node, ast.Constant) and isinstance(node.value, (int, float)):
185
+ return node.value
186
+ if isinstance(node, ast.Name):
187
+ return resolve(node.id)
188
+ raise ModelError(f"disallowed token in derived expr: {ast.dump(node)}")
189
+ try:
190
+ return ev(ast.parse(expr, mode="eval"))
191
+ except SyntaxError as e:
192
+ raise ModelError(f"unparseable derived expr {expr!r}: {e}")
193
+
194
+
195
+ def metric(key, date_from=None, date_to=None, team_id=None, _seen=None):
196
+ """Resolve a registered metric to a number for a window (+optional BU). The only public
197
+ compute path β€” every surface that shows this metric goes through here."""
198
+ _seen = _seen or set()
199
+ if key in _seen:
200
+ raise ModelError(f"circular metric reference at {key!r}")
201
+ _seen = _seen | {key}
202
+ m = _model()["metrics"].get(key)
203
+ if not m:
204
+ raise ModelError(f"unknown metric {key!r}")
205
+ agg = m.get("agg")
206
+ if agg == "ratio":
207
+ num = metric(m["numerator"], date_from, date_to, team_id, _seen)
208
+ den = metric(m["denominator"], date_from, date_to, team_id, _seen)
209
+ return (num / den) if den else 0.0
210
+ if agg == "derived":
211
+ return _eval_expr(m["expr"], lambda k: metric(k, date_from, date_to, team_id, _seen))
212
+ t = _model()["topics"][m["topic"]]
213
+ dom = _domain(t, date_from, date_to, team_id)
214
+ if agg == "sum":
215
+ # ⚠ `negate` applies HERE too, not only in `_measure_sql`. It existed for GL income on a
216
+ # STORE-ONLY topic, so the live path never met it β€” the first metric that is both negated
217
+ # and parity-checked (`returns`) would otherwise compare a negative live number against a
218
+ # positive store one, and the "parity failure" would look like a data problem.
219
+ got = O.sum_field(t["entity"], dom, m["field"])
220
+ return -got if m.get("negate") else got
221
+ if agg == "count_distinct":
222
+ return O.distinct_count(t["entity"], dom, m["field"])
223
+ if agg == "count":
224
+ return O.get_odoo().search_count(t["entity"], dom)
225
+ raise ModelError(f"metric {key!r}: unknown agg {agg!r}")
226
+
227
+
228
+ # ---------------------------------------------------------------- validate contracts
229
+
230
+ def _v_order_level_revenue(date_from, date_to, team_id):
231
+ """Independent check: line-level Ξ£ price_subtotal == order-level Ξ£ amount_untaxed."""
232
+ import modules.sales as S
233
+ line = metric("revenue", date_from, date_to, team_id)
234
+ order = O.sum_field("sale.order", S.order_domain(date_from, date_to, team_id), "amount_untaxed")
235
+ return line, order
236
+
237
+
238
+ def _v_order_count(date_from, date_to, team_id):
239
+ """Exact reconciliation: order-level count == distinct line-parent orders + line-LESS orders.
240
+ (Confirmed orders with zero lines exist in the data β€” 2 found at first proof, 2026-07-11;
241
+ counting them explicitly makes the identity exact instead of hiding them in a tolerance,
242
+ and surfaces them as a data-health signal.)"""
243
+ import modules.sales as S
244
+ order_level = metric("orders", date_from, date_to, team_id)
245
+ dom = S.order_domain(date_from, date_to, team_id)
246
+ from_lines = O.distinct_count("sale.order.line",
247
+ O.sale_line_domain(date_from, date_to, team_id), "order_id")
248
+ empty = O.get_odoo().search_count("sale.order", dom + [("order_line", "=", False)])
249
+ return order_level, from_lines + empty
250
+
251
+
252
+ def _v_gl_opex(date_from, date_to, team_id):
253
+ """Store-path opex == the live Odoo aggregate with the Expenses-module domain (posted,
254
+ expense-type accounts), to the cent. team_id N/A (company-level)."""
255
+ res = store_query("gl_lines", ["opex"], date_from=date_from, date_to=date_to)
256
+ ours = res["rows"][0]["opex"] if res["rows"] else 0
257
+ dom = [("move_id.state", "=", "posted"),
258
+ ("account_id.account_type", "in", ["expense", "expense_depreciation"])]
259
+ if date_from:
260
+ dom.append(("date", ">=", date_from))
261
+ if date_to:
262
+ dom.append(("date", "<=", date_to))
263
+ live = O.sum_field("account.move.line", dom, "balance")
264
+ return ours, live
265
+
266
+
267
+ def _v_ar_outstanding(date_from, date_to, team_id):
268
+ """Store-path AR outstanding == live Odoo aggregate (posted out_invoice/out_refund residuals).
269
+ No date window β€” outstanding is an as-of-now balance."""
270
+ res = store_query("receivables", ["ar_outstanding"])
271
+ ours = res["rows"][0]["ar_outstanding"] if res["rows"] else 0
272
+ live = O.sum_field("account.move",
273
+ [("state", "=", "posted"), ("move_type", "in", ["out_invoice", "out_refund"])],
274
+ "amount_residual_signed")
275
+ return ours, live
276
+
277
+
278
+ def _v_returns_sign(date_from, date_to, team_id):
279
+ """Independent check on the NEGATION: `returns` must equal the ABSOLUTE untaxed total of the
280
+ same posted credit notes, read straight from Odoo without going through the metric.
281
+
282
+ A lost negation makes returns negative; a doubled one makes it negative again. Both read as
283
+ "no returns" rather than as an error, and neither is visible to any self-consistency check β€”
284
+ the store and the live path would agree with each other perfectly while both being wrong.
285
+ """
286
+ ours = metric("returns", date_from, date_to, team_id)
287
+ raw = O.sum_field("account.move",
288
+ _BUILDERS["credit_note_domain"](date_from, date_to, team_id),
289
+ "amount_untaxed_signed")
290
+ return ours, abs(raw)
291
+
292
+
293
+ def _v_line_vs_move_grain(date_from, date_to, team_id):
294
+ """Independent check on the invoice-LINE topic: the store's netted product-line total must
295
+ equal the same figure read straight from LIVE Odoo.
296
+
297
+ Two things this catches that self-consistency cannot. (1) A LOST CREDIT-NOTE NEGATION β€”
298
+ `price_subtotal` is stored POSITIVE on a refund line, so a dropped subtraction inflates
299
+ revenue while every internal check still agrees. (2) A WRONG `display_type` FILTER β€” the
300
+ invoice tables carry 'cogs' and 'payment_term' rows alongside 'product' ones, and including
301
+ them changes the total silently rather than erroring.
302
+
303
+ Company-level (team_id None) uses the strongest possible reference: the MOVE-grain
304
+ `amount_untaxed_signed`, a different table entirely. Per-BU there is no move-grain analogue
305
+ (account.move carries no business unit β€” every invoice sits on team 1), so the reference is
306
+ the live line-level aggregate reached through the sale-order dot path: still an independent
307
+ engine and query path from the DuckDB store, which is what the contract requires.
308
+ """
309
+ ours = (store_query("invoice_lines", ["invoiced_line_sales"],
310
+ date_from=date_from, date_to=date_to, team_id=team_id)
311
+ .get("rows") or [{}])[0].get("invoiced_line_sales") or 0.0
312
+ if team_id is None:
313
+ raw = O.sum_field("account.move",
314
+ [("state", "=", "posted"),
315
+ ("move_type", "in", ["out_invoice", "out_refund"]),
316
+ ("invoice_date", ">=", date_from), ("invoice_date", "<=", date_to)],
317
+ "amount_untaxed_signed")
318
+ return ours, raw
319
+ dom = [("parent_state", "=", "posted"), ("display_type", "=", "product"),
320
+ ("date", ">=", date_from), ("date", "<=", date_to),
321
+ ("sale_line_ids.order_id.team_id", "=", team_id)]
322
+ gross = O.sum_field("account.move.line", dom + [("move_type", "=", "out_invoice")],
323
+ "price_subtotal")
324
+ refunds = O.sum_field("account.move.line", dom + [("move_type", "=", "out_refund")],
325
+ "price_subtotal")
326
+ return ours, (gross or 0) - (refunds or 0)
327
+
328
+
329
+ _VALIDATORS = {
330
+ "returns_sign": _v_returns_sign,
331
+ "line_vs_move_grain": _v_line_vs_move_grain,
332
+ "order_level_revenue": _v_order_level_revenue,
333
+ "order_count": _v_order_count,
334
+ "gl_opex": _v_gl_opex,
335
+ "ar_outstanding": _v_ar_outstanding,
336
+ }
337
+
338
+
339
+ def validate_metric(key, date_from=None, date_to=None, team_id=None, tol=0.01):
340
+ """Run a metric's declared independent cross-check. Returns the standard check dict
341
+ (the validate.py convention: check/ok/gap)."""
342
+ m = _model()["metrics"].get(key)
343
+ if not m:
344
+ raise ModelError(f"unknown metric {key!r}")
345
+ contract = m.get("validate")
346
+ if not contract:
347
+ return {"check": f"semantic.{key}: no independent contract declared", "ok": True, "gap": 0.0,
348
+ "note": "declare one in model/metrics when an independent aggregate exists"}
349
+ fn = _VALIDATORS.get(contract["method"])
350
+ if not fn:
351
+ raise ModelError(f"metric {key!r}: unknown validate method {contract['method']!r}")
352
+ ours, independent = fn(date_from, date_to, team_id)
353
+ gap = abs((ours or 0) - (independent or 0))
354
+ return {"check": f"semantic.{key} == {contract['method']} ({date_from}..{date_to}, team={team_id})",
355
+ "ok": gap <= tol, "gap": round(gap, 4), "ours": ours, "independent": independent}
356
+
357
+
358
+ def validate(pre=None):
359
+ """Module-convention validate(): run every declared contract over YTD, all-BU + per-BU."""
360
+ import core.periods as P
361
+ yf, yt = P.ytd()
362
+ out = []
363
+ for key, m in _model()["metrics"].items():
364
+ if not m.get("validate"):
365
+ continue
366
+ out.append(validate_metric(key, yf, yt, None))
367
+ topic = _model()["topics"][m["topic"]]
368
+ if (topic.get("store") or {}).get("team_col"): # company-level topics have no BU runs
369
+ for tid in O.TEAM_IDS:
370
+ out.append(validate_metric(key, yf, yt, tid))
371
+ return out
372
+
373
+
374
+ # ---------------------------------------------------------------- store path (OM-2)
375
+ # The compiler behind run_semantic_query: registered metrics + whitelisted dims β†’ DuckDB SQL over
376
+ # the tenant store (harness/datastore.py). EVERY identifier comes from the versioned model files
377
+ # (trusted); every VALUE is parameterized β€” the small model only ever picks keys, so there is no
378
+ # injection surface. Live XML-RPC stays the validate() path (the store never validates itself).
379
+
380
+ def _store_con():
381
+ import harness.datastore as DS
382
+ if not DS.ready(): # a mid-backfill store returns PARTIAL totals silently
383
+ raise ModelError("the data cache is still warming up after a restart (a one-time first "
384
+ "sync) β€” the dashboards work now from live data; ask me again in a "
385
+ "moment and I'll have it")
386
+ return DS.ro_cursor() # cursor on the shared instance β€” no sync-writer contention
387
+
388
+
389
+ def _measure_sql(m, alias):
390
+ """SQL for a base measure. Supports FILTERED measures (`store_filter_sql` β€” the Omni
391
+ filtered-measure concept: sum(CASE WHEN … )) and `negate: true` (e.g. GL income, stored
392
+ credit-negative, reported positive)."""
393
+ agg, f = m.get("agg"), m.get("field")
394
+ flt = render_scope(m["store_filter_sql"]) if m.get("store_filter_sql") else None
395
+ if agg == "sum":
396
+ core = f"sum(CASE WHEN {flt} THEN {alias}.{f} ELSE 0 END)" if flt else f"sum({alias}.{f})"
397
+ elif agg == "count_distinct":
398
+ core = (f"count(DISTINCT CASE WHEN {flt} THEN {alias}.{f} END)" if flt
399
+ else f"count(DISTINCT {alias}.{f})")
400
+ elif agg == "count":
401
+ core = f"count(CASE WHEN {flt} THEN 1 END)" if flt else "count(*)"
402
+ else:
403
+ raise ModelError(f"metric {m['key']!r}: agg {agg!r} has no store expression")
404
+ return f"-({core})" if m.get("negate") else core
405
+
406
+
407
+ def _expand_measures(keys):
408
+ """Expand ratio/derived metrics into the base measures SQL must aggregate, keeping the
409
+ requested keys for post-compute. Returns (base_keys, requested_keys)."""
410
+ mts = _model()["metrics"]
411
+ base, seen = [], set()
412
+
413
+ def need(k):
414
+ if k in seen:
415
+ return
416
+ seen.add(k)
417
+ m = mts.get(k)
418
+ if not m:
419
+ raise ModelError(f"unknown metric {k!r}")
420
+ if m["agg"] == "ratio":
421
+ need(m["numerator"]); need(m["denominator"])
422
+ elif m["agg"] == "derived":
423
+ import ast as _ast
424
+ for node in _ast.walk(_ast.parse(m["expr"], mode="eval")):
425
+ if isinstance(node, _ast.Name):
426
+ need(node.id)
427
+ else:
428
+ if k not in base:
429
+ base.append(k)
430
+ for k in keys:
431
+ need(k)
432
+ return base, list(keys)
433
+
434
+
435
+ def _post_compute(row, key):
436
+ m = _model()["metrics"][key]
437
+ if m["agg"] == "ratio":
438
+ num, den = _post_compute(row, m["numerator"]), _post_compute(row, m["denominator"])
439
+ return (num / den) if den else 0.0
440
+ if m["agg"] == "derived":
441
+ return _eval_expr(m["expr"], lambda k: _post_compute(row, k))
442
+ return row.get(key) or 0.0
443
+
444
+
445
+ # A metric's declared `format` -> the grid field type the compiler and the cells both read.
446
+ # One mapping, so a new metric cannot arrive as an untyped column.
447
+ _FORMAT_TYPE = {"usd": "currency", "int": "int", "pct": "pct"}
448
+
449
+
450
+ def _clean_tree(tree, cols):
451
+ """Run a filter tree through the SAME validator the client's view state goes through.
452
+
453
+ `filter_sql`'s documented input contract is "feed me CLEANED trees" β€” it is faithful to the
454
+ TS engine rather than fail-closed on garbage, because rejecting garbage is the validator's
455
+ job. The UI path always cleans (view state is sanitised on the way into the store), but
456
+ store_query/store_rows are callable directly, and an uncleaned tree is where the two
457
+ diverge: `{"value": null}` is ACTIVE in TS (and throws on a text field) and INACTIVE here.
458
+
459
+ Normalising at the entry point makes that unreachable rather than merely unlikely, and
460
+ brings the depth / width / node-count caps to the server side too. `aios_grid` is a leaf
461
+ module with zero imports of its own, so this cannot cycle.
462
+ """
463
+ from aios_grid import (clean_filter_tree, COHORT_FIELD as _COHORT_FIELD,
464
+ MAX_FILTER_DEPTH, MAX_FILTER_NODES, MAX_FILTER_SIBLINGS)
465
+
466
+ # REFUSE an over-large tree rather than TRUNCATE it. clean_filter_tree caps siblings per
467
+ # level, total nodes and depth by DROPPING the excess β€” correct on the client, where the
468
+ # cap is a DoS guard on a per-row render loop and a dropped condition is the lesser evil.
469
+ # On the server it is not: dropping AND-conditions WIDENS the result, and store_rows then
470
+ # reports a row_count and scope-wide totals that look authoritative for a query the caller
471
+ # never asked for. That is precisely the silent [:N] that [[no-unverifiable-aggregates]]
472
+ # forbids. A caller that exceeds the caps gets an error, not a quietly different answer.
473
+ total = 0
474
+
475
+ def _walk(nodes, depth):
476
+ nonlocal total
477
+ nodes = list(nodes or [])
478
+ if len(nodes) > MAX_FILTER_SIBLINGS:
479
+ raise ModelError(f"filter tree has {len(nodes)} conditions at one level; the limit "
480
+ f"is {MAX_FILTER_SIBLINGS}. Refusing rather than dropping the "
481
+ f"excess, which would silently widen the result.")
482
+ for node in nodes:
483
+ if not isinstance(node, dict):
484
+ continue
485
+ total += 1
486
+ if isinstance(node.get('children'), list):
487
+ if depth >= MAX_FILTER_DEPTH:
488
+ raise ModelError(f"filter tree nests deeper than {MAX_FILTER_DEPTH} levels; "
489
+ f"refusing rather than dropping the deepest group.")
490
+ _walk(node['children'], depth + 1)
491
+
492
+ _walk(tree, 1)
493
+ if total > MAX_FILTER_NODES:
494
+ raise ModelError(f"filter tree has {total} nodes; the limit is {MAX_FILTER_NODES}. "
495
+ f"Refusing rather than truncating, which would silently widen it.")
496
+
497
+ # CG-8. A MEASURE condition ("Sales in the last 90 days > 5,000") is answered by
498
+ # harness/measure_filter.py, which resolves it to a SET OF CUSTOMER IDS. It has no meaning
499
+ # on this path, and both ways it could arrive here are silent:
500
+ # - on a topic without that column, `clean_filter_tree` DROPS it as unknown β€” widening the
501
+ # query while store_rows reports an authoritative row_count for something nobody asked
502
+ # for (the same class as the sibling/depth caps above);
503
+ # - on sales_lines at row grain, `revenue` IS a real column, so it would compile as a
504
+ # per-LINE predicate with the window silently discarded β€” a different question answered
505
+ # confidently, which is worse than dropping it.
506
+ # The discriminator is the `window`, not the column name: no column condition has one.
507
+ windowed = []
508
+ cohorts = []
509
+
510
+ def _find(nodes):
511
+ for node in nodes or []:
512
+ if not isinstance(node, dict):
513
+ continue
514
+ if isinstance(node.get('children'), list):
515
+ _find(node['children'])
516
+ elif node.get('window') is not None:
517
+ windowed.append(str(node.get('colId')))
518
+ elif node.get('colId') == _COHORT_FIELD:
519
+ cohorts.append(str(node.get('value')))
520
+
521
+ _find(tree)
522
+ # Owner item 5, and the same refusal for the same reason. A cohort leaf names a
523
+ # hand-curated set of CUSTOMERS held per user in the tenant store β€” this path knows nothing
524
+ # about it, and `clean_filter_tree` below would drop it as an unknown key, widening the
525
+ # query while store_rows still reported an authoritative row_count. If a caller ever needs
526
+ # it here, the fix is to pass the membership in, not to let it fall through.
527
+ if cohorts:
528
+ raise ModelError(
529
+ f"filter tree carries cohort-membership condition(s) {sorted(set(cohorts))} β€” "
530
+ f"cohort membership is host state, not a column of this topic. Refusing rather than "
531
+ f"dropping it, which would silently widen the result under an authoritative count.")
532
+ if windowed:
533
+ raise ModelError(
534
+ f"filter tree carries measure condition(s) over a date window {sorted(set(windowed))}"
535
+ f" β€” those are resolved to a set of ids by harness.measure_filter, not compiled into "
536
+ f"this query. Refusing rather than dropping or mis-compiling them, either of which "
537
+ f"would silently answer a different question under an authoritative count.")
538
+ return clean_filter_tree(tree, set(cols))
539
+
540
+
541
+ def _measure_row_sql(m, alias):
542
+ """A sum-metric's per-ROW value: the same expression `_measure_sql` aggregates, without the
543
+ aggregate wrapper. At line grain `revenue` IS `l.price_subtotal` for that line β€” deriving it
544
+ from the metric rather than hand-writing the column is what keeps ONE definition of the
545
+ number (the drift the semantic layer exists to prevent). Only `sum` has a row value; a
546
+ count/count_distinct metric is a property of a SET, not of a row."""
547
+ if m.get("agg") != "sum":
548
+ return None
549
+ f = m.get("field")
550
+ flt = render_scope(m["store_filter_sql"]) if m.get("store_filter_sql") else None
551
+ core = f"CASE WHEN {flt} THEN {alias}.{f} ELSE 0 END" if flt else f"{alias}.{f}"
552
+ return f"-({core})" if m.get("negate") else core
553
+
554
+
555
+ def store_columns(topic, include_measures=True, grain="aggregate"):
556
+ """The `{colId: {sql, type, aggregate}}` spec `harness.filter_sql` compiles against (CG-1).
557
+
558
+ `grain="row"` is the LINE-grain projection (CG-2): sum-metrics resolve to their own raw
559
+ field with `aggregate=False`, so a line table filters and sorts on real row values in WHERE
560
+ rather than needing HAVING. count-style metrics are dropped β€” they have no row value.
561
+
562
+ This is the seam between the grid's field contract and the semantic model: the grid speaks
563
+ field KEYS, the compiler needs SQL expressions and a field TYPE, and only the model may say
564
+ what a key resolves to. Nothing about a topic leaks into filter_sql itself.
565
+
566
+ <dim> -> the dim's NAME column (what the grid displays and what a user means when
567
+ they type "contains floral"), filtered PRE-aggregation in WHERE. Correct
568
+ even in a grouped query: every row of a group shares its group's name.
569
+ <dim>_id -> the raw id, for exact machine-keyed filtering.
570
+ date -> the topic's date column, so a grid can filter/sort it like any other field.
571
+ <metric> -> the aggregate expression, marked `aggregate` so the caller routes it to
572
+ HAVING rather than WHERE (see store_query β€” that path is not built yet).
573
+ """
574
+ t = _model()["topics"].get(topic)
575
+ if not t or "store" not in t:
576
+ raise ModelError(f"topic {topic!r} has no store binding")
577
+ s = t["store"]
578
+ cols = {}
579
+ for key, d in (s.get("dims") or {}).items():
580
+ cols[key] = {"sql": d.get("name_col") or d["col"], "type": "text", "aggregate": False}
581
+ # Only expose a separate id column when the dim actually HAS one. `city` is its own
582
+ # name (col == name_col), so a `city_id` would be a text column typed int β€” filtering
583
+ # it numerically would quietly compare 0 against 0 for every row.
584
+ if d.get("name_col") and d["name_col"] != d["col"]:
585
+ cols[f"{key}_id"] = {"sql": d["col"], "type": "int", "aggregate": False}
586
+ if s.get("date_col"):
587
+ cols["date"] = {"sql": s["date_col"], "type": "date", "aggregate": False}
588
+ if include_measures:
589
+ for k, m in _model()["metrics"].items():
590
+ if m.get("topic") != topic or m.get("agg") in ("ratio", "derived"):
591
+ continue # ratio/derived are post-computed, not SQL
592
+ # The field TYPE comes from the metric's declared format, never a guess: `units` is
593
+ # format:int, and typing it currency would render 791,960 units as dollars.
594
+ ftype = _FORMAT_TYPE.get(m.get("format"), "currency")
595
+ if grain == "row":
596
+ row_sql = _measure_row_sql(m, s["alias"])
597
+ if row_sql:
598
+ cols[k] = {"sql": row_sql, "type": ftype, "aggregate": False}
599
+ else:
600
+ cols[k] = {"sql": _measure_sql(m, s["alias"]),
601
+ "type": ftype, "aggregate": True}
602
+ return cols
603
+
604
+
605
+ def store_query(topic, measures, group_by=None, grain=None, date_from=None, date_to=None,
606
+ team_id=None, filters=None, sort=None, limit=1000, exclude_services=False,
607
+ filter_tree=None, filter_conj="and", today=None):
608
+ """The semantic query over the tenant store β€” the engine behind the Analyst's
609
+ run_semantic_query tool and the saved-view re-runner. All keys whitelisted against the model."""
610
+ t = _model()["topics"].get(topic)
611
+ if not t or "store" not in t:
612
+ raise ModelError(f"topic {topic!r} has no store binding")
613
+ s = t["store"]
614
+ alias = s["alias"]
615
+ dims = s.get("dims") or {}
616
+ limit = max(1, min(int(limit or 1000), 5000))
617
+
618
+ base_keys, requested = _expand_measures(list(measures or []))
619
+ if not base_keys:
620
+ raise ModelError("at least one measure required")
621
+ # Cross-topic base metrics (e.g. aov = revenue Γ· orders, where orders lives on sales_orders):
622
+ # in a SCALAR query they resolve via a nested scalar query on their HOME topic (correct grain β€”
623
+ # never count(*) on the wrong table); in a GROUPED query they are refused (split the request).
624
+ foreign = [k for k in base_keys if _model()["metrics"][k]["topic"] != topic]
625
+ if foreign and (group_by or grain):
626
+ raise ModelError(f"metrics {foreign} belong to another topic β€” cross-topic measures are "
627
+ f"scalar-only; run them against their own topic when grouping")
628
+ base_keys = [k for k in base_keys if k not in foreign]
629
+ foreign_vals = {}
630
+ for k in foreign:
631
+ sub = store_query(_model()["metrics"][k]["topic"], [k], date_from=date_from,
632
+ date_to=date_to, team_id=team_id)
633
+ foreign_vals[k] = sub["rows"][0][k] if sub["rows"] else 0
634
+ if not base_keys: # purely foreign scalar request
635
+ return {"topic": topic, "rows": [foreign_vals], "row_count": 1, "sql": "(nested)",
636
+ "measures": requested, "group_by": [], "grain": None}
637
+
638
+ select, group_cols, params = [], [], []
639
+ gb = [g for g in (group_by or []) if g]
640
+ for g in gb:
641
+ if g not in dims:
642
+ raise ModelError(f"group_by {g!r} not a dim of {topic!r} (allowed: {list(dims)})")
643
+ d = dims[g]
644
+ select.append(f"{d['col']} AS {g}_id" if d.get("name_col") else f"{d['col']} AS {g}")
645
+ group_cols.append(d["col"])
646
+ if d.get("name_col"):
647
+ select.append(f"max({d['name_col']}) AS {g}")
648
+ if grain:
649
+ if grain not in ("month", "week", "day"):
650
+ raise ModelError("grain must be month|week|day")
651
+ select.insert(0, f"date_trunc('{grain}', CAST({s['date_col']} AS TIMESTAMP)) AS period")
652
+ group_cols.insert(0, f"date_trunc('{grain}', CAST({s['date_col']} AS TIMESTAMP))")
653
+ for k in base_keys:
654
+ select.append(f"{_measure_sql(_model()['metrics'][k], alias)} AS {k}")
655
+
656
+ where = [render_scope(s["scope_sql"].strip())]
657
+ # TYPE-CORRECT date bounds (the 2025-07-01 lesson): columns hold ISO strings in TWO shapes β€”
658
+ # datetimes ('YYYY-MM-DD HH:MM:SS', sale date_order) and bare dates ('YYYY-MM-DD', GL date).
659
+ # Lexical string comparison EXCLUDES the window's first day for bare dates ('2025-07-01' <
660
+ # '2025-07-01 00:00:00'), which silently dropped a whole day of GL ($49,467). Cast both sides.
661
+ if date_from:
662
+ where.append(f"CAST({s['date_col']} AS TIMESTAMP) >= ?")
663
+ params.append(f"{date_from} 00:00:00")
664
+ if date_to:
665
+ where.append(f"CAST({s['date_col']} AS TIMESTAMP) <= ?")
666
+ params.append(f"{date_to} 23:59:59")
667
+ if team_id:
668
+ if not s.get("team_col"):
669
+ raise ModelError(f"topic {topic!r} is company-level β€” it has no business-unit filter")
670
+ where.append(f"{s['team_col']} = ?"); params.append(int(team_id))
671
+ if exclude_services and s.get("service_filter"):
672
+ where.append(render_scope(s["service_filter"]))
673
+ for dim_key, vals in (filters or {}).items():
674
+ if dim_key not in dims:
675
+ raise ModelError(f"filter dim {dim_key!r} not allowed (use: {list(dims)})")
676
+ # id dims filter by int; TEXT dims (e.g. city) filter by the value itself β€” either way
677
+ # every VALUE stays parameterized (the whitelist covers identifiers only)
678
+ def _coerce(v):
679
+ try:
680
+ return int(v)
681
+ except (TypeError, ValueError):
682
+ return str(v)
683
+ ids = [_coerce(v) for v in (vals if isinstance(vals, (list, tuple)) else [vals])]
684
+ where.append(f"{dims[dim_key]['col']} IN ({','.join('?' for _ in ids)})")
685
+ params.extend(ids)
686
+
687
+ # The grid's filter TREE (CG-1). `filters` above stays the Analyst's flat dim=IN shape;
688
+ # this is the nested, 11-operator contract the table UI emits. Compiled by
689
+ # harness/filter_sql, which is held in lock-step with the TS engine and the validator by
690
+ # aios-web/verify_filter_engine.py. Appended AFTER the dim filters so params stay in
691
+ # positional order with `where`.
692
+ if filter_tree:
693
+ from harness import filter_sql as _fs
694
+ cols = store_columns(topic)
695
+ pred = _fs.compile_filter_tree(_clean_tree(filter_tree, cols), filter_conj, cols,
696
+ today=today)
697
+ if pred is not None:
698
+ if pred.uses_aggregate:
699
+ bad = sorted(c for c in pred.columns_used if cols[c].get("aggregate"))
700
+ raise ModelError(
701
+ f"filter_tree references measure(s) {bad} β€” a measure filter has to land in "
702
+ f"HAVING, and that path is deliberately NOT built: CG-1 ships the WHERE path "
703
+ f"because its consumer (CG-2, the line-grain sales table) does not aggregate. "
704
+ f"Filter on dims, or aggregate first and filter the result.")
705
+ where.append(pred.sql)
706
+ params.extend(pred.params)
707
+
708
+ sql = (f"SELECT {', '.join(select)} FROM {s['table']} {alias} {s.get('join','')} "
709
+ f"WHERE {' AND '.join(where)}")
710
+ if group_cols:
711
+ sql += f" GROUP BY {', '.join(group_cols)}"
712
+ if sort:
713
+ key = sort.lstrip("-")
714
+ if key not in base_keys + gb + (["period"] if grain else []):
715
+ raise ModelError(f"sort {sort!r} must reference a selected measure/dim")
716
+ sql += f" ORDER BY {key} {'DESC' if sort.startswith('-') else 'ASC'}"
717
+ elif grain:
718
+ sql += " ORDER BY period"
719
+ sql += f" LIMIT {limit}"
720
+
721
+ con = _store_con()
722
+ try:
723
+ cur = con.execute(sql, params)
724
+ cols = [d[0] for d in cur.description]
725
+ rows = [dict(zip(cols, r)) for r in cur.fetchall()]
726
+ finally:
727
+ con.close()
728
+ for row in rows: # ratio/derived post-compute per group
729
+ row.update(foreign_vals) # scalar cross-topic components
730
+ for k in requested:
731
+ if _model()["metrics"][k]["agg"] in ("ratio", "derived"):
732
+ row[k] = _post_compute(row, k)
733
+ if "period" in row and row["period"] is not None:
734
+ row["period"] = str(row["period"])[:10]
735
+ return {"topic": topic, "rows": rows, "row_count": len(rows), "sql": sql,
736
+ "measures": requested, "group_by": gb, "grain": grain}
737
+
738
+
739
+ def store_rows(topic, date_from=None, date_to=None, team_id=None, filter_tree=None,
740
+ filter_conj="and", search=None, sorts=None, limit=200, offset=0,
741
+ aggregates=None, exclude_services=False, id_sql=None, member_ids=None,
742
+ today=None):
743
+ """ROW-grain fetch β€” the line-grain counterpart to store_query's aggregate path (CG-2).
744
+
745
+ store_query answers "what is the number"; this answers "which rows". At line grain the
746
+ payload is the binding constraint (254,837 sale_order_lines against 1,550 customers β€” the
747
+ whole book would be ~96 MB at the customer table's measured 377 B/row), so rows come back
748
+ WINDOWED.
749
+
750
+ A window is only honest if the counts and totals are not windowed with it
751
+ ([[no-unverifiable-aggregates]]). So this returns THREE independent numbers, each from its
752
+ own query over the FULL scope, never from `len(rows)`:
753
+
754
+ row_count rows matching the filter across the whole scope -> the N of "N of M"
755
+ total_count rows in the scope with no filter at all -> the M
756
+ aggregates the topic's OWN metric definitions, summed over the whole FILTERED scope
757
+
758
+ That is the difference between a windowed fetch and a silent `[:N]` cap: the user is told
759
+ how many rows exist and shown totals for all of them, while receiving only the page they
760
+ can see.
761
+
762
+ Filters/sorts/search compile through harness.filter_sql, so a line table means EXACTLY what
763
+ the client engine means at customer grain β€” that equivalence is what verify_filter_engine.py
764
+ exists to hold.
765
+
766
+ Dates are returned already truncated to the 10-char ISO shape the filter compares against,
767
+ so what is displayed and what is filtered are the same string. Numerics come back RAW; the
768
+ payload layer rounds them (aios_grid._round), and the compiler mirrors that rounding.
769
+ """
770
+ t = _model()["topics"].get(topic)
771
+ if not t or "store" not in t:
772
+ raise ModelError(f"topic {topic!r} has no store binding")
773
+ from harness import filter_sql as _fs
774
+ s = t["store"]
775
+ alias = s["alias"]
776
+ cols = store_columns(topic, grain="row")
777
+ id_sql = id_sql or f"{alias}.id"
778
+ limit = max(1, min(int(limit or 200), 5000))
779
+ offset = max(0, int(offset or 0))
780
+
781
+ # --- the scope every one of the three queries shares -------------------------------
782
+ scope, scope_params = [render_scope(s["scope_sql"].strip())], []
783
+ if date_from:
784
+ scope.append(f"CAST({s['date_col']} AS TIMESTAMP) >= ?")
785
+ scope_params.append(f"{date_from} 00:00:00")
786
+ if date_to:
787
+ scope.append(f"CAST({s['date_col']} AS TIMESTAMP) <= ?")
788
+ scope_params.append(f"{date_to} 23:59:59")
789
+ if team_id:
790
+ if not s.get("team_col"):
791
+ raise ModelError(f"topic {topic!r} is company-level β€” it has no business-unit filter")
792
+ scope.append(f"{s['team_col']} = ?")
793
+ scope_params.append(int(team_id))
794
+ if exclude_services and s.get("service_filter"):
795
+ scope.append(render_scope(s["service_filter"]))
796
+
797
+ # --- the user's narrowing, on top of the scope --------------------------------------
798
+ narrow, narrow_params = [], []
799
+ pred = _fs.compile_filter_tree(_clean_tree(filter_tree or [], cols), filter_conj, cols,
800
+ today=today,
801
+ member_ids=member_ids or None, id_sql=id_sql)
802
+ if pred is not None:
803
+ if pred.uses_aggregate: # cannot happen at grain="row"; guard anyway
804
+ raise ModelError(f"row-grain filter referenced an aggregate: {sorted(pred.columns_used)}")
805
+ narrow.append(pred.sql)
806
+ narrow_params.extend(pred.params)
807
+ spred = _fs.compile_search(search, cols)
808
+ if spred is not None:
809
+ narrow.append(spred.sql)
810
+ narrow_params.extend(spred.params)
811
+
812
+ frm = f"{s['table']} {alias} {s.get('join', '')}"
813
+ scope_where = " AND ".join(scope)
814
+ all_where = " AND ".join(scope + narrow)
815
+ con = _store_con()
816
+ try:
817
+ # 1. the WINDOW of rows
818
+ select = [f"{id_sql} AS _rid"] + [
819
+ (f"SUBSTR(CAST({c['sql']} AS VARCHAR), 1, 10) AS {k}" if c["type"] == "date"
820
+ else f"{c['sql']} AS {k}")
821
+ for k, c in cols.items()]
822
+ order = _fs.compile_order_by(sorts, cols, tiebreak_sql=id_sql) or f"{id_sql} ASC"
823
+ sql = (f"SELECT {', '.join(select)} FROM {frm} WHERE {all_where} "
824
+ f"ORDER BY {order} LIMIT {limit} OFFSET {offset}")
825
+ cur = con.execute(sql, scope_params + narrow_params)
826
+ names = [d[0] for d in cur.description]
827
+ rows = [dict(zip(names, r)) for r in cur.fetchall()]
828
+
829
+ # 2. the two counts β€” over the WHOLE scope, never len(rows)
830
+ count_sql = f"SELECT count(*) FROM {frm} WHERE {all_where}"
831
+ row_count = con.execute(count_sql, scope_params + narrow_params).fetchone()[0]
832
+ total_count = con.execute(f"SELECT count(*) FROM {frm} WHERE {scope_where}",
833
+ scope_params).fetchone()[0]
834
+
835
+ # 3. the aggregates β€” the topic's OWN metric definitions over the whole FILTERED scope
836
+ mts = _model()["metrics"]
837
+ keys = [k for k in (aggregates if aggregates is not None
838
+ else [k for k, m in mts.items()
839
+ if m.get("topic") == topic and m.get("agg") == "sum"])]
840
+ aggs = {}
841
+ if keys:
842
+ for k in keys:
843
+ if k not in mts or mts[k].get("topic") != topic:
844
+ raise ModelError(f"aggregate {k!r} is not a metric of topic {topic!r}")
845
+ agg_sql = ("SELECT " + ", ".join(f"{_measure_sql(mts[k], alias)} AS {k}" for k in keys)
846
+ + f" FROM {frm} WHERE {all_where}")
847
+ cur = con.execute(agg_sql, scope_params + narrow_params)
848
+ aggs = dict(zip([d[0] for d in cur.description], cur.fetchone()))
849
+ finally:
850
+ con.close()
851
+
852
+ return {
853
+ "topic": topic,
854
+ "rows": rows,
855
+ "columns": {k: {"type": c["type"]} for k, c in cols.items()},
856
+ "window": {"offset": offset, "limit": limit, "returned": len(rows)},
857
+ "row_count": row_count, # N β€” matches the filter, across the WHOLE scope
858
+ "total_count": total_count, # M β€” the unfiltered scope
859
+ "aggregates": aggs, # over the whole FILTERED scope, not the window
860
+ "sql": sql,
861
+ "count_sql": count_sql,
862
+ }
863
+
864
+
865
+ def store_field_values(topic, dim, search=None, limit=25):
866
+ """Resolve real filterable values for a dim (the Analyst's get_field_values tool β€” fixes
867
+ 'NYC Florist' vs the actual record name BEFORE querying)."""
868
+ t = _model()["topics"].get(topic)
869
+ if not t or "store" not in t:
870
+ raise ModelError(f"topic {topic!r} has no store binding")
871
+ d = (t["store"].get("dims") or {}).get(dim)
872
+ if not d:
873
+ raise ModelError(f"unknown dim {dim!r} for topic {topic!r}")
874
+ if not d.get("name_col"):
875
+ raise ModelError(f"dim {dim!r} has no name column (filter by id)")
876
+ s = t["store"]
877
+ sql = (f"SELECT DISTINCT {d['col']} AS id, {d['name_col']} AS name "
878
+ f"FROM {s['table']} {s['alias']} {s.get('join','')} "
879
+ f"WHERE {render_scope(s['scope_sql'].strip())}")
880
+ params = []
881
+ if search:
882
+ sql += f" AND {d['name_col']} ILIKE ?"
883
+ params.append(f"%{search}%")
884
+ sql += f" ORDER BY name LIMIT {max(1, min(int(limit), 100))}"
885
+ con = _store_con()
886
+ try:
887
+ return [{"id": r[0], "name": r[1]} for r in con.execute(sql, params).fetchall()]
888
+ finally:
889
+ con.close()
890
+
891
+
892
+ def store_parity(date_from=None, date_to=None):
893
+ """THE OM-2 gate: the store path must equal the live path to the cent, per metric per scope."""
894
+ import core.periods as P
895
+ if not date_from:
896
+ date_from, date_to = P.ytd()
897
+ out = []
898
+ for key in ("revenue", "margin", "units", "orders", "customers", "returns", "invoiced"):
899
+ m = _model()["metrics"][key]
900
+ # ⚠ A COMPANY-LEVEL topic is checked at company scope ONLY. `store_query` RAISES when a
901
+ # team is asked of a topic with no team_col, and the live builder refuses too β€” so
902
+ # looping the BUs here would fail the gate for a topic that is correct. Parity for a
903
+ # company-level number means company-level parity; pretending otherwise would either
904
+ # crash or, worse, compare two numbers that silently ignored the team.
905
+ _store = (_model()["topics"].get(m["topic"]) or {}).get("store") or {}
906
+ scopes = (None, *O.TEAM_IDS) if _store.get("team_col") else (None,)
907
+ for tid in scopes:
908
+ live = metric(key, date_from, date_to, tid)
909
+ res = store_query(m["topic"], [key], date_from=date_from, date_to=date_to, team_id=tid)
910
+ ours = res["rows"][0][key] if res["rows"] else 0
911
+ gap = abs((ours or 0) - (live or 0))
912
+ out.append({"check": f"store.{key} == live.{key} (team={tid})",
913
+ "ok": gap <= 0.01, "gap": round(gap, 4), "store": ours, "live": live})
914
+ # The agent dim (2026-07-17): store agent-filtered revenue must equal live revenue over the
915
+ # agent's FIRST-agent book β€” a fully independent path (live partner read + line-domain sum)
916
+ # against the store's res_partner.agent_id attribution. Largest book = the sharpest check.
917
+ recs = O.search_read("res.partner", [("active", "in", [True, False]),
918
+ ("agent_ids", "!=", False)], ["agent_ids"], limit=100000)
919
+ books = {}
920
+ for r in recs:
921
+ a = (r.get("agent_ids") or [None])[0]
922
+ if a:
923
+ books.setdefault(a, []).append(r["id"])
924
+ if books:
925
+ top = max(books, key=lambda k: len(books[k]))
926
+ live = O.sum_field("sale.order.line",
927
+ O.sale_line_domain(date_from, date_to, partner_ids=books[top]),
928
+ "price_subtotal")
929
+ res = store_query("sales_lines", ["revenue"], date_from=date_from, date_to=date_to,
930
+ filters={"agent": [top]})
931
+ ours = res["rows"][0]["revenue"] if res["rows"] else 0
932
+ gap = abs((ours or 0) - (live or 0))
933
+ out.append({"check": f"store.revenue[agent={top}] == live book sum (first-agent, "
934
+ f"{len(books[top])} customers)",
935
+ "ok": gap <= 0.01, "gap": round(gap, 4), "store": ours, "live": live})
936
+ return out
royalimports_os/harness/skills/__init__.py ADDED
@@ -0,0 +1 @@
 
 
1
+ """Portable skills β€” written once against the canonical model, run on any compatible connector."""
royalimports_os/harness/skills/sales.py ADDED
@@ -0,0 +1,168 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Sales skill β€” PORTABLE. The de-Odoo'd replacement for modules/sales.py: every metric is expressed
2
+ against the canonical model and runs on any connector with the SALES capability (INVOICING for
3
+ returns). No model names, no `amount_untaxed`, no Odoo β€” the connector translates.
4
+
5
+ `validate(tenant)` parity-checks each metric against the current Odoo module, so a port only replaces
6
+ the original once it ties out to the cent (same discipline as the modules' own validate()).
7
+
8
+ Date windows come from core.periods (calendar math is source-agnostic).
9
+ """
10
+ from __future__ import annotations
11
+ import sys
12
+ from pathlib import Path
13
+ sys.path.insert(0, str(Path(__file__).resolve().parents[2])) # royalimports_os/
14
+ import core.periods as P
15
+ from harness.canonical import Capability, Entity, Query, Filter
16
+ from harness.base import Tenant
17
+
18
+ REQUIRES = (Capability.SALES,)
19
+
20
+
21
+ # ------------------------------------------------------------------ tenant scope -> canonical filters
22
+ def _scope(tenant: Tenant, business_unit: str | None):
23
+ """The client's scope CONFIG (was hard-coded Odoo constants) as canonical filters β€” valid on both
24
+ SALES_ORDER and SALES_ORDER_LINE (status/business_unit/customer exist on both)."""
25
+ cfg = tenant.config
26
+ fs = []
27
+ if cfg.get('confirmed_statuses'):
28
+ fs.append(Filter('status', 'in', list(cfg['confirmed_statuses'])))
29
+ bus = cfg.get('business_units') or {}
30
+ if business_unit and business_unit in bus:
31
+ fs.append(Filter('business_unit', 'in', [bus[business_unit]]))
32
+ elif bus:
33
+ fs.append(Filter('business_unit', 'in', list(bus.values())))
34
+ if cfg.get('excluded_customer_ids'):
35
+ fs.append(Filter('customer', 'not_in', list(cfg['excluded_customer_ids'])))
36
+ return tuple(fs)
37
+
38
+
39
+ def _sum(conn, entity, measure, df, dt, filters):
40
+ return sum(r[measure] for r in conn.aggregate(
41
+ Query(entity, (measure,), 'sum', date_from=df, date_to=dt, filters=filters)))
42
+
43
+
44
+ def _count(conn, entity, df, dt, filters):
45
+ return sum(r['count'] for r in conn.aggregate(
46
+ Query(entity, (), 'count', date_from=df, date_to=dt, filters=filters)))
47
+
48
+
49
+ # ------------------------------------------------------------------ metrics (portable)
50
+ def revenue(tenant, date_from, date_to, business_unit=None):
51
+ return _sum(tenant.source(Capability.SALES), Entity.SALES_ORDER, 'amount',
52
+ date_from, date_to, _scope(tenant, business_unit))
53
+
54
+
55
+ def headline(tenant, business_unit=None, t=None):
56
+ conn = tenant.source(Capability.SALES)
57
+ t = t or P.today()
58
+ yf, yt = P.ytd(t)
59
+ lf, lt = P.ytd_last_year(t)
60
+ sc = _scope(tenant, business_unit)
61
+ rev = _sum(conn, Entity.SALES_ORDER, 'amount', yf, yt, sc)
62
+ rev_ly = _sum(conn, Entity.SALES_ORDER, 'amount', lf, lt, sc)
63
+ orders = _count(conn, Entity.SALES_ORDER, yf, yt, sc)
64
+ custs = conn.distinct_count(Query(Entity.SALES_ORDER, date_from=yf, date_to=yt, filters=sc), 'customer')
65
+ return {'ytd_revenue': rev, 'ytd_revenue_ly': rev_ly, 'yoy_pct': P.yoy_pct(rev, rev_ly),
66
+ 'ytd_orders': orders, 'ytd_customers': custs, 'aov': (rev / orders) if orders else 0.0}
67
+
68
+
69
+ def monthly_trend(tenant, n=13, business_unit=None, t=None):
70
+ conn = tenant.source(Capability.SALES)
71
+ t = t or P.today()
72
+ first = P.month_starts(n, t)[0][1]
73
+ span_from = f"{int(first[:4]) - 1:04d}{first[4:]}" # a year before the first shown month
74
+ q = Query(Entity.SALES_ORDER, ('amount',), 'sum', grain='month',
75
+ date_from=span_from, date_to=t.isoformat(), filters=_scope(tenant, business_unit))
76
+ m = {r['period']: r['amount'] for r in conn.aggregate(q)}
77
+ rows = []
78
+ for ym, _s, _e in P.month_starts(n, t):
79
+ y, mo = int(ym[:4]) - 1, int(ym[5:7])
80
+ this, last = m.get(ym, 0.0), m.get(f'{y:04d}-{mo:02d}', 0.0)
81
+ rows.append({'month': ym, 'revenue': this, 'revenue_ly': last, 'yoy_pct': P.yoy_pct(this, last)})
82
+ return rows
83
+
84
+
85
+ def top_customers(tenant, limit=25, business_unit=None, t=None):
86
+ conn = tenant.source(Capability.SALES)
87
+ yf, yt = P.ytd(t or P.today())
88
+ q = Query(Entity.SALES_ORDER, ('amount',), 'sum', group_by=('customer',),
89
+ date_from=yf, date_to=yt, filters=_scope(tenant, business_unit))
90
+ rows = [{'pid': r['customer']['id'], 'customer': r['customer']['name'],
91
+ 'revenue': r['amount'], 'orders': r['count']}
92
+ for r in conn.aggregate(q) if r.get('customer')]
93
+ return sorted(rows, key=lambda x: -x['revenue'])[:limit]
94
+
95
+
96
+ def top_skus(tenant, limit=25, business_unit=None, t=None):
97
+ conn = tenant.source(Capability.SALES)
98
+ yf, yt = P.ytd(t or P.today())
99
+ q = Query(Entity.SALES_ORDER_LINE, ('amount', 'qty', 'margin'), 'sum', group_by=('product',),
100
+ date_from=yf, date_to=yt, filters=_scope(tenant, business_unit))
101
+ rows = []
102
+ for r in conn.aggregate(q):
103
+ if not r.get('product'):
104
+ continue
105
+ lines = r['count'] or 0
106
+ rows.append({'product': r['product']['name'], 'pid': r['product']['id'], 'revenue': r['amount'],
107
+ 'qty': r['qty'], 'margin': r['margin'],
108
+ 'profit_per_order': (r['margin'] / lines) if lines else 0.0})
109
+ return sorted(rows, key=lambda x: -x['revenue'])[:limit]
110
+
111
+
112
+ def by_category(tenant, limit=20, business_unit=None, t=None):
113
+ conn = tenant.source(Capability.SALES)
114
+ yf, yt = P.ytd(t or P.today())
115
+ q = Query(Entity.SALES_ORDER_LINE, ('amount',), 'sum', group_by=('category',),
116
+ date_from=yf, date_to=yt, filters=_scope(tenant, business_unit))
117
+ rows = [{'category': r['category'], 'revenue': r['amount']} for r in conn.aggregate(q)]
118
+ return sorted(rows, key=lambda x: -x['revenue'])[:limit]
119
+
120
+
121
+ def returns_monthly(tenant, n=13, t=None):
122
+ """Returns (credit notes) per month β€” needs INVOICING, consolidated (no BU split)."""
123
+ conn = tenant.source(Capability.INVOICING)
124
+ t = t or P.today()
125
+ first = P.month_starts(n, t)[0][1]
126
+ span_from = f"{int(first[:4]) - 1:04d}{first[4:]}"
127
+ ex = tenant.config.get('excluded_customer_ids') or []
128
+ filters = [Filter('doc_type', 'eq', 'out_refund'), Filter('status', 'eq', 'posted')]
129
+ if ex:
130
+ filters.append(Filter('customer', 'not_in', list(ex)))
131
+ q = Query(Entity.INVOICE, ('amount',), 'sum', grain='month',
132
+ date_from=span_from, date_to=t.isoformat(), filters=tuple(filters))
133
+ m = {r['period']: -r['amount'] for r in conn.aggregate(q)} # out_refund amount is negative -> flip
134
+ rows = []
135
+ for ym, _s, _e in P.month_starts(n, t):
136
+ y, mo = int(ym[:4]) - 1, int(ym[5:7])
137
+ this, last = m.get(ym, 0.0), m.get(f'{y:04d}-{mo:02d}', 0.0)
138
+ rows.append({'month': ym, 'revenue': this, 'revenue_ly': last, 'yoy_pct': P.yoy_pct(this, last)})
139
+ return rows
140
+
141
+
142
+ # ------------------------------------------------------------------ parity validation (port gate)
143
+ def validate(tenant, t=None):
144
+ """Reconcile the ported skill to the current Odoo module β€” a port ships only when it ties out."""
145
+ import modules.sales as M
146
+ t = t or P.today()
147
+ yf, yt = P.ytd(t)
148
+ checks = []
149
+
150
+ def chk(name, a, b, tol=1.0):
151
+ checks.append({'check': name, 'a': round(a, 2), 'b': round(b, 2),
152
+ 'gap': round(a - b, 2), 'ok': abs(a - b) <= tol})
153
+
154
+ h, mh = headline(tenant), M.headline()
155
+ chk('headline YTD revenue == module', h['ytd_revenue'], mh['ytd_revenue'])
156
+ chk('headline YTD orders == module', h['ytd_orders'], mh['ytd_orders'])
157
+ chk('headline YTD customers == module', h['ytd_customers'], mh['ytd_customers'])
158
+ chk('Ξ£ top_customers == module', sum(r['revenue'] for r in top_customers(tenant, 10**9)),
159
+ sum(r['revenue'] for r in M.top_customers(limit=10**9)))
160
+ chk('Ξ£ by_category == module', sum(r['revenue'] for r in by_category(tenant, 10**9)),
161
+ sum(r['revenue'] for r in M.by_category(limit=10**9)))
162
+ chk('Ξ£ top_skus revenue == module', sum(r['revenue'] for r in top_skus(tenant, 10**9)),
163
+ sum(r['revenue'] for r in M.top_skus(limit=10**9)))
164
+ chk('monthly_trend Ξ£ this-yr == module', sum(r['revenue'] for r in monthly_trend(tenant)),
165
+ sum(r['revenue'] for r in M.monthly_trend()))
166
+ chk('returns_monthly Ξ£ this-yr == module', sum(r['revenue'] for r in returns_monthly(tenant)),
167
+ sum(r['revenue'] for r in M.returns_monthly()))
168
+ return checks
royalimports_os/harness/tables.py ADDED
@@ -0,0 +1,129 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """harness/tables.py β€” a topic-backed TABLE for the grid (CG-2).
2
+
3
+ The grid's first table (customers) is built from `modules.customer_data.pool()`: one row per
4
+ customer, the whole book shipped as component args, filtered client-side. That works at 1,550
5
+ rows and does not work at line grain β€” `sales_lines` is 201,558 rows / ~89 MB.
6
+
7
+ This module builds the SECOND table: rows come from `semantic.store_rows`, so the fetch is
8
+ WINDOWED and the filtering is SQL. Its whole job is to hand the grid a payload in the same
9
+ host-neutral shape the React component already speaks, plus the one thing a windowed table must
10
+ carry that a whole-book table never needed:
11
+
12
+ counts = {matched, total, shown, windowed}
13
+
14
+ `matched` and `total` come from their own queries over the FULL scope; `shown` is the size of
15
+ this window. The component renders "showing SHOWN of MATCHED (filtered from TOTAL)" from those
16
+ numbers and NEVER from `rows.length` β€” which is the difference between an honest window and a
17
+ silent `[:N]` wearing a total ([[no-unverifiable-aggregates]]).
18
+
19
+ The field CONTRACT is derived from the topic, not hand-written: labels and types come from the
20
+ model's dims and metrics, so a new metric cannot arrive as an untyped, unlabelled column and the
21
+ grid can never disagree with the semantic layer about what a column means.
22
+ """
23
+ from harness import semantic as _sem
24
+
25
+ #: Topic-backed tables the grid can open. `key` is the storage key its saved views live under β€”
26
+ #: separate from the customer table's, so the two tables' views can never collide.
27
+ TOPIC_TABLES = {
28
+ "sales_lines": {
29
+ "label": "Sales lines",
30
+ "storage_key": "sales_lines_table_workspace",
31
+ # the columns worth defaulting to visible, in order; everything else stays available
32
+ "default_visible": ["date", "order_partner", "product", "category", "state", "agent",
33
+ "team", "revenue", "units", "margin"],
34
+ # a line-grain table is unusable sorted by nothing in particular
35
+ "default_sorts": [{"colId": "date", "dir": "desc"}],
36
+ },
37
+ }
38
+
39
+ #: how many rows one window carries. Small enough that the payload stays a few hundred KB at
40
+ #: ~440 B/row, large enough to scroll. The honest counts make the window visible rather than
41
+ #: silent, so this is a performance knob and not a correctness one.
42
+ PAGE_ROWS = 200
43
+
44
+
45
+ def _labels(topic):
46
+ """colId -> human label, taken from the MODEL (dim labels, metric labels)."""
47
+ t = _sem.topics()[topic]
48
+ out = {}
49
+ for key, d in ((t.get("store") or {}).get("dims") or {}).items():
50
+ out[key] = d.get("label") or key.replace("_", " ").title()
51
+ out[f"{key}_id"] = f"{out[key]} id"
52
+ out["date"] = "Date"
53
+ for k, m in _sem.metrics().items():
54
+ if m.get("topic") == topic:
55
+ out[k] = m.get("label") or k
56
+ return out
57
+
58
+
59
+ def table_fields(topic):
60
+ """The grid field contract for a topic, derived from the semantic model.
61
+
62
+ Every column is `source='odoo'` β€” read-only. A line is a fact from the ERP; unlike the
63
+ customer table there is no overlay stratum here (notes/tags hang off a CUSTOMER, not off an
64
+ individual order line), so the two-strata split simply does not arise.
65
+ """
66
+ cfg = TOPIC_TABLES[topic]
67
+ cols = _sem.store_columns(topic, grain="row")
68
+ labels = _labels(topic)
69
+ order = [k for k in cfg["default_visible"] if k in cols]
70
+ order += [k for k in cols if k not in order]
71
+ return [{
72
+ "key": k,
73
+ "label": labels.get(k, k.replace("_", " ").title()),
74
+ "type": cols[k]["type"],
75
+ "source": "odoo",
76
+ "default": k in cfg["default_visible"],
77
+ } for k in order]
78
+
79
+
80
+ def table_payload(topic, config=None, offset=0, limit=None, team_id=None,
81
+ date_from=None, date_to=None):
82
+ """Fetch ONE window of a topic-backed table plus its scope-wide counts and totals.
83
+
84
+ `config` is a grid ViewConfig (filters / filterConj / sorts / search) β€” the same shape the
85
+ customer table persists, so a saved view means the same thing on either table.
86
+
87
+ Returns the component payload: {fields, rows, counts, aggregates, storageKey}.
88
+ """
89
+ if topic not in TOPIC_TABLES:
90
+ raise _sem.ModelError(f"no topic-backed table registered for {topic!r}")
91
+ cfg = TOPIC_TABLES[topic]
92
+ config = config or {}
93
+ limit = PAGE_ROWS if limit is None else limit
94
+
95
+ got = _sem.store_rows(
96
+ topic,
97
+ filter_tree=config.get("filters") or [],
98
+ filter_conj=config.get("filterConj") or "and",
99
+ search=config.get("search") or None,
100
+ sorts=config.get("sorts") or cfg["default_sorts"],
101
+ member_ids=config.get("memberPids") or None,
102
+ limit=limit, offset=offset,
103
+ team_id=team_id, date_from=date_from, date_to=date_to,
104
+ )
105
+
106
+ rows = []
107
+ for i, r in enumerate(got["rows"]):
108
+ # `pid` is the grid's row identity contract (selection, detail, overlay all key on it).
109
+ # At line grain that identity is the order LINE id, which store_rows returns as _rid.
110
+ row = {k: v for k, v in r.items() if k != "_rid"}
111
+ row["pid"] = r["_rid"]
112
+ rows.append(row)
113
+
114
+ return {
115
+ "fields": table_fields(topic),
116
+ "rows": rows,
117
+ # THE honest window. `matched`/`total` are scope-wide queries; `shown` is this window.
118
+ # A caller must never recompute `matched` from len(rows).
119
+ "counts": {
120
+ "shown": len(rows),
121
+ "matched": got["row_count"],
122
+ "total": got["total_count"],
123
+ "windowed": True,
124
+ },
125
+ "aggregates": got["aggregates"], # over the whole FILTERED scope, not the window
126
+ "storageKey": cfg["storage_key"],
127
+ "label": cfg["label"],
128
+ "window": got["window"],
129
+ }
royalimports_os/harness/telemetry.py ADDED
@@ -0,0 +1,94 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """harness/telemetry.py β€” usage telemetry v0 (OM-3; principle 7: the improvement loop is
2
+ structural). Append-only JSONL of what people actually do β€” pages opened, explore queries run,
3
+ Analyst questions asked β€” the observe-half of just-in-time modeling: what's used gets promoted
4
+ into the governed model; what's ignored gets pruned.
5
+
6
+ Fail-silent by design: telemetry must never break a page or a query. Local file v0
7
+ (data/store/usage.jsonl, tenant data, git-ignored); mined in-session for now, store-backed later.
8
+ """
9
+ import json
10
+ import sys
11
+ import time
12
+ import traceback
13
+ from pathlib import Path
14
+
15
+ PATH = Path(__file__).resolve().parents[1] / "data" / "store" / "usage.jsonl"
16
+ ERR_PATH = PATH.parent / "errors.jsonl"
17
+
18
+
19
+ def log(kind, **fields):
20
+ try:
21
+ PATH.parent.mkdir(parents=True, exist_ok=True)
22
+ with PATH.open("a", encoding="utf-8") as f:
23
+ f.write(json.dumps({"ts": time.strftime("%Y-%m-%d %H:%M:%S"),
24
+ "kind": kind, **fields}, default=str) + "\n")
25
+ except Exception:
26
+ pass
27
+
28
+
29
+ def error(where, exc=None, **fields):
30
+ """THE central error sink (owner rule 2026-07-23: ALL errors are logged). One JSONL line per
31
+ caught error (data/store/errors.jsonl β€” per-deployment, git-ignored) + a mirror line on
32
+ stderr so it also lands in the Space container logs. Call it from EVERY except block that
33
+ swallows or downgrades an exception β€” a failure with nowhere to land is how the app breaks
34
+ silently (the 2026-07-23 drawer fire). Never raises."""
35
+ try:
36
+ rec = {"ts": time.strftime("%Y-%m-%d %H:%M:%S"), "where": str(where)[:120], **fields}
37
+ if exc is not None:
38
+ rec["type"] = type(exc).__name__
39
+ rec["msg"] = str(exc)[:400]
40
+ try:
41
+ tb = "".join(traceback.format_exception(type(exc), exc, exc.__traceback__,
42
+ limit=8))
43
+ if tb.strip():
44
+ rec["trace"] = tb[-1800:]
45
+ except Exception:
46
+ pass
47
+ ERR_PATH.parent.mkdir(parents=True, exist_ok=True)
48
+ with ERR_PATH.open("a", encoding="utf-8") as f:
49
+ f.write(json.dumps(rec, default=str) + "\n")
50
+ print(f"[app-error] {rec['ts']} {rec['where']}: "
51
+ f"{rec.get('type', '')} {rec.get('msg', '')}", file=sys.stderr, flush=True)
52
+ except Exception:
53
+ pass
54
+
55
+
56
+ def read_errors(limit=500):
57
+ """Tail of the error log (newest last) β€” the Data Health admin section reads this."""
58
+ if not ERR_PATH.exists():
59
+ return []
60
+ out = []
61
+ try:
62
+ with ERR_PATH.open(encoding="utf-8") as f:
63
+ for line in f:
64
+ try:
65
+ out.append(json.loads(line))
66
+ except json.JSONDecodeError:
67
+ continue
68
+ except Exception:
69
+ return out
70
+ return out[-limit:]
71
+
72
+
73
+ def read(limit=5000):
74
+ if not PATH.exists():
75
+ return []
76
+ out = []
77
+ with PATH.open(encoding="utf-8") as f:
78
+ for line in f:
79
+ try:
80
+ out.append(json.loads(line))
81
+ except json.JSONDecodeError:
82
+ continue
83
+ return out[-limit:]
84
+
85
+
86
+ def summary():
87
+ """Counts by kind + the most-touched pages/topics β€” the promote-or-prune shortlist."""
88
+ from collections import Counter
89
+ ev = read()
90
+ kinds = Counter(e.get("kind") for e in ev)
91
+ pages = Counter(e.get("page") for e in ev if e.get("kind") == "page")
92
+ topics = Counter(e.get("topic") for e in ev if e.get("topic"))
93
+ return {"events": len(ev), "by_kind": dict(kinds),
94
+ "top_pages": pages.most_common(10), "top_topics": topics.most_common(10)}