Deploy AIOS web (React glide grid + FastAPI slice)
Browse filesThis view is limited to 50 files because it contains too many changes. Β See raw diff
- Dockerfile +62 -0
- README.md +13 -10
- api/aios_session.py +124 -0
- api/deps.py +129 -0
- api/main.py +240 -0
- api/pages.py +362 -0
- api/pages_sales.py +270 -0
- api/routes_admin.py +325 -0
- api/routes_auth.py +143 -0
- api/routes_customers.py +208 -0
- api/routes_grid.py +155 -0
- api/routes_nav.py +34 -0
- api/routes_pages.py +65 -0
- requirements.txt +12 -0
- royalimports_os/aios_grid.py +1221 -0
- royalimports_os/core/auth_google.py +91 -0
- royalimports_os/core/context.py +59 -0
- royalimports_os/core/grid_events.py +1070 -0
- royalimports_os/core/links.py +59 -0
- royalimports_os/core/management_pl.py +161 -0
- royalimports_os/core/odoo.py +201 -0
- royalimports_os/core/periods.py +155 -0
- royalimports_os/core/perms.py +164 -0
- royalimports_os/core/registry.py +137 -0
- royalimports_os/core/store.py +307 -0
- royalimports_os/core/store_backend.py +85 -0
- royalimports_os/core/store_pg.py +249 -0
- royalimports_os/core/table_store.py +281 -0
- royalimports_os/core/user_tables.py +179 -0
- royalimports_os/core/users.py +238 -0
- royalimports_os/evals/analyst_golden.yml +104 -0
- royalimports_os/harness/__init__.py +3 -0
- royalimports_os/harness/analyst.py +265 -0
- royalimports_os/harness/base.py +123 -0
- royalimports_os/harness/canonical.py +101 -0
- royalimports_os/harness/chatlog.py +102 -0
- royalimports_os/harness/connectors/__init__.py +1 -0
- royalimports_os/harness/connectors/odoo.py +147 -0
- royalimports_os/harness/connectors/quickbooks.py +37 -0
- royalimports_os/harness/datastore.py +750 -0
- royalimports_os/harness/evals.py +134 -0
- royalimports_os/harness/filter_sql.py +654 -0
- royalimports_os/harness/measure_filter.py +666 -0
- royalimports_os/harness/routines.py +259 -0
- royalimports_os/harness/runtime.py +186 -0
- royalimports_os/harness/semantic.py +936 -0
- royalimports_os/harness/skills/__init__.py +1 -0
- royalimports_os/harness/skills/sales.py +168 -0
- royalimports_os/harness/tables.py +129 -0
- royalimports_os/harness/telemetry.py +94 -0
Dockerfile
ADDED
|
@@ -0,0 +1,62 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# AIOS web β the React shell + FastAPI over the royalimports_os data layer. THE CANONICAL IMAGE:
|
| 2 |
+
# this is the one that actually runs, and the one a host migration re-points (C1e).
|
| 3 |
+
#
|
| 4 |
+
# β WHY A SECOND DOCKERFILE EXISTS (ops/Dockerfile.api), stated here so nobody "tidies" the wrong
|
| 5 |
+
# one away. It is NOT a duplicated service β it is the SAME service built from a DIFFERENT BUILD
|
| 6 |
+
# CONTEXT:
|
| 7 |
+
# * HF Spaces: the context is the Space repo root, which `deploy_web.py` uploads FLATTENED β
|
| 8 |
+
# `api/`, `web/`, `royalimports_os/`, `requirements.txt` all at top level. Hence the paths here.
|
| 9 |
+
# * ops/compose.yml: the context is the local repo root, where the same files live at
|
| 10 |
+
# `aios-web/api/`, `aios-web/web/`, `royalimports_os/`. Different prefixes, same content.
|
| 11 |
+
# β THE REAL FIX, and it is a follow-up rather than something to do the day of a first deploy:
|
| 12 |
+
# make `deploy_web.py` upload with the REPO layout so both contexts are identical, then this file
|
| 13 |
+
# serves both and `ops/Dockerfile.api` is deleted. Until then `ops/verify_portability.py` gates
|
| 14 |
+
# the two against each other on every property that matters (same app object, same launcher
|
| 15 |
+
# contract, non-root, healthchecked, no baked secrets), so they cannot drift silently.
|
| 16 |
+
#
|
| 17 |
+
# Two stages: node builds the React bundle, python serves it + the JSON API. Login is the app's own
|
| 18 |
+
# branded page over a signed session cookie (wave 1 removed the HTTP Basic gate).
|
| 19 |
+
|
| 20 |
+
# ---- stage 1: build the React bundle ----
|
| 21 |
+
FROM node:22-slim AS web
|
| 22 |
+
WORKDIR /web
|
| 23 |
+
# Resolve deps FRESH from the public registry (the local lockfile pinned a mirror-only lodash
|
| 24 |
+
# 4.18.1 that isn't on npmjs.org; a fresh install gets the public 4.17.x that glide accepts).
|
| 25 |
+
COPY web/package.json ./
|
| 26 |
+
RUN npm install --no-audit --no-fund --no-package-lock
|
| 27 |
+
COPY web/ ./
|
| 28 |
+
RUN rm -f package-lock.json && npm run build # -> /web/dist
|
| 29 |
+
|
| 30 |
+
# ---- stage 2: python API serving the bundle ----
|
| 31 |
+
FROM python:3.12-slim
|
| 32 |
+
WORKDIR /app
|
| 33 |
+
COPY requirements.txt ./
|
| 34 |
+
RUN pip install --no-cache-dir -r requirements.txt
|
| 35 |
+
COPY api/ ./api/
|
| 36 |
+
COPY royalimports_os/ ./royalimports_os/ # the reused data layer (modules/core/harness)
|
| 37 |
+
COPY --from=web /web/dist ./web/dist
|
| 38 |
+
|
| 39 |
+
# Run unprivileged β the C2 security baseline, and matched to ops/Dockerfile.api so the two images
|
| 40 |
+
# do not differ on a property that would only surface after a migration. uid 1000 is what HF
|
| 41 |
+
# Spaces expects.
|
| 42 |
+
RUN useradd --create-home --uid 1000 user && chown -R user:user /app
|
| 43 |
+
USER user
|
| 44 |
+
|
| 45 |
+
ENV RI_DIR=/app/royalimports_os \
|
| 46 |
+
PORT=7860 \
|
| 47 |
+
PYTHONUNBUFFERED=1
|
| 48 |
+
# β AIOS_INSECURE_SSL is deliberately NOT set. It disables TLS verification for every outbound
|
| 49 |
+
# call in the process and exists for one developer laptop. ops/verify_portability.py fails if any
|
| 50 |
+
# deploy config sets it.
|
| 51 |
+
EXPOSE 7860
|
| 52 |
+
|
| 53 |
+
# Liveness via stdlib rather than curl β it keeps the image free of an apt layer that exists only
|
| 54 |
+
# for the healthcheck.
|
| 55 |
+
HEALTHCHECK --interval=30s --timeout=5s --start-period=60s --retries=3 \
|
| 56 |
+
CMD python -c "import os,urllib.request,sys; sys.exit(0 if urllib.request.urlopen(f'http://127.0.0.1:{os.environ.get(\"PORT\",\"7860\")}/api/health',timeout=4).status==200 else 1)"
|
| 57 |
+
|
| 58 |
+
# β THE PORT COMES FROM THE ENVIRONMENT, ALWAYS. The application itself reads PORT nowhere β it is
|
| 59 |
+
# a launcher argument β which is what makes moving hosts a config change instead of a code change.
|
| 60 |
+
# Secrets (Odoo creds, APP_PASSWORD, AIOS_SESSION_SECRET, APP_BASE_URL) arrive as env at RUN time
|
| 61 |
+
# and are never baked; the repo-root .dockerignore drops **/.env from the build context.
|
| 62 |
+
CMD ["sh", "-c", "uvicorn api.main:app --host 0.0.0.0 --port ${PORT}"]
|
README.md
CHANGED
|
@@ -1,10 +1,13 @@
|
|
| 1 |
-
---
|
| 2 |
-
title: Loopable
|
| 3 |
-
emoji:
|
| 4 |
-
colorFrom:
|
| 5 |
-
colorTo: gray
|
| 6 |
-
sdk: docker
|
| 7 |
-
|
| 8 |
-
|
| 9 |
-
|
| 10 |
-
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
---
|
| 2 |
+
title: Loopable
|
| 3 |
+
emoji: π
|
| 4 |
+
colorFrom: indigo
|
| 5 |
+
colorTo: gray
|
| 6 |
+
sdk: docker
|
| 7 |
+
app_port: 7860
|
| 8 |
+
pinned: false
|
| 9 |
+
---
|
| 10 |
+
|
| 11 |
+
Loopable β the AIOS platform shell: a React front end (Airtable-grade grid + dashboard pages)
|
| 12 |
+
over a FastAPI service reusing the royalimports_os data layer. Login-gated: a branded login page over an HMAC-signed session cookie, with per-user
|
| 13 |
+
business-unit scope and module grants. Separate from the Streamlit app.
|
api/aios_session.py
ADDED
|
@@ -0,0 +1,124 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""aios_session.py β X3: the signed, STATELESS session cookie (EXIT-3a, 2026-07-30).
|
| 2 |
+
|
| 3 |
+
aios_session = base64url(json {t,u,e,i,x}) . base64url(HMAC-SHA256)
|
| 4 |
+
|
| 5 |
+
`t` tenant slug Β· `u` username Β· `e` the user's revocation epoch Β· `i` idle deadline Β·
|
| 6 |
+
`x` absolute expiry. Signed with `AIOS_SESSION_SECRET` using stdlib `hmac` β no new dependency,
|
| 7 |
+
and nothing to store server-side, which is the whole point: EXIT-4 requires a process that holds
|
| 8 |
+
no per-tenant state, and a session TABLE is per-tenant state. D2's Postgres sessions-MIRROR (an
|
| 9 |
+
audit view, not the source of truth) arrives with C-2 and is deliberately deferred.
|
| 10 |
+
|
| 11 |
+
WHY STATELESS STILL REVOKES. A cookie nobody can delete sounds like a cookie nobody can revoke.
|
| 12 |
+
The answer is the `epoch` integer on the user record (`core.users.epoch` / `bump_epoch`): the
|
| 13 |
+
cookie carries the epoch it was minted under, every verification re-reads the account's current
|
| 14 |
+
epoch, and a password change or a deactivation bumps it β so every outstanding cookie for that
|
| 15 |
+
user dies at once, without a session table.
|
| 16 |
+
|
| 17 |
+
β NO FALLBACK SECRET, EVER. With `AIOS_SESSION_SECRET` unset this module mints a RANDOM
|
| 18 |
+
per-process key: sessions then die on restart, which is an inconvenience. A hardcoded default
|
| 19 |
+
would instead let anyone who has read this repo forge a cookie for any user of any tenant β an
|
| 20 |
+
inconvenience is the correct trade against a forgery key. `EPHEMERAL_SECRET` records which
|
| 21 |
+
happened so the app can say so at startup and a gate can assert it.
|
| 22 |
+
"""
|
| 23 |
+
import base64
|
| 24 |
+
import hmac
|
| 25 |
+
import json
|
| 26 |
+
import os
|
| 27 |
+
import secrets
|
| 28 |
+
import time
|
| 29 |
+
from hashlib import sha256
|
| 30 |
+
|
| 31 |
+
COOKIE_NAME = "aios_session"
|
| 32 |
+
IDLE_SECONDS = 8 * 60 * 60 # D2: reissued on every authenticated request
|
| 33 |
+
ABSOLUTE_SECONDS = 30 * 24 * 60 * 60 # D2: a hard ceiling no amount of activity extends
|
| 34 |
+
|
| 35 |
+
_env_secret = os.environ.get("AIOS_SESSION_SECRET") or ""
|
| 36 |
+
EPHEMERAL_SECRET = not _env_secret
|
| 37 |
+
_SECRET = _env_secret.encode("utf-8") if _env_secret else secrets.token_bytes(32)
|
| 38 |
+
|
| 39 |
+
|
| 40 |
+
def _b64(raw: bytes) -> str:
|
| 41 |
+
return base64.urlsafe_b64encode(raw).decode("ascii").rstrip("=")
|
| 42 |
+
|
| 43 |
+
|
| 44 |
+
def _unb64(txt: str) -> bytes:
|
| 45 |
+
return base64.urlsafe_b64decode(txt + "=" * (-len(txt) % 4))
|
| 46 |
+
|
| 47 |
+
|
| 48 |
+
def _sign(payload: bytes) -> str:
|
| 49 |
+
return _b64(hmac.new(_SECRET, payload, sha256).digest())
|
| 50 |
+
|
| 51 |
+
|
| 52 |
+
def mint(tenant, username, epoch, now=None, absolute_expiry=None):
|
| 53 |
+
"""A fresh cookie value. `absolute_expiry` carries over on reissue so refreshing the idle
|
| 54 |
+
window can never extend the 30-day ceiling β that would make the ceiling unreachable."""
|
| 55 |
+
now = int(now if now is not None else time.time())
|
| 56 |
+
claims = {"t": str(tenant), "u": str(username), "e": int(epoch),
|
| 57 |
+
"i": now + IDLE_SECONDS,
|
| 58 |
+
"x": int(absolute_expiry if absolute_expiry is not None
|
| 59 |
+
else now + ABSOLUTE_SECONDS)}
|
| 60 |
+
payload = json.dumps(claims, separators=(",", ":"), sort_keys=True).encode("utf-8")
|
| 61 |
+
return f"{_b64(payload)}.{_sign(payload)}", claims
|
| 62 |
+
|
| 63 |
+
|
| 64 |
+
def read(value, now=None):
|
| 65 |
+
"""Parse + verify a cookie value β claims dict, or None.
|
| 66 |
+
|
| 67 |
+
None for EVERY failure mode β bad shape, bad signature, expired, unparseable. The caller
|
| 68 |
+
answers 401 and must not learn which: an error that distinguishes "signature wrong" from
|
| 69 |
+
"expired" tells a forger whether their key is right.
|
| 70 |
+
"""
|
| 71 |
+
if not value or not isinstance(value, str) or value.count(".") != 1:
|
| 72 |
+
return None
|
| 73 |
+
body, sig = value.split(".", 1)
|
| 74 |
+
try:
|
| 75 |
+
payload = _unb64(body)
|
| 76 |
+
except Exception:
|
| 77 |
+
return None
|
| 78 |
+
# compare_digest on the SIGNATURE, always β a byte-wise early exit is a timing oracle.
|
| 79 |
+
if not hmac.compare_digest(sig, _sign(payload)):
|
| 80 |
+
return None
|
| 81 |
+
try:
|
| 82 |
+
claims = json.loads(payload.decode("utf-8"))
|
| 83 |
+
except Exception:
|
| 84 |
+
return None
|
| 85 |
+
if not isinstance(claims, dict):
|
| 86 |
+
return None
|
| 87 |
+
if not isinstance(claims.get("u"), str) or not isinstance(claims.get("t"), str):
|
| 88 |
+
return None
|
| 89 |
+
if not claims["u"] or not claims["t"]:
|
| 90 |
+
return None
|
| 91 |
+
try:
|
| 92 |
+
idle, absolute, epoch = int(claims["i"]), int(claims["x"]), int(claims["e"])
|
| 93 |
+
except (KeyError, TypeError, ValueError):
|
| 94 |
+
return None
|
| 95 |
+
now = int(now if now is not None else time.time())
|
| 96 |
+
if now >= idle or now >= absolute:
|
| 97 |
+
return None
|
| 98 |
+
claims["e"], claims["i"], claims["x"] = epoch, idle, absolute
|
| 99 |
+
return claims
|
| 100 |
+
|
| 101 |
+
|
| 102 |
+
def is_secure_request(request):
|
| 103 |
+
"""Secure flag ON unless this is plain-HTTP localhost. Set unconditionally in production;
|
| 104 |
+
unset on a local http:// dev origin, where a Secure cookie is simply never sent back and
|
| 105 |
+
login would appear to succeed and then not work."""
|
| 106 |
+
host = (request.headers.get("host") or "").split(":")[0].lower()
|
| 107 |
+
proto = (request.headers.get("x-forwarded-proto")
|
| 108 |
+
or request.url.scheme or "http").split(",")[0].strip().lower()
|
| 109 |
+
if proto == "https":
|
| 110 |
+
return True
|
| 111 |
+
return host not in ("localhost", "127.0.0.1", "[::1]", "::1")
|
| 112 |
+
|
| 113 |
+
|
| 114 |
+
def set_cookie(response, request, value):
|
| 115 |
+
response.set_cookie(
|
| 116 |
+
COOKIE_NAME, value, httponly=True, samesite="lax",
|
| 117 |
+
secure=is_secure_request(request), max_age=ABSOLUTE_SECONDS, path="/")
|
| 118 |
+
|
| 119 |
+
|
| 120 |
+
def clear_cookie(response, request):
|
| 121 |
+
# Same attributes as when it was set: a browser matches on path/secure/samesite when
|
| 122 |
+
# deleting, and a mismatched delete silently leaves the cookie in place.
|
| 123 |
+
response.delete_cookie(COOKIE_NAME, path="/", httponly=True, samesite="lax",
|
| 124 |
+
secure=is_secure_request(request))
|
api/deps.py
ADDED
|
@@ -0,0 +1,129 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""deps.py β the request-scoped session + tenant resolution every v1 route depends on.
|
| 2 |
+
|
| 3 |
+
ONE place answers "who is asking, for which tenant, and what may they see?", because a
|
| 4 |
+
permission check that exists in four routes has four chances to be forgotten. The routes take
|
| 5 |
+
`session: Session = Depends(require_session)` and receive an object that has ALREADY failed
|
| 6 |
+
closed if anything was wrong.
|
| 7 |
+
|
| 8 |
+
FAIL-CLOSED, spelled out (X2): no cookie β **401**. Cookie present but unverifiable, expired,
|
| 9 |
+
epoch-revoked, or naming an unknown tenant β **401**. Authenticated but not granted the surface β
|
| 10 |
+
**403**. Never an empty 200: an empty list is a legitimate answer meaning "no rows", and using it
|
| 11 |
+
to mean "you are not allowed" is how a permission bug becomes invisible.
|
| 12 |
+
"""
|
| 13 |
+
import os
|
| 14 |
+
import sys
|
| 15 |
+
from dataclasses import dataclass
|
| 16 |
+
from pathlib import Path
|
| 17 |
+
|
| 18 |
+
from fastapi import Depends, HTTPException, Request, Response
|
| 19 |
+
|
| 20 |
+
import aios_session
|
| 21 |
+
|
| 22 |
+
_RI = Path(os.environ.get("RI_DIR") or Path(__file__).resolve().parents[2] / "royalimports_os")
|
| 23 |
+
if str(_RI) not in sys.path:
|
| 24 |
+
sys.path.insert(0, str(_RI))
|
| 25 |
+
|
| 26 |
+
import core.perms as perms # noqa: E402
|
| 27 |
+
import core.users as users # noqa: E402
|
| 28 |
+
from harness import runtime # noqa: E402
|
| 29 |
+
|
| 30 |
+
|
| 31 |
+
def err(status, code, message):
|
| 32 |
+
"""The one error shape (X2): non-2xx JSON `{"error": {"code", "message"}}`."""
|
| 33 |
+
return HTTPException(status_code=status, detail={"error": {"code": code, "message": message}})
|
| 34 |
+
|
| 35 |
+
|
| 36 |
+
@dataclass
|
| 37 |
+
class Session:
|
| 38 |
+
"""A verified request identity. `user` is the PUBLIC record (never a hash or a salt)."""
|
| 39 |
+
tenant: str
|
| 40 |
+
user: dict
|
| 41 |
+
claims: dict
|
| 42 |
+
runtime: object
|
| 43 |
+
|
| 44 |
+
@property
|
| 45 |
+
def uname(self):
|
| 46 |
+
return self.user.get("username", "")
|
| 47 |
+
|
| 48 |
+
@property
|
| 49 |
+
def admin(self):
|
| 50 |
+
return perms.is_admin(self.user)
|
| 51 |
+
|
| 52 |
+
def require(self, module_key):
|
| 53 |
+
"""403 unless this session may open `module_key`. Returns nothing β it is a gate."""
|
| 54 |
+
if not perms.may_open(self.user, module_key):
|
| 55 |
+
raise err(403, "forbidden", f"your account does not have access to {module_key}")
|
| 56 |
+
|
| 57 |
+
|
| 58 |
+
def _user_for(claims):
|
| 59 |
+
"""The account named by a verified cookie, or None β with the epoch check that makes a
|
| 60 |
+
stateless cookie revocable.
|
| 61 |
+
|
| 62 |
+
β THE RULE THIS FUNCTION MUST OBEY: **it has to accept exactly the identities `users.verify`
|
| 63 |
+
issues.** Any divergence produces the worst failure shape there is β a login that returns 200
|
| 64 |
+
with a cookie and then 401s every request after it. This function got that wrong once (the
|
| 65 |
+
emergency-master branch below ran only when NO record existed, while `verify` grants the master
|
| 66 |
+
identity even when one does), so it is now written as a deliberate mirror of `verify`'s
|
| 67 |
+
structure: try the record, else the master.
|
| 68 |
+
|
| 69 |
+
TWO CASES, in `verify`'s own order:
|
| 70 |
+
* THE RECORD, if it is readable and satisfies everything a session adds on top of a login:
|
| 71 |
+
the account is active, and its CURRENT epoch equals the cookie's β otherwise the session
|
| 72 |
+
is revoked (a password change or a deactivation bumped it).
|
| 73 |
+
* THE EMERGENCY MASTER β username 'admin' with APP_PASSWORD configured β exactly as
|
| 74 |
+
`verify`'s last branch grants it, whether or not a record exists. This is what stops a
|
| 75 |
+
store outage, a self-deactivation or a bumped epoch locking the owner out of the product.
|
| 76 |
+
β AND IT MEANS AN `admin` SESSION IS NOT EPOCH-REVOCABLE while APP_PASSWORD is set. That
|
| 77 |
+
is not a weakening: whoever holds APP_PASSWORD can simply log in again, so bumping admin's
|
| 78 |
+
epoch never revoked them in the first place. Rotating APP_PASSWORD is how you revoke it.
|
| 79 |
+
Every OTHER account is fully epoch-revocable, which is asserted in `verify_api.py`.
|
| 80 |
+
* anything else β None. An unknown username is NOT admitted just because its signature was
|
| 81 |
+
valid: a signature proves the cookie is ours, not that the account still exists.
|
| 82 |
+
"""
|
| 83 |
+
uname = (claims.get("u") or "").strip().lower()
|
| 84 |
+
if not uname:
|
| 85 |
+
return None
|
| 86 |
+
try:
|
| 87 |
+
reg = users.registry() or {}
|
| 88 |
+
except Exception:
|
| 89 |
+
reg = {}
|
| 90 |
+
rec = reg.get(uname)
|
| 91 |
+
if (rec and rec.get("active", True)
|
| 92 |
+
and int(rec.get("epoch") or 0) == int(claims.get("e") or 0)):
|
| 93 |
+
return users._public(uname, rec)
|
| 94 |
+
if uname == "admin" and os.environ.get("APP_PASSWORD", ""):
|
| 95 |
+
return {"username": "admin", "name": "Administrator", "role": "admin",
|
| 96 |
+
"bus": "all", "modules": "all", "epoch": int(claims.get("e") or 0)}
|
| 97 |
+
return None
|
| 98 |
+
|
| 99 |
+
|
| 100 |
+
def require_session(request: Request, response: Response) -> Session:
|
| 101 |
+
"""Verify the cookie, resolve the tenant, and REISSUE the cookie (the 8h idle window is
|
| 102 |
+
refreshed on use; the 30-day absolute expiry is carried over, never extended)."""
|
| 103 |
+
raw = request.cookies.get(aios_session.COOKIE_NAME)
|
| 104 |
+
if not raw:
|
| 105 |
+
raise err(401, "no_session", "sign in to continue")
|
| 106 |
+
claims = aios_session.read(raw)
|
| 107 |
+
if not claims:
|
| 108 |
+
raise err(401, "invalid_session", "your session has expired β sign in again")
|
| 109 |
+
user = _user_for(claims)
|
| 110 |
+
if not user:
|
| 111 |
+
raise err(401, "invalid_session", "your session has expired β sign in again")
|
| 112 |
+
try:
|
| 113 |
+
rt = runtime.get_runtime(claims["t"])
|
| 114 |
+
except KeyError:
|
| 115 |
+
# An unknown tenant slug is a 401, not a 404: the cookie is not a valid identity for this
|
| 116 |
+
# deployment, and confirming which slugs exist would answer a question we were not asked.
|
| 117 |
+
raise err(401, "invalid_session", "your session has expired β sign in again")
|
| 118 |
+
fresh, _ = aios_session.mint(claims["t"], user["username"], int(user.get("epoch") or 0),
|
| 119 |
+
absolute_expiry=claims["x"])
|
| 120 |
+
aios_session.set_cookie(response, request, fresh)
|
| 121 |
+
return Session(tenant=claims["t"], user=user, claims=claims, runtime=rt)
|
| 122 |
+
|
| 123 |
+
|
| 124 |
+
def module_gate(module_key):
|
| 125 |
+
"""A dependency that 401s without a session and 403s without the grant for `module_key`."""
|
| 126 |
+
def _dep(session: Session = Depends(require_session)) -> Session:
|
| 127 |
+
session.require(module_key)
|
| 128 |
+
return session
|
| 129 |
+
return _dep
|
api/main.py
ADDED
|
@@ -0,0 +1,240 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""AIOS web API β the ONE shared, stateless process the Streamlit exit is aimed at.
|
| 2 |
+
|
| 3 |
+
It REUSES the `royalimports_os/` data layer verbatim (nothing re-implemented): the canonical,
|
| 4 |
+
reconciled Odoo model stays server-side and the browser only ever sees derived JSON. Serves the
|
| 5 |
+
JSON API under `/api/*` and the built React bundle (`aios-web/web/dist`) at `/`.
|
| 6 |
+
|
| 7 |
+
WHAT CHANGED IN EXIT WAVE 1 (2026-07-30) β three things, all of them load-bearing:
|
| 8 |
+
|
| 9 |
+
* **Real sessions replace HTTP Basic.** The whole app used to sit behind one shared
|
| 10 |
+
`APP_PASSWORD` with the browser's native Basic prompt, which means every caller was the same
|
| 11 |
+
anonymous principal and no route could scope anything. Now a signed stateless cookie carries
|
| 12 |
+
a real `core/users` identity (X3), and every v1 route resolves BU + own-book scope from the
|
| 13 |
+
user RECORD. `/api/health` stays unauthenticated (liveness only).
|
| 14 |
+
* **The overlay fork is deleted.** `aios-web/api/data/overlay.json` was a second writable home
|
| 15 |
+
for user-owned fields the Streamlit app keeps in the tenant store. ONE store now (C1c).
|
| 16 |
+
* **The SSL shim is env-gated.** It used to run at import, unconditionally.
|
| 17 |
+
|
| 18 |
+
β TENANT RESOLUTION IS PER REQUEST (X7). Nothing tenant-shaped is held at module level; the
|
| 19 |
+
session's tenant claim resolves through `harness.runtime.get_runtime`, which is LRU-bounded. This
|
| 20 |
+
is the rule the ~30β80 MB/tenant target depends on β EXIT-0 measured the alternative at a ~0.6 GB
|
| 21 |
+
commit FLOOR per tenant, duplicated within 2% per tenant, nothing shared.
|
| 22 |
+
"""
|
| 23 |
+
import os
|
| 24 |
+
import sys
|
| 25 |
+
from pathlib import Path
|
| 26 |
+
|
| 27 |
+
# --- the local Odoo SSL quirk, now BEHIND A GATE -------------------------------------------------
|
| 28 |
+
# The Windows trust store reports the (valid) Odoo cert as expired, so local runs need an
|
| 29 |
+
# unverified default context; the HF Space and the container are unaffected. This used to run
|
| 30 |
+
# unconditionally at import β i.e. the shipped container disabled TLS verification for every
|
| 31 |
+
# outbound HTTPS call it ever made, to Odoo and to everything else, forever. That is a
|
| 32 |
+
# man-in-the-middle away from being someone else's data. It is now opt-in, must be set
|
| 33 |
+
# deliberately, and is never on by default.
|
| 34 |
+
# β NEVER set AIOS_INSECURE_SSL in a deployed environment. It exists for one developer laptop.
|
| 35 |
+
if os.environ.get("AIOS_INSECURE_SSL") == "1":
|
| 36 |
+
import ssl
|
| 37 |
+
ssl._create_default_https_context = ssl._create_unverified_context # noqa: S323
|
| 38 |
+
|
| 39 |
+
# --- reuse the tenant #0 data layer verbatim. RI_DIR overrides the location in the container
|
| 40 |
+
# (where the layout differs from the local sibling-dir default). ---
|
| 41 |
+
_HERE = Path(__file__).resolve()
|
| 42 |
+
_RI = Path(os.environ.get("RI_DIR") or (_HERE.parents[2] / "royalimports_os"))
|
| 43 |
+
for _p in (str(_RI), str(_HERE.parent)):
|
| 44 |
+
if _p not in sys.path:
|
| 45 |
+
sys.path.insert(0, _p)
|
| 46 |
+
|
| 47 |
+
from dotenv import load_dotenv # noqa: E402
|
| 48 |
+
load_dotenv(_RI / ".env") # Odoo creds + APP_PASSWORD, git-ignored, never committed/printed
|
| 49 |
+
|
| 50 |
+
from fastapi import Body, Depends, FastAPI, Request # noqa: E402
|
| 51 |
+
from fastapi.responses import JSONResponse # noqa: E402
|
| 52 |
+
from fastapi.staticfiles import StaticFiles # noqa: E402
|
| 53 |
+
from starlette.exceptions import HTTPException as StarletteHTTPException # noqa: E402
|
| 54 |
+
|
| 55 |
+
import aios_session # noqa: E402
|
| 56 |
+
import routes_admin # noqa: E402
|
| 57 |
+
import routes_auth # noqa: E402
|
| 58 |
+
import routes_customers # noqa: E402
|
| 59 |
+
import routes_grid # noqa: E402
|
| 60 |
+
import routes_nav # noqa: E402
|
| 61 |
+
import routes_pages # noqa: E402
|
| 62 |
+
from core import grid_events # noqa: E402
|
| 63 |
+
from deps import Session, module_gate # noqa: E402
|
| 64 |
+
|
| 65 |
+
_WEB_DIST = _HERE.parents[1] / "web" / "dist"
|
| 66 |
+
|
| 67 |
+
# --- THE CANONICAL FIELD CONTRACT: a startup assertion, and the referee gate's subject ----------
|
| 68 |
+
# Every field tags its semantic TYPE and its SOURCE. `source='odoo'` is READ-ONLY (Odoo is never
|
| 69 |
+
# written); `source='overlay'` is the editable stratum that lives OUTSIDE Odoo. Loaded from the
|
| 70 |
+
# ONE canonical file shared with the embedded host (`royalimports_os/aios_grid_fields.json`), so
|
| 71 |
+
# embed == standalone by construction.
|
| 72 |
+
#
|
| 73 |
+
# TWO REASONS THIS IS HERE and not folded into the routes:
|
| 74 |
+
# 1. It FAILS LOUDLY at import when the file is missing β which means the deploy or the RI_DIR
|
| 75 |
+
# layout is broken, and discovering that from a 500 on the first customer request instead of
|
| 76 |
+
# at startup costs a debugging session. (The pre-wave main.py had this guard; the EXIT-2a
|
| 77 |
+
# rewrite dropped it and this restores it.)
|
| 78 |
+
# 2. `aios-web/verify_fields_contract.py` β the cross-side REFEREE β reads `FIELDS` and
|
| 79 |
+
# `_PASSTHROUGH_KEYS` from this module to prove the canonical file, `aios_grid.py` and this
|
| 80 |
+
# API have not drifted. The rewrite removed them and the referee went red; retargeting the
|
| 81 |
+
# gate would have been the wrong repair ([[gate-can-report-green-on-nothing]]: retarget, do
|
| 82 |
+
# not delete β but only when the subject genuinely moved. Here it should not have moved).
|
| 83 |
+
#
|
| 84 |
+
# β THE ROUTES SERVE A SUPERSET OF THIS. `routes_customers._payload` derives its field list from
|
| 85 |
+
# `aios_grid.fields_from_workspace(ws)`, which is `FIELDS` PLUS the session user's own custom_ and
|
| 86 |
+
# measure_ columns β the same list the Streamlit host renders. That is the point: the standalone
|
| 87 |
+
# shell now sees the user's own columns instead of the bare base contract. `FIELDS` is the
|
| 88 |
+
# canonical FLOOR, asserted below to be exactly what `aios_grid` starts from.
|
| 89 |
+
import json as _json_contract # noqa: E402
|
| 90 |
+
_FIELDS_PATH = _RI / "aios_grid_fields.json"
|
| 91 |
+
if not _FIELDS_PATH.is_file():
|
| 92 |
+
raise FileNotFoundError(
|
| 93 |
+
f"AIOS web API: canonical field contract missing at {_FIELDS_PATH}. Set RI_DIR to the "
|
| 94 |
+
"royalimports_os root (it also carries the data layer this API imports)."
|
| 95 |
+
)
|
| 96 |
+
_contract = _json_contract.loads(_FIELDS_PATH.read_text(encoding="utf-8"))
|
| 97 |
+
FIELDS = _contract["fields"] if isinstance(_contract, dict) else _contract
|
| 98 |
+
# text/status/date pass through untouched; every OTHER odoo field is numeric -> rounded. Derived
|
| 99 |
+
# from field TYPE (not a hand-kept key list) so a new text/date field can never be wrongly rounded.
|
| 100 |
+
_PASSTHROUGH_KEYS = {f["key"] for f in FIELDS
|
| 101 |
+
if f["source"] == "odoo" and f["type"] in ("text", "status", "date")}
|
| 102 |
+
|
| 103 |
+
app = FastAPI(title="AIOS web API")
|
| 104 |
+
|
| 105 |
+
|
| 106 |
+
@app.exception_handler(StarletteHTTPException)
|
| 107 |
+
def _error_shape(request: Request, exc: StarletteHTTPException):
|
| 108 |
+
"""ONE error shape for every non-2xx (X2): `{"error": {"code", "message"}}`.
|
| 109 |
+
|
| 110 |
+
`deps.err` already raises detail in that shape; anything FastAPI raises on its own (a 404, a
|
| 111 |
+
422 from a malformed path param) is wrapped here so a client never has to branch on two
|
| 112 |
+
different error bodies.
|
| 113 |
+
"""
|
| 114 |
+
detail = exc.detail
|
| 115 |
+
if isinstance(detail, dict) and "error" in detail:
|
| 116 |
+
body = detail
|
| 117 |
+
else:
|
| 118 |
+
body = {"error": {"code": f"http_{exc.status_code}", "message": str(detail)}}
|
| 119 |
+
return JSONResponse(body, status_code=exc.status_code,
|
| 120 |
+
headers=getattr(exc, "headers", None))
|
| 121 |
+
|
| 122 |
+
|
| 123 |
+
@app.exception_handler(grid_events.StoreUnavailable)
|
| 124 |
+
def _store_unavailable(request: Request, exc: grid_events.StoreUnavailable):
|
| 125 |
+
"""THE SAFETY NET for "the store is down" β 503, from anywhere.
|
| 126 |
+
|
| 127 |
+
β WHY THIS IS APP-LEVEL AND NOT A `try` PER ROUTE. It was a try per route first, and a
|
| 128 |
+
`StoreUnavailable` raised while BUILDING THE PAYLOAD β before the route reached its own
|
| 129 |
+
try/except β surfaced as a 500. A store outage is a normal operational state and every route
|
| 130 |
+
here touches the store at least twice (the workspace read, then the write), so "remember to
|
| 131 |
+
wrap it" is a rule that gets forgotten once and then reports the wrong thing. One handler
|
| 132 |
+
means a store outage can only ever be a 503, whichever call raised it.
|
| 133 |
+
|
| 134 |
+
The seam raises this only when the caller passed no `fallback_ws` β i.e. exactly on this
|
| 135 |
+
adapter, which has no durable session dict to degrade into. A 200 over a write that
|
| 136 |
+
evaporated is the failure the whole rule exists to prevent.
|
| 137 |
+
"""
|
| 138 |
+
return JSONResponse(
|
| 139 |
+
{"error": {"code": "store_unavailable",
|
| 140 |
+
"message": "the tenant store is unavailable β no change was saved"}},
|
| 141 |
+
status_code=503)
|
| 142 |
+
|
| 143 |
+
|
| 144 |
+
@app.get("/api/health")
|
| 145 |
+
def health():
|
| 146 |
+
"""Unauthenticated LIVENESS only β it must answer before anyone can sign in, so it may not
|
| 147 |
+
reveal anything about the deployment beyond "the process is up". No version, no tenant list,
|
| 148 |
+
no config: a health endpoint is the one URL every scanner finds first."""
|
| 149 |
+
return {"ok": True}
|
| 150 |
+
|
| 151 |
+
|
| 152 |
+
app.include_router(routes_auth.router)
|
| 153 |
+
app.include_router(routes_nav.router)
|
| 154 |
+
app.include_router(routes_customers.router)
|
| 155 |
+
app.include_router(routes_grid.router)
|
| 156 |
+
# EXIT wave 2: the Y1 page-data envelope (one route for every ported dashboard) and Y4's user
|
| 157 |
+
# administration. `routes_pages` imports `pages`, which lazily imports each `pages_*` builder β so
|
| 158 |
+
# a new page is a builder module plus one registry line, and nothing here changes.
|
| 159 |
+
app.include_router(routes_pages.router)
|
| 160 |
+
app.include_router(routes_admin.router)
|
| 161 |
+
|
| 162 |
+
|
| 163 |
+
# --- DEPRECATED ALIASES (removed when S2's shell flips; kept so the current bundle keeps working)
|
| 164 |
+
# They are the v1 handlers with the v1 session requirement β NOT the old unauthenticated Basic
|
| 165 |
+
# behavior. An alias that kept the old auth would be a bypass of everything above it.
|
| 166 |
+
_GATE = module_gate(routes_customers.MODULE)
|
| 167 |
+
|
| 168 |
+
|
| 169 |
+
@app.get("/api/customers", deprecated=True)
|
| 170 |
+
def _customers_alias(session: Session = Depends(_GATE)):
|
| 171 |
+
return routes_customers._payload(session)
|
| 172 |
+
|
| 173 |
+
|
| 174 |
+
@app.patch("/api/customers/{pid}", deprecated=True)
|
| 175 |
+
def _patch_alias(pid: int, body: dict = Body(default=None),
|
| 176 |
+
session: Session = Depends(_GATE)):
|
| 177 |
+
return routes_customers.patch_customer(pid, body, session)
|
| 178 |
+
|
| 179 |
+
|
| 180 |
+
def _assert_contract_floor():
|
| 181 |
+
"""The canonical FIELDS must be exactly what `aios_grid` starts an empty workspace from.
|
| 182 |
+
|
| 183 |
+
This is what stops `FIELDS` becoming a constant that exists only to satisfy a gate. If the
|
| 184 |
+
canonical JSON and `aios_grid.fields_from_workspace({})` ever disagree, the standalone API and
|
| 185 |
+
the embedded host are serving two different schemas and the grid's own contract has forked β
|
| 186 |
+
the exact drift `verify_fields_contract.py` was written to catch, now also caught at startup
|
| 187 |
+
on whatever machine is actually running.
|
| 188 |
+
"""
|
| 189 |
+
import aios_grid
|
| 190 |
+
base = [f for f in aios_grid.fields_from_workspace({}) if not f.get("custom")]
|
| 191 |
+
if [f["key"] for f in base] != [f["key"] for f in FIELDS]:
|
| 192 |
+
raise RuntimeError(
|
| 193 |
+
"AIOS web API: the canonical field contract and aios_grid.fields_from_workspace({}) "
|
| 194 |
+
"disagree on the base field set β embed and standalone would serve different schemas. "
|
| 195 |
+
"Run aios-web/verify_fields_contract.py.")
|
| 196 |
+
|
| 197 |
+
|
| 198 |
+
def _startup_notes():
|
| 199 |
+
"""Say the two things an operator must know, once, at import β never in a response body."""
|
| 200 |
+
if aios_session.EPHEMERAL_SECRET:
|
| 201 |
+
print("[aios-api] AIOS_SESSION_SECRET is not set β signing with a random per-process "
|
| 202 |
+
"key. Sessions will not survive a restart and will not work across workers. "
|
| 203 |
+
"Set it in production.")
|
| 204 |
+
if os.environ.get("AIOS_INSECURE_SSL") == "1":
|
| 205 |
+
print("[aios-api] AIOS_INSECURE_SSL=1 β TLS verification is DISABLED for outbound "
|
| 206 |
+
"requests. Local development only; never in a deployed environment.")
|
| 207 |
+
|
| 208 |
+
|
| 209 |
+
_assert_contract_floor()
|
| 210 |
+
_startup_notes()
|
| 211 |
+
|
| 212 |
+
|
| 213 |
+
# β THE STATIC MOUNT IS NOW UNAUTHENTICATED, and that is a change this wave made on purpose.
|
| 214 |
+
# Before EXIT-3a, `BasicAuth` middleware gated the WHOLE app including this mount. A branded login
|
| 215 |
+
# page cannot live behind a password prompt, so the shell's own assets must be public β which they
|
| 216 |
+
# are: `index.html`, the JS/CSS bundle and the favicon reveal nothing.
|
| 217 |
+
#
|
| 218 |
+
# β WHAT THAT SILENTLY DE-GATED, caught in review rather than in production. `web/dist/` also
|
| 219 |
+
# carries `sample_customers.json`, a DEV FIXTURE copied from `web/public/`. Its 8 customers are
|
| 220 |
+
# synthetic, but the `agent` column holds REAL EMPLOYEE NAMES, and it went from Basic-gated to
|
| 221 |
+
# publicly fetchable in this commit. Nothing needs it: both the API bridge and `useCustomerData`
|
| 222 |
+
# deleted their sample fallback on purpose ("NOTHING HERE FALLS BACK TO sample_customers.json"),
|
| 223 |
+
# and it survives only because it sits in `web/public/`. So it is refused here β 404, the same
|
| 224 |
+
# answer as any other path that is not part of the app.
|
| 225 |
+
#
|
| 226 |
+
# The right long-term fix is deleting it from `web/public/` (S2's lane β flagged in the mailbox);
|
| 227 |
+
# this guard is what makes the API safe regardless of what the bundle happens to contain.
|
| 228 |
+
_DEV_FIXTURES = {"sample_customers.json"}
|
| 229 |
+
|
| 230 |
+
|
| 231 |
+
class _AppStatic(StaticFiles):
|
| 232 |
+
async def get_response(self, path, scope):
|
| 233 |
+
if Path(path).name in _DEV_FIXTURES:
|
| 234 |
+
raise StarletteHTTPException(status_code=404, detail="Not Found")
|
| 235 |
+
return await super().get_response(path, scope)
|
| 236 |
+
|
| 237 |
+
|
| 238 |
+
# static bundle LAST so /api/* wins; html=True serves index.html at / plus the built assets
|
| 239 |
+
if _WEB_DIST.is_dir():
|
| 240 |
+
app.mount("/", _AppStatic(directory=str(_WEB_DIST), html=True), name="web")
|
api/pages.py
ADDED
|
@@ -0,0 +1,362 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""pages.py β Y1: the page-data envelope, and the ONE place its rules live (W2-2).
|
| 2 |
+
|
| 3 |
+
THE POINT OF THIS FILE IS WAVE 3. Collections and Procurement must land by adding a BUILDER, not
|
| 4 |
+
by inventing a shape β so every rule the envelope has (the closed `fmt` enum, the drill grammar,
|
| 5 |
+
the no-silent-caps contract, how BU scope is resolved) lives here as a constructor a builder calls,
|
| 6 |
+
never as a convention a builder is expected to remember. A rule that exists only in a docstring is
|
| 7 |
+
a rule the second page gets wrong.
|
| 8 |
+
|
| 9 |
+
WHAT A BUILDER OWES, and nothing more:
|
| 10 |
+
metrics(team_id, granularity, today) -> a dict of plain data, SCOPE-SHAPED ONLY
|
| 11 |
+
blocks(metrics, team_id, granularity) -> [block, ...] built with the constructors below
|
| 12 |
+
TITLE / SUBTITLE / MODULE / CONTROLS
|
| 13 |
+
`build_envelope` does the rest: it resolves the BU, caches the metrics on the SCOPE key, assembles
|
| 14 |
+
the per-request half, and returns the envelope.
|
| 15 |
+
|
| 16 |
+
β THE CACHE LINE, WHICH IS WHERE WAVE 1 SHIPPED A CROSS-USER LEAK. What may be cached on a
|
| 17 |
+
`(page, team_id, doc_mode, day)` key is the BUILDER'S METRICS β the answer to a question two users
|
| 18 |
+
with the same scope are asking identically. What may NEVER be cached there is anything with a
|
| 19 |
+
per-USER stratum, and on this envelope that is `controls` (its `bu.options` come from
|
| 20 |
+
`perms.allowed_bu_labels(user)`, i.e. from the user RECORD). Wave 1's defect was exactly this shape
|
| 21 |
+
one layer down: a whole payload cached on a scope key while carrying per-user fields and overlay
|
| 22 |
+
cells. It survived 129 green checks because every fixture user had a distinct scope pair. So the
|
| 23 |
+
split is structural here, not remembered: `metrics()` cannot see the session at all β it takes
|
| 24 |
+
`team_id`, and `verify_api.py` asserts the cached object holds nothing user-shaped.
|
| 25 |
+
|
| 26 |
+
β THE CLIENT FORMATS. The server never sends a pre-formatted number, so one formatter change fixes
|
| 27 |
+
every page β and `fmt` is a CLOSED enum for the same reason. Two vocabularies exist and they are
|
| 28 |
+
not interchangeable: `FMTS` for kpi/table/validation, and viz `FieldType` for a chart block's
|
| 29 |
+
`fields[]` (those objects are fed straight into `chartData`). `money` vs `currency` is the trap.
|
| 30 |
+
"""
|
| 31 |
+
import time
|
| 32 |
+
|
| 33 |
+
# `deps` FIRST, deliberately: it is what puts `royalimports_os/` on `sys.path`, so importing it
|
| 34 |
+
# ahead of any `core.*` makes this module importable in whatever order a caller reaches it.
|
| 35 |
+
from deps import Session, err, perms # noqa: I001
|
| 36 |
+
|
| 37 |
+
import core.context as ctxlib # noqa: E402
|
| 38 |
+
|
| 39 |
+
# ββ the two vocabularies βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 40 |
+
#: The Y1 client-format enum for kpi / table / validation values. CLOSED on purpose: a page that
|
| 41 |
+
#: needs a new one adds it HERE and to the client's `fmt` at the same time, which is the property
|
| 42 |
+
#: that keeps one formatter authoritative. A table column with NO `fmt` renders verbatim β that is
|
| 43 |
+
#: how a text column is expressed without opening the enum.
|
| 44 |
+
FMTS = ("money", "num", "pct", "int", "date")
|
| 45 |
+
|
| 46 |
+
#: The viz `FieldType` values a chart block's `fields[]` may use β a DIFFERENT vocabulary, because
|
| 47 |
+
#: `chartData(spec, rows, fieldByKey)` consumes those objects directly. `currency`, not `money`.
|
| 48 |
+
VIZ_TYPES = ("text", "status", "currency", "int", "pct", "date")
|
| 49 |
+
|
| 50 |
+
#: `chartData`'s own display cap (`MAX_BUCKETS`). A payload must fit it at the SOURCE: the engine
|
| 51 |
+
#: truncates past 12 and, on a date axis, sorts ASCENDING first β so a 13-month series loses the
|
| 52 |
+
#: CURRENT month, the one anybody actually looks at. Mirrored here so a builder can size its own
|
| 53 |
+
#: series without importing anything from the client tree.
|
| 54 |
+
MAX_BUCKETS = 12
|
| 55 |
+
|
| 56 |
+
#: Drill kinds the client is expected to understand. `decomp` names a WINDOW + a scope; the others
|
| 57 |
+
#: name an ENTITY. β THIS WAVE THEY ARE DECLARATIONS β there is no resolver route (Β§5 Y1 amendment
|
| 58 |
+
#: 4). Each descriptor is COMPLETE, so wave 3 adds the resolver with no shape change, and Y6's
|
| 59 |
+
#: panel must say it is thin rather than look finished.
|
| 60 |
+
DRILL_KINDS = ("decomp", "customer", "sku", "rep", "bu")
|
| 61 |
+
|
| 62 |
+
|
| 63 |
+
def _fmt_ok(fmt, allowed, where):
|
| 64 |
+
if fmt is not None and fmt not in allowed:
|
| 65 |
+
# A wrong `fmt` renders β wrongly β and nothing complains. Refusing it at construction is
|
| 66 |
+
# the only moment anybody finds out, so this raises rather than coercing.
|
| 67 |
+
raise ValueError(f"{where}: fmt {fmt!r} is not one of {allowed}")
|
| 68 |
+
return fmt
|
| 69 |
+
|
| 70 |
+
|
| 71 |
+
# ββ block constructors βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 72 |
+
def kpi(key, label, value, fmt="money", delta=None, delta_fmt=None, delta_label=None,
|
| 73 |
+
delta_dir=None, note=None, drill=None):
|
| 74 |
+
"""One KPI card. `delta` is a NUMBER or None β never a string.
|
| 75 |
+
|
| 76 |
+
THE ZERO-REVENUE RULE, ported from `app.py:2846`: a period with no orders yet must not show an
|
| 77 |
+
alarming β100%. It sends NO number (`delta=None`) plus a `delta_label` and `delta_dir='off'`,
|
| 78 |
+
so no formatter is involved and the card states the situation instead of computing a ratio
|
| 79 |
+
against nothing.
|
| 80 |
+
|
| 81 |
+
THE BIG-RATIO RULE IS THE CLIENT'S: a tiny LY base makes `+14,975%` read as noise, and
|
| 82 |
+
Streamlit rewrites it per-card as `+150Γ LY`. That belongs in ONE formatter (`fmt.pct` renders
|
| 83 |
+
|v| β₯ 999 as a multiple), so the server just sends the number.
|
| 84 |
+
"""
|
| 85 |
+
out = {"key": key, "label": label, "value": value,
|
| 86 |
+
"fmt": _fmt_ok(fmt, FMTS, f"kpi {key}")}
|
| 87 |
+
if delta is not None:
|
| 88 |
+
out["delta"] = delta
|
| 89 |
+
out["delta_fmt"] = _fmt_ok(delta_fmt or "pct", FMTS, f"kpi {key} delta")
|
| 90 |
+
if delta_label:
|
| 91 |
+
out["delta_label"] = delta_label
|
| 92 |
+
if delta_dir:
|
| 93 |
+
out["delta_dir"] = delta_dir # up | down | flat | off β a TONE hint, not the sign
|
| 94 |
+
if note:
|
| 95 |
+
out["note"] = note
|
| 96 |
+
if drill:
|
| 97 |
+
out["drill"] = drill
|
| 98 |
+
return out
|
| 99 |
+
|
| 100 |
+
|
| 101 |
+
def kpis(key, items):
|
| 102 |
+
return {"type": "kpis", "key": key, "items": list(items)}
|
| 103 |
+
|
| 104 |
+
|
| 105 |
+
def section(key, label, note=None):
|
| 106 |
+
out = {"type": "section", "key": key, "label": label}
|
| 107 |
+
if note:
|
| 108 |
+
out["note"] = note
|
| 109 |
+
return out
|
| 110 |
+
|
| 111 |
+
|
| 112 |
+
def column(key, label, fmt=None, align=None):
|
| 113 |
+
"""A table column. `fmt` omitted β render verbatim (that is how text columns work)."""
|
| 114 |
+
out = {"key": key, "label": label}
|
| 115 |
+
if fmt:
|
| 116 |
+
out["fmt"] = _fmt_ok(fmt, FMTS, f"column {key}")
|
| 117 |
+
if align:
|
| 118 |
+
out["align"] = align
|
| 119 |
+
elif fmt in ("money", "num", "pct", "int"):
|
| 120 |
+
out["align"] = "right" # numbers align right by default, everywhere
|
| 121 |
+
return out
|
| 122 |
+
|
| 123 |
+
|
| 124 |
+
def table(key, columns, rows, total=None, drill=None, download=None, empty=None):
|
| 125 |
+
"""A table block. `shown`/`total` ALWAYS ship β `shown == total` still sends both.
|
| 126 |
+
|
| 127 |
+
Owner rule [[no-unverifiable-aggregates]]: a silent `[:N]` cap is a defect. `total` defaults to
|
| 128 |
+
`len(rows)` so a builder that forgot to truncate cannot accidentally claim it did not.
|
| 129 |
+
"""
|
| 130 |
+
rows = list(rows)
|
| 131 |
+
out = {"type": "table", "key": key, "columns": list(columns), "rows": rows,
|
| 132 |
+
"shown": len(rows), "total": int(total if total is not None else len(rows))}
|
| 133 |
+
if drill:
|
| 134 |
+
out["drill"] = drill
|
| 135 |
+
if download:
|
| 136 |
+
out["download"] = download
|
| 137 |
+
if empty:
|
| 138 |
+
out["empty"] = empty
|
| 139 |
+
return out
|
| 140 |
+
|
| 141 |
+
|
| 142 |
+
def chart(key, spec, series, fields, rows, x_order=None, drill=None, total=None):
|
| 143 |
+
"""A chart block: ONE x, N SERIES.
|
| 144 |
+
|
| 145 |
+
β WHY `series[]` AND NOT `ChartSpec.splitBy`. The viz engine has no series dimension:
|
| 146 |
+
`chartData()` buckets on `x` only and `DashboardView` draws one flat `buckets[]`. `splitBy` is
|
| 147 |
+
persisted, validated and offered in the chart editor but never reaches the arithmetic or the
|
| 148 |
+
renderer, so a `splitBy` chart draws identically to one without it. A YoY comparison is
|
| 149 |
+
therefore N `chartData` calls over the SAME wide rows, one per series `y` β which also means
|
| 150 |
+
nothing about `chartData` changes and `verify_charts.py`'s verdict stays put, which W2-5's
|
| 151 |
+
pure-move proof requires.
|
| 152 |
+
|
| 153 |
+
β WHY `x_order`. The engine sorts buckets by VALUE unless the x field is date-family β a trend
|
| 154 |
+
would render as a revenue-ordered sawtooth. And date-family is not an escape: it buckets to
|
| 155 |
+
`YYYY-MM`, so a weekly axis would collapse into months. So the SERVER declares the chronology
|
| 156 |
+
and the client re-orders the buckets it got back.
|
| 157 |
+
"""
|
| 158 |
+
for f in fields:
|
| 159 |
+
if f.get("type") not in VIZ_TYPES:
|
| 160 |
+
raise ValueError(f"chart {key}: field {f.get('key')!r} type {f.get('type')!r} is not a "
|
| 161 |
+
f"viz FieldType {VIZ_TYPES} β `money` is the kpi/table vocabulary")
|
| 162 |
+
rows = list(rows)
|
| 163 |
+
out = {"type": "chart", "key": key, "spec": dict(spec), "series": list(series),
|
| 164 |
+
"fields": list(fields), "rows": rows,
|
| 165 |
+
"shown": len(rows), "total": int(total if total is not None else len(rows))}
|
| 166 |
+
if x_order is not None:
|
| 167 |
+
out["x_order"] = list(x_order)
|
| 168 |
+
if drill:
|
| 169 |
+
out["drill"] = drill
|
| 170 |
+
return out
|
| 171 |
+
|
| 172 |
+
|
| 173 |
+
def validation(checks, note=None):
|
| 174 |
+
"""The reconciliation panel, from a module's `validate()` output.
|
| 175 |
+
|
| 176 |
+
`actual` is the module's `a` β the number the page shows. `expected` is its `b` β the
|
| 177 |
+
INDEPENDENT aggregate it must tie to. The signed gap rides in `note` rather than being
|
| 178 |
+
recomputed client-side, because the module's own tolerance decided `ok` and a second
|
| 179 |
+
subtraction in the client could disagree with it.
|
| 180 |
+
"""
|
| 181 |
+
out = {"type": "validation", "key": "validation",
|
| 182 |
+
"checks": [{"name": c.get("check") or c.get("name") or "",
|
| 183 |
+
"ok": bool(c.get("ok")),
|
| 184 |
+
"actual": c.get("a"), "expected": c.get("b"),
|
| 185 |
+
"fmt": "money" if isinstance(c.get("a"), float) else "num",
|
| 186 |
+
"note": f"gap {c.get('gap')}"} for c in (checks or [])]}
|
| 187 |
+
if note:
|
| 188 |
+
out["note"] = note
|
| 189 |
+
return out
|
| 190 |
+
|
| 191 |
+
|
| 192 |
+
# ββ drill descriptors: ONE grammar, three placements βββββββββββββββββββββββββββββββββββββββββββββ
|
| 193 |
+
def decomp_drill(window_row, scope_type="period", scope_value=None, label=None):
|
| 194 |
+
"""A WINDOW + scope descriptor. Mirrors `ui/drawers._decomp_desc` key-for-key, so the eventual
|
| 195 |
+
resolver can serve both front-ends from one function β and accepts both row shapes the sales
|
| 196 |
+
module produces (`date_from`/`date_to` on the scorecard, `start`/`end` on a trend row)."""
|
| 197 |
+
return {"kind": "decomp", "label": label,
|
| 198 |
+
"scope_type": scope_type, "scope_value": scope_value,
|
| 199 |
+
"date_from": window_row.get("date_from") or window_row.get("start"),
|
| 200 |
+
"date_to": window_row.get("date_to") or window_row.get("end"),
|
| 201 |
+
"cmp_from": window_row.get("cmp_from"), "cmp_to": window_row.get("cmp_to")}
|
| 202 |
+
|
| 203 |
+
|
| 204 |
+
def entity_drill(kind, id_key, label_key=None):
|
| 205 |
+
"""Block-level: every row of this block opens the same kind of entity panel."""
|
| 206 |
+
if kind not in DRILL_KINDS:
|
| 207 |
+
raise ValueError(f"drill kind {kind!r} is not one of {DRILL_KINDS}")
|
| 208 |
+
out = {"kind": kind, "id_key": id_key}
|
| 209 |
+
if label_key:
|
| 210 |
+
out["label_key"] = label_key
|
| 211 |
+
return out
|
| 212 |
+
|
| 213 |
+
|
| 214 |
+
def row_drill(row_key="_drill"):
|
| 215 |
+
"""Block-level: each row carries its OWN descriptor under `row_key`."""
|
| 216 |
+
return {"kind": "decomp", "row_key": row_key}
|
| 217 |
+
|
| 218 |
+
|
| 219 |
+
def truncate(rows, limit):
|
| 220 |
+
"""`(rows[:limit], total)` β the ONLY sanctioned way to cap a list. Returning the total beside
|
| 221 |
+
the slice is what makes "showing N of M" possible; a bare `[:N]` is the defect."""
|
| 222 |
+
rows = list(rows)
|
| 223 |
+
return rows[:limit], len(rows)
|
| 224 |
+
|
| 225 |
+
|
| 226 |
+
# ββ BU scope: the one line that could fail open ββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 227 |
+
def resolve_bu(session: Session, bu):
|
| 228 |
+
"""The effective `team_id` for this request. `bu` NARROWS; it never authorises.
|
| 229 |
+
|
| 230 |
+
β THE RULE, AND WHY IT IS SHAPED THIS WAY. `perms.scope_team_id` derives scope from the user
|
| 231 |
+
RECORD precisely because a UI selector defaults to 'All' β consolidated, and mirroring that
|
| 232 |
+
default on an API with no selector would BE the leak. So a query parameter cannot widen:
|
| 233 |
+
|
| 234 |
+
* absent / 'all' β `scope_team_id(user)`, which is None ONLY when the user may see every BU
|
| 235 |
+
and is the PINNED team_id otherwise. A Royal-only user asking for 'all' gets 6, and 'all'
|
| 236 |
+
for them honestly means "everything you may see".
|
| 237 |
+
* an explicit id β must be in `allowed_team_ids(user)`, else **403**. Never silently coerced
|
| 238 |
+
to their own BU: a coerced parameter hides a client bug behind data that looks right.
|
| 239 |
+
* anything else β 400.
|
| 240 |
+
"""
|
| 241 |
+
raw = "" if bu is None else str(bu).strip().lower()
|
| 242 |
+
if raw in ("", "all"):
|
| 243 |
+
return perms.scope_team_id(session.user)
|
| 244 |
+
try:
|
| 245 |
+
want = int(raw)
|
| 246 |
+
except (TypeError, ValueError):
|
| 247 |
+
raise err(400, "bad_bu", "bu must be 'all' or a business-unit id")
|
| 248 |
+
if want not in perms.allowed_team_ids(session.user):
|
| 249 |
+
raise err(403, "bu_forbidden",
|
| 250 |
+
"your account does not have access to that business unit")
|
| 251 |
+
return want
|
| 252 |
+
|
| 253 |
+
|
| 254 |
+
def bu_control(session: Session, team_id):
|
| 255 |
+
"""The BU picker, built from the user's OWN permitted labels β which is exactly why `controls`
|
| 256 |
+
is assembled per request and never cached on a scope key."""
|
| 257 |
+
labels = perms.allowed_bu_labels(session.user)
|
| 258 |
+
options = []
|
| 259 |
+
for lab in labels:
|
| 260 |
+
if lab == "All":
|
| 261 |
+
options.append({"value": "all", "label": "All"})
|
| 262 |
+
continue
|
| 263 |
+
tid = ctxlib.BRANDS.get(lab)
|
| 264 |
+
if tid is not None:
|
| 265 |
+
options.append({"value": tid, "label": lab})
|
| 266 |
+
return {"key": "bu", "kind": "bu", "label": "Business Unit",
|
| 267 |
+
"value": "all" if team_id is None else team_id, "options": options}
|
| 268 |
+
|
| 269 |
+
|
| 270 |
+
def choice_control(key, label, value, options):
|
| 271 |
+
return {"key": key, "kind": "choice", "label": label, "value": value,
|
| 272 |
+
"options": [{"value": v, "label": l} for v, l in options]}
|
| 273 |
+
|
| 274 |
+
|
| 275 |
+
def bu_label(team_id):
|
| 276 |
+
"""'All Business Units' / 'Fisch' / 'Royal' β for prose, never for a permission decision."""
|
| 277 |
+
if team_id is None:
|
| 278 |
+
return "All Business Units"
|
| 279 |
+
for lab, tid in ctxlib.BRANDS.items():
|
| 280 |
+
if tid == team_id:
|
| 281 |
+
return lab
|
| 282 |
+
return str(team_id)
|
| 283 |
+
|
| 284 |
+
|
| 285 |
+
# ββ the registry + the assembler βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 286 |
+
#: page key -> builder module. Wave 3 adds `'ar'` and `'procurement'` HERE and nowhere else β the
|
| 287 |
+
#: route, the grant wall, the BU resolution, the caching and the envelope are already written.
|
| 288 |
+
#: β The key must be the REGISTRY key, because it is what `session.require()` gates on.
|
| 289 |
+
_BUILDERS = {}
|
| 290 |
+
|
| 291 |
+
|
| 292 |
+
def register(key, builder):
|
| 293 |
+
_BUILDERS[str(key)] = builder
|
| 294 |
+
|
| 295 |
+
|
| 296 |
+
def builder_for(key):
|
| 297 |
+
"""The builder for `key`, or None. A missing key is a 404 at the route β NOT a 403: an
|
| 298 |
+
unported page is a fact about our roadmap, not about this user's grants."""
|
| 299 |
+
if not _BUILDERS:
|
| 300 |
+
import pages_sales # noqa: F401 (self-registers on import)
|
| 301 |
+
return _BUILDERS.get(str(key or "").strip().lower())
|
| 302 |
+
|
| 303 |
+
|
| 304 |
+
def known_pages():
|
| 305 |
+
builder_for("") # force the lazy import
|
| 306 |
+
return sorted(_BUILDERS)
|
| 307 |
+
|
| 308 |
+
|
| 309 |
+
#: How long a scope's metrics stay warm. Matches `routes_customers._CACHE_TTL` β the underlying
|
| 310 |
+
#: pull is the same order of cost and a second TTL would be a second thing to reason about.
|
| 311 |
+
_CACHE_TTL = 900
|
| 312 |
+
#: The metrics cache is bounded for the reason every cache in this process is: an entry per
|
| 313 |
+
#: (page Γ scope Γ day) that is never evicted is a memory leak with a tenant-shaped growth curve.
|
| 314 |
+
_CACHE_MAX = 24
|
| 315 |
+
|
| 316 |
+
|
| 317 |
+
def _cached_metrics(session: Session, page_key, team_id, granularity, builder):
|
| 318 |
+
"""The SCOPE-shaped half, cached on the runtime (per tenant, LRU-bounded).
|
| 319 |
+
|
| 320 |
+
The key carries the DAY: every metric here is relative to "today" (YTD, WTD, the trailing 12
|
| 321 |
+
months), so an entry that outlives midnight serves yesterday's question with today's label.
|
| 322 |
+
"""
|
| 323 |
+
rt = session.runtime
|
| 324 |
+
today = time.strftime("%Y-%m-%d")
|
| 325 |
+
key = ("page", page_key, team_id, granularity, today)
|
| 326 |
+
hit = rt.pool_cache.get(key)
|
| 327 |
+
now = time.time()
|
| 328 |
+
if hit and now - hit[0] < _CACHE_TTL:
|
| 329 |
+
return hit[1]
|
| 330 |
+
metrics = builder.metrics(team_id=team_id, granularity=granularity)
|
| 331 |
+
rt.pool_cache[key] = (now, metrics)
|
| 332 |
+
# A `while`, not a one-shot slice: evicting a fixed batch leaves the map above its own bound
|
| 333 |
+
# whenever it is more than that batch over, which is a bound that does not bind.
|
| 334 |
+
# β This dict is SHARED with `routes_customers._pool_rows`'s pool entries (they key on
|
| 335 |
+
# `("pool", β¦)`, these on `("page", β¦)`), so either side's eviction can drop the other's entry.
|
| 336 |
+
# Harmless β a miss is a rebuild β and deliberately one bounded map per tenant rather than two.
|
| 337 |
+
while len(rt.pool_cache) > _CACHE_MAX:
|
| 338 |
+
rt.pool_cache.pop(min(rt.pool_cache, key=lambda k: rt.pool_cache[k][0]), None)
|
| 339 |
+
return metrics
|
| 340 |
+
|
| 341 |
+
|
| 342 |
+
def build_envelope(session: Session, page_key, bu=None, period=None):
|
| 343 |
+
"""The Y1 envelope for `page_key`, scoped to this session. Raises the contracted errors."""
|
| 344 |
+
builder = builder_for(page_key)
|
| 345 |
+
if builder is None:
|
| 346 |
+
raise err(404, "unknown_page", f"no page data is published for {page_key!r}")
|
| 347 |
+
# The GRANT WALL, on the builder's registry key. 403, never an empty 200 β an empty page reads
|
| 348 |
+
# as "you have no data" and is how a permission bug hides in plain sight.
|
| 349 |
+
session.require(builder.MODULE)
|
| 350 |
+
team_id = resolve_bu(session, bu)
|
| 351 |
+
granularity = builder.resolve_granularity(period)
|
| 352 |
+
metrics = _cached_metrics(session, page_key, team_id, granularity, builder)
|
| 353 |
+
return {
|
| 354 |
+
"key": page_key,
|
| 355 |
+
"title": builder.TITLE,
|
| 356 |
+
"subtitle": builder.subtitle(team_id),
|
| 357 |
+
# UTC with an explicit Z. A naive local timestamp from a container whose TZ nobody set is
|
| 358 |
+
# indistinguishable from a wrong one.
|
| 359 |
+
"as_of": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime(metrics["pulled_at"])),
|
| 360 |
+
"controls": [bu_control(session, team_id)] + builder.controls(granularity),
|
| 361 |
+
"blocks": builder.blocks(metrics, team_id, granularity),
|
| 362 |
+
}
|
api/pages_sales.py
ADDED
|
@@ -0,0 +1,270 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""pages_sales.py β the Sales page builder (W2-2). The FIRST Y1 builder, and the template.
|
| 2 |
+
|
| 3 |
+
It reuses `royalimports_os/modules/sales.py` VERBATIM β the reconciled, validated data layer the
|
| 4 |
+
Streamlit page already reads β so there is exactly one definition of every number and the API is a
|
| 5 |
+
projection, not a second implementation. Nothing here computes revenue.
|
| 6 |
+
|
| 7 |
+
WHAT A BUILDER IS, in three functions:
|
| 8 |
+
metrics(team_id, granularity) the SCOPE-shaped data. Cached by `pages._cached_metrics`.
|
| 9 |
+
β It cannot see the session, by design β that is what makes
|
| 10 |
+
"nothing user-shaped is cached" a structural property rather
|
| 11 |
+
than a rule somebody has to remember.
|
| 12 |
+
blocks(metrics, team_id, β¦) the ordered Y1 blocks, built with `pages`' constructors.
|
| 13 |
+
controls(granularity) the page's own toolbar (the BU picker is added by `pages`).
|
| 14 |
+
|
| 15 |
+
β THE ONE LEAK THIS FILE HAD TO CLOSE. `sales.headline(team_id=β¦)` returns a `by_team` map that
|
| 16 |
+
**always covers BOTH business units** β its own docstring says "by_team always shows both" β so a
|
| 17 |
+
Royal-only request's headline carries Fisch revenue. `metrics()` therefore STRIPS `by_team` when a
|
| 18 |
+
BU is pinned, at the source, before it is cached: the other BU's numbers never enter the cache entry
|
| 19 |
+
for a scoped user, so a later edit that emits the block unconditionally renders nothing instead of
|
| 20 |
+
leaking. Fail-closed by construction, not by review. Asserted in `verify_api.py`.
|
| 21 |
+
|
| 22 |
+
β `by_rep` IS NOT THE STREAMLIT PAGE'S "By agent". `sales.by_rep` groups by Odoo `user_id` (the
|
| 23 |
+
salesperson on the order); the Streamlit page shows `customers.by_dimension('agent')` (the
|
| 24 |
+
CUSTOMER's assigned agent β a different number, see [[invoice-line-agent-commission]]). The split
|
| 25 |
+
doc's brief names `by_rep`, so `by_rep` is what ships; the divergence is flagged in the S1 mailbox
|
| 26 |
+
for the owner rather than resolved in code.
|
| 27 |
+
"""
|
| 28 |
+
import sys as _sys
|
| 29 |
+
import time
|
| 30 |
+
|
| 31 |
+
import core.odoo as O
|
| 32 |
+
import core.periods as P
|
| 33 |
+
import modules.sales as sales
|
| 34 |
+
|
| 35 |
+
import pages
|
| 36 |
+
|
| 37 |
+
MODULE = "sales" # the REGISTRY key β what the grant wall gates on
|
| 38 |
+
TITLE = "Sales"
|
| 39 |
+
|
| 40 |
+
_GRANULARITIES = (("monthly", "Monthly"), ("weekly", "Weekly"))
|
| 41 |
+
|
| 42 |
+
|
| 43 |
+
def resolve_granularity(period):
|
| 44 |
+
"""`?period=` β the trend granularity.
|
| 45 |
+
|
| 46 |
+
An unrecognised value is COERCED to the default rather than refused, and that asymmetry with
|
| 47 |
+
`bu` is deliberate: `bu` is a SCOPE question, where a coerced parameter would silently show the
|
| 48 |
+
wrong book, so it 403s. `period` is a DISPLAY question, and the envelope echoes the resolved
|
| 49 |
+
value back in `controls[].value`, so a coerced parameter cannot look accepted.
|
| 50 |
+
"""
|
| 51 |
+
return "weekly" if str(period or "").strip().lower() in ("weekly", "week") else "monthly"
|
| 52 |
+
|
| 53 |
+
|
| 54 |
+
def controls(granularity):
|
| 55 |
+
return [pages.choice_control("period", "Granularity", granularity, _GRANULARITIES)]
|
| 56 |
+
|
| 57 |
+
|
| 58 |
+
def subtitle(team_id):
|
| 59 |
+
return ("Confirmed orders only (Odoo state sale/done), house accounts excluded. Floral "
|
| 60 |
+
"wholesale is seasonal, so every comparison is against the SAME period last year.")
|
| 61 |
+
|
| 62 |
+
|
| 63 |
+
# ββ the scope-shaped half ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 64 |
+
def metrics(team_id=None, granularity="monthly"):
|
| 65 |
+
"""Everything the blocks need, for ONE scope. Pure data β no session, no user, no formatting."""
|
| 66 |
+
# Pin the recognition basis explicitly. It is thread-local and defaults to 'order', but a
|
| 67 |
+
# worker thread reused from a container that once served 'invoice' would otherwise inherit a
|
| 68 |
+
# different definition of revenue mid-page, and nothing in the payload would say so.
|
| 69 |
+
O.set_doc_mode("order")
|
| 70 |
+
t = P.today()
|
| 71 |
+
|
| 72 |
+
head = sales.headline(t=t, team_id=team_id)
|
| 73 |
+
if team_id is not None:
|
| 74 |
+
# β See the module docstring: `by_team` always covers both BUs. Strip it BEFORE caching.
|
| 75 |
+
head = dict(head, by_team={})
|
| 76 |
+
|
| 77 |
+
if granularity == "weekly":
|
| 78 |
+
trend = sales.weekly_trend(n_weeks=pages.MAX_BUCKETS, t=t, team_id=team_id)
|
| 79 |
+
else:
|
| 80 |
+
trend = sales.monthly_trend(n=pages.MAX_BUCKETS, t=t, team_id=team_id)
|
| 81 |
+
|
| 82 |
+
return {
|
| 83 |
+
"pulled_at": time.time(),
|
| 84 |
+
"today": P._d(t),
|
| 85 |
+
"scorecard": sales.period_scorecard(t=t, team_id=team_id),
|
| 86 |
+
"headline": head,
|
| 87 |
+
"trend": trend,
|
| 88 |
+
# limit=1e9 so the TOTAL is knowable: the module slices after aggregating, so asking for
|
| 89 |
+
# everything costs the same read and lets the block report "showing 25 of 812" instead of
|
| 90 |
+
# capping silently. A bare `[:N]` with no total is the defect [[no-unverifiable-aggregates]]
|
| 91 |
+
# exists to name.
|
| 92 |
+
"by_rep": sales.by_rep(t=t, limit=10 ** 9, team_id=team_id),
|
| 93 |
+
"top_customers": sales.top_customers(t=t, limit=10 ** 9, team_id=team_id),
|
| 94 |
+
"validation": sales.validate(t=t, team_id=team_id),
|
| 95 |
+
}
|
| 96 |
+
|
| 97 |
+
|
| 98 |
+
# ββ the blocks βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 99 |
+
_LIST_LIMIT = 25
|
| 100 |
+
|
| 101 |
+
|
| 102 |
+
def _delta_dir(v):
|
| 103 |
+
if v is None:
|
| 104 |
+
return None
|
| 105 |
+
return "up" if v > 0 else "down" if v < 0 else "flat"
|
| 106 |
+
|
| 107 |
+
|
| 108 |
+
def _scorecard_kpis(scorecard):
|
| 109 |
+
"""One card per period, every one of them drillable to its exact window."""
|
| 110 |
+
items = []
|
| 111 |
+
for r in scorecard:
|
| 112 |
+
drill = pages.decomp_drill(r, "period", None, f"Sales Β· {r['label']}")
|
| 113 |
+
if not r["revenue"]:
|
| 114 |
+
# No orders yet in this window. Sending a β100% here is the alarming lie
|
| 115 |
+
# `app.py:2846` guards against; send no number at all.
|
| 116 |
+
items.append(pages.kpi(r["key"], r["label"], 0.0, "money",
|
| 117 |
+
delta_label="no orders yet", delta_dir="off", drill=drill))
|
| 118 |
+
continue
|
| 119 |
+
items.append(pages.kpi(r["key"], r["label"], r["revenue"], "money",
|
| 120 |
+
delta=r["yoy_pct"], delta_fmt="pct",
|
| 121 |
+
delta_label="vs LY" if r["yoy_pct"] is not None else None,
|
| 122 |
+
delta_dir=_delta_dir(r["yoy_pct"]), drill=drill))
|
| 123 |
+
return pages.kpis("scorecard", items)
|
| 124 |
+
|
| 125 |
+
|
| 126 |
+
def _ytd_context_kpis(scorecard, head):
|
| 127 |
+
"""The three YTD numbers the Streamlit page puts in a caption (`app.py:2858`). As KPI cards
|
| 128 |
+
rather than prose, so each one carries the drill the caption could not."""
|
| 129 |
+
ytd = next((r for r in scorecard if r["key"] == "ytd"), scorecard[-1] if scorecard else {})
|
| 130 |
+
drill = pages.decomp_drill(ytd, "period", None, "Sales Β· Year to date")
|
| 131 |
+
return pages.kpis("ytd_context", [
|
| 132 |
+
pages.kpi("ytd_orders", "Orders (YTD)", head.get("ytd_orders") or 0, "int", drill=drill),
|
| 133 |
+
pages.kpi("ytd_aov", "Average order value", head.get("aov") or 0.0, "money", drill=drill),
|
| 134 |
+
pages.kpi("ytd_customers", "Active customers (YTD)", head.get("ytd_customers") or 0,
|
| 135 |
+
"int", drill=drill),
|
| 136 |
+
])
|
| 137 |
+
|
| 138 |
+
|
| 139 |
+
def _trend_block(trend, granularity, today):
|
| 140 |
+
"""The YoY trend: ONE x, TWO series, chronology declared by the server.
|
| 141 |
+
|
| 142 |
+
See `pages.chart` for why this is `series[]` + `x_order[]` rather than `ChartSpec.splitBy` and
|
| 143 |
+
a date-typed axis. `spec.y` is set to the FIRST series so a naive single-series
|
| 144 |
+
`chartData(spec, β¦)` call still draws the primary bars instead of a "choose a number field"
|
| 145 |
+
problem card.
|
| 146 |
+
"""
|
| 147 |
+
label_key = "week" if granularity == "weekly" else "month"
|
| 148 |
+
rows, x_order = [], []
|
| 149 |
+
for r in trend:
|
| 150 |
+
label = str(r.get(label_key) or "")
|
| 151 |
+
rows.append({
|
| 152 |
+
"period": label,
|
| 153 |
+
"revenue": r.get("revenue") or 0.0,
|
| 154 |
+
"revenue_ly": r.get("revenue_ly") or 0.0,
|
| 155 |
+
# The in-progress period compares a PARTIAL window against a full one, so the client
|
| 156 |
+
# fades it and drops its YoY (`app.py:2867-2875`). One rule for both granularities:
|
| 157 |
+
# the window that CONTAINS today. ISO strings, so string order is date order.
|
| 158 |
+
"partial": bool(r.get("start") and r.get("end")
|
| 159 |
+
and str(r["start"]) <= today <= str(r["end"])),
|
| 160 |
+
"_drill": pages.decomp_drill(r, "period", None, f"Sales Β· {label}"),
|
| 161 |
+
})
|
| 162 |
+
x_order.append(label)
|
| 163 |
+
unit = "Week of" if granularity == "weekly" else "Month"
|
| 164 |
+
spec = {"id": "sales-trend", "kind": "bar", "x": "period", "y": "revenue", "agg": "sum",
|
| 165 |
+
"title": "Revenue trend", "palette": "brand",
|
| 166 |
+
"axis": {"x": {"label": unit}, "y": {"format": "currency"}}}
|
| 167 |
+
fields = [{"key": "period", "label": unit, "type": "text", "source": "odoo"},
|
| 168 |
+
{"key": "revenue", "label": "Revenue", "type": "currency", "source": "odoo"},
|
| 169 |
+
{"key": "revenue_ly", "label": "Revenue LY", "type": "currency", "source": "odoo"}]
|
| 170 |
+
series = [{"y": "revenue", "label": "This year"},
|
| 171 |
+
{"y": "revenue_ly", "label": "Last year"}]
|
| 172 |
+
return pages.chart("trend", spec, series, fields, rows, x_order=x_order,
|
| 173 |
+
drill=pages.row_drill())
|
| 174 |
+
|
| 175 |
+
|
| 176 |
+
def _by_bu_block(head, scorecard):
|
| 177 |
+
"""YTD per Business Unit β CONSOLIDATED VIEWS ONLY.
|
| 178 |
+
|
| 179 |
+
Omitted entirely when a BU is pinned, matching `app.py:2898`'s `if tid is None`. `metrics()` has
|
| 180 |
+
already emptied `by_team` in that case, so this cannot accidentally render the other BU even if
|
| 181 |
+
the guard here were removed β belt and braces on the one number that must never cross.
|
| 182 |
+
"""
|
| 183 |
+
by_team = head.get("by_team") or {}
|
| 184 |
+
if not by_team:
|
| 185 |
+
return None
|
| 186 |
+
ytd = next((r for r in scorecard if r["key"] == "ytd"), scorecard[-1] if scorecard else {})
|
| 187 |
+
rows = []
|
| 188 |
+
for tid in O.TEAM_IDS:
|
| 189 |
+
name = O.TEAM_NAMES.get(tid)
|
| 190 |
+
e = by_team.get(name)
|
| 191 |
+
if not e:
|
| 192 |
+
continue
|
| 193 |
+
rows.append({
|
| 194 |
+
"bu": name, "team_id": tid,
|
| 195 |
+
"revenue": e.get("ytd") or 0.0, "revenue_ly": e.get("ytd_ly") or 0.0,
|
| 196 |
+
"yoy_pct": P.yoy_pct(e.get("ytd") or 0.0, e.get("ytd_ly") or 0.0),
|
| 197 |
+
"_drill": pages.decomp_drill(ytd, "brand", tid, f"Sales Β· {name} (YTD)"),
|
| 198 |
+
})
|
| 199 |
+
return pages.table(
|
| 200 |
+
"by_bu",
|
| 201 |
+
[pages.column("bu", "Business Unit"),
|
| 202 |
+
pages.column("revenue", "YTD $", "money"),
|
| 203 |
+
pages.column("revenue_ly", "LY $", "money"),
|
| 204 |
+
pages.column("yoy_pct", "YoY", "pct")],
|
| 205 |
+
rows, drill=pages.row_drill(), download={"filename": "sales_by_business_unit"},
|
| 206 |
+
empty="No confirmed orders in either Business Unit this year.")
|
| 207 |
+
|
| 208 |
+
|
| 209 |
+
def _by_rep_block(by_rep):
|
| 210 |
+
rows, total = pages.truncate(by_rep, _LIST_LIMIT)
|
| 211 |
+
return pages.table(
|
| 212 |
+
"by_rep",
|
| 213 |
+
[pages.column("rep", "Salesperson"),
|
| 214 |
+
pages.column("revenue", "YTD $", "money"),
|
| 215 |
+
pages.column("orders", "Orders", "int")],
|
| 216 |
+
rows, total=total, drill=pages.entity_drill("rep", "uid", "rep"),
|
| 217 |
+
download={"filename": "sales_by_rep"},
|
| 218 |
+
empty="No confirmed orders are attributed to a salesperson in this scope.")
|
| 219 |
+
|
| 220 |
+
|
| 221 |
+
def _top_customers_block(top_customers):
|
| 222 |
+
rows, total = pages.truncate(top_customers, _LIST_LIMIT)
|
| 223 |
+
return pages.table(
|
| 224 |
+
"top_customers",
|
| 225 |
+
[pages.column("customer", "Customer"),
|
| 226 |
+
pages.column("revenue", "YTD $", "money"),
|
| 227 |
+
pages.column("orders", "Orders", "int")],
|
| 228 |
+
rows, total=total, drill=pages.entity_drill("customer", "pid", "customer"),
|
| 229 |
+
download={"filename": "top_customers"},
|
| 230 |
+
empty="No confirmed orders in this scope yet.")
|
| 231 |
+
|
| 232 |
+
|
| 233 |
+
def blocks(metrics, team_id=None, granularity="monthly"):
|
| 234 |
+
"""The ORDERED blocks, mirroring `app.py:2833-2961`'s reading order.
|
| 235 |
+
|
| 236 |
+
β The BU table is present only on a consolidated view, so the client must not assume a fixed
|
| 237 |
+
block count or index β it renders `blocks` in order and skips a type it does not know.
|
| 238 |
+
"""
|
| 239 |
+
sc = metrics["scorecard"]
|
| 240 |
+
head = metrics["headline"]
|
| 241 |
+
out = [
|
| 242 |
+
_scorecard_kpis(sc),
|
| 243 |
+
_ytd_context_kpis(sc, head),
|
| 244 |
+
pages.section("trend_h", "Revenue trend",
|
| 245 |
+
"This period versus the same period last year. Click any bar to decompose "
|
| 246 |
+
"that period by customer and SKU. The in-progress period is marked partial "
|
| 247 |
+
"and carries no YoY."),
|
| 248 |
+
_trend_block(metrics["trend"], granularity, metrics["today"]),
|
| 249 |
+
]
|
| 250 |
+
bu = _by_bu_block(head, sc)
|
| 251 |
+
if bu is not None:
|
| 252 |
+
out.append(pages.section("by_bu_h", "By Business Unit",
|
| 253 |
+
"YTD revenue per Business Unit versus last year."))
|
| 254 |
+
out.append(bu)
|
| 255 |
+
out.append(pages.section("by_rep_h", "By salesperson",
|
| 256 |
+
"YTD revenue per Odoo salesperson on the order."))
|
| 257 |
+
out.append(_by_rep_block(metrics["by_rep"]))
|
| 258 |
+
out.append(pages.section("top_customers_h", "Top customers",
|
| 259 |
+
"Largest customers by YTD revenue."))
|
| 260 |
+
out.append(_top_customers_block(metrics["top_customers"]))
|
| 261 |
+
out.append(pages.validation(
|
| 262 |
+
metrics["validation"],
|
| 263 |
+
"Every headline number reconciles to an independent Odoo aggregate, and every "
|
| 264 |
+
"decomposition sums back to it."))
|
| 265 |
+
return out
|
| 266 |
+
|
| 267 |
+
|
| 268 |
+
#: Self-registers on import, so adding a page is one file plus one import β `pages.builder_for`
|
| 269 |
+
#: does the import and never needs a per-page branch.
|
| 270 |
+
pages.register("sales", _sys.modules[__name__])
|
api/routes_admin.py
ADDED
|
@@ -0,0 +1,325 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""routes_admin.py β Y4: user administration + the session's own settings (W2-3).
|
| 2 |
+
|
| 3 |
+
These routes are the standalone shell's replacement for `app.py`'s `users_dialog` / `settings_dialog`
|
| 4 |
+
modals, over the SAME `core/users` account store β so both front-ends administer one set of
|
| 5 |
+
accounts and the eventual OIDC migration (D-3) swaps the CREDENTIAL check, not the user model.
|
| 6 |
+
|
| 7 |
+
ADMIN-ONLY, FAIL-CLOSED. `admin_gate` is a role check (`perms.is_admin`), mirroring the Streamlit
|
| 8 |
+
dialog's `if not is_admin()`. A default record has `role: 'user'`, so an unreadable or partial record
|
| 9 |
+
is denied rather than admitted. `verify_api.py` proves it by having a viewer TRY every route.
|
| 10 |
+
|
| 11 |
+
β THIS IS THE FIRST WRITER OF `modules` THAT HAS EVER EXISTED. `core.users.set_access(modules=β¦)`
|
| 12 |
+
has no caller anywhere in the shipped app β module grants are only settable by editing the store
|
| 13 |
+
JSON out of band. That matters because the record format carries TWO documented fail-OPENS, both
|
| 14 |
+
asserted in `verify_api.py` section A:
|
| 15 |
+
|
| 16 |
+
* `modules: []` is FALSY, so `perms.allowed_modules` reads it as 'all' = UNRESTRICTED. An admin
|
| 17 |
+
clearing every checkbox to lock an account down would grant it everything.
|
| 18 |
+
* `bus: [<unknown id>]` falls through `allowed_bus_labels`'s "no recognisable label" branch to
|
| 19 |
+
`['All','Fisch','Royal']` β so one typo'd BU id is FULL cross-BU access, in the model whose
|
| 20 |
+
whole point is strict isolation.
|
| 21 |
+
|
| 22 |
+
Y4 says do not "fix" `modules: []` in this wave, and that is right: the READER is mirrored in
|
| 23 |
+
`ui/session.py` and `core/perms.py` and diverging one of them mid-wave breaks lock-step. But the
|
| 24 |
+
WRITER is new, and it can simply refuse to create either footgun. So both are 400s here and both
|
| 25 |
+
read semantics are untouched β the seam is write-strict / read-unchanged.
|
| 26 |
+
|
| 27 |
+
β A STORE OUTAGE IS A 503, NEVER AN EMPTY LIST. `users.registry()` swallows a failed read into `{}`
|
| 28 |
+
one level down, and serving that as `{"users": []}` would tell an administrator their tenant has no
|
| 29 |
+
accounts. Same rule as the write path: an empty 200 is never how this API says "something is wrong".
|
| 30 |
+
"""
|
| 31 |
+
import re
|
| 32 |
+
|
| 33 |
+
from fastapi import APIRouter, Body, Depends, Response
|
| 34 |
+
|
| 35 |
+
import core.registry as registry
|
| 36 |
+
import core.store as store
|
| 37 |
+
|
| 38 |
+
from deps import Session, err, perms, require_session, users
|
| 39 |
+
from routes_auth import _public_user
|
| 40 |
+
|
| 41 |
+
router = APIRouter(prefix="/api/v1")
|
| 42 |
+
|
| 43 |
+
#: A username is a STORE KEY (`users.json`) and also the key the table workspace is filed under
|
| 44 |
+
#: (`data[username]`), so it is constrained rather than trusted: lowercase, no separators, no
|
| 45 |
+
#: whitespace, nothing that could traverse or collide once it becomes part of a path or a filename.
|
| 46 |
+
_UNAME_RE = re.compile(r"^[a-z0-9][a-z0-9._-]{1,31}$")
|
| 47 |
+
#: Passwords are PBKDF2-200k, which is only as strong as what it is given. The Streamlit dialog
|
| 48 |
+
#: enforces nothing; this does, and the two are allowed to differ because only one of them is
|
| 49 |
+
#: reachable from the internet.
|
| 50 |
+
_MIN_PW = 8
|
| 51 |
+
_ROLES = ("user", "admin")
|
| 52 |
+
|
| 53 |
+
|
| 54 |
+
def admin_gate(session: Session = Depends(require_session)) -> Session:
|
| 55 |
+
"""401 without a session, 403 without the admin role. Role, not a module grant β mirroring
|
| 56 |
+
`app.py`'s `users_dialog`, whose only wall is `is_admin()`."""
|
| 57 |
+
if not perms.is_admin(session.user):
|
| 58 |
+
raise err(403, "forbidden", "administrators only")
|
| 59 |
+
return session
|
| 60 |
+
|
| 61 |
+
|
| 62 |
+
def _require_store():
|
| 63 |
+
"""A 503 the moment the store cannot serve, so no route below can report emptiness as truth."""
|
| 64 |
+
try:
|
| 65 |
+
ok = store.available()
|
| 66 |
+
except Exception:
|
| 67 |
+
ok = False
|
| 68 |
+
if not ok:
|
| 69 |
+
raise err(503, "store_unavailable",
|
| 70 |
+
"the tenant store is unavailable β accounts cannot be read or changed")
|
| 71 |
+
|
| 72 |
+
|
| 73 |
+
def _registry():
|
| 74 |
+
_require_store()
|
| 75 |
+
try:
|
| 76 |
+
reg = users.registry() or {}
|
| 77 |
+
except Exception:
|
| 78 |
+
raise err(503, "store_unavailable", "the tenant store is unavailable")
|
| 79 |
+
return reg
|
| 80 |
+
|
| 81 |
+
|
| 82 |
+
def _view(uname, rec):
|
| 83 |
+
"""What an administrator may see about an account. NEVER `salt` or `hash` β this function is
|
| 84 |
+
the only projection these routes use, so there is one place that can leak and it does not."""
|
| 85 |
+
return {"username": uname,
|
| 86 |
+
"name": rec.get("name") or uname,
|
| 87 |
+
"role": rec.get("role", "user"),
|
| 88 |
+
"bus": rec.get("bus", "all"),
|
| 89 |
+
"bu_labels": perms.allowed_bu_labels(rec),
|
| 90 |
+
"modules": rec.get("modules", "all"),
|
| 91 |
+
"agent": rec.get("agent") or None,
|
| 92 |
+
"email": rec.get("email") or None,
|
| 93 |
+
"active": bool(rec.get("active", True)),
|
| 94 |
+
# The session-revocation handle. An admin needs it: it is the only visible evidence
|
| 95 |
+
# that "sign them out everywhere" actually happened.
|
| 96 |
+
"epoch": int(rec.get("epoch") or 0)}
|
| 97 |
+
|
| 98 |
+
|
| 99 |
+
# ββ request validation: every rule below is fail-closed ββββββββββββββββββββββββββββββββββββββββββ
|
| 100 |
+
def _clean_username(v):
|
| 101 |
+
uname = str(v or "").strip().lower()
|
| 102 |
+
if not _UNAME_RE.match(uname):
|
| 103 |
+
raise err(400, "bad_username",
|
| 104 |
+
"a username is 2-32 characters: lowercase letters, digits, dot, dash or "
|
| 105 |
+
"underscore, starting with a letter or digit")
|
| 106 |
+
return uname
|
| 107 |
+
|
| 108 |
+
|
| 109 |
+
def _clean_password(v):
|
| 110 |
+
pw = str(v or "")
|
| 111 |
+
if len(pw) < _MIN_PW:
|
| 112 |
+
raise err(400, "weak_password", f"a password must be at least {_MIN_PW} characters")
|
| 113 |
+
return pw
|
| 114 |
+
|
| 115 |
+
|
| 116 |
+
def _clean_role(v):
|
| 117 |
+
role = str(v or "").strip().lower()
|
| 118 |
+
if role not in _ROLES:
|
| 119 |
+
raise err(400, "bad_role", f"role must be one of {list(_ROLES)}")
|
| 120 |
+
return role
|
| 121 |
+
|
| 122 |
+
|
| 123 |
+
def _clean_bus(v):
|
| 124 |
+
"""'all', or a non-empty list of KNOWN business-unit ids.
|
| 125 |
+
|
| 126 |
+
Refusing an unknown id is the whole point: `allowed_bus_labels` treats a list with no
|
| 127 |
+
recognisable label as full access, so `bus: [7]` would be a cross-BU grant created by a typo.
|
| 128 |
+
"""
|
| 129 |
+
if isinstance(v, str):
|
| 130 |
+
if v.strip().lower() == "all":
|
| 131 |
+
return "all"
|
| 132 |
+
raise err(400, "bad_bus", "bus must be 'all' or a list of business-unit ids")
|
| 133 |
+
if not isinstance(v, (list, tuple)) or not v:
|
| 134 |
+
raise err(400, "bad_bus",
|
| 135 |
+
"bus must be 'all' or a NON-EMPTY list of business-unit ids (an empty list "
|
| 136 |
+
"would read as unrestricted)")
|
| 137 |
+
out = []
|
| 138 |
+
for b in v:
|
| 139 |
+
try:
|
| 140 |
+
b = int(b)
|
| 141 |
+
except (TypeError, ValueError):
|
| 142 |
+
raise err(400, "bad_bus", "a business-unit id must be a number")
|
| 143 |
+
if b not in users.BU_LABELS:
|
| 144 |
+
raise err(400, "bad_bus",
|
| 145 |
+
f"{b} is not a known business unit {sorted(users.BU_LABELS)} β an "
|
| 146 |
+
"unrecognised id would widen access to every BU")
|
| 147 |
+
out.append(b)
|
| 148 |
+
return sorted(set(out))
|
| 149 |
+
|
| 150 |
+
|
| 151 |
+
def _clean_modules(v):
|
| 152 |
+
"""'all', or a non-empty list of KNOWN module keys.
|
| 153 |
+
|
| 154 |
+
`[]` is refused because it READS as unrestricted (see the module docstring). An unknown key is
|
| 155 |
+
refused because `may_open` is fail-closed on it: an admin who typed `sale` for `sales` would
|
| 156 |
+
silently lock the account out of the page they meant to grant.
|
| 157 |
+
"""
|
| 158 |
+
if isinstance(v, str):
|
| 159 |
+
if v.strip().lower() == "all":
|
| 160 |
+
return "all"
|
| 161 |
+
raise err(400, "bad_modules", "modules must be 'all' or a list of module keys")
|
| 162 |
+
if not isinstance(v, (list, tuple)) or not v:
|
| 163 |
+
raise err(400, "bad_modules",
|
| 164 |
+
"modules must be 'all' or a NON-EMPTY list of module keys β an empty list "
|
| 165 |
+
"reads as UNRESTRICTED, which is the opposite of locking an account down")
|
| 166 |
+
known = set(registry.BY_KEY) | set(perms._LEGACY_KEYS)
|
| 167 |
+
out, bad = [], []
|
| 168 |
+
for k in v:
|
| 169 |
+
k = str(k or "").strip()
|
| 170 |
+
(out if k in known else bad).append(k)
|
| 171 |
+
if bad:
|
| 172 |
+
raise err(400, "bad_modules", f"unknown module keys: {sorted(bad)}")
|
| 173 |
+
return sorted(set(out))
|
| 174 |
+
|
| 175 |
+
|
| 176 |
+
# ββ the routes βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 177 |
+
@router.get("/admin/users")
|
| 178 |
+
def list_users(session: Session = Depends(admin_gate)):
|
| 179 |
+
reg = _registry()
|
| 180 |
+
return {"users": [_view(u, reg[u]) for u in sorted(reg)]}
|
| 181 |
+
|
| 182 |
+
|
| 183 |
+
@router.post("/admin/users", status_code=201)
|
| 184 |
+
def create_user(body: dict = Body(default=None), session: Session = Depends(admin_gate)):
|
| 185 |
+
"""Create an account. **409 if the username already exists β `PATCH` is the update path.**
|
| 186 |
+
|
| 187 |
+
β WHY 409 AND NOT AN UPSERT. `core.users.create_user` writes a fresh `_record()`, and a fresh
|
| 188 |
+
record has NO `epoch` β so overwriting an existing account used to drop its epoch back to 0 and
|
| 189 |
+
RESURRECT every cookie minted before its last password change. `core/users.py` now
|
| 190 |
+
preserves-and-bumps on overwrite (which also closes it for `app.py`'s dialog, where "Add /
|
| 191 |
+
update a user" calls the same function), but this route still refuses: an upsert that silently
|
| 192 |
+
replaces an account's password, role and BU access because someone reused a username is not an
|
| 193 |
+
update anybody asked for.
|
| 194 |
+
"""
|
| 195 |
+
body = body or {}
|
| 196 |
+
_require_store()
|
| 197 |
+
uname = _clean_username(body.get("username"))
|
| 198 |
+
pw = _clean_password(body.get("password"))
|
| 199 |
+
role = _clean_role(body.get("role") or "user")
|
| 200 |
+
bus = _clean_bus(body.get("bus") if body.get("bus") is not None else "all")
|
| 201 |
+
modules = _clean_modules(body.get("modules") if body.get("modules") is not None else "all")
|
| 202 |
+
name = str(body.get("name") or "").strip() or uname
|
| 203 |
+
agent = str(body.get("agent") or "").strip() or None
|
| 204 |
+
email = str(body.get("email") or "").strip() or None
|
| 205 |
+
|
| 206 |
+
if uname in _registry():
|
| 207 |
+
raise err(409, "user_exists", f"{uname} already exists β PATCH it to change it")
|
| 208 |
+
users.create_user(uname, pw, name, role=role, bus=bus, modules=modules,
|
| 209 |
+
agent=agent, email=email)
|
| 210 |
+
reg = _registry()
|
| 211 |
+
if uname not in reg:
|
| 212 |
+
# The store accepted the write and does not have it. Reporting 201 here would be the
|
| 213 |
+
# "200 over a write that evaporated" failure the whole store-outage rule exists to prevent.
|
| 214 |
+
raise err(503, "store_unavailable", "the account was not saved β try again")
|
| 215 |
+
return {"user": _view(uname, reg[uname])}
|
| 216 |
+
|
| 217 |
+
|
| 218 |
+
@router.patch("/admin/users/{username}")
|
| 219 |
+
def update_user(username: str, body: dict = Body(default=None),
|
| 220 |
+
session: Session = Depends(admin_gate)):
|
| 221 |
+
"""Change access on an existing account. Only the keys PRESENT in the body change.
|
| 222 |
+
|
| 223 |
+
β NO EPOCH BUMP, and that is not an omission. `deps._user_for` re-reads the record on every
|
| 224 |
+
request, so a narrowed role / BU / module grant applies to the target's LIVE session on its very
|
| 225 |
+
next call β bumping the epoch would only add a forced re-login on top. `active: false` is the
|
| 226 |
+
exception and it bumps, inside `core.users.set_active`, because a disabled account must lose its
|
| 227 |
+
session rather than keep working until the cookie expires.
|
| 228 |
+
"""
|
| 229 |
+
body = body if isinstance(body, dict) else {}
|
| 230 |
+
reg = _registry()
|
| 231 |
+
uname = str(username or "").strip().lower()
|
| 232 |
+
if uname not in reg:
|
| 233 |
+
raise err(404, "no_such_user", f"no account named {uname!r}")
|
| 234 |
+
if not body:
|
| 235 |
+
raise err(400, "empty_patch", "no fields to update")
|
| 236 |
+
|
| 237 |
+
unknown = sorted(set(body) - {"name", "role", "bus", "modules", "active", "agent", "email"})
|
| 238 |
+
if unknown:
|
| 239 |
+
# A silently-ignored field is how a UI ends up believing it saved something it did not.
|
| 240 |
+
raise err(400, "unknown_fields", f"cannot update: {unknown}")
|
| 241 |
+
|
| 242 |
+
is_self = uname == session.uname
|
| 243 |
+
role = _clean_role(body["role"]) if "role" in body else None
|
| 244 |
+
active = bool(body["active"]) if "active" in body else None
|
| 245 |
+
# THE ONE-CLICK LOCKOUT GUARD. APP_PASSWORD remains the real backstop for 'admin', so this is
|
| 246 |
+
# not the thing that keeps the product reachable β it just stops an administrator removing
|
| 247 |
+
# their own access with a toggle and having to go find the master password.
|
| 248 |
+
if is_self and role is not None and role != "admin":
|
| 249 |
+
raise err(400, "self_demote",
|
| 250 |
+
"you cannot remove your own administrator role β ask another admin")
|
| 251 |
+
if is_self and active is False:
|
| 252 |
+
raise err(400, "self_deactivate", "you cannot deactivate your own account")
|
| 253 |
+
|
| 254 |
+
kw = {}
|
| 255 |
+
if role is not None:
|
| 256 |
+
kw["role"] = role
|
| 257 |
+
if "name" in body:
|
| 258 |
+
kw["name"] = str(body["name"] or "").strip() or uname
|
| 259 |
+
if "bus" in body:
|
| 260 |
+
kw["bus"] = _clean_bus(body["bus"])
|
| 261 |
+
if "modules" in body:
|
| 262 |
+
kw["modules"] = _clean_modules(body["modules"])
|
| 263 |
+
if "agent" in body:
|
| 264 |
+
kw["agent"] = str(body["agent"] or "").strip() # '' clears the link
|
| 265 |
+
if "email" in body:
|
| 266 |
+
kw["email"] = str(body["email"] or "").strip()
|
| 267 |
+
if kw:
|
| 268 |
+
users.set_access(uname, **kw)
|
| 269 |
+
if active is not None and active != bool(reg[uname].get("active", True)):
|
| 270 |
+
users.set_active(uname, active) # bumps the epoch: kills live sessions
|
| 271 |
+
|
| 272 |
+
fresh = _registry()
|
| 273 |
+
return {"user": _view(uname, fresh[uname])}
|
| 274 |
+
|
| 275 |
+
|
| 276 |
+
@router.post("/admin/users/{username}/password", status_code=204)
|
| 277 |
+
def set_password(username: str, body: dict = Body(default=None),
|
| 278 |
+
session: Session = Depends(admin_gate)):
|
| 279 |
+
"""Rotate a password. **Revokes every outstanding session for that account** β the epoch bump
|
| 280 |
+
happens inside the same read-modify-write as the hash (`core.users.set_password`), so the two
|
| 281 |
+
can never disagree.
|
| 282 |
+
|
| 283 |
+
204 with no body: there is nothing to say that the caller did not already know, and echoing the
|
| 284 |
+
account back would invite a client to diff a response for a password change.
|
| 285 |
+
"""
|
| 286 |
+
reg = _registry()
|
| 287 |
+
uname = str(username or "").strip().lower()
|
| 288 |
+
if uname not in reg:
|
| 289 |
+
raise err(404, "no_such_user", f"no account named {uname!r}")
|
| 290 |
+
pw = _clean_password((body or {}).get("password"))
|
| 291 |
+
before = int(reg[uname].get("epoch") or 0)
|
| 292 |
+
users.set_password(uname, pw)
|
| 293 |
+
after = int((_registry().get(uname) or {}).get("epoch") or 0)
|
| 294 |
+
if after <= before:
|
| 295 |
+
# The whole value of this route is the revocation. If the epoch did not move, the sessions
|
| 296 |
+
# were not revoked, and answering 204 would say they were.
|
| 297 |
+
raise err(503, "store_unavailable",
|
| 298 |
+
"the password change did not persist β outstanding sessions were NOT revoked")
|
| 299 |
+
return Response(status_code=204)
|
| 300 |
+
|
| 301 |
+
|
| 302 |
+
@router.get("/settings")
|
| 303 |
+
def settings(session: Session = Depends(require_session)):
|
| 304 |
+
"""The session's OWN settings β any authenticated user, not admin-only.
|
| 305 |
+
|
| 306 |
+
`user` comes from `routes_auth._public_user`, deliberately reusing the ONE definition of what a
|
| 307 |
+
client may know about itself (never a hash, never the epoch). `scope` restates it as the two
|
| 308 |
+
things the Settings UI shows, and `tenant` names only THIS tenant β enumerating the others
|
| 309 |
+
would answer a question about our customer list.
|
| 310 |
+
"""
|
| 311 |
+
modules = perms.allowed_modules(session.user)
|
| 312 |
+
return {
|
| 313 |
+
"user": _public_user(session.user),
|
| 314 |
+
"scope": {
|
| 315 |
+
"bus": perms.allowed_bu_labels(session.user),
|
| 316 |
+
"team_id": perms.scope_team_id(session.user),
|
| 317 |
+
"agent": perms.scope_agent(session.user),
|
| 318 |
+
"modules": sorted(modules) if modules is not None else "all",
|
| 319 |
+
},
|
| 320 |
+
"tenant": {"key": session.tenant, "name": getattr(session.runtime, "name",
|
| 321 |
+
session.tenant)},
|
| 322 |
+
# CHROME ONLY. Every /admin route re-checks the role server-side; this exists so the client
|
| 323 |
+
# does not paint a "Manage users" button that 403s.
|
| 324 |
+
"admin": perms.is_admin(session.user),
|
| 325 |
+
}
|
api/routes_auth.py
ADDED
|
@@ -0,0 +1,143 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""routes_auth.py β X2's auth leg: login / logout / me (EXIT-3a).
|
| 2 |
+
|
| 3 |
+
Password auth against the SAME `core/users` accounts the Streamlit `gate()` uses, so both
|
| 4 |
+
front-ends share one identity today. The amended D-3 (our own OIDC client against Authentik, with
|
| 5 |
+
a branded login page) lands when D-1/D-2 unblock (owner blocker B-4); this is the interim leg it
|
| 6 |
+
replaces, and it is deliberately the same account store so the migration is a swap of the
|
| 7 |
+
CREDENTIAL check, not of the user model.
|
| 8 |
+
"""
|
| 9 |
+
import hmac
|
| 10 |
+
import os
|
| 11 |
+
import time
|
| 12 |
+
from collections import OrderedDict
|
| 13 |
+
|
| 14 |
+
from fastapi import APIRouter, Body, Depends, Request, Response
|
| 15 |
+
|
| 16 |
+
import aios_session
|
| 17 |
+
from deps import Session, err, require_session, users, perms
|
| 18 |
+
|
| 19 |
+
router = APIRouter(prefix="/api/v1/auth")
|
| 20 |
+
|
| 21 |
+
#: THE SERVER OWNS BACKOFF (X5): the login page shows an inline error and never paces itself, so
|
| 22 |
+
#: the pacing has to be here or it does not exist. Bounded so it cannot grow into a memory leak.
|
| 23 |
+
#: β HONEST LIMIT: per-PROCESS. Behind several workers an attacker gets this budget per worker,
|
| 24 |
+
#: and it resets on deploy. It raises the cost of online guessing against PBKDF2-200k; it is not
|
| 25 |
+
#: a substitute for the real thing, which arrives with Authentik (D-1) owning rate limits centrally.
|
| 26 |
+
#: β KEYED BY (TENANT, USERNAME), not by username. Two tenants can each have an 'admin' or a
|
| 27 |
+
#: 'boss', and a shared bucket would mean failed logins against tenant B lock the same name out of
|
| 28 |
+
#: tenant A β a cross-tenant coupling in the one wave whose point is that tenants cannot reach
|
| 29 |
+
#: each other. It is also a (weak) existence oracle: a 429 for a name you never tried here would
|
| 30 |
+
#: tell you somebody else did.
|
| 31 |
+
_FAILS = OrderedDict()
|
| 32 |
+
_MAX_TRACKED = 2048
|
| 33 |
+
_LOCKOUT_AFTER = 8
|
| 34 |
+
_LOCKOUT_SECONDS = 60
|
| 35 |
+
|
| 36 |
+
|
| 37 |
+
def _throttled(key, now=None):
|
| 38 |
+
now = now or time.time()
|
| 39 |
+
hit = _FAILS.get(key)
|
| 40 |
+
if not hit:
|
| 41 |
+
return 0
|
| 42 |
+
count, last = hit
|
| 43 |
+
if count < _LOCKOUT_AFTER:
|
| 44 |
+
return 0
|
| 45 |
+
remaining = int(_LOCKOUT_SECONDS - (now - last))
|
| 46 |
+
return remaining if remaining > 0 else 0
|
| 47 |
+
|
| 48 |
+
|
| 49 |
+
def _note_failure(key, now=None):
|
| 50 |
+
now = now or time.time()
|
| 51 |
+
count, last = _FAILS.get(key, (0, 0.0))
|
| 52 |
+
# A window that has fully elapsed starts the count again β a user who mistypes twice today
|
| 53 |
+
# and twice next week is not an attacker.
|
| 54 |
+
if now - last > _LOCKOUT_SECONDS:
|
| 55 |
+
count = 0
|
| 56 |
+
_FAILS[key] = (count + 1, now)
|
| 57 |
+
_FAILS.move_to_end(key)
|
| 58 |
+
while len(_FAILS) > _MAX_TRACKED:
|
| 59 |
+
_FAILS.popitem(last=False)
|
| 60 |
+
|
| 61 |
+
|
| 62 |
+
def _clear_failures(key):
|
| 63 |
+
_FAILS.pop(key, None)
|
| 64 |
+
|
| 65 |
+
|
| 66 |
+
def _public_user(user):
|
| 67 |
+
"""What the client is allowed to know about itself. Never a hash, a salt, or the epoch β
|
| 68 |
+
the epoch is a server-side revocation handle and a client has no use for it."""
|
| 69 |
+
return {"username": user.get("username", ""), "name": user.get("name", ""),
|
| 70 |
+
"role": user.get("role", "user"),
|
| 71 |
+
"bus": perms.allowed_bu_labels(user),
|
| 72 |
+
"team_id": perms.scope_team_id(user),
|
| 73 |
+
"agent": perms.scope_agent(user),
|
| 74 |
+
"modules": sorted(perms.allowed_modules(user) or []) or "all",
|
| 75 |
+
"landing": perms.landing_page(user)}
|
| 76 |
+
|
| 77 |
+
|
| 78 |
+
@router.post("/login")
|
| 79 |
+
def login(request: Request, response: Response, body: dict = Body(default=None)):
|
| 80 |
+
body = body or {}
|
| 81 |
+
uname = str(body.get("username") or "").strip().lower()
|
| 82 |
+
pw = str(body.get("password") or "")
|
| 83 |
+
tenant = str(body.get("tenant") or "royal-imports").strip().lower()
|
| 84 |
+
|
| 85 |
+
# The tenant is validated BEFORE the credential check so an unknown slug can never mint a
|
| 86 |
+
# cookie β but it answers with the SAME 401 as a bad password, because "that tenant does not
|
| 87 |
+
# exist" is a fact about our customer list.
|
| 88 |
+
from harness import runtime
|
| 89 |
+
try:
|
| 90 |
+
runtime.get_runtime(tenant)
|
| 91 |
+
except KeyError:
|
| 92 |
+
raise err(401, "invalid_credentials", "that username and password do not match")
|
| 93 |
+
|
| 94 |
+
throttle_key = (tenant, uname)
|
| 95 |
+
wait = _throttled(throttle_key)
|
| 96 |
+
if wait:
|
| 97 |
+
raise err(429, "too_many_attempts",
|
| 98 |
+
f"too many failed attempts β try again in {wait} seconds")
|
| 99 |
+
|
| 100 |
+
user = users.verify(uname, pw) if (uname and pw) else None
|
| 101 |
+
if not user:
|
| 102 |
+
# CONSTANT-TIME-ISH 401. `users.verify` returns immediately for an unknown username (no
|
| 103 |
+
# record, no PBKDF2) and burns ~200k iterations for a known one, so the response time
|
| 104 |
+
# answers "does this account exist?" to anyone with a stopwatch. Burning one equivalent
|
| 105 |
+
# hash on the failure path removes that oracle. It costs nothing on the happy path.
|
| 106 |
+
try:
|
| 107 |
+
users._hash(pw or "x", "00" * 16)
|
| 108 |
+
except Exception:
|
| 109 |
+
pass
|
| 110 |
+
_note_failure(throttle_key)
|
| 111 |
+
raise err(401, "invalid_credentials", "that username and password do not match")
|
| 112 |
+
|
| 113 |
+
_clear_failures(throttle_key)
|
| 114 |
+
value, _claims = aios_session.mint(tenant, user["username"], int(user.get("epoch") or 0))
|
| 115 |
+
aios_session.set_cookie(response, request, value)
|
| 116 |
+
return {"user": _public_user(user)}
|
| 117 |
+
|
| 118 |
+
|
| 119 |
+
@router.post("/logout", status_code=204)
|
| 120 |
+
def logout(request: Request):
|
| 121 |
+
"""204 and the cookie is cleared. Deliberately NOT session-gated: logging out must work from
|
| 122 |
+
an already-invalid session, or a user holding a broken cookie has no way to get rid of it.
|
| 123 |
+
|
| 124 |
+
β THE COOKIE IS CLEARED ON THE RETURNED RESPONSE, not on an injected `response` param. When a
|
| 125 |
+
handler RETURNS a Response object, FastAPI ships that object β anything written to the
|
| 126 |
+
injected `response` is silently dropped. The first version of this route took `response:
|
| 127 |
+
Response`, called `clear_cookie` on it, then returned a fresh `Response(204)`: the 204 was
|
| 128 |
+
correct, no `Set-Cookie` was ever sent, and the session stayed alive after a "successful"
|
| 129 |
+
logout. Caught by asserting `/auth/me` is 401 AFTER the logout rather than trusting the 204.
|
| 130 |
+
|
| 131 |
+
β This clears the BROWSER's copy only β the signed value stays cryptographically valid until
|
| 132 |
+
it expires, which is the honest cost of a stateless session. "Sign me out everywhere" is
|
| 133 |
+
`core.users.bump_epoch` (a password change already does it); D2's Postgres session mirror
|
| 134 |
+
makes per-device revocation possible and arrives with C-2.
|
| 135 |
+
"""
|
| 136 |
+
out = Response(status_code=204)
|
| 137 |
+
aios_session.clear_cookie(out, request)
|
| 138 |
+
return out
|
| 139 |
+
|
| 140 |
+
|
| 141 |
+
@router.get("/me")
|
| 142 |
+
def me(session: Session = Depends(require_session)):
|
| 143 |
+
return {"user": _public_user(session.user), "tenant": session.tenant}
|
api/routes_customers.py
ADDED
|
@@ -0,0 +1,208 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""routes_customers.py β X2's read + write of the customer table, BU-SCOPED (EXIT-3b / EXIT-2a).
|
| 2 |
+
|
| 3 |
+
Two things happen here that did not happen in the pre-wave `main.py`:
|
| 4 |
+
|
| 5 |
+
1. **ROWS ARE SCOPED TO THE SESSION.** The old `/api/customers` served `cl.pool()` β the whole
|
| 6 |
+
book, to anyone holding the shared APP_PASSWORD. Now the pool is built with
|
| 7 |
+
`(team_id, agent_name)` derived from the USER RECORD, so a Royal-only user never receives a
|
| 8 |
+
Fisch row and an agent-linked login never receives another rep's book. The scope is applied
|
| 9 |
+
at the QUERY, not as a post-filter, so there is no moment at which the other BU's rows exist
|
| 10 |
+
in this response.
|
| 11 |
+
|
| 12 |
+
2. **THE OVERLAY FORK IS GONE.** `aios-web/api/data/overlay.json` was a SECOND writable home
|
| 13 |
+
for the same user-owned fields the Streamlit app keeps in the tenant store β two truths, and
|
| 14 |
+
whichever process you asked last was right. Reads and writes now both go through
|
| 15 |
+
`modules.customer_data`'s table-workspace functions: ONE store (C1c, ARCHITECTURE Β§1a rule 3).
|
| 16 |
+
|
| 17 |
+
β ONE STORE IS NOT YET ONE CACHE (strangler-period, booked honestly). `core.store.get()` is
|
| 18 |
+
cache-first per PROCESS, so a write from the Streamlit container is invisible to a running API
|
| 19 |
+
container until its cache is refreshed, and vice versa. Deleting the fork removes the second
|
| 20 |
+
SOURCE OF TRUTH; it does not make the two runtimes coherent. The fix is X4/Postgres (task C-4,
|
| 21 |
+
owner-blocked on B-3), and no test here may claim read-your-writes ACROSS runtimes β a TestClient
|
| 22 |
+
proof is single-process and would report green on exactly the thing that is still broken.
|
| 23 |
+
"""
|
| 24 |
+
import time
|
| 25 |
+
|
| 26 |
+
from fastapi import APIRouter, Body, Depends
|
| 27 |
+
|
| 28 |
+
from deps import Session, err, module_gate, perms
|
| 29 |
+
|
| 30 |
+
router = APIRouter(prefix="/api/v1")
|
| 31 |
+
|
| 32 |
+
#: The surface these routes serve. Both legs are gated on it, so a user without the grant gets a
|
| 33 |
+
#: 403 rather than an empty table that looks like "you have no customers".
|
| 34 |
+
MODULE = "customer_data"
|
| 35 |
+
|
| 36 |
+
_CACHE_TTL = 900 # the pool build is slow (Odoo + reconciliation); 15 min, as before
|
| 37 |
+
|
| 38 |
+
|
| 39 |
+
def _pool_rows(session: Session):
|
| 40 |
+
"""The reconciled Odoo pool for this session's SCOPE β the slow part, and the only part that
|
| 41 |
+
may be shared between users.
|
| 42 |
+
|
| 43 |
+
β WHAT MAY BE CACHED HERE, AND WHY THE LINE IS EXACTLY HERE. The cache key is
|
| 44 |
+
`(team_id, agent)` and the cached value is the raw pool: Odoo-source columns only. That is
|
| 45 |
+
genuinely scope-shaped β two users with the same BU and the same book are asking the same
|
| 46 |
+
question, and `cl.pool()` is expensive (an Odoo pull plus reconciliation).
|
| 47 |
+
|
| 48 |
+
The FULL PAYLOAD is NOT cacheable on this key, and caching it here was a real defect I shipped
|
| 49 |
+
and then removed. `fields` comes from `fields_from_workspace(ws)` β that user's own `custom_`
|
| 50 |
+
and `measure_` columns β and every overlay cell comes from `ws['overlays']`; the table
|
| 51 |
+
workspace is read as `data[username]` and written as `patch_table_overlay(uname, β¦)`, i.e.
|
| 52 |
+
PER USER. So two Royal-only users with no agent link share `(6, None)` and the second one
|
| 53 |
+
would have been served the FIRST one's private notes and private columns. The scope key was
|
| 54 |
+
right for the pool and wrong for everything wrapped around it.
|
| 55 |
+
|
| 56 |
+
It survived a green 129-check battery because every fixture user had a DISTINCT
|
| 57 |
+
`(team_id, agent)` pair, so no two of them ever collided on the key β the test set could not
|
| 58 |
+
express the bug. `verify_api.py` now carries a same-scope second user for exactly this.
|
| 59 |
+
"""
|
| 60 |
+
import modules.customer_data as cl
|
| 61 |
+
|
| 62 |
+
rt = session.runtime
|
| 63 |
+
team_id = perms.scope_team_id(session.user)
|
| 64 |
+
agent = perms.scope_agent(session.user)
|
| 65 |
+
key = ("pool", team_id, agent)
|
| 66 |
+
hit = rt.pool_cache.get(key)
|
| 67 |
+
now = time.time()
|
| 68 |
+
if hit and now - hit[0] < _CACHE_TTL:
|
| 69 |
+
return hit[1]
|
| 70 |
+
# β THE SCOPE GOES INTO THE BUILDER. `pool(agent_name, team_id)` is the same reconciled
|
| 71 |
+
# builder the Streamlit page uses β passing the scope here is what makes the isolation a
|
| 72 |
+
# property of the QUERY instead of a filter someone can forget to apply downstream.
|
| 73 |
+
rows_src = cl.pool(agent, team_id)
|
| 74 |
+
rt.pool_cache[key] = (now, rows_src)
|
| 75 |
+
# Bounded: a scope cache that only ever grows is a memory leak in a shared process.
|
| 76 |
+
if len(rt.pool_cache) > 16:
|
| 77 |
+
for stale in sorted(rt.pool_cache, key=lambda k: rt.pool_cache[k][0])[:8]:
|
| 78 |
+
rt.pool_cache.pop(stale, None)
|
| 79 |
+
return rows_src
|
| 80 |
+
|
| 81 |
+
|
| 82 |
+
def _payload(session: Session):
|
| 83 |
+
"""`{fields, rows, today, pulled_at}` β X2 fixes this at the EXACT shape the pre-wave
|
| 84 |
+
`/api/customers` served, which `verify_fields_contract.py` referees.
|
| 85 |
+
|
| 86 |
+
The per-USER half (this user's field list, this user's overlay cells) is assembled fresh on
|
| 87 |
+
every request from one workspace read. That read is a single store `get`, which is cheap next
|
| 88 |
+
to the pool build β and it is the only arrangement in which nothing user-shaped can outlive
|
| 89 |
+
the request that asked for it. See `_pool_rows` for what the cache holds and why.
|
| 90 |
+
"""
|
| 91 |
+
import aios_grid
|
| 92 |
+
from core import grid_events
|
| 93 |
+
|
| 94 |
+
rows_src = _pool_rows(session)
|
| 95 |
+
|
| 96 |
+
# ONE workspace read serves both the field list and the overlay values (doing it per stratum
|
| 97 |
+
# would triple the store reads per request).
|
| 98 |
+
ws = grid_events.table_workspace(_ctx_for(session, []), allowed_pids=None)
|
| 99 |
+
fields = aios_grid.fields_from_workspace(ws)
|
| 100 |
+
odoo_keys = [f["key"] for f in fields if f.get("source") == "odoo"]
|
| 101 |
+
overlay_keys = [f["key"] for f in fields if f.get("source") == "overlay"]
|
| 102 |
+
passthrough = {f["key"] for f in fields
|
| 103 |
+
if f.get("source") == "odoo" and f.get("type") in ("text", "status", "date")}
|
| 104 |
+
overlays = ws.get("overlays") or {}
|
| 105 |
+
|
| 106 |
+
# β `rows_src` IS THE SHARED CACHED LIST β read it, never mutate it. Every row below is a NEW
|
| 107 |
+
# dict; writing an overlay value onto `r` instead of `row` would put one user's private cell
|
| 108 |
+
# into the copy the next same-scope user is served, which is the leak this split exists to
|
| 109 |
+
# prevent, re-introduced one line lower down.
|
| 110 |
+
rows = []
|
| 111 |
+
for r in rows_src:
|
| 112 |
+
pid = r.get("pid")
|
| 113 |
+
row = {"pid": pid, "_created": r.get("_created") or ""}
|
| 114 |
+
for k in odoo_keys:
|
| 115 |
+
v = r.get(k)
|
| 116 |
+
row[k] = v if k in passthrough else _round(v)
|
| 117 |
+
stored = overlays.get(str(pid)) or {}
|
| 118 |
+
for k in overlay_keys:
|
| 119 |
+
row[k] = stored.get(k, "")
|
| 120 |
+
rows.append(row)
|
| 121 |
+
|
| 122 |
+
return {"fields": fields, "rows": rows,
|
| 123 |
+
# `today` rides the payload because every relative date condition must resolve against
|
| 124 |
+
# the TENANT's day, never the browser's β a client that falls back to its own clock
|
| 125 |
+
# disagrees with the server for everyone west of it.
|
| 126 |
+
"today": time.strftime("%Y-%m-%d"),
|
| 127 |
+
"pulled_at": time.strftime("%Y-%m-%d %H:%M")}
|
| 128 |
+
|
| 129 |
+
|
| 130 |
+
def _round(v):
|
| 131 |
+
# bool is a subclass of int β guard it so True/False never becomes 1/0. Mirrors
|
| 132 |
+
# royalimports_os/aios_grid.py._round exactly (embed == standalone on boolean-valued fields).
|
| 133 |
+
return round(v) if isinstance(v, (int, float)) and not isinstance(v, bool) else v
|
| 134 |
+
|
| 135 |
+
|
| 136 |
+
def _ctx_for(session: Session, pids):
|
| 137 |
+
"""An EventCtx for the READ path β no fallback workspace, so a store outage is a 503 rather
|
| 138 |
+
than a phantom in-memory workspace an API request cannot persist."""
|
| 139 |
+
from core import grid_events
|
| 140 |
+
return grid_events.EventCtx(
|
| 141 |
+
uname=session.uname, allowed_pids=frozenset(pids or ()), fields=[],
|
| 142 |
+
admin=session.admin, fallback_ws=None, seen_ids={})
|
| 143 |
+
|
| 144 |
+
|
| 145 |
+
def allowed_pids(session: Session):
|
| 146 |
+
"""The pids this session may touch β the POOL's own ids, so the write wall and the read scope
|
| 147 |
+
can never disagree.
|
| 148 |
+
|
| 149 |
+
Reads `_pool_rows` rather than `_payload`: the wall only needs identities, and going through
|
| 150 |
+
the full payload would pay for a workspace read and a row assembly on every write.
|
| 151 |
+
"""
|
| 152 |
+
return frozenset(r["pid"] for r in _pool_rows(session) if r.get("pid") is not None)
|
| 153 |
+
|
| 154 |
+
|
| 155 |
+
@router.get("/customers")
|
| 156 |
+
def customers(session: Session = Depends(module_gate(MODULE))):
|
| 157 |
+
return _payload(session)
|
| 158 |
+
|
| 159 |
+
|
| 160 |
+
@router.patch("/customers/{pid}")
|
| 161 |
+
def patch_customer(pid: int, body: dict = Body(default=None),
|
| 162 |
+
session: Session = Depends(module_gate(MODULE))):
|
| 163 |
+
"""Write the EDITABLE overlay stratum only β Odoo stays read-only, forever.
|
| 164 |
+
|
| 165 |
+
Routed through `core.grid_events.handle_one` as an `overlay_patch` event rather than writing
|
| 166 |
+
the store directly: that handler is where the per-key `permissions.edit` wall, the pid wall
|
| 167 |
+
and the truncation rules live, and a second implementation of those would be a second set of
|
| 168 |
+
them to keep in step. The response reports what was ACCEPTED, which is not always what was
|
| 169 |
+
asked for.
|
| 170 |
+
"""
|
| 171 |
+
from core import grid_events
|
| 172 |
+
|
| 173 |
+
updates = dict(body or {})
|
| 174 |
+
if not updates:
|
| 175 |
+
raise err(400, "empty_patch", "no fields to update")
|
| 176 |
+
pool = allowed_pids(session)
|
| 177 |
+
if pid not in pool:
|
| 178 |
+
# 403, not 404: the pid may well exist β it is simply not in this session's book, and
|
| 179 |
+
# saying "no such customer" would confirm the opposite to anyone who guessed right.
|
| 180 |
+
raise err(403, "out_of_scope", "that customer is not in your book")
|
| 181 |
+
payload = _payload(session)
|
| 182 |
+
ctx = grid_events.EventCtx(
|
| 183 |
+
uname=session.uname, allowed_pids=pool, fields=payload["fields"],
|
| 184 |
+
admin=session.admin, fallback_ws=None, seen_ids={})
|
| 185 |
+
try:
|
| 186 |
+
grid_events.handle_one(
|
| 187 |
+
{"id": f"patch:{pid}:{time.time_ns()}", "type": "overlay_patch",
|
| 188 |
+
"pid": pid, "updates": updates}, ctx)
|
| 189 |
+
except grid_events.StoreUnavailable:
|
| 190 |
+
raise err(503, "store_unavailable",
|
| 191 |
+
"the tenant store is unavailable β your change was not saved")
|
| 192 |
+
|
| 193 |
+
# What actually landed, read back from the store rather than echoed from the request: a
|
| 194 |
+
# refused key or a truncated value must not be reported as accepted.
|
| 195 |
+
stored = (grid_events.table_workspace(_ctx_for(session, pool), allowed_pids=None)
|
| 196 |
+
.get("overlays") or {}).get(str(pid)) or {}
|
| 197 |
+
accepted = {k: stored.get(k) for k in updates if k in stored}
|
| 198 |
+
refused = sorted(k for k in updates if k not in accepted or stored.get(k) != str(updates[k]))
|
| 199 |
+
# No cache to patch: the overlay stratum is re-read from the store on every `_payload`, so
|
| 200 |
+
# read-your-writes within this runtime is a property of the design rather than of a
|
| 201 |
+
# write-through step somebody has to remember. (It was a write-through step while the whole
|
| 202 |
+
# payload was cached on a scope key β the arrangement that leaked one user's notes to
|
| 203 |
+
# another. See `_pool_rows`.) Cross-RUNTIME coherence is still not claimed: the module
|
| 204 |
+
# docstring says why, and Postgres is the fix.
|
| 205 |
+
out = {"ok": True, "pid": pid, "updates": accepted}
|
| 206 |
+
if refused:
|
| 207 |
+
out["refused"] = refused
|
| 208 |
+
return out
|
api/routes_grid.py
ADDED
|
@@ -0,0 +1,155 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""routes_grid.py β X2's write seam over HTTP: the SECOND adapter on `core.grid_events` (EXIT-1c).
|
| 2 |
+
|
| 3 |
+
`POST /api/v1/grid/events` takes the component's event objects VERBATIM β the same objects the
|
| 4 |
+
Streamlit host receives through the component value slot, unchanged β and runs them through the
|
| 5 |
+
same `core.grid_events.handle_events` the Streamlit adapter runs. That is the whole point of
|
| 6 |
+
EXIT-1a: one implementation of every permission wall, two transports. A route that re-validated
|
| 7 |
+
anything here would be a second wall to keep in step, and the two would drift on the first change.
|
| 8 |
+
|
| 9 |
+
DEDUP IS PER REQUEST, and that is a deliberate limit, not an oversight. The Streamlit adapter's
|
| 10 |
+
`seen_ids` lives in `st.session_state` β genuinely per user session β because the client resends
|
| 11 |
+
its recent 24-event window on every emit inside one page session. A stateless API has no such
|
| 12 |
+
dict, and inventing a per-user server-side one would be exactly the resident per-tenant state
|
| 13 |
+
EXIT-4a exists to remove. So: ids are deduped WITHIN a request body (the resend window's whole
|
| 14 |
+
purpose β a batch that repeats an id processes it once), and a genuinely replayed request is
|
| 15 |
+
handled by the operations being idempotent. The one event where a replay is observable is
|
| 16 |
+
`add_to_list` (it would add the same pids to the same cohort twice β a set union, so the
|
| 17 |
+
membership is unchanged, but the toast count repeats). Noted rather than papered over; a
|
| 18 |
+
server-side idempotency key belongs with D2's session mirror (C-2).
|
| 19 |
+
|
| 20 |
+
STORE DOWN = 503. `fallback_ws` is None on this adapter, so `core.grid_events` raises
|
| 21 |
+
`StoreUnavailable` rather than writing to an in-memory workspace no API request could ever read
|
| 22 |
+
back. A 200 over a write that evaporated is the failure this rule exists to prevent.
|
| 23 |
+
"""
|
| 24 |
+
import time
|
| 25 |
+
|
| 26 |
+
from fastapi import APIRouter, Body, Depends
|
| 27 |
+
|
| 28 |
+
from deps import Session, err, module_gate
|
| 29 |
+
|
| 30 |
+
router = APIRouter(prefix="/api/v1")
|
| 31 |
+
|
| 32 |
+
MODULE = "customer_data"
|
| 33 |
+
|
| 34 |
+
#: The client's own resend window (`app.py`'s `[-24:]`). A body larger than this is not a
|
| 35 |
+
#: legitimate client β refuse it rather than doing 500 store writes on one request.
|
| 36 |
+
_MAX_EVENTS = 24
|
| 37 |
+
|
| 38 |
+
|
| 39 |
+
def _ctx(session: Session, fields, pids, **kw):
|
| 40 |
+
from core import grid_events
|
| 41 |
+
return grid_events.EventCtx(
|
| 42 |
+
uname=session.uname, allowed_pids=pids, fields=fields, admin=session.admin,
|
| 43 |
+
fallback_ws=None, seen_ids={}, **kw)
|
| 44 |
+
|
| 45 |
+
|
| 46 |
+
#: The two surfaces the ONE grid serves. `cohort` is the same table over hand-curated SETS rather
|
| 47 |
+
#: than over the whole scoped pool β `app.py:page_cohort` calls the same `_table_grid` with
|
| 48 |
+
#: `cohort_mode=True, scope_key='cohort'`.
|
| 49 |
+
_SCOPES = ("customer", "cohort")
|
| 50 |
+
|
| 51 |
+
|
| 52 |
+
def _scope_or_400(raw):
|
| 53 |
+
"""β REFUSE AN UNKNOWN SCOPE, never default it. A typo silently served as `customer` would
|
| 54 |
+
hand a user the whole book on a page they opened to see one cohort β the widening direction,
|
| 55 |
+
which is the one that must fail closed."""
|
| 56 |
+
scope = (raw or "customer").strip().lower()
|
| 57 |
+
if scope not in _SCOPES:
|
| 58 |
+
raise err(400, "bad_scope",
|
| 59 |
+
f"scope must be one of {', '.join(_SCOPES)} β refusing to guess")
|
| 60 |
+
return scope
|
| 61 |
+
|
| 62 |
+
|
| 63 |
+
@router.get("/workspace")
|
| 64 |
+
def workspace(scope: str = "customer",
|
| 65 |
+
session: Session = Depends(module_gate(MODULE))):
|
| 66 |
+
"""The durable table workspace for this session: views, fields, overlays, folders.
|
| 67 |
+
|
| 68 |
+
β WHY THIS EXISTS (S1βS2, 2026-07-30 β an X2 AMENDMENT, see the split doc). X2 fixes
|
| 69 |
+
`/customers` at the exact shape `verify_fields_contract.py` referees, so the workspace cannot
|
| 70 |
+
ride along in it without forking the thing that gate keeps single-sourced. Without a
|
| 71 |
+
workspace route the standalone shell's write path would be WRITE-ONLY: events persist
|
| 72 |
+
server-side and nothing reads them back on reload, so a saved view looks lost to the user
|
| 73 |
+
even though the store has it. This route closes that read-back gap at its own URL.
|
| 74 |
+
|
| 75 |
+
`allowed_pids` is passed so a SHARED view's `memberPids` are re-scoped to THIS reader β the
|
| 76 |
+
wave-9 leak rule. Omitting it would hand a Fisch-scoped user a member list built by a
|
| 77 |
+
full-access user.
|
| 78 |
+
"""
|
| 79 |
+
from core import grid_events
|
| 80 |
+
from routes_customers import allowed_pids
|
| 81 |
+
|
| 82 |
+
scope = _scope_or_400(scope)
|
| 83 |
+
try:
|
| 84 |
+
pids = allowed_pids(session)
|
| 85 |
+
ws = grid_events.table_workspace(_ctx(session, [], pids), allowed_pids=pids)
|
| 86 |
+
except grid_events.StoreUnavailable:
|
| 87 |
+
raise err(503, "store_unavailable", "the tenant store is unavailable")
|
| 88 |
+
|
| 89 |
+
# β THE SURFACE STAMP β a MIRROR of `app.py:5830-5836`, and it must stay one. These four keys
|
| 90 |
+
# are the ONLY thing that tells the client which surface it is drawing: `CustomerGrid.tsx:685`
|
| 91 |
+
# reads `payload.workspace.cohortMode` and nothing else. Until this route could express them,
|
| 92 |
+
# the standalone shell had no way to render Cohort at all β the write path already accepted
|
| 93 |
+
# `scopeKey` (below), so the read path was the whole gap.
|
| 94 |
+
#
|
| 95 |
+
# β Deliberately NOT `hideViews`. The Streamlit Cohort page does not set it either: `cohortMode`
|
| 96 |
+
# is what swaps the Views panel for the cohort list, so setting both would be two switches for
|
| 97 |
+
# one behaviour and they would drift.
|
| 98 |
+
ws = dict(ws) # never stamp the object the store handed back
|
| 99 |
+
ws["scopeKey"] = scope
|
| 100 |
+
if scope == "cohort":
|
| 101 |
+
ws["cohortMode"] = True
|
| 102 |
+
# The create pane offers "this cohort table only" (default) vs "all customer tables".
|
| 103 |
+
ws["scopeChoice"] = True
|
| 104 |
+
return {"workspace": ws}
|
| 105 |
+
|
| 106 |
+
|
| 107 |
+
@router.post("/grid/events")
|
| 108 |
+
def grid_events_route(body: dict = Body(default=None),
|
| 109 |
+
session: Session = Depends(module_gate(MODULE))):
|
| 110 |
+
"""`{events: [<component event objects, verbatim>]}` β `{results, doc?, toast?}`."""
|
| 111 |
+
from core import grid_events
|
| 112 |
+
from routes_customers import _payload, allowed_pids
|
| 113 |
+
|
| 114 |
+
events = (body or {}).get("events")
|
| 115 |
+
if events is None and isinstance(body, dict) and body.get("type"):
|
| 116 |
+
events = [body] # a single event object, the legacy shape
|
| 117 |
+
if not isinstance(events, list):
|
| 118 |
+
raise err(400, "bad_events", "expected {events: [...]}")
|
| 119 |
+
if len(events) > _MAX_EVENTS:
|
| 120 |
+
raise err(400, "too_many_events",
|
| 121 |
+
f"at most {_MAX_EVENTS} events per request (the client's resend window)")
|
| 122 |
+
|
| 123 |
+
payload = _payload(session)
|
| 124 |
+
pids = frozenset(r["pid"] for r in payload["rows"] if r.get("pid") is not None)
|
| 125 |
+
# Validated with the SAME predicate as the read route. An unrecognised scopeKey used to be
|
| 126 |
+
# passed through verbatim, and `core.grid_events` only ever compares it to 'cohort' β so a
|
| 127 |
+
# typo degraded silently to customer-scope behaviour on a WRITE. Read and write must agree on
|
| 128 |
+
# what a scope is, or the surface you read is not the surface you wrote.
|
| 129 |
+
ctx = _ctx(session, payload["fields"], pids,
|
| 130 |
+
scope_key=_scope_or_400((body or {}).get("scopeKey")))
|
| 131 |
+
|
| 132 |
+
# Per-event results so the client can tell which of a batch landed β the component's own
|
| 133 |
+
# bridge has no response channel at all, so this is strictly more than the embed gets.
|
| 134 |
+
results = []
|
| 135 |
+
try:
|
| 136 |
+
for one in events:
|
| 137 |
+
eid = str(one.get("id") or "") if isinstance(one, dict) else ""
|
| 138 |
+
rerender = grid_events.handle_one(one, ctx)
|
| 139 |
+
results.append({"id": eid, "rerender": bool(rerender)})
|
| 140 |
+
except grid_events.StoreUnavailable:
|
| 141 |
+
raise err(503, "store_unavailable",
|
| 142 |
+
"the tenant store is unavailable β none of your changes were saved")
|
| 143 |
+
|
| 144 |
+
out = {"results": results, "rerender": any(r["rerender"] for r in results)}
|
| 145 |
+
if ctx.out.doc is not None:
|
| 146 |
+
out["doc"] = ctx.out.doc
|
| 147 |
+
if ctx.out.toast is not None:
|
| 148 |
+
out["toast"] = ctx.out.toast
|
| 149 |
+
# β NOTHING IS INVALIDATED HERE, on purpose. The runtime cache holds ONLY the scope-shaped
|
| 150 |
+
# Odoo pool (see `routes_customers._pool_rows`), and no event on this route can change an
|
| 151 |
+
# Odoo column β Odoo is read-only. Everything an event DOES change (overlays, fields, views,
|
| 152 |
+
# folders, cohorts) is re-read from the store on the next request. An earlier version cleared
|
| 153 |
+
# `pool_cache` after an `overlay_patch`, which threw away an expensive Odoo pull to refresh
|
| 154 |
+
# data that was never in it.
|
| 155 |
+
return out
|
api/routes_nav.py
ADDED
|
@@ -0,0 +1,34 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""routes_nav.py β X2's `GET /api/v1/nav`: the registry, filtered by what this session may open.
|
| 2 |
+
|
| 3 |
+
SERVER-FILTERED, not client-filtered. The client renders what it is given and never decides who
|
| 4 |
+
may see what β a nav that hides a link the API would still serve is a UI courtesy, not a
|
| 5 |
+
permission. `core.perms.nav_pages` is the single predicate (shared with `may_open`'s page gate),
|
| 6 |
+
so the nav and the 403 can never disagree about a grant.
|
| 7 |
+
|
| 8 |
+
The full rule set β archived is invisible to everyone, `group_only` rows are excluded so no
|
| 9 |
+
`parent` reference can dangle, `nav: False` surfaces ship as `chrome: 'utility'`, and the
|
| 10 |
+
`prefs.json` Library preference is deliberately NOT applied β is documented on
|
| 11 |
+
`core.perms.nav_pages`, which is where it belongs: one place, both callers.
|
| 12 |
+
"""
|
| 13 |
+
from fastapi import APIRouter, Depends
|
| 14 |
+
|
| 15 |
+
from deps import Session, perms, require_session
|
| 16 |
+
|
| 17 |
+
router = APIRouter(prefix="/api/v1")
|
| 18 |
+
|
| 19 |
+
|
| 20 |
+
@router.get("/nav")
|
| 21 |
+
def nav(session: Session = Depends(require_session)):
|
| 22 |
+
"""`{pages: [{key,label,source?,chrome}], landing}`.
|
| 23 |
+
|
| 24 |
+
Note what this does NOT do: it never returns an empty `pages` list as a way of saying "you
|
| 25 |
+
are not allowed". A session that may open nothing at all is a misconfigured account, and it
|
| 26 |
+
gets an explicit 403 β an empty 200 is indistinguishable from "the registry is empty" and is
|
| 27 |
+
how a permission bug hides in plain sight (X2's never-an-empty-200 rule).
|
| 28 |
+
"""
|
| 29 |
+
pages = perms.nav_pages(session.user)
|
| 30 |
+
if not pages:
|
| 31 |
+
from deps import err
|
| 32 |
+
raise err(403, "no_surfaces",
|
| 33 |
+
"your account has no dashboards assigned β ask an administrator")
|
| 34 |
+
return {"pages": pages, "landing": perms.landing_page(session.user)}
|
api/routes_pages.py
ADDED
|
@@ -0,0 +1,65 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""routes_pages.py β X2/Y1's read of a whole dashboard page: `GET /api/v1/pages/{key}`.
|
| 2 |
+
|
| 3 |
+
ONE route serves every ported page. That is the wave-3 promise made mechanical: Collections and
|
| 4 |
+
Procurement arrive as a builder registered in `pages.py`, and they inherit this route's session
|
| 5 |
+
wall, grant wall, BU resolution, caching and error shape without a line being written here.
|
| 6 |
+
|
| 7 |
+
WHY THE GATE IS INSIDE THE HANDLER and not `Depends(module_gate(...))`. `module_gate` closes over a
|
| 8 |
+
FIXED module key at import; the key here is a PATH PARAMETER, so the check has to happen once the
|
| 9 |
+
key is known. It is the same predicate (`Session.require` β `perms.may_open`), reached from
|
| 10 |
+
`pages.build_envelope` so a future builder cannot be added without it.
|
| 11 |
+
|
| 12 |
+
FAIL-CLOSED, and the order matters:
|
| 13 |
+
no cookie -> 401 (require_session)
|
| 14 |
+
unknown / unported page key -> 404 (a fact about our roadmap, not about this user)
|
| 15 |
+
no grant for the page -> 403 (never an empty 200 β an empty page reads as "no data")
|
| 16 |
+
a BU the account may not see -> 403 (never coerced to their own BU)
|
| 17 |
+
a malformed bu -> 400
|
| 18 |
+
"""
|
| 19 |
+
from fastapi import APIRouter, Depends, Query
|
| 20 |
+
|
| 21 |
+
import pages
|
| 22 |
+
# β IMPORTED HERE AT MODULE LEVEL ON PURPOSE, even though `pages.builder_for` also imports it
|
| 23 |
+
# lazily. `deploy_web.py` ships a hand-maintained file set, and W2-11 adds a `check_include()`-style
|
| 24 |
+
# IMPORT WALK to catch what that set misses β a walk that only follows module-level imports would
|
| 25 |
+
# never reach a builder registered from inside a function body, and a missing module CRASHED the
|
| 26 |
+
# live Space on 2026-07-04. One visible import makes the builder reachable by static analysis; the
|
| 27 |
+
# lazy import in `pages.py` stays as the fallback that keeps the registry self-assembling.
|
| 28 |
+
# (No circularity: `pages` is fully loaded by the line above before `pages_sales` imports it back.)
|
| 29 |
+
import pages_sales # noqa: F401
|
| 30 |
+
from deps import Session, require_session
|
| 31 |
+
|
| 32 |
+
router = APIRouter(prefix="/api/v1")
|
| 33 |
+
|
| 34 |
+
|
| 35 |
+
@router.get("/pages/{key}")
|
| 36 |
+
def page_data(key: str,
|
| 37 |
+
bu: str = Query(default=None,
|
| 38 |
+
description="'all' or a business-unit id. NARROWS the session's own "
|
| 39 |
+
"scope; it can never widen it."),
|
| 40 |
+
period: str = Query(default=None,
|
| 41 |
+
description="Page-specific display parameter. The resolved value "
|
| 42 |
+
"is echoed in controls[].value."),
|
| 43 |
+
session: Session = Depends(require_session)):
|
| 44 |
+
"""The Y1 envelope for one page, scoped to this session's BU + own-book grant."""
|
| 45 |
+
return pages.build_envelope(session, key, bu=bu, period=period)
|
| 46 |
+
|
| 47 |
+
|
| 48 |
+
@router.get("/pages")
|
| 49 |
+
def page_index(session: Session = Depends(require_session)):
|
| 50 |
+
"""Which pages have published data, filtered to what this session may open.
|
| 51 |
+
|
| 52 |
+
The shell needs this to know whether a nav entry leads to a ported page or to the Streamlit
|
| 53 |
+
host during the strangler period β and it is `may_open`-filtered for the same reason
|
| 54 |
+
`/api/v1/nav` is: the client renders what it is given and never decides who may see what.
|
| 55 |
+
|
| 56 |
+
β AN EMPTY LIST IS A LEGITIMATE 200 HERE, and that is not a hole in the never-an-empty-200 rule.
|
| 57 |
+
`/api/v1/nav` 403s on empty because a session that may open NOTHING is a misconfigured account.
|
| 58 |
+
This route answers "which of your pages have been PORTED yet", and during the strangler period
|
| 59 |
+
"none of them" is the ordinary, truthful answer for a user granted only surfaces that still
|
| 60 |
+
live in the Streamlit host. Returning 403 would say they lack access to pages they can see.
|
| 61 |
+
"""
|
| 62 |
+
from deps import perms
|
| 63 |
+
keys = [k for k in pages.known_pages()
|
| 64 |
+
if perms.may_open(session.user, (pages.builder_for(k).MODULE))]
|
| 65 |
+
return {"pages": keys}
|
requirements.txt
ADDED
|
@@ -0,0 +1,12 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# AIOS web API β the FastAPI backend + the royalimports_os data-layer deps it reuses.
|
| 2 |
+
# (streamlit is included only because the shared modules/core may import it at load; it is never
|
| 3 |
+
# run here. No altair/plotly/openpyxl/reportlab β those are Streamlit-UI/export only.)
|
| 4 |
+
fastapi>=0.139
|
| 5 |
+
uvicorn[standard]>=0.30
|
| 6 |
+
python-dotenv>=1.0
|
| 7 |
+
pandas>=2.0
|
| 8 |
+
requests>=2.28
|
| 9 |
+
huggingface_hub>=0.20
|
| 10 |
+
duckdb>=1.0
|
| 11 |
+
pyyaml>=6.0
|
| 12 |
+
streamlit==1.58.0
|
royalimports_os/aios_grid.py
ADDED
|
@@ -0,0 +1,1221 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""aios_grid β embed the AIOS React/glide Airtable-style grid inside Streamlit.
|
| 2 |
+
|
| 3 |
+
This is REUSABLE MODULE INFRASTRUCTURE: the same self-contained grid that runs as the
|
| 4 |
+
standalone aios-web app is inlined into a single HTML file and hosted inside a Streamlit
|
| 5 |
+
component. The React bundle reads its data from `window.__AIOS_GRID__` (an object
|
| 6 |
+
`{fields, rows}`) that we inject into the page <head> before the app's module script runs
|
| 7 |
+
β so there is NO backend and NO /api call in the Streamlit container; the browser only ever
|
| 8 |
+
sees the derived JSON we hand it.
|
| 9 |
+
|
| 10 |
+
Usage (from any page):
|
| 11 |
+
import aios_grid
|
| 12 |
+
aios_grid.render(aios_grid.rows_from_pool(pool_rows), aios_grid.FIELDS)
|
| 13 |
+
|
| 14 |
+
Design notes:
|
| 15 |
+
* ZERO app-internal imports (no `modules.*` / `core.*`). FIELDS is a literal and
|
| 16 |
+
`rows_from_pool` takes already-built rows β so this helper is tenant/module-agnostic and
|
| 17 |
+
sidesteps deploy_hf.py's import guard entirely.
|
| 18 |
+
* The built HTML ships as `aios_grid_embed.html` in THIS directory (added to
|
| 19 |
+
deploy_hf.py INCLUDE). Build it with `npm run build:embed` in aios-web/web, then copy
|
| 20 |
+
dist-embed/index.html -> royalimports_os/aios_grid_embed.html (see build_embed.py).
|
| 21 |
+
For LOCAL dev before that copy, we fall back to reading dist-embed/index.html directly.
|
| 22 |
+
* The preferred host is a Streamlit Components v1 bridge. It sends data/view/schema args
|
| 23 |
+
into the React app and returns guarded events for persistence in the tenant store. The old
|
| 24 |
+
injected-HTML path remains a read/local-write fallback when component assets are absent.
|
| 25 |
+
"""
|
| 26 |
+
import json
|
| 27 |
+
import re
|
| 28 |
+
from pathlib import Path
|
| 29 |
+
|
| 30 |
+
_HERE = Path(__file__).resolve().parent
|
| 31 |
+
|
| 32 |
+
# Where the inlined single-file build lives. FIRST match wins:
|
| 33 |
+
# 1. the shipped copy in this dir (what deploy_hf.py uploads to the Space)
|
| 34 |
+
# 2. the raw build output in the sibling aios-web tree (local dev, pre-copy)
|
| 35 |
+
_EMBED_CANDIDATES = [
|
| 36 |
+
_HERE / "aios_grid_embed.html",
|
| 37 |
+
_HERE.parent / "aios-web" / "web" / "dist-embed" / "index.html",
|
| 38 |
+
]
|
| 39 |
+
_COMPONENT_CANDIDATES = [
|
| 40 |
+
_HERE / "aios_grid_component",
|
| 41 |
+
_HERE.parent / "aios-web" / "web" / "dist-embed",
|
| 42 |
+
]
|
| 43 |
+
_DECLARED_COMPONENTS = {}
|
| 44 |
+
|
| 45 |
+
# --- the FIELD CONTRACT β loaded from the CANONICAL source `aios_grid_fields.json` in THIS
|
| 46 |
+
# directory (the SINGLE source of truth, shared with aios-web/api/main.py). source='odoo' is
|
| 47 |
+
# READ-ONLY; source='overlay' is the editable stratum (notes/tags) that lives OUTSIDE Odoo.
|
| 48 |
+
#
|
| 49 |
+
# Why a sibling JSON and NOT an import: aios_grid.py's contract is ZERO app-internal imports so
|
| 50 |
+
# it stays tenant/module-agnostic and sidesteps deploy_hf.py's import guard. A JSON next to the
|
| 51 |
+
# module preserves that exactly β no import, no guard interaction β while still single-sourcing
|
| 52 |
+
# the values (a build-time copy from another tree would reintroduce the drift we're removing).
|
| 53 |
+
# The file ships to the Space via deploy_hf.py INCLUDE. Edit the JSON, then run
|
| 54 |
+
# aios-web/verify_fields_contract.py. ---
|
| 55 |
+
_FIELDS_PATH = _HERE / "aios_grid_fields.json"
|
| 56 |
+
|
| 57 |
+
|
| 58 |
+
def _load_fields():
|
| 59 |
+
if not _FIELDS_PATH.is_file():
|
| 60 |
+
raise FileNotFoundError(
|
| 61 |
+
f"aios_grid: canonical field contract missing at {_FIELDS_PATH}. It is the single "
|
| 62 |
+
"source of truth for the grid schema and MUST ship (deploy_hf.py INCLUDE lists it)."
|
| 63 |
+
)
|
| 64 |
+
doc = json.loads(_FIELDS_PATH.read_text(encoding="utf-8"))
|
| 65 |
+
fields = doc.get("fields") if isinstance(doc, dict) else doc
|
| 66 |
+
if not isinstance(fields, list) or not fields:
|
| 67 |
+
raise ValueError(f"aios_grid: {_FIELDS_PATH} has no 'fields' list.")
|
| 68 |
+
return fields
|
| 69 |
+
|
| 70 |
+
|
| 71 |
+
FIELDS = _load_fields()
|
| 72 |
+
|
| 73 |
+
# text/date fields pass through untouched; every OTHER odoo (numeric) field is rounded β
|
| 74 |
+
# mirrors aios-web/api/main.py _payload() exactly so embed == standalone byte-for-byte.
|
| 75 |
+
|
| 76 |
+
|
| 77 |
+
def _round(v):
|
| 78 |
+
return round(v) if isinstance(v, (int, float)) and not isinstance(v, bool) else v
|
| 79 |
+
|
| 80 |
+
|
| 81 |
+
# Types a USER may create from the column menu (owner item 7, 2026-07-26). Mirrors
|
| 82 |
+
# customer-grid/types.ts CREATABLE_TYPES; verify_fields_contract.py holds the two in step.
|
| 83 |
+
# select β a single-select with its own `options` (the "Status" a user wants to add; distinct
|
| 84 |
+
# from the Odoo-sourced `status` lifecycle, which is not user-defined)
|
| 85 |
+
# user β an assignee, whose choices come from the HOST's real user list, never from here
|
| 86 |
+
#: `multiselect` (wave-2 item 5, 2026-07-27): the Airtable-style MULTI select β declared
|
| 87 |
+
#: options like `select`, but the cell holds a comma-joined SET and the row belongs to every
|
| 88 |
+
#: member of it (the `multi` grouping/cell contract the Cohorts column established).
|
| 89 |
+
#: Wave-5 item 11 (2026-07-27): `checkbox` (cell = bool; the overlay stores '1' or '') Β·
|
| 90 |
+
#: `phone` / `email` / `url` (text-family with per-type rendering) Β· `rating` (top-level
|
| 91 |
+
#: `max`, SVG stars client-side β never emoji) Β· `created_time` (READ-ONLY, renders the row's
|
| 92 |
+
#: `_created`) Β· `formula` (READ-ONLY, client-computed from the row's other cells).
|
| 93 |
+
CUSTOM_FIELD_TYPES = {"text", "select", "multiselect", "user", "int", "currency", "pct", "date",
|
| 94 |
+
"checkbox", "phone", "email", "url", "rating", "created_time", "formula"}
|
| 95 |
+
#: User-created types whose CELLS are read-only: their values are computed (formula β client
|
| 96 |
+
#: side, any error degrades to BLANK) or system-owned (created_time = the row's `_created`).
|
| 97 |
+
#: Emitted with the cohort column's read-only mechanism β `source: 'odoo'` + `derived` β so
|
| 98 |
+
#: the client never offers an editor and the host never accepts a cell write for them.
|
| 99 |
+
READONLY_CUSTOM_TYPES = {"created_time", "formula"}
|
| 100 |
+
MAX_FIELD_OPTIONS = 50
|
| 101 |
+
MAX_FORMULA_LEN = 500
|
| 102 |
+
#: `rating` bounds. Airtable caps at 10; below 2 a rating is a checkbox.
|
| 103 |
+
RATING_MAX_DEFAULT, RATING_MAX_MIN, RATING_MAX_MAX = 5, 2, 10
|
| 104 |
+
|
| 105 |
+
#: Key prefix of a FORMULA-MEASURE column (owner item 7, 2026-07-27): a user-created field that
|
| 106 |
+
#: IS a measure over a window β `Sales Β· the last 90 days` as a column. Mirrors the client's
|
| 107 |
+
#: `measure_` keys in CustomerGrid.createField. Distinct from `custom_` because the two strata
|
| 108 |
+
#: could not be more different: `custom_` is the EDITABLE overlay (user-typed values), while a
|
| 109 |
+
#: measure field is READ-ONLY and its values are computed by the host per render.
|
| 110 |
+
MEASURE_FIELD_PREFIX = "measure_"
|
| 111 |
+
#: The numeric types a measure can render as (semantic._FORMAT_TYPE's range).
|
| 112 |
+
MEASURE_FIELD_TYPES = {"currency", "int", "pct"}
|
| 113 |
+
|
| 114 |
+
|
| 115 |
+
def _clean_options(raw):
|
| 116 |
+
"""Choices for a `select`: strings, trimmed, de-duplicated case-insensitively, capped.
|
| 117 |
+
Mirrors types.ts parseOptions β a choice list that means one thing in the picker and
|
| 118 |
+
another in the filter dropdown is a column with two vocabularies."""
|
| 119 |
+
out, seen = [], set()
|
| 120 |
+
for v in list(raw or [])[:MAX_FIELD_OPTIONS * 2]:
|
| 121 |
+
if not isinstance(v, (str, int, float)) or isinstance(v, bool):
|
| 122 |
+
continue
|
| 123 |
+
s = str(v).strip()[:120]
|
| 124 |
+
if not s or s.lower() in seen:
|
| 125 |
+
continue
|
| 126 |
+
seen.add(s.lower())
|
| 127 |
+
out.append(s)
|
| 128 |
+
return out[:MAX_FIELD_OPTIONS]
|
| 129 |
+
|
| 130 |
+
|
| 131 |
+
def _clean_rating_max(raw):
|
| 132 |
+
"""A rating's star count, bounded. Anything unusable is the default, not a refusal β the
|
| 133 |
+
field still holds its 1..max integers either way."""
|
| 134 |
+
try:
|
| 135 |
+
return max(RATING_MAX_MIN, min(int(raw), RATING_MAX_MAX))
|
| 136 |
+
except (TypeError, ValueError):
|
| 137 |
+
return RATING_MAX_DEFAULT
|
| 138 |
+
|
| 139 |
+
|
| 140 |
+
#: The field types a number-style display format may apply to. `formula` is here because its
|
| 141 |
+
#: RESULT is a number the client renders; `pct` already renders in points and takes decimals.
|
| 142 |
+
_NUMBER_FORMAT_TYPES = {"int", "currency", "pct", "formula"}
|
| 143 |
+
|
| 144 |
+
|
| 145 |
+
def _clean_format(raw, ftype):
|
| 146 |
+
"""Per-type DISPLAY format (wave-5 item 10), fail-closed: unknown keys are DROPPED, wrong
|
| 147 |
+
types return None (the property is simply absent). Rendering-only β a format can change how
|
| 148 |
+
a value reads, never what it is, which is why this needs no parity gate of its own."""
|
| 149 |
+
if not isinstance(raw, dict):
|
| 150 |
+
return None
|
| 151 |
+
out = {}
|
| 152 |
+
if ftype in _NUMBER_FORMAT_TYPES:
|
| 153 |
+
if isinstance(raw.get("thousands"), bool):
|
| 154 |
+
out["thousands"] = raw["thousands"]
|
| 155 |
+
if raw.get("decimals") is not None:
|
| 156 |
+
try:
|
| 157 |
+
d = int(raw["decimals"])
|
| 158 |
+
except (TypeError, ValueError):
|
| 159 |
+
d = None
|
| 160 |
+
if d is not None and 0 <= d <= 4:
|
| 161 |
+
out["decimals"] = d
|
| 162 |
+
if isinstance(raw.get("abbrev"), bool):
|
| 163 |
+
out["abbrev"] = raw["abbrev"]
|
| 164 |
+
elif ftype in ("date", "created_time"):
|
| 165 |
+
if isinstance(raw.get("time"), bool):
|
| 166 |
+
out["time"] = raw["time"]
|
| 167 |
+
if raw.get("tz") in ("local", "utc"):
|
| 168 |
+
out["tz"] = raw["tz"]
|
| 169 |
+
return out or None
|
| 170 |
+
|
| 171 |
+
|
| 172 |
+
def _clean_permissions(raw):
|
| 173 |
+
"""`{edit: 'everyone' | 'creator' | 'admins'}` or None (wave-5 item 1). WHO may set it is
|
| 174 |
+
the host handler's business (creator/admin, enforced fail-closed there); this validates
|
| 175 |
+
only the shape, like every other property here."""
|
| 176 |
+
if isinstance(raw, dict) and raw.get("edit") in ("everyone", "creator", "admins"):
|
| 177 |
+
return {"edit": raw["edit"]}
|
| 178 |
+
return None
|
| 179 |
+
|
| 180 |
+
|
| 181 |
+
_FORMULA_CHARS = re.compile(r"^[\w\s{}()+\-*/.,<>=!]*$")
|
| 182 |
+
_FORMULA_REF = re.compile(r"\{([^{}]*)\}")
|
| 183 |
+
|
| 184 |
+
|
| 185 |
+
def _clean_formula(raw, valid_keys=None):
|
| 186 |
+
"""STRUCTURAL passthrough for a formula field's expression (wave-5 item 9).
|
| 187 |
+
|
| 188 |
+
Meaning is NOT checked here: the CLIENT engine owns the grammar (arithmetic over `{field}`
|
| 189 |
+
refs, ABS/ROUND/MIN/MAX/IF), and any evaluation error degrades to a BLANK cell β never a
|
| 190 |
+
wrong number. That is the `_clean_window` split one stratum up, and deliberately NOT a
|
| 191 |
+
Python mirror of the grammar: a second engine is the filter_sql-class drift risk.
|
| 192 |
+
Structure IS checked β charset, length, balanced parens, well-formed non-empty `{refs}` β
|
| 193 |
+
and at WRITE time (`valid_keys` given) every ref must name a field this table has, fail
|
| 194 |
+
closed. At READ time refs are left alone: a referenced field deleted later must blank the
|
| 195 |
+
CELLS, not vaporise the column.
|
| 196 |
+
"""
|
| 197 |
+
if not isinstance(raw, str):
|
| 198 |
+
return None
|
| 199 |
+
s = raw.strip()
|
| 200 |
+
if not s or len(s) > MAX_FORMULA_LEN or not _FORMULA_CHARS.match(s):
|
| 201 |
+
return None
|
| 202 |
+
depth = 0
|
| 203 |
+
for ch in s:
|
| 204 |
+
if ch == "(":
|
| 205 |
+
depth += 1
|
| 206 |
+
elif ch == ")":
|
| 207 |
+
depth -= 1
|
| 208 |
+
if depth < 0:
|
| 209 |
+
return None
|
| 210 |
+
if depth:
|
| 211 |
+
return None
|
| 212 |
+
refs = _FORMULA_REF.findall(s)
|
| 213 |
+
leftover = _FORMULA_REF.sub("", s)
|
| 214 |
+
if "{" in leftover or "}" in leftover: # unbalanced / nested braces
|
| 215 |
+
return None
|
| 216 |
+
if any(not r.strip() for r in refs): # a `{}` ref names nothing
|
| 217 |
+
return None
|
| 218 |
+
if valid_keys is not None and any(r not in valid_keys for r in refs):
|
| 219 |
+
return None
|
| 220 |
+
return s
|
| 221 |
+
|
| 222 |
+
|
| 223 |
+
def _field_extras(saved, ftype):
|
| 224 |
+
"""createdBy / permissions / format / scope β the validated passthrough the created strata
|
| 225 |
+
share (wave 5). `createdBy` is only ever WRITTEN host-side (the handler stamps it); here it
|
| 226 |
+
is carried so the client can gate its menus and the handler can enforce against it.
|
| 227 |
+
`scope` (wave-6 item 9): 'cohort' marks a field created as cohort-specific β the handler
|
| 228 |
+
stamps it at create (cohort page only) and preserves it like createdBy; carried here so the
|
| 229 |
+
client can label the field, filtered OUT of other pages by fields_from_workspace."""
|
| 230 |
+
out = {}
|
| 231 |
+
who = saved.get("createdBy")
|
| 232 |
+
if isinstance(who, str) and who.strip():
|
| 233 |
+
out["createdBy"] = who.strip()[:80]
|
| 234 |
+
perms = _clean_permissions(saved.get("permissions"))
|
| 235 |
+
if perms:
|
| 236 |
+
out["permissions"] = perms
|
| 237 |
+
fmt = _clean_format(saved.get("format"), ftype)
|
| 238 |
+
if fmt:
|
| 239 |
+
out["format"] = fmt
|
| 240 |
+
if saved.get("scope") == "cohort":
|
| 241 |
+
out["scope"] = "cohort"
|
| 242 |
+
return out
|
| 243 |
+
|
| 244 |
+
|
| 245 |
+
#: The DERIVED column listing the cohorts a customer is in (owner, 2026-07-27).
|
| 246 |
+
#:
|
| 247 |
+
#: NOT in `aios_grid_fields.json`, deliberately. That contract is per-TABLE and shared with the
|
| 248 |
+
#: standalone API and the dev sample; a cohort is per-USER, so the column exists exactly when the
|
| 249 |
+
#: caller has cohorts β the same condition under which the `__cohort__` FILTER field is offered.
|
| 250 |
+
#: Putting it in the canonical contract would mean an always-present column that is empty for
|
| 251 |
+
#: everyone else, plus three consumers to keep in step for a value none of them can produce.
|
| 252 |
+
COHORT_COLUMN = 'cohorts'
|
| 253 |
+
|
| 254 |
+
|
| 255 |
+
def cohort_field(label='Cohorts'):
|
| 256 |
+
"""The derived cohort column's descriptor.
|
| 257 |
+
|
| 258 |
+
`source: 'odoo'` is doing ONE job here and it is not provenance: the client keys editability
|
| 259 |
+
off `source == 'overlay'`, and `_cl_handle_grid_event` accepts cell writes only for overlay
|
| 260 |
+
keys. So 'odoo' is what makes this column READ-ONLY at both ends. `derived: True` is what
|
| 261 |
+
stops the column menu calling it a "source field" on that basis.
|
| 262 |
+
|
| 263 |
+
β `filterable: False`, and the replacement is the `Where [Cohort] [β¦]` CONDITION, not another
|
| 264 |
+
column. Text ops over a joined string would ALMOST work and disagree at the edges β `contains
|
| 265 |
+
"VIP"` also matches a cohort called "VIP club" β and a filter that is nearly right is worse
|
| 266 |
+
than one that is absent.
|
| 267 |
+
"""
|
| 268 |
+
return {
|
| 269 |
+
'key': COHORT_COLUMN, 'label': label, 'type': 'text', 'source': 'odoo',
|
| 270 |
+
'default': False, 'filterable': False, 'derived': True, 'multi': True,
|
| 271 |
+
'note': 'The cohorts this customer belongs to, newest membership last. A cohort is a '
|
| 272 |
+
'LIST, so grouping by this column puts a customer under EVERY cohort they are '
|
| 273 |
+
'in β the group counts therefore add up to more than the record count, which '
|
| 274 |
+
'stays the number of distinct customers. Read-only: a cohort changes only on '
|
| 275 |
+
'the Cohort page. To FILTER by one, use the Cohort condition in Filter.',
|
| 276 |
+
}
|
| 277 |
+
|
| 278 |
+
|
| 279 |
+
def cohort_cells(cohorts, allowed_pids=None):
|
| 280 |
+
"""`{pid: {'cohorts': 'Q3 calls, Lost'}}` from `[{id,name,pids}, ...]`.
|
| 281 |
+
|
| 282 |
+
Built per render from the caller's OWN cohorts and handed to `rows_from_pool` as `derived`,
|
| 283 |
+
never merged into the cached pool rows β those are shared across users, and stamping one
|
| 284 |
+
user's cohorts onto them would leak the membership to everybody else on the next render.
|
| 285 |
+
"""
|
| 286 |
+
cells = {}
|
| 287 |
+
for c in cohorts or []:
|
| 288 |
+
# β The COMMA is the separator the client splits on to group a customer into EVERY
|
| 289 |
+
# cohort they are in, so it cannot also occur inside a name. Cohort names are free text
|
| 290 |
+
# ("Q3 calls, west" is a name somebody will type), so a comma is replaced here rather
|
| 291 |
+
# than left to break the split silently β one group called "Q3 calls" and another called
|
| 292 |
+
# "west" would be two lists that do not exist. The cost is cosmetic and confined to the
|
| 293 |
+
# cell; the Cohort page still shows the name the user typed.
|
| 294 |
+
name = str(c.get('name') or c.get('id') or '').replace(',', ' ').strip()
|
| 295 |
+
for pid in c.get('pids') or ():
|
| 296 |
+
if allowed_pids is not None and pid not in allowed_pids:
|
| 297 |
+
continue
|
| 298 |
+
cells.setdefault(pid, []).append(name)
|
| 299 |
+
return {pid: {COHORT_COLUMN: ', '.join(names)} for pid, names in cells.items()}
|
| 300 |
+
|
| 301 |
+
|
| 302 |
+
def clean_measure_field(raw, offered):
|
| 303 |
+
"""Validate one UNTRUSTED formula-measure field (owner item 7) against the caller's OFFER.
|
| 304 |
+
|
| 305 |
+
`offered` is `{measure key: {label, type}}` from `measure_filter.measure_fields(team_id)` β
|
| 306 |
+
the same admission the condition builder uses, so a field can only name a measure this
|
| 307 |
+
caller could also filter by. Returns the canonical stored shape, or None (fail closed).
|
| 308 |
+
|
| 309 |
+
`source:'odoo'` + `derived:True` is the cohort column's read-only mechanism, reused:
|
| 310 |
+
the client keys editability off `source == 'overlay'` and the host accepts cell writes only
|
| 311 |
+
for overlay keys, so a measure column cannot be typed into at either end. `filterable:False`
|
| 312 |
+
because the REPLACEMENT is the measure CONDITION with the same measure and window β the
|
| 313 |
+
governed, gate-proved path (CG-8/CG-12) β not text ops over a derived cell.
|
| 314 |
+
"""
|
| 315 |
+
if not isinstance(raw, dict):
|
| 316 |
+
return None
|
| 317 |
+
key = str(raw.get("key") or "")
|
| 318 |
+
if not key.startswith(MEASURE_FIELD_PREFIX) or len(key) > 80:
|
| 319 |
+
return None
|
| 320 |
+
spec = raw.get("measure")
|
| 321 |
+
if not isinstance(spec, dict):
|
| 322 |
+
return None
|
| 323 |
+
m = (offered or {}).get(spec.get("key"))
|
| 324 |
+
if not m:
|
| 325 |
+
return None # not admitted for this caller -> fail closed
|
| 326 |
+
window = _clean_window(spec.get("window"))
|
| 327 |
+
if window is None:
|
| 328 |
+
return None # a measure column with no period is not a column
|
| 329 |
+
mtype = m.get("type") if m.get("type") in MEASURE_FIELD_TYPES else "currency"
|
| 330 |
+
return {
|
| 331 |
+
"key": key,
|
| 332 |
+
"label": str(raw.get("label") or m.get("label") or "Measure")[:120],
|
| 333 |
+
"type": mtype,
|
| 334 |
+
"source": "odoo",
|
| 335 |
+
"default": True,
|
| 336 |
+
"custom": True,
|
| 337 |
+
"derived": True,
|
| 338 |
+
"filterable": False,
|
| 339 |
+
"agg": "sum" if mtype in ("currency", "int") else None,
|
| 340 |
+
"note": str(raw.get("note") or "")[:2000],
|
| 341 |
+
"measure": {"key": str(spec.get("key"))[:80], "window": window},
|
| 342 |
+
}
|
| 343 |
+
|
| 344 |
+
|
| 345 |
+
def measure_fields_of(fields):
|
| 346 |
+
"""The formula-measure columns among `fields` β the ones whose values the host must compute
|
| 347 |
+
per render (see `rows_from_pool`'s `derived`)."""
|
| 348 |
+
return [f for f in fields or [] if isinstance(f.get("measure"), dict)]
|
| 349 |
+
|
| 350 |
+
|
| 351 |
+
def fields_from_workspace(workspace=None, cohorts=False, scope_key=None):
|
| 352 |
+
"""Overlay persisted notes/custom fields onto the immutable source-field contract.
|
| 353 |
+
|
| 354 |
+
`cohorts=True` appends the derived cohort column β see `cohort_field`. Appended LAST and
|
| 355 |
+
`default: False`, so it never displaces a column somebody already reads; the Fields menu is
|
| 356 |
+
where you turn it on.
|
| 357 |
+
|
| 358 |
+
Three saved strata pass through: notes on base fields, `custom_` overlay fields (editable),
|
| 359 |
+
and `measure_` formula-measure fields (read-only, host-computed β owner item 7). A measure
|
| 360 |
+
field was validated against the caller's measure OFFER when it was written
|
| 361 |
+
(`clean_measure_field`); here only its SHAPE is re-checked, because this module has zero
|
| 362 |
+
app-internal imports and cannot know the offer. A measure that has since become
|
| 363 |
+
unanswerable (a BU scope on a company-level measure) degrades to a BLANK column at value
|
| 364 |
+
time, never to an error.
|
| 365 |
+
|
| 366 |
+
`scope_key` (wave-6 item 9) names the PAGE doing the asking ('customer' / 'cohort'). A
|
| 367 |
+
saved def carrying `scope` is emitted only when it matches β so a cohort-specific field
|
| 368 |
+
never appears on the Customer table. FAIL-CLOSED: a caller that passes no scope_key sees
|
| 369 |
+
only unscoped (global) fields; base contract fields are never scoped.
|
| 370 |
+
"""
|
| 371 |
+
saved = dict((workspace or {}).get("fields") or {})
|
| 372 |
+
out = []
|
| 373 |
+
base_keys = {field["key"] for field in FIELDS}
|
| 374 |
+
for base in FIELDS:
|
| 375 |
+
meta = saved.get(base["key"]) or {}
|
| 376 |
+
field = dict(base)
|
| 377 |
+
if isinstance(meta.get("note"), str):
|
| 378 |
+
field["note"] = meta["note"][:2000]
|
| 379 |
+
# Wave-5 item 10: a saved DISPLAY format on any base field (a preset included) β how a
|
| 380 |
+
# number or date READS, per user. Rendering-only, so this is the whole acceptance.
|
| 381 |
+
fmt = _clean_format(meta.get("format"), base.get("type"))
|
| 382 |
+
if fmt:
|
| 383 |
+
field["format"] = fmt
|
| 384 |
+
# PRESET measure fields (wave-2 item 8): a preset+measure base may take a saved
|
| 385 |
+
# window/label override. Wave 6 deleted every preset member (the owner's
|
| 386 |
+
# no-buildable-presets rule) so this branch is currently MEMBERLESS β kept as the
|
| 387 |
+
# measure_ path's twin for any future preset-carrying contract, and because deleting
|
| 388 |
+
# it would silently change what a re-added preset means.
|
| 389 |
+
if base.get("preset") and isinstance(base.get("measure"), dict):
|
| 390 |
+
saved_measure = meta.get("measure") if isinstance(meta.get("measure"), dict) else {}
|
| 391 |
+
window = _clean_window(saved_measure.get("window"))
|
| 392 |
+
if window is not None:
|
| 393 |
+
field["measure"] = {"key": base["measure"]["key"], "window": window}
|
| 394 |
+
if isinstance(meta.get("label"), str) and meta["label"].strip():
|
| 395 |
+
field["label"] = meta["label"][:120]
|
| 396 |
+
out.append(field)
|
| 397 |
+
for key, field in saved.items():
|
| 398 |
+
if key in base_keys or not isinstance(field, dict):
|
| 399 |
+
continue
|
| 400 |
+
if field.get("scope") and field.get("scope") != scope_key:
|
| 401 |
+
continue # a cohort-specific field on another page (wave-6 item 9)
|
| 402 |
+
if (str(key).startswith(MEASURE_FIELD_PREFIX)
|
| 403 |
+
and isinstance(field.get("measure"), dict)):
|
| 404 |
+
window = _clean_window(field["measure"].get("window"))
|
| 405 |
+
mkey = str(field["measure"].get("key") or "")
|
| 406 |
+
if window is None or not mkey:
|
| 407 |
+
continue
|
| 408 |
+
mtype = field.get("type") if field.get("type") in MEASURE_FIELD_TYPES else "currency"
|
| 409 |
+
out.append({
|
| 410 |
+
"key": str(key)[:80],
|
| 411 |
+
"label": str(field.get("label") or "Measure")[:120],
|
| 412 |
+
"type": mtype,
|
| 413 |
+
"source": "odoo",
|
| 414 |
+
"default": bool(field.get("default", True)),
|
| 415 |
+
"custom": True,
|
| 416 |
+
"derived": True,
|
| 417 |
+
"filterable": False,
|
| 418 |
+
"agg": "sum" if mtype in ("currency", "int") else None,
|
| 419 |
+
"note": str(field.get("note") or "")[:2000],
|
| 420 |
+
"measure": {"key": mkey[:80], "window": window},
|
| 421 |
+
**_field_extras(field, mtype),
|
| 422 |
+
})
|
| 423 |
+
continue
|
| 424 |
+
if not str(key).startswith("custom_"):
|
| 425 |
+
continue
|
| 426 |
+
ftype = field.get("type")
|
| 427 |
+
if ftype not in CUSTOM_FIELD_TYPES:
|
| 428 |
+
continue
|
| 429 |
+
if ftype in READONLY_CUSTOM_TYPES:
|
| 430 |
+
# Wave-5 items 9/11: the read-only user-created pair. Emitted with the cohort
|
| 431 |
+
# column's mechanism (source 'odoo' + derived) so the client never offers an
|
| 432 |
+
# editor and the host's overlay-write guard excludes them by construction.
|
| 433 |
+
# FILTERABLE since wave 6 (owner item 6): their values live client-side
|
| 434 |
+
# (formula computes over the row, created_time renders `_created`), this
|
| 435 |
+
# table's counts are client-mode, and the windowed count path never sees
|
| 436 |
+
# these tables β so the client engine answers them soundly.
|
| 437 |
+
entry = {
|
| 438 |
+
"key": str(key)[:80],
|
| 439 |
+
"label": str(field.get("label") or "Untitled")[:120],
|
| 440 |
+
"type": ftype,
|
| 441 |
+
"source": "odoo",
|
| 442 |
+
"derived": True,
|
| 443 |
+
"filterable": True,
|
| 444 |
+
"default": bool(field.get("default", True)),
|
| 445 |
+
"custom": True,
|
| 446 |
+
"note": str(field.get("note") or "")[:2000],
|
| 447 |
+
**_field_extras(field, ftype),
|
| 448 |
+
}
|
| 449 |
+
if ftype == "formula":
|
| 450 |
+
formula = _clean_formula(field.get("formula"))
|
| 451 |
+
if formula is None:
|
| 452 |
+
continue # a formula field without a formula is nothing
|
| 453 |
+
entry["formula"] = formula
|
| 454 |
+
out.append(entry)
|
| 455 |
+
continue
|
| 456 |
+
if field.get("source") != "overlay":
|
| 457 |
+
continue
|
| 458 |
+
options = (_clean_options(field.get("options"))
|
| 459 |
+
if ftype in ("select", "multiselect") else [])
|
| 460 |
+
if ftype in ("select", "multiselect") and not options:
|
| 461 |
+
# A select with no surviving choices can never hold a value. Dropping the COLUMN
|
| 462 |
+
# would lose the user's data; degrading it to text keeps every stored value
|
| 463 |
+
# readable and lets them re-add choices.
|
| 464 |
+
ftype = "text"
|
| 465 |
+
out.append({
|
| 466 |
+
"key": str(key)[:80],
|
| 467 |
+
"label": str(field.get("label") or "Untitled")[:120],
|
| 468 |
+
"type": ftype,
|
| 469 |
+
"source": "overlay",
|
| 470 |
+
"default": bool(field.get("default", True)),
|
| 471 |
+
"custom": True,
|
| 472 |
+
"agg": field.get("agg") if field.get("agg") in {"sum"} else None,
|
| 473 |
+
"note": str(field.get("note") or "")[:2000],
|
| 474 |
+
**({"options": options} if ftype in ("select", "multiselect") else {}),
|
| 475 |
+
# comma-joined SET semantics (the Cohorts column's contract): the row belongs to
|
| 476 |
+
# every member, groups count it under each, the toolbar count stays distinct.
|
| 477 |
+
**({"multi": True} if ftype == "multiselect" else {}),
|
| 478 |
+
**({"max": _clean_rating_max(field.get("max"))} if ftype == "rating" else {}),
|
| 479 |
+
**_field_extras(field, ftype),
|
| 480 |
+
})
|
| 481 |
+
if cohorts:
|
| 482 |
+
out.append(cohort_field())
|
| 483 |
+
return out
|
| 484 |
+
|
| 485 |
+
|
| 486 |
+
def rows_from_pool(pool_rows, fields=None, overlays=None, derived=None):
|
| 487 |
+
"""Map customer_data.pool() dicts -> the API row shape the grid expects:
|
| 488 |
+
pid + each Odoo field (numeric fields rounded, text/date passed through) + the
|
| 489 |
+
persisted external overlay. Mirrors the standalone API payload contract.
|
| 490 |
+
|
| 491 |
+
`derived` is `{pid: {key: value}}` for columns the HOST computes per render rather than
|
| 492 |
+
reads off the pool row β today just the cohort column. A separate argument from `overlays`
|
| 493 |
+
on purpose: `overlays` is the PERSISTED user stratum, and putting a value there that is
|
| 494 |
+
never written back would make the dict mean two things.
|
| 495 |
+
"""
|
| 496 |
+
fields = fields or FIELDS
|
| 497 |
+
odoo_fields = [field for field in fields if field["source"] == "odoo"]
|
| 498 |
+
overlay_fields = [field for field in fields if field["source"] == "overlay"]
|
| 499 |
+
derived_keys = [field["key"] for field in fields if field.get("derived")]
|
| 500 |
+
overlays = overlays or {}
|
| 501 |
+
derived = derived or {}
|
| 502 |
+
out = []
|
| 503 |
+
for r in pool_rows:
|
| 504 |
+
pid = r.get("pid")
|
| 505 |
+
# `_created` (wave-5 item 11) rides every row like `pid` does β the datum the
|
| 506 |
+
# `created_time` field type renders, regardless of that field's own key. Not a Field:
|
| 507 |
+
# it has no column of its own until a user creates one. `lat`/`lon` (wave-7 W11) ride
|
| 508 |
+
# the same way: the Map VIEW's data, nullable, deliberately not a column.
|
| 509 |
+
row = {"pid": pid, "_created": r.get("_created") or "",
|
| 510 |
+
"lat": r.get("lat"), "lon": r.get("lon")}
|
| 511 |
+
for field in odoo_fields:
|
| 512 |
+
k = field["key"]
|
| 513 |
+
if field.get("derived"):
|
| 514 |
+
continue # not on the pool row β filled from `derived` below
|
| 515 |
+
v = r.get(k)
|
| 516 |
+
row[k] = v if field["type"] in {"text", "status", "date"} else _round(v)
|
| 517 |
+
saved = overlays.get(str(pid), {}) or {}
|
| 518 |
+
for field in overlay_fields:
|
| 519 |
+
row[field["key"]] = saved.get(field["key"], "")
|
| 520 |
+
got = derived.get(pid) or {}
|
| 521 |
+
for k in derived_keys:
|
| 522 |
+
# '' not None: a customer in no cohort has an EMPTY cohort list, and `is empty` on a
|
| 523 |
+
# text column is the question somebody will ask of it.
|
| 524 |
+
row[k] = got.get(k, "")
|
| 525 |
+
out.append(row)
|
| 526 |
+
return out
|
| 527 |
+
|
| 528 |
+
|
| 529 |
+
# --- the FILTER-TREE contract (mirrors customer-grid/types.ts) ---------------
|
| 530 |
+
# Ops the Airtable-parity condition builder can emit. isEmpty/isNotEmpty are
|
| 531 |
+
# VALUE-FREE (they legitimately carry no value and must never be dropped for it).
|
| 532 |
+
FILTER_OPS = {'contains', 'doesNotContain', 'eq', 'neq', 'isEmpty', 'isNotEmpty',
|
| 533 |
+
'gt', 'gte', 'lt', 'lte', 'between', 'within'}
|
| 534 |
+
#: The RESERVED pseudo-column of a cohort-membership leaf (owner item 5, 2026-07-26). It is not
|
| 535 |
+
#: a Field and never will be β a cohort is a hand-curated SET, so making it a column would mean
|
| 536 |
+
#: a cell per row per cohort. Mirrors customer-grid/types.ts COHORT_FIELD.
|
| 537 |
+
COHORT_FIELD = '__cohort__'
|
| 538 |
+
#: Ops a cohort leaf may carry (owner, 2026-07-27): set operators over a SET of cohorts.
|
| 539 |
+
#: Anything else is dropped. Deliberately DISJOINT from FILTER_OPS β see types.ts COHORT_OPS:
|
| 540 |
+
#: a set op reaching a column leaf would fall through the client engine's switch to "no
|
| 541 |
+
#: narrowing", so keeping the vocabularies apart makes the existing fail-closed drop do the work.
|
| 542 |
+
COHORT_OPS = {'anyOf', 'allOf', 'noneOf'}
|
| 543 |
+
#: The single-cohort ops this leaf shipped with, kept as PERMANENT aliases and REWRITTEN here:
|
| 544 |
+
#: `is part of [one]` is `is any of [that one]`, so a saved view keeps answering and upgrades the
|
| 545 |
+
#: next time it is written. Mirrors types.ts COHORT_OP_ALIASES.
|
| 546 |
+
COHORT_OP_ALIASES = {'eq': 'anyOf', 'neq': 'noneOf'}
|
| 547 |
+
#: How many cohorts one condition may name. Mirrors types.ts MAX_COHORT_IDS.
|
| 548 |
+
MAX_COHORT_IDS = 20
|
| 549 |
+
|
| 550 |
+
|
| 551 |
+
def parse_cohort_ids(value):
|
| 552 |
+
"""The cohorts a leaf names, parsed out of `value`. Mirrors types.ts `cohortIds()`.
|
| 553 |
+
|
| 554 |
+
Comma-separated in one string because `FilterRule.value` is what all four layers persist and
|
| 555 |
+
round-trip, and a one-element list is byte-identical to what the single-cohort leaf already
|
| 556 |
+
stored β so every shipped view parses with no migration. Safe because a cohort id is built
|
| 557 |
+
from `[a-z0-9_]` only (modules/cohort.new_id), so a comma cannot occur inside one.
|
| 558 |
+
"""
|
| 559 |
+
out = []
|
| 560 |
+
for raw in ('' if value is None else str(value)).split(','):
|
| 561 |
+
cid = raw.strip()[:120]
|
| 562 |
+
if not cid or cid in out:
|
| 563 |
+
continue
|
| 564 |
+
out.append(cid)
|
| 565 |
+
if len(out) >= MAX_COHORT_IDS:
|
| 566 |
+
break
|
| 567 |
+
return out
|
| 568 |
+
# Airtable allows 3 nesting levels (root conditions -> group -> group), then grays
|
| 569 |
+
# the button out. MAX_FILTER_DEPTH in types.ts must stay in lock-step with this.
|
| 570 |
+
MAX_FILTER_DEPTH = 3
|
| 571 |
+
MAX_FILTER_NODES = 100 # total nodes across the whole tree
|
| 572 |
+
MAX_FILTER_SIBLINGS = 50 # per level
|
| 573 |
+
|
| 574 |
+
|
| 575 |
+
#: Shape of a measure condition's date window. aios_grid has ZERO app-internal imports by
|
| 576 |
+
#: design, so it does NOT know the window VOCABULARY β `harness/windows.py` owns that, mirrored
|
| 577 |
+
#: in `customer-grid/windows.ts`, and a third copy here is exactly the drift those two already
|
| 578 |
+
#: need a gate to prevent. This validates SHAPE only.
|
| 579 |
+
WINDOW_MAX_N = 3650
|
| 580 |
+
|
| 581 |
+
|
| 582 |
+
def _clean_window(raw):
|
| 583 |
+
"""Structural passthrough for a measure condition's `{kind, n?, from?, to?}` window.
|
| 584 |
+
|
| 585 |
+
Meaning is NOT checked here: an unrecognised `kind` survives this function and is REFUSED by
|
| 586 |
+
`harness.measure_filter.resolve_rule`, the layer that owns the vocabulary. Splitting it this
|
| 587 |
+
way keeps the grid module reusable and keeps one definition of what "last quarter" means.
|
| 588 |
+
"""
|
| 589 |
+
if not isinstance(raw, dict):
|
| 590 |
+
return None
|
| 591 |
+
kind = raw.get('kind')
|
| 592 |
+
if not isinstance(kind, str) or not kind or len(kind) > 40:
|
| 593 |
+
return None
|
| 594 |
+
out = {'kind': kind}
|
| 595 |
+
if raw.get('n') is not None:
|
| 596 |
+
try:
|
| 597 |
+
out['n'] = max(1, min(int(raw['n']), WINDOW_MAX_N))
|
| 598 |
+
except (TypeError, ValueError):
|
| 599 |
+
return None
|
| 600 |
+
for side in ('from', 'to'):
|
| 601 |
+
if raw.get(side) not in (None, ''):
|
| 602 |
+
out[side] = str(raw[side])[:32]
|
| 603 |
+
return out
|
| 604 |
+
|
| 605 |
+
|
| 606 |
+
def _clean_rhs(raw, valid_keys):
|
| 607 |
+
"""CG-9 β validate `{kind, colId, window?}`, the "compare against another attribute" side.
|
| 608 |
+
|
| 609 |
+
SHAPE and KEY only: `colId` must be something this table has (the caller widens `valid_keys`
|
| 610 |
+
with the measure keys, exactly as it does for the left side), and a measure rhs must carry a
|
| 611 |
+
window. What the window MEANS is `harness/windows.py`'s business, same split as `_clean_window`.
|
| 612 |
+
"""
|
| 613 |
+
if not isinstance(raw, dict):
|
| 614 |
+
return None
|
| 615 |
+
kind = raw.get('kind')
|
| 616 |
+
if kind not in ('field', 'measure', 'stat'):
|
| 617 |
+
return None
|
| 618 |
+
if kind == 'stat':
|
| 619 |
+
# A STATISTIC carries no column: the population is the comparand. Shape only β which
|
| 620 |
+
# statistics exist is `harness/measure_filter.STATS`'s business, and an unrecognised one
|
| 621 |
+
# is REFUSED there rather than guessed at, exactly like an unrecognised window kind.
|
| 622 |
+
stat = raw.get('stat')
|
| 623 |
+
if not isinstance(stat, str) or not stat or len(stat) > 24:
|
| 624 |
+
return None
|
| 625 |
+
return {'kind': 'stat', 'stat': stat}
|
| 626 |
+
col = raw.get('colId')
|
| 627 |
+
if col not in valid_keys:
|
| 628 |
+
return None
|
| 629 |
+
out = {'kind': kind, 'colId': col}
|
| 630 |
+
if kind == 'measure':
|
| 631 |
+
window = _clean_window(raw.get('window'))
|
| 632 |
+
if window is None:
|
| 633 |
+
return None # a measure comparand with no period is not a question
|
| 634 |
+
out['window'] = window
|
| 635 |
+
return out
|
| 636 |
+
|
| 637 |
+
|
| 638 |
+
#: View DISPLAY MODES beside the grid (wave-6 item 10; 'map' wave-7 W11; 'dashboard' wave-8
|
| 639 |
+
#: I19). Mirrors customer-grid/types.ts DISPLAY_MODES; 'grid' is what an absent/unknown
|
| 640 |
+
#: display means, so it is never stored.
|
| 641 |
+
#: β 'dashboard' is RETAINED FOREVER (wave-9 I10, contract C2). The owner renamed the mode to
|
| 642 |
+
#: "Chart" (a Dashboard MODULE is coming and the two would collide), but this set is the
|
| 643 |
+
#: gatekeeper for a STORED value: `_clean_display` DROPS an unknown mode, so removing
|
| 644 |
+
#: 'dashboard' here would silently downgrade every already-saved chart view to grid β and live
|
| 645 |
+
#: views are sitting in mode:'dashboard' right now (wave 8's own close-out records one). The
|
| 646 |
+
#: rename is therefore a stored-value MIGRATION, not a constant rename: accept 'dashboard' on
|
| 647 |
+
#: READ forever, only ever WRITE 'chart'.
|
| 648 |
+
DISPLAY_MODES = {'grid', 'list', 'calendar', 'kanban', 'map', 'dashboard', 'chart'}
|
| 649 |
+
|
| 650 |
+
#: Old wire value -> the value we store today. Applied AFTER the membership test so an unknown
|
| 651 |
+
#: mode is still rejected rather than accidentally aliased.
|
| 652 |
+
_LEGACY_MODES = {'dashboard': 'chart'}
|
| 653 |
+
|
| 654 |
+
#: Chart kinds a chart-mode view may hold (wave-8 I19, contract C2). Mirrors the client's
|
| 655 |
+
#: union. Deliberately small: the owner asked to "start with simple charts" and expand, and a
|
| 656 |
+
#: kind the client cannot draw is worse than one that does not exist yet.
|
| 657 |
+
CHART_KINDS = {'bar', 'line', 'area', 'donut', 'kpi'}
|
| 658 |
+
CHART_AGGS = {'sum', 'avg', 'count', 'min', 'max'}
|
| 659 |
+
MAX_CHARTS = 12 #: per view β a dashboard, not an unbounded render loop
|
| 660 |
+
MAX_CHART_TITLE = 60
|
| 661 |
+
|
| 662 |
+
#: Wave-9 I11 (contract C2) β chart customisation, host-validated.
|
| 663 |
+
#:
|
| 664 |
+
#: `palette` names a colour JOB, never a colour. A browser must not be able to post a raw hex:
|
| 665 |
+
#: the four names below map to the four jobs a palette can do (identity / magnitude / polarity)
|
| 666 |
+
#: and resolve to brand ramps client-side, so a tenant restyle cannot be defeated by a stored
|
| 667 |
+
#: literal. STATUS colours (good/warning/serious/critical) are deliberately NOT selectable β
|
| 668 |
+
#: they are reserved signal, and reusing them as "series 4" is how a chart starts lying.
|
| 669 |
+
CHART_PALETTES = {'brand', 'categorical', 'sequential', 'diverging'}
|
| 670 |
+
CHART_FORMATS = {'auto', 'number', 'currency', 'percent', 'compact'}
|
| 671 |
+
MAX_AXIS_LABEL = 40
|
| 672 |
+
#: `size` is the I10 drag. Width is in GRID COLUMNS (a 12-column board), height in px.
|
| 673 |
+
CHART_W_RANGE = (1, 12)
|
| 674 |
+
CHART_H_RANGE = (120, 800)
|
| 675 |
+
|
| 676 |
+
|
| 677 |
+
def _clean_chart(raw, valid_keys):
|
| 678 |
+
"""One dashboard chart, fail-closed. Returns None if the chart cannot be drawn.
|
| 679 |
+
|
| 680 |
+
A chart's `y` is optional (absent = count of rows, which is what "how many customers per
|
| 681 |
+
state" means). `x` is NOT: a chart with no category axis has nothing to plot against, and
|
| 682 |
+
silently keeping it would put an empty card on the dashboard with no way to tell why.
|
| 683 |
+
"""
|
| 684 |
+
if not isinstance(raw, dict):
|
| 685 |
+
return None
|
| 686 |
+
kind = raw.get('kind')
|
| 687 |
+
if kind not in CHART_KINDS:
|
| 688 |
+
return None
|
| 689 |
+
x = raw.get('x')
|
| 690 |
+
if x not in valid_keys:
|
| 691 |
+
return None # dead category ref -> the chart goes, not the board
|
| 692 |
+
cid = raw.get('id')
|
| 693 |
+
if not isinstance(cid, str) or not cid.strip():
|
| 694 |
+
return None # the client owns chart ids; an unidentified card
|
| 695 |
+
# cannot be edited or removed, so it must not persist
|
| 696 |
+
out = {'id': cid.strip()[:64], 'kind': kind, 'x': x,
|
| 697 |
+
'agg': raw.get('agg') if raw.get('agg') in CHART_AGGS else 'sum'}
|
| 698 |
+
if raw.get('y') in valid_keys:
|
| 699 |
+
out['y'] = raw['y']
|
| 700 |
+
else:
|
| 701 |
+
# no measurable column -> the only honest aggregation left is "how many rows"
|
| 702 |
+
out['agg'] = 'count'
|
| 703 |
+
title = raw.get('title')
|
| 704 |
+
if isinstance(title, str) and title.strip():
|
| 705 |
+
out['title'] = title.strip()[:MAX_CHART_TITLE]
|
| 706 |
+
|
| 707 |
+
# ββ wave-9 I11 (contract C2): customisation ββββββββββββββββββββββββββββββββββββββββββββ
|
| 708 |
+
# `splitBy` is the field whose values become the SERIES. It is deliberately not called
|
| 709 |
+
# `colorBy`: that name already means two other things here (`config.colorBy` = row
|
| 710 |
+
# colouring, `display.colorField` = map pin colour) and a third sense would be unreadable.
|
| 711 |
+
if raw.get('splitBy') in valid_keys and raw['splitBy'] != x:
|
| 712 |
+
out['splitBy'] = raw['splitBy']
|
| 713 |
+
# Stacking is only a question once there are series to stack, and only for the two kinds
|
| 714 |
+
# that can express it. Anywhere else it is dropped rather than stored as a lie the client
|
| 715 |
+
# would have to re-decide.
|
| 716 |
+
if out.get('splitBy') and kind in ('bar', 'area') and raw.get('stacked') is True:
|
| 717 |
+
out['stacked'] = True
|
| 718 |
+
if raw.get('palette') in CHART_PALETTES:
|
| 719 |
+
out['palette'] = raw['palette']
|
| 720 |
+
|
| 721 |
+
axis = raw.get('axis')
|
| 722 |
+
if isinstance(axis, dict):
|
| 723 |
+
# β ONE y-scale, always. There is no second-axis key here and there must never be:
|
| 724 |
+
# two y-scales on one frame can manufacture any correlation you like by rescaling, and
|
| 725 |
+
# the honest alternatives are two charts, small multiples, or indexing to a common base.
|
| 726 |
+
# Ruled explicitly in contract C2 against the "Tableau versatility" brief.
|
| 727 |
+
clean_axis = {}
|
| 728 |
+
for side in ('x', 'y'):
|
| 729 |
+
spec = axis.get(side)
|
| 730 |
+
if not isinstance(spec, dict):
|
| 731 |
+
continue
|
| 732 |
+
one = {}
|
| 733 |
+
lab = spec.get('label')
|
| 734 |
+
if isinstance(lab, str) and lab.strip():
|
| 735 |
+
one['label'] = lab.strip()[:MAX_AXIS_LABEL]
|
| 736 |
+
if spec.get('format') in CHART_FORMATS:
|
| 737 |
+
one['format'] = spec['format']
|
| 738 |
+
if one:
|
| 739 |
+
clean_axis[side] = one
|
| 740 |
+
if clean_axis:
|
| 741 |
+
out['axis'] = clean_axis
|
| 742 |
+
|
| 743 |
+
size = raw.get('size')
|
| 744 |
+
if isinstance(size, dict):
|
| 745 |
+
one = {}
|
| 746 |
+
for key, (lo, hi) in (('w', CHART_W_RANGE), ('h', CHART_H_RANGE)):
|
| 747 |
+
try:
|
| 748 |
+
one[key] = max(lo, min(hi, int(size[key])))
|
| 749 |
+
except (KeyError, TypeError, ValueError):
|
| 750 |
+
pass # a partial size is fine: the client defaults the missing axis
|
| 751 |
+
if one:
|
| 752 |
+
out['size'] = one
|
| 753 |
+
return out
|
| 754 |
+
|
| 755 |
+
|
| 756 |
+
def _clean_display(raw, valid_keys):
|
| 757 |
+
"""Structural passthrough for a view's `config.display` (wave-6 item 10), fail-closed.
|
| 758 |
+
|
| 759 |
+
`{mode, dateField?, stackField?, titleField?, colorField?, sizeField?, charts?}` β mode
|
| 760 |
+
must be a known non-grid mode (grid is the absent default, so storing it would be a second
|
| 761 |
+
way to say nothing); every field ref must name a field this table has (a ref to a deleted
|
| 762 |
+
field is DROPPED and the client falls back to its per-mode default); unknown keys are
|
| 763 |
+
dropped. What each mode MEANS β calendar wants a date-family field, kanban a select-family
|
| 764 |
+
stack, map a single-select to colour by and a numeric to size by β is the client's
|
| 765 |
+
business: it is the only layer that renders them, and a wrong-typed ref degrades to that
|
| 766 |
+
surface's default rather than to an error (the `_clean_window` split).
|
| 767 |
+
|
| 768 |
+
Wave-8 (contract C2) adds the map encodings (`colorField` I3, `sizeField` I5) and
|
| 769 |
+
dashboard `charts` (I19). A chart whose x/y names a deleted field is dropped INDIVIDUALLY β
|
| 770 |
+
never the whole array, because losing one column should not cost the user a dashboard they
|
| 771 |
+
spent time building.
|
| 772 |
+
"""
|
| 773 |
+
if not isinstance(raw, dict):
|
| 774 |
+
return None
|
| 775 |
+
mode = raw.get('mode')
|
| 776 |
+
if mode not in DISPLAY_MODES or mode == 'grid':
|
| 777 |
+
return None
|
| 778 |
+
# Wave-9 I10 (C2): normalise the legacy wire value AFTER the membership test, so an unknown
|
| 779 |
+
# mode is still rejected rather than accidentally aliased into a real one. Every already
|
| 780 |
+
# saved 'dashboard' view reads back as 'chart' from here on; nothing writes 'dashboard'.
|
| 781 |
+
mode = _LEGACY_MODES.get(mode, mode)
|
| 782 |
+
out = {'mode': mode}
|
| 783 |
+
for ref in ('dateField', 'stackField', 'titleField', 'colorField', 'sizeField'):
|
| 784 |
+
if raw.get(ref) in valid_keys:
|
| 785 |
+
out[ref] = raw[ref]
|
| 786 |
+
charts = raw.get('charts')
|
| 787 |
+
if isinstance(charts, list):
|
| 788 |
+
clean = [c for c in (_clean_chart(x, valid_keys) for x in charts[:MAX_CHARTS]) if c]
|
| 789 |
+
# de-dupe by id: two cards sharing an id are one card as far as the client's keyed
|
| 790 |
+
# render is concerned, and the second would silently shadow the first
|
| 791 |
+
seen, uniq = set(), []
|
| 792 |
+
for c in clean:
|
| 793 |
+
if c['id'] in seen:
|
| 794 |
+
continue
|
| 795 |
+
seen.add(c['id'])
|
| 796 |
+
uniq.append(c)
|
| 797 |
+
if uniq:
|
| 798 |
+
out['charts'] = uniq
|
| 799 |
+
return out
|
| 800 |
+
|
| 801 |
+
|
| 802 |
+
#: FOLDERS over the saved views / cohorts sidebars (wave-8 I11, contract C4).
|
| 803 |
+
#:
|
| 804 |
+
#: β Folder membership is stored as a SIDE MAP (`itemFolders`), not as a `folderId` ON each
|
| 805 |
+
#: view or cohort β a deliberate amendment to C4's first wording, recorded in the split doc.
|
| 806 |
+
#: Two reasons. (1) A cohort lives in a DIFFERENT store (`customer_cohorts`, keyed by cohort
|
| 807 |
+
#: id) and adding a `folders` key beside those ids would collide with a cohort whose generated
|
| 808 |
+
#: id happened to be 'folders'. (2) Folder placement is a per-user ORGANISING act, not part of
|
| 809 |
+
#: what a view IS: keeping it out of the view config means duplicating or exporting a view does
|
| 810 |
+
#: not drag a folder reference along with it. One map, one home, both surfaces.
|
| 811 |
+
FOLDER_SURFACES = {"views", "cohorts"}
|
| 812 |
+
MAX_FOLDERS = 60
|
| 813 |
+
MAX_FOLDER_NAME = 80
|
| 814 |
+
|
| 815 |
+
#: Wave-9 I15 (contract C5) β a user-chosen folder icon, as {shape, tone}.
|
| 816 |
+
#:
|
| 817 |
+
#: Both halves are WHITELISTS, never free values: `shape` names geometry the client already
|
| 818 |
+
#: draws (one source, `iconShapes.ts`, read by both painters) and `tone` names a palette token,
|
| 819 |
+
#: not a colour β so a browser cannot post a hex and defeat a tenant restyle, and a shape the
|
| 820 |
+
#: client cannot render can never reach the store.
|
| 821 |
+
#: β MIRRORS CLIENT'S `iconShapes.ts` ENUMERATION EXACTLY (C5: CLIENT enumerates, HOST mirrors β
|
| 822 |
+
#: posted in the split doc 2026-07-29, HOST adopted it the same day, replacing a provisional
|
| 823 |
+
#: 12-shape guess of mine that contained shapes the client cannot draw). Do not extend this set
|
| 824 |
+
#: without the matching client geometry: an unknown shape falls back to the default folder mark,
|
| 825 |
+
#: which is also I14's "existing folders get the folder icon" for every pre-wave-9 folder.
|
| 826 |
+
FOLDER_ICON_SHAPES = {"folder", "star", "flag", "tag", "bookmark", "box", "circle", "square"}
|
| 827 |
+
#: Tones are the C1 pastels β FILLS ONLY, never text (the standing palette rule). 'grey' is the
|
| 828 |
+
#: default, and is CLIENT's key name: not 'neutral', which is what HOST first guessed.
|
| 829 |
+
FOLDER_ICON_TONES = {"blue", "green", "yellow", "red", "grey"}
|
| 830 |
+
FOLDER_ICON_DEFAULT_TONE = "grey"
|
| 831 |
+
|
| 832 |
+
|
| 833 |
+
#: Wave-9 I17 (contract C4) β who may EDIT a saved view.
|
| 834 |
+
#:
|
| 835 |
+
#: β READ THIS BEFORE BUILDING ON IT. Views are stored PER USER today
|
| 836 |
+
#: (`core/table_store.TableStore.workspace` reads `store.get(table_key)[username]`), so one
|
| 837 |
+
#: user's views are invisible to every other user and "collaborative" has nothing to act on
|
| 838 |
+
#: yet. This validator is therefore CORRECT-BUT-INERT plumbing: it makes the setting durable
|
| 839 |
+
#: and fail-closed now, so that when shared views land the permission does not need a data
|
| 840 |
+
#: migration and no saved view is retro-restricted. It does NOT make anything shared, and
|
| 841 |
+
#: nothing in the app currently reads it to grant or deny cross-user access.
|
| 842 |
+
#: Recorded as the C4 amendment in .claude/wiki/research/grid-wave9-split.md.
|
| 843 |
+
VIEW_EDIT_MODES = {'personal', 'collaborative', 'users'}
|
| 844 |
+
MAX_VIEW_USERS = 50
|
| 845 |
+
|
| 846 |
+
|
| 847 |
+
def clean_view_permissions(raw, default, known_users=None):
|
| 848 |
+
"""{edit, users?} β fail-closed on both halves.
|
| 849 |
+
|
| 850 |
+
`default` is supplied by the CALLER because it splits by path, and that split is a
|
| 851 |
+
permission rule rather than a formatting one: absent on a view that already exists means a
|
| 852 |
+
pre-wave-9 view and must stay 'collaborative' (retro-restricting somebody's saved view is a
|
| 853 |
+
silent takeaway), while absent on CREATE must be 'personal' (a new view must never be
|
| 854 |
+
anyone-can-edit purely by omission).
|
| 855 |
+
|
| 856 |
+
`known_users` (when given) is the real account list: an unknown name is DROPPED, and an
|
| 857 |
+
'users' grant left with nobody in it collapses to 'personal' rather than to everyone.
|
| 858 |
+
"""
|
| 859 |
+
mode = (raw or {}).get('edit') if isinstance(raw, dict) else None
|
| 860 |
+
if mode not in VIEW_EDIT_MODES:
|
| 861 |
+
mode = default if default in VIEW_EDIT_MODES else 'personal'
|
| 862 |
+
if mode != 'users':
|
| 863 |
+
return {'edit': mode}
|
| 864 |
+
names, seen = [], set()
|
| 865 |
+
for u in list((raw or {}).get('users') or [])[:MAX_VIEW_USERS]:
|
| 866 |
+
u = str(u or '').strip()
|
| 867 |
+
if not u or u.lower() in seen:
|
| 868 |
+
continue
|
| 869 |
+
if known_users is not None and u not in known_users:
|
| 870 |
+
continue # fail-closed: a name we cannot resolve grants nothing
|
| 871 |
+
seen.add(u.lower())
|
| 872 |
+
names.append(u)
|
| 873 |
+
if not names:
|
| 874 |
+
return {'edit': 'personal'} # an empty grant is NOT "everyone"
|
| 875 |
+
return {'edit': 'users', 'users': names}
|
| 876 |
+
|
| 877 |
+
|
| 878 |
+
def clean_folder_icon(raw):
|
| 879 |
+
"""{shape, tone} or None. Fail-closed on both halves, independently.
|
| 880 |
+
|
| 881 |
+
A folder with a valid shape but a junk tone keeps the shape and defaults the tone rather
|
| 882 |
+
than losing the icon entirely β losing a user's pick because one half was wrong is the kind
|
| 883 |
+
of silent data loss the folder events already avoid elsewhere.
|
| 884 |
+
"""
|
| 885 |
+
if not isinstance(raw, dict):
|
| 886 |
+
return None
|
| 887 |
+
shape = raw.get("shape")
|
| 888 |
+
if shape not in FOLDER_ICON_SHAPES:
|
| 889 |
+
return None
|
| 890 |
+
tone = raw.get("tone")
|
| 891 |
+
return {"shape": shape,
|
| 892 |
+
"tone": tone if tone in FOLDER_ICON_TONES else FOLDER_ICON_DEFAULT_TONE}
|
| 893 |
+
|
| 894 |
+
|
| 895 |
+
def clean_folders(raw):
|
| 896 |
+
"""Validate the per-surface folder lists, fail-closed. {surface: [{id, name, order}]}."""
|
| 897 |
+
out = {}
|
| 898 |
+
for surface in FOLDER_SURFACES:
|
| 899 |
+
items, seen = [], set()
|
| 900 |
+
for f in list((raw or {}).get(surface) or [])[:MAX_FOLDERS]:
|
| 901 |
+
if not isinstance(f, dict):
|
| 902 |
+
continue
|
| 903 |
+
fid = str(f.get("id") or "").strip()[:80]
|
| 904 |
+
name = str(f.get("name") or "").strip()[:MAX_FOLDER_NAME]
|
| 905 |
+
if not fid or not name or fid in seen:
|
| 906 |
+
continue # an unidentified or unnamed folder cannot be shown or edited
|
| 907 |
+
seen.add(fid)
|
| 908 |
+
try:
|
| 909 |
+
order = int(f.get("order", len(items)))
|
| 910 |
+
except (TypeError, ValueError):
|
| 911 |
+
order = len(items)
|
| 912 |
+
row = {"id": fid, "name": name, "order": order}
|
| 913 |
+
icon = clean_folder_icon(f.get("icon")) # wave-9 I15 (C5); absent = default mark
|
| 914 |
+
if icon:
|
| 915 |
+
row["icon"] = icon
|
| 916 |
+
items.append(row)
|
| 917 |
+
items.sort(key=lambda x: x["order"])
|
| 918 |
+
for i, f in enumerate(items):
|
| 919 |
+
f["order"] = i # re-index so `order` is always dense and total
|
| 920 |
+
if items:
|
| 921 |
+
out[surface] = items
|
| 922 |
+
return out
|
| 923 |
+
|
| 924 |
+
|
| 925 |
+
def clean_item_folders(raw, folders, valid_ids):
|
| 926 |
+
"""{surface: {itemId: folderId}} β dropping any placement whose ITEM or FOLDER is gone.
|
| 927 |
+
|
| 928 |
+
This is what makes a deleted folder's contents fall back to the root rather than vanish:
|
| 929 |
+
nothing stores "this item is in no folder", so an unresolvable placement simply disappears
|
| 930 |
+
and the item renders at the top level. Same for an item that was deleted elsewhere β its
|
| 931 |
+
stale placement can never resurrect it, because the sidebars render ITEMS and consult this
|
| 932 |
+
map, never the other way round.
|
| 933 |
+
"""
|
| 934 |
+
out = {}
|
| 935 |
+
for surface in FOLDER_SURFACES:
|
| 936 |
+
fids = {f["id"] for f in (folders or {}).get(surface, [])}
|
| 937 |
+
ok = {}
|
| 938 |
+
for item_id, fid in ((raw or {}).get(surface) or {}).items():
|
| 939 |
+
if not isinstance(item_id, str) or not isinstance(fid, str):
|
| 940 |
+
continue
|
| 941 |
+
if item_id in (valid_ids or {}).get(surface, ()) and fid in fids:
|
| 942 |
+
ok[item_id[:120]] = fid
|
| 943 |
+
if ok:
|
| 944 |
+
out[surface] = ok
|
| 945 |
+
return out
|
| 946 |
+
|
| 947 |
+
|
| 948 |
+
def clean_filter_tree(raw, valid_keys, depth=1, budget=None, cohort_ids=None):
|
| 949 |
+
"""Recursively validate an UNTRUSTED filter tree (conditions + nested groups).
|
| 950 |
+
|
| 951 |
+
Returns a clean tree of leaf conditions ({colId, op, value, value2}) and
|
| 952 |
+
groups ({conj, children}). Module-agnostic on purpose: any module embedding
|
| 953 |
+
the grid validates its own view state through this one function.
|
| 954 |
+
|
| 955 |
+
Fail-closed PER NODE: anything unrecognised is DROPPED rather than raised β
|
| 956 |
+
the same contract the rest of the view sanitiser follows, so one bad rule can
|
| 957 |
+
never cost a user their whole saved view. Depth, per-level width and total
|
| 958 |
+
node count are all capped: the tree is re-evaluated for every row on every
|
| 959 |
+
render, so an unbounded structure would be a persistent client-side DoS.
|
| 960 |
+
Empty groups are dropped (they carry no meaning once persisted).
|
| 961 |
+
|
| 962 |
+
`cohort_ids` is the set of cohorts the CALLER may see. A cohort leaf naming anything else is
|
| 963 |
+
DROPPED here rather than left for the engine β a deleted cohort would otherwise leave a
|
| 964 |
+
condition that can only match nothing, so `List is not [deleted]` would show an empty table
|
| 965 |
+
forever with no way to tell why. `None` means this host has no cohorts, and then every
|
| 966 |
+
cohort leaf is dropped: fail-closed, like every other unknown key.
|
| 967 |
+
"""
|
| 968 |
+
if budget is None:
|
| 969 |
+
budget = [MAX_FILTER_NODES]
|
| 970 |
+
out = []
|
| 971 |
+
for node in list(raw or [])[:MAX_FILTER_SIBLINGS]:
|
| 972 |
+
if budget[0] <= 0:
|
| 973 |
+
break
|
| 974 |
+
if not isinstance(node, dict):
|
| 975 |
+
continue
|
| 976 |
+
if isinstance(node.get('children'), list): # a condition GROUP
|
| 977 |
+
if depth >= MAX_FILTER_DEPTH:
|
| 978 |
+
continue # too deep -> drop
|
| 979 |
+
budget[0] -= 1
|
| 980 |
+
children = clean_filter_tree(node['children'], valid_keys,
|
| 981 |
+
depth + 1, budget, cohort_ids)
|
| 982 |
+
if children:
|
| 983 |
+
out.append({'conj': 'or' if node.get('conj') == 'or' else 'and',
|
| 984 |
+
'children': children})
|
| 985 |
+
continue
|
| 986 |
+
if node.get('colId') == COHORT_FIELD: # a cohort-membership leaf
|
| 987 |
+
op = COHORT_OP_ALIASES.get(node.get('op'), node.get('op'))
|
| 988 |
+
named = parse_cohort_ids(node.get('value'))
|
| 989 |
+
# ALL of them, or the leaf goes. A set that quietly lost a member asks a DIFFERENT
|
| 990 |
+
# question, and for `noneOf` a strictly wider one: `is none of [A, B]` degrading to
|
| 991 |
+
# `is none of [A]` would show every row in B under a count nobody would doubt. This
|
| 992 |
+
# is the same all-or-nothing the single-cohort leaf already had, extended to a set.
|
| 993 |
+
if op in COHORT_OPS and named and all(c in (cohort_ids or ()) for c in named):
|
| 994 |
+
budget[0] -= 1
|
| 995 |
+
out.append({'colId': COHORT_FIELD, 'op': op,
|
| 996 |
+
'value': ','.join(named), 'value2': ''})
|
| 997 |
+
continue
|
| 998 |
+
if node.get('colId') in valid_keys and node.get('op') in FILTER_OPS:
|
| 999 |
+
budget[0] -= 1
|
| 1000 |
+
# `or ''` would be wrong here: it maps every FALSY value to '', and '' is the
|
| 1001 |
+
# signal for "inactive". A numeric 0 (or 0.0, or False) is a real value the client
|
| 1002 |
+
# treats as active β `0 === ""` is false in TS β so `revenue = 0` would silently
|
| 1003 |
+
# stop filtering and show every row instead of the zero-revenue ones.
|
| 1004 |
+
val, val2 = node.get('value'), node.get('value2')
|
| 1005 |
+
leaf = {'colId': node['colId'], 'op': node['op'],
|
| 1006 |
+
'value': ('' if val is None else str(val))[:500],
|
| 1007 |
+
'value2': ('' if val2 is None else str(val2))[:500]}
|
| 1008 |
+
# CG-8. A MEASURE condition ("Sales, in the last 90 days, > 5,000") carries two
|
| 1009 |
+
# extra members: a stable client-generated `id`, which is how the server's answer
|
| 1010 |
+
# finds its way back to the condition that asked (positional matching silently
|
| 1011 |
+
# re-associates every answer the moment a user deletes a condition), and the
|
| 1012 |
+
# `window`. Emitted ONLY when the input has them β a column condition's cleaned
|
| 1013 |
+
# shape is unchanged, so every persisted view deserialises byte-identically and
|
| 1014 |
+
# `clean_filter_tree` stays idempotent (verify_filter_engine.py asserts that by
|
| 1015 |
+
# exact structural comparison).
|
| 1016 |
+
rid = node.get('id')
|
| 1017 |
+
if rid not in (None, ''):
|
| 1018 |
+
leaf['id'] = str(rid)[:64]
|
| 1019 |
+
window = _clean_window(node.get('window'))
|
| 1020 |
+
if window is not None:
|
| 1021 |
+
leaf['window'] = window
|
| 1022 |
+
# Owner items 3 + 4, carried under the SAME rule as CG-8's `id`/`window`: emitted
|
| 1023 |
+
# only when the input has them, so a plain column condition's cleaned shape is
|
| 1024 |
+
# byte-identical to what it always was and `clean_filter_tree` stays idempotent
|
| 1025 |
+
# (verify_filter_engine.py asserts that by exact structural comparison). Drop the
|
| 1026 |
+
# carry-through and the next autosave silently strips a date condition back to a
|
| 1027 |
+
# bare comparison against an empty value β i.e. back to INACTIVE.
|
| 1028 |
+
date_window = _clean_window(node.get('dateWindow'))
|
| 1029 |
+
if date_window is not None:
|
| 1030 |
+
leaf['dateWindow'] = date_window
|
| 1031 |
+
mode = node.get('dateMode')
|
| 1032 |
+
# SHAPE only. An unrecognised mode survives here and is refused by
|
| 1033 |
+
# `windows.resolve_anchor`, which returns None and makes the condition match
|
| 1034 |
+
# NOTHING β the same split as `_clean_window`, and the reason this module can stay
|
| 1035 |
+
# free of the date vocabulary it would otherwise have to keep in step.
|
| 1036 |
+
if isinstance(mode, str) and 0 < len(mode) <= 40:
|
| 1037 |
+
leaf['dateMode'] = mode
|
| 1038 |
+
rhs = _clean_rhs(node.get('rhs'), valid_keys)
|
| 1039 |
+
if rhs is not None:
|
| 1040 |
+
leaf['rhs'] = rhs
|
| 1041 |
+
out.append(leaf)
|
| 1042 |
+
return out
|
| 1043 |
+
|
| 1044 |
+
|
| 1045 |
+
def _default_view_config(fields):
|
| 1046 |
+
shown = [field["key"] for field in fields
|
| 1047 |
+
if field.get("default") is not False or field["source"] == "overlay"]
|
| 1048 |
+
hidden = [field["key"] for field in fields if field["key"] not in shown]
|
| 1049 |
+
return {
|
| 1050 |
+
# `filters` is the ROOT of the filter tree: leaf conditions and/or nested
|
| 1051 |
+
# condition groups ({conj, children}); `filterConj` joins the root level.
|
| 1052 |
+
"filters": [], "filterConj": "and",
|
| 1053 |
+
"sorts": [], "groupBy": None, "colorBy": None,
|
| 1054 |
+
"rowHeightMode": "short", "order": shown + hidden, "visible": shown,
|
| 1055 |
+
"widths": {}, "memberPids": [],
|
| 1056 |
+
}
|
| 1057 |
+
|
| 1058 |
+
|
| 1059 |
+
def views_from_defs(defs, saved_views, fields):
|
| 1060 |
+
"""Convert legacy list formulas into the shared serializable SavedView contract."""
|
| 1061 |
+
base = _default_view_config(fields)
|
| 1062 |
+
views = [{
|
| 1063 |
+
"id": "all-customers", "name": "All customers", "kind": "system",
|
| 1064 |
+
"locked": True, "config": dict(base),
|
| 1065 |
+
}]
|
| 1066 |
+
op_map = {">=": "gte", ">": "gt", "<=": "lte", "<": "lt", "=": "eq",
|
| 1067 |
+
"contains": "contains"}
|
| 1068 |
+
for name, definition in (defs or {}).items():
|
| 1069 |
+
filters = []
|
| 1070 |
+
for rule in definition.get("rules") or []:
|
| 1071 |
+
if rule.get("field") not in {field["key"] for field in fields}:
|
| 1072 |
+
continue
|
| 1073 |
+
filters.append({
|
| 1074 |
+
"colId": rule["field"],
|
| 1075 |
+
"op": op_map.get(rule.get("op"), "eq"),
|
| 1076 |
+
"value": str(rule.get("value") if rule.get("value") is not None else ""),
|
| 1077 |
+
})
|
| 1078 |
+
sort = str(definition.get("sort") or "")
|
| 1079 |
+
sorts = ([{"colId": sort.lstrip("-"),
|
| 1080 |
+
"dir": "desc" if sort.startswith("-") else "asc"}]
|
| 1081 |
+
if sort.lstrip("-") in {field["key"] for field in fields} else [])
|
| 1082 |
+
views.append({
|
| 1083 |
+
"id": "list:" + str(name),
|
| 1084 |
+
"name": str(name),
|
| 1085 |
+
"kind": "list",
|
| 1086 |
+
"note": str(definition.get("note") or ""),
|
| 1087 |
+
"config": {
|
| 1088 |
+
**base, "filters": filters, "sorts": sorts,
|
| 1089 |
+
"memberPids": [int(pid) for pid in definition.get("members") or []
|
| 1090 |
+
if isinstance(pid, int) or str(pid).isdigit()],
|
| 1091 |
+
},
|
| 1092 |
+
})
|
| 1093 |
+
index = {view["id"]: i for i, view in enumerate(views)}
|
| 1094 |
+
for view_id, saved in (saved_views or {}).items():
|
| 1095 |
+
if not isinstance(saved, dict) or not isinstance(saved.get("config"), dict):
|
| 1096 |
+
continue
|
| 1097 |
+
clean = dict(saved)
|
| 1098 |
+
clean["id"] = str(view_id)
|
| 1099 |
+
if view_id in index:
|
| 1100 |
+
views[index[view_id]] = clean
|
| 1101 |
+
else:
|
| 1102 |
+
views.append(clean)
|
| 1103 |
+
return views
|
| 1104 |
+
|
| 1105 |
+
|
| 1106 |
+
def embed_html_path():
|
| 1107 |
+
"""The first existing candidate path for the inlined single-file build, or None."""
|
| 1108 |
+
for p in _EMBED_CANDIDATES:
|
| 1109 |
+
if p.is_file():
|
| 1110 |
+
return p
|
| 1111 |
+
return None
|
| 1112 |
+
|
| 1113 |
+
|
| 1114 |
+
def scope_counts(shown, matched, total):
|
| 1115 |
+
"""The honest 'N of M' a SERVER-WINDOWED table must carry (CG-2).
|
| 1116 |
+
|
| 1117 |
+
`matched` and `total` MUST come from their own queries over the whole scope. Never pass
|
| 1118 |
+
`len(rows)` as `matched` β that is the silent [:N] this exists to prevent: the page would
|
| 1119 |
+
report the window size as though it were the result size.
|
| 1120 |
+
|
| 1121 |
+
Refuses the shapes that could only be a mistake, because a wrong count here is invisible on
|
| 1122 |
+
screen (it looks like a smaller dataset, not like an error).
|
| 1123 |
+
"""
|
| 1124 |
+
shown, matched, total = int(shown), int(matched), int(total)
|
| 1125 |
+
if matched > total:
|
| 1126 |
+
raise ValueError(f"matched ({matched}) exceeds total ({total}) β a filter cannot match "
|
| 1127 |
+
f"more rows than the scope holds")
|
| 1128 |
+
if shown > matched:
|
| 1129 |
+
raise ValueError(f"shown ({shown}) exceeds matched ({matched}) β the window cannot hold "
|
| 1130 |
+
f"more rows than the filter matched")
|
| 1131 |
+
return {"shown": shown, "matched": matched, "total": total, "windowed": True}
|
| 1132 |
+
|
| 1133 |
+
|
| 1134 |
+
def build_html(rows, fields, *, workspace=None, path=None, counts=None):
|
| 1135 |
+
"""PURE: read the inlined single-file build and inject the data handshake.
|
| 1136 |
+
|
| 1137 |
+
Returns a self-contained HTML string with
|
| 1138 |
+
<script>window.__AIOS_GRID__ = {"fields": ..., "rows": ...};</script>
|
| 1139 |
+
inserted immediately after the opening <head> tag β a CLASSIC (non-module) script, so it
|
| 1140 |
+
executes during parse BEFORE the app's deferred `type="module"` script reads the global.
|
| 1141 |
+
Raises FileNotFoundError if no build is present (callers should try/except and fall back).
|
| 1142 |
+
"""
|
| 1143 |
+
p = Path(path) if path else embed_html_path()
|
| 1144 |
+
if p is None or not Path(p).is_file():
|
| 1145 |
+
raise FileNotFoundError(
|
| 1146 |
+
"aios_grid: no embed build found. Run `npm run build:embed` in aios-web/web and "
|
| 1147 |
+
"copy dist-embed/index.html -> royalimports_os/aios_grid_embed.html "
|
| 1148 |
+
"(python build_embed.py). Looked in: "
|
| 1149 |
+
+ " ; ".join(str(c) for c in _EMBED_CANDIDATES))
|
| 1150 |
+
html = Path(p).read_text(encoding="utf-8")
|
| 1151 |
+
# ensure_ascii=False keeps names readable; default=str is a safety net for stray
|
| 1152 |
+
# date/Decimal; the </ escape prevents a value containing "</script>" from breaking out.
|
| 1153 |
+
data = {"fields": fields, "rows": rows, "workspace": workspace or {}}
|
| 1154 |
+
# Only a WINDOWED table carries counts. Omitting the key leaves the client counting the rows
|
| 1155 |
+
# it holds, which is correct when it holds all of them.
|
| 1156 |
+
if counts:
|
| 1157 |
+
data["counts"] = counts
|
| 1158 |
+
payload = json.dumps(data, ensure_ascii=False, default=str).replace("</", "<\\/")
|
| 1159 |
+
tag = "<script>window.__AIOS_GRID__ = " + payload + ";</script>"
|
| 1160 |
+
new_html, n = re.subn(r"(<head\b[^>]*>)", lambda m: m.group(1) + tag, html, count=1)
|
| 1161 |
+
if n == 0: # no <head> (shouldn't happen) β still run it first
|
| 1162 |
+
new_html = tag + html
|
| 1163 |
+
return new_html
|
| 1164 |
+
|
| 1165 |
+
|
| 1166 |
+
def component_dir():
|
| 1167 |
+
for path in _COMPONENT_CANDIDATES:
|
| 1168 |
+
if (path / "index.html").is_file():
|
| 1169 |
+
return path
|
| 1170 |
+
return None
|
| 1171 |
+
|
| 1172 |
+
|
| 1173 |
+
def render(rows, fields, *, workspace=None, height=680, key="aios_grid",
|
| 1174 |
+
user_options=None, measures=None, measure_sets=None, today=None, viewer=None):
|
| 1175 |
+
"""Render the Airtable-style grid and return its latest guarded host event.
|
| 1176 |
+
|
| 1177 |
+
The preferred custom-component path forwards `key` and supports two-way persistence.
|
| 1178 |
+
If only the legacy HTML build exists, it renders safely with no return event.
|
| 1179 |
+
|
| 1180 |
+
`user_options` are the assignable people for `user`-typed overlay columns. They come from
|
| 1181 |
+
the HOST (the tenant's real accounts) rather than from anything the grid could invent, so
|
| 1182 |
+
an assignee is always someone who can actually log in and see the row.
|
| 1183 |
+
|
| 1184 |
+
`measures` / `measure_sets` are CG-8. A measure is FILTERABLE BUT NOT DISPLAYABLE: "Sales in
|
| 1185 |
+
the last 90 days" is a question about a window, not a column of this table, and rendering it
|
| 1186 |
+
as a column would silently pick a window for the user. The HOST resolves each condition to a
|
| 1187 |
+
set of row ids (`{ruleId: [pid, ...]}`) and the client treats that set as a membership test β
|
| 1188 |
+
which is why this module can offer measure filtering while keeping ZERO app-internal imports:
|
| 1189 |
+
it never computes a measure, it only carries one.
|
| 1190 |
+
|
| 1191 |
+
`today` is the TENANT'S current date as an ISO string, and every relative date condition
|
| 1192 |
+
("is within the past month", "is before one month ago") resolves against it. It comes from
|
| 1193 |
+
the host rather than from `new Date()` in the browser for the same three reasons
|
| 1194 |
+
harness/windows.py takes it as a parameter: a viewer a day ahead of the server would resolve
|
| 1195 |
+
a different month than the server summed, a window must not shift between the count and the
|
| 1196 |
+
rows, and the tenant's day boundary is the tenant's. Omit it and those conditions match
|
| 1197 |
+
NOTHING with a visible marker β never everything.
|
| 1198 |
+
"""
|
| 1199 |
+
import streamlit.components.v1 as _c
|
| 1200 |
+
directory = component_dir()
|
| 1201 |
+
payload = {"fields": fields, "rows": rows, "workspace": workspace or {},
|
| 1202 |
+
"userOptions": [str(u)[:120] for u in (user_options or [])][:500],
|
| 1203 |
+
"measures": list(measures or []),
|
| 1204 |
+
"today": str(today) if today else None,
|
| 1205 |
+
# Wave-5 item 1: who is looking, host-computed β what gates the client's
|
| 1206 |
+
# "Edit field permissions" menu entry. The HOST-side enforcement is the wall;
|
| 1207 |
+
# this is the courtesy that keeps the menu honest.
|
| 1208 |
+
"viewer": ({"name": str(viewer.get("name") or "")[:80],
|
| 1209 |
+
"isAdmin": bool(viewer.get("isAdmin"))}
|
| 1210 |
+
if isinstance(viewer, dict) else None),
|
| 1211 |
+
"measureSets": {str(k): list(v) for k, v in (measure_sets or {}).items()}}
|
| 1212 |
+
if directory is not None:
|
| 1213 |
+
cache_key = str(directory)
|
| 1214 |
+
component = _DECLARED_COMPONENTS.get(cache_key)
|
| 1215 |
+
if component is None:
|
| 1216 |
+
component = _c.declare_component("aios_customer_grid", path=cache_key)
|
| 1217 |
+
_DECLARED_COMPONENTS[cache_key] = component
|
| 1218 |
+
return component(payload=payload, height=height, key=key, default=None)
|
| 1219 |
+
html = build_html(rows, fields, workspace=workspace)
|
| 1220 |
+
_c.html(html, height=height, scrolling=False)
|
| 1221 |
+
return None
|
royalimports_os/core/auth_google.py
ADDED
|
@@ -0,0 +1,91 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Sign in with Google β TEMPLATE ONLY (wave-7 W10, 2026-07-28). Nothing here runs in
|
| 2 |
+
production yet: every flow function raises until the wiring checklist in
|
| 3 |
+
docs/GOOGLE_LOGIN_TODO.md is completed and GOOGLE_LOGIN_ENABLED=1 is set.
|
| 4 |
+
|
| 5 |
+
DESIGN (recorded now so the full build has a contract to meet):
|
| 6 |
+
- OAuth 2.0 authorization-code flow with PKCE against Google Identity
|
| 7 |
+
(https://accounts.google.com/.well-known/openid-configuration). No SDK dependency β
|
| 8 |
+
three HTTPS calls (authorize redirect, token exchange, JWKS fetch) keep the surface
|
| 9 |
+
auditable and the requirements.txt unchanged until we commit.
|
| 10 |
+
- Google is an IDENTITY, not an ACCOUNT SOURCE. A Google sign-in maps to an EXISTING
|
| 11 |
+
core/users.py account via its `email` attribute (set_access(..., email=...)); an
|
| 12 |
+
unknown email FAILS CLOSED with "no account for this Google identity" β Google login
|
| 13 |
+
never creates users, so the per-BU/per-module grant model stays the only door.
|
| 14 |
+
- The password gate stays byte-for-byte intact beside it. APP_PASSWORD remains the
|
| 15 |
+
bootstrap master; removing password auth is a separate, owner-approved step.
|
| 16 |
+
|
| 17 |
+
ENV (placeholders β see .env.example additions in docs/GOOGLE_LOGIN_TODO.md):
|
| 18 |
+
GOOGLE_OAUTH_CLIENT_ID OAuth client id from Google Cloud console
|
| 19 |
+
GOOGLE_OAUTH_CLIENT_SECRET its secret (Space secret / .env β never committed)
|
| 20 |
+
GOOGLE_OAUTH_REDIRECT_URI e.g. https://royal-imports-cfo-os.hf.space/ (must be
|
| 21 |
+
registered VERBATIM in the console; localhost:8501 for dev)
|
| 22 |
+
GOOGLE_LOGIN_ENABLED '1' arms the flow; anything else keeps this a template
|
| 23 |
+
"""
|
| 24 |
+
from __future__ import annotations
|
| 25 |
+
|
| 26 |
+
import os
|
| 27 |
+
|
| 28 |
+
AUTH_ENDPOINT = "https://accounts.google.com/o/oauth2/v2/auth"
|
| 29 |
+
TOKEN_ENDPOINT = "https://oauth2.googleapis.com/token"
|
| 30 |
+
JWKS_URI = "https://www.googleapis.com/oauth2/v3/certs"
|
| 31 |
+
SCOPES = "openid email profile"
|
| 32 |
+
|
| 33 |
+
|
| 34 |
+
def configured() -> bool:
|
| 35 |
+
"""True when the console credentials exist β the LOGIN BUTTON renders iff this is True.
|
| 36 |
+
(Separate from enabled(): a configured-but-disabled deploy shows nothing either, so a
|
| 37 |
+
half-finished setup never paints a dead button.)"""
|
| 38 |
+
return bool(os.environ.get("GOOGLE_OAUTH_CLIENT_ID")) and enabled()
|
| 39 |
+
|
| 40 |
+
|
| 41 |
+
def enabled() -> bool:
|
| 42 |
+
return os.environ.get("GOOGLE_LOGIN_ENABLED", "") == "1"
|
| 43 |
+
|
| 44 |
+
|
| 45 |
+
def auth_url(state: str, code_challenge: str) -> str:
|
| 46 |
+
"""The Google consent-screen URL for the authorization-code + PKCE flow.
|
| 47 |
+
|
| 48 |
+
`state` must be an unguessable per-session token the callback VERIFIES (CSRF);
|
| 49 |
+
`code_challenge` is BASE64URL(SHA256(code_verifier)) with the verifier held in the
|
| 50 |
+
session. Template guard: raises until the TODO checklist lands.
|
| 51 |
+
"""
|
| 52 |
+
raise NotImplementedError(
|
| 53 |
+
"Google login is a template β complete docs/GOOGLE_LOGIN_TODO.md, then implement "
|
| 54 |
+
"auth_url() (urlencode client_id, redirect_uri, response_type=code, scope, state, "
|
| 55 |
+
"code_challenge, code_challenge_method=S256 onto AUTH_ENDPOINT)."
|
| 56 |
+
)
|
| 57 |
+
|
| 58 |
+
|
| 59 |
+
def exchange_code(code: str, code_verifier: str) -> dict:
|
| 60 |
+
"""POST the authorization code to TOKEN_ENDPOINT β {'id_token', 'access_token', ...}.
|
| 61 |
+
Template guard: raises until implemented (requests.post with client_id/secret,
|
| 62 |
+
redirect_uri, grant_type=authorization_code, code, code_verifier)."""
|
| 63 |
+
raise NotImplementedError(
|
| 64 |
+
"Google login is a template β implement exchange_code() per docs/GOOGLE_LOGIN_TODO.md."
|
| 65 |
+
)
|
| 66 |
+
|
| 67 |
+
|
| 68 |
+
def verify_id_token(id_token: str) -> dict:
|
| 69 |
+
"""Validate the JWT against Google's JWKS (signature, iss, aud=client_id, exp) and
|
| 70 |
+
return its claims ({'email', 'email_verified', 'name', ...}). MUST reject
|
| 71 |
+
email_verified=False. Template guard: raises until implemented."""
|
| 72 |
+
raise NotImplementedError(
|
| 73 |
+
"Google login is a template β implement verify_id_token() per docs/GOOGLE_LOGIN_TODO.md."
|
| 74 |
+
)
|
| 75 |
+
|
| 76 |
+
|
| 77 |
+
def account_for_claims(claims: dict):
|
| 78 |
+
"""Map verified Google claims β the core/users.py account, FAIL-CLOSED.
|
| 79 |
+
|
| 80 |
+
The one function already real: the mapping rule is the design's heart and testable
|
| 81 |
+
today. Returns the users.registry() record whose `email` equals the verified claim
|
| 82 |
+
(case-insensitive), only if the account is active; None otherwise β never a new user.
|
| 83 |
+
"""
|
| 84 |
+
email = str(claims.get("email") or "").strip().lower()
|
| 85 |
+
if not email or claims.get("email_verified") is not True:
|
| 86 |
+
return None
|
| 87 |
+
from core import users
|
| 88 |
+
for username, u in (users.registry() or {}).items():
|
| 89 |
+
if str(u.get("email") or "").strip().lower() == email and u.get("active", True):
|
| 90 |
+
return users._public(username, u)
|
| 91 |
+
return None
|
royalimports_os/core/context.py
ADDED
|
@@ -0,0 +1,59 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Request context β the single object that carries the GLOBAL filters through a page render.
|
| 2 |
+
|
| 3 |
+
Instead of threading date_from / date_to / team_id / ... through every function, a page builds
|
| 4 |
+
one `Ctx` from the sidebar controls and passes it (or its primitives) down. Modules read only
|
| 5 |
+
what they need. Add a new global filter once here and every module can opt into it.
|
| 6 |
+
|
| 7 |
+
The DBA (doing-business-as) brand filter lives here as `team_id`:
|
| 8 |
+
None -> All / consolidated (Fisch + Royal)
|
| 9 |
+
5 -> Fisch
|
| 10 |
+
6 -> Royal
|
| 11 |
+
HQ / company-level modules (cash flow, balance sheet, AR/AP, consolidated P&L) ignore brand and
|
| 12 |
+
call `ctx.consolidated()`; operational modules (sales, customers, products, margins) honour it.
|
| 13 |
+
"""
|
| 14 |
+
from dataclasses import dataclass, replace
|
| 15 |
+
import core.odoo as O
|
| 16 |
+
import core.periods as P
|
| 17 |
+
|
| 18 |
+
# label -> team_id used by the sidebar Brand (DBA) selector
|
| 19 |
+
BRANDS = {'All': None, 'Fisch': 5, 'Royal': 6}
|
| 20 |
+
|
| 21 |
+
|
| 22 |
+
@dataclass(frozen=True)
|
| 23 |
+
class Ctx:
|
| 24 |
+
date_from: str
|
| 25 |
+
date_to: str
|
| 26 |
+
period_label: str = 'Year to date'
|
| 27 |
+
team_id: int | None = None # None = All DBAs (consolidated); 5 = Fisch; 6 = Royal
|
| 28 |
+
doc_mode: str = 'order' # reserved: 'order' vs 'invoice' recognition basis
|
| 29 |
+
|
| 30 |
+
@property
|
| 31 |
+
def brand(self) -> str:
|
| 32 |
+
return O.TEAM_NAMES.get(self.team_id, 'All') if self.team_id else 'All'
|
| 33 |
+
|
| 34 |
+
@property
|
| 35 |
+
def teams(self) -> list:
|
| 36 |
+
"""Team ids in scope (single brand, or both when consolidated)."""
|
| 37 |
+
return [self.team_id] if self.team_id else list(O.TEAM_IDS)
|
| 38 |
+
|
| 39 |
+
@property
|
| 40 |
+
def window(self) -> tuple:
|
| 41 |
+
return self.date_from, self.date_to
|
| 42 |
+
|
| 43 |
+
def consolidated(self) -> 'Ctx':
|
| 44 |
+
"""All-DBA copy β HQ modules use this so the brand filter never scopes company numbers."""
|
| 45 |
+
return replace(self, team_id=None)
|
| 46 |
+
|
| 47 |
+
def with_period(self, date_from, date_to, label=None) -> 'Ctx':
|
| 48 |
+
return replace(self, date_from=date_from, date_to=date_to,
|
| 49 |
+
period_label=label or self.period_label)
|
| 50 |
+
|
| 51 |
+
|
| 52 |
+
def from_ui(period_label, period_window, brand_label='All', doc_mode='order') -> Ctx:
|
| 53 |
+
df, dt = period_window
|
| 54 |
+
return Ctx(df, dt, period_label, BRANDS.get(brand_label), doc_mode)
|
| 55 |
+
|
| 56 |
+
|
| 57 |
+
def default(t=None) -> Ctx:
|
| 58 |
+
df, dt = P.ytd(t)
|
| 59 |
+
return Ctx(df, dt, 'Year to date', None)
|
royalimports_os/core/grid_events.py
ADDED
|
@@ -0,0 +1,1070 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""core/grid_events.py β THE WRITE SEAM, lifted out of Streamlit (EXIT-1a, X1, 2026-07-30).
|
| 2 |
+
|
| 3 |
+
This is the grid's event handler: the code that takes an UNTRUSTED event object from the
|
| 4 |
+
browser component and decides what may be persisted. It was `app.py:_cl_handle_grid_event` /
|
| 5 |
+
`_cl_handle_one` and moved here as a PURE MOVE β no behavior change inside a move (MA principle
|
| 6 |
+
5). Every body below is the app.py body verbatim; the ONLY mechanical rewrites are the
|
| 7 |
+
Streamlit couplings the contract names:
|
| 8 |
+
|
| 9 |
+
st.session_state['_cl_grid_seen_ids'] -> ctx.seen_ids (adapter-owned dedup state)
|
| 10 |
+
st.session_state['_cl_doc_payload'] -> ctx.out.doc (EventResult.doc)
|
| 11 |
+
st.session_state['_cohort_toast'] -> ctx.out.toast (EventResult.toast)
|
| 12 |
+
st.session_state['cl_table_workspace'] -> ctx.fallback_ws (store-unavailable fallback)
|
| 13 |
+
is_admin() (the bare call inside doc_delete, and _cl_table_workspace's admin=None default)
|
| 14 |
+
-> ctx.admin
|
| 15 |
+
|
| 16 |
+
There is NO `import streamlit` in this file and there must never be one β `verify_seam.py`
|
| 17 |
+
enforces that with a `sys.meta_path` block, which is the only proof that survives refactoring.
|
| 18 |
+
|
| 19 |
+
TWO ADAPTERS, ONE HANDLER:
|
| 20 |
+
* the Streamlit adapter β app.py keeps `_cl_handle_grid_event(event, uname, allowed_pids, β¦)`
|
| 21 |
+
with its old name and signature, builds an EventCtx whose `seen_ids` and `fallback_ws` are
|
| 22 |
+
session-state-backed dicts, and applies the Result to session state.
|
| 23 |
+
* the HTTP adapter β `aios-web/api` builds an EventCtx with `fallback_ws=None`. A None
|
| 24 |
+
fallback with the store unavailable is a **503**, never a silent in-memory workspace: an
|
| 25 |
+
API request has no durable session dict to fall back to, so pretending to persist would
|
| 26 |
+
lose the write with a 200 on it.
|
| 27 |
+
|
| 28 |
+
β LAYERING, stated honestly. `core/` is not supposed to import `modules/` (ARCHITECTURE Β§1's
|
| 29 |
+
one-directional rule). This handler genuinely straddles: it is the WRITE half of the customer
|
| 30 |
+
table and its persistence verbs live in `modules.customer_data` / `modules.cohort`. The
|
| 31 |
+
contracted path for the strangler is `core/grid_events.py` (both adapters import it from here),
|
| 32 |
+
so the import stays and the violation is RECORDED rather than hidden. It is the one file in
|
| 33 |
+
`core/` that points up; when EXIT-5 ports the surfaces, the persistence verbs are what move
|
| 34 |
+
down, and this note is the marker for that work.
|
| 35 |
+
"""
|
| 36 |
+
import datetime as dt
|
| 37 |
+
from dataclasses import dataclass, field
|
| 38 |
+
from typing import MutableMapping, Optional
|
| 39 |
+
|
| 40 |
+
import core.store as store
|
| 41 |
+
import core.table_store as _tstore # wave-9 I17: the shared-view authorisation predicates
|
| 42 |
+
import core.users as users
|
| 43 |
+
import harness.telemetry as _tel # _tel.error() = THE error sink (all caught errors log)
|
| 44 |
+
import modules.cohort as cohort_mod
|
| 45 |
+
import modules.customer_data as cl_mod
|
| 46 |
+
|
| 47 |
+
#: C5 (owner item 12): customer documents. The cap is enforced HOST-side on the DECODED byte
|
| 48 |
+
#: count β the client checks the same number first, but a client check only spares the user a
|
| 49 |
+
#: pointless upload; this is what bounds what the tenant's dataset can be made to hold.
|
| 50 |
+
#: MOVED from app.py:1410β1412 with the handler (app.py re-imports these during the strangler).
|
| 51 |
+
_DOCS_KEY = 'customer_docs'
|
| 52 |
+
_DOC_MAX_BYTES = 5 * 1024 * 1024
|
| 53 |
+
_DOC_MAX_PER_ROW = 50
|
| 54 |
+
|
| 55 |
+
#: The props of a field def the CLIENT actually draws β the wave-6 no-blip comparison set.
|
| 56 |
+
#: Host-only stamps (createdBy, permissions) are deliberately absent: the client renders
|
| 57 |
+
#: nothing from its own copy of those, and a permissions refusal is answered explicitly.
|
| 58 |
+
_FIELD_ECHO_PROPS = ('label', 'type', 'note', 'options', 'max', 'formula', 'agg',
|
| 59 |
+
'scope', 'format', 'measure')
|
| 60 |
+
|
| 61 |
+
|
| 62 |
+
class StoreUnavailable(RuntimeError):
|
| 63 |
+
"""The store is down AND the caller offered no durable fallback (HTTP adapter β 503).
|
| 64 |
+
|
| 65 |
+
The Streamlit adapter never sees this: it passes a session-state-backed `fallback_ws`, so
|
| 66 |
+
its store-down path degrades to the same captioned in-memory workspace it always had. An
|
| 67 |
+
API caller has no such dict, and the honest answer to "persist this" with nowhere to
|
| 68 |
+
persist it is an error β not a 200 over a write that evaporates.
|
| 69 |
+
"""
|
| 70 |
+
|
| 71 |
+
|
| 72 |
+
@dataclass
|
| 73 |
+
class EventResult:
|
| 74 |
+
"""What the HANDLER needs to tell its adapter β the three things it cannot do itself.
|
| 75 |
+
|
| 76 |
+
`rerender` is the old return value with its old meaning: "this page must re-render from
|
| 77 |
+
server state", NOT "handled" (see `handle_one`'s docstring β the distinction is the
|
| 78 |
+
difference between a snappy grid and a sluggish one).
|
| 79 |
+
"""
|
| 80 |
+
rerender: bool = False
|
| 81 |
+
doc: Optional[dict] = None # doc_fetch's answer β the bridge has no response channel
|
| 82 |
+
toast: Optional[str] = None # add_to_list's confirmation
|
| 83 |
+
|
| 84 |
+
|
| 85 |
+
@dataclass
|
| 86 |
+
class EventCtx:
|
| 87 |
+
"""Everything the handler is allowed to know, and nothing about how it is being served.
|
| 88 |
+
|
| 89 |
+
β `allowed_pids` is a SECURITY parameter: it is the pool this caller may touch at all, and
|
| 90 |
+
every row-scoped event (overlay_patch, doc_*, cohort membership, add_to_list, memberPids)
|
| 91 |
+
is intersected with it. `admin` and `uname` are likewise the HOST's statement of identity β
|
| 92 |
+
never anything the browser said about itself.
|
| 93 |
+
|
| 94 |
+
`seen_ids` is ADAPTER-OWNED dedup state (the event log resends its recent window on every
|
| 95 |
+
emit, so consumed ids must stay consumed): Streamlit passes a session-state dict, HTTP
|
| 96 |
+
passes the caller's per-session mapping.
|
| 97 |
+
|
| 98 |
+
`fallback_ws` is the store-unavailable workspace home, the OUTER dict keyed by username
|
| 99 |
+
(exactly what `st.session_state.setdefault('cl_table_workspace', {})` is). HTTP passes
|
| 100 |
+
None, and None + store down raises StoreUnavailable (β 503).
|
| 101 |
+
|
| 102 |
+
`out` is the sink for the two parked answers, so `handle_one` can keep returning a bare
|
| 103 |
+
bool exactly as it did in app.py while doc/toast still escape. (Additive amendment to X1's
|
| 104 |
+
field list, 2026-07-30 β defaulted; recorded in the S1 mailbox.)
|
| 105 |
+
"""
|
| 106 |
+
uname: str
|
| 107 |
+
allowed_pids: frozenset
|
| 108 |
+
fields: list
|
| 109 |
+
measure_keys: frozenset = frozenset()
|
| 110 |
+
resolved_ids: frozenset = frozenset()
|
| 111 |
+
cohort_ids: frozenset = frozenset()
|
| 112 |
+
measure_offer: tuple = ()
|
| 113 |
+
admin: bool = False
|
| 114 |
+
scope_key: str = 'customer'
|
| 115 |
+
seen_ids: Optional[MutableMapping] = None
|
| 116 |
+
fallback_ws: Optional[MutableMapping] = None
|
| 117 |
+
out: EventResult = field(default_factory=EventResult)
|
| 118 |
+
|
| 119 |
+
def __post_init__(self):
|
| 120 |
+
# A caller that forgets `seen_ids` gets per-call dedup rather than a crash β the SET is
|
| 121 |
+
# what makes re-delivery idempotent, and its absence must not be the thing that decides
|
| 122 |
+
# whether an event is processed at all.
|
| 123 |
+
if self.seen_ids is None:
|
| 124 |
+
self.seen_ids = {}
|
| 125 |
+
|
| 126 |
+
|
| 127 |
+
# ------------------------------------------------------------------ the workspace family (X1 amd.)
|
| 128 |
+
def table_workspace(ctx, allowed_pids=None):
|
| 129 |
+
"""Durable Airtable-style view/schema/overlay state, with an honest session fallback.
|
| 130 |
+
|
| 131 |
+
Wave-9 I17: the returned `views` are this user's OWN views MERGED with every view SHARED
|
| 132 |
+
with them. Own views win a same-id collision β a personal view is the user's own object and
|
| 133 |
+
must never be shadowed by somebody else's share.
|
| 134 |
+
|
| 135 |
+
β `allowed_pids` is a SECURITY parameter, not a convenience. A shared view carries
|
| 136 |
+
`memberPids`, which the writer's scope validated on the way IN β but the READER may have a
|
| 137 |
+
narrower scope. Handing a Fisch-scoped user a view whose member list was built by a
|
| 138 |
+
full-access user would leak the other BU's customers straight through the strict-isolation
|
| 139 |
+
rule. So membership is re-filtered against the READER on the way OUT. Pass it wherever the
|
| 140 |
+
result reaches a renderer or an event handler; omitting it drops memberPids entirely rather
|
| 141 |
+
than trusting the stored list (fail-closed: a smaller view, never a wider one).
|
| 142 |
+
|
| 143 |
+
MOVED from app.py:5323 (`_cl_table_workspace`). `admin` was a parameter defaulting to a live
|
| 144 |
+
`is_admin()` call; it now always comes from `ctx.admin`, which is identical at the one call
|
| 145 |
+
site because app.py computes `admin = bool(is_admin())` once per render. `allowed_pids`
|
| 146 |
+
stays an EXPLICIT argument because the folder leg deliberately omits it.
|
| 147 |
+
"""
|
| 148 |
+
uname = ctx.uname
|
| 149 |
+
if store.available():
|
| 150 |
+
ws = cl_mod.table_workspace(uname)
|
| 151 |
+
try:
|
| 152 |
+
shared = cl_mod.shared_table_views(uname, bool(ctx.admin))
|
| 153 |
+
except Exception as _se:
|
| 154 |
+
_tel.error('sharedviews:read', _se)
|
| 155 |
+
shared = {}
|
| 156 |
+
if shared:
|
| 157 |
+
merged = dict(shared)
|
| 158 |
+
merged.update(ws.get('views') or {}) # own views win the id collision
|
| 159 |
+
ws['views'] = {vid: scope_view(v, uname, allowed_pids)
|
| 160 |
+
for vid, v in merged.items()}
|
| 161 |
+
return ws
|
| 162 |
+
# Store down. Streamlit degrades to its captioned session workspace exactly as before; an
|
| 163 |
+
# API caller has no durable dict and must be told, not quietly served memory.
|
| 164 |
+
if ctx.fallback_ws is None:
|
| 165 |
+
raise StoreUnavailable('table workspace unavailable: no store and no session fallback')
|
| 166 |
+
return ctx.fallback_ws.setdefault(
|
| 167 |
+
uname, {'views': {}, 'fields': {}, 'overlays': {}})
|
| 168 |
+
|
| 169 |
+
|
| 170 |
+
def scope_view(view, uname, allowed_pids):
|
| 171 |
+
"""Re-scope a view to the READER. A no-op for a view the reader created.
|
| 172 |
+
|
| 173 |
+
Two things travel inside a shared view that are SCOPE, not presentation:
|
| 174 |
+
1. `memberPids` β a customer list. Re-filtered against the reader's permitted pool.
|
| 175 |
+
2. cohort conditions β a cohort id is a permission handle owned by ONE user. For a reader
|
| 176 |
+
who does not own it the leaf is UNANSWERABLE, and partial evaluation of an unanswerable
|
| 177 |
+
leaf WIDENS `is none of` ([[cg-cohort-set-conditions]]) β i.e. it would show MORE rows,
|
| 178 |
+
not fewer. So the whole filter tree is dropped for a foreign reader rather than
|
| 179 |
+
evaluated with a hole in it. A visibly unfiltered shared view is recoverable; a
|
| 180 |
+
silently widened one is a leak.
|
| 181 |
+
"""
|
| 182 |
+
if not isinstance(view, dict) or view.get('createdBy') == uname:
|
| 183 |
+
return view
|
| 184 |
+
out = dict(view)
|
| 185 |
+
cfg = dict(out.get('config') or {})
|
| 186 |
+
mp = cfg.get('memberPids')
|
| 187 |
+
if isinstance(mp, list):
|
| 188 |
+
cfg['memberPids'] = ([p for p in mp if p in (allowed_pids or ())]
|
| 189 |
+
if allowed_pids is not None else [])
|
| 190 |
+
if tree_has_cohort(cfg.get('filters')):
|
| 191 |
+
cfg['filters'] = []
|
| 192 |
+
cfg['_scopedOut'] = 'cohort' # the client says so rather than pretending
|
| 193 |
+
out['config'] = cfg
|
| 194 |
+
return out
|
| 195 |
+
|
| 196 |
+
|
| 197 |
+
def tree_has_cohort(node):
|
| 198 |
+
"""True when a filter tree references the Cohorts column anywhere (any depth)."""
|
| 199 |
+
import aios_grid as _agc
|
| 200 |
+
col = _agc.COHORT_COLUMN
|
| 201 |
+
if isinstance(node, list):
|
| 202 |
+
return any(tree_has_cohort(n) for n in node)
|
| 203 |
+
if not isinstance(node, dict):
|
| 204 |
+
return False
|
| 205 |
+
if node.get('colId') == col or node.get('field') == col:
|
| 206 |
+
return True
|
| 207 |
+
return tree_has_cohort(node.get('conditions') or node.get('children'))
|
| 208 |
+
|
| 209 |
+
|
| 210 |
+
def measure_leaves(nodes):
|
| 211 |
+
"""Every leaf of a filter tree that carries a date WINDOW β i.e. every measure condition.
|
| 212 |
+
|
| 213 |
+
Keyed off the window rather than the column name on purpose: a window is the one thing no
|
| 214 |
+
column condition has, so this cannot mistake `revenue_ytd > 5000` for a measure.
|
| 215 |
+
"""
|
| 216 |
+
out = []
|
| 217 |
+
for node in nodes or []:
|
| 218 |
+
if not isinstance(node, dict):
|
| 219 |
+
continue
|
| 220 |
+
if isinstance(node.get('children'), list):
|
| 221 |
+
out.extend(measure_leaves(node['children']))
|
| 222 |
+
elif node.get('window') is not None:
|
| 223 |
+
out.append(node)
|
| 224 |
+
return out
|
| 225 |
+
|
| 226 |
+
|
| 227 |
+
# ------------------------------------------------------------------ private handler helpers
|
| 228 |
+
def _field_echo_equal(accepted, raw):
|
| 229 |
+
"""Does the ACCEPTED def match what the client already renders? (wave-6 item 3)
|
| 230 |
+
|
| 231 |
+
The no-blip law: a field_upsert whose acceptance changed NOTHING the client draws needs no
|
| 232 |
+
re-render β the client is the ORIGIN of the change, its optimistic copy is already right,
|
| 233 |
+
and the rerun it would buy costs a full payload re-serialisation plus an iframe remount
|
| 234 |
+
(the "blip" the owner named). STRICT per-prop equality after collapsing the empty forms
|
| 235 |
+
('' / [] / {} / absent all render identically): any normalisation a validator applied is a
|
| 236 |
+
real difference the client must be SHOWN. Fail-closed β unsure means not equal means rerun.
|
| 237 |
+
"""
|
| 238 |
+
empty = ('', [], {})
|
| 239 |
+
for k in _FIELD_ECHO_PROPS:
|
| 240 |
+
a, b = accepted.get(k), raw.get(k) if isinstance(raw, dict) else None
|
| 241 |
+
if k == 'scope':
|
| 242 |
+
# 'global' and absent are the SAME rendering (an unscoped field) β the event may
|
| 243 |
+
# say 'global' explicitly while the stored def says it by omission.
|
| 244 |
+
a = None if a == 'global' else a
|
| 245 |
+
b = None if b == 'global' else b
|
| 246 |
+
if (None if a in empty else a) != (None if b in empty else b):
|
| 247 |
+
return False
|
| 248 |
+
return True
|
| 249 |
+
|
| 250 |
+
|
| 251 |
+
def _known_usernames():
|
| 252 |
+
"""Real account names, for C4's fail-closed `users` grant. None = "could not resolve".
|
| 253 |
+
|
| 254 |
+
None means DO NOT FILTER, deliberately. Returning an empty set on a transient registry
|
| 255 |
+
error would silently collapse a user's stored 'specific users' grant to 'personal' on the
|
| 256 |
+
very next autosave β destroying a setting because a read blipped. The grant is inert today
|
| 257 |
+
anyway (views are per-user; nothing reads it to allow cross-user access), so the honest
|
| 258 |
+
trade is: never invent access, never destroy a stored intent either.
|
| 259 |
+
"""
|
| 260 |
+
try:
|
| 261 |
+
reg = set(users.registry() or {})
|
| 262 |
+
except Exception as _ue:
|
| 263 |
+
_tel.error('viewperms:users', _ue)
|
| 264 |
+
return None
|
| 265 |
+
# β EMPTY IS "UNRESOLVED", NOT "NOBODY EXISTS". registry() returns {} rather than raising
|
| 266 |
+
# when the user store cannot be read (a store blip logs `store:get:users` and hands back an
|
| 267 |
+
# empty dict) β and a working app ALWAYS has at least the bootstrap admin. Treating {} as a
|
| 268 |
+
# real answer would filter every name out of a stored grant and collapse it to 'personal'
|
| 269 |
+
# on the next autosave. Caught by testing this in BARE mode with no store token.
|
| 270 |
+
return reg or None
|
| 271 |
+
|
| 272 |
+
|
| 273 |
+
# ------------------------------------------------------------------ THE SEAM
|
| 274 |
+
def handle_events(events, ctx):
|
| 275 |
+
"""The event LOG entry point β one EventResult. The `[-24:]` window lives here.
|
| 276 |
+
|
| 277 |
+
β THE VALUE IS AN EVENT LOG `{'events': [...]}`, NOT ONE EVENT (2026-07-27). A component
|
| 278 |
+
has exactly one value slot, and two quick emits share it: creating a field emits
|
| 279 |
+
`field_upsert` and the insertColumn autosave emits `view_upsert` 420ms later β with the
|
| 280 |
+
server still inside the first rerun, the second write clobbered the slot before any run
|
| 281 |
+
read it, and the field creation VANISHED with no error anywhere (measured live; the same
|
| 282 |
+
race sat latent under overlay-field creation, masked by localStorage). The client now
|
| 283 |
+
sends its recent window of events; each is processed exactly once, keyed by its id. The
|
| 284 |
+
single-event shape is still accepted β it is what an older client in a stale tab sends.
|
| 285 |
+
|
| 286 |
+
Accepts everything the component value can be: the log dict, a bare list of events (what
|
| 287 |
+
the HTTP body's `events` array unwraps to), or one legacy single-event object.
|
| 288 |
+
|
| 289 |
+
`EventResult.rerender` is True when ANY processed event demands a re-render from server
|
| 290 |
+
state β the same OR-fold `_cl_handle_grid_event` did.
|
| 291 |
+
"""
|
| 292 |
+
if isinstance(events, dict) and isinstance(events.get('events'), list):
|
| 293 |
+
seq = events['events'][-24:]
|
| 294 |
+
elif isinstance(events, list):
|
| 295 |
+
seq = events[-24:]
|
| 296 |
+
else:
|
| 297 |
+
seq = [events]
|
| 298 |
+
rerun = False
|
| 299 |
+
for one in seq:
|
| 300 |
+
rerun = handle_one(one, ctx) or rerun
|
| 301 |
+
ctx.out.rerender = rerun
|
| 302 |
+
return ctx.out
|
| 303 |
+
|
| 304 |
+
|
| 305 |
+
def handle_one(event, ctx):
|
| 306 |
+
"""Validate ONE untrusted component event.
|
| 307 |
+
|
| 308 |
+
Odoo-source fields are never accepted as cell updates. View and schema objects are
|
| 309 |
+
user-scoped; row patches additionally require the pid to be in the currently permitted
|
| 310 |
+
Customer List pool.
|
| 311 |
+
|
| 312 |
+
RETURN VALUE = "this page must re-render from server state", NOT "handled".
|
| 313 |
+
The distinction is the difference between a snappy grid and a sluggish one.
|
| 314 |
+
Persisting is not by itself a reason to rerun: Streamlit has ALREADY re-run the
|
| 315 |
+
script once simply to deliver the component value, and the client is the ORIGIN
|
| 316 |
+
of the change, so its own state is correct before the server hears about it. An
|
| 317 |
+
extra st.rerun() therefore buys nothing and costs a second full page run.
|
| 318 |
+
Measured on the 1,550-row book (2026-07-25): one filter tweak produced TWO
|
| 319 |
+
page_customer_data runs ~2.8s apart, each re-serialising the whole ~576 KB row
|
| 320 |
+
payload β and the handler on the second run is a guaranteed no-op, because the
|
| 321 |
+
event-id dedupe two lines below has already consumed that id.
|
| 322 |
+
|
| 323 |
+
THE WAVE-6 NO-BLIP LAW (item 3, 2026-07-27) β True ONLY when the next render
|
| 324 |
+
genuinely differs from what the client already draws:
|
| 325 |
+
field_upsert β False when the accepted def ECHOES the client's copy
|
| 326 |
+
(_field_echo_equal); True on any validator mutation, any
|
| 327 |
+
permissions refusal, and ALWAYS when a measure window changed
|
| 328 |
+
(values must be re-resolved server-side).
|
| 329 |
+
overlay_patch β False when every requested update was accepted verbatim;
|
| 330 |
+
True the moment anything was refused or truncated.
|
| 331 |
+
field_delete β False: the client removed the column optimistically and the
|
| 332 |
+
store scrub is invisible to the next render.
|
| 333 |
+
field_duplicate β True only when host-held VALUES must flow back (custom_ cell
|
| 334 |
+
copies, measure_ recomputes); formula/created_time clones
|
| 335 |
+
compute client-side.
|
| 336 |
+
view_delete β False (the deleted view simply stops being served).
|
| 337 |
+
view_upsert β unchanged: False, except an unresolved measure condition.
|
| 338 |
+
cohort_* / add_to_list β True: the lists payload is server truth, and these are
|
| 339 |
+
not the hot path.
|
| 340 |
+
|
| 341 |
+
The id dedupe is a SET (not a last-id slot): the event log resends the whole recent
|
| 342 |
+
window on every emit, so every id must stay consumed, not just the latest one.
|
| 343 |
+
"""
|
| 344 |
+
# The ctx fields the moved body reads as plain locals β bound once here so every branch
|
| 345 |
+
# below is textually the app.py body (which is what makes this move reviewable).
|
| 346 |
+
uname = ctx.uname
|
| 347 |
+
allowed_pids = ctx.allowed_pids
|
| 348 |
+
fields = ctx.fields
|
| 349 |
+
measure_keys = ctx.measure_keys
|
| 350 |
+
resolved_ids = ctx.resolved_ids
|
| 351 |
+
cohort_ids = ctx.cohort_ids
|
| 352 |
+
measure_offer = ctx.measure_offer
|
| 353 |
+
admin = ctx.admin
|
| 354 |
+
scope_key = ctx.scope_key
|
| 355 |
+
|
| 356 |
+
if not isinstance(event, dict):
|
| 357 |
+
return False
|
| 358 |
+
event_id = str(event.get('id') or '')[:180]
|
| 359 |
+
seen = ctx.seen_ids
|
| 360 |
+
if not event_id or event_id in seen:
|
| 361 |
+
return False
|
| 362 |
+
seen[event_id] = True
|
| 363 |
+
if len(seen) > 400: # a session-lifetime set, kept bounded
|
| 364 |
+
for stale in list(seen)[:200]:
|
| 365 |
+
seen.pop(stale, None)
|
| 366 |
+
kind = event.get('type')
|
| 367 |
+
field_by_key = {f['key']: f for f in fields}
|
| 368 |
+
valid_keys = set(field_by_key)
|
| 369 |
+
overlay_keys = {f['key'] for f in fields if f.get('source') == 'overlay'}
|
| 370 |
+
ws = table_workspace(ctx, allowed_pids=allowed_pids)
|
| 371 |
+
|
| 372 |
+
def _session_ready():
|
| 373 |
+
ws.setdefault('views', {})
|
| 374 |
+
ws.setdefault('fields', {})
|
| 375 |
+
ws.setdefault('overlays', {})
|
| 376 |
+
|
| 377 |
+
if kind == 'view_upsert':
|
| 378 |
+
raw = event.get('view')
|
| 379 |
+
if not isinstance(raw, dict) or not isinstance(raw.get('config'), dict):
|
| 380 |
+
return False
|
| 381 |
+
view_id = str(raw.get('id') or '').strip()[:120]
|
| 382 |
+
name = str(raw.get('name') or '').strip()[:120]
|
| 383 |
+
if not view_id or not name:
|
| 384 |
+
return False
|
| 385 |
+
cfg = raw['config']
|
| 386 |
+
# The filter TREE (leaf conditions and/or nested condition groups) is
|
| 387 |
+
# validated by the module-agnostic grid contract helper β imported lazily
|
| 388 |
+
# here for the same reason the page does it: a missing embed build must
|
| 389 |
+
# never break page import.
|
| 390 |
+
import aios_grid as _ag
|
| 391 |
+
# A MEASURE key is valid in a FILTER and nowhere else (CG-8): it may not be ordered,
|
| 392 |
+
# shown, grouped or coloured by, because it is not a column of this table. Hence the
|
| 393 |
+
# widened key set here and the plain `valid_keys` everywhere below.
|
| 394 |
+
# `cohort_ids` is a PERMISSION list, not a convenience: a cohort rule naming anything
|
| 395 |
+
# else is dropped here. Leaving it in would persist a condition the engine can only
|
| 396 |
+
# answer with "nothing", so `Cohort is not [a list I deleted]` would show an empty table
|
| 397 |
+
# forever with nothing on screen explaining why.
|
| 398 |
+
filters = _ag.clean_filter_tree(cfg.get('filters'),
|
| 399 |
+
valid_keys | set(measure_keys),
|
| 400 |
+
cohort_ids=set(cohort_ids))
|
| 401 |
+
sorts = []
|
| 402 |
+
for rule in list(cfg.get('sorts') or [])[:20]:
|
| 403 |
+
if (isinstance(rule, dict) and rule.get('colId') in valid_keys
|
| 404 |
+
and rule.get('dir') in ('asc', 'desc')):
|
| 405 |
+
sorts.append({'colId': rule['colId'], 'dir': rule['dir']})
|
| 406 |
+
order = list(dict.fromkeys(
|
| 407 |
+
key for key in list(cfg.get('order') or []) if key in valid_keys))
|
| 408 |
+
order += [field['key'] for field in fields if field['key'] not in order]
|
| 409 |
+
visible = list(dict.fromkeys(
|
| 410 |
+
key for key in list(cfg.get('visible') or []) if key in valid_keys))
|
| 411 |
+
widths = {}
|
| 412 |
+
raw_widths = cfg.get('widths') if isinstance(cfg.get('widths'), dict) else {}
|
| 413 |
+
for key, value in raw_widths.items():
|
| 414 |
+
if key in valid_keys:
|
| 415 |
+
try:
|
| 416 |
+
widths[key] = max(60, min(600, round(float(value))))
|
| 417 |
+
except (TypeError, ValueError):
|
| 418 |
+
pass
|
| 419 |
+
config = {
|
| 420 |
+
'filters': filters,
|
| 421 |
+
# conjunction joining the ROOT-level conditions; anything but an
|
| 422 |
+
# explicit 'or' means 'and' (legacy views had an implicit AND).
|
| 423 |
+
'filterConj': 'or' if cfg.get('filterConj') == 'or' else 'and',
|
| 424 |
+
'sorts': sorts,
|
| 425 |
+
'groupBy': cfg.get('groupBy') if cfg.get('groupBy') in valid_keys else None,
|
| 426 |
+
'colorBy': cfg.get('colorBy') if cfg.get('colorBy') in valid_keys else None,
|
| 427 |
+
'rowHeightMode': (cfg.get('rowHeightMode')
|
| 428 |
+
if cfg.get('rowHeightMode') in ('short', 'medium', 'tall')
|
| 429 |
+
else 'short'),
|
| 430 |
+
'order': order,
|
| 431 |
+
'visible': visible,
|
| 432 |
+
'widths': widths,
|
| 433 |
+
'memberPids': [pid for pid in list(cfg.get('memberPids') or [])
|
| 434 |
+
if isinstance(pid, int) and pid in allowed_pids],
|
| 435 |
+
}
|
| 436 |
+
# Wave-6 item 10: a non-grid display mode rides the view config, validated
|
| 437 |
+
# structurally (mode whitelist, refs must be this table's fields).
|
| 438 |
+
display = _ag._clean_display(cfg.get('display'), valid_keys)
|
| 439 |
+
if display:
|
| 440 |
+
config['display'] = display
|
| 441 |
+
# Wave-6 item 11 (pin): how many leading columns stay frozen while scrolling
|
| 442 |
+
# sideways. Clamped 1..8 β 0 would unfreeze the identity column the whole
|
| 443 |
+
# surface pivots on; absent = the legacy single frozen column.
|
| 444 |
+
if cfg.get('frozenCount') is not None:
|
| 445 |
+
try:
|
| 446 |
+
config['frozenCount'] = max(1, min(8, int(cfg['frozenCount'])))
|
| 447 |
+
except (TypeError, ValueError):
|
| 448 |
+
pass
|
| 449 |
+
# ββ wave-9 C3 (lock) + C4 (permissions) ββββββββββββββββββββββββββββββββββββββββββββ
|
| 450 |
+
# The PRIOR stored view decides both: whether this is a create or an update (C4's
|
| 451 |
+
# default splits on exactly that) and who is allowed to change the lock (C3).
|
| 452 |
+
prior = (ws.get('views') or {}).get(view_id) or {}
|
| 453 |
+
# β THE SHARED WALL (wave-9 I17). `ws` is already visibility-filtered, so a view the
|
| 454 |
+
# caller may not SEE reads as absent β which would look like a CREATE and let anybody
|
| 455 |
+
# overwrite somebody else's shared view by guessing its id. Ask the store for the raw
|
| 456 |
+
# record and refuse before deciding anything else. Fail-closed on both branches.
|
| 457 |
+
_raw_shared = cl_mod.shared_table_view(view_id) if store.available() else None
|
| 458 |
+
if _raw_shared is not None and not prior:
|
| 459 |
+
return False # exists, not visible to this caller: refuse
|
| 460 |
+
if _raw_shared is not None and not _tstore._may_edit(_raw_shared, uname, admin):
|
| 461 |
+
return False # visible but not editable by this caller
|
| 462 |
+
is_new = not prior
|
| 463 |
+
# C4: the creator is HOST-STAMPED and never read from the browser β the same rule as
|
| 464 |
+
# the field-level createdBy. An existing view keeps whoever made it.
|
| 465 |
+
creator = prior.get('createdBy') or uname
|
| 466 |
+
# Only the creator or an admin may change the PERMISSIONS themselves. A collaborator who
|
| 467 |
+
# could rewrite them could grant themselves sole ownership of someone else's view, or
|
| 468 |
+
# quietly widen a users-scoped view to everyone β privilege escalation by edit.
|
| 469 |
+
if not is_new and not _tstore._may_administer(prior, uname, admin):
|
| 470 |
+
raw = dict(raw)
|
| 471 |
+
raw['permissions'] = prior.get('permissions')
|
| 472 |
+
raw['locked'] = prior.get('locked')
|
| 473 |
+
# β A SYSTEM VIEW IS PER-USER STATE AND MUST NEVER BE SHARED. Caught in the wave-9
|
| 474 |
+
# integration pass, before deploy: `all-customers` is every user's DEFAULT view and it
|
| 475 |
+
# carries their own column widths, visible fields and row height. C4's read-default
|
| 476 |
+
# ('collaborative' when permissions are absent, so nobody's saved view is
|
| 477 |
+
# retro-restricted) is right for a view somebody AUTHORED β but applied to the system
|
| 478 |
+
# view it routed `all-customers` into the shared bucket, and the first user to resize a
|
| 479 |
+
# column would have published their layout to everyone. Verified in the store:
|
| 480 |
+
# shared_table_views() was returning ['all-customers'].
|
| 481 |
+
# System views are pinned personal, so they stay in each user's own workspace exactly as
|
| 482 |
+
# before this wave. `kind` is host-validated two lines below; read it here from `raw` and
|
| 483 |
+
# treat the stored kind as authoritative for an existing view.
|
| 484 |
+
_kind = (prior.get('kind') if not is_new
|
| 485 |
+
else (raw.get('kind') if raw.get('kind') in ('system', 'list', 'custom')
|
| 486 |
+
else 'custom'))
|
| 487 |
+
if _kind == 'system' or view_id == 'all-customers':
|
| 488 |
+
perms = {'edit': 'personal'}
|
| 489 |
+
else:
|
| 490 |
+
perms = _ag.clean_view_permissions(
|
| 491 |
+
raw.get('permissions'),
|
| 492 |
+
default='personal' if is_new else 'collaborative',
|
| 493 |
+
known_users=_known_usernames())
|
| 494 |
+
# C3: only the creator or an admin may SET or CLEAR the lock; anyone else keeps whatever
|
| 495 |
+
# is already stored. Enforced here rather than hidden in the client. This composes with
|
| 496 |
+
# C4 and does not override it: C4 governs who may edit the CONTENTS, C3 who may freeze
|
| 497 |
+
# the display MODE.
|
| 498 |
+
# β Inert in practice TODAY β a user can only ever write their own per-user workspace,
|
| 499 |
+
# so `uname == creator` always holds. Correct plumbing for shared views, not a live gate.
|
| 500 |
+
may_lock = admin or uname == creator
|
| 501 |
+
locked = bool(raw.get('locked')) if may_lock else bool(prior.get('locked'))
|
| 502 |
+
# C3: the LOCK's actual job β a locked view's display mode is frozen, so a locked Kanban
|
| 503 |
+
# stays Kanban. This half IS live per-user and is the part the owner asked for. Filters,
|
| 504 |
+
# sorts and columns stay editable by design.
|
| 505 |
+
if locked and not is_new:
|
| 506 |
+
was = (prior.get('config') or {}).get('display')
|
| 507 |
+
now = config.get('display')
|
| 508 |
+
if (was or {}).get('mode') != (now or {}).get('mode'):
|
| 509 |
+
if was:
|
| 510 |
+
config['display'] = was
|
| 511 |
+
else:
|
| 512 |
+
config.pop('display', None)
|
| 513 |
+
view = {'id': view_id, 'name': name,
|
| 514 |
+
'kind': raw.get('kind') if raw.get('kind') in ('system', 'list', 'custom')
|
| 515 |
+
else 'custom',
|
| 516 |
+
'locked': locked,
|
| 517 |
+
'createdBy': creator,
|
| 518 |
+
'permissions': perms,
|
| 519 |
+
'note': str(raw.get('note') or '')[:2000],
|
| 520 |
+
'config': config}
|
| 521 |
+
if store.available():
|
| 522 |
+
cl_mod.save_table_view(uname, view)
|
| 523 |
+
else:
|
| 524 |
+
_session_ready()
|
| 525 |
+
ws['views'][view_id] = view
|
| 526 |
+
# PERSISTED, but deliberately NO rerun β see the return-value contract in
|
| 527 |
+
# this function's docstring. This is the hot path: every filter, sort,
|
| 528 |
+
# group, colour, column-resize and column-toggle autosaves through here.
|
| 529 |
+
#
|
| 530 |
+
# CG-8 is the ONE exception, and it is self-limiting. A measure condition is answered
|
| 531 |
+
# server-side, so if this view carries one whose answer was NOT in the payload we just
|
| 532 |
+
# shipped, the client is showing a pending marker with nothing coming. Ask for exactly
|
| 533 |
+
# one more run β and only then, so the common case (a column filter) still costs zero.
|
| 534 |
+
# If the top-of-run read of the component's own value already covered it, this is False
|
| 535 |
+
# and nothing extra happens; if that read is ever stale, this is what stops the
|
| 536 |
+
# condition hanging unresolved.
|
| 537 |
+
#
|
| 538 |
+
# β COMPLETE rules only (wave-6 item 3, found by the live roundtrip on this rev):
|
| 539 |
+
# picking a measure FIELD mints a valueless rule whose autosave used to land here as
|
| 540 |
+
# "unresolved" and trigger a rerun that could resolve NOTHING β `rule_complete` refuses
|
| 541 |
+
# half-asked questions β so the only effect was a remount that closed the builder
|
| 542 |
+
# popover mid-build (the wave-5 "gate flake" runs, now explained). A rule still being
|
| 543 |
+
# typed stays client-side PENDING by design; the run that matters fires when its VALUE
|
| 544 |
+
# arrives and the question becomes answerable.
|
| 545 |
+
from harness import measure_filter as _mfc
|
| 546 |
+
unresolved = {str(r.get('id') or '') for r in measure_leaves(filters)
|
| 547 |
+
if _mfc.rule_complete(r)} - set(resolved_ids)
|
| 548 |
+
return bool(unresolved - {''})
|
| 549 |
+
|
| 550 |
+
if kind == 'view_delete':
|
| 551 |
+
view_id = str(event.get('viewId') or '')[:120]
|
| 552 |
+
if not view_id or view_id == 'all-customers':
|
| 553 |
+
return False
|
| 554 |
+
# Wave-9 I17: deleting a SHARED view is narrower than editing it β creator or admin
|
| 555 |
+
# only. A collaborator can change what a shared view shows; they cannot destroy
|
| 556 |
+
# everybody else's copy of it, because there is only one copy.
|
| 557 |
+
if store.available():
|
| 558 |
+
_raw_shared = cl_mod.shared_table_view(view_id)
|
| 559 |
+
if _raw_shared is not None and not _tstore._may_administer(
|
| 560 |
+
_raw_shared, uname, admin):
|
| 561 |
+
return False
|
| 562 |
+
cl_mod.delete_table_view(uname, view_id)
|
| 563 |
+
else:
|
| 564 |
+
_session_ready()
|
| 565 |
+
ws['views'].pop(view_id, None)
|
| 566 |
+
# Wave-6 item 3: the client removed the view optimistically; a deleted view simply
|
| 567 |
+
# stops being served, so the next render differs in nothing the client still shows.
|
| 568 |
+
return False
|
| 569 |
+
|
| 570 |
+
if kind == 'field_upsert':
|
| 571 |
+
raw = event.get('field')
|
| 572 |
+
if not isinstance(raw, dict):
|
| 573 |
+
return False
|
| 574 |
+
key = str(raw.get('key') or '')[:80]
|
| 575 |
+
import aios_grid as _ag2
|
| 576 |
+
if key.startswith(_ag2.MEASURE_FIELD_PREFIX):
|
| 577 |
+
# A FORMULA-MEASURE column (owner item 7). Validated against THIS caller's measure
|
| 578 |
+
# offer β the same admission the condition builder uses β so a field can only name a
|
| 579 |
+
# measure this user could also filter by. Fail closed on anything else.
|
| 580 |
+
field = _ag2.clean_measure_field(raw, {m['key']: m for m in measure_offer})
|
| 581 |
+
if field is None:
|
| 582 |
+
return False
|
| 583 |
+
fmt = _ag2._clean_format(raw.get('format'), field.get('type'))
|
| 584 |
+
if fmt:
|
| 585 |
+
field['format'] = fmt
|
| 586 |
+
elif key in field_by_key and not key.startswith('custom_'):
|
| 587 |
+
base = field_by_key[key]
|
| 588 |
+
field = {**base, 'note': str(raw.get('note') or '')[:2000]}
|
| 589 |
+
# Wave-5 item 10: a DISPLAY format on any base field (presets included) β how a
|
| 590 |
+
# number/date reads, per user. Rendering-only; validated fail-closed per type.
|
| 591 |
+
fmt = _ag2._clean_format(raw.get('format'), base.get('type'))
|
| 592 |
+
if fmt:
|
| 593 |
+
field['format'] = fmt
|
| 594 |
+
else:
|
| 595 |
+
field.pop('format', None) # cleared -> back to the type's default render
|
| 596 |
+
# PRESET measure fields (wave-2 item 8): the browser may change the PERIOD (and the
|
| 597 |
+
# auto-renamed label) of revenue_ytd / revenue_ly / orders_24m β nothing else. The
|
| 598 |
+
# whitelist IS the base contract: only a field that shipped with preset+measure can
|
| 599 |
+
# take a window here, and the measure KEY is pinned to the base's (fail closed).
|
| 600 |
+
if (base.get('preset') and isinstance(base.get('measure'), dict)
|
| 601 |
+
and isinstance(raw.get('measure'), dict)):
|
| 602 |
+
window = _ag2._clean_window(raw['measure'].get('window'))
|
| 603 |
+
if window is not None:
|
| 604 |
+
field['measure'] = {'key': base['measure']['key'], 'window': window}
|
| 605 |
+
if isinstance(raw.get('label'), str) and raw['label'].strip():
|
| 606 |
+
field['label'] = str(raw['label'])[:120]
|
| 607 |
+
elif key.startswith('custom_') and raw.get('type') in _ag2.CUSTOM_FIELD_TYPES:
|
| 608 |
+
ftype = raw['type']
|
| 609 |
+
readonly = ftype in _ag2.READONLY_CUSTOM_TYPES
|
| 610 |
+
# The editable family must SAY it is overlay (the client's own contract); the
|
| 611 |
+
# read-only pair (formula / created_time) is stored under the cohort column's
|
| 612 |
+
# mechanism instead, so no cell write can ever be accepted for it.
|
| 613 |
+
if not readonly and raw.get('source') != 'overlay':
|
| 614 |
+
return False
|
| 615 |
+
field = {'key': key, 'label': str(raw.get('label') or 'Untitled')[:120],
|
| 616 |
+
'type': ftype, 'source': 'odoo' if readonly else 'overlay',
|
| 617 |
+
'default': True, 'custom': True,
|
| 618 |
+
'note': str(raw.get('note') or '')[:2000]}
|
| 619 |
+
if readonly:
|
| 620 |
+
field['derived'] = True
|
| 621 |
+
# FILTERABLE since wave 6 (owner item 6): formula/created_time values live
|
| 622 |
+
# client-side, and this table's counts are client-mode β the client engine
|
| 623 |
+
# answers conditions on them soundly. Mirrors fields_from_workspace.
|
| 624 |
+
field['filterable'] = True
|
| 625 |
+
if raw.get('agg') == 'sum' and not readonly:
|
| 626 |
+
field['agg'] = 'sum'
|
| 627 |
+
# A `select`/`multiselect` carries its own vocabulary; a `user` does NOT β its
|
| 628 |
+
# choices are the tenant's accounts, resolved fresh on every render, so persisting
|
| 629 |
+
# a snapshot here would go stale the moment somebody joins or leaves.
|
| 630 |
+
if ftype in ('select', 'multiselect'):
|
| 631 |
+
field['options'] = _ag2._clean_options(raw.get('options'))
|
| 632 |
+
if not field['options']:
|
| 633 |
+
return False # a select with no choices can never hold a value
|
| 634 |
+
if ftype == 'rating':
|
| 635 |
+
field['max'] = _ag2._clean_rating_max(raw.get('max'))
|
| 636 |
+
if ftype == 'formula':
|
| 637 |
+
# WRITE-time is where refs must exist (fail closed); read-time leaves them
|
| 638 |
+
# alone so a later-deleted ref blanks the CELLS, not the column.
|
| 639 |
+
formula = _ag2._clean_formula(raw.get('formula'), valid_keys)
|
| 640 |
+
if formula is None:
|
| 641 |
+
return False
|
| 642 |
+
field['formula'] = formula
|
| 643 |
+
fmt = _ag2._clean_format(raw.get('format'), ftype)
|
| 644 |
+
if fmt:
|
| 645 |
+
field['format'] = fmt
|
| 646 |
+
else:
|
| 647 |
+
return False
|
| 648 |
+
# Wave-5 item 1 β createdBy + permissions on the CREATED strata (custom_/measure_).
|
| 649 |
+
# `createdBy` is stamped HOST-SIDE at create and preserved on every later write: the
|
| 650 |
+
# client never supplies it, so a browser cannot claim someone else's field. A
|
| 651 |
+
# permissions CHANGE is accepted only from the creator or an admin; a legacy field
|
| 652 |
+
# with no createdBy is admin-only (fail closed, both directions). The table workspace
|
| 653 |
+
# is per-user today, so this wall becomes load-bearing when sharing arrives β but the
|
| 654 |
+
# admin path and the legacy path are enforceable (and NC'd) right now.
|
| 655 |
+
needs_values = False
|
| 656 |
+
refused_perms = False
|
| 657 |
+
if key.startswith('custom_') or key.startswith(_ag2.MEASURE_FIELD_PREFIX):
|
| 658 |
+
prior = ws.get('fields', {}).get(key)
|
| 659 |
+
prior = prior if isinstance(prior, dict) else None
|
| 660 |
+
if prior is None:
|
| 661 |
+
field['createdBy'] = uname
|
| 662 |
+
# Wave-6 item 9: scope is chosen at CREATE, on the cohort page only β
|
| 663 |
+
# 'cohort' marks the field cohort-specific; 'global' or absent stays
|
| 664 |
+
# unscoped, so an old bundle that sends no scope keeps today's behavior.
|
| 665 |
+
if scope_key == 'cohort' and raw.get('scope') == 'cohort':
|
| 666 |
+
field['scope'] = 'cohort'
|
| 667 |
+
else:
|
| 668 |
+
if isinstance(prior.get('createdBy'), str) and prior['createdBy'].strip():
|
| 669 |
+
field['createdBy'] = prior['createdBy']
|
| 670 |
+
# Scope is PRESERVED like createdBy: the client copy never moves an
|
| 671 |
+
# existing field between scopes.
|
| 672 |
+
if prior.get('scope') == 'cohort':
|
| 673 |
+
field['scope'] = 'cohort'
|
| 674 |
+
want = _ag2._clean_permissions(raw.get('permissions'))
|
| 675 |
+
have = _ag2._clean_permissions((prior or {}).get('permissions'))
|
| 676 |
+
owner = field.get('createdBy')
|
| 677 |
+
if want is not None and (admin or (owner and owner == uname)):
|
| 678 |
+
field['permissions'] = want
|
| 679 |
+
else:
|
| 680 |
+
if have is not None:
|
| 681 |
+
field['permissions'] = have
|
| 682 |
+
# Asked for a permissions change and did not get it: the client's
|
| 683 |
+
# optimistic copy is a lie that must be repainted (wave-6 item 3).
|
| 684 |
+
refused_perms = want is not None and want != have
|
| 685 |
+
# A NEW measure column, or an existing one whose window moved: the VALUES
|
| 686 |
+
# are host-computed, so the next render genuinely differs.
|
| 687 |
+
if key.startswith(_ag2.MEASURE_FIELD_PREFIX):
|
| 688 |
+
needs_values = prior is None or prior.get('measure') != field.get('measure')
|
| 689 |
+
if store.available():
|
| 690 |
+
cl_mod.save_table_field(uname, field)
|
| 691 |
+
else:
|
| 692 |
+
_session_ready()
|
| 693 |
+
if not field.get('custom') and not str(field.get('note') or '').strip():
|
| 694 |
+
ws['fields'].pop(key, None)
|
| 695 |
+
else:
|
| 696 |
+
ws['fields'][key] = field
|
| 697 |
+
if needs_values or refused_perms:
|
| 698 |
+
return True
|
| 699 |
+
# THE NO-BLIP RETURN (wave-6 item 3): an accepted def that echoes what the client
|
| 700 |
+
# already renders needs no rerun β see _field_echo_equal and the docstring's law.
|
| 701 |
+
return not _field_echo_equal(field, raw)
|
| 702 |
+
|
| 703 |
+
if kind in ('cohort_add', 'cohort_remove', 'cohort_rename', 'cohort_delete'):
|
| 704 |
+
# Cohort membership/lifecycle edits from the grid's cohort panel (wave-2 item 2 β
|
| 705 |
+
# replaces the retired Manage popover, so rename/delete keep a surface). Fail-closed:
|
| 706 |
+
# only the caller's OWN cohorts (all_for), and member pids are intersected with the
|
| 707 |
+
# caller's permitted pool β the same guard add_to_list rides.
|
| 708 |
+
if not store.available():
|
| 709 |
+
return False
|
| 710 |
+
cohort_id = str(event.get('cohortId') or '').strip()[:120]
|
| 711 |
+
if not cohort_id or cohort_id not in cohort_mod.all_for(uname):
|
| 712 |
+
return False
|
| 713 |
+
try:
|
| 714 |
+
if kind == 'cohort_rename':
|
| 715 |
+
name = str(event.get('name') or '').strip()[:120]
|
| 716 |
+
if not name:
|
| 717 |
+
return False
|
| 718 |
+
cohort_mod.rename(uname, cohort_id, name)
|
| 719 |
+
elif kind == 'cohort_delete':
|
| 720 |
+
cohort_mod.delete(uname, cohort_id)
|
| 721 |
+
else:
|
| 722 |
+
raw_pids = event.get('pids')
|
| 723 |
+
if not isinstance(raw_pids, list):
|
| 724 |
+
return False
|
| 725 |
+
pids = [p for p in raw_pids if isinstance(p, int) and p in allowed_pids]
|
| 726 |
+
if not pids:
|
| 727 |
+
return False
|
| 728 |
+
if kind == 'cohort_add':
|
| 729 |
+
cohort_mod.add_members(uname, cohort_id, pids)
|
| 730 |
+
else:
|
| 731 |
+
cohort_mod.remove_members(uname, cohort_id, pids)
|
| 732 |
+
except ValueError as e:
|
| 733 |
+
_tel.error('cohort:panel-edit', e)
|
| 734 |
+
return False
|
| 735 |
+
# True: the lists payload (names, membership, counts) must reflect this on the next
|
| 736 |
+
# render β the panel is showing server truth, not local optimism.
|
| 737 |
+
return True
|
| 738 |
+
|
| 739 |
+
if kind in ('doc_add', 'doc_delete', 'doc_fetch'):
|
| 740 |
+
# DOCUMENTS on a customer row (owner item 12, contract C5). Metadata in the store key
|
| 741 |
+
# `customer_docs`; the BYTES live at their own path in the tenant dataset, never in a
|
| 742 |
+
# JSON blob β see core.store.upload_bytes for why that distinction is load-bearing.
|
| 743 |
+
import base64 as _b64c
|
| 744 |
+
import re as _re_mod
|
| 745 |
+
if not store.available():
|
| 746 |
+
return False
|
| 747 |
+
pid = event.get('pid')
|
| 748 |
+
doc_id = str(event.get('docId') or '').strip()[:80]
|
| 749 |
+
# The pid wall is the SAME one every other row-scoped event rides: a caller may only
|
| 750 |
+
# touch customers inside their own permitted pool. Without it, documents would be the
|
| 751 |
+
# one surface where a scoped agent could read another book's rows.
|
| 752 |
+
if not isinstance(pid, int) or pid not in allowed_pids or not doc_id:
|
| 753 |
+
return False
|
| 754 |
+
meta_all = store.get(_DOCS_KEY) or {}
|
| 755 |
+
row = list((meta_all.get(str(pid)) or []))
|
| 756 |
+
|
| 757 |
+
if kind == 'doc_add':
|
| 758 |
+
raw_b64 = event.get('data_b64')
|
| 759 |
+
name = str(event.get('name') or 'document')[:200]
|
| 760 |
+
if not isinstance(raw_b64, str) or not raw_b64:
|
| 761 |
+
return False
|
| 762 |
+
try:
|
| 763 |
+
blob = _b64c.b64decode(raw_b64, validate=True)
|
| 764 |
+
except Exception:
|
| 765 |
+
return False # not base64 -> not a file we will store
|
| 766 |
+
# Re-check the DECODED size host-side. The client checks it too, but a client check
|
| 767 |
+
# is a courtesy: this is the number that bounds what the tenant's repo can be made
|
| 768 |
+
# to hold by anyone who can reach the endpoint.
|
| 769 |
+
if not blob or len(blob) > _DOC_MAX_BYTES:
|
| 770 |
+
return False
|
| 771 |
+
if len(row) >= _DOC_MAX_PER_ROW or any(d.get('id') == doc_id for d in row):
|
| 772 |
+
return False
|
| 773 |
+
safe = _re_mod.sub(r'[^A-Za-z0-9._-]', '_', name)[:80] or 'file'
|
| 774 |
+
path = f'docs/{pid}/{doc_id}_{safe}'
|
| 775 |
+
try:
|
| 776 |
+
store.upload_bytes(path, blob, message=f'doc {doc_id} for customer {pid}')
|
| 777 |
+
except Exception as _ue:
|
| 778 |
+
_tel.error('docs:upload', _ue)
|
| 779 |
+
return False
|
| 780 |
+
row.append({'id': doc_id, 'name': name,
|
| 781 |
+
'mime': str(event.get('mime') or 'application/octet-stream')[:120],
|
| 782 |
+
'size': len(blob), 'by': uname,
|
| 783 |
+
'ts': dt.datetime.now().strftime('%Y-%m-%dT%H:%M:%S'),
|
| 784 |
+
'path': path})
|
| 785 |
+
elif kind == 'doc_delete':
|
| 786 |
+
hit = next((d for d in row if d.get('id') == doc_id), None)
|
| 787 |
+
if not hit:
|
| 788 |
+
return False
|
| 789 |
+
# THE WALL: the uploader or an admin. Enforced here regardless of what the client
|
| 790 |
+
# chose to render β `canDelete` in the payload only hides a button.
|
| 791 |
+
if hit.get('by') != uname and not admin:
|
| 792 |
+
return False
|
| 793 |
+
store.delete_path(hit.get('path') or '')
|
| 794 |
+
row = [d for d in row if d.get('id') != doc_id]
|
| 795 |
+
else: # doc_fetch
|
| 796 |
+
hit = next((d for d in row if d.get('id') == doc_id), None)
|
| 797 |
+
if not hit:
|
| 798 |
+
return False
|
| 799 |
+
blob = store.download_bytes(hit.get('path') or '')
|
| 800 |
+
if blob is None:
|
| 801 |
+
return False
|
| 802 |
+
# The bridge is a fire-and-forget event LOG with no response channel, so the answer
|
| 803 |
+
# is PARKED on the Result and rides the next render's payload (C5).
|
| 804 |
+
ctx.out.doc = {
|
| 805 |
+
'pid': pid, 'docId': doc_id, 'name': hit.get('name') or 'document',
|
| 806 |
+
'mime': hit.get('mime') or 'application/octet-stream',
|
| 807 |
+
'data_b64': _b64c.b64encode(blob).decode('ascii')}
|
| 808 |
+
return True # re-render to deliver it
|
| 809 |
+
|
| 810 |
+
def _write(cur):
|
| 811 |
+
cur[str(pid)] = row
|
| 812 |
+
return cur
|
| 813 |
+
store.update(_DOCS_KEY, _write, flush='async')
|
| 814 |
+
return True
|
| 815 |
+
|
| 816 |
+
if kind in ('folder_create', 'folder_rename', 'folder_delete', 'folder_duplicate',
|
| 817 |
+
'item_move'):
|
| 818 |
+
# FOLDERS over the Views / Cohorts sidebars (owner item 11, contract C4). All five
|
| 819 |
+
# events touch ONE home β the table workspace's `folders` + `itemFolders` β so a cohort
|
| 820 |
+
# can be filed without the cohort STORE learning about folders at all.
|
| 821 |
+
import aios_grid as _agf
|
| 822 |
+
if not store.available():
|
| 823 |
+
return False
|
| 824 |
+
surface = str(event.get('surface') or '')
|
| 825 |
+
if surface not in _agf.FOLDER_SURFACES:
|
| 826 |
+
return False
|
| 827 |
+
# The ids this user may actually file: their own views, their own cohorts. Fail-closed β
|
| 828 |
+
# an item_move naming somebody else's cohort is dropped by clean_item_folders anyway,
|
| 829 |
+
# but refusing here means we never write it in the first place.
|
| 830 |
+
def _own_ids(sfc, cur_ws):
|
| 831 |
+
if sfc == 'views':
|
| 832 |
+
return set((cur_ws.get('views') or {}))
|
| 833 |
+
return set(cohort_mod.all_for(uname))
|
| 834 |
+
|
| 835 |
+
def _mutate(cur_ws):
|
| 836 |
+
folders = _agf.clean_folders(cur_ws.get('folders'))
|
| 837 |
+
placed = dict(cur_ws.get('itemFolders') or {})
|
| 838 |
+
lst = list(folders.get(surface) or [])
|
| 839 |
+
# β the wire field is `folderId` on EVERY folder event (CLIENT's types.ts is the
|
| 840 |
+
# built contract; the split doc's first draft said `id`). Converged here rather
|
| 841 |
+
# than asking the client to re-emit β see the C4 reconciliation note in the doc.
|
| 842 |
+
fid = str(event.get('folderId') or '').strip()[:80]
|
| 843 |
+
if kind == 'folder_create':
|
| 844 |
+
name = str(event.get('name') or '').strip()[:_agf.MAX_FOLDER_NAME]
|
| 845 |
+
if not fid or not name or len(lst) >= _agf.MAX_FOLDERS:
|
| 846 |
+
return None
|
| 847 |
+
if any(f['id'] == fid for f in lst):
|
| 848 |
+
return None
|
| 849 |
+
row = {'id': fid, 'name': name, 'order': len(lst)}
|
| 850 |
+
icon = _agf.clean_folder_icon(event.get('icon')) # wave-9 I15 (C5)
|
| 851 |
+
if icon:
|
| 852 |
+
row['icon'] = icon
|
| 853 |
+
lst.append(row)
|
| 854 |
+
elif kind == 'folder_rename':
|
| 855 |
+
name = str(event.get('name') or '').strip()[:_agf.MAX_FOLDER_NAME]
|
| 856 |
+
hit = next((f for f in lst if f['id'] == fid), None)
|
| 857 |
+
if not hit or not name:
|
| 858 |
+
return None
|
| 859 |
+
hit['name'] = name
|
| 860 |
+
# wave-9 I15 (C5): the rename pane is also where the icon is changed. An event
|
| 861 |
+
# that carries NO icon key leaves the existing one alone (a rename must not
|
| 862 |
+
# silently strip a chosen icon); an explicit null clears it back to default.
|
| 863 |
+
if 'icon' in event:
|
| 864 |
+
icon = _agf.clean_folder_icon(event.get('icon'))
|
| 865 |
+
if icon:
|
| 866 |
+
hit['icon'] = icon
|
| 867 |
+
else:
|
| 868 |
+
hit.pop('icon', None)
|
| 869 |
+
elif kind == 'folder_delete':
|
| 870 |
+
if not any(f['id'] == fid for f in lst):
|
| 871 |
+
return None
|
| 872 |
+
lst = [f for f in lst if f['id'] != fid]
|
| 873 |
+
# CONTENTS MOVE TO ROOT, never delete. Dropping the placements is exactly that:
|
| 874 |
+
# an item with no placement renders at the top level.
|
| 875 |
+
placed[surface] = {k: v for k, v in (placed.get(surface) or {}).items()
|
| 876 |
+
if v != fid}
|
| 877 |
+
elif kind == 'folder_duplicate':
|
| 878 |
+
new_id = str(event.get('newId') or '').strip()[:80]
|
| 879 |
+
src = next((f for f in lst if f['id'] == fid), None)
|
| 880 |
+
if not src or not new_id or len(lst) >= _agf.MAX_FOLDERS:
|
| 881 |
+
return None
|
| 882 |
+
if any(f['id'] == new_id for f in lst):
|
| 883 |
+
return None
|
| 884 |
+
lst.append({'id': new_id, 'name': f"{src['name']} copy"[:_agf.MAX_FOLDER_NAME],
|
| 885 |
+
'order': len(lst)})
|
| 886 |
+
# β The folder's CONTENTS are duplicated by the client emitting the ordinary
|
| 887 |
+
# per-item duplicate events it already owns (view_upsert with a fresh id, the
|
| 888 |
+
# cohort copy path) and then item_move-ing them here. Copying views/cohorts
|
| 889 |
+
# inside this handler would be a SECOND implementation of "duplicate an item",
|
| 890 |
+
# and the two would drift the first time either one changed.
|
| 891 |
+
else: # item_move
|
| 892 |
+
item_id = str(event.get('itemId') or '').strip()[:120]
|
| 893 |
+
target = event.get('folderId')
|
| 894 |
+
if not item_id or item_id not in _own_ids(surface, cur_ws):
|
| 895 |
+
return None
|
| 896 |
+
cur = dict(placed.get(surface) or {})
|
| 897 |
+
if target is None:
|
| 898 |
+
cur.pop(item_id, None) # back to the root
|
| 899 |
+
else:
|
| 900 |
+
tid = str(target)[:80]
|
| 901 |
+
if not any(f['id'] == tid for f in lst):
|
| 902 |
+
return None
|
| 903 |
+
cur[item_id] = tid
|
| 904 |
+
placed[surface] = cur
|
| 905 |
+
folders[surface] = lst
|
| 906 |
+
clean = _agf.clean_folders(folders)
|
| 907 |
+
return clean, _agf.clean_item_folders(
|
| 908 |
+
placed, clean,
|
| 909 |
+
{sfc: _own_ids(sfc, cur_ws) for sfc in _agf.FOLDER_SURFACES})
|
| 910 |
+
|
| 911 |
+
# β NO allowed_pids ON PURPOSE β this is app.py's bare `_cl_table_workspace(uname)`
|
| 912 |
+
# call at the old :6268, preserved exactly. Omitting the pool drops memberPids from
|
| 913 |
+
# any foreign shared view (fail-closed), which is harmless here because the folder
|
| 914 |
+
# legs read only view IDS and the folder maps. Passing the pool would be a behavior
|
| 915 |
+
# change inside a pure move.
|
| 916 |
+
_res = _mutate(table_workspace(ctx))
|
| 917 |
+
if _res is None:
|
| 918 |
+
return False # refused: nothing is written
|
| 919 |
+
cl_mod.save_table_folders(uname, _res[0], _res[1])
|
| 920 |
+
# True: the sidebars render from server state (which folder holds what), not from local
|
| 921 |
+
# optimism β the same contract the cohort panel's membership edits ride.
|
| 922 |
+
return True
|
| 923 |
+
|
| 924 |
+
if kind == 'field_delete':
|
| 925 |
+
# Delete a USER-CREATED column outright (owner gap closed 2026-07-27). The prefix IS
|
| 926 |
+
# the permission: only the created strata (`custom_` overlay fields, `measure_` formula
|
| 927 |
+
# columns) are deletable β a base field's key fails the prefix test no matter what the
|
| 928 |
+
# browser claims, so the table's contract cannot be deleted from the client. The store
|
| 929 |
+
# scrubs the field's stored cell values with it (delete_table_field); views naming the
|
| 930 |
+
# key self-heal on their next autosave, the rule every stale colId already rides.
|
| 931 |
+
key = str(event.get('key') or '')[:80]
|
| 932 |
+
import aios_grid as _ag3
|
| 933 |
+
if not (key.startswith('custom_') or key.startswith(_ag3.MEASURE_FIELD_PREFIX)):
|
| 934 |
+
return False
|
| 935 |
+
if store.available():
|
| 936 |
+
cl_mod.delete_table_field(uname, key)
|
| 937 |
+
else:
|
| 938 |
+
_session_ready()
|
| 939 |
+
ws['fields'].pop(key, None)
|
| 940 |
+
for _row in ws['overlays'].values():
|
| 941 |
+
if isinstance(_row, dict):
|
| 942 |
+
_row.pop(key, None)
|
| 943 |
+
# Wave-6 item 3: the client dropped the column optimistically; the store scrub is
|
| 944 |
+
# invisible to the next render (the values leave WITH the column).
|
| 945 |
+
return False
|
| 946 |
+
|
| 947 |
+
if kind == 'field_duplicate':
|
| 948 |
+
# Wave-5 item 1. The DESTINATION key is client-generated (the established pattern β
|
| 949 |
+
# the client must know it synchronously to place the clone right of the source), and
|
| 950 |
+
# both keys must sit in the SAME created stratum: base odoo fields offer no Duplicate,
|
| 951 |
+
# because duplicating the source-of-truth into an editable copy would need a
|
| 952 |
+
# value-snapshot mechanism nobody asked for. Overlay VALUES are copied only for
|
| 953 |
+
# `custom_` sources (a measure clone recomputes; a formula clone re-evaluates).
|
| 954 |
+
# The duplicator becomes the clone's creator β it is a NEW field they made.
|
| 955 |
+
src_key = str(event.get('sourceKey') or '')[:80]
|
| 956 |
+
new_key = str(event.get('key') or '')[:80]
|
| 957 |
+
import aios_grid as _ag5
|
| 958 |
+
prefix = ('custom_' if src_key.startswith('custom_')
|
| 959 |
+
else _ag5.MEASURE_FIELD_PREFIX
|
| 960 |
+
if src_key.startswith(_ag5.MEASURE_FIELD_PREFIX) else None)
|
| 961 |
+
if (prefix is None or not new_key.startswith(prefix) or new_key == src_key
|
| 962 |
+
or new_key in field_by_key):
|
| 963 |
+
return False
|
| 964 |
+
src = ws.get('fields', {}).get(src_key)
|
| 965 |
+
if not isinstance(src, dict):
|
| 966 |
+
return False
|
| 967 |
+
clone = dict(src)
|
| 968 |
+
clone['key'] = new_key
|
| 969 |
+
label = str(event.get('label') or '').strip()[:120]
|
| 970 |
+
clone['label'] = label or (str(src.get('label') or 'Untitled')[:110] + ' copy')
|
| 971 |
+
clone['createdBy'] = uname
|
| 972 |
+
# Wave-6 item 9: the clone inherits the source's scope via dict(src); on the cohort
|
| 973 |
+
# page the duplicate surface may explicitly place it instead (same create-time rule
|
| 974 |
+
# as field_upsert β a page that is not the cohort page cannot mint a cohort field).
|
| 975 |
+
if scope_key == 'cohort' and event.get('scope') in ('cohort', 'global'):
|
| 976 |
+
if event['scope'] == 'cohort':
|
| 977 |
+
clone['scope'] = 'cohort'
|
| 978 |
+
else:
|
| 979 |
+
clone.pop('scope', None)
|
| 980 |
+
if store.available():
|
| 981 |
+
cl_mod.duplicate_table_field(uname, src_key, new_key, clone)
|
| 982 |
+
else:
|
| 983 |
+
_session_ready()
|
| 984 |
+
ws['fields'][new_key] = clone
|
| 985 |
+
if src_key.startswith('custom_'):
|
| 986 |
+
for _row in ws['overlays'].values():
|
| 987 |
+
if isinstance(_row, dict) and src_key in _row:
|
| 988 |
+
_row[new_key] = _row[src_key]
|
| 989 |
+
# Wave-6 item 3: True only when host-held VALUES must flow back β copied overlay
|
| 990 |
+
# cells (custom_ sources) or host-computed measure values. formula/created_time
|
| 991 |
+
# clones compute client-side, so the optimistic insert is already complete.
|
| 992 |
+
return src.get('type') not in ('formula', 'created_time')
|
| 993 |
+
|
| 994 |
+
if kind == 'add_to_list':
|
| 995 |
+
# "Add to list" (owner item 6): a view's current matches become fixed cohort members.
|
| 996 |
+
#
|
| 997 |
+
# β `pids` arrives from the BROWSER. It is intersected with `allowed_pids` β the pool this
|
| 998 |
+
# user may actually see β before anything is stored. The client computing the set is a
|
| 999 |
+
# correctness decision (it is the only engine that already knows the answer, and asking a
|
| 1000 |
+
# second engine the same question is the drift verify_filter_engine.py exists to prevent);
|
| 1001 |
+
# it is NOT a trust decision.
|
| 1002 |
+
raw_pids = event.get('pids')
|
| 1003 |
+
if not isinstance(raw_pids, list):
|
| 1004 |
+
return False
|
| 1005 |
+
pids = [p for p in raw_pids if isinstance(p, int) and p in allowed_pids]
|
| 1006 |
+
if not pids:
|
| 1007 |
+
return False
|
| 1008 |
+
cohort_id = str(event.get('cohortId') or '').strip()[:120]
|
| 1009 |
+
name = str(event.get('name') or '').strip()[:120]
|
| 1010 |
+
if not store.available():
|
| 1011 |
+
return False
|
| 1012 |
+
try:
|
| 1013 |
+
if not cohort_id:
|
| 1014 |
+
if not name:
|
| 1015 |
+
return False
|
| 1016 |
+
cohort_id = cohort_mod.create(uname, name, pids)
|
| 1017 |
+
else:
|
| 1018 |
+
if cohort_id not in cohort_mod.all_for(uname):
|
| 1019 |
+
return False # not this user's cohort β fail closed, do not create
|
| 1020 |
+
cohort_mod.add_members(uname, cohort_id, pids)
|
| 1021 |
+
except ValueError as e:
|
| 1022 |
+
_tel.error('cohort:add_to_list', e)
|
| 1023 |
+
return False
|
| 1024 |
+
# True: the Cohort page's list and counts must reflect this, and the toast below only
|
| 1025 |
+
# renders on a fresh run.
|
| 1026 |
+
ctx.out.toast = (
|
| 1027 |
+
f"Added {len(pids):,} customer{'' if len(pids) == 1 else 's'} to "
|
| 1028 |
+
f"{name or 'the list'}.")
|
| 1029 |
+
return True
|
| 1030 |
+
|
| 1031 |
+
if kind == 'overlay_patch':
|
| 1032 |
+
pid = event.get('pid')
|
| 1033 |
+
if not isinstance(pid, int) or pid not in allowed_pids:
|
| 1034 |
+
return False
|
| 1035 |
+
updates = {}
|
| 1036 |
+
refused = False
|
| 1037 |
+
raw_updates = event.get('updates') if isinstance(event.get('updates'), dict) else {}
|
| 1038 |
+
for key, value in raw_updates.items():
|
| 1039 |
+
if key not in overlay_keys or not isinstance(value, (str, int, float)):
|
| 1040 |
+
refused = True
|
| 1041 |
+
continue
|
| 1042 |
+
# Wave-5 item 1: `permissions.edit` is enforced HERE, per key β the client hiding
|
| 1043 |
+
# its editor is courtesy, this is the wall. Fail-closed: a restricted field with
|
| 1044 |
+
# no readable createdBy admits only admins.
|
| 1045 |
+
perms = (field_by_key.get(key) or {}).get('permissions') or {}
|
| 1046 |
+
edit = perms.get('edit')
|
| 1047 |
+
if edit == 'admins' and not admin:
|
| 1048 |
+
refused = True
|
| 1049 |
+
continue
|
| 1050 |
+
if edit == 'creator' and not (
|
| 1051 |
+
admin or (field_by_key.get(key) or {}).get('createdBy') == uname):
|
| 1052 |
+
refused = True
|
| 1053 |
+
continue
|
| 1054 |
+
clean = str(value)[:10000]
|
| 1055 |
+
if isinstance(value, str) and clean != value:
|
| 1056 |
+
# Truncated: the stored truth is shorter than the client's copy. (An
|
| 1057 |
+
# int/float arriving as its string form renders identically β not a refusal.)
|
| 1058 |
+
refused = True
|
| 1059 |
+
updates[key] = clean
|
| 1060 |
+
if updates:
|
| 1061 |
+
if store.available():
|
| 1062 |
+
cl_mod.patch_table_overlay(uname, pid, updates)
|
| 1063 |
+
else:
|
| 1064 |
+
_session_ready()
|
| 1065 |
+
ws['overlays'].setdefault(str(pid), {}).update(updates)
|
| 1066 |
+
# Wave-6 item 3: the client's cell already shows the typed value β rerun ONLY when
|
| 1067 |
+
# some of what it shows was refused or clamped and must be repainted to the truth.
|
| 1068 |
+
return refused
|
| 1069 |
+
|
| 1070 |
+
return False
|
royalimports_os/core/links.py
ADDED
|
@@ -0,0 +1,59 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Deep links into the app β ?page=<key>&entity=<kind>:<id>&src=<source>.
|
| 2 |
+
|
| 3 |
+
The app (app.py) consumes these params once per session right after login and routes to the page,
|
| 4 |
+
opening the entity drawer when an entity is present. Digest/alert/mention emails build their links
|
| 5 |
+
here so every emailed number lands the reader on the exact row it came from. `src` tags where the
|
| 6 |
+
click came from (digest / alert / mention) β the app logs it as the click-through read receipt.
|
| 7 |
+
"""
|
| 8 |
+
import os
|
| 9 |
+
import sys
|
| 10 |
+
from urllib.parse import quote
|
| 11 |
+
|
| 12 |
+
#: β THE ONE HOST LITERAL LEFT IN THE RUNTIME PATH, and it is a MIGRATION HAZARD β named here
|
| 13 |
+
#: rather than buried in a default argument, because the day we change hosts this is what silently
|
| 14 |
+
#: keeps mailing customers a link to the DEAD one. Found by `ops/verify_portability.py` on its
|
| 15 |
+
#: first run (2026-07-30).
|
| 16 |
+
#:
|
| 17 |
+
#: Why it is still here instead of raising: `deeplink()` has exactly one family of callers
|
| 18 |
+
#: (`modules/digest.py`, 7 sites, all building email hrefs), `DIGEST_ENABLED` defaults to ON, and
|
| 19 |
+
#: no deploy sets `APP_BASE_URL` today β so refusing to guess would break a LIVE feature to fix a
|
| 20 |
+
#: latent one. Instead: both deploy scripts now push `APP_BASE_URL`, which makes the fallback
|
| 21 |
+
#: unreachable in a deployed environment, and reaching it prints an operator warning.
|
| 22 |
+
#:
|
| 23 |
+
#: β EXIT CONDITION for this constant: once a deploy has set `APP_BASE_URL` and it is confirmed
|
| 24 |
+
#: present in the running environment, delete the fallback and let `deeplink()` raise instead. It
|
| 25 |
+
#: is declared in the portability gate's RESIDUALS so the gate reports it honestly rather than
|
| 26 |
+
#: reporting green over it.
|
| 27 |
+
_LEGACY_HF_FALLBACK = 'https://royal-imports-cfo-os.hf.space'
|
| 28 |
+
|
| 29 |
+
BASE_URL = (os.environ.get('APP_BASE_URL') or _LEGACY_HF_FALLBACK).rstrip('/')
|
| 30 |
+
|
| 31 |
+
#: Warned LAZILY and ONCE β at the first link actually built, not at import. An import-time print
|
| 32 |
+
#: fires in every gate, every subprocess and every unit test that so much as touches `core`, which
|
| 33 |
+
#: is how a real warning becomes noise nobody reads.
|
| 34 |
+
_WARNED = [False]
|
| 35 |
+
|
| 36 |
+
|
| 37 |
+
def _warn_once_if_defaulted():
|
| 38 |
+
if _WARNED[0] or os.environ.get('APP_BASE_URL'):
|
| 39 |
+
return
|
| 40 |
+
_WARNED[0] = True
|
| 41 |
+
print(f"[links] APP_BASE_URL is not set - emailed deep links will point at "
|
| 42 |
+
f"{_LEGACY_HF_FALLBACK}, which is correct ONLY while that is where the app lives. "
|
| 43 |
+
f"Set APP_BASE_URL in the deployed environment.", file=sys.stderr)
|
| 44 |
+
|
| 45 |
+
# entity kinds the URL scheme supports (must stay in sync with app.py's _desc_from_param)
|
| 46 |
+
LINK_KINDS = ('customer', 'sku', 'agent', 'account')
|
| 47 |
+
|
| 48 |
+
|
| 49 |
+
def deeplink(page=None, kind=None, ident=None, src=None):
|
| 50 |
+
"""An absolute app URL. deeplink('customer_data', 'customer', 1234, src='digest')."""
|
| 51 |
+
_warn_once_if_defaulted()
|
| 52 |
+
parts = []
|
| 53 |
+
if page:
|
| 54 |
+
parts.append(f"page={quote(str(page), safe='')}")
|
| 55 |
+
if kind in LINK_KINDS and ident not in (None, ''):
|
| 56 |
+
parts.append(f"entity={quote(f'{kind}:{ident}', safe=':')}")
|
| 57 |
+
if src:
|
| 58 |
+
parts.append(f"src={quote(str(src), safe='')}")
|
| 59 |
+
return BASE_URL + ('/?' + '&'.join(parts) if parts else '/')
|
royalimports_os/core/management_pl.py
ADDED
|
@@ -0,0 +1,161 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""core/management_pl.py β reproduce the operating model's MANAGEMENT P&Ls
|
| 2 |
+
(FFS-M / RI-M / GD-M / BU-M) monthly from RAW ODOO, Sep-2024 cutover onward. READ-ONLY.
|
| 3 |
+
|
| 4 |
+
Decoded from v42 (see 17. Application development/financial_analysis/MODEL_LOGIC_MAP.md Β§10):
|
| 5 |
+
|
| 6 |
+
-M[entity][account] = clean_entity[account] # live from Odoo (PROVEN to the cent)
|
| 7 |
+
+ hq_overlay[entity][account] # HQ cost allocated to the entity
|
| 8 |
+
+ adj_overrides[entity][account] # tiny, transition-month only
|
| 9 |
+
|
| 10 |
+
clean_entity = account.analytic.line grouped by general_account_id (GL row) x account_id
|
| 11 |
+
(analytic = ENTITY column). Analytic ids: Fisch=1 Royal=2 GiftwareDeals=3
|
| 12 |
+
HQ=4 Internal=5. Sections by account_type. This is build_income_statements.py
|
| 13 |
+
(verified exact). Proven live: Oct-2025 acct 51000 -> Fisch 327,531.85 etc.
|
| 14 |
+
|
| 15 |
+
hq_overlay / adj_overrides come from data/mgmt_overlay.json (extracted read-only from v42 by
|
| 16 |
+
build_mgmt_overlay.py). The HQ allocation is HARDCODED in the model (PNL (HQ-*) tabs are literal
|
| 17 |
+
values, not formulas), so storing it is faithful to how the model is actually maintained.
|
| 18 |
+
|
| 19 |
+
Consolidation: BU-M = FFS-M + RI-M + GD-M (the Internal analytic is eliminated; HQ is allocated out).
|
| 20 |
+
|
| 21 |
+
Pure: data in -> plain dicts out. No Streamlit, no writes.
|
| 22 |
+
"""
|
| 23 |
+
from __future__ import annotations
|
| 24 |
+
import json
|
| 25 |
+
import os
|
| 26 |
+
from functools import lru_cache
|
| 27 |
+
|
| 28 |
+
from core import odoo as O
|
| 29 |
+
|
| 30 |
+
# ---- entity <-> Odoo analytic account ids (confirmed live) --------------------------------------
|
| 31 |
+
ENTITY_ANALYTIC = {'FFS': 1, 'RI': 2, 'GD': 3, 'HQ': 4, 'Internal': 5}
|
| 32 |
+
ANALYTIC_ENTITY = {v: k for k, v in ENTITY_ANALYTIC.items()}
|
| 33 |
+
BUS = ['FFS', 'RI', 'GD'] # the business units that get a -M statement
|
| 34 |
+
ENTITY_LABEL = {'FFS': 'Fisch (FFS)', 'RI': 'Royal (RI)', 'GD': 'Giftware Deals (GD)', 'BU': 'Consolidated (BU)'}
|
| 35 |
+
|
| 36 |
+
# P&L sections by Odoo account_type (mirrors build_income_statements.py)
|
| 37 |
+
INCOME_TYPES = {'income'}
|
| 38 |
+
OTHER_INCOME_TYPES = {'income_other'}
|
| 39 |
+
COGS_TYPES = {'expense_direct_cost'}
|
| 40 |
+
EXPENSE_TYPES = {'expense'}
|
| 41 |
+
DEPREC_TYPES = {'expense_depreciation'}
|
| 42 |
+
PL_TYPES = INCOME_TYPES | OTHER_INCOME_TYPES | COGS_TYPES | EXPENSE_TYPES | DEPREC_TYPES
|
| 43 |
+
|
| 44 |
+
_OVERLAY_PATH = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))),
|
| 45 |
+
'data', 'mgmt_overlay.json')
|
| 46 |
+
|
| 47 |
+
|
| 48 |
+
def month_bounds(year: int, month: int) -> tuple[str, str]:
|
| 49 |
+
import calendar
|
| 50 |
+
last = calendar.monthrange(year, month)[1]
|
| 51 |
+
return f'{year}-{month:02d}-01', f'{year}-{month:02d}-{last:02d}'
|
| 52 |
+
|
| 53 |
+
|
| 54 |
+
@lru_cache(maxsize=1)
|
| 55 |
+
def account_master() -> dict:
|
| 56 |
+
"""{gl_account_id: {'code','name','type'}} for every account."""
|
| 57 |
+
rows = O.search_read('account.account', [], ['id', 'code', 'name', 'account_type'])
|
| 58 |
+
return {r['id']: {'code': r.get('code') or '', 'name': r.get('name') or '',
|
| 59 |
+
'type': r.get('account_type')} for r in rows}
|
| 60 |
+
|
| 61 |
+
|
| 62 |
+
def _display(acc_type: str, raw: float) -> float:
|
| 63 |
+
"""Income shown raw; costs/expenses/depreciation flipped positive (model convention)."""
|
| 64 |
+
if acc_type in INCOME_TYPES or acc_type in OTHER_INCOME_TYPES:
|
| 65 |
+
return raw
|
| 66 |
+
return -raw
|
| 67 |
+
|
| 68 |
+
|
| 69 |
+
@lru_cache(maxsize=64)
|
| 70 |
+
def clean_entities(year: int, month: int) -> dict:
|
| 71 |
+
"""The clean per-entity P&L straight from Odoo analytic lines, for one month.
|
| 72 |
+
|
| 73 |
+
Returns {entity_key: {account_code: display_amount}} for FFS/RI/GD/HQ/Internal, restricted to
|
| 74 |
+
P&L account types. This is the LIVE, proven-exact foundation (= build_income_statements.py).
|
| 75 |
+
"""
|
| 76 |
+
start, end = month_bounds(year, month)
|
| 77 |
+
rows = O.read_group('account.analytic.line',
|
| 78 |
+
domain=[('date', '>=', start), ('date', '<=', end)],
|
| 79 |
+
fields=['amount:sum'], groupby=['general_account_id', 'account_id'], lazy=False)
|
| 80 |
+
am = account_master()
|
| 81 |
+
out = {k: {} for k in ENTITY_ANALYTIC}
|
| 82 |
+
for r in rows:
|
| 83 |
+
gen = r.get('general_account_id'); ana = r.get('account_id')
|
| 84 |
+
if not gen or not ana:
|
| 85 |
+
continue
|
| 86 |
+
gid, aid = gen[0], ana[0]
|
| 87 |
+
ent = ANALYTIC_ENTITY.get(aid)
|
| 88 |
+
acc = am.get(gid)
|
| 89 |
+
if ent is None or not acc or acc['type'] not in PL_TYPES:
|
| 90 |
+
continue
|
| 91 |
+
amt = _display(acc['type'], r.get('amount') or 0.0)
|
| 92 |
+
if abs(amt) < 0.005:
|
| 93 |
+
continue
|
| 94 |
+
out[ent][acc['code']] = out[ent].get(acc['code'], 0.0) + amt
|
| 95 |
+
return out
|
| 96 |
+
|
| 97 |
+
|
| 98 |
+
@lru_cache(maxsize=1)
|
| 99 |
+
def _overlay() -> dict:
|
| 100 |
+
"""The stored HQ-allocation + transition-reconciliation overlay (build_mgmt_overlay.py).
|
| 101 |
+
Shape: {'YYYY-MM': {'FFS': {acct: amt}, ...}}. These are financial figures, so they live in the
|
| 102 |
+
PRIVATE HF Dataset store on the Space (never in the public Space repo); locally they load from the
|
| 103 |
+
committed JSON. File first, so local runs need no token. Empty -> -M = clean entity only."""
|
| 104 |
+
try:
|
| 105 |
+
with open(_OVERLAY_PATH, encoding='utf-8') as f:
|
| 106 |
+
return json.load(f)
|
| 107 |
+
except (FileNotFoundError, json.JSONDecodeError):
|
| 108 |
+
pass
|
| 109 |
+
try:
|
| 110 |
+
from core import store
|
| 111 |
+
return store.get('mgmt_overlay') or {}
|
| 112 |
+
except Exception:
|
| 113 |
+
return {}
|
| 114 |
+
|
| 115 |
+
|
| 116 |
+
def management_pl(year: int, month: int, entity: str) -> dict:
|
| 117 |
+
"""The management P&L for one entity-month: {account_code: amount} = clean + overlay.
|
| 118 |
+
|
| 119 |
+
entity in {'FFS','RI','GD'}; 'BU' returns the consolidation (sum of the three).
|
| 120 |
+
"""
|
| 121 |
+
if entity == 'BU':
|
| 122 |
+
agg: dict = {}
|
| 123 |
+
for bu in BUS:
|
| 124 |
+
for code, amt in management_pl(year, month, bu).items():
|
| 125 |
+
agg[code] = agg.get(code, 0.0) + amt
|
| 126 |
+
return agg
|
| 127 |
+
clean = dict(clean_entities(year, month).get(entity, {}))
|
| 128 |
+
ov = _overlay().get(f'{year}-{month:02d}', {}).get(entity, {})
|
| 129 |
+
for code, amt in ov.items():
|
| 130 |
+
clean[code] = clean.get(code, 0.0) + amt
|
| 131 |
+
return clean
|
| 132 |
+
|
| 133 |
+
|
| 134 |
+
# ---- section roll-ups (for the statement view) --------------------------------------------------
|
| 135 |
+
def _bucket(code: str) -> str | None:
|
| 136 |
+
acc = next((a for a in account_master().values() if a['code'] == code), None)
|
| 137 |
+
if not acc:
|
| 138 |
+
return None
|
| 139 |
+
t = acc['type']
|
| 140 |
+
if t in INCOME_TYPES: return 'income'
|
| 141 |
+
if t in OTHER_INCOME_TYPES: return 'other_income'
|
| 142 |
+
if t in COGS_TYPES: return 'cogs'
|
| 143 |
+
if t in EXPENSE_TYPES: return 'expense'
|
| 144 |
+
if t in DEPREC_TYPES: return 'deprec'
|
| 145 |
+
return None
|
| 146 |
+
|
| 147 |
+
|
| 148 |
+
def statement(year: int, month: int, entity: str) -> dict:
|
| 149 |
+
"""Section subtotals + net for one entity-month (income / cogs / gross / opex / deprec / net)."""
|
| 150 |
+
lines = management_pl(year, month, entity)
|
| 151 |
+
sec = {'income': 0.0, 'other_income': 0.0, 'cogs': 0.0, 'expense': 0.0, 'deprec': 0.0}
|
| 152 |
+
for code, amt in lines.items():
|
| 153 |
+
b = _bucket(code)
|
| 154 |
+
if b:
|
| 155 |
+
sec[b] += amt
|
| 156 |
+
income = sec['income'] + sec['other_income']
|
| 157 |
+
gross = income - sec['cogs']
|
| 158 |
+
net = gross - sec['expense'] - sec['deprec']
|
| 159 |
+
return {'income': income, 'cogs': sec['cogs'], 'gross_profit': gross,
|
| 160 |
+
'operating_expense': sec['expense'], 'depreciation': sec['deprec'], 'net_profit': net,
|
| 161 |
+
'lines': lines}
|
royalimports_os/core/odoo.py
ADDED
|
@@ -0,0 +1,201 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""CFO-OS shared Odoo data layer (READ-ONLY).
|
| 2 |
+
|
| 3 |
+
Reuses the existing write-guarded client (ffs_dashboard/odoo_client.py) so there is
|
| 4 |
+
a single source of truth for the connection. Adds:
|
| 5 |
+
- cached singleton client
|
| 6 |
+
- read_group / search_read passthroughs with light cleanup
|
| 7 |
+
- business constants (teams, excluded partners)
|
| 8 |
+
- the "sales universe" base domain (confirmed orders, RI+FFS only)
|
| 9 |
+
|
| 10 |
+
NOTHING here writes to Odoo β the underlying client hard-blocks writes.
|
| 11 |
+
"""
|
| 12 |
+
import os
|
| 13 |
+
import sys
|
| 14 |
+
import queue as _queue
|
| 15 |
+
import threading
|
| 16 |
+
from concurrent.futures import ThreadPoolExecutor
|
| 17 |
+
from functools import lru_cache
|
| 18 |
+
from pathlib import Path
|
| 19 |
+
|
| 20 |
+
# Self-contained read-only client lives at the app root (royalimports_os/odoo_client.py) with its
|
| 21 |
+
# own .env β so this whole folder can be moved/hosted anywhere without external paths.
|
| 22 |
+
_ROOT = Path(__file__).resolve().parents[1] # royalimports_os/
|
| 23 |
+
if str(_ROOT) not in sys.path:
|
| 24 |
+
sys.path.insert(0, str(_ROOT))
|
| 25 |
+
from odoo_client import OdooClient # noqa: E402 (read-only, write-guarded)
|
| 26 |
+
|
| 27 |
+
# ---- business constants ----
|
| 28 |
+
TEAM_IDS = [5, 6]
|
| 29 |
+
TEAM_NAMES = {5: 'Fisch', 6: 'Royal'}
|
| 30 |
+
# Partners to exclude from analytics (not part of RI/FFS). Sourced from env so no
|
| 31 |
+
# customer name is ever hardcoded in the (potentially public) repo; set as a Space secret.
|
| 32 |
+
EXCLUDE_PARTNER_NAMES = {n.strip() for n in os.environ.get('EXCLUDE_PARTNER_NAMES', '').split(',') if n.strip()}
|
| 33 |
+
|
| 34 |
+
# Confirmed sale.order.line scope for RI+FFS (the analytics "sales universe").
|
| 35 |
+
SALE_LINE_BASE = [
|
| 36 |
+
('state', 'in', ['sale', 'done']),
|
| 37 |
+
('order_id.team_id', 'in', TEAM_IDS),
|
| 38 |
+
('product_id', '!=', False),
|
| 39 |
+
]
|
| 40 |
+
# Posted customer invoices/credit notes (AR + revenue recognised).
|
| 41 |
+
INVOICE_BASE = [
|
| 42 |
+
('move_type', 'in', ['out_invoice', 'out_refund']),
|
| 43 |
+
('state', '=', 'posted'),
|
| 44 |
+
]
|
| 45 |
+
|
| 46 |
+
|
| 47 |
+
@lru_cache(maxsize=1)
|
| 48 |
+
def _global_client():
|
| 49 |
+
"""The single read-only client used for normal (main-thread) calls."""
|
| 50 |
+
return OdooClient()
|
| 51 |
+
|
| 52 |
+
|
| 53 |
+
# ---- concurrent read pool -------------------------------------------------------------------
|
| 54 |
+
# Each Odoo XML-RPC call is a round-trip; a heavy drawer fires a dozen of them and they add up to
|
| 55 |
+
# many seconds when run one-by-one. `parallel()` runs independent READ tasks across a small pool of
|
| 56 |
+
# extra connections, with each task transparently using its own connection via a thread-local β so
|
| 57 |
+
# the existing read_group/search_read/sum_field helpers need no change. READ-ONLY only.
|
| 58 |
+
_POOL_MAX = 8
|
| 59 |
+
_tlocal = threading.local()
|
| 60 |
+
_pool_q = _queue.Queue()
|
| 61 |
+
_pool_n = 0
|
| 62 |
+
_pool_lock = threading.Lock()
|
| 63 |
+
|
| 64 |
+
|
| 65 |
+
def _checkout():
|
| 66 |
+
global _pool_n
|
| 67 |
+
try:
|
| 68 |
+
return _pool_q.get_nowait()
|
| 69 |
+
except _queue.Empty:
|
| 70 |
+
with _pool_lock:
|
| 71 |
+
grow = _pool_n < _POOL_MAX
|
| 72 |
+
if grow:
|
| 73 |
+
_pool_n += 1
|
| 74 |
+
return OdooClient() if grow else _pool_q.get() # else block until one frees
|
| 75 |
+
|
| 76 |
+
|
| 77 |
+
def _checkin(c):
|
| 78 |
+
_pool_q.put(c)
|
| 79 |
+
|
| 80 |
+
|
| 81 |
+
def get_odoo():
|
| 82 |
+
"""The active read-only client. Inside a parallel() task this is that task's pooled connection;
|
| 83 |
+
otherwise it's the shared singleton. Transparent to every read_group/search_read/... caller."""
|
| 84 |
+
c = getattr(_tlocal, 'client', None)
|
| 85 |
+
return c if c is not None else _global_client()
|
| 86 |
+
|
| 87 |
+
|
| 88 |
+
def set_doc_mode(mode):
|
| 89 |
+
"""Recognition basis for THIS thread's domain builds: 'order' (all confirmed orders) or 'invoice'
|
| 90 |
+
(fully-invoiced orders only). Cached bundles call this before pulling; parallel() propagates it to
|
| 91 |
+
each worker. Thread-local, so concurrent user sessions never race on each other's basis."""
|
| 92 |
+
_tlocal.doc_mode = 'invoice' if mode == 'invoice' else 'order'
|
| 93 |
+
|
| 94 |
+
|
| 95 |
+
def doc_mode():
|
| 96 |
+
return getattr(_tlocal, 'doc_mode', 'order')
|
| 97 |
+
|
| 98 |
+
|
| 99 |
+
def parallel(tasks):
|
| 100 |
+
"""Run independent zero-arg READ callables concurrently across the connection pool; returns
|
| 101 |
+
results in order. Inside each task, get_odoo() (hence read_group/search_read/sum_field/
|
| 102 |
+
search_count) uses that task's own connection. Never use for writes."""
|
| 103 |
+
tasks = list(tasks)
|
| 104 |
+
if not tasks:
|
| 105 |
+
return []
|
| 106 |
+
if len(tasks) == 1:
|
| 107 |
+
return [tasks[0]()]
|
| 108 |
+
|
| 109 |
+
caller_mode = doc_mode() # propagate the submitter's basis to each worker
|
| 110 |
+
def _run(fn):
|
| 111 |
+
c = _checkout()
|
| 112 |
+
_tlocal.client = c
|
| 113 |
+
_tlocal.doc_mode = caller_mode
|
| 114 |
+
try:
|
| 115 |
+
return fn()
|
| 116 |
+
finally:
|
| 117 |
+
_tlocal.client = None
|
| 118 |
+
_tlocal.doc_mode = 'order'
|
| 119 |
+
_checkin(c)
|
| 120 |
+
with ThreadPoolExecutor(max_workers=min(len(tasks), _POOL_MAX)) as ex:
|
| 121 |
+
return list(ex.map(_run, tasks))
|
| 122 |
+
|
| 123 |
+
|
| 124 |
+
def parallel_map(task_dict):
|
| 125 |
+
"""parallel() over a {name: callable} mapping -> {name: result}."""
|
| 126 |
+
keys = list(task_dict)
|
| 127 |
+
return dict(zip(keys, parallel([task_dict[k] for k in keys])))
|
| 128 |
+
|
| 129 |
+
|
| 130 |
+
def m2o_id(v):
|
| 131 |
+
return v[0] if isinstance(v, list) and v else None
|
| 132 |
+
|
| 133 |
+
|
| 134 |
+
def m2o_name(v):
|
| 135 |
+
return v[1] if isinstance(v, list) and len(v) > 1 else ''
|
| 136 |
+
|
| 137 |
+
|
| 138 |
+
@lru_cache(maxsize=1)
|
| 139 |
+
def excluded_partner_ids():
|
| 140 |
+
"""Resolve the excluded partner names (from EXCLUDE_PARTNER_NAMES) to ids, for fast
|
| 141 |
+
domain filtering. Cached for the process."""
|
| 142 |
+
o = get_odoo()
|
| 143 |
+
if not EXCLUDE_PARTNER_NAMES:
|
| 144 |
+
return tuple()
|
| 145 |
+
rows = o.search_read('res.partner',
|
| 146 |
+
[('name', 'in', list(EXCLUDE_PARTNER_NAMES))], ['id'])
|
| 147 |
+
return tuple(r['id'] for r in rows)
|
| 148 |
+
|
| 149 |
+
|
| 150 |
+
def sale_line_domain(date_from=None, date_to=None, team_id=None, extra=None, partner_ids=None):
|
| 151 |
+
"""Build a sale.order.line domain in the RI+FFS confirmed-order scope, excluding
|
| 152 |
+
the configured house accounts, with optional date window / single team / extra clauses.
|
| 153 |
+
partner_ids (a collection, possibly empty) restricts to those customers β the carrier for
|
| 154 |
+
the Customer module's Agent filter; None = no partner restriction (an empty set matches none)."""
|
| 155 |
+
dom = list(SALE_LINE_BASE)
|
| 156 |
+
if team_id is not None:
|
| 157 |
+
dom = [d for d in dom if not (isinstance(d, tuple) and d[0] == 'order_id.team_id')]
|
| 158 |
+
dom.append(('order_id.team_id', '=', team_id))
|
| 159 |
+
if date_from:
|
| 160 |
+
dom.append(('order_id.date_order', '>=', f'{date_from} 00:00:00'))
|
| 161 |
+
if date_to:
|
| 162 |
+
dom.append(('order_id.date_order', '<=', f'{date_to} 23:59:59'))
|
| 163 |
+
ex = excluded_partner_ids()
|
| 164 |
+
if ex:
|
| 165 |
+
dom.append(('order_partner_id', 'not in', list(ex)))
|
| 166 |
+
if partner_ids is not None:
|
| 167 |
+
dom.append(('order_partner_id', 'in', list(partner_ids)))
|
| 168 |
+
if doc_mode() == 'invoice':
|
| 169 |
+
dom.append(('order_id.invoice_status', '=', 'invoiced'))
|
| 170 |
+
if extra:
|
| 171 |
+
dom.extend(extra)
|
| 172 |
+
return dom
|
| 173 |
+
|
| 174 |
+
|
| 175 |
+
def read_group(model, domain, fields, groupby, **kw):
|
| 176 |
+
return get_odoo().read_group(model, domain=domain, fields=fields, groupby=groupby, **kw)
|
| 177 |
+
|
| 178 |
+
|
| 179 |
+
def search_read(model, domain=None, fields=None, **kw):
|
| 180 |
+
return get_odoo().search_read(model, domain=domain or [], fields=fields or [], **kw)
|
| 181 |
+
|
| 182 |
+
|
| 183 |
+
def sum_field(model, domain, field):
|
| 184 |
+
"""One-line aggregate sum of `field` over a domain (uses read_group, no row fetch).
|
| 185 |
+
|
| 186 |
+
Robust to the empty match set: a groupby=[] read_group over zero rows makes Odoo return a
|
| 187 |
+
None aggregate, which the server's XML-RPC layer cannot marshal ("cannot marshal None").
|
| 188 |
+
An empty set simply means the sum is 0.0, so we treat that specific Fault as zero."""
|
| 189 |
+
try:
|
| 190 |
+
g = get_odoo().read_group(model, domain=domain, fields=[f'{field}:sum'], groupby=[], lazy=False)
|
| 191 |
+
except Exception as e: # noqa: BLE001 β narrow check below
|
| 192 |
+
if 'cannot marshal None' in str(e):
|
| 193 |
+
return 0.0
|
| 194 |
+
raise
|
| 195 |
+
return (g[0].get(field) or 0.0) if g else 0.0
|
| 196 |
+
|
| 197 |
+
|
| 198 |
+
def distinct_count(model, domain, field):
|
| 199 |
+
"""Distinct count of a (m2o) field over a domain via grouped read_group."""
|
| 200 |
+
g = get_odoo().read_group(model, domain=domain, fields=[field], groupby=[field], lazy=False)
|
| 201 |
+
return sum(1 for r in g if r.get(field))
|
royalimports_os/core/periods.py
ADDED
|
@@ -0,0 +1,155 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Period helpers β floral wholesale is highly seasonal, so YoY / same-period
|
| 2 |
+
comparisons are first-class. Everything returns ISO date strings (yyyy-mm-dd).
|
| 3 |
+
|
| 4 |
+
`today` is injectable for testing/reproducibility.
|
| 5 |
+
"""
|
| 6 |
+
import datetime as dt
|
| 7 |
+
|
| 8 |
+
|
| 9 |
+
def _d(x):
|
| 10 |
+
return x.isoformat()
|
| 11 |
+
|
| 12 |
+
|
| 13 |
+
def today():
|
| 14 |
+
return dt.date.today()
|
| 15 |
+
|
| 16 |
+
|
| 17 |
+
def ytd(t=None):
|
| 18 |
+
"""(start, end) for year-to-date through `t`."""
|
| 19 |
+
t = t or today()
|
| 20 |
+
return _d(dt.date(t.year, 1, 1)), _d(t)
|
| 21 |
+
|
| 22 |
+
|
| 23 |
+
def ytd_last_year(t=None):
|
| 24 |
+
"""Same calendar window one year earlier (Jan 1 LY β same month/day LY).
|
| 25 |
+
Clamps Feb 29 β Feb 28 in non-leap years."""
|
| 26 |
+
t = t or today()
|
| 27 |
+
try:
|
| 28 |
+
end = dt.date(t.year - 1, t.month, t.day)
|
| 29 |
+
except ValueError:
|
| 30 |
+
end = dt.date(t.year - 1, t.month, t.day - 1)
|
| 31 |
+
return _d(dt.date(t.year - 1, 1, 1)), end if isinstance(end, str) else _d(end)
|
| 32 |
+
|
| 33 |
+
|
| 34 |
+
def ltm(t=None):
|
| 35 |
+
"""Trailing 12 months ending `t`."""
|
| 36 |
+
t = t or today()
|
| 37 |
+
return _d(t - dt.timedelta(days=365)), _d(t)
|
| 38 |
+
|
| 39 |
+
|
| 40 |
+
def prior_ltm(t=None):
|
| 41 |
+
t = t or today()
|
| 42 |
+
return _d(t - dt.timedelta(days=730)), _d(t - dt.timedelta(days=365))
|
| 43 |
+
|
| 44 |
+
|
| 45 |
+
def mtd(t=None):
|
| 46 |
+
t = t or today()
|
| 47 |
+
return _d(dt.date(t.year, t.month, 1)), _d(t)
|
| 48 |
+
|
| 49 |
+
|
| 50 |
+
def wtd(t=None):
|
| 51 |
+
"""(Monday of this ISO week, t) β week-to-date."""
|
| 52 |
+
t = t or today()
|
| 53 |
+
return _d(t - dt.timedelta(days=t.weekday())), _d(t)
|
| 54 |
+
|
| 55 |
+
|
| 56 |
+
def qtd(t=None):
|
| 57 |
+
"""(first day of this calendar quarter, t) β quarter-to-date."""
|
| 58 |
+
t = t or today()
|
| 59 |
+
q_start_month = ((t.month - 1) // 3) * 3 + 1
|
| 60 |
+
return _d(dt.date(t.year, q_start_month, 1)), _d(t)
|
| 61 |
+
|
| 62 |
+
|
| 63 |
+
def quarter_range(t=None):
|
| 64 |
+
"""(first day, last day) of the calendar quarter containing t."""
|
| 65 |
+
t = t or today()
|
| 66 |
+
qs = ((t.month - 1) // 3) * 3 + 1
|
| 67 |
+
first = dt.date(t.year, qs, 1)
|
| 68 |
+
nxt = dt.date(t.year + (1 if qs + 3 > 12 else 0), ((qs + 2) % 12) + 1, 1)
|
| 69 |
+
return _d(first), _d(nxt - dt.timedelta(days=1))
|
| 70 |
+
|
| 71 |
+
|
| 72 |
+
def shift_year(date_from, date_to, weeks=False):
|
| 73 |
+
"""Same window one year earlier. For week-aligned comparisons (weeks=True) shift by exactly 52
|
| 74 |
+
weeks (364 days) so the weekday lines up; otherwise shift by calendar year (clamping Feb 29)."""
|
| 75 |
+
f = dt.date.fromisoformat(date_from)
|
| 76 |
+
t = dt.date.fromisoformat(date_to)
|
| 77 |
+
if weeks:
|
| 78 |
+
return _d(f - dt.timedelta(days=364)), _d(t - dt.timedelta(days=364))
|
| 79 |
+
|
| 80 |
+
def back(d):
|
| 81 |
+
try:
|
| 82 |
+
return dt.date(d.year - 1, d.month, d.day)
|
| 83 |
+
except ValueError:
|
| 84 |
+
return dt.date(d.year - 1, d.month, d.day - 1)
|
| 85 |
+
return _d(back(f)), _d(back(t))
|
| 86 |
+
|
| 87 |
+
|
| 88 |
+
def last_n_days(n, t=None):
|
| 89 |
+
t = t or today()
|
| 90 |
+
return _d(t - dt.timedelta(days=n)), _d(t)
|
| 91 |
+
|
| 92 |
+
|
| 93 |
+
def year_range(year):
|
| 94 |
+
return f'{year}-01-01', f'{year}-12-31'
|
| 95 |
+
|
| 96 |
+
|
| 97 |
+
def month_range(year, month):
|
| 98 |
+
"""(first day, last day) of a given calendar month, ISO strings."""
|
| 99 |
+
first = dt.date(year, month, 1)
|
| 100 |
+
nxt = dt.date(year + (1 if month == 12 else 0), 1 if month == 12 else month + 1, 1)
|
| 101 |
+
return _d(first), _d(nxt - dt.timedelta(days=1))
|
| 102 |
+
|
| 103 |
+
|
| 104 |
+
def period_options(t=None, n_months=15):
|
| 105 |
+
"""Ordered {label: (date_from, date_to)} for the close/reconciliation period picker."""
|
| 106 |
+
t = t or today()
|
| 107 |
+
opts = {}
|
| 108 |
+
opts['This month (to date)'] = (_d(dt.date(t.year, t.month, 1)), _d(t))
|
| 109 |
+
lm_year, lm_month = (t.year, t.month - 1) if t.month > 1 else (t.year - 1, 12)
|
| 110 |
+
lf, lt = month_range(lm_year, lm_month)
|
| 111 |
+
opts[f'Last month ({lf[:7]})'] = (lf, lt)
|
| 112 |
+
opts['Quarter to date'] = (_d(dt.date(t.year, ((t.month - 1) // 3) * 3 + 1, 1)), _d(t))
|
| 113 |
+
opts['Year to date'] = (_d(dt.date(t.year, 1, 1)), _d(t))
|
| 114 |
+
for label, mf, mt in reversed(month_starts(n_months, t)):
|
| 115 |
+
if label == f'{t.year}-{t.month:02d}' or label == lf[:7]:
|
| 116 |
+
continue
|
| 117 |
+
opts[label] = (mf, mt)
|
| 118 |
+
return opts
|
| 119 |
+
|
| 120 |
+
|
| 121 |
+
def month_starts(n_back=13, t=None):
|
| 122 |
+
"""List of (yyyy-mm, start, end) for the last n_back months including current."""
|
| 123 |
+
t = t or today()
|
| 124 |
+
out = []
|
| 125 |
+
y, m = t.year, t.month
|
| 126 |
+
for _ in range(n_back):
|
| 127 |
+
start = dt.date(y, m, 1)
|
| 128 |
+
end = (dt.date(y + (m // 12), (m % 12) + 1, 1) - dt.timedelta(days=1))
|
| 129 |
+
out.append((f'{y}-{m:02d}', _d(start), _d(end)))
|
| 130 |
+
m -= 1
|
| 131 |
+
if m == 0:
|
| 132 |
+
m = 12
|
| 133 |
+
y -= 1
|
| 134 |
+
return list(reversed(out))
|
| 135 |
+
|
| 136 |
+
|
| 137 |
+
def week_starts(n_back=12, t=None):
|
| 138 |
+
"""List of (label, start, end) for the last n_back ISO weeks (MonβSun) including the current
|
| 139 |
+
(partial) week. label = the Monday as 'MM-DD' (compact, chart-friendly); end is clamped to t
|
| 140 |
+
for the current week so weekly totals never include future days."""
|
| 141 |
+
t = t or today()
|
| 142 |
+
this_mon = t - dt.timedelta(days=t.weekday())
|
| 143 |
+
out = []
|
| 144 |
+
for i in range(n_back):
|
| 145 |
+
mon = this_mon - dt.timedelta(weeks=i)
|
| 146 |
+
sun = mon + dt.timedelta(days=6)
|
| 147 |
+
end = min(sun, t)
|
| 148 |
+
out.append((f'{mon.month:02d}-{mon.day:02d}', _d(mon), _d(end)))
|
| 149 |
+
return list(reversed(out))
|
| 150 |
+
|
| 151 |
+
|
| 152 |
+
def yoy_pct(this_v, last_v):
|
| 153 |
+
if last_v:
|
| 154 |
+
return (this_v - last_v) / last_v * 100.0
|
| 155 |
+
return None
|
royalimports_os/core/perms.py
ADDED
|
@@ -0,0 +1,164 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""core/perms.py β the permission + scope predicates as PURE functions of a user record.
|
| 2 |
+
|
| 3 |
+
EXIT-3b (2026-07-30). `ui/session.py` holds the same predicates keyed off `st.session_state`;
|
| 4 |
+
these take the user record explicitly, so the API can answer "what may this session see?" without
|
| 5 |
+
a Streamlit session. ONE implementation of each rule, two callers β and `verify_api.py` asserts
|
| 6 |
+
lock-step against `ui/session.py`'s source (the [[date-window-vocabulary]] discipline: a mirrored
|
| 7 |
+
vocabulary that is not gated has already drifted).
|
| 8 |
+
|
| 9 |
+
β THE TRAP THIS FILE EXISTS TO AVOID. `ui/session.brand_team()` reads the sidebar's BU selector
|
| 10 |
+
and defaults to `'All'` β `team_id=None` β CONSOLIDATED. Mirroring it literally on the API would
|
| 11 |
+
be a fail-OPEN: an API session has no selector, so a Royal-only user would be handed Fisch rows β
|
| 12 |
+
the exact leak strict isolation exists to prevent. The scope must be derived from the USER RECORD
|
| 13 |
+
(`bus`), never from a UI default. `scope_team_id()` below is that derivation.
|
| 14 |
+
|
| 15 |
+
Fail-closed everywhere: an unknown module key is denied, an unreadable `bus` narrows rather than
|
| 16 |
+
widens, and a user with no grant sees nothing rather than everything.
|
| 17 |
+
"""
|
| 18 |
+
import core.context as ctxlib
|
| 19 |
+
import core.registry as registry
|
| 20 |
+
import core.users as users
|
| 21 |
+
|
| 22 |
+
# Old module keys migrate ON READ β grants, Library prefs and ?page= deep links keep working
|
| 23 |
+
# without ever editing a stored user record. MUST stay identical to ui/session._LEGACY_KEYS
|
| 24 |
+
# (gated in verify_api.py): 'myday' shipped before the Customer List rename, 'customer_list'
|
| 25 |
+
# before the 2026-07-26 rename to 'customer_data', 'map' joined in wave 8 when the Map PAGE
|
| 26 |
+
# became a VIEW of the customer table. A chain is fine only if every hop resolves to the CURRENT
|
| 27 |
+
# key, so 'myday' points at the END of the chain, not at the middle.
|
| 28 |
+
# β 'customers' is deliberately NOT here. It is a real registry key β the parent of
|
| 29 |
+
# customer_data + cohort β and may_open() gives a parent grant to its children. Mapping it away
|
| 30 |
+
# would silently narrow every "customers" grant to one child.
|
| 31 |
+
_LEGACY_KEYS = {'myday': 'customer_data', 'customer_list': 'customer_data',
|
| 32 |
+
'map': 'customer_data', 'library': 'settings'}
|
| 33 |
+
|
| 34 |
+
|
| 35 |
+
def is_admin(user):
|
| 36 |
+
return (user or {}).get('role') == 'admin'
|
| 37 |
+
|
| 38 |
+
|
| 39 |
+
def allowed_modules(user):
|
| 40 |
+
"""Registry keys this user may open β None means ALL (admins / unrestricted users)."""
|
| 41 |
+
m = (user or {}).get('modules', 'all')
|
| 42 |
+
if not m or m == 'all':
|
| 43 |
+
return None
|
| 44 |
+
return {_LEGACY_KEYS.get(k, k) for k in m}
|
| 45 |
+
|
| 46 |
+
|
| 47 |
+
def may_open(user, key):
|
| 48 |
+
"""True iff `user` may OPEN module `key`. A PARENT grant covers its sub-modules (granting
|
| 49 |
+
'customers' grants 'customer_data'). Fail-closed β an unknown key stays denied."""
|
| 50 |
+
am = allowed_modules(user)
|
| 51 |
+
if am is None or key in am:
|
| 52 |
+
return True
|
| 53 |
+
parent = registry.BY_KEY.get(key, {}).get('parent')
|
| 54 |
+
return parent in am if parent else False
|
| 55 |
+
|
| 56 |
+
|
| 57 |
+
# ------------------------------------------------------------------ BU (business unit) scope
|
| 58 |
+
def allowed_bu_labels(user):
|
| 59 |
+
"""The BU labels this user may see β `core.users.allowed_bus_labels`, which is already a
|
| 60 |
+
pure function of the record. Named here so callers have one import for the whole rule set."""
|
| 61 |
+
return users.allowed_bus_labels(user)
|
| 62 |
+
|
| 63 |
+
|
| 64 |
+
def allowed_team_ids(user):
|
| 65 |
+
"""The concrete Odoo team_ids this user may see, as a frozenset. `All` expands to every BU."""
|
| 66 |
+
labels = allowed_bu_labels(user)
|
| 67 |
+
if 'All' in labels:
|
| 68 |
+
return frozenset(t for t in ctxlib.BRANDS.values() if t is not None)
|
| 69 |
+
return frozenset(ctxlib.BRANDS[l] for l in labels
|
| 70 |
+
if l in ctxlib.BRANDS and ctxlib.BRANDS[l] is not None)
|
| 71 |
+
|
| 72 |
+
|
| 73 |
+
def scope_team_id(user):
|
| 74 |
+
"""The `team_id` an API request must query with β the ONE number that enforces BU isolation.
|
| 75 |
+
|
| 76 |
+
None means CONSOLIDATED, and it is returned ONLY when the user may genuinely see every BU
|
| 77 |
+
(`bus='all'`, or a multi-BU grant, both of which put 'All' in the label list). A user
|
| 78 |
+
permitted exactly one BU gets that BU's id PINNED, never None.
|
| 79 |
+
|
| 80 |
+
β This is the opposite default from `ui/session.brand_team()`, which falls back to 'All'
|
| 81 |
+
because a UI selector has to start somewhere. Here there is no selector to read, so the
|
| 82 |
+
fallback would BE the leak. A single-BU user's rows are pinned at the query, not filtered
|
| 83 |
+
afterwards, so there is no window in which the other BU's rows exist in the response.
|
| 84 |
+
"""
|
| 85 |
+
labels = allowed_bu_labels(user)
|
| 86 |
+
if 'All' in labels:
|
| 87 |
+
return None
|
| 88 |
+
ids = sorted(allowed_team_ids(user))
|
| 89 |
+
return ids[0] if len(ids) == 1 else None
|
| 90 |
+
|
| 91 |
+
|
| 92 |
+
def scope_agent(user):
|
| 93 |
+
"""The sales agent whose OWN BOOK this session is confined to, or None for the whole book.
|
| 94 |
+
|
| 95 |
+
β Mirrors `page_customer_data` (app.py), which reads `u.get('agent')` with NO admin
|
| 96 |
+
exemption β deliberately different from `page_agent`'s `None if is_admin() else β¦`. The
|
| 97 |
+
customer TABLE is the surface `/api/v1/customers` serves, so its rule is the one that
|
| 98 |
+
travels: an account linked to an agent sees that agent's book, admin or not. Getting this
|
| 99 |
+
wrong in the permissive direction would hand a restricted sales-agent login the whole book.
|
| 100 |
+
"""
|
| 101 |
+
return (user or {}).get('agent') or None
|
| 102 |
+
|
| 103 |
+
|
| 104 |
+
# ------------------------------------------------------------------ nav projection
|
| 105 |
+
def nav_pages(user):
|
| 106 |
+
"""The nav this user may see: registry order, `may_open`-filtered, archived EXCLUDED.
|
| 107 |
+
|
| 108 |
+
Shape (X2's `{key,label,source?,parent?}` plus ONE additive field, `chrome`):
|
| 109 |
+
[{key, label, source?, chrome: 'main'|'utility'}, β¦]
|
| 110 |
+
|
| 111 |
+
Four rules, each of which has bitten before:
|
| 112 |
+
|
| 113 |
+
* ARCHIVED IS INVISIBLE TO EVERYONE ([[module-archive-semantics]]) β not "hidden unless
|
| 114 |
+
admin". 18 of the 28 registry rows are archived and none may appear here.
|
| 115 |
+
|
| 116 |
+
* `group_only` ROWS ARE EXCLUDED ENTIRELY, and therefore **no `parent` is ever emitted**.
|
| 117 |
+
Wave-9 I8 flipped the host nav to a FLAT list: children became top-level and the folder
|
| 118 |
+
key was dropped, because a `group_only` key names a FAMILY and has no `PAGE_FUNCS` entry
|
| 119 |
+
β rendering a row for it is a white screen one click away (app.py:8182-8187 says exactly
|
| 120 |
+
this). `customers` is the only such row. Emitting `parent: 'customers'` on its children
|
| 121 |
+
while the parent itself is filtered out would hand the client a DANGLING reference to a
|
| 122 |
+
row that is not in the payload, which is worse than a flat list β so the flat list is
|
| 123 |
+
what ships, matching the surface a user actually sees today.
|
| 124 |
+
|
| 125 |
+
* `nav: False` IS NOT `archived`. `settings`, `analyst` and `dictionary` are LIVE surfaces
|
| 126 |
+
the host renders outside the module list (Analyst is the unrestricted landing page,
|
| 127 |
+
Settings sits in the chrome). Dropping them would leave the shell unable to reach two of
|
| 128 |
+
the nine live surfaces, so they ship flagged `chrome: 'utility'` and the client places
|
| 129 |
+
them where the host does. `chrome: 'main'` is the module list.
|
| 130 |
+
|
| 131 |
+
* NOT filtered by `ui/session.user_enabled_modules()`. That is a PREFERENCE read from
|
| 132 |
+
`royalimports_os/data/store/prefs.json` β a local file on one box. Consuming it here
|
| 133 |
+
would put per-user FILESYSTEM state inside a process EXIT-4 requires to be stateless,
|
| 134 |
+
and the same user's nav would differ per container. It is a Library setting, not a
|
| 135 |
+
permission; the permission gate is `may_open`, and that is what this applies.
|
| 136 |
+
"""
|
| 137 |
+
out = []
|
| 138 |
+
for m in registry.REGISTRY:
|
| 139 |
+
if m.get('archived') or m.get('group_only'):
|
| 140 |
+
continue
|
| 141 |
+
if not may_open(user, m['key']):
|
| 142 |
+
continue
|
| 143 |
+
row = {'key': m['key'], 'label': m['label'],
|
| 144 |
+
'chrome': 'main' if m.get('nav', True) else 'utility'}
|
| 145 |
+
if m.get('source'):
|
| 146 |
+
row['source'] = m['source']
|
| 147 |
+
out.append(row)
|
| 148 |
+
return out
|
| 149 |
+
|
| 150 |
+
|
| 151 |
+
def landing_page(user):
|
| 152 |
+
"""Where a restricted user lands: the first MAIN-chrome page in their grant. None for
|
| 153 |
+
unrestricted users (the client lands on the Analyst, as the host does).
|
| 154 |
+
|
| 155 |
+
Mirrors `ui/session.first_allowed_module` minus its archived last-resort leg β that leg
|
| 156 |
+
exists so a misconfigured Streamlit login is not bricked; an API caller gets an explicit
|
| 157 |
+
answer instead, and serving an archived key would contradict the rule above.
|
| 158 |
+
"""
|
| 159 |
+
if allowed_modules(user) is None:
|
| 160 |
+
return None
|
| 161 |
+
for row in nav_pages(user):
|
| 162 |
+
if row['chrome'] == 'main':
|
| 163 |
+
return row['key']
|
| 164 |
+
return None
|
royalimports_os/core/registry.py
ADDED
|
@@ -0,0 +1,137 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Module registry β the single source of truth for the dashboards.
|
| 2 |
+
|
| 3 |
+
`app.py` builds the sidebar nav from this, and decides whether the global Brand (DBA) filter
|
| 4 |
+
applies to a page. `validate.py` runs each module's validate() from this. Adding a module =
|
| 5 |
+
add one row here + a page function in app.py + a module in modules/.
|
| 6 |
+
|
| 7 |
+
Fields:
|
| 8 |
+
key stable id (matches the modules/<key>.py file and the app page_<key> function)
|
| 9 |
+
label sidebar/title text (no emojis)
|
| 10 |
+
brand True -> DBA-filterable (honours the Fisch/Royal selector)
|
| 11 |
+
False -> consolidated/company-level (ignores the brand selector)
|
| 12 |
+
hq True -> HQ / company-level (cash flow, balance sheet, AR/AP, consolidated)
|
| 13 |
+
validate True -> module exposes validate() and is included in validate.py
|
| 14 |
+
parent set -> this row is a SUB-MODULE of <parent key>: it leaves the Workflows dropdown and
|
| 15 |
+
renders as a sub-nav button when its parent (or a sibling) is the active module; a
|
| 16 |
+
parent grant covers its sub-modules (owner nav redesign 2026-07-23)
|
| 17 |
+
archived True -> RETIRED from the UI for EVERYONE (owner 2026-07-27, supersedes the
|
| 18 |
+
2026-07-23 per-user-restore semantics): no nav entry, no Settings>Modules checkbox,
|
| 19 |
+
prewarm and validate.py skip it. Code and data layer stay in the repo but are NOT
|
| 20 |
+
maintained β spend no effort on archived modules until the owner un-archives.
|
| 21 |
+
2026-07-23 pass: backorders, pricecomp, o2c, bookings, spend, expenses.
|
| 22 |
+
2026-07-27 pass (keep = Sales + Customers family + Agents + Procurement +
|
| 23 |
+
Collections + Analyst/plumbing): products, assortment, financial, management,
|
| 24 |
+
pricing, inventory, vendor, warehouse, returns, outreach, complexity, health.
|
| 25 |
+
note one-line scope note shown in the UI / docs
|
| 26 |
+
"""
|
| 27 |
+
# WAVE-9 I8 β `'source'`: which connected system a module's data comes from.
|
| 28 |
+
#
|
| 29 |
+
# The owner's model: every nav entry is "a piece of database from a source", so the nav shows the
|
| 30 |
+
# source beside the name ("Sales Β· Odoo"). It is a REGISTRY FACT rather than a string in the nav
|
| 31 |
+
# code precisely so the nav needs no edit when a second connector lands β a user-created blank
|
| 32 |
+
# table simply carries a different source, and a module with no `source` (Metric Dictionary,
|
| 33 |
+
# Settings, the Analyst) renders no badge because it is not a database at all.
|
| 34 |
+
# See [[connector-onboarding]]: the plug-in seam is datastore.py, and Odoo has no delegated
|
| 35 |
+
# OAuth, so the "+ New" connect flow is the credential-form shape, not an OAuth redirect.
|
| 36 |
+
REGISTRY = [
|
| 37 |
+
{'key': 'sales', 'label': 'Sales', 'brand': True, 'hq': False, 'validate': True, 'source': 'Odoo',
|
| 38 |
+
'note': 'Revenue, YoY, seasonality, reps, customers, SKUs β by BU.'},
|
| 39 |
+
# ARCHIVED AS A PAGE (owner item 14, wave 8): "archive the current form of the Customer
|
| 40 |
+
# dashboard completely, i want to redesign it, exactly with the backend we have". So the
|
| 41 |
+
# PAGE goes and the DATA LAYER stays β modules/customers.py still powers the Customer table's
|
| 42 |
+
# metrics and still validates against Odoo, which is what the redesign will be built on.
|
| 43 |
+
#
|
| 44 |
+
# β NOT 'archived': True. Archived means invisible to everyone, and this row is the parent of
|
| 45 |
+
# customer_data + cohort β archiving it would take the whole family out of the nav and strand
|
| 46 |
+
# its children. 'group_only' says exactly what is true: this key names a FOLDER, never a
|
| 47 |
+
# destination. It has no PAGE_FUNCS entry, the nav never renders it as a leaf, and a stale
|
| 48 |
+
# deep link to it redirects to the family's first visible member.
|
| 49 |
+
{'key': 'customers', 'label': 'Customers', 'brand': True, 'hq': False, 'validate': True,
|
| 50 |
+
'group_only': True,
|
| 51 |
+
'note': 'Folder for the customer surfaces (Customer table, Cohort). The old Customers '
|
| 52 |
+
'dashboard was retired for redesign (owner item 14, 2026-07-28); its metric layer '
|
| 53 |
+
'(modules/customers.py) still backs the table and still validates against Odoo.'},
|
| 54 |
+
# LABEL renamed 'Customer List' -> 'Data' (owner 2026-07-25) -> 'Customer' (owner item 5,
|
| 55 |
+
# 2026-07-27); KEY renamed 'customer_list' -> 'customer_data' (owner 2026-07-26). The key
|
| 56 |
+
# rename is safe ONLY because _LEGACY_KEYS (ui/session.py) migrates grants, Library prefs
|
| 57 |
+
# and ?page= deep links ON READ β stored user records are never edited, exactly as
|
| 58 |
+
# myday->customer_list was done. The STORE keys are deliberately NOT renamed:
|
| 59 |
+
# 'customer_table_workspace' and 'customer_lists' hold real user data, and renaming them
|
| 60 |
+
# would orphan every saved view. Label-only changes need none of that machinery.
|
| 61 |
+
{'key': 'customer_data', 'label': 'Customer', 'brand': True, 'hq': False, 'validate': False, 'source': 'Odoo', 'parent': 'customers',
|
| 62 |
+
'note': 'Build, filter and save customer lists: the whole scoped book with per-customer '
|
| 63 |
+
'metrics; saved lists are visible, editable formulas (Call list and Win-back ship '
|
| 64 |
+
'as templates) plus hand-picked members. Sub-module of Customers (owner nav redesign 2026-07-23; replaced My Day).'},
|
| 65 |
+
# Cohort (owner item 6, 2026-07-26): the SAME table as Data over a FIXED set. The
|
| 66 |
+
# difference is membership semantics, not layout β a saved view re-populates as the
|
| 67 |
+
# data moves, a cohort only changes when a person edits it. validate:False because
|
| 68 |
+
# there is no Odoo aggregate to reconcile a hand-picked list against; its metrics are
|
| 69 |
+
# the customers module's, already validated there.
|
| 70 |
+
{'key': 'cohort', 'label': 'Cohort', 'brand': True, 'hq': False, 'validate': False, 'source': 'Odoo', 'parent': 'customers',
|
| 71 |
+
'note': 'Hand-curated, unchanging customer lists. Same grid as Data (filter, sort, '
|
| 72 |
+
'group, custom columns) but the SET is fixed: it changes only when someone '
|
| 73 |
+
'adds or removes a member. Populated from a view via ... > Add to list.'},
|
| 74 |
+
{'key': 'agent', 'label': 'Agents', 'brand': True, 'hq': False, 'validate': True, 'source': 'Odoo',
|
| 75 |
+
'note': 'Per-agent book: sales (custom dates), returns, top SKUs, full customer list incl. inactive. Returns consolidated.'},
|
| 76 |
+
{'key': 'products', 'label': 'SKU', 'brand': True, 'hq': False, 'validate': True, 'archived': True,
|
| 77 |
+
'note': 'SKU movers, zombies, velocity, coverage, drawers β by BU (sales-derived).'},
|
| 78 |
+
{'key': 'assortment', 'label': 'Assortment', 'brand': True, 'hq': False, 'validate': True, 'archived': True,
|
| 79 |
+
'note': 'Facet-level performance (category/color/occasion/collection/season) + season readiness β by BU.'},
|
| 80 |
+
{'key': 'financial', 'label': 'Financial', 'brand': True, 'hq': False, 'validate': True, 'archived': True,
|
| 81 |
+
'note': 'Gross margin by BU/category/SKU. Cash-conversion cycle is HQ-consolidated.'},
|
| 82 |
+
{'key': 'pricing', 'label': 'Pricing', 'brand': True, 'hq': False, 'validate': True, 'archived': True,
|
| 83 |
+
'note': 'Per-SKU margin/markup/LTM sales + allocated net P&L (channel-rate cost-to-SKU) for pricing decisions.'},
|
| 84 |
+
{'key': 'management', 'label': 'Management P&L', 'brand': False, 'hq': True, 'validate': True, 'archived': True,
|
| 85 |
+
'nav': False, # rendered INSIDE the Financial module (view toggle) β archived WITH financial
|
| 86 |
+
'note': 'Operating-model management income statements (FFS/RI/GD/BU-M) reproduced from raw Odoo, Sep-2024+.'},
|
| 87 |
+
{'key': 'inventory', 'label': 'Inventory', 'brand': False, 'hq': False, 'validate': True, 'archived': True,
|
| 88 |
+
'note': 'On-hand stock is physically consolidated (shared warehouse); not DBA-split.'},
|
| 89 |
+
{'key': 'procurement', 'label': 'Procurement', 'brand': False, 'hq': False, 'validate': True, 'source': 'Odoo',
|
| 90 |
+
'note': 'Biweekly buy list: lead-time-cover trigger, 8-month demand baseline + YTD trend, by supplier.'},
|
| 91 |
+
{'key': 'vendor', 'label': 'Vendors', 'brand': False, 'hq': False, 'validate': False, 'archived': True,
|
| 92 |
+
'note': 'Vendor master data + the SKUs each supplies; multiple-vendor / cheapest-price view.'},
|
| 93 |
+
{'key': 'warehouse', 'label': 'Warehouse', 'brand': False, 'hq': False, 'validate': True, 'archived': True,
|
| 94 |
+
'note': 'Movement & efficiency: on-time ship, Pick-Pack-Ship cycle times, throughput, backlog age, shrinkage. Physical ops (consolidated).'},
|
| 95 |
+
{'key': 'ar', 'label': 'Collections', 'brand': False, 'hq': True, 'validate': True,
|
| 96 |
+
'source': 'Odoo',
|
| 97 |
+
'note': 'Receivables are company-level (consolidated).'},
|
| 98 |
+
{'key': 'returns', 'label': 'Returns', 'brand': False, 'hq': True, 'validate': True, 'archived': True,
|
| 99 |
+
'note': 'Credit-note lens: refund concentration by SKU (quality) and customer (behavior). Company-level.'},
|
| 100 |
+
{'key': 'outreach', 'label': 'Outreach', 'brand': True, 'hq': False, 'validate': False, 'archived': True,
|
| 101 |
+
'note': 'Campaign emails to customer segments: templates, suppression, send log, revenue attribution. Sending is admin-gated behind SAFE_MODE.'},
|
| 102 |
+
{'key': 'backorders', 'label': 'Backorders', 'brand': True, 'hq': False, 'validate': True, 'archived': True,
|
| 103 |
+
'note': 'Confirmed-undelivered order lines aged vs promise date, valued, with a supply-aware next action per row (ship / expedite / call). Wholesale scope.'},
|
| 104 |
+
{'key': 'pricecomp', 'label': 'Price Compliance', 'brand': True, 'hq': False, 'validate': True, 'archived': True,
|
| 105 |
+
'note': 'Selling below the customer pricelist tier (LTM, per customer x SKU): the pocket-price floor worklist with annualized leak $. Sub-30% ratios usually mean a stale or pack-basis RULE β fix the rule, not the rep.'},
|
| 106 |
+
{'key': 'o2c', 'label': 'Cash Timing', 'brand': True, 'hq': False, 'validate': True, 'archived': True,
|
| 107 |
+
'note': 'Order-to-cash stage decomposition (order-to-ship / ship-to-invoice / invoice-to-paid, each with its owner) + the terms-gap rollup: contractual vs actual days per payment term with the free-credit $ it strands.'},
|
| 108 |
+
{'key': 'bookings', 'label': 'Order Book', 'brand': True, 'hq': False, 'validate': True, 'archived': True,
|
| 109 |
+
'note': 'Pre-season booking coverage: cumulative booked $ for Aug-Dec delivery vs last year same-week; category TY-vs-LY-as-of-date. The container-program decision, months ahead.'},
|
| 110 |
+
{'key': 'spend', 'label': 'Spend & Payables', 'brand': False, 'hq': True, 'validate': True, 'archived': True,
|
| 111 |
+
'note': 'Vendor-bill side: payment-terms capture (early-pay cash surrendered), duplicate-bill review, the opex spend cube (fragmentation, consolidate+rebid) and freight recovery. Company-level.'},
|
| 112 |
+
{'key': 'complexity', 'label': 'SKU Complexity', 'brand': False, 'hq': False, 'validate': True, 'archived': True,
|
| 113 |
+
'note': 'BCG tail rationalization: every SKU re-costed by ACTIVITY (lines, picks, returns) + 25%/yr carrying; KILL candidates rest on hard math (GM minus carrying), REVIEW on the pooled activity estimate. All-channel.'},
|
| 114 |
+
{'key': 'expenses', 'label': 'Expenses', 'brand': False, 'hq': True, 'validate': True, 'archived': True,
|
| 115 |
+
'note': 'Operating expense from the GL (all expense-type accounts; COGS excluded): trend, operating leverage (opex % of revenue), the YoY cost bridge, XmR control-limit spike watch list, fixed/variable split and a drill-to-ledger category directory. Company-level.'},
|
| 116 |
+
{'key': 'health', 'label': 'Data Health', 'brand': False, 'hq': True, 'validate': True, 'archived': True,
|
| 117 |
+
'note': 'Close/reconciliation scan; period-filtered, mostly company-level. ARCHITECTURE '
|
| 118 |
+
'(owner 2026-07-23): every discrepancy / potential-error marker lives here β '
|
| 119 |
+
'procurement mapping gaps, count-trust (unverified on-hand counts), untracked-on-'
|
| 120 |
+
'order β so operational workflows stay clean for doing the work.'},
|
| 121 |
+
{'key': 'dictionary', 'label': 'Metric Dictionary', 'brand': False, 'hq': True, 'validate': True,
|
| 122 |
+
'nav': False, # sidebar sentinel button (like Feedback) β not a Dashboards/Workflows entry
|
| 123 |
+
'note': 'PAGE RETIRED 2026-07-23 (owner: broken/not customer-facing) β row kept ONLY so validate.py keeps running the semantic-layer contracts (the Analyst grounding proof). No nav, no page.'},
|
| 124 |
+
{'key': 'settings', 'label': 'Settings', 'brand': False, 'hq': True, 'validate': False,
|
| 125 |
+
'nav': False, # sidebar sentinel (account group) β owner IA 2026-07-12
|
| 126 |
+
'note': 'User scope settings: the Business Unit toggle (strict isolation) and the Data basis '
|
| 127 |
+
'toggle (Orders vs Invoiced) moved here from the sidebar.'},
|
| 128 |
+
{'key': 'analyst', 'label': 'AIOS Analyst', 'brand': False, 'hq': True, 'validate': False,
|
| 129 |
+
'nav': False, # sidebar sentinel button; eval gate = harness/evals.py (run pre-ship, NOT in validate.py β live LLM cost)
|
| 130 |
+
'note': 'Ask the business a question in plain language: a small AI model calls governed tools over the semantic layer β answers carry their query trace and drill links. AI-generated output (Art. 50 labeled).'},
|
| 131 |
+
]
|
| 132 |
+
|
| 133 |
+
BY_KEY = {m['key']: m for m in REGISTRY}
|
| 134 |
+
|
| 135 |
+
|
| 136 |
+
def brand_filterable(key) -> bool:
|
| 137 |
+
return bool(BY_KEY.get(key, {}).get('brand'))
|
royalimports_os/core/store.py
ADDED
|
@@ -0,0 +1,307 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Tiny JSON persistence on a PRIVATE Hugging Face Dataset repo β the platform's ONLY writable store
|
| 2 |
+
(Odoo stays strictly read-only). Holds the user registry, customer notes and per-user tags.
|
| 3 |
+
|
| 4 |
+
Auth uses HF_TOKEN (a Space secret / .env value with write access to the royal-imports org). The
|
| 5 |
+
repo is created private on first write. Reads use a short in-process cache that writes refresh, so
|
| 6 |
+
within a running instance reads are consistent with writes; a fresh instance re-fetches the latest.
|
| 7 |
+
"""
|
| 8 |
+
import io
|
| 9 |
+
import os
|
| 10 |
+
import json
|
| 11 |
+
import threading
|
| 12 |
+
import time as _time
|
| 13 |
+
from pathlib import Path
|
| 14 |
+
|
| 15 |
+
try:
|
| 16 |
+
from dotenv import load_dotenv
|
| 17 |
+
load_dotenv(Path(__file__).resolve().parents[1] / '.env')
|
| 18 |
+
except Exception:
|
| 19 |
+
pass
|
| 20 |
+
|
| 21 |
+
from huggingface_hub import HfApi, hf_hub_download
|
| 22 |
+
from huggingface_hub.utils import EntryNotFoundError, RepositoryNotFoundError
|
| 23 |
+
|
| 24 |
+
REPO = os.environ.get('OS_DATA_REPO', 'royal-imports/cfo-os-data')
|
| 25 |
+
_LOCK = threading.RLock()
|
| 26 |
+
_CACHE = {}
|
| 27 |
+
# --- async flush state (wave-7 W3, 2026-07-28) --------------------------------------------
|
| 28 |
+
# One typed character in a Notes cell used to cost THREE synchronous network calls inside the
|
| 29 |
+
# component round-trip (strict re-download + create_repo + full re-upload = one hub commit per
|
| 30 |
+
# edit β measured seconds each, and a commit-budget hazard). update(..., flush='async') now
|
| 31 |
+
# applies the mutation to the in-process cache synchronously (read-your-writes: every reader
|
| 32 |
+
# already consults _CACHE first) and a background worker coalesces uploads. The restart-wipe
|
| 33 |
+
# protection is UNCHANGED where it matters: the FIRST update of a key in a fresh process still
|
| 34 |
+
# strict-reads (a transient read error aborts the write, never merges into {}); after that the
|
| 35 |
+
# cache we last wrote IS the latest known state, so re-downloading it per keystroke bought
|
| 36 |
+
# nothing but latency. Known, accepted: with a second concurrent writer (local QA against the
|
| 37 |
+
# shared store) the stale window widens from ~one round-trip to ~the flush gap.
|
| 38 |
+
_OWNED = {} # name -> True once _CACHE[name] reflects our own last accepted write
|
| 39 |
+
_DIRTY = {} # name -> cache is ahead of the hub
|
| 40 |
+
_FLUSHERS = {} # name -> live worker thread
|
| 41 |
+
_REPO_OK = [False] # create_repo(exist_ok=True) confirmed once per process
|
| 42 |
+
_FLUSH_DELAY = 2.0 # coalesce a burst of edits into one commit
|
| 43 |
+
_FLUSH_MIN_GAP = 20.0 # floor between commits of one key (the 256-commits/hr budget)
|
| 44 |
+
_LAST_FLUSH = {} # name -> monotonic time of the last completed upload
|
| 45 |
+
|
| 46 |
+
|
| 47 |
+
def _token():
|
| 48 |
+
return os.environ.get('HF_TOKEN') or None
|
| 49 |
+
|
| 50 |
+
|
| 51 |
+
def available():
|
| 52 |
+
"""True when a token is configured β callers degrade gracefully (read-only UI) without one."""
|
| 53 |
+
return bool(_token())
|
| 54 |
+
|
| 55 |
+
|
| 56 |
+
def _download(name):
|
| 57 |
+
"""Parse {name}.json into a dict. Returns {} ONLY when the file is genuinely absent. Any
|
| 58 |
+
other failure (network, auth, rate-limit, corrupt/partial JSON) RAISES β a transient read
|
| 59 |
+
error must never be mistaken for an empty store, or a caller could overwrite real data with
|
| 60 |
+
an empty dict (this is what wiped the user registry on restart)."""
|
| 61 |
+
try:
|
| 62 |
+
path = hf_hub_download(REPO, f'{name}.json', repo_type='dataset', token=_token(),
|
| 63 |
+
force_download=True)
|
| 64 |
+
with open(path, encoding='utf-8') as f:
|
| 65 |
+
return json.load(f)
|
| 66 |
+
except (EntryNotFoundError, RepositoryNotFoundError, FileNotFoundError):
|
| 67 |
+
return {}
|
| 68 |
+
|
| 69 |
+
|
| 70 |
+
def get(name, fresh=False):
|
| 71 |
+
"""Load {name}.json as a dict. LENIENT (for display reads): on a transient fetch error, hand
|
| 72 |
+
back the last known-good cached value if we have one, else {}. Callers that MUTATE the store
|
| 73 |
+
must go through update()/_read_strict, which RAISE on such errors so they never clobber real
|
| 74 |
+
data with a phantom-empty read."""
|
| 75 |
+
with _LOCK:
|
| 76 |
+
if not fresh and name in _CACHE:
|
| 77 |
+
return json.loads(json.dumps(_CACHE[name])) # hand back a copy
|
| 78 |
+
try:
|
| 79 |
+
data = _download(name)
|
| 80 |
+
except Exception as e:
|
| 81 |
+
try: # lazy import β core stays layer-clean
|
| 82 |
+
import harness.telemetry as _tel # noqa: PLC0415
|
| 83 |
+
_tel.error(f'store:get:{name}', e,
|
| 84 |
+
fallback='cached' if name in _CACHE else 'empty')
|
| 85 |
+
except Exception:
|
| 86 |
+
pass
|
| 87 |
+
if name in _CACHE:
|
| 88 |
+
return json.loads(json.dumps(_CACHE[name]))
|
| 89 |
+
return {}
|
| 90 |
+
_CACHE[name] = data
|
| 91 |
+
return json.loads(json.dumps(data))
|
| 92 |
+
|
| 93 |
+
|
| 94 |
+
def _read_strict(name):
|
| 95 |
+
"""Like get(fresh=True) but RAISES on any transient read error ({} only for a genuinely
|
| 96 |
+
absent file). Read-modify-write callers use this so a failed read ABORTS the write instead
|
| 97 |
+
of merging changes into an empty dict and overwriting the store."""
|
| 98 |
+
with _LOCK:
|
| 99 |
+
data = _download(name)
|
| 100 |
+
_CACHE[name] = data
|
| 101 |
+
return json.loads(json.dumps(data))
|
| 102 |
+
|
| 103 |
+
|
| 104 |
+
def exists(name):
|
| 105 |
+
"""True if {name}.json is confirmed present, False if confirmed absent, and True on ANY
|
| 106 |
+
uncertainty β so a caller that would seed or overwrite the store never clobbers a registry
|
| 107 |
+
we merely failed to reach."""
|
| 108 |
+
try:
|
| 109 |
+
files = HfApi(token=_token()).list_repo_files(REPO, repo_type='dataset')
|
| 110 |
+
return f'{name}.json' in files
|
| 111 |
+
except RepositoryNotFoundError:
|
| 112 |
+
return False
|
| 113 |
+
except Exception:
|
| 114 |
+
return True
|
| 115 |
+
|
| 116 |
+
|
| 117 |
+
def _upload(name, data):
|
| 118 |
+
"""The raw hub write. create_repo is confirmed ONCE per process β it was a full extra API
|
| 119 |
+
round-trip on every put(), pure latency after the first call ever succeeded."""
|
| 120 |
+
api = HfApi(token=_token())
|
| 121 |
+
if not _REPO_OK[0]:
|
| 122 |
+
api.create_repo(REPO, repo_type='dataset', private=True, exist_ok=True)
|
| 123 |
+
_REPO_OK[0] = True
|
| 124 |
+
buf = io.BytesIO(json.dumps(data, ensure_ascii=False, indent=2).encode('utf-8'))
|
| 125 |
+
api.upload_file(path_or_fileobj=buf, path_in_repo=f'{name}.json', repo_id=REPO,
|
| 126 |
+
repo_type='dataset', commit_message=f'update {name}')
|
| 127 |
+
|
| 128 |
+
|
| 129 |
+
def upload_bytes(path_in_repo, data, message=None):
|
| 130 |
+
"""Persist RAW BYTES at an arbitrary path in the tenant dataset (wave-8 C5, documents).
|
| 131 |
+
|
| 132 |
+
The rest of this module is JSON-per-key; a customer's attachment is neither JSON nor
|
| 133 |
+
per-key, so it gets its own path rather than being base64'd into somebody's blob. That
|
| 134 |
+
matters for a reason beyond tidiness: the workspace blob is READ ON EVERY PAGE RENDER, and
|
| 135 |
+
a few megabytes of base64 inside it would be paid on every keystroke by every user of that
|
| 136 |
+
table, not just by whoever opened the document.
|
| 137 |
+
|
| 138 |
+
Bytes are never cached in-process: they are large, cold, and fetched one at a time on an
|
| 139 |
+
explicit user action.
|
| 140 |
+
"""
|
| 141 |
+
if not available():
|
| 142 |
+
raise RuntimeError('No HF_TOKEN configured β persistence is unavailable.')
|
| 143 |
+
with _LOCK:
|
| 144 |
+
api = HfApi(token=_token())
|
| 145 |
+
if not _REPO_OK[0]:
|
| 146 |
+
api.create_repo(REPO, repo_type='dataset', private=True, exist_ok=True)
|
| 147 |
+
_REPO_OK[0] = True
|
| 148 |
+
api.upload_file(path_or_fileobj=io.BytesIO(data), path_in_repo=path_in_repo,
|
| 149 |
+
repo_id=REPO, repo_type='dataset',
|
| 150 |
+
commit_message=message or f'add {path_in_repo}')
|
| 151 |
+
|
| 152 |
+
|
| 153 |
+
def download_bytes(path_in_repo):
|
| 154 |
+
"""Raw bytes back, or None when absent/unreachable.
|
| 155 |
+
|
| 156 |
+
None rather than an exception: a missing attachment must degrade to "that file is gone",
|
| 157 |
+
never to a broken page β the same contract every other read here follows.
|
| 158 |
+
"""
|
| 159 |
+
if not available():
|
| 160 |
+
return None
|
| 161 |
+
try:
|
| 162 |
+
local = hf_hub_download(REPO, path_in_repo, repo_type='dataset', token=_token())
|
| 163 |
+
return Path(local).read_bytes()
|
| 164 |
+
except (EntryNotFoundError, RepositoryNotFoundError):
|
| 165 |
+
return None
|
| 166 |
+
except Exception:
|
| 167 |
+
return None
|
| 168 |
+
|
| 169 |
+
|
| 170 |
+
def delete_path(path_in_repo):
|
| 171 |
+
"""Remove a stored file. Absent is SUCCESS β deleting what is already gone is the goal."""
|
| 172 |
+
if not available():
|
| 173 |
+
return False
|
| 174 |
+
try:
|
| 175 |
+
HfApi(token=_token()).delete_file(path_in_repo=path_in_repo, repo_id=REPO,
|
| 176 |
+
repo_type='dataset',
|
| 177 |
+
commit_message=f'delete {path_in_repo}')
|
| 178 |
+
return True
|
| 179 |
+
except (EntryNotFoundError, RepositoryNotFoundError):
|
| 180 |
+
return True
|
| 181 |
+
except Exception:
|
| 182 |
+
return False
|
| 183 |
+
|
| 184 |
+
|
| 185 |
+
def put(name, data):
|
| 186 |
+
"""Persist {name}.json (creates the private repo on first write). Updates the in-process cache."""
|
| 187 |
+
if not available():
|
| 188 |
+
raise RuntimeError('No HF_TOKEN configured β persistence is unavailable.')
|
| 189 |
+
with _LOCK:
|
| 190 |
+
_upload(name, data)
|
| 191 |
+
_CACHE[name] = json.loads(json.dumps(data))
|
| 192 |
+
_OWNED[name] = True
|
| 193 |
+
_LAST_FLUSH[name] = _time.monotonic()
|
| 194 |
+
return _CACHE[name]
|
| 195 |
+
|
| 196 |
+
|
| 197 |
+
def _flush_worker(name):
|
| 198 |
+
"""Coalescing uploader for one key. Runs while dirty; retries with backoff on failure β
|
| 199 |
+
the cache stays the truth and the dirty mark survives, so nothing is ever lost silently."""
|
| 200 |
+
backoff = 2.0
|
| 201 |
+
while True:
|
| 202 |
+
_time.sleep(_FLUSH_DELAY)
|
| 203 |
+
with _LOCK:
|
| 204 |
+
gap = _time.monotonic() - _LAST_FLUSH.get(name, 0.0)
|
| 205 |
+
if gap < _FLUSH_MIN_GAP:
|
| 206 |
+
wait = _FLUSH_MIN_GAP - gap
|
| 207 |
+
elif not _DIRTY.get(name):
|
| 208 |
+
_FLUSHERS.pop(name, None)
|
| 209 |
+
return
|
| 210 |
+
else:
|
| 211 |
+
_DIRTY[name] = False
|
| 212 |
+
snapshot = json.loads(json.dumps(_CACHE.get(name, {})))
|
| 213 |
+
wait = None
|
| 214 |
+
if wait is not None:
|
| 215 |
+
_time.sleep(wait)
|
| 216 |
+
continue
|
| 217 |
+
try:
|
| 218 |
+
_upload(name, snapshot)
|
| 219 |
+
with _LOCK:
|
| 220 |
+
_LAST_FLUSH[name] = _time.monotonic()
|
| 221 |
+
backoff = 2.0
|
| 222 |
+
except Exception as e:
|
| 223 |
+
with _LOCK:
|
| 224 |
+
_DIRTY[name] = True # keep it queued; cache is still the truth
|
| 225 |
+
try:
|
| 226 |
+
import harness.telemetry as _tel # noqa: PLC0415
|
| 227 |
+
_tel.error(f'store:flush:{name}', e)
|
| 228 |
+
except Exception:
|
| 229 |
+
pass
|
| 230 |
+
_time.sleep(backoff)
|
| 231 |
+
backoff = min(backoff * 2, 60.0)
|
| 232 |
+
|
| 233 |
+
|
| 234 |
+
def _schedule_flush(name):
|
| 235 |
+
"""Mark dirty + ensure exactly one live worker for the key. Caller holds _LOCK."""
|
| 236 |
+
_DIRTY[name] = True
|
| 237 |
+
t = _FLUSHERS.get(name)
|
| 238 |
+
if t is not None and t.is_alive():
|
| 239 |
+
return
|
| 240 |
+
t = threading.Thread(target=_flush_worker, args=(name,), daemon=True,
|
| 241 |
+
name=f'store-flush-{name}')
|
| 242 |
+
_FLUSHERS[name] = t
|
| 243 |
+
t.start()
|
| 244 |
+
|
| 245 |
+
|
| 246 |
+
def flush(name=None, timeout=30.0):
|
| 247 |
+
"""Block until pending async writes for `name` (or ALL keys) reach the hub β QA gates and
|
| 248 |
+
shutdown hooks use this; the app itself never needs to."""
|
| 249 |
+
deadline = _time.monotonic() + timeout
|
| 250 |
+
names = [name] if name else None
|
| 251 |
+
while _time.monotonic() < deadline:
|
| 252 |
+
with _LOCK:
|
| 253 |
+
pending = {k for k, v in _DIRTY.items() if v} | {
|
| 254 |
+
k for k, t in _FLUSHERS.items() if t.is_alive()}
|
| 255 |
+
if names is not None:
|
| 256 |
+
pending &= set(names)
|
| 257 |
+
if not pending:
|
| 258 |
+
return True
|
| 259 |
+
_time.sleep(0.2)
|
| 260 |
+
return False
|
| 261 |
+
|
| 262 |
+
|
| 263 |
+
def _flush_now_at_exit():
|
| 264 |
+
"""Process-exit safety net: push any dirty keys synchronously, bypassing the coalescing
|
| 265 |
+
gap β a Space restart must not eat the last seconds of the async window."""
|
| 266 |
+
with _LOCK:
|
| 267 |
+
dirty = [k for k, v in _DIRTY.items() if v]
|
| 268 |
+
snaps = {k: json.loads(json.dumps(_CACHE.get(k, {}))) for k in dirty}
|
| 269 |
+
for k in dirty:
|
| 270 |
+
_DIRTY[k] = False
|
| 271 |
+
for k, d in snaps.items():
|
| 272 |
+
try:
|
| 273 |
+
_upload(k, d)
|
| 274 |
+
except Exception:
|
| 275 |
+
pass # exiting anyway β nothing left to reschedule
|
| 276 |
+
|
| 277 |
+
|
| 278 |
+
import atexit # noqa: E402 (registered after the helpers it needs)
|
| 279 |
+
atexit.register(_flush_now_at_exit)
|
| 280 |
+
|
| 281 |
+
|
| 282 |
+
def update(name, fn, flush='sync'):
|
| 283 |
+
"""Read-modify-write. STRICT first read (raises on a transient error rather than starting
|
| 284 |
+
from an empty dict β a network blip during 'add user' can never wipe the accounts), apply
|
| 285 |
+
fn(data)->data, persist.
|
| 286 |
+
|
| 287 |
+
flush='sync' (default) = exactly the historical behavior: fresh strict read + blocking
|
| 288 |
+
upload. flush='async' (the table-workspace hot path, wave-7 W3) = the mutation applies to
|
| 289 |
+
the in-process cache and returns immediately; a coalescing worker commits to the hub. The
|
| 290 |
+
strict read still guards the FIRST async update of a key per process; after we own the
|
| 291 |
+
cache, it IS the latest known state and re-downloading it per edit was pure latency."""
|
| 292 |
+
with _LOCK:
|
| 293 |
+
if flush == 'async' and _OWNED.get(name) and name in _CACHE:
|
| 294 |
+
data = json.loads(json.dumps(_CACHE[name]))
|
| 295 |
+
else:
|
| 296 |
+
data = _read_strict(name)
|
| 297 |
+
result = fn(data)
|
| 298 |
+
data = result if result is not None else data
|
| 299 |
+
if flush == 'async':
|
| 300 |
+
if not available():
|
| 301 |
+
raise RuntimeError('No HF_TOKEN configured β persistence is unavailable.')
|
| 302 |
+
_CACHE[name] = json.loads(json.dumps(data))
|
| 303 |
+
_OWNED[name] = True
|
| 304 |
+
_schedule_flush(name)
|
| 305 |
+
return data
|
| 306 |
+
put(name, data)
|
| 307 |
+
return data
|
royalimports_os/core/store_backend.py
ADDED
|
@@ -0,0 +1,85 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""core/store_backend.py β which store is active: `STORE_BACKEND=hf|pg` (X4 / EXIT-2b).
|
| 2 |
+
|
| 3 |
+
import core.store_backend as store # new code
|
| 4 |
+
store.get('users') # goes to whichever backend is selected
|
| 5 |
+
|
| 6 |
+
β WHAT THIS DOES **NOT** DO, stated first because assuming otherwise would be a silent data
|
| 7 |
+
split. Flipping `STORE_BACKEND=pg` does NOT redirect the ~40 existing callers that say
|
| 8 |
+
`import core.store as store` β they are bound to the HF module directly. Rewiring them is task
|
| 9 |
+
**C-4** (the HFβPostgres cutover with its dual-read window), and C-4 cannot start until the
|
| 10 |
+
database exists (owner blocker **B-3**). Until then this selector is the SEAM, exercised by new
|
| 11 |
+
code and by the gate, and the default is `hf`, which is exactly today's behaviour.
|
| 12 |
+
|
| 13 |
+
Doing it any other way would be worse. The tempting shortcut β have `core/store.py` itself
|
| 14 |
+
delegate β would mean every existing caller silently changes backend the moment an env var is set
|
| 15 |
+
in some shell, with no dual-read window and no way to compare the two stores' contents first.
|
| 16 |
+
"One store" is the goal; getting there through a flag nobody audited would produce two.
|
| 17 |
+
|
| 18 |
+
THE DEFAULT IS `hf` AND STAYS `hf` until C-4 says otherwise. An unrecognised value is a
|
| 19 |
+
configuration error and RAISES rather than falling back β a typo'd backend name that quietly
|
| 20 |
+
served the old store would be discovered by data going missing.
|
| 21 |
+
"""
|
| 22 |
+
import os
|
| 23 |
+
|
| 24 |
+
import core.store as _hf
|
| 25 |
+
|
| 26 |
+
_NAMES = ('available', 'get', 'exists', 'put', 'update', 'upload_bytes', 'download_bytes',
|
| 27 |
+
'delete_path', 'flush')
|
| 28 |
+
|
| 29 |
+
|
| 30 |
+
def name():
|
| 31 |
+
"""The selected backend name, validated. `hf` unless explicitly told otherwise."""
|
| 32 |
+
raw = (os.environ.get('STORE_BACKEND') or 'hf').strip().lower()
|
| 33 |
+
if raw not in ('hf', 'pg'):
|
| 34 |
+
raise RuntimeError(
|
| 35 |
+
f"STORE_BACKEND={raw!r} is not a backend. Use 'hf' (the HF Dataset store, the "
|
| 36 |
+
f"default) or 'pg' (Postgres, needs DATABASE_URL). Refusing to guess: a typo that "
|
| 37 |
+
f"silently served the other store is how data ends up in two places.")
|
| 38 |
+
return raw
|
| 39 |
+
|
| 40 |
+
|
| 41 |
+
def active():
|
| 42 |
+
"""The backend MODULE. Resolved per call, so a test can flip the env var without a reimport."""
|
| 43 |
+
if name() == 'pg':
|
| 44 |
+
import core.store_pg as _pg
|
| 45 |
+
return _pg
|
| 46 |
+
return _hf
|
| 47 |
+
|
| 48 |
+
|
| 49 |
+
# --- the nine delegates. Written out rather than generated with setattr so that reading this file
|
| 50 |
+
# tells you the whole interface, and so a new store operation cannot be added to core/store.py
|
| 51 |
+
# without something here failing to match (verify_store_pg.py compares the signatures).
|
| 52 |
+
def available():
|
| 53 |
+
return active().available()
|
| 54 |
+
|
| 55 |
+
|
| 56 |
+
def get(name_, fresh=False):
|
| 57 |
+
return active().get(name_, fresh=fresh)
|
| 58 |
+
|
| 59 |
+
|
| 60 |
+
def exists(name_):
|
| 61 |
+
return active().exists(name_)
|
| 62 |
+
|
| 63 |
+
|
| 64 |
+
def put(name_, data):
|
| 65 |
+
return active().put(name_, data)
|
| 66 |
+
|
| 67 |
+
|
| 68 |
+
def update(name_, fn, flush='sync'):
|
| 69 |
+
return active().update(name_, fn, flush=flush)
|
| 70 |
+
|
| 71 |
+
|
| 72 |
+
def upload_bytes(path_in_repo, data, message=None):
|
| 73 |
+
return active().upload_bytes(path_in_repo, data, message=message)
|
| 74 |
+
|
| 75 |
+
|
| 76 |
+
def download_bytes(path_in_repo):
|
| 77 |
+
return active().download_bytes(path_in_repo)
|
| 78 |
+
|
| 79 |
+
|
| 80 |
+
def delete_path(path_in_repo):
|
| 81 |
+
return active().delete_path(path_in_repo)
|
| 82 |
+
|
| 83 |
+
|
| 84 |
+
def flush(name_=None, timeout=30.0):
|
| 85 |
+
return active().flush(name_, timeout=timeout)
|
royalimports_os/core/store_pg.py
ADDED
|
@@ -0,0 +1,249 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""core/store_pg.py β `core/store.py`'s interface, backed by Postgres (X4 / EXIT-2b, 2026-07-30).
|
| 2 |
+
|
| 3 |
+
THE INTERFACE IS THE CONTRACT, and it is `core/store.py`'s nine functions:
|
| 4 |
+
|
| 5 |
+
available() Β· get(name, fresh=False) Β· exists(name) Β· put(name, data)
|
| 6 |
+
update(name, fn, flush='sync') Β· upload_bytes(path, data, message=None)
|
| 7 |
+
download_bytes(path) Β· delete_path(path) Β· flush(name=None, timeout=30.0)
|
| 8 |
+
|
| 9 |
+
Same names, same signatures, same return types, same failure semantics β so switching backends is
|
| 10 |
+
an env var (`STORE_BACKEND=hf|pg`) and not a rewrite of every caller. `core/store_backend.py`
|
| 11 |
+
does the selection; nothing above it needs to know which store it is talking to.
|
| 12 |
+
|
| 13 |
+
β NOT VERIFIED AGAINST A REAL SERVER β owner blocker **B-3** (choose + provision managed
|
| 14 |
+
Postgres, US-region, co-located with Ashburn). `verify_store_pg.py` is written and it SKIPS LOUDLY
|
| 15 |
+
without `DATABASE_URL`; it is not reported as passing, and this module is not the default backend.
|
| 16 |
+
The schema is `harness/pg/schema.sql`. Sequenced around the blocker, never stubbed past it.
|
| 17 |
+
|
| 18 |
+
WHAT POSTGRES FIXES, precisely (C1c β these are the reasons, not a preference):
|
| 19 |
+
* **`get()` is cache-first on the HF store**, so a write from another process is invisible to a
|
| 20 |
+
running app until restart. Observed live in wave 11. Here a read is a SELECT: cross-process
|
| 21 |
+
coherence is the default rather than a thing to remember.
|
| 22 |
+
* **`update()` was read-modify-write with no concurrency control** β two writers raced and the
|
| 23 |
+
loser vanished silently. Here it runs inside ONE transaction with `SELECT β¦ FOR UPDATE`, so
|
| 24 |
+
the second writer waits and then sees the first writer's value.
|
| 25 |
+
* **A write was an HTTP commit** against a 256-commits/hour repo budget, which is why the hot
|
| 26 |
+
path needed a coalescing async flusher at all. A Postgres write is a write; `flush='async'`
|
| 27 |
+
stays in the signature and becomes a no-op, because there is nothing to coalesce.
|
| 28 |
+
|
| 29 |
+
β ONE DELIBERATE DIFFERENCE FROM THE HF BACKEND, and it is a fix. `store.get()` swallows transient
|
| 30 |
+
read failures and hands back a cached-or-empty dict, because a display read must not break a page.
|
| 31 |
+
That leniency is exactly what wiped the user registry once (an empty read merged into a write), and
|
| 32 |
+
`_read_strict` exists to opt out of it. Postgres has no equivalent "the network blipped" state
|
| 33 |
+
worth hiding: a connection failure RAISES from `get()` here. Callers that must not break on a read
|
| 34 |
+
should catch it β silently returning `{}` from a database that is simply unreachable is how an
|
| 35 |
+
empty dict gets written back over real data.
|
| 36 |
+
"""
|
| 37 |
+
import json
|
| 38 |
+
import os
|
| 39 |
+
import threading
|
| 40 |
+
|
| 41 |
+
_URL_ENV = 'DATABASE_URL'
|
| 42 |
+
_LOCK = threading.RLock()
|
| 43 |
+
_POOL = {'pool': None, 'url': None, 'schema': None}
|
| 44 |
+
|
| 45 |
+
|
| 46 |
+
def _url():
|
| 47 |
+
return os.environ.get(_URL_ENV) or None
|
| 48 |
+
|
| 49 |
+
|
| 50 |
+
def _schema_for(tenant_slug=None):
|
| 51 |
+
"""`t_<slug>` with `-` normalised to `_` (a hyphen is not legal in an unquoted identifier).
|
| 52 |
+
|
| 53 |
+
Mirrors `control.provision_tenant_schema` exactly β the two MUST agree or a write lands in a
|
| 54 |
+
schema the DDL never created. `AIOS_TENANT` names the tenant this process serves; it defaults
|
| 55 |
+
to tenant #0 so the default behaviour is unchanged, the same rule `harness.datastore.path_for`
|
| 56 |
+
follows for the DuckDB filename.
|
| 57 |
+
"""
|
| 58 |
+
slug = (tenant_slug or os.environ.get('AIOS_TENANT') or 'royal-imports').strip().lower()
|
| 59 |
+
return 't_' + slug.replace('-', '_')
|
| 60 |
+
|
| 61 |
+
|
| 62 |
+
def _pool():
|
| 63 |
+
"""The psycopg connection pool, opened once per process.
|
| 64 |
+
|
| 65 |
+
psycopg is imported LAZILY and only here: `STORE_BACKEND=hf` must not require the dependency
|
| 66 |
+
to be installed at all, which is what keeps this file safe to commit before B-3 unblocks.
|
| 67 |
+
"""
|
| 68 |
+
with _LOCK:
|
| 69 |
+
url = _url()
|
| 70 |
+
if not url:
|
| 71 |
+
raise RuntimeError(
|
| 72 |
+
f'{_URL_ENV} is not set β the Postgres store backend has nothing to connect to.')
|
| 73 |
+
if _POOL['pool'] is not None and _POOL['url'] == url:
|
| 74 |
+
return _POOL['pool']
|
| 75 |
+
try:
|
| 76 |
+
from psycopg_pool import ConnectionPool
|
| 77 |
+
except ImportError as e: # pragma: no cover - depends on B-3
|
| 78 |
+
raise RuntimeError(
|
| 79 |
+
'STORE_BACKEND=pg needs psycopg[binary,pool]. It is deliberately NOT in '
|
| 80 |
+
'requirements.txt until the database exists (owner blocker B-3), so that the '
|
| 81 |
+
'default hf backend installs nothing it does not use.') from e
|
| 82 |
+
if _POOL['pool'] is not None:
|
| 83 |
+
try:
|
| 84 |
+
_POOL['pool'].close()
|
| 85 |
+
except Exception:
|
| 86 |
+
pass
|
| 87 |
+
# min_size=0 so a process that never touches the store opens no connection at all β the
|
| 88 |
+
# shared API serves plenty of requests (health, static, a cached payload) that never read.
|
| 89 |
+
_POOL['pool'] = ConnectionPool(url, min_size=0, max_size=8, open=True, timeout=10.0)
|
| 90 |
+
_POOL['url'] = url
|
| 91 |
+
return _POOL['pool']
|
| 92 |
+
|
| 93 |
+
|
| 94 |
+
def available():
|
| 95 |
+
"""True when a connection can actually be MADE, not merely when a URL is configured.
|
| 96 |
+
|
| 97 |
+
`core.store.available()` only checks for a token, which is all it can cheaply do. Here the
|
| 98 |
+
honest answer needs a round-trip, so the result is memoised per process: callers use this to
|
| 99 |
+
decide whether to degrade the UI, and a `SELECT 1` per render would be absurd.
|
| 100 |
+
"""
|
| 101 |
+
if not _url():
|
| 102 |
+
return False
|
| 103 |
+
if _POOL.get('ok'):
|
| 104 |
+
return True
|
| 105 |
+
try:
|
| 106 |
+
with _pool().connection() as con:
|
| 107 |
+
con.execute('SELECT 1')
|
| 108 |
+
_POOL['ok'] = True
|
| 109 |
+
return True
|
| 110 |
+
except Exception:
|
| 111 |
+
return False
|
| 112 |
+
|
| 113 |
+
|
| 114 |
+
def _table(kind, tenant_slug=None):
|
| 115 |
+
"""`"t_slug"."store_kv"` β the qualified table, safely quoted."""
|
| 116 |
+
from psycopg import sql
|
| 117 |
+
return sql.SQL('{}.{}').format(sql.Identifier(_schema_for(tenant_slug)),
|
| 118 |
+
sql.Identifier(kind))
|
| 119 |
+
|
| 120 |
+
|
| 121 |
+
def _q(template, kind, tenant_slug=None):
|
| 122 |
+
"""Compose `template` with `{tbl}` = the QUALIFIED table and `{bare}` = the table name alone.
|
| 123 |
+
|
| 124 |
+
β THE TWO ARE NOT INTERCHANGEABLE, and getting it wrong is a runtime syntax error nothing
|
| 125 |
+
local would catch (no database here β owner blocker B-3). Inside `ON CONFLICT β¦ DO UPDATE`,
|
| 126 |
+
Postgres refers to the conflicting row by the table's ALIAS, which defaults to the bare
|
| 127 |
+
table name: `store_kv.rev` is valid, `t_royal_imports.store_kv.rev` is not. So the FROM
|
| 128 |
+
position takes `{tbl}` and the DO-UPDATE position takes `{bare}`.
|
| 129 |
+
|
| 130 |
+
Identifiers go through `psycopg.sql.Identifier`, so the schema name β derived from a tenant
|
| 131 |
+
slug β cannot be an injection vector even though it is interpolated. The slug is ALSO
|
| 132 |
+
regex-constrained at its source (`control.tenants`), which is the belt to this brace.
|
| 133 |
+
"""
|
| 134 |
+
from psycopg import sql
|
| 135 |
+
return sql.SQL(template).format(tbl=_table(kind, tenant_slug),
|
| 136 |
+
bare=sql.Identifier(kind))
|
| 137 |
+
|
| 138 |
+
|
| 139 |
+
def get(name, fresh=False, tenant_slug=None):
|
| 140 |
+
"""The value stored under `name`, or `{}` when the key is genuinely absent.
|
| 141 |
+
|
| 142 |
+
`fresh` is accepted and IGNORED: it exists in the HF signature to bypass a process cache, and
|
| 143 |
+
there is no cache here β every read is a SELECT. Keeping the parameter means callers do not
|
| 144 |
+
branch on the backend.
|
| 145 |
+
"""
|
| 146 |
+
with _pool().connection() as con:
|
| 147 |
+
row = con.execute(
|
| 148 |
+
_q('SELECT value FROM {tbl} WHERE key = %s', 'store_kv', tenant_slug),
|
| 149 |
+
(str(name),)).fetchone()
|
| 150 |
+
return dict(row[0]) if row and isinstance(row[0], dict) else (row[0] if row else {})
|
| 151 |
+
|
| 152 |
+
|
| 153 |
+
def exists(name, tenant_slug=None):
|
| 154 |
+
"""True/False β and unlike the HF backend this is NEVER "True on uncertainty".
|
| 155 |
+
|
| 156 |
+
`core.store.exists` returns True on any error so a caller that would seed the store cannot
|
| 157 |
+
clobber a registry it merely failed to reach. That guard exists because a FILE HOST cannot
|
| 158 |
+
distinguish "absent" from "unreachable". Postgres can: an absent row is a fact, and an
|
| 159 |
+
unreachable server raises instead of answering.
|
| 160 |
+
"""
|
| 161 |
+
with _pool().connection() as con:
|
| 162 |
+
row = con.execute(
|
| 163 |
+
_q('SELECT 1 FROM {tbl} WHERE key = %s', 'store_kv', tenant_slug),
|
| 164 |
+
(str(name),)).fetchone()
|
| 165 |
+
return row is not None
|
| 166 |
+
|
| 167 |
+
|
| 168 |
+
def put(name, data, tenant_slug=None):
|
| 169 |
+
with _pool().connection() as con:
|
| 170 |
+
con.execute(
|
| 171 |
+
_q('INSERT INTO {tbl} (key, value) VALUES (%s, %s::jsonb) '
|
| 172 |
+
'ON CONFLICT (key) DO UPDATE SET value = EXCLUDED.value, '
|
| 173 |
+
'updated_at = now(), rev = {bare}.rev + 1', 'store_kv', tenant_slug),
|
| 174 |
+
(str(name), json.dumps(data)))
|
| 175 |
+
return data
|
| 176 |
+
|
| 177 |
+
|
| 178 |
+
def update(name, fn, flush='sync', tenant_slug=None):
|
| 179 |
+
"""Read-modify-write inside ONE transaction, with the row LOCKED for the duration.
|
| 180 |
+
|
| 181 |
+
This is the operation the HF backend could not make safe. There, `update` read over HTTP,
|
| 182 |
+
applied `fn`, and uploaded β so two concurrent writers both read the old value and the second
|
| 183 |
+
upload silently discarded the first writer's change. `SELECT β¦ FOR UPDATE` makes the second
|
| 184 |
+
writer WAIT and then apply `fn` to the first writer's result, which is what "read-modify-write"
|
| 185 |
+
was always supposed to mean.
|
| 186 |
+
|
| 187 |
+
`flush` is accepted and ignored (see the module docstring): there is no commit budget to
|
| 188 |
+
coalesce against, so 'async' has nothing to defer.
|
| 189 |
+
"""
|
| 190 |
+
with _pool().connection() as con:
|
| 191 |
+
with con.transaction():
|
| 192 |
+
row = con.execute(
|
| 193 |
+
_q('SELECT value FROM {tbl} WHERE key = %s FOR UPDATE',
|
| 194 |
+
'store_kv', tenant_slug),
|
| 195 |
+
(str(name),)).fetchone()
|
| 196 |
+
data = dict(row[0]) if row and isinstance(row[0], dict) else (row[0] if row else {})
|
| 197 |
+
result = fn(data)
|
| 198 |
+
data = result if result is not None else data
|
| 199 |
+
con.execute(
|
| 200 |
+
_q('INSERT INTO {tbl} (key, value) VALUES (%s, %s::jsonb) '
|
| 201 |
+
'ON CONFLICT (key) DO UPDATE SET value = EXCLUDED.value, '
|
| 202 |
+
'updated_at = now(), rev = {bare}.rev + 1', 'store_kv', tenant_slug),
|
| 203 |
+
(str(name), json.dumps(data)))
|
| 204 |
+
return data
|
| 205 |
+
|
| 206 |
+
|
| 207 |
+
def upload_bytes(path_in_repo, data, message=None, tenant_slug=None):
|
| 208 |
+
with _pool().connection() as con:
|
| 209 |
+
con.execute(
|
| 210 |
+
_q('INSERT INTO {tbl} (path, bytes, message) VALUES (%s, %s, %s) '
|
| 211 |
+
'ON CONFLICT (path) DO UPDATE SET bytes = EXCLUDED.bytes, '
|
| 212 |
+
'message = EXCLUDED.message, updated_at = now()', 'store_blobs', tenant_slug),
|
| 213 |
+
(str(path_in_repo), bytes(data), message))
|
| 214 |
+
|
| 215 |
+
|
| 216 |
+
def download_bytes(path_in_repo, tenant_slug=None):
|
| 217 |
+
"""Raw bytes, or None when absent β the same contract as the HF backend: a missing attachment
|
| 218 |
+
degrades to "that file is gone", never to a broken page."""
|
| 219 |
+
with _pool().connection() as con:
|
| 220 |
+
row = con.execute(
|
| 221 |
+
_q('SELECT bytes FROM {tbl} WHERE path = %s', 'store_blobs', tenant_slug),
|
| 222 |
+
(str(path_in_repo),)).fetchone()
|
| 223 |
+
return bytes(row[0]) if row else None
|
| 224 |
+
|
| 225 |
+
|
| 226 |
+
def delete_path(path_in_repo, tenant_slug=None):
|
| 227 |
+
"""Absent is SUCCESS β deleting what is already gone is the goal (HF backend's contract)."""
|
| 228 |
+
with _pool().connection() as con:
|
| 229 |
+
con.execute(_q('DELETE FROM {tbl} WHERE path = %s', 'store_blobs', tenant_slug),
|
| 230 |
+
(str(path_in_repo),))
|
| 231 |
+
return True
|
| 232 |
+
|
| 233 |
+
|
| 234 |
+
def flush(name=None, timeout=30.0):
|
| 235 |
+
"""A no-op that returns True: every write above is already committed when it returns. Kept in
|
| 236 |
+
the interface because QA gates and shutdown hooks call it and must not branch on the backend."""
|
| 237 |
+
return True
|
| 238 |
+
|
| 239 |
+
|
| 240 |
+
def close():
|
| 241 |
+
"""Release the pool (tests, and a process that is shutting down cleanly)."""
|
| 242 |
+
with _LOCK:
|
| 243 |
+
if _POOL['pool'] is not None:
|
| 244 |
+
try:
|
| 245 |
+
_POOL['pool'].close()
|
| 246 |
+
finally:
|
| 247 |
+
_POOL['pool'] = None
|
| 248 |
+
_POOL['url'] = None
|
| 249 |
+
_POOL.pop('ok', None)
|
royalimports_os/core/table_store.py
ADDED
|
@@ -0,0 +1,281 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""The generic per-user TABLE WORKSPACE store β the persistence half of the table-page factory.
|
| 2 |
+
|
| 3 |
+
One durable store key holds one table OBJECT's per-user Airtable-style state:
|
| 4 |
+
|
| 5 |
+
{username: {'views': {view_id: SavedView},
|
| 6 |
+
'fields': {field_key: Field}, # notes + custom_ + measure_ strata
|
| 7 |
+
'overlays': {str(pid): {field_key: value}}}}
|
| 8 |
+
|
| 9 |
+
`make(table_key)` returns the six operations a table page's host loop needs, closed over that
|
| 10 |
+
key. The Customer table's ops (`modules/customer_data.py`, key 'customer_table_workspace') are
|
| 11 |
+
these exact functions β the logic MOVED here 2026-07-27 so that duplicating the Customer table
|
| 12 |
+
pattern to a new object is a registry row + a config, not a copy of the store plumbing
|
| 13 |
+
(owner directive: the table-page factory).
|
| 14 |
+
|
| 15 |
+
A LIST (membership/formula semantics) is deliberately a different store from a VIEW
|
| 16 |
+
(presentation/query state) β see modules/customer_data.py's customer_lists key.
|
| 17 |
+
"""
|
| 18 |
+
import core.store as store
|
| 19 |
+
|
| 20 |
+
#: Wave-9 I17 β the SHARED bucket. Views whose permissions make them visible to anyone but
|
| 21 |
+
#: their creator live here instead of in a personal workspace, under a key that cannot collide
|
| 22 |
+
#: with a username (usernames come from core/users.py and are never dunder-wrapped; `is_shared`
|
| 23 |
+
#: guards it anyway). ONE HOME PER VIEW, never both: a view moved to personal is REMOVED from
|
| 24 |
+
#: here, and a view shared is removed from its creator's workspace. Two homes would mean two
|
| 25 |
+
#: divergent copies the moment either was edited.
|
| 26 |
+
SHARED_KEY = '__shared__'
|
| 27 |
+
|
| 28 |
+
|
| 29 |
+
def _may_see(view, viewer, is_admin=False):
|
| 30 |
+
"""Visibility for ONE shared view, fail-closed.
|
| 31 |
+
|
| 32 |
+
'collaborative' = everyone who can already open the module (the caller has gated that).
|
| 33 |
+
'users' = the named users, plus the creator, plus admins β an admin who could not
|
| 34 |
+
see a view could not administer it either.
|
| 35 |
+
Anything unrecognised returns False rather than defaulting open: an unreadable permission
|
| 36 |
+
must never widen access ([[aios-permissioning]] β no fail-open defaults).
|
| 37 |
+
"""
|
| 38 |
+
if not isinstance(view, dict):
|
| 39 |
+
return False
|
| 40 |
+
if view.get('createdBy') == viewer or is_admin:
|
| 41 |
+
return True
|
| 42 |
+
perms = view.get('permissions') or {}
|
| 43 |
+
edit = perms.get('edit')
|
| 44 |
+
if edit == 'collaborative':
|
| 45 |
+
return True
|
| 46 |
+
if edit == 'users':
|
| 47 |
+
return viewer in set(perms.get('users') or ())
|
| 48 |
+
return False # 'personal', absent, or junk
|
| 49 |
+
|
| 50 |
+
|
| 51 |
+
def _may_edit(view, viewer, is_admin=False):
|
| 52 |
+
"""Who may WRITE a shared view. Same set as visibility today β the owner's item asks 'who
|
| 53 |
+
can edit' and lists who 'can have access', i.e. seeing and editing are one grant. Kept as a
|
| 54 |
+
separate function so they can diverge (a future read-only share) without hunting callers."""
|
| 55 |
+
return _may_see(view, viewer, is_admin)
|
| 56 |
+
|
| 57 |
+
|
| 58 |
+
def _may_administer(view, viewer, is_admin=False):
|
| 59 |
+
"""Who may change a view's PERMISSIONS, or delete it: the creator or an admin ONLY.
|
| 60 |
+
|
| 61 |
+
Deliberately narrower than _may_edit. If a collaborator could rewrite `permissions` they
|
| 62 |
+
could grant themselves sole ownership of somebody else's view, or quietly widen a
|
| 63 |
+
users-scoped view to everyone β the classic privilege-escalation-by-edit hole.
|
| 64 |
+
"""
|
| 65 |
+
if not isinstance(view, dict):
|
| 66 |
+
return False
|
| 67 |
+
return bool(is_admin) or view.get('createdBy') == viewer
|
| 68 |
+
|
| 69 |
+
|
| 70 |
+
def is_shared(view):
|
| 71 |
+
"""A view belongs in the shared bucket when its permissions reach beyond its creator."""
|
| 72 |
+
return ((view or {}).get('permissions') or {}).get('edit') in ('collaborative', 'users')
|
| 73 |
+
|
| 74 |
+
|
| 75 |
+
class TableStore:
|
| 76 |
+
"""The six store operations for one table object's workspace, closed over its store key."""
|
| 77 |
+
|
| 78 |
+
def __init__(self, table_key):
|
| 79 |
+
self.table_key = table_key
|
| 80 |
+
|
| 81 |
+
# ---------------------------------------------------------------- read
|
| 82 |
+
def workspace(self, username):
|
| 83 |
+
"""One user's durable workspace: always the full three-strata shape."""
|
| 84 |
+
try:
|
| 85 |
+
ws = (store.get(self.table_key) or {}).get(username, {}) or {}
|
| 86 |
+
except Exception:
|
| 87 |
+
ws = {}
|
| 88 |
+
return {
|
| 89 |
+
'views': dict(ws.get('views') or {}),
|
| 90 |
+
'fields': dict(ws.get('fields') or {}),
|
| 91 |
+
'overlays': dict(ws.get('overlays') or {}),
|
| 92 |
+
# wave-8 I11 (C4): folders over the saved views / cohorts sidebars. A FOURTH
|
| 93 |
+
# stratum rather than a key on each item β see aios_grid.clean_folders for why
|
| 94 |
+
# (a cohort lives in another store, and filing is an organising act, not part of
|
| 95 |
+
# what a view is). Absent for every workspace saved before this wave, which is
|
| 96 |
+
# exactly "no folders yet".
|
| 97 |
+
'folders': dict(ws.get('folders') or {}),
|
| 98 |
+
'itemFolders': dict(ws.get('itemFolders') or {}),
|
| 99 |
+
}
|
| 100 |
+
|
| 101 |
+
# ---------------------------------------------------------------- write
|
| 102 |
+
def _update(self, username, change):
|
| 103 |
+
def _up(data):
|
| 104 |
+
ws = data.setdefault(username, {})
|
| 105 |
+
ws.setdefault('views', {})
|
| 106 |
+
ws.setdefault('fields', {})
|
| 107 |
+
ws.setdefault('overlays', {})
|
| 108 |
+
ws.setdefault('folders', {})
|
| 109 |
+
ws.setdefault('itemFolders', {})
|
| 110 |
+
change(ws)
|
| 111 |
+
return data
|
| 112 |
+
# flush='async' (wave-7 W3): this is THE hot path β every autosaved filter tweak,
|
| 113 |
+
# column note and typed overlay cell lands here inside the component round-trip, and
|
| 114 |
+
# the historical synchronous hub commit cost seconds per edit. The mutation applies to
|
| 115 |
+
# the in-process cache (read-your-writes for every subsequent render); the hub write
|
| 116 |
+
# coalesces in the background. Registry/auth writes elsewhere stay flush='sync'.
|
| 117 |
+
store.update(self.table_key, _up, flush='async')
|
| 118 |
+
|
| 119 |
+
def save_folders(self, username, folders, item_folders):
|
| 120 |
+
"""Replace the folder stratum wholesale (wave-8 I11).
|
| 121 |
+
|
| 122 |
+
Wholesale rather than per-folder because the caller has ALREADY validated the complete
|
| 123 |
+
picture through aios_grid.clean_folders / clean_item_folders, and those two are
|
| 124 |
+
interdependent: a placement is only legal while its folder exists, so committing them
|
| 125 |
+
separately would leave a window where a reader sees an item filed into a folder that is
|
| 126 |
+
not there yet. One write, one consistent state.
|
| 127 |
+
"""
|
| 128 |
+
def _set(ws):
|
| 129 |
+
ws['folders'] = dict(folders or {})
|
| 130 |
+
ws['itemFolders'] = dict(item_folders or {})
|
| 131 |
+
self._update(username, _set)
|
| 132 |
+
|
| 133 |
+
def shared_views(self, viewer, is_admin=False):
|
| 134 |
+
"""Every SHARED view this viewer may see, by id (wave-9 I17).
|
| 135 |
+
|
| 136 |
+
Read-only and independent of the viewer's own workspace: the caller merges. Returns
|
| 137 |
+
only what `_may_see` allows, so a caller cannot accidentally render somebody else's
|
| 138 |
+
personal view by forgetting to filter.
|
| 139 |
+
"""
|
| 140 |
+
try:
|
| 141 |
+
bucket = ((store.get(self.table_key) or {}).get(SHARED_KEY) or {}).get('views') or {}
|
| 142 |
+
except Exception:
|
| 143 |
+
return {}
|
| 144 |
+
return {vid: dict(v) for vid, v in bucket.items()
|
| 145 |
+
if _may_see(v, viewer, is_admin)}
|
| 146 |
+
|
| 147 |
+
def shared_view(self, view_id):
|
| 148 |
+
"""One shared view RAW β no visibility filter. For authorisation decisions only: a
|
| 149 |
+
caller must know a view exists and who owns it before it can decide whether the actor
|
| 150 |
+
may touch it. Never hand the result to a renderer without checking `_may_see`."""
|
| 151 |
+
try:
|
| 152 |
+
return ((store.get(self.table_key) or {}).get(SHARED_KEY) or {}
|
| 153 |
+
).get('views', {}).get(str(view_id))
|
| 154 |
+
except Exception:
|
| 155 |
+
return None
|
| 156 |
+
|
| 157 |
+
def save_view(self, username, view, shared=None):
|
| 158 |
+
"""Upsert a SavedView into its ONE home β personal workspace or the shared bucket.
|
| 159 |
+
|
| 160 |
+
`shared` defaults to reading the view's own permissions (`is_shared`). Whichever home
|
| 161 |
+
it lands in, the view is REMOVED from the other, so a view can never exist as two
|
| 162 |
+
copies that diverge on the next edit.
|
| 163 |
+
|
| 164 |
+
β AUTHORISATION IS THE CALLER'S JOB and must happen BEFORE this is called β this layer
|
| 165 |
+
moves data and does not know who is asking. `_cl_handle_one` is the wall.
|
| 166 |
+
"""
|
| 167 |
+
view_id = str((view or {}).get('id') or '').strip()
|
| 168 |
+
if not view_id:
|
| 169 |
+
raise ValueError('view id is required')
|
| 170 |
+
if username == SHARED_KEY:
|
| 171 |
+
raise ValueError('reserved username')
|
| 172 |
+
to_shared = is_shared(view) if shared is None else bool(shared)
|
| 173 |
+
payload = dict(view)
|
| 174 |
+
|
| 175 |
+
def _up(data):
|
| 176 |
+
if to_shared:
|
| 177 |
+
bucket = data.setdefault(SHARED_KEY, {})
|
| 178 |
+
bucket.setdefault('views', {})[view_id] = payload
|
| 179 |
+
# it may have lived in the creator's workspace before being shared
|
| 180 |
+
owner = data.get(payload.get('createdBy') or username) or {}
|
| 181 |
+
(owner.get('views') or {}).pop(view_id, None)
|
| 182 |
+
else:
|
| 183 |
+
ws = data.setdefault(username, {})
|
| 184 |
+
ws.setdefault('views', {})[view_id] = payload
|
| 185 |
+
(data.get(SHARED_KEY, {}).get('views') or {}).pop(view_id, None)
|
| 186 |
+
return data
|
| 187 |
+
|
| 188 |
+
store.update(self.table_key, _up, flush='async')
|
| 189 |
+
|
| 190 |
+
def delete_view(self, username, view_id):
|
| 191 |
+
"""Delete a custom/list view override. The system all-rows view is guarded by caller.
|
| 192 |
+
|
| 193 |
+
Removes from BOTH homes: the caller has already authorised the delete, and leaving a
|
| 194 |
+
stale copy in the other bucket would resurrect the view on the next read.
|
| 195 |
+
"""
|
| 196 |
+
vid = str(view_id)
|
| 197 |
+
|
| 198 |
+
def _up(data):
|
| 199 |
+
(data.get(username, {}).get('views') or {}).pop(vid, None)
|
| 200 |
+
(data.get(SHARED_KEY, {}).get('views') or {}).pop(vid, None)
|
| 201 |
+
return data
|
| 202 |
+
|
| 203 |
+
store.update(self.table_key, _up, flush='async')
|
| 204 |
+
|
| 205 |
+
def save_field(self, username, field):
|
| 206 |
+
"""Persist a column note or a user-created (custom_/measure_) field definition."""
|
| 207 |
+
key = str((field or {}).get('key') or '').strip()
|
| 208 |
+
if not key:
|
| 209 |
+
raise ValueError('field key is required')
|
| 210 |
+
|
| 211 |
+
def _save(ws):
|
| 212 |
+
# A cleared note on an immutable source field returns to the canonical schema
|
| 213 |
+
# instead of leaving a meaningless override row. Custom fields remain even with
|
| 214 |
+
# an empty note β and so does a PRESET field carrying a measure-window override
|
| 215 |
+
# (wave-2 item 8) or a DISPLAY-format override (wave-5 item 10): the note may be
|
| 216 |
+
# empty but the period/format is real user state.
|
| 217 |
+
if (not field.get('custom') and not str(field.get('note') or '').strip()
|
| 218 |
+
and not isinstance(field.get('measure'), dict)
|
| 219 |
+
and not isinstance(field.get('format'), dict)):
|
| 220 |
+
ws['fields'].pop(key, None)
|
| 221 |
+
else:
|
| 222 |
+
ws['fields'][key] = dict(field)
|
| 223 |
+
|
| 224 |
+
self._update(username, _save)
|
| 225 |
+
|
| 226 |
+
def duplicate_field(self, username, source_key, new_key, field):
|
| 227 |
+
"""Clone a user-created field in ONE store transaction (wave-5 item 1): the new
|
| 228 |
+
definition plus β for `custom_` overlay sources only β every stored cell value under
|
| 229 |
+
the source key. One transaction, because a def without its values (or values without a
|
| 230 |
+
def) is exactly the orphan state delete_field exists to prevent, in reverse.
|
| 231 |
+
The caller validated both keys (same created stratum) and stamped the clone's
|
| 232 |
+
createdBy; this layer only moves data."""
|
| 233 |
+
source_key = str(source_key or '').strip()
|
| 234 |
+
new_key = str(new_key or '').strip()
|
| 235 |
+
if not source_key or not new_key or source_key == new_key:
|
| 236 |
+
raise ValueError('duplicate_field needs two distinct keys')
|
| 237 |
+
|
| 238 |
+
def _dup(ws):
|
| 239 |
+
ws['fields'][new_key] = dict(field)
|
| 240 |
+
if source_key.startswith('custom_'):
|
| 241 |
+
for row in ws['overlays'].values():
|
| 242 |
+
if isinstance(row, dict) and source_key in row:
|
| 243 |
+
row[new_key] = row[source_key]
|
| 244 |
+
|
| 245 |
+
self._update(username, _dup)
|
| 246 |
+
|
| 247 |
+
def delete_field(self, username, key):
|
| 248 |
+
"""Delete a USER-CREATED field definition outright (owner gap closed 2026-07-27).
|
| 249 |
+
|
| 250 |
+
Only the created strata ever reach here (`custom_` overlay fields, `measure_` formula
|
| 251 |
+
columns β the caller enforces the prefix). The stored overlay VALUES for the key are
|
| 252 |
+
scrubbed with it: a deleted column's cells must not linger as orphan data that would
|
| 253 |
+
silently resurface if the key were ever reused. Views referencing the key self-heal on
|
| 254 |
+
their next autosave (an unknown colId is dropped) β the rule every stale key rides.
|
| 255 |
+
"""
|
| 256 |
+
key = str(key or '').strip()
|
| 257 |
+
if not key:
|
| 258 |
+
return
|
| 259 |
+
|
| 260 |
+
def _drop(ws):
|
| 261 |
+
ws['fields'].pop(key, None)
|
| 262 |
+
for row in ws['overlays'].values():
|
| 263 |
+
if isinstance(row, dict):
|
| 264 |
+
row.pop(key, None)
|
| 265 |
+
|
| 266 |
+
self._update(username, _drop)
|
| 267 |
+
|
| 268 |
+
def patch_overlay(self, username, pid, updates):
|
| 269 |
+
"""Patch only the external editable stratum; never writes to the source system."""
|
| 270 |
+
clean = dict(updates or {})
|
| 271 |
+
if not clean:
|
| 272 |
+
return
|
| 273 |
+
|
| 274 |
+
def _patch(ws):
|
| 275 |
+
ws['overlays'].setdefault(str(int(pid)), {}).update(clean)
|
| 276 |
+
|
| 277 |
+
self._update(username, _patch)
|
| 278 |
+
|
| 279 |
+
|
| 280 |
+
def make(table_key):
|
| 281 |
+
return TableStore(table_key)
|
royalimports_os/core/user_tables.py
ADDED
|
@@ -0,0 +1,179 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""USER-CREATED TABLES (wave-9 I8 / contract C6) β databases that are not a connector.
|
| 2 |
+
|
| 3 |
+
The owner's "+ New" flow has two branches: connect a SOURCE (Odoo today) or *"create a completely
|
| 4 |
+
new database blank, where this time the user can actually insert row that they want."* The first
|
| 5 |
+
branch is the connector seam and lives in `harness/datastore.py`; THIS module is the second.
|
| 6 |
+
|
| 7 |
+
WHY A SEPARATE STORE AND NOT A REGISTRY ROW. `core/registry.py` is a static Python list read at
|
| 8 |
+
import time β it is the catalogue of modules the PRODUCT ships, and a tenant cannot append to it
|
| 9 |
+
at runtime without editing code. A user-created table is tenant data, so it lives in the tenant
|
| 10 |
+
store and is MERGED into the nav beside the registry rows. That also keeps the archived/allowed
|
| 11 |
+
machinery honest: a user table is never "archived", it is deleted, and it is never in
|
| 12 |
+
`allowed_modules` because it is not a module.
|
| 13 |
+
|
| 14 |
+
SHAPE (one store key, `user_tables`):
|
| 15 |
+
|
| 16 |
+
{table_key: {'key', 'label', 'source', 'createdBy', 'created', 'fields': [...], 'rows': {...}}}
|
| 17 |
+
|
| 18 |
+
`source` is deliberately NOT 'Odoo' β the nav badge is generated from it, so a blank table reads
|
| 19 |
+
"Blank" and can never be mistaken for connected data. `rows` is `{row_id: {field_key: value}}`,
|
| 20 |
+
which is the same overlay shape `TableStore` already uses, so the grid's write path needs no new
|
| 21 |
+
storage concept.
|
| 22 |
+
|
| 23 |
+
β ROW WRITES ARE ONLY EVER LEGAL HERE. An Odoo-sourced table is READ-ONLY at the source and
|
| 24 |
+
`_cl_handle_one`'s row_add / row_delete refuse unless `is_user_table(key)` β a user must not be
|
| 25 |
+
able to invent a customer in Odoo by typing into a grid.
|
| 26 |
+
"""
|
| 27 |
+
import datetime as _dt
|
| 28 |
+
import re
|
| 29 |
+
|
| 30 |
+
import core.store as store
|
| 31 |
+
|
| 32 |
+
STORE_KEY = 'user_tables'
|
| 33 |
+
MAX_TABLES = 40
|
| 34 |
+
MAX_LABEL = 60
|
| 35 |
+
MAX_ROWS = 5000
|
| 36 |
+
#: the badge the nav paints for a table with no connector behind it
|
| 37 |
+
BLANK_SOURCE = 'Blank'
|
| 38 |
+
#: every user table's key is prefixed, so it can never collide with a registry module key
|
| 39 |
+
KEY_PREFIX = 'ut_'
|
| 40 |
+
|
| 41 |
+
|
| 42 |
+
def _slug(label):
|
| 43 |
+
s = re.sub(r'[^a-z0-9]+', '_', str(label or '').strip().lower()).strip('_')
|
| 44 |
+
return (s or 'table')[:40]
|
| 45 |
+
|
| 46 |
+
|
| 47 |
+
def all_tables():
|
| 48 |
+
"""{key: definition} β every user-created table in this tenant."""
|
| 49 |
+
try:
|
| 50 |
+
return dict(store.get(STORE_KEY) or {})
|
| 51 |
+
except Exception:
|
| 52 |
+
return {}
|
| 53 |
+
|
| 54 |
+
|
| 55 |
+
def get(table_key):
|
| 56 |
+
return all_tables().get(str(table_key))
|
| 57 |
+
|
| 58 |
+
|
| 59 |
+
def is_user_table(table_key):
|
| 60 |
+
"""THE permission predicate for row writes. Cheap and total: a key that is not in this store
|
| 61 |
+
is not a user table, so it cannot accept invented rows."""
|
| 62 |
+
return str(table_key or '').startswith(KEY_PREFIX) and get(table_key) is not None
|
| 63 |
+
|
| 64 |
+
|
| 65 |
+
def create(label, username, fields=None):
|
| 66 |
+
"""Create a blank table. Returns its key, or None if refused.
|
| 67 |
+
|
| 68 |
+
Refuses rather than raises on the ordinary conditions (no name, duplicate, cap reached) β
|
| 69 |
+
the caller is a UI form and a refusal is a message, not an exception.
|
| 70 |
+
"""
|
| 71 |
+
label = str(label or '').strip()[:MAX_LABEL]
|
| 72 |
+
if not label:
|
| 73 |
+
return None
|
| 74 |
+
existing = all_tables()
|
| 75 |
+
if len(existing) >= MAX_TABLES:
|
| 76 |
+
return None
|
| 77 |
+
key = KEY_PREFIX + _slug(label)
|
| 78 |
+
if key in existing: # a second table of the same name gets a suffix
|
| 79 |
+
for n in range(2, 50):
|
| 80 |
+
if f'{key}_{n}' not in existing:
|
| 81 |
+
key = f'{key}_{n}'
|
| 82 |
+
break
|
| 83 |
+
else:
|
| 84 |
+
return None
|
| 85 |
+
# A blank table still needs ONE column to be a table at all: a row with no fields cannot be
|
| 86 |
+
# displayed, edited or identified. 'name' is the identity column, the same role `customer`
|
| 87 |
+
# plays in the Customer table.
|
| 88 |
+
defn = {
|
| 89 |
+
'key': key,
|
| 90 |
+
'label': label,
|
| 91 |
+
'source': BLANK_SOURCE,
|
| 92 |
+
'createdBy': username,
|
| 93 |
+
'created': _dt.datetime.now().strftime('%Y-%m-%dT%H:%M:%S'),
|
| 94 |
+
'fields': list(fields or [{'key': 'name', 'label': 'Name', 'type': 'text',
|
| 95 |
+
'source': 'overlay', 'default': True}]),
|
| 96 |
+
'rows': {},
|
| 97 |
+
}
|
| 98 |
+
|
| 99 |
+
def _add(cur):
|
| 100 |
+
cur[key] = defn
|
| 101 |
+
return cur
|
| 102 |
+
|
| 103 |
+
store.update(STORE_KEY, _add, flush='sync') # a creation is not a hot path; commit it
|
| 104 |
+
return key
|
| 105 |
+
|
| 106 |
+
|
| 107 |
+
def delete(table_key):
|
| 108 |
+
def _drop(cur):
|
| 109 |
+
cur.pop(str(table_key), None)
|
| 110 |
+
return cur
|
| 111 |
+
store.update(STORE_KEY, _drop, flush='sync')
|
| 112 |
+
|
| 113 |
+
|
| 114 |
+
def add_row(table_key, values=None, username=None):
|
| 115 |
+
"""Append one row. Returns the new row id, or None if refused."""
|
| 116 |
+
if not is_user_table(table_key):
|
| 117 |
+
return None # β never on a connector-backed table
|
| 118 |
+
defn = get(table_key) or {}
|
| 119 |
+
rows = defn.get('rows') or {}
|
| 120 |
+
if len(rows) >= MAX_ROWS:
|
| 121 |
+
return None
|
| 122 |
+
rid = str(max((int(r) for r in rows if str(r).isdigit()), default=0) + 1)
|
| 123 |
+
valid = {f['key'] for f in (defn.get('fields') or [])}
|
| 124 |
+
clean = {k: v for k, v in (values or {}).items() if k in valid}
|
| 125 |
+
|
| 126 |
+
def _add(cur):
|
| 127 |
+
t = cur.get(str(table_key))
|
| 128 |
+
if t is not None:
|
| 129 |
+
t.setdefault('rows', {})[rid] = clean
|
| 130 |
+
return cur
|
| 131 |
+
|
| 132 |
+
store.update(STORE_KEY, _add, flush='async')
|
| 133 |
+
return rid
|
| 134 |
+
|
| 135 |
+
|
| 136 |
+
def delete_row(table_key, row_id):
|
| 137 |
+
if not is_user_table(table_key):
|
| 138 |
+
return False
|
| 139 |
+
|
| 140 |
+
def _drop(cur):
|
| 141 |
+
t = cur.get(str(table_key))
|
| 142 |
+
if t is not None:
|
| 143 |
+
(t.get('rows') or {}).pop(str(row_id), None)
|
| 144 |
+
return cur
|
| 145 |
+
|
| 146 |
+
store.update(STORE_KEY, _drop, flush='async')
|
| 147 |
+
return True
|
| 148 |
+
|
| 149 |
+
|
| 150 |
+
def may_open(table_key, viewer, is_admin=False):
|
| 151 |
+
"""FAIL-CLOSED visibility: the creator, or an admin. Nothing else.
|
| 152 |
+
|
| 153 |
+
A user table cannot be gated by `allowed_modules` β a module grant written last month cannot
|
| 154 |
+
describe a table created this morning β so it needs its own rule, and the safe rule is the
|
| 155 |
+
narrow one. Tables are stored tenant-wide under one key, so without this ANY user (including
|
| 156 |
+
a BU-scoped sales agent) could open a table somebody else created just by knowing its key.
|
| 157 |
+
Sharing a user table with named colleagues is a follow-on, and it should reuse the shared-
|
| 158 |
+
VIEW vocabulary rather than inventing a second one.
|
| 159 |
+
"""
|
| 160 |
+
t = get(table_key)
|
| 161 |
+
if not t:
|
| 162 |
+
return False
|
| 163 |
+
return bool(is_admin) or t.get('createdBy') == viewer
|
| 164 |
+
|
| 165 |
+
|
| 166 |
+
def nav_entries(viewer=None, is_admin=False):
|
| 167 |
+
"""Registry-SHAPED dicts so the nav can render user tables beside real modules with no
|
| 168 |
+
special-casing: the same keys the flat nav reads (`key`, `label`, `source`).
|
| 169 |
+
|
| 170 |
+
Filtered by `may_open`, so the nav cannot offer a row the dispatcher would refuse.
|
| 171 |
+
"""
|
| 172 |
+
out = []
|
| 173 |
+
for key, t in sorted(all_tables().items(), key=lambda kv: (kv[1].get('label') or '').lower()):
|
| 174 |
+
if viewer is not None and not may_open(key, viewer, is_admin):
|
| 175 |
+
continue
|
| 176 |
+
out.append({'key': key, 'label': t.get('label') or key,
|
| 177 |
+
'source': t.get('source') or BLANK_SOURCE,
|
| 178 |
+
'user_table': True, 'nav': True, 'validate': False})
|
| 179 |
+
return out
|
royalimports_os/core/users.py
ADDED
|
@@ -0,0 +1,238 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Per-user accounts for the platform, persisted in the HF Dataset store (users.json).
|
| 2 |
+
|
| 3 |
+
Passwords are salted + PBKDF2-HMAC-SHA256 (200k iterations) β never stored or logged in plaintext.
|
| 4 |
+
A bootstrap 'admin' account is seeded from APP_PASSWORD so the owner can always log in and create
|
| 5 |
+
users; APP_PASSWORD also works as an emergency master for 'admin' if the registry is unreachable.
|
| 6 |
+
|
| 7 |
+
Each account carries BU access ('all' or a list of team-ids [5=Fisch, 6=Royal]) which drives
|
| 8 |
+
allowed_bus() β the basis for per-Business-Unit permissioning (a Royal-only user never sees Fisch).
|
| 9 |
+
"""
|
| 10 |
+
import os
|
| 11 |
+
import hmac
|
| 12 |
+
import hashlib
|
| 13 |
+
import secrets
|
| 14 |
+
|
| 15 |
+
import core.store as store
|
| 16 |
+
|
| 17 |
+
BU_LABELS = {5: 'Fisch', 6: 'Royal'}
|
| 18 |
+
_ITER = 200_000
|
| 19 |
+
|
| 20 |
+
|
| 21 |
+
def _hash(pw, salt):
|
| 22 |
+
return hashlib.pbkdf2_hmac('sha256', str(pw).encode('utf-8'), bytes.fromhex(salt), _ITER).hex()
|
| 23 |
+
|
| 24 |
+
|
| 25 |
+
def _record(pw, name, role, bus, active=True, modules='all', agent=None, email=None):
|
| 26 |
+
salt = secrets.token_hex(16)
|
| 27 |
+
return {'salt': salt, 'hash': _hash(pw, salt), 'name': name, 'role': role,
|
| 28 |
+
'bus': bus, 'active': active, 'modules': modules,
|
| 29 |
+
'agent': agent or None, 'email': email or None}
|
| 30 |
+
|
| 31 |
+
|
| 32 |
+
def registry():
|
| 33 |
+
return store.get('users')
|
| 34 |
+
|
| 35 |
+
|
| 36 |
+
def ensure_bootstrap():
|
| 37 |
+
"""Seed an 'admin' account from APP_PASSWORD ONLY on a truly fresh store (no users file yet).
|
| 38 |
+
Idempotent; no-op if the store is unavailable (the app then falls back to the master-password
|
| 39 |
+
path in verify()).
|
| 40 |
+
|
| 41 |
+
Critically, this NEVER overwrites an existing registry: it seeds only when store.exists('users')
|
| 42 |
+
is definitively False. A transient read failure at startup used to return {} and make this
|
| 43 |
+
re-seed just {admin} over the real accounts β that is the bug that wiped users on restart."""
|
| 44 |
+
if not store.available():
|
| 45 |
+
return
|
| 46 |
+
if store.exists('users'): # present, or uncertain -> never clobber
|
| 47 |
+
return
|
| 48 |
+
try:
|
| 49 |
+
reg = store.get('users', fresh=True)
|
| 50 |
+
except Exception:
|
| 51 |
+
return
|
| 52 |
+
if reg:
|
| 53 |
+
return
|
| 54 |
+
master = os.environ.get('APP_PASSWORD', '')
|
| 55 |
+
if not master:
|
| 56 |
+
return
|
| 57 |
+
try:
|
| 58 |
+
store.put('users', {'admin': _record(master, 'Administrator', 'admin', 'all')})
|
| 59 |
+
except Exception:
|
| 60 |
+
pass
|
| 61 |
+
|
| 62 |
+
|
| 63 |
+
def _public(username, u):
|
| 64 |
+
return {'username': username, 'name': u.get('name', username),
|
| 65 |
+
'role': u.get('role', 'user'), 'bus': u.get('bus', 'all'),
|
| 66 |
+
'modules': u.get('modules', 'all'),
|
| 67 |
+
'agent': u.get('agent'), 'email': u.get('email'),
|
| 68 |
+
'epoch': int(u.get('epoch') or 0)}
|
| 69 |
+
|
| 70 |
+
|
| 71 |
+
# ------------------------------------------------------------------ session revocation (X3)
|
| 72 |
+
# The API's session cookie is SIGNED AND STATELESS: there is no server-side session table to
|
| 73 |
+
# delete from, so "log this user out everywhere" needs a number that lives with the account. The
|
| 74 |
+
# cookie carries the epoch it was minted under; bumping the account's epoch makes every
|
| 75 |
+
# outstanding cookie for that user fail verification on its next use. Absent == 0, so every
|
| 76 |
+
# record written before this wave is valid without a migration.
|
| 77 |
+
def epoch(username):
|
| 78 |
+
"""The current session epoch for `username`. None when there is no such account.
|
| 79 |
+
|
| 80 |
+
None is NOT 0. 0 is "this account exists and has never been revoked"; None is "no record" β
|
| 81 |
+
which the session verifier must treat as a reason to refuse, not as a default to compare
|
| 82 |
+
against. (The APP_PASSWORD emergency-master admin has no record at all; the verifier handles
|
| 83 |
+
that case explicitly rather than inventing an epoch for it here.)
|
| 84 |
+
"""
|
| 85 |
+
username = (username or '').strip().lower()
|
| 86 |
+
try:
|
| 87 |
+
u = (store.get('users') or {}).get(username)
|
| 88 |
+
except Exception:
|
| 89 |
+
return None
|
| 90 |
+
return int((u or {}).get('epoch') or 0) if u else None
|
| 91 |
+
|
| 92 |
+
|
| 93 |
+
def bump_epoch(username):
|
| 94 |
+
"""Revoke every outstanding API session for this account."""
|
| 95 |
+
username = (username or '').strip().lower()
|
| 96 |
+
|
| 97 |
+
def _set(reg):
|
| 98 |
+
u = reg.get(username)
|
| 99 |
+
if u:
|
| 100 |
+
u['epoch'] = int(u.get('epoch') or 0) + 1
|
| 101 |
+
return reg
|
| 102 |
+
store.update('users', _set)
|
| 103 |
+
|
| 104 |
+
|
| 105 |
+
def verify(username, pw):
|
| 106 |
+
"""Return a public user dict on success, else None. APP_PASSWORD is an emergency master for the
|
| 107 |
+
'admin' login even if the store is unreachable, so the owner is never locked out."""
|
| 108 |
+
username = (username or '').strip().lower()
|
| 109 |
+
if not username or not pw:
|
| 110 |
+
return None
|
| 111 |
+
master = os.environ.get('APP_PASSWORD', '')
|
| 112 |
+
try:
|
| 113 |
+
# read fresh so accounts created moments ago (UI or out-of-band) are recognised at once
|
| 114 |
+
reg = store.get('users', fresh=True)
|
| 115 |
+
except Exception:
|
| 116 |
+
reg = {}
|
| 117 |
+
u = reg.get(username)
|
| 118 |
+
if u and u.get('active', True) and hmac.compare_digest(_hash(pw, u['salt']), u['hash']):
|
| 119 |
+
return _public(username, u)
|
| 120 |
+
# emergency master: admin + APP_PASSWORD always works (covers first run / store outage)
|
| 121 |
+
if username == 'admin' and master and hmac.compare_digest(str(pw), master):
|
| 122 |
+
# β CARRY THE RECORD'S CURRENT EPOCH when there is a record to read, so the session cookie
|
| 123 |
+
# the API mints from this dict AGREES with the stored account.
|
| 124 |
+
#
|
| 125 |
+
# This is not what stops the emergency lockout β `deps._user_for`'s master fallback does
|
| 126 |
+
# that, and a negative control confirmed the lockout is gone with or without this line.
|
| 127 |
+
# What it fixes is subtler and is a SCOPE question: a cookie whose epoch disagrees with the
|
| 128 |
+
# record falls through to that master fallback, which hands back a SYNTHETIC identity
|
| 129 |
+
# (`bus: 'all'`, `modules: 'all'`). An admin whose record narrows either field would
|
| 130 |
+
# therefore be silently WIDENED to consolidated, all-module access for the life of that
|
| 131 |
+
# session. Matching the epoch means the record branch wins and the account's real scope
|
| 132 |
+
# applies, leaving the master fallback as the true last resort it is meant to be.
|
| 133 |
+
# 0 when the store is unreachable, which is the case this branch was written for.
|
| 134 |
+
return {'username': 'admin', 'name': 'Administrator', 'role': 'admin', 'bus': 'all',
|
| 135 |
+
'modules': 'all', 'epoch': int((reg.get('admin') or {}).get('epoch') or 0)}
|
| 136 |
+
return None
|
| 137 |
+
|
| 138 |
+
|
| 139 |
+
def create_user(username, pw, name, role='user', bus='all', modules='all',
|
| 140 |
+
agent=None, email=None):
|
| 141 |
+
"""Create β or, from app.py's dialog, OVERWRITE β an account.
|
| 142 |
+
|
| 143 |
+
β X3: OVERWRITING AN ACCOUNT MUST NOT RESURRECT ITS OLD SESSIONS. `_record()` builds a fresh
|
| 144 |
+
record with no `epoch` key, i.e. absent == 0. So re-saving an existing username used to reset
|
| 145 |
+
the epoch to 0, and every cookie minted before that account's last password rotation started
|
| 146 |
+
verifying again β a silent un-revocation. `app.py`'s "Add / update a user" calls this function
|
| 147 |
+
for BOTH add and update, so the hole was reachable from the shipped UI.
|
| 148 |
+
|
| 149 |
+
Epoch revocation is only ever as strong as the NARROWEST write path that touches the record, so
|
| 150 |
+
the carry-and-bump lives here rather than in each caller: an overwrite is at least as
|
| 151 |
+
session-invalidating as a password change, and it usually IS one.
|
| 152 |
+
"""
|
| 153 |
+
username = (username or '').strip().lower()
|
| 154 |
+
if not username or not pw:
|
| 155 |
+
raise ValueError('username and password are required')
|
| 156 |
+
|
| 157 |
+
def _add(reg):
|
| 158 |
+
prior = reg.get(username) or {}
|
| 159 |
+
rec = _record(pw, name or username, role, bus, modules=modules,
|
| 160 |
+
agent=agent, email=email)
|
| 161 |
+
if prior:
|
| 162 |
+
rec['epoch'] = int(prior.get('epoch') or 0) + 1
|
| 163 |
+
reg[username] = rec
|
| 164 |
+
return reg
|
| 165 |
+
store.update('users', _add)
|
| 166 |
+
|
| 167 |
+
|
| 168 |
+
def set_password(username, pw):
|
| 169 |
+
username = (username or '').strip().lower()
|
| 170 |
+
|
| 171 |
+
def _set(reg):
|
| 172 |
+
u = reg.get(username)
|
| 173 |
+
if u:
|
| 174 |
+
u['salt'] = secrets.token_hex(16)
|
| 175 |
+
u['hash'] = _hash(pw, u['salt'])
|
| 176 |
+
# X3: a password change revokes every outstanding API session for the account. Bumped
|
| 177 |
+
# INSIDE the same read-modify-write as the hash so the two can never disagree β a
|
| 178 |
+
# separate update() could rotate the password and leave old cookies live if the second
|
| 179 |
+
# write failed.
|
| 180 |
+
u['epoch'] = int(u.get('epoch') or 0) + 1
|
| 181 |
+
return reg
|
| 182 |
+
store.update('users', _set)
|
| 183 |
+
|
| 184 |
+
|
| 185 |
+
def set_active(username, active):
|
| 186 |
+
username = (username or '').strip().lower()
|
| 187 |
+
|
| 188 |
+
def _set(reg):
|
| 189 |
+
if username in reg:
|
| 190 |
+
reg[username]['active'] = bool(active)
|
| 191 |
+
# X3: DEACTIVATION must kill live sessions, not just future logins β otherwise a
|
| 192 |
+
# disabled account keeps working until its cookie expires. Bumped on reactivation too:
|
| 193 |
+
# cheap, and it means a re-enabled account never resurrects a stale cookie.
|
| 194 |
+
reg[username]['epoch'] = int(reg[username].get('epoch') or 0) + 1
|
| 195 |
+
return reg
|
| 196 |
+
store.update('users', _set)
|
| 197 |
+
|
| 198 |
+
|
| 199 |
+
def set_access(username, role=None, bus=None, modules=None, agent=None, email=None,
|
| 200 |
+
name=None):
|
| 201 |
+
"""Update access fields. agent/email: pass '' to clear, None to leave unchanged β
|
| 202 |
+
the userβagent link scopes the Customer List page / digests to that agent's book.
|
| 203 |
+
|
| 204 |
+
`name` follows the same None-means-unchanged idiom. It is here because it had no setter at all:
|
| 205 |
+
a display name could previously only be changed by re-creating the record through
|
| 206 |
+
`create_user`, i.e. by also resetting the password (and, before the fix above, the session
|
| 207 |
+
epoch). Y4's `PATCH {name?}` needs the narrow write, not the destructive one."""
|
| 208 |
+
username = (username or '').strip().lower()
|
| 209 |
+
|
| 210 |
+
def _set(reg):
|
| 211 |
+
u = reg.get(username)
|
| 212 |
+
if u:
|
| 213 |
+
if name is not None:
|
| 214 |
+
u['name'] = name
|
| 215 |
+
if role is not None:
|
| 216 |
+
u['role'] = role
|
| 217 |
+
if bus is not None:
|
| 218 |
+
u['bus'] = bus
|
| 219 |
+
if modules is not None:
|
| 220 |
+
u['modules'] = modules
|
| 221 |
+
if agent is not None:
|
| 222 |
+
u['agent'] = agent or None
|
| 223 |
+
if email is not None:
|
| 224 |
+
u['email'] = email or None
|
| 225 |
+
return reg
|
| 226 |
+
store.update('users', _set)
|
| 227 |
+
|
| 228 |
+
|
| 229 |
+
def allowed_bus_labels(user):
|
| 230 |
+
"""BU labels this user may select. 'all' -> All+Fisch+Royal; a single BU -> just that BU (no
|
| 231 |
+
'All', so the other BU is never reachable); multiple -> All + each."""
|
| 232 |
+
bus = (user or {}).get('bus', 'all')
|
| 233 |
+
if bus == 'all':
|
| 234 |
+
return ['All', 'Fisch', 'Royal']
|
| 235 |
+
labels = [BU_LABELS[b] for b in bus if b in BU_LABELS]
|
| 236 |
+
if not labels:
|
| 237 |
+
return ['All', 'Fisch', 'Royal']
|
| 238 |
+
return (['All'] + labels) if len(labels) > 1 else labels
|
royalimports_os/evals/analyst_golden.yml
ADDED
|
@@ -0,0 +1,104 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Analyst golden eval set v1 (OM-4 gate, 2026-07-11; truths REFRESHED 2026-07-17 after the
|
| 2 |
+
# Jul-15 Odoo mass-recompute + hard-delete reconciliation β closed windows DO drift when docs
|
| 3 |
+
# are edited/deleted; refresh truths from the parity-proven layer, never loosen tolerances).
|
| 4 |
+
# Original header: truths computed from the PARITY-PROVEN
|
| 5 |
+
# semantic layer over CLOSED windows (H1 2026 = 2026-01-01..2026-06-30) so live data drift cannot
|
| 6 |
+
# break them. Checks are DETERMINISTIC (number within tolerance + required substrings + tool-use
|
| 7 |
+
# assertions) β the validate() culture applied to AI answers. Run: harness/evals.py.
|
| 8 |
+
# expect.value tolerance: abs or rel (fraction). expect.contains: ALL substrings (case-insensitive).
|
| 9 |
+
# expect.tools: tool names that MUST appear in the trace (the answer must come FROM the tools).
|
| 10 |
+
# expect.not_tools: tool names that must NOT appear (the over-refusal guard β an answerable ask
|
| 11 |
+
# must never end in report_gap; a gap ask must never end in a forced wrong query).
|
| 12 |
+
items:
|
| 13 |
+
- id: rev_h1
|
| 14 |
+
ask: "What was our wholesale revenue for the first half of 2026 (Jan 1 through Jun 30)?"
|
| 15 |
+
expect: {value: 3501508.33, rel: 0.001, tools: [run_semantic_query]}
|
| 16 |
+
- id: rev_h1_fisch
|
| 17 |
+
ask: "Fisch revenue only, H1 2026 (Jan 1 - Jun 30)?"
|
| 18 |
+
expect: {value: 2494244.70, rel: 0.001, tools: [run_semantic_query]}
|
| 19 |
+
- id: rev_h1_royal
|
| 20 |
+
ask: "And Royal's revenue for the same H1 2026 window?"
|
| 21 |
+
expect: {value: 1007263.63, rel: 0.001, tools: [run_semantic_query]}
|
| 22 |
+
- id: rev_june
|
| 23 |
+
ask: "Revenue for June 2026?"
|
| 24 |
+
expect: {value: 535046.06, rel: 0.001, tools: [run_semantic_query]}
|
| 25 |
+
- id: margin_h1
|
| 26 |
+
ask: "Gross margin dollars for H1 2026 (Jan-Jun), wholesale?"
|
| 27 |
+
expect: {value: 2067509.45, rel: 0.001, tools: [run_semantic_query]}
|
| 28 |
+
- id: margin_pct_h1
|
| 29 |
+
ask: "What gross margin percentage did we run in H1 2026?"
|
| 30 |
+
expect: {value: 0.59, abs: 0.005, allow_pct_form: true, tools: [run_semantic_query]}
|
| 31 |
+
- id: orders_h1
|
| 32 |
+
ask: "How many confirmed wholesale orders did we take in H1 2026 (Jan 1 - Jun 30)?"
|
| 33 |
+
expect: {value: 4989, abs: 1, tools: [run_semantic_query]}
|
| 34 |
+
- id: customers_h1
|
| 35 |
+
ask: "How many distinct customers bought from us in H1 2026?"
|
| 36 |
+
expect: {value: 1037, abs: 1, tools: [run_semantic_query]}
|
| 37 |
+
- id: top_customer
|
| 38 |
+
ask: "Who was our biggest customer by revenue in H1 2026?"
|
| 39 |
+
expect: {contains: ["Poppy Flowers"], tools: [run_semantic_query], not_tools: [report_gap]}
|
| 40 |
+
- id: opex_h1
|
| 41 |
+
ask: "Total operating expense on the GL for H1 2026 (Jan 1 - Jun 30)?"
|
| 42 |
+
expect: {value: 4683428.81, rel: 0.001, tools: [run_semantic_query]}
|
| 43 |
+
- id: top_expense
|
| 44 |
+
ask: "What is our single largest expense account in H1 2026?"
|
| 45 |
+
expect: {contains: ["FBA"], tools: [run_semantic_query]}
|
| 46 |
+
- id: chart_request
|
| 47 |
+
ask: "Make me a line chart of monthly wholesale revenue for H1 2026."
|
| 48 |
+
expect: {artifact: chart, tools: [run_semantic_query, make_chart]}
|
| 49 |
+
- id: bu_guard
|
| 50 |
+
ask: "Show one single revenue line combining Fisch and Royal together per month for H1 2026."
|
| 51 |
+
expect: {tools: [run_semantic_query]} # combined-total ok; the guard is it must NOT error
|
| 52 |
+
- id: name_resolution
|
| 53 |
+
ask: "How much revenue did the customer 'flower' β I don't remember the exact name, something with FLOWERS in it β bring in H1 2026? Pick the top match."
|
| 54 |
+
expect: {tools: [get_field_values, run_semantic_query]}
|
| 55 |
+
- id: map_request
|
| 56 |
+
ask: "Put my top 20 customers by H1 2026 revenue on a map."
|
| 57 |
+
expect: {artifact: chart, tools: [run_semantic_query, make_chart]}
|
| 58 |
+
# --- the gap loop + location/YoY competence (added 2026-07-16 with the debug fixes) ---
|
| 59 |
+
- id: city_filter
|
| 60 |
+
ask: "Which customer in Brooklyn brought us the most revenue in H1 2026 (Jan 1 - Jun 30)?"
|
| 61 |
+
expect: {value: 48785.00, rel: 0.001, contains: ["J KAY"],
|
| 62 |
+
tools: [get_field_values, run_semantic_query], not_tools: [report_gap]}
|
| 63 |
+
- id: yoy_decliners
|
| 64 |
+
ask: "Which Fisch customer declined the most in revenue in H1 2026 (Jan 1 - Jun 30) versus the same window of 2025? Give the dollar decline."
|
| 65 |
+
expect: {value: 16451.25, rel: 0.005, contains: ["JUNE"],
|
| 66 |
+
tools: [run_semantic_query, transform_result], not_tools: [report_gap]}
|
| 67 |
+
# --- the agent dimension (added 2026-07-17: res.partner.agent_ids m2m sync β the #1 logged
|
| 68 |
+
# gap, closed). agent_book is VERBATIM the old gap_refusal ask: yesterday's honest refusal
|
| 69 |
+
# must become today's answer.
|
| 70 |
+
- id: agent_book
|
| 71 |
+
ask: "Who are the top customers handled by our salesperson Naomi in H1 2026?"
|
| 72 |
+
expect: {contains: ["Poppy Flowers"], tools: [get_field_values, run_semantic_query],
|
| 73 |
+
not_tools: [report_gap]}
|
| 74 |
+
- id: agent_revenue
|
| 75 |
+
ask: "How much revenue did agent Naomi's book of customers bring in H1 2026 (Jan 1 - Jun 30)?"
|
| 76 |
+
expect: {value: 872869.98, rel: 0.005, tools: [run_semantic_query], not_tools: [report_gap]}
|
| 77 |
+
# --- the TWO agent sources (added 2026-07-28). Odoo holds a customer-master book AND a
|
| 78 |
+
# per-invoice-line commission agent; they name different people, and the commission table also
|
| 79 |
+
# carries internal SALESPEOPLE who are not agents. These three lock the distinction: the
|
| 80 |
+
# unallocated bucket must be answerable at all, and Anna/Shantal must never be called agents.
|
| 81 |
+
- id: agent_unallocated
|
| 82 |
+
ask: "Using the invoice-line basis, how much of Fisch's 2025 sales was NOT allocated to an agent?"
|
| 83 |
+
expect: {value: 1447579.59, rel: 0.01, tools: [run_semantic_query], not_tools: [report_gap]}
|
| 84 |
+
- id: agent_not_salesperson
|
| 85 |
+
ask: "Who are our agents on the Fisch invoice lines in 2025, and what did each generate?"
|
| 86 |
+
expect: {contains: ["Pasternak"], not_contains: ["Anna", "Shantal"],
|
| 87 |
+
tools: [run_semantic_query], not_tools: [report_gap]}
|
| 88 |
+
- id: agent_allocated_2025
|
| 89 |
+
ask: "On the invoice-line basis, how much of Fisch's 2025 sales was allocated to a real agent?"
|
| 90 |
+
expect: {value: 2995664.75, rel: 0.01, tools: [run_semantic_query], not_tools: [report_gap]}
|
| 91 |
+
- id: gap_refusal
|
| 92 |
+
ask: "How many units of our best-selling SKU do we physically have in stock right now?"
|
| 93 |
+
expect: {tools: [report_gap]} # stock-on-hand has NO topic in the store β must self-report, never guess
|
| 94 |
+
# --- the analytics transform library (added 2026-07-18: 60-op Tableau-class set). Truths from
|
| 95 |
+
# the parity-proven semantic layer over the CLOSED H1 2026 window.
|
| 96 |
+
- id: abc_a_tier
|
| 97 |
+
ask: "Using an ABC / Pareto analysis of H1 2026 revenue (Jan 1 - Jun 30), how many customers fall in the A tier β the vital few that make up the first 80% of revenue?"
|
| 98 |
+
expect: {value: 311, abs: 6, tools: [run_semantic_query, transform_result], not_tools: [report_gap]}
|
| 99 |
+
- id: ytd_customers
|
| 100 |
+
ask: "How many distinct customers had we sold to cumulatively from Jan 1 through the end of March 2026?"
|
| 101 |
+
expect: {value: 790, abs: 1, tools: [run_semantic_query], not_tools: [report_gap]}
|
| 102 |
+
- id: concentration_top5
|
| 103 |
+
ask: "What percent of our H1 2026 revenue (Jan 1 - Jun 30) is accounted for by our top 5 customers?"
|
| 104 |
+
expect: {value: 7.84, abs: 0.5, tools: [run_semantic_query], not_tools: [report_gap]}
|
royalimports_os/harness/__init__.py
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Harness β the multi-source financial-dashboarding platform (connectors + canonical model + skills)."""
|
| 2 |
+
from .canonical import Capability, Entity, Query, Filter
|
| 3 |
+
from .base import Connector, Tenant, Skill, register_skill, skills_for
|
royalimports_os/harness/analyst.py
ADDED
|
@@ -0,0 +1,265 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""harness/analyst.py β the AIOS Analyst loop (OM-4, 2026-07-11).
|
| 2 |
+
|
| 3 |
+
A CHEAP small model (via OpenRouter, OpenAI-compatible) drives the compounding tool registry
|
| 4 |
+
(harness/tools.py) to answer questions, build charts, and spawn dashboards over the semantic
|
| 5 |
+
layer. The model NEVER writes SQL β it picks registry keys, guided by the in-product SKILL
|
| 6 |
+
recipes (model/skills/*.skill.yml) injected into its system prompt. Plan: omni-adoption addendum.
|
| 7 |
+
|
| 8 |
+
Architecture (mirrors the Omni coordinator pattern):
|
| 9 |
+
ask(question) -> bounded tool loop: chat -> tool_calls -> tools.dispatch() -> results -> chat β¦
|
| 10 |
+
until a plain answer or MAX_ITERS. chat_fn is INJECTABLE: live = OpenRouter POST; tests = a
|
| 11 |
+
scripted double β the loop mechanics are provable offline (no key, no tokens).
|
| 12 |
+
|
| 13 |
+
Correctness posture: bounded iterations (never endless); tool errors go BACK to the model as
|
| 14 |
+
readable envelopes (it self-corrects); the final answer carries the tool trace (auditability +
|
| 15 |
+
the drill guarantee); exhausted-budget is reported AS exhausted, never as success (loop rules).
|
| 16 |
+
Cost posture: DEFAULT_MODEL is a cheap tool-calling model; override per-tenant/env; prompt = the
|
| 17 |
+
skills + topic summary (small, cacheable), NOT raw data.
|
| 18 |
+
"""
|
| 19 |
+
import json
|
| 20 |
+
import os
|
| 21 |
+
from pathlib import Path
|
| 22 |
+
|
| 23 |
+
import harness.tools as T
|
| 24 |
+
|
| 25 |
+
# Provider LADDER (owner 2026-07-12: "use Groq and Cerebras"): all OpenAI-compatible chat
|
| 26 |
+
# endpoints; tried in order, failing over on payment/rate/server errors β an exhausted or
|
| 27 |
+
# rate-limited provider degrades to the next instead of taking the Analyst down (the
|
| 28 |
+
# OpenRouter-402 incident). Pin one with ANALYST_PROVIDER=groq|cerebras|openrouter; override
|
| 29 |
+
# its model with ANALYST_MODEL. Every model here must re-pass the eval gate (Β§4b).
|
| 30 |
+
PROVIDERS = [
|
| 31 |
+
{"name": "groq", "env": "GROQ_API_KEY",
|
| 32 |
+
"url": "https://api.groq.com/openai/v1/chat/completions",
|
| 33 |
+
"model": "llama-3.3-70b-versatile"},
|
| 34 |
+
{"name": "cerebras", "env": "CEREBRAS_API_KEY",
|
| 35 |
+
"url": "https://api.cerebras.ai/v1/chat/completions",
|
| 36 |
+
"model": "gpt-oss-120b"}, # this account's tool-capable model (from /v1/models)
|
| 37 |
+
{"name": "openrouter", "env": "OPENROUTER_API_KEY",
|
| 38 |
+
"url": "https://openrouter.ai/api/v1/chat/completions",
|
| 39 |
+
"model": "openai/gpt-4o-mini"},
|
| 40 |
+
]
|
| 41 |
+
DEFAULT_MODEL = None # None = the selected provider's default model
|
| 42 |
+
MAX_ITERS = 8
|
| 43 |
+
|
| 44 |
+
|
| 45 |
+
def available_providers():
|
| 46 |
+
pin = os.environ.get("ANALYST_PROVIDER")
|
| 47 |
+
ps = [p for p in PROVIDERS if os.environ.get(p["env"])]
|
| 48 |
+
if pin:
|
| 49 |
+
ps = [p for p in ps if p["name"] == pin] or ps
|
| 50 |
+
return ps
|
| 51 |
+
|
| 52 |
+
|
| 53 |
+
def llm_configured():
|
| 54 |
+
return bool(available_providers())
|
| 55 |
+
SKILLS_DIR = Path(__file__).resolve().parents[1] / "model" / "skills"
|
| 56 |
+
|
| 57 |
+
|
| 58 |
+
def _skills_text():
|
| 59 |
+
parts = []
|
| 60 |
+
for f in sorted(SKILLS_DIR.glob("*.skill.yml")):
|
| 61 |
+
parts.append(f.read_text(encoding="utf-8"))
|
| 62 |
+
return "\n\n".join(parts) or "(no skill files installed)"
|
| 63 |
+
|
| 64 |
+
|
| 65 |
+
def system_prompt():
|
| 66 |
+
"""The small model's standing instructions: the platform rules + the skill recipes.
|
| 67 |
+
Stable content (cache-friendly); per-question content goes in the user turn."""
|
| 68 |
+
topics = T.list_topics()
|
| 69 |
+
topic_summary = "\n".join(
|
| 70 |
+
f"- {t['topic']}: {t['label']} | metrics: {', '.join(t['metrics'])} | dims: {', '.join(t['dims'])}"
|
| 71 |
+
for t in topics)
|
| 72 |
+
return f"""You are the AIOS Analyst for a wholesale business. You answer data questions by
|
| 73 |
+
calling TOOLS over a governed semantic layer. THE RULES (non-negotiable):
|
| 74 |
+
1. You NEVER write SQL or invent field names β only registry keys from the tools' schemas.
|
| 75 |
+
2. Start unknown asks with list_topics/describe_topic. Resolve typed names (customers, products)
|
| 76 |
+
with get_field_values BEFORE filtering. Business units: Fisch=5, Royal=6 β never mix them in
|
| 77 |
+
one series unless explicitly comparing.
|
| 78 |
+
3. Every number you present must come from a tool result and keep its result_id (drillable).
|
| 79 |
+
If a tool errors, read the error and correct your call β do not make numbers up, ever.
|
| 80 |
+
4. Artifact actions (save_view, compose_dashboard, schedule_report, create_alert) need the
|
| 81 |
+
user's explicit confirmation first.
|
| 82 |
+
5. Answer plainly and lead with the finding. State the window and scope you used β READ the
|
| 83 |
+
window from the tool result's 'window' field, never guess it: a query without dates covers
|
| 84 |
+
ALL recorded history (there is no hidden default window). Time-bound asks ("this year",
|
| 85 |
+
"past 4 weeks") must pass explicit date_from/date_to.
|
| 86 |
+
6. A single-number question gets THE SINGLE NUMBER first (the total for the asked window/scope),
|
| 87 |
+
THEN any breakdown. Never answer a "what was X" question with only components: no monthly
|
| 88 |
+
lists without the total, no per-BU splits when the consolidated figure was asked. If the ask
|
| 89 |
+
is consolidated, query WITHOUT group_by (or sum before answering).
|
| 90 |
+
7. NEVER state an aggregate you did not obtain from a tool result. If you want to mention a
|
| 91 |
+
total across the rows you fetched, either run the scalar query for it or ADD UP those exact
|
| 92 |
+
rows β a round "approximately" figure is a fabrication and forbidden. When in doubt, present
|
| 93 |
+
the rows without a total. Joining or diffing two result sets BY HAND (e.g. this year vs last
|
| 94 |
+
year per customer) is equally forbidden β that is transform_result's job ({{"op":"yoy"}} adds
|
| 95 |
+
<m>_ly, <m>_delta, <m>_yoy_pct per row); if no transform can compute it, say you cannot.
|
| 96 |
+
8. THE GAP PROTOCOL. If β and only if β after checking the schema (list_topics/describe_topic)
|
| 97 |
+
no registered dimension, metric, transform or chart kind can answer, do NOT guess and do NOT
|
| 98 |
+
force a wrong query: call report_gap(kind, missing, question, workaround) ONCE, then answer
|
| 99 |
+
in one honest sentence what is missing and offer the nearest ask that IS answerable. Before
|
| 100 |
+
concluding a gap, check the TRANSFORMS: year-over-year / decline / growth comparisons (yoy),
|
| 101 |
+
rankings (top_n/sort/head), shares, running totals and distributions are ALL answerable via
|
| 102 |
+
transform_result β a derived-analytics question is never a gap. Both directions are
|
| 103 |
+
failures: refusing without report_gap, and calling report_gap for something the tools
|
| 104 |
+
support.
|
| 105 |
+
9. NEVER emit markdown images, data: URIs, or links to charts β you cannot draw. Charts exist
|
| 106 |
+
ONLY through make_chart / make_table; the platform renders them below your answer. Refer to
|
| 107 |
+
a chart as "the chart below". Derived analytics (top-N, YoY, shares, running totals, moving
|
| 108 |
+
averages, ranks, bins) exist ONLY through transform_result β never arithmetic in your head.
|
| 109 |
+
|
| 110 |
+
CHART PICKER (the form follows the comparison β pick by what the question compares):
|
| 111 |
+
- trend over time -> line (few periods -> bar); seasonal/two-dim intensity -> heatmap
|
| 112 |
+
- this year vs last year by period -> transform_result [{{"op":"yoy"}}] then yoy_bars
|
| 113 |
+
- ranking / "top N" -> transform_result top_n then ranked_bar (sorted, labeled)
|
| 114 |
+
- part-to-whole -> pie|donut ONLY when <=6 slices (top_n first); over time -> stacked_bar;
|
| 115 |
+
shares of a shifting total -> stacked_pct; hierarchy of shares -> treemap
|
| 116 |
+
- what drove the change / contribution -> waterfall (signed steps; platform adds the Total)
|
| 117 |
+
- concentration ("do the top X carry the book?") -> pareto
|
| 118 |
+
- correlation of two measures -> scatter; + a third measure -> bubble (size)
|
| 119 |
+
- distribution of a measure -> histogram (or transform bin -> bar)
|
| 120 |
+
- stage conversion -> funnel; actual vs target -> bullet (y=actual, y2=target)
|
| 121 |
+
- level + rate together (e.g. GM$ + GM%) -> combo (y=bars left, y2=line right)
|
| 122 |
+
- same small chart per segment -> facet on line|bar|area|scatter
|
| 123 |
+
- exact figures, many columns, lists -> make_table (totals row included)
|
| 124 |
+
- customer geography -> map. When two forms tie, pick the simpler one.
|
| 125 |
+
|
| 126 |
+
DATASETS:
|
| 127 |
+
{topic_summary}
|
| 128 |
+
|
| 129 |
+
YOUR SKILL RECIPES (follow these plans; adjust the [bracketed] parameters):
|
| 130 |
+
{_skills_text()}"""
|
| 131 |
+
|
| 132 |
+
|
| 133 |
+
_DEAD_STATUS = {401, 402, 403} # payment/auth: skip the provider immediately, no retry
|
| 134 |
+
_RETRY_STATUS = {429, 500, 502, 503} # rate/server: retry with backoff before failing over
|
| 135 |
+
_MAX_ANSWER_TOKENS = 1500 # answers are short; also keeps low-credit providers usable
|
| 136 |
+
_COOLDOWN = {} # provider name -> unix ts until which we skip it (in-proc)
|
| 137 |
+
|
| 138 |
+
|
| 139 |
+
def _live_chat(messages, tools, model, tool_choice="auto"):
|
| 140 |
+
"""POST down the provider ladder with LATENCY-FIRST failover (a chat user is waiting):
|
| 141 |
+
a rate-limited provider is SKIPPED IMMEDIATELY in favor of the next one β we only sleep
|
| 142 |
+
when the ENTIRE ladder is limited (then short ladder-level backoffs). A LONG Retry-After
|
| 143 |
+
(daily quota, e.g. Groq TPD) puts the provider in an in-process COOLDOWN; 401/402/403 skip
|
| 144 |
+
the provider outright."""
|
| 145 |
+
import time as _time
|
| 146 |
+
|
| 147 |
+
import requests
|
| 148 |
+
provs = available_providers()
|
| 149 |
+
if not provs:
|
| 150 |
+
raise RuntimeError("no LLM provider configured β set GROQ_API_KEY / CEREBRAS_API_KEY / "
|
| 151 |
+
"OPENROUTER_API_KEY in royalimports_os/.env")
|
| 152 |
+
last_err = None
|
| 153 |
+
# Two patience profiles: CHAT (default) fails over fast β a user is waiting; BATCH
|
| 154 |
+
# (ANALYST_PATIENT=1, set by the eval gate / insight runner) waits out per-minute rate
|
| 155 |
+
# windows across more ladder rounds instead of exhausting the ladder in seconds.
|
| 156 |
+
patient = os.environ.get("ANALYST_PATIENT") == "1"
|
| 157 |
+
round_sleeps = (5, 10, 20, 30, 45, 60) if patient else (2, 6, 12)
|
| 158 |
+
for rnd in range(len(round_sleeps)): # ladder rounds
|
| 159 |
+
saw_retryable = False
|
| 160 |
+
for p in provs:
|
| 161 |
+
if _COOLDOWN.get(p["name"], 0) > _time.time():
|
| 162 |
+
last_err = RuntimeError(f"{p['name']}: cooling down (quota)")
|
| 163 |
+
continue
|
| 164 |
+
try:
|
| 165 |
+
r = requests.post(p["url"], timeout=120,
|
| 166 |
+
headers={"Authorization": f"Bearer {os.environ[p['env']]}",
|
| 167 |
+
"HTTP-Referer": "https://aios.local",
|
| 168 |
+
"X-Title": "AIOS Analyst"},
|
| 169 |
+
json={"model": (model or os.environ.get("ANALYST_MODEL")
|
| 170 |
+
or p["model"]),
|
| 171 |
+
"messages": messages, "tools": tools,
|
| 172 |
+
"tool_choice": tool_choice, "temperature": 0.1,
|
| 173 |
+
"max_tokens": _MAX_ANSWER_TOKENS})
|
| 174 |
+
if r.status_code in _DEAD_STATUS:
|
| 175 |
+
last_err = RuntimeError(f"{p['name']}: HTTP {r.status_code}")
|
| 176 |
+
_COOLDOWN[p["name"]] = _time.time() + 300 # don't re-poke dead auth/payment
|
| 177 |
+
continue
|
| 178 |
+
if r.status_code in _RETRY_STATUS:
|
| 179 |
+
last_err = RuntimeError(f"{p['name']}: HTTP {r.status_code}")
|
| 180 |
+
try:
|
| 181 |
+
wait = float(r.headers.get("retry-after") or 0)
|
| 182 |
+
except ValueError:
|
| 183 |
+
wait = 0
|
| 184 |
+
if wait > 60: # daily/long quota β cool down
|
| 185 |
+
_COOLDOWN[p["name"]] = _time.time() + wait
|
| 186 |
+
else:
|
| 187 |
+
saw_retryable = True # NO sleep β next provider NOW
|
| 188 |
+
continue
|
| 189 |
+
r.raise_for_status()
|
| 190 |
+
usage = r.json().get("usage", {})
|
| 191 |
+
return r.json()["choices"][0]["message"], usage
|
| 192 |
+
except requests.RequestException as e:
|
| 193 |
+
last_err = e
|
| 194 |
+
continue # network blip β next provider
|
| 195 |
+
if not saw_retryable:
|
| 196 |
+
break # nothing retryable left anywhere
|
| 197 |
+
_time.sleep(round_sleeps[rnd]) # whole ladder limited β pause a round
|
| 198 |
+
raise RuntimeError(f"all LLM providers failed (last: {last_err})")
|
| 199 |
+
|
| 200 |
+
|
| 201 |
+
def ask(question, history=None, chat_fn=None, model=DEFAULT_MODEL, max_iters=MAX_ITERS,
|
| 202 |
+
telemetry_kind="analyst"):
|
| 203 |
+
"""Run one Analyst turn. Returns {answer, artifacts, tool_trace, iterations, exhausted,
|
| 204 |
+
usage}. chat_fn(messages, tools, model) -> (assistant_message_dict, usage_dict)."""
|
| 205 |
+
chat = chat_fn or _live_chat
|
| 206 |
+
tools = T.openai_tools()
|
| 207 |
+
messages = [{"role": "system", "content": system_prompt()}]
|
| 208 |
+
messages += list(history or [])
|
| 209 |
+
# The model has no clock: anchor relative windows ("last week", "this month") to today.
|
| 210 |
+
# Lives in the USER turn so the system prompt stays byte-stable (prompt-cache friendly).
|
| 211 |
+
import datetime as _dt
|
| 212 |
+
messages.append({"role": "user",
|
| 213 |
+
"content": f"(today is {_dt.date.today().isoformat()})\n{question}"})
|
| 214 |
+
trace, artifacts, usage_total = [], [], {"prompt_tokens": 0, "completion_tokens": 0}
|
| 215 |
+
|
| 216 |
+
for i in range(max_iters):
|
| 217 |
+
msg, usage = chat(messages, tools, model)
|
| 218 |
+
for k in usage_total:
|
| 219 |
+
usage_total[k] += usage.get(k, 0) or 0
|
| 220 |
+
tool_calls = msg.get("tool_calls") or []
|
| 221 |
+
# Free-tier models INTERMITTENTLY answer a data question without calling any tool
|
| 222 |
+
# (~1 in 4 observed on gpt-oss-120b β the "no chart" defect). Deterministic fix: if
|
| 223 |
+
# the VERY FIRST turn used no tools, redo it once with tool_choice='required'. Meta
|
| 224 |
+
# questions just get a harmless schema lookup before answering.
|
| 225 |
+
if i == 0 and not tool_calls and chat is _live_chat:
|
| 226 |
+
msg, usage = _live_chat(messages, tools, model, tool_choice="required")
|
| 227 |
+
for k in usage_total:
|
| 228 |
+
usage_total[k] += usage.get(k, 0) or 0
|
| 229 |
+
tool_calls = msg.get("tool_calls") or []
|
| 230 |
+
messages.append({"role": "assistant", "content": msg.get("content") or "",
|
| 231 |
+
**({"tool_calls": tool_calls} if tool_calls else {})})
|
| 232 |
+
if not tool_calls:
|
| 233 |
+
answer = (msg.get("content") or "").strip()
|
| 234 |
+
if not answer:
|
| 235 |
+
# A model occasionally returns NO content and NO tool calls (observed live
|
| 236 |
+
# 2026-07-16: the reply bubble was simply empty and the next turn hallucinated).
|
| 237 |
+
# Never surface silence β report the failure honestly instead.
|
| 238 |
+
answer = ("(the model returned an empty answer β nothing was computed; "
|
| 239 |
+
"please rephrase or ask again)")
|
| 240 |
+
import harness.telemetry as _tel
|
| 241 |
+
_tel.log(telemetry_kind, question=question[:200],
|
| 242 |
+
model=model or (available_providers() or [{}])[0].get("name", "auto"),
|
| 243 |
+
iterations=i + 1,
|
| 244 |
+
tools=[t["tool"] for t in trace],
|
| 245 |
+
tokens=usage_total["prompt_tokens"] + usage_total["completion_tokens"])
|
| 246 |
+
return {"answer": answer, "artifacts": artifacts,
|
| 247 |
+
"tool_trace": trace, "iterations": i + 1, "exhausted": False,
|
| 248 |
+
"usage": usage_total}
|
| 249 |
+
for tc in tool_calls:
|
| 250 |
+
name = tc["function"]["name"]
|
| 251 |
+
args = tc["function"].get("arguments") or "{}"
|
| 252 |
+
res = T.dispatch(name, args)
|
| 253 |
+
trace.append({"tool": name, "args": args if isinstance(args, str) else json.dumps(args),
|
| 254 |
+
"ok": res["ok"], "error": res.get("error")})
|
| 255 |
+
if res["ok"] and isinstance(res["data"], dict) and (
|
| 256 |
+
"chart" in res["data"] or "kpi" in res["data"] or "table" in res["data"]
|
| 257 |
+
or "dashboard" in res["data"]):
|
| 258 |
+
artifacts.append(res["data"])
|
| 259 |
+
messages.append({"role": "tool", "tool_call_id": tc.get("id", name),
|
| 260 |
+
"content": json.dumps(res, default=str)[:8000]})
|
| 261 |
+
# budget exhausted β report honestly (never as success)
|
| 262 |
+
return {"answer": "(stopped: tool-call budget exhausted before a final answer β "
|
| 263 |
+
"narrow the question or raise max_iters)",
|
| 264 |
+
"artifacts": artifacts, "tool_trace": trace, "iterations": max_iters,
|
| 265 |
+
"exhausted": True, "usage": usage_total}
|
royalimports_os/harness/base.py
ADDED
|
@@ -0,0 +1,123 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Harness core β the platform contract that turns 'a dashboard' into 'a product'.
|
| 2 |
+
|
| 3 |
+
Three concepts:
|
| 4 |
+
Connector a source adapter (Odoo / QuickBooks / Amazon / β¦). Declares its capabilities and
|
| 5 |
+
answers canonical Queries. Adding a source = subclass Connector.
|
| 6 |
+
Tenant one client (Royal Imports, β¦). Bundles the connectors they've hooked up + their scope
|
| 7 |
+
config (business units, excluded accounts, channel map, branding). Multi-tenant lives here.
|
| 8 |
+
Skill a portable dashboard/metric written ONCE against the canonical model; it declares the
|
| 9 |
+
capabilities it needs so a tenant only ever sees skills its sources can serve.
|
| 10 |
+
|
| 11 |
+
The UI (app.py) becomes a thin renderer over (tenant β available skills). Nothing above the
|
| 12 |
+
connector layer knows what Odoo is.
|
| 13 |
+
"""
|
| 14 |
+
from __future__ import annotations
|
| 15 |
+
from abc import ABC, abstractmethod
|
| 16 |
+
from dataclasses import dataclass, field
|
| 17 |
+
from typing import Callable
|
| 18 |
+
from .canonical import Capability, Query, Entity
|
| 19 |
+
|
| 20 |
+
|
| 21 |
+
class Connector(ABC):
|
| 22 |
+
"""A data source. Implement `capabilities()` + `aggregate()` (and optionally `records()`), map the
|
| 23 |
+
canonical model to the source, and the source is usable by every compatible skill."""
|
| 24 |
+
key: str = 'connector'
|
| 25 |
+
name: str = 'Connector'
|
| 26 |
+
|
| 27 |
+
@abstractmethod
|
| 28 |
+
def capabilities(self) -> set[Capability]:
|
| 29 |
+
"""The canonical capabilities this source can serve for the given account."""
|
| 30 |
+
|
| 31 |
+
@abstractmethod
|
| 32 |
+
def aggregate(self, q: Query) -> list[dict]:
|
| 33 |
+
"""Answer an aggregate Query β list of canonical dicts (keys = measures/group_by, + 'count')."""
|
| 34 |
+
|
| 35 |
+
def records(self, q: Query) -> list[dict]:
|
| 36 |
+
"""Optional row-level fetch (canonical dicts). Default: not supported."""
|
| 37 |
+
raise NotImplementedError(f'{self.key}: row-level records() not implemented')
|
| 38 |
+
|
| 39 |
+
def distinct_count(self, q: Query, fieldname: str) -> int:
|
| 40 |
+
"""Number of distinct values of a canonical field over the query's scope (e.g. active
|
| 41 |
+
customers). Default: derive from an aggregate group_by; connectors may override efficiently."""
|
| 42 |
+
from dataclasses import replace
|
| 43 |
+
rows = self.aggregate(replace(q, measures=(), aggregate='count', group_by=(fieldname,)))
|
| 44 |
+
return sum(1 for r in rows if r.get(fieldname) not in (None, False))
|
| 45 |
+
|
| 46 |
+
def supports(self, cap: Capability) -> bool:
|
| 47 |
+
return cap in self.capabilities()
|
| 48 |
+
|
| 49 |
+
def entities(self) -> set[Entity]:
|
| 50 |
+
"""Canonical entities this connector can materialize (advertised by subclasses)."""
|
| 51 |
+
return set()
|
| 52 |
+
|
| 53 |
+
|
| 54 |
+
@dataclass
|
| 55 |
+
class Tenant:
|
| 56 |
+
"""One client of the platform. `sources` maps a connector key β Connector instance. `config` holds
|
| 57 |
+
the client's scope semantics (which is what used to be hard-coded in core/odoo.py):
|
| 58 |
+
business_units {label: source_scope_value} e.g. {'Fisch': 5, 'Royal': 6}
|
| 59 |
+
confirmed_statuses [...] which order states count as 'real'
|
| 60 |
+
excluded_customers [ids] house / inter-co accounts to drop
|
| 61 |
+
channels {label: match} e.g. {'Amazon': <partner>} (marketplace split)
|
| 62 |
+
branding {...} palette, name, logo
|
| 63 |
+
A skill reads tenant.config for scope and asks tenant.source(capability) for the right connector."""
|
| 64 |
+
key: str
|
| 65 |
+
name: str
|
| 66 |
+
sources: dict[str, Connector] = field(default_factory=dict)
|
| 67 |
+
config: dict = field(default_factory=dict)
|
| 68 |
+
|
| 69 |
+
def add(self, conn: Connector) -> 'Tenant':
|
| 70 |
+
self.sources[conn.key] = conn
|
| 71 |
+
return self
|
| 72 |
+
|
| 73 |
+
def capabilities(self) -> set[Capability]:
|
| 74 |
+
caps: set[Capability] = set()
|
| 75 |
+
for c in self.sources.values():
|
| 76 |
+
caps |= c.capabilities()
|
| 77 |
+
return caps
|
| 78 |
+
|
| 79 |
+
def source(self, cap: Capability, prefer: str | None = None) -> Connector:
|
| 80 |
+
"""The connector that serves `cap` (a tenant may hook several sources; `prefer` picks one,
|
| 81 |
+
else first that supports it). Raises if no connected source provides the capability."""
|
| 82 |
+
if prefer and prefer in self.sources and self.sources[prefer].supports(cap):
|
| 83 |
+
return self.sources[prefer]
|
| 84 |
+
for c in self.sources.values():
|
| 85 |
+
if c.supports(cap):
|
| 86 |
+
return c
|
| 87 |
+
raise LookupError(f"tenant {self.key!r} has no source for capability {cap.value!r}")
|
| 88 |
+
|
| 89 |
+
def has(self, *caps: Capability) -> bool:
|
| 90 |
+
mine = self.capabilities()
|
| 91 |
+
return all(c in mine for c in caps)
|
| 92 |
+
|
| 93 |
+
|
| 94 |
+
# --------------------------------------------------------------------------- skill registry
|
| 95 |
+
@dataclass(frozen=True)
|
| 96 |
+
class Skill:
|
| 97 |
+
"""A portable dashboard. `requires` are the capabilities it needs; the harness only offers it to a
|
| 98 |
+
tenant whose connected sources cover them. `render` is the UI entrypoint (given tenant + context)."""
|
| 99 |
+
key: str
|
| 100 |
+
label: str
|
| 101 |
+
requires: tuple[Capability, ...]
|
| 102 |
+
render: Callable | None = None
|
| 103 |
+
hq: bool = False # consolidated / company-level (ignores the business-unit filter)
|
| 104 |
+
|
| 105 |
+
def available_for(self, tenant: Tenant) -> bool:
|
| 106 |
+
return tenant.has(*self.requires)
|
| 107 |
+
|
| 108 |
+
|
| 109 |
+
_SKILLS: dict[str, Skill] = {}
|
| 110 |
+
|
| 111 |
+
|
| 112 |
+
def register_skill(skill: Skill) -> Skill:
|
| 113 |
+
_SKILLS[skill.key] = skill
|
| 114 |
+
return skill
|
| 115 |
+
|
| 116 |
+
|
| 117 |
+
def skills_for(tenant: Tenant) -> list[Skill]:
|
| 118 |
+
"""The skills a tenant can actually run, given its connected sources β this is the dynamic nav."""
|
| 119 |
+
return [s for s in _SKILLS.values() if s.available_for(tenant)]
|
| 120 |
+
|
| 121 |
+
|
| 122 |
+
def all_skills() -> list[Skill]:
|
| 123 |
+
return list(_SKILLS.values())
|
royalimports_os/harness/canonical.py
ADDED
|
@@ -0,0 +1,101 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Canonical financial data model β the SHARED vocabulary of the harness.
|
| 2 |
+
|
| 3 |
+
Skills speak ONLY in these terms; connectors (Odoo / QuickBooks / Amazon / β¦) translate them to
|
| 4 |
+
their own source. This is the seam that decouples the dashboards from any single system: add a new
|
| 5 |
+
source by writing a connector that maps its API onto these entities/fields β every skill that needs
|
| 6 |
+
only the capabilities that source provides then works, unchanged.
|
| 7 |
+
|
| 8 |
+
Design rules:
|
| 9 |
+
- Entities and canonical field names are source-agnostic (no `amount_untaxed`, no `sale.order`).
|
| 10 |
+
- A skill builds a `Query` (what business question), a connector answers it (how, per source).
|
| 11 |
+
- Capabilities let a skill declare what it needs and a tenant show only the dashboards its
|
| 12 |
+
connected sources can actually serve.
|
| 13 |
+
"""
|
| 14 |
+
from __future__ import annotations
|
| 15 |
+
from dataclasses import dataclass, field
|
| 16 |
+
from enum import Enum
|
| 17 |
+
from typing import Any
|
| 18 |
+
|
| 19 |
+
|
| 20 |
+
class Capability(str, Enum):
|
| 21 |
+
"""What a connector can serve. Skills require a set of these; a tenant's nav is the union of
|
| 22 |
+
what its connectors provide. (A pure-accounting source like QuickBooks has LEDGER+INVOICING+
|
| 23 |
+
CUSTOMERS but not SKU-level SALES velocity; Amazon has SALES but no LEDGER; Odoo has all.)"""
|
| 24 |
+
SALES = 'sales' # orders / order lines (SKU-level revenue, units)
|
| 25 |
+
INVOICING = 'invoicing' # customer invoices + credit notes (AR, returns)
|
| 26 |
+
LEDGER = 'ledger' # GL accounts + journal lines (P&L, balance sheet)
|
| 27 |
+
PAYABLES = 'payables' # vendor bills / AP
|
| 28 |
+
INVENTORY = 'inventory' # on-hand stock levels + valuation
|
| 29 |
+
PRODUCTS = 'products' # product master (SKU, category, cost)
|
| 30 |
+
CUSTOMERS = 'customers' # customer master (name, geo, assigned agent)
|
| 31 |
+
|
| 32 |
+
|
| 33 |
+
class Entity(str, Enum):
|
| 34 |
+
"""Canonical business objects. Each connector declares which it can materialize."""
|
| 35 |
+
SALES_ORDER = 'sales_order'
|
| 36 |
+
SALES_ORDER_LINE = 'sales_order_line'
|
| 37 |
+
INVOICE = 'invoice' # customer invoice / credit note
|
| 38 |
+
JOURNAL_LINE = 'journal_line' # GL posting
|
| 39 |
+
GL_ACCOUNT = 'gl_account'
|
| 40 |
+
VENDOR_BILL = 'vendor_bill'
|
| 41 |
+
STOCK_LEVEL = 'stock_level'
|
| 42 |
+
PRODUCT = 'product'
|
| 43 |
+
CUSTOMER = 'customer'
|
| 44 |
+
|
| 45 |
+
|
| 46 |
+
# Canonical field names per entity β the ONLY field identifiers a skill may use. Connectors own the
|
| 47 |
+
# mapping to their native fields. `date` is the entity's primary time axis (used by Query windows).
|
| 48 |
+
FIELDS: dict[Entity, set[str]] = {
|
| 49 |
+
Entity.SALES_ORDER: {'date', 'amount', 'customer', 'salesperson', 'agent', 'status', 'business_unit', 'channel'},
|
| 50 |
+
Entity.SALES_ORDER_LINE: {'date', 'amount', 'qty', 'margin', 'product', 'category', 'customer', 'status', 'business_unit'},
|
| 51 |
+
Entity.INVOICE: {'date', 'amount', 'customer', 'doc_type', 'status', 'salesperson', 'business_unit'},
|
| 52 |
+
Entity.JOURNAL_LINE: {'date', 'amount', 'account', 'account_type', 'business_unit', 'status'},
|
| 53 |
+
Entity.GL_ACCOUNT: {'code', 'name', 'type'},
|
| 54 |
+
Entity.VENDOR_BILL: {'date', 'amount', 'vendor', 'status'},
|
| 55 |
+
Entity.STOCK_LEVEL: {'product', 'qty', 'value', 'location'},
|
| 56 |
+
Entity.PRODUCT: {'id', 'code', 'name', 'category', 'cost', 'active'},
|
| 57 |
+
Entity.CUSTOMER: {'id', 'name', 'city', 'state', 'agent', 'active'},
|
| 58 |
+
}
|
| 59 |
+
|
| 60 |
+
# Allowed filter/aggregate vocabulary (connectors translate these; they never see source operators).
|
| 61 |
+
OPS = frozenset({'eq', 'ne', 'in', 'not_in', 'gte', 'lte', 'gt', 'lt', 'like'})
|
| 62 |
+
AGGS = frozenset({'sum', 'count', 'avg', 'min', 'max'})
|
| 63 |
+
GRAINS = frozenset({'day', 'week', 'month', 'quarter', 'year'}) # time-bucket for a trend
|
| 64 |
+
|
| 65 |
+
|
| 66 |
+
@dataclass(frozen=True)
|
| 67 |
+
class Filter:
|
| 68 |
+
"""A canonical predicate: (canonical_field, op, value). op β OPS."""
|
| 69 |
+
fieldname: str
|
| 70 |
+
op: str
|
| 71 |
+
value: Any
|
| 72 |
+
|
| 73 |
+
def __post_init__(self):
|
| 74 |
+
if self.op not in OPS:
|
| 75 |
+
raise ValueError(f'unknown op {self.op!r}; allowed {sorted(OPS)}')
|
| 76 |
+
|
| 77 |
+
|
| 78 |
+
@dataclass(frozen=True)
|
| 79 |
+
class Query:
|
| 80 |
+
"""A source-agnostic analytical request. `measures` are canonical fields aggregated by `aggregate`,
|
| 81 |
+
optionally grouped by canonical `group_by` fields, filtered by `filters`, over [date_from, date_to]
|
| 82 |
+
on the entity's `date` axis. A connector returns a list of plain dicts keyed by the canonical
|
| 83 |
+
measure/group names (+ 'count'). Skills post-process those dicts β never source rows."""
|
| 84 |
+
entity: Entity
|
| 85 |
+
measures: tuple[str, ...] = ()
|
| 86 |
+
aggregate: str = 'sum'
|
| 87 |
+
group_by: tuple[str, ...] = ()
|
| 88 |
+
filters: tuple[Filter, ...] = ()
|
| 89 |
+
date_from: str | None = None
|
| 90 |
+
date_to: str | None = None
|
| 91 |
+
grain: str | None = None # bucket the date axis into a trend; result rows carry 'period' (e.g. '2026-05')
|
| 92 |
+
|
| 93 |
+
def __post_init__(self):
|
| 94 |
+
if self.aggregate not in AGGS:
|
| 95 |
+
raise ValueError(f'unknown aggregate {self.aggregate!r}')
|
| 96 |
+
if self.grain is not None and self.grain not in GRAINS:
|
| 97 |
+
raise ValueError(f'unknown grain {self.grain!r}; allowed {sorted(GRAINS)}')
|
| 98 |
+
valid = FIELDS.get(self.entity, set())
|
| 99 |
+
for f in list(self.measures) + list(self.group_by) + [x.fieldname for x in self.filters]:
|
| 100 |
+
if f not in valid:
|
| 101 |
+
raise ValueError(f'{f!r} is not a canonical field of {self.entity.value} (have {sorted(valid)})')
|
royalimports_os/harness/chatlog.py
ADDED
|
@@ -0,0 +1,102 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""harness/chatlog.py β durable Analyst conversations (2026-07-16).
|
| 2 |
+
|
| 3 |
+
Every Analyst conversation is a logged SESSION that survives reruns, page navigation and Space
|
| 4 |
+
restarts. Persistence rides the platform's only writable store (core/store.py β the private HF
|
| 5 |
+
dataset), the SAME mechanism as users/notes β Odoo stays read-only.
|
| 6 |
+
|
| 7 |
+
Shape (store key 'analyst_sessions'): { username: [ session, ... ] } (newest first), each
|
| 8 |
+
session = {id, title, started_at, updated_at, ui:[[role,content,artifacts],...],
|
| 9 |
+
llm:[{role,content},...]}.
|
| 10 |
+
ui = the redisplay transcript; llm = the trimmed model-context transcript.
|
| 11 |
+
|
| 12 |
+
Write posture (per the design review): the LIVE turn stays in st.session_state for a snappy chat;
|
| 13 |
+
we persist in a BACKGROUND daemon thread (core/store.update is a full dataset commit β never block
|
| 14 |
+
the UI on it), and bound growth (MAX_SESSIONS per user, MAX_TURNS per session) so the JSON stays
|
| 15 |
+
small. A failed/absent store degrades to in-memory only (chat still works this session).
|
| 16 |
+
"""
|
| 17 |
+
import threading
|
| 18 |
+
import time
|
| 19 |
+
import uuid
|
| 20 |
+
|
| 21 |
+
import core.store as store
|
| 22 |
+
|
| 23 |
+
KEY = "analyst_sessions"
|
| 24 |
+
MAX_SESSIONS = 40 # keep the last N conversations per user (bounds the JSON)
|
| 25 |
+
MAX_TURNS = 60 # ui/llm entries kept per conversation
|
| 26 |
+
TITLE_LEN = 60
|
| 27 |
+
|
| 28 |
+
|
| 29 |
+
def new_id():
|
| 30 |
+
return uuid.uuid4().hex[:12]
|
| 31 |
+
|
| 32 |
+
|
| 33 |
+
def new_session(sid=None):
|
| 34 |
+
now = time.strftime("%Y-%m-%d %H:%M:%S")
|
| 35 |
+
return {"id": sid or new_id(), "title": "", "started_at": now, "updated_at": now,
|
| 36 |
+
"ui": [], "llm": []}
|
| 37 |
+
|
| 38 |
+
|
| 39 |
+
def _safe(obj):
|
| 40 |
+
"""JSON round-trip a value (chart/kpi artifacts) so only serializable content is stored;
|
| 41 |
+
anything that won't serialize is dropped rather than breaking the whole write."""
|
| 42 |
+
import json
|
| 43 |
+
try:
|
| 44 |
+
return json.loads(json.dumps(obj, default=str))
|
| 45 |
+
except Exception:
|
| 46 |
+
return None
|
| 47 |
+
|
| 48 |
+
|
| 49 |
+
def load_sessions(username):
|
| 50 |
+
"""The user's conversations, newest first (lenient read β never raises on a store blip)."""
|
| 51 |
+
if not store.available():
|
| 52 |
+
return []
|
| 53 |
+
try:
|
| 54 |
+
all_s = store.get(KEY) or {}
|
| 55 |
+
except Exception:
|
| 56 |
+
return []
|
| 57 |
+
return list(all_s.get(username, []))
|
| 58 |
+
|
| 59 |
+
|
| 60 |
+
def touch(session, question):
|
| 61 |
+
"""Stamp updated_at and derive a title from the first user question."""
|
| 62 |
+
session["updated_at"] = time.strftime("%Y-%m-%d %H:%M:%S")
|
| 63 |
+
if not session.get("title") and question:
|
| 64 |
+
t = " ".join(str(question).split())
|
| 65 |
+
session["title"] = t[:TITLE_LEN] + ("β¦" if len(t) > TITLE_LEN else "")
|
| 66 |
+
# bound in-memory transcript length (keep the most recent turns)
|
| 67 |
+
if len(session.get("ui", [])) > MAX_TURNS:
|
| 68 |
+
session["ui"] = session["ui"][-MAX_TURNS:]
|
| 69 |
+
if len(session.get("llm", [])) > MAX_TURNS:
|
| 70 |
+
session["llm"] = session["llm"][-MAX_TURNS:]
|
| 71 |
+
return session
|
| 72 |
+
|
| 73 |
+
|
| 74 |
+
def upsert(sessions, session):
|
| 75 |
+
"""Return the sessions list with `session` moved to the front (deduped by id), capped."""
|
| 76 |
+
out = [s for s in (sessions or []) if s.get("id") != session.get("id")]
|
| 77 |
+
out.insert(0, session)
|
| 78 |
+
return out[:MAX_SESSIONS]
|
| 79 |
+
|
| 80 |
+
|
| 81 |
+
def _persist(username, session):
|
| 82 |
+
# sanitize artifacts for storage (charts are plain dicts; drop any non-serializable stragglers)
|
| 83 |
+
stored = dict(session)
|
| 84 |
+
stored["ui"] = [[role, content, [a for a in (_safe(x) for x in (arts or [])) if a is not None]]
|
| 85 |
+
for (role, content, arts) in session.get("ui", [])]
|
| 86 |
+
|
| 87 |
+
def _fn(data):
|
| 88 |
+
data = data or {}
|
| 89 |
+
data[username] = upsert(data.get(username, []), stored)
|
| 90 |
+
return data
|
| 91 |
+
try:
|
| 92 |
+
store.update(KEY, _fn)
|
| 93 |
+
except Exception:
|
| 94 |
+
pass # store blip / no token β in-memory only this run
|
| 95 |
+
|
| 96 |
+
|
| 97 |
+
def save_async(username, session):
|
| 98 |
+
"""Persist in a daemon thread so the chat never waits on a dataset commit."""
|
| 99 |
+
if not store.available():
|
| 100 |
+
return
|
| 101 |
+
threading.Thread(target=_persist, args=(username, dict(session)),
|
| 102 |
+
daemon=True, name="analyst-chatlog").start()
|
royalimports_os/harness/connectors/__init__.py
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
"""Source connectors β one module per system (odoo, quickbooks, amazon, ...)."""
|
royalimports_os/harness/connectors/odoo.py
ADDED
|
@@ -0,0 +1,147 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Odoo connector β the reference implementation.
|
| 2 |
+
|
| 3 |
+
ALL Odoo specifics (model names, field names, XML-RPC quirks, the read_group dot-path gotchas) live
|
| 4 |
+
here and nowhere else. It maps the canonical model onto the existing, battle-tested `core.odoo` data
|
| 5 |
+
layer (read-only, write-guarded) so nothing is re-plumbed β the harness just gives it a source-agnostic
|
| 6 |
+
face. Every other connector (QuickBooks, Amazon, β¦) is a sibling of this file.
|
| 7 |
+
"""
|
| 8 |
+
from __future__ import annotations
|
| 9 |
+
import sys
|
| 10 |
+
from dataclasses import replace
|
| 11 |
+
from functools import lru_cache
|
| 12 |
+
from pathlib import Path
|
| 13 |
+
sys.path.insert(0, str(Path(__file__).resolve().parents[2])) # royalimports_os/
|
| 14 |
+
import core.odoo as O
|
| 15 |
+
from harness.canonical import Capability, Entity, Query, Filter
|
| 16 |
+
|
| 17 |
+
|
| 18 |
+
@lru_cache(maxsize=1)
|
| 19 |
+
def _product_category_map():
|
| 20 |
+
"""product_id -> main category name (Odoo-side enrichment; reuses the existing resolver)."""
|
| 21 |
+
import modules.sales as _sales
|
| 22 |
+
return _sales._product_cat()
|
| 23 |
+
|
| 24 |
+
|
| 25 |
+
def _replace_group(q: Query, new_group_by):
|
| 26 |
+
return replace(q, group_by=new_group_by, grain=None)
|
| 27 |
+
|
| 28 |
+
_OPS = {'eq': '=', 'ne': '!=', 'in': 'in', 'not_in': 'not in',
|
| 29 |
+
'gte': '>=', 'lte': '<=', 'gt': '>', 'lt': '<', 'like': 'ilike'}
|
| 30 |
+
|
| 31 |
+
# Canonical entity -> Odoo model + (canonical field -> Odoo field) + its time axis.
|
| 32 |
+
_MAP = {
|
| 33 |
+
Entity.SALES_ORDER: {
|
| 34 |
+
'model': 'sale.order', 'date': 'date_order', 'dt': True,
|
| 35 |
+
'f': {'amount': 'amount_untaxed', 'customer': 'partner_id', 'salesperson': 'user_id',
|
| 36 |
+
'status': 'state', 'business_unit': 'team_id'}},
|
| 37 |
+
Entity.SALES_ORDER_LINE: {
|
| 38 |
+
'model': 'sale.order.line', 'date': 'order_id.date_order', 'dt': True,
|
| 39 |
+
'f': {'amount': 'price_subtotal', 'qty': 'product_uom_qty', 'margin': 'margin',
|
| 40 |
+
'product': 'product_id', 'customer': 'order_partner_id',
|
| 41 |
+
'status': 'order_id.state', 'business_unit': 'order_id.team_id'}},
|
| 42 |
+
Entity.INVOICE: {
|
| 43 |
+
'model': 'account.move', 'date': 'invoice_date', 'dt': False,
|
| 44 |
+
'f': {'amount': 'amount_untaxed_signed', 'customer': 'partner_id', 'doc_type': 'move_type',
|
| 45 |
+
'status': 'state', 'salesperson': 'invoice_user_id', 'business_unit': 'team_id'}},
|
| 46 |
+
Entity.STOCK_LEVEL: {
|
| 47 |
+
'model': 'stock.quant', 'date': None, 'dt': False,
|
| 48 |
+
'f': {'product': 'product_id', 'qty': 'quantity', 'location': 'location_id'}},
|
| 49 |
+
Entity.PRODUCT: {
|
| 50 |
+
'model': 'product.product', 'date': None, 'dt': False,
|
| 51 |
+
'f': {'id': 'id', 'code': 'default_code', 'name': 'name', 'category': 'categ_id',
|
| 52 |
+
'cost': 'standard_price', 'active': 'active'}},
|
| 53 |
+
Entity.CUSTOMER: {
|
| 54 |
+
'model': 'res.partner', 'date': None, 'dt': False,
|
| 55 |
+
'f': {'id': 'id', 'name': 'name', 'city': 'city', 'state': 'state_id',
|
| 56 |
+
'agent': 'agent_ids', 'active': 'active'}},
|
| 57 |
+
}
|
| 58 |
+
|
| 59 |
+
|
| 60 |
+
class OdooConnector:
|
| 61 |
+
key = 'odoo'
|
| 62 |
+
name = 'Odoo ERP'
|
| 63 |
+
|
| 64 |
+
def __init__(self, doc_mode: str = 'order'):
|
| 65 |
+
self._doc_mode = doc_mode
|
| 66 |
+
|
| 67 |
+
def capabilities(self) -> set[Capability]:
|
| 68 |
+
return {Capability.SALES, Capability.INVOICING, Capability.LEDGER, Capability.PAYABLES,
|
| 69 |
+
Capability.INVENTORY, Capability.PRODUCTS, Capability.CUSTOMERS}
|
| 70 |
+
|
| 71 |
+
def entities(self) -> set[Entity]:
|
| 72 |
+
return set(_MAP)
|
| 73 |
+
|
| 74 |
+
def supports(self, cap: Capability) -> bool:
|
| 75 |
+
return cap in self.capabilities()
|
| 76 |
+
|
| 77 |
+
# ------------------------------------------------------------------ query translation
|
| 78 |
+
def _domain(self, q: Query, meta):
|
| 79 |
+
dom = []
|
| 80 |
+
if q.date_from and meta['date']:
|
| 81 |
+
dom.append((meta['date'], '>=', q.date_from + (' 00:00:00' if meta['dt'] else '')))
|
| 82 |
+
if q.date_to and meta['date']:
|
| 83 |
+
dom.append((meta['date'], '<=', q.date_to + (' 23:59:59' if meta['dt'] else '')))
|
| 84 |
+
for flt in q.filters:
|
| 85 |
+
dom.append((meta['f'].get(flt.fieldname, flt.fieldname), _OPS[flt.op], flt.value))
|
| 86 |
+
return dom
|
| 87 |
+
|
| 88 |
+
def aggregate(self, q: Query) -> list[dict]:
|
| 89 |
+
meta = _MAP[q.entity]
|
| 90 |
+
O.set_doc_mode(self._doc_mode)
|
| 91 |
+
# 'category' isn't a native line field (dot-path groupby fails) β enrich from the product master.
|
| 92 |
+
if 'category' in q.group_by:
|
| 93 |
+
return self._by_category(q, meta)
|
| 94 |
+
dom = self._domain(q, meta)
|
| 95 |
+
measures = [f"{meta['f'].get(m, m)}:{q.aggregate}" for m in q.measures]
|
| 96 |
+
groupby = [meta['f'].get(g, g) for g in q.group_by]
|
| 97 |
+
if q.grain and meta['date']: # time-bucket -> Odoo's date:grain groupby
|
| 98 |
+
groupby.append(f"{meta['date']}:{q.grain}")
|
| 99 |
+
rows = O.read_group(meta['model'], dom, measures, groupby, lazy=False)
|
| 100 |
+
return [self._canon_row(r, q, meta) for r in rows]
|
| 101 |
+
|
| 102 |
+
def _canon_row(self, r, q, meta):
|
| 103 |
+
rec = {}
|
| 104 |
+
for m in q.measures:
|
| 105 |
+
rec[m] = r.get(meta['f'].get(m, m)) or 0.0
|
| 106 |
+
for g in q.group_by:
|
| 107 |
+
v = r.get(meta['f'].get(g, g))
|
| 108 |
+
rec[g] = ({'id': O.m2o_id(v), 'name': O.m2o_name(v)} if isinstance(v, (list, tuple)) else v)
|
| 109 |
+
if q.grain and meta['date']:
|
| 110 |
+
rng = (r.get('__range') or {}).get(f"{meta['date']}:{q.grain}") or {}
|
| 111 |
+
rec['period'] = (rng.get('from') or '')[:7]
|
| 112 |
+
rec['count'] = r.get('__count') or 0
|
| 113 |
+
return rec
|
| 114 |
+
|
| 115 |
+
def _by_category(self, q, meta):
|
| 116 |
+
"""Group SALES_ORDER_LINE by main product category by grouping on product then folding
|
| 117 |
+
product->category in-connector (source-specific enrichment lives here, not in the skill)."""
|
| 118 |
+
catmap = _product_category_map()
|
| 119 |
+
sub = _replace_group(q, tuple(g for g in q.group_by if g != 'category') + ('product',))
|
| 120 |
+
agg = {}
|
| 121 |
+
for r in self.aggregate(sub):
|
| 122 |
+
prod = r.get('product') or {}
|
| 123 |
+
cat = catmap.get(prod.get('id') if isinstance(prod, dict) else prod, '(uncategorized)')
|
| 124 |
+
e = agg.setdefault(cat, {'category': cat, 'count': 0})
|
| 125 |
+
for m in q.measures:
|
| 126 |
+
e[m] = e.get(m, 0.0) + (r.get(m) or 0.0)
|
| 127 |
+
e['count'] += r.get('count') or 0
|
| 128 |
+
return sorted(agg.values(), key=lambda x: -(x.get(q.measures[0]) if q.measures else 0))
|
| 129 |
+
|
| 130 |
+
def distinct_count(self, q: Query, fieldname: str) -> int:
|
| 131 |
+
meta = _MAP[q.entity]
|
| 132 |
+
O.set_doc_mode(self._doc_mode)
|
| 133 |
+
return O.distinct_count(meta['model'], self._domain(q, meta), meta['f'].get(fieldname, fieldname))
|
| 134 |
+
|
| 135 |
+
def records(self, q: Query) -> list[dict]:
|
| 136 |
+
meta = _MAP[q.entity]
|
| 137 |
+
O.set_doc_mode(self._doc_mode)
|
| 138 |
+
fields = [meta['f'].get(m, m) for m in (q.measures or tuple(meta['f']))]
|
| 139 |
+
rows = O.search_read(meta['model'], self._domain(q, meta), fields)
|
| 140 |
+
inv = {v: k for k, v in meta['f'].items()}
|
| 141 |
+
return [{inv.get(k, k): (O.m2o_name(v) if isinstance(v, (list, tuple)) else v)
|
| 142 |
+
for k, v in r.items()} for r in rows]
|
| 143 |
+
|
| 144 |
+
# convenience for tenant setup: resolve names β ids for scope config
|
| 145 |
+
@staticmethod
|
| 146 |
+
def excluded_customer_ids() -> list[int]:
|
| 147 |
+
return list(O.excluded_partner_ids())
|
royalimports_os/harness/connectors/quickbooks.py
ADDED
|
@@ -0,0 +1,37 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""QuickBooks Online connector β STUB (scaffolding for the next real integration).
|
| 2 |
+
|
| 3 |
+
It exists to prove the harness is source-agnostic: it declares the capabilities QBO actually has
|
| 4 |
+
(accounting + AR + customers, but NOT SKU-level sales velocity or warehouse inventory), so the
|
| 5 |
+
platform will automatically OFFER ledger/AR/customer skills on a QuickBooks-only tenant and HIDE the
|
| 6 |
+
SKU/inventory skills β no per-skill if-statements anywhere. Fill in aggregate() against the QBO
|
| 7 |
+
Reports/Query API (OAuth2) to make it live.
|
| 8 |
+
"""
|
| 9 |
+
from __future__ import annotations
|
| 10 |
+
from harness.canonical import Capability, Entity, Query
|
| 11 |
+
|
| 12 |
+
|
| 13 |
+
class QuickBooksConnector:
|
| 14 |
+
key = 'quickbooks'
|
| 15 |
+
name = 'QuickBooks Online'
|
| 16 |
+
|
| 17 |
+
def __init__(self, realm_id: str | None = None, token: str | None = None):
|
| 18 |
+
self.realm_id = realm_id
|
| 19 |
+
self._token = token # OAuth2 access token (never logged)
|
| 20 |
+
|
| 21 |
+
def capabilities(self) -> set[Capability]:
|
| 22 |
+
# QBO gives you the books + receivables + the customer list; it does not model SKU-level
|
| 23 |
+
# order lines or physical stock the way an ERP/marketplace does.
|
| 24 |
+
return {Capability.LEDGER, Capability.INVOICING, Capability.PAYABLES, Capability.CUSTOMERS}
|
| 25 |
+
|
| 26 |
+
def entities(self) -> set[Entity]:
|
| 27 |
+
return {Entity.INVOICE, Entity.JOURNAL_LINE, Entity.GL_ACCOUNT, Entity.VENDOR_BILL, Entity.CUSTOMER}
|
| 28 |
+
|
| 29 |
+
def supports(self, cap: Capability) -> bool:
|
| 30 |
+
return cap in self.capabilities()
|
| 31 |
+
|
| 32 |
+
def aggregate(self, q: Query) -> list[dict]:
|
| 33 |
+
raise NotImplementedError(
|
| 34 |
+
"QuickBooksConnector.aggregate: TODO β translate the canonical Query to the QBO "
|
| 35 |
+
"Query/Reports API (GET /v3/company/{realm}/query) and map rows back to canonical dicts. "
|
| 36 |
+
"Entity map: INVOICE->Invoice/CreditMemo, JOURNAL_LINE->JournalEntry lines, "
|
| 37 |
+
"GL_ACCOUNT->Account, VENDOR_BILL->Bill, CUSTOMER->Customer.")
|
royalimports_os/harness/datastore.py
ADDED
|
@@ -0,0 +1,750 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""harness/store.py β the tenant data store (OM-1, 2026-07-11).
|
| 2 |
+
|
| 3 |
+
Incremental Odoo β DuckDB mirror: the local analytical store that saved views/dashboards (OM-3)
|
| 4 |
+
and the AI Analyst's tools (OM-4) query at warehouse speed, instead of hammering live XML-RPC
|
| 5 |
+
per question.
|
| 6 |
+
Plan: .claude/wiki/research/omni-adoption.md Part IV (OM-1) + [[odoo-open-source]] BUILD-NOW #1.
|
| 7 |
+
|
| 8 |
+
Design rules:
|
| 9 |
+
- The store is a RAW mirror (unscoped). Business scope (wholesale teams, GIFTWARE exclusion,
|
| 10 |
+
confirmed-only) is applied by the SEMANTIC layer at query time (OM-2) β one source of truth.
|
| 11 |
+
- Sync is CHECKPOINTED + BOUNDED + RESUMABLE (loop-library discipline): backfill paginates by
|
| 12 |
+
id; live mode advances a write_date cursor; every run is bounded by max_batches and safe to
|
| 13 |
+
kill/re-run (upserts are idempotent).
|
| 14 |
+
- validate() compares the store against LIVE Odoo (raw fidelity: counts + monthly sums to the
|
| 15 |
+
cent) β the store never validates itself. Odoo hard-deletes (unlink) don't move write_date;
|
| 16 |
+
the count checks are the drift detector for those.
|
| 17 |
+
- Client data: the .duckdb file lives under data/store/ (git-ignored). READ-ONLY on Odoo.
|
| 18 |
+
"""
|
| 19 |
+
import datetime as dt
|
| 20 |
+
import json
|
| 21 |
+
import os
|
| 22 |
+
import sys
|
| 23 |
+
import time
|
| 24 |
+
from pathlib import Path
|
| 25 |
+
|
| 26 |
+
import duckdb
|
| 27 |
+
|
| 28 |
+
import core.odoo as O
|
| 29 |
+
|
| 30 |
+
_STORE_DIR = Path(__file__).resolve().parents[1] / "data" / "store"
|
| 31 |
+
|
| 32 |
+
# β MUTABLE ON PURPOSE, and read LATE by every function below (Python resolves a module global at
|
| 33 |
+
# CALL time), which is how `provision_tenant.py` already points a fresh instance at its own file.
|
| 34 |
+
# EXIT-4a (X7) formalises that: `AIOS_DUCKDB_PATH` lets a container name the file without an edit,
|
| 35 |
+
# `path_for()` owns the per-tenant naming convention, and `use_path()` is the SAFE way to switch β
|
| 36 |
+
# see its docstring for why a bare assignment is not.
|
| 37 |
+
# The DEFAULT is unchanged: <app root>/data/store/royal.duckdb.
|
| 38 |
+
DB_PATH = Path(os.environ.get("AIOS_DUCKDB_PATH") or (_STORE_DIR / "royal.duckdb"))
|
| 39 |
+
|
| 40 |
+
|
| 41 |
+
def path_for(tenant_key):
|
| 42 |
+
"""The canonical DuckDB file for a tenant slug. `royal-imports` keeps the historical
|
| 43 |
+
`royal.duckdb` name so tenant #0's existing 175 MB file is not orphaned by a rename.
|
| 44 |
+
|
| 45 |
+
File-per-tenant IS the isolation model (C1c): DuckDB has no row-level security, so the only
|
| 46 |
+
boundary that holds is the operating system's β one file, one tenant.
|
| 47 |
+
"""
|
| 48 |
+
key = str(tenant_key or "").strip().lower()
|
| 49 |
+
if key in ("", "royal-imports"):
|
| 50 |
+
return DB_PATH if key == "" else Path(
|
| 51 |
+
os.environ.get("AIOS_DUCKDB_PATH") or (_STORE_DIR / "royal.duckdb"))
|
| 52 |
+
safe = "".join(c if (c.isalnum() or c in "-_") else "_" for c in key)[:60]
|
| 53 |
+
return _STORE_DIR / f"{safe}.duckdb"
|
| 54 |
+
|
| 55 |
+
|
| 56 |
+
def use_path(path):
|
| 57 |
+
"""Repoint the store at `path`, CLOSING the process singleton first.
|
| 58 |
+
|
| 59 |
+
β A bare `datastore.DB_PATH = β¦` is not enough and is the more dangerous half of this
|
| 60 |
+
operation. `_instance()` caches ONE open connection for the life of the process, so after a
|
| 61 |
+
plain reassignment every read keeps being served from the PREVIOUSLY opened file β a silent
|
| 62 |
+
cross-tenant read, returning real rows that belong to somebody else. Closing the connection
|
| 63 |
+
and clearing the readiness memo (which is also per-file) is what makes the switch honest.
|
| 64 |
+
|
| 65 |
+
β AND CLOSING THE CONNECTION IS STILL NOT ENOUGH ON ITS OWN, because `ro_con()` caches a
|
| 66 |
+
cursor in a `threading.local`: this function can only clear the CALLING thread's, and
|
| 67 |
+
`ro_con`'s liveness probe (`SELECT 1`) SUCCEEDS on a cursor whose underlying connection was
|
| 68 |
+
closed out from under it in some cases β so another thread would keep answering from the old
|
| 69 |
+
file with no error to notice. The generation counter below is what closes that: `ro_con`
|
| 70 |
+
compares the generation its cursor was opened under and reopens when it has moved, which is
|
| 71 |
+
a check no individual thread can forget to do.
|
| 72 |
+
|
| 73 |
+
Returns the new path. Callers: `provision_tenant.py`, and any single-tenant worker process.
|
| 74 |
+
NOT a per-request operation β one process serves one analytical store at a time, which is why
|
| 75 |
+
`harness.runtime` hands out the PATH rather than switching the global underneath a request.
|
| 76 |
+
"""
|
| 77 |
+
global DB_PATH
|
| 78 |
+
with _INSTANCE_LOCK:
|
| 79 |
+
con = _INSTANCE.get("con")
|
| 80 |
+
if con is not None:
|
| 81 |
+
try:
|
| 82 |
+
con.close()
|
| 83 |
+
except Exception:
|
| 84 |
+
pass
|
| 85 |
+
_INSTANCE["con"] = None
|
| 86 |
+
_RO_TLS.__dict__.pop("cur", None)
|
| 87 |
+
_READY["ok"] = False
|
| 88 |
+
_GENERATION[0] += 1 # every other thread's cached cursor is now stale
|
| 89 |
+
DB_PATH = Path(path)
|
| 90 |
+
return DB_PATH
|
| 91 |
+
|
| 92 |
+
# Entity specs: Odoo model β store table. m2o fields land as <name>_id + <name>_name.
|
| 93 |
+
# 'archivable' entities are pulled with active in [True, False] (the re-SKU/merge rule).
|
| 94 |
+
ENTITIES = {
|
| 95 |
+
"sale_order": {
|
| 96 |
+
"model": "sale.order", "archivable": False,
|
| 97 |
+
"fields": ["name", "date_order", "partner_id", "team_id", "user_id", "state",
|
| 98 |
+
"amount_untaxed", "invoice_status", "write_date"],
|
| 99 |
+
},
|
| 100 |
+
"sale_order_line": {
|
| 101 |
+
"model": "sale.order.line", "archivable": False,
|
| 102 |
+
"fields": ["order_id", "order_partner_id", "product_id", "product_uom_qty",
|
| 103 |
+
"price_subtotal", "margin", "purchase_price", "write_date"],
|
| 104 |
+
},
|
| 105 |
+
"res_partner": {
|
| 106 |
+
"model": "res.partner", "archivable": True,
|
| 107 |
+
# `agent` (bool, labelled "Creditor/Agent") is THE discriminator between a real sales
|
| 108 |
+
# AGENT and an internal SALESPERSON who merely carries commission lines β the commission
|
| 109 |
+
# table itself cannot tell them apart. `salesman_as_agent` ("convert salesman into agent")
|
| 110 |
+
# marks agents who came from staff; the owner ruling on whether those count as agents is
|
| 111 |
+
# OUTSTANDING, so both flags are synced and neither reading is baked in.
|
| 112 |
+
"fields": ["name", "team_id", "city", "state_id", "country_id", "active",
|
| 113 |
+
"agent", "salesman_as_agent", "write_date"],
|
| 114 |
+
"backfill_fields": ["agent", "salesman_as_agent"], # added 2026-07-28
|
| 115 |
+
# m2m fields land as <base>_id (the FIRST linked id β the Customers-module convention:
|
| 116 |
+
# a customer's assigned agent is agent_ids[0], ~one per customer, MECE) + <field> as a
|
| 117 |
+
# JSON list of ALL ids. Declared explicitly: a 2-element m2m read would otherwise be
|
| 118 |
+
# indistinguishable from an m2o (id, name) pair in _flatten.
|
| 119 |
+
"m2m": ["agent_ids"],
|
| 120 |
+
},
|
| 121 |
+
"product_product": {
|
| 122 |
+
"model": "product.product", "archivable": True,
|
| 123 |
+
"fields": ["default_code", "name", "categ_id", "type", "standard_price", "active",
|
| 124 |
+
"write_date"],
|
| 125 |
+
},
|
| 126 |
+
"account_move": {
|
| 127 |
+
"model": "account.move", "archivable": False,
|
| 128 |
+
"fields": ["name", "move_type", "state", "invoice_date", "invoice_date_due",
|
| 129 |
+
"partner_id", "amount_untaxed_signed", "amount_residual_signed",
|
| 130 |
+
"payment_state", "write_date"],
|
| 131 |
+
},
|
| 132 |
+
"account_move_line": {
|
| 133 |
+
"model": "account.move.line", "archivable": False,
|
| 134 |
+
# display_type/move_type/parent_state/price_subtotal/product_id added 2026-07-28 so the
|
| 135 |
+
# invoice-LINE grain is queryable (topic `invoice_lines`).
|
| 136 |
+
# β display_type values here are 'product', 'cogs', 'payment_term', 'line_note',
|
| 137 |
+
# 'line_section'. Commission rows attach to 'cogs' lines as well as 'product' ones
|
| 138 |
+
# (96,936 of 147,160 in 2025). Filtering to 'product' is a GRAIN guard: measured
|
| 139 |
+
# 2026-07-29, omitting it does not move revenue (non-product lines carry
|
| 140 |
+
# price_subtotal = 0) but it inflates commission ROW COUNTS ~80% and would corrupt any
|
| 141 |
+
# count-of-lines measure.
|
| 142 |
+
"fields": ["move_id", "account_id", "partner_id", "date", "debit", "credit",
|
| 143 |
+
"balance", "display_type", "move_type", "parent_state", "price_subtotal",
|
| 144 |
+
"product_id", "write_date"],
|
| 145 |
+
"backfill_fields": ["display_type", "move_type", "parent_state", "price_subtotal",
|
| 146 |
+
"product_id"], # added 2026-07-28
|
| 147 |
+
# the BU bridge: account.move carries NO business unit (every invoice sits on team 1 β
|
| 148 |
+
# see [[invoice-bu-attribution]]), so Fisch/Royal on the invoice basis is only reachable
|
| 149 |
+
# through the originating sale order. sale_line_id (the FIRST linked sale line) is
|
| 150 |
+
# lossless here: ZERO 2025 invoice lines span more than one team (measured).
|
| 151 |
+
"m2m": ["sale_line_ids"],
|
| 152 |
+
},
|
| 153 |
+
"account_invoice_line_agent": {
|
| 154 |
+
# The OCA sale-commission module: one row per (invoice line Γ agent). This is the SECOND
|
| 155 |
+
# agent source β per-line commission attribution β and it names a different population
|
| 156 |
+
# than res_partner.agent_ids (the customer-master book). See topic `invoice_lines`.
|
| 157 |
+
# β TWO similarly-named FKs, do not confuse them:
|
| 158 |
+
# object_id -> account_move_line (THE grain; join on this)
|
| 159 |
+
# invoice_id -> account_move (the parent document)
|
| 160 |
+
"model": "account.invoice.line.agent", "archivable": False,
|
| 161 |
+
"fields": ["agent_id", "commission_id", "amount", "invoice_id", "object_id",
|
| 162 |
+
"invoice_date", "settled", "write_date"],
|
| 163 |
+
},
|
| 164 |
+
"account_account": {
|
| 165 |
+
# NOTE: no `active` field on account.account in this Odoo version (learned 2026-07-11 β
|
| 166 |
+
# a ('active','in',...) domain 500s); treat as non-archivable.
|
| 167 |
+
"model": "account.account", "archivable": False,
|
| 168 |
+
"fields": ["code", "name", "account_type", "write_date"],
|
| 169 |
+
},
|
| 170 |
+
}
|
| 171 |
+
|
| 172 |
+
BATCH = 2000
|
| 173 |
+
|
| 174 |
+
# Fields that are genuinely BOOLEAN. This list is load-bearing in TWO places: _cols types the
|
| 175 |
+
# column, and _flatten must NOT collapse a real False into NULL. Odoo returns False both for
|
| 176 |
+
# "empty" and for "boolean false", so a bool field missing from here silently becomes NULL β and
|
| 177 |
+
# for res_partner.agent that would erase the ONE flag separating an AGENT from a SALESPERSON
|
| 178 |
+
# (see [[invoice-line-agent-commission]]): every row would read "not an agent" indistinguishably
|
| 179 |
+
# from "unknown".
|
| 180 |
+
BOOL_FIELDS = ("active", "agent", "salesman_as_agent", "settled", "commission_free")
|
| 181 |
+
|
| 182 |
+
|
| 183 |
+
import threading as _thr
|
| 184 |
+
|
| 185 |
+
_RO_TLS = _thr.local()
|
| 186 |
+
_READY = {"ok": False}
|
| 187 |
+
_INSTANCE = {"con": None}
|
| 188 |
+
_INSTANCE_LOCK = _thr.Lock()
|
| 189 |
+
#: Bumped by `use_path()` whenever the store FILE changes. Every per-thread cursor is stamped with
|
| 190 |
+
#: the generation it was opened under, so a thread that never called `use_path` still cannot keep
|
| 191 |
+
#: reading the previous tenant's file β see `ro_con`. A plain "is the cursor alive?" probe cannot
|
| 192 |
+
#: answer that question, which is why this counter exists rather than more probing.
|
| 193 |
+
_GENERATION = [0]
|
| 194 |
+
|
| 195 |
+
|
| 196 |
+
def mark_ready():
|
| 197 |
+
"""In-process signal that the store is fully synced (every entity live). The sync writer calls
|
| 198 |
+
this after an all-live pass so a reader never has to (re)discover usability by touching the
|
| 199 |
+
file. Belt-and-suspenders alongside the shared-instance model below."""
|
| 200 |
+
_READY["ok"] = True
|
| 201 |
+
|
| 202 |
+
|
| 203 |
+
def _instance():
|
| 204 |
+
"""THE one process-wide read-write DuckDB connection. Readers take .cursor() off it and the
|
| 205 |
+
sync writer uses it too β DuckDB serves concurrent reads + a single writer from ONE instance
|
| 206 |
+
via MVCC, so a read NEVER contends with a sync pass for the file lock.
|
| 207 |
+
|
| 208 |
+
This is the fix for the 'connection error unless I refresh' bug (2026-07-16): an independent
|
| 209 |
+
read-only `duckdb.connect()` FAILS while the writer holds the lock ('Can't open a connection
|
| 210 |
+
to same database file with a different configuration than existing connections'); a cursor on
|
| 211 |
+
the shared instance does not. Openers must ensure the file is fully present first β ensure_seed
|
| 212 |
+
writes atomically (os.replace) and every read path guards on DB_PATH.exists() β so the instance
|
| 213 |
+
never binds a partial or absent file. (Trade-off: the app now holds the write lock for its whole
|
| 214 |
+
life, so a SEPARATE process β e.g. mcp_server.py β cannot open the same store file concurrently;
|
| 215 |
+
in production only the app runs, and that was already true during any sync pass.)"""
|
| 216 |
+
con = _INSTANCE["con"]
|
| 217 |
+
if con is not None:
|
| 218 |
+
return con
|
| 219 |
+
with _INSTANCE_LOCK:
|
| 220 |
+
if _INSTANCE["con"] is None:
|
| 221 |
+
DB_PATH.parent.mkdir(parents=True, exist_ok=True)
|
| 222 |
+
_INSTANCE["con"] = duckdb.connect(str(DB_PATH))
|
| 223 |
+
return _INSTANCE["con"]
|
| 224 |
+
|
| 225 |
+
|
| 226 |
+
def ro_cursor():
|
| 227 |
+
"""A read cursor on the shared instance β the contention-free way to read the store."""
|
| 228 |
+
return _instance().cursor()
|
| 229 |
+
|
| 230 |
+
|
| 231 |
+
def ready():
|
| 232 |
+
"""True once EVERY entity has completed its backfill (phase 'live'). Gate all store READS on
|
| 233 |
+
this: a mid-backfill store would return PARTIAL totals silently β worse than an error. Positive
|
| 234 |
+
results cache for the process (backfill never regresses). Reads via a shared-instance cursor,
|
| 235 |
+
so it is never mis-reported as 'not ready' just because a background sync pass is mid-flight."""
|
| 236 |
+
if _READY["ok"]:
|
| 237 |
+
return True
|
| 238 |
+
if not DB_PATH.exists():
|
| 239 |
+
return False
|
| 240 |
+
try:
|
| 241 |
+
cur = ro_cursor()
|
| 242 |
+
try:
|
| 243 |
+
phases = {r[0]: r[1] for r in
|
| 244 |
+
cur.execute("SELECT entity, phase FROM _sync_state").fetchall()}
|
| 245 |
+
finally:
|
| 246 |
+
cur.close()
|
| 247 |
+
except Exception:
|
| 248 |
+
return False
|
| 249 |
+
if all(phases.get(k) == "live" for k in ENTITIES):
|
| 250 |
+
_READY["ok"] = True
|
| 251 |
+
return True
|
| 252 |
+
return False
|
| 253 |
+
|
| 254 |
+
|
| 255 |
+
def ro_con():
|
| 256 |
+
"""Per-thread cached read CURSOR on the shared instance for page/module store reads (the
|
| 257 |
+
open cost dominates repeated small queries β the Expenses retrofit measured 7.7sβ1.0s). No
|
| 258 |
+
lock contention with the sync writer; a cursor that has gone bad is reopened once. REFUSES
|
| 259 |
+
until ready() β callers either fall back to live reads (modules) or relay a readable message
|
| 260 |
+
(Analyst).
|
| 261 |
+
|
| 262 |
+
β THE GENERATION CHECK IS A CORRECTNESS GUARD, NOT AN OPTIMISATION (EXIT-4a). The liveness
|
| 263 |
+
probe below asks "does this cursor still work?", which is a different question from "is this
|
| 264 |
+
cursor still pointing at the file we are supposed to be reading?". After `use_path()` switches
|
| 265 |
+
tenants, a cursor cached in ANOTHER thread's `threading.local` can still answer `SELECT 1` β
|
| 266 |
+
and would then serve that thread real, plausible rows from the PREVIOUS tenant's file, with no
|
| 267 |
+
exception anywhere. `use_path` bumps the generation; a cursor opened under an older one is
|
| 268 |
+
discarded here, which is the only place every thread is guaranteed to pass through."""
|
| 269 |
+
cur = getattr(_RO_TLS, "cur", None)
|
| 270 |
+
if cur is not None and getattr(_RO_TLS, "gen", None) != _GENERATION[0]:
|
| 271 |
+
try:
|
| 272 |
+
cur.close()
|
| 273 |
+
except Exception:
|
| 274 |
+
pass
|
| 275 |
+
cur, _RO_TLS.cur = None, None # the store moved underneath this thread
|
| 276 |
+
if cur is not None:
|
| 277 |
+
try:
|
| 278 |
+
cur.execute("SELECT 1") # cheap liveness probe
|
| 279 |
+
return cur
|
| 280 |
+
except Exception:
|
| 281 |
+
_RO_TLS.cur = None # stale β reopen below
|
| 282 |
+
if not ready():
|
| 283 |
+
raise RuntimeError("the tenant data store is completing its FIRST sync on this "
|
| 284 |
+
"deployment β dashboards work meanwhile (live reads); retry "
|
| 285 |
+
"store queries in a few minutes")
|
| 286 |
+
cur = ro_cursor()
|
| 287 |
+
_RO_TLS.cur = cur
|
| 288 |
+
_RO_TLS.gen = _GENERATION[0] # stamp WHICH file this cursor belongs to
|
| 289 |
+
return cur
|
| 290 |
+
|
| 291 |
+
|
| 292 |
+
SEED_DATASET = os.environ.get("STORE_SEED_DATASET", "royal-imports/cfo-os-data")
|
| 293 |
+
|
| 294 |
+
|
| 295 |
+
def ensure_seed():
|
| 296 |
+
"""Fresh/ephemeral deployment (HF Space disks reset on every rebuild): hydrate the store
|
| 297 |
+
from the PRIVATE dataset's seed snapshot β Analyst-ready in ~a minute instead of a 20-40 min
|
| 298 |
+
XML-RPC backfill; the auto-sync then closes the gap via write_date cursors. Also restores
|
| 299 |
+
the pilot views/routines when absent. Fail-quiet: no seed/token β normal backfill."""
|
| 300 |
+
import shutil
|
| 301 |
+
if DB_PATH.exists():
|
| 302 |
+
return False
|
| 303 |
+
tok = os.environ.get("HF_TOKEN")
|
| 304 |
+
if not tok:
|
| 305 |
+
return False
|
| 306 |
+
try:
|
| 307 |
+
from huggingface_hub import hf_hub_download
|
| 308 |
+
DB_PATH.parent.mkdir(parents=True, exist_ok=True)
|
| 309 |
+
p = hf_hub_download(SEED_DATASET, "store_seed/royal.duckdb",
|
| 310 |
+
repo_type="dataset", token=tok)
|
| 311 |
+
# Atomic install: copy to a temp path then os.replace β a reader that guards on
|
| 312 |
+
# DB_PATH.exists() must never bind a half-written file into the shared instance.
|
| 313 |
+
tmp = DB_PATH.with_name(DB_PATH.name + ".tmp")
|
| 314 |
+
shutil.copyfile(p, tmp)
|
| 315 |
+
os.replace(tmp, DB_PATH)
|
| 316 |
+
for fn in ("views.json", "routines.json"):
|
| 317 |
+
tgt = DB_PATH.parent / fn
|
| 318 |
+
if not tgt.exists():
|
| 319 |
+
try:
|
| 320 |
+
q = hf_hub_download(SEED_DATASET, f"store_seed/{fn}",
|
| 321 |
+
repo_type="dataset", token=tok)
|
| 322 |
+
shutil.copyfile(q, tgt)
|
| 323 |
+
except Exception:
|
| 324 |
+
pass
|
| 325 |
+
return True
|
| 326 |
+
except Exception:
|
| 327 |
+
return False
|
| 328 |
+
|
| 329 |
+
|
| 330 |
+
def connect():
|
| 331 |
+
"""A cursor on the shared instance for the SYNC WRITER (and status/validate). Returned as a
|
| 332 |
+
cursor so callers' con.close() frees the cursor without closing the process-wide instance;
|
| 333 |
+
one writer cursor is active at a time (the sync thread is sequential), readers use others."""
|
| 334 |
+
cur = _instance().cursor()
|
| 335 |
+
cur.execute("""CREATE TABLE IF NOT EXISTS _sync_state (
|
| 336 |
+
entity VARCHAR PRIMARY KEY, phase VARCHAR, last_id BIGINT,
|
| 337 |
+
cursor_wd VARCHAR, rows BIGINT, updated_at VARCHAR)""")
|
| 338 |
+
return cur
|
| 339 |
+
|
| 340 |
+
|
| 341 |
+
def _flatten(rec, fields, m2m=()):
|
| 342 |
+
out = {"id": rec["id"]}
|
| 343 |
+
for f in m2m:
|
| 344 |
+
v = rec.get(f) or []
|
| 345 |
+
base = f.removesuffix("_ids")
|
| 346 |
+
out[f"{base}_id"] = v[0] if v else None
|
| 347 |
+
out[f] = json.dumps(list(v)) if v else None
|
| 348 |
+
for f in fields:
|
| 349 |
+
v = rec.get(f)
|
| 350 |
+
if isinstance(v, (list, tuple)) and len(v) == 2 and isinstance(v[0], int):
|
| 351 |
+
out[f"{f.removesuffix('_id')}_id"] = v[0]
|
| 352 |
+
out[f"{f.removesuffix('_id')}_name"] = str(v[1])
|
| 353 |
+
elif isinstance(v, (list, tuple)):
|
| 354 |
+
out[f] = json.dumps(v)
|
| 355 |
+
elif v is False and f not in BOOL_FIELDS: # Odoo False = NULL for non-bool fields
|
| 356 |
+
out[f] = None
|
| 357 |
+
else:
|
| 358 |
+
out[f] = v
|
| 359 |
+
return out
|
| 360 |
+
|
| 361 |
+
|
| 362 |
+
def _cols(spec):
|
| 363 |
+
cols = ["id BIGINT PRIMARY KEY"]
|
| 364 |
+
for f in spec["fields"]:
|
| 365 |
+
base = f.removesuffix("_id")
|
| 366 |
+
if f.endswith("_id"):
|
| 367 |
+
cols += [f"{base}_id BIGINT", f"{base}_name VARCHAR"]
|
| 368 |
+
elif f in BOOL_FIELDS:
|
| 369 |
+
cols.append(f"{f} BOOLEAN")
|
| 370 |
+
elif f in ("amount_untaxed", "amount_untaxed_signed", "amount_residual_signed",
|
| 371 |
+
"price_subtotal", "margin", "purchase_price", "product_uom_qty",
|
| 372 |
+
"standard_price", "debit", "credit", "balance", "amount"):
|
| 373 |
+
cols.append(f"{f} DOUBLE")
|
| 374 |
+
else:
|
| 375 |
+
cols.append(f"{f} VARCHAR")
|
| 376 |
+
for f in spec.get("m2m") or []:
|
| 377 |
+
base = f.removesuffix("_ids")
|
| 378 |
+
cols += [f"{base}_id BIGINT", f"{f} VARCHAR"]
|
| 379 |
+
return cols
|
| 380 |
+
|
| 381 |
+
|
| 382 |
+
def _ensure_table(con, key, spec):
|
| 383 |
+
con.execute(f"CREATE TABLE IF NOT EXISTS {key} ({', '.join(_cols(spec))})")
|
| 384 |
+
|
| 385 |
+
|
| 386 |
+
def _ensure_columns(con, key, spec):
|
| 387 |
+
"""Schema evolution: existing stores/seeds predate columns a newer spec introduces β
|
| 388 |
+
CREATE TABLE IF NOT EXISTS won't add them. Returns the newly added column names."""
|
| 389 |
+
have = {r[1] for r in con.execute(f"PRAGMA table_info('{key}')").fetchall()}
|
| 390 |
+
added = []
|
| 391 |
+
for c in _cols(spec):
|
| 392 |
+
name = c.split()[0]
|
| 393 |
+
if name not in have:
|
| 394 |
+
con.execute(f"ALTER TABLE {key} ADD COLUMN {c}")
|
| 395 |
+
added.append(name)
|
| 396 |
+
return added
|
| 397 |
+
|
| 398 |
+
|
| 399 |
+
def _field_cols(f):
|
| 400 |
+
"""The store column(s) one spec field materialises into (m2o fields become _id + _name)."""
|
| 401 |
+
if f.endswith("_id"):
|
| 402 |
+
base = f.removesuffix("_id")
|
| 403 |
+
return [f"{base}_id", f"{base}_name"]
|
| 404 |
+
return [f]
|
| 405 |
+
|
| 406 |
+
|
| 407 |
+
def _backfill_columns(con, key, spec, log=print):
|
| 408 |
+
"""Scalar/m2o schema evolution β the twin of _backfill_m2m, and for the same reason.
|
| 409 |
+
|
| 410 |
+
_ensure_columns ALTERs a new column in, but existing rows NEVER re-sync (WE moved the schema;
|
| 411 |
+
their write_date didn't move), so a newly added field stays NULL forever β silently, and NULL
|
| 412 |
+
is indistinguishable from a legitimately empty value. That is how a store starts answering
|
| 413 |
+
'no rows match' instead of erroring. Re-pull just the affected fields for every row,
|
| 414 |
+
paginated by id, marked durably so a crash retries rather than declaring completion.
|
| 415 |
+
|
| 416 |
+
The fields are DECLARED in the spec (`backfill_fields`), never inferred from "which columns
|
| 417 |
+
did _ensure_columns just add" β that inference is wrong on the second run, when the ALTER has
|
| 418 |
+
already happened and the list comes back empty while the data is still missing. Declaring it
|
| 419 |
+
makes the migration re-entrant and reviewable. Leave the entry in place after it completes;
|
| 420 |
+
the durable marker, not the absence of the declaration, is what stops it re-running.
|
| 421 |
+
"""
|
| 422 |
+
fields = [f for f in (spec.get("backfill_fields") or []) if f != "write_date"]
|
| 423 |
+
if not fields:
|
| 424 |
+
return
|
| 425 |
+
marker = f"{key}.cols.{','.join(sorted(fields))}"
|
| 426 |
+
st = con.execute("SELECT phase, last_id, rows FROM _sync_state WHERE entity=?",
|
| 427 |
+
[marker]).fetchone()
|
| 428 |
+
if st and st[0] == "done":
|
| 429 |
+
return
|
| 430 |
+
# RESUMABLE, not restart-from-zero: these passes run to ~1M rows, and a migration that loses
|
| 431 |
+
# an hour of work to one dropped connection gets skipped by the next person under time
|
| 432 |
+
# pressure. Progress is checkpointed every page; 'done' is written only at the end, so an
|
| 433 |
+
# interrupted run resumes at the last committed id instead of re-pulling or, worse, declaring
|
| 434 |
+
# completion it never reached.
|
| 435 |
+
last, n = (st[1] or 0, st[2] or 0) if st else (0, 0)
|
| 436 |
+
if last:
|
| 437 |
+
log(f" {key}: column backfill resuming at id {last:,} ({n:,} rows already done)")
|
| 438 |
+
cols = [c for f in fields for c in _field_cols(f)]
|
| 439 |
+
setter = ", ".join(f"{c}=?" for c in cols)
|
| 440 |
+
while True:
|
| 441 |
+
recs = O.search_read(spec["model"], _base_domain(spec) + [("id", ">", last)],
|
| 442 |
+
fields, limit=5000, order="id asc")
|
| 443 |
+
if not recs:
|
| 444 |
+
break
|
| 445 |
+
rows = [_flatten(r, fields) for r in recs]
|
| 446 |
+
con.executemany(f"UPDATE {key} SET {setter} WHERE id=?",
|
| 447 |
+
[[r.get(c) for c in cols] + [r["id"]] for r in rows])
|
| 448 |
+
last, n = recs[-1]["id"], n + len(recs)
|
| 449 |
+
con.execute("INSERT OR REPLACE INTO _sync_state VALUES (?,?,?,?,?,?)",
|
| 450 |
+
[marker, "running", last, None, n, time.strftime("%Y-%m-%d %H:%M:%S")])
|
| 451 |
+
log(f" {key}: column backfill {'+'.join(fields)} -> id {last:,} ({n:,} rows)")
|
| 452 |
+
con.execute("INSERT OR REPLACE INTO _sync_state VALUES (?,?,?,?,?,?)",
|
| 453 |
+
[marker, "done", last, None, n, time.strftime("%Y-%m-%d %H:%M:%S")])
|
| 454 |
+
log(f" {key}: column backfill complete -> {n:,} rows")
|
| 455 |
+
|
| 456 |
+
|
| 457 |
+
def _backfill_m2m(con, key, spec, log=print):
|
| 458 |
+
"""One-time targeted backfill after a schema migration: existing rows never re-sync (their
|
| 459 |
+
write_date didn't move when WE added a column), so pull every record where the m2m field is
|
| 460 |
+
SET and update in place. Rows with the field empty keep NULL β correct and free. Completion
|
| 461 |
+
is a durable _sync_state marker ('<key>.m2m.<field>') β NOT 'column just added': a crash
|
| 462 |
+
between the ALTER and this backfill (seen live: the SSL-failed first run) must retry."""
|
| 463 |
+
for f in spec.get("m2m") or []:
|
| 464 |
+
marker = f"{key}.m2m.{f}"
|
| 465 |
+
if con.execute("SELECT 1 FROM _sync_state WHERE entity=?", [marker]).fetchone():
|
| 466 |
+
continue
|
| 467 |
+
base = f.removesuffix("_ids")
|
| 468 |
+
# PAGINATED by id. A single limit=100000 call silently TRUNCATED here: account_move_line
|
| 469 |
+
# has 228,067 rows with sale_line_ids set, so 56% of the BU bridge would have gone
|
| 470 |
+
# missing with no error β a partial backfill that reads as a complete one (the
|
| 471 |
+
# [[no-unverifiable-aggregates]] no-silent-caps rule).
|
| 472 |
+
rows, last, page = [], 0, 20000
|
| 473 |
+
while True:
|
| 474 |
+
recs = O.search_read(spec["model"],
|
| 475 |
+
_base_domain(spec) + [(f, "!=", False), ("id", ">", last)],
|
| 476 |
+
[f], limit=page, order="id asc")
|
| 477 |
+
if not recs:
|
| 478 |
+
break
|
| 479 |
+
rows += [(r[f][0], json.dumps(list(r[f])), r["id"]) for r in recs if r.get(f)]
|
| 480 |
+
last = recs[-1]["id"]
|
| 481 |
+
log(f" {key}: m2m {f} scanned to id {last:,} ({len(rows):,} rows)")
|
| 482 |
+
if rows:
|
| 483 |
+
con.executemany(f"UPDATE {key} SET {base}_id=?, {f}=? WHERE id=?", rows)
|
| 484 |
+
con.execute("INSERT OR REPLACE INTO _sync_state VALUES (?,?,?,?,?,?)",
|
| 485 |
+
[marker, "done", 0, None, len(rows), time.strftime("%Y-%m-%d %H:%M:%S")])
|
| 486 |
+
log(f" {key}: m2m backfill {f} -> {len(rows):,} rows updated")
|
| 487 |
+
|
| 488 |
+
|
| 489 |
+
def _upsert(con, key, spec, recs):
|
| 490 |
+
if not recs:
|
| 491 |
+
return
|
| 492 |
+
rows = [_flatten(r, spec["fields"], spec.get("m2m") or ()) for r in recs]
|
| 493 |
+
colnames = [c.split()[0] for c in _cols(spec)]
|
| 494 |
+
ids = [r["id"] for r in rows]
|
| 495 |
+
con.execute(f"DELETE FROM {key} WHERE id IN ({','.join(map(str, ids))})")
|
| 496 |
+
con.executemany(
|
| 497 |
+
f"INSERT INTO {key} ({', '.join(colnames)}) VALUES ({', '.join('?' for _ in colnames)})",
|
| 498 |
+
[[r.get(c) for c in colnames] for r in rows])
|
| 499 |
+
|
| 500 |
+
|
| 501 |
+
def _state(con, key):
|
| 502 |
+
row = con.execute("SELECT phase, last_id, cursor_wd, rows FROM _sync_state WHERE entity=?",
|
| 503 |
+
[key]).fetchone()
|
| 504 |
+
return {"phase": row[0], "last_id": row[1], "cursor_wd": row[2], "rows": row[3]} if row else None
|
| 505 |
+
|
| 506 |
+
|
| 507 |
+
def _save_state(con, key, phase, last_id, cursor_wd):
|
| 508 |
+
n = con.execute(f"SELECT count(*) FROM {key}").fetchone()[0]
|
| 509 |
+
con.execute("""INSERT OR REPLACE INTO _sync_state VALUES (?,?,?,?,?,?)""",
|
| 510 |
+
[key, phase, last_id, cursor_wd, n, time.strftime("%Y-%m-%d %H:%M:%S")])
|
| 511 |
+
|
| 512 |
+
|
| 513 |
+
def _base_domain(spec):
|
| 514 |
+
return [("active", "in", [True, False])] if spec["archivable"] else []
|
| 515 |
+
|
| 516 |
+
|
| 517 |
+
def sync_entity(key, max_batches=200, batch=BATCH, log=print):
|
| 518 |
+
"""One bounded, resumable sync pass for an entity. Backfill (by id) β live (by write_date).
|
| 519 |
+
Returns (phase, pulled_rows)."""
|
| 520 |
+
spec = ENTITIES[key]
|
| 521 |
+
con = connect()
|
| 522 |
+
_ensure_table(con, key, spec)
|
| 523 |
+
_ensure_columns(con, key, spec)
|
| 524 |
+
_backfill_columns(con, key, spec, log=log)
|
| 525 |
+
_backfill_m2m(con, key, spec, log=log)
|
| 526 |
+
st = _state(con, key) or {"phase": "backfill", "last_id": 0, "cursor_wd": None, "rows": 0}
|
| 527 |
+
fields = spec["fields"] + list(spec.get("m2m") or [])
|
| 528 |
+
pulled = 0
|
| 529 |
+
try:
|
| 530 |
+
if st["phase"] == "backfill":
|
| 531 |
+
last_id = st["last_id"] or 0
|
| 532 |
+
for i in range(max_batches):
|
| 533 |
+
dom = _base_domain(spec) + [("id", ">", last_id)]
|
| 534 |
+
recs = O.search_read(spec["model"], dom, ["id"] + fields,
|
| 535 |
+
limit=batch, order="id asc")
|
| 536 |
+
if not recs:
|
| 537 |
+
# backfill complete β switch to live mode anchored at max write_date seen
|
| 538 |
+
wd = con.execute(f"SELECT max(write_date) FROM {key}").fetchone()[0]
|
| 539 |
+
_save_state(con, key, "live", last_id, wd or "1970-01-01 00:00:00")
|
| 540 |
+
log(f" {key}: BACKFILL COMPLETE ({st['rows'] + pulled:,} rows) -> live mode")
|
| 541 |
+
return "live", pulled
|
| 542 |
+
_upsert(con, key, spec, recs)
|
| 543 |
+
last_id = recs[-1]["id"]
|
| 544 |
+
pulled += len(recs)
|
| 545 |
+
_save_state(con, key, "backfill", last_id, None)
|
| 546 |
+
if (i + 1) % 10 == 0:
|
| 547 |
+
log(f" {key}: backfill β¦{pulled:,} rows (idβ€{last_id})")
|
| 548 |
+
log(f" {key}: backfill PAUSED at {pulled:,} rows this run (bounded; resumes)")
|
| 549 |
+
return "backfill", pulled
|
| 550 |
+
# live mode: write_date >= cursor (idempotent overlap; upsert dedupes). MASS-STAMP
|
| 551 |
+
# FALLBACK (found live 2026-07-17): a batch job can stamp >batch rows with ONE
|
| 552 |
+
# write_date second (an Odoo recompute stamped ~88k account_move rows '2026-07-15
|
| 553 |
+
# 16:35:03'; stored values carry microseconds, XML-RPC strings don't) β then the
|
| 554 |
+
# write_date cursor can NEVER advance and every pass re-pulls the same first page
|
| 555 |
+
# forever. When a full batch leaves the cursor unchanged we switch to an ID-WALK over
|
| 556 |
+
# write_date >= cursor (order id asc, id > tie), checkpointed as 'cursor|<id>' in
|
| 557 |
+
# cursor_wd so a killed walk resumes; on completion the cursor jumps one second past
|
| 558 |
+
# the stamp (safe: the walk covered every row in that second, and the stamp is in the
|
| 559 |
+
# past β guarded by a 2-minute recency check).
|
| 560 |
+
raw = st["cursor_wd"] or "1970-01-01 00:00:00"
|
| 561 |
+
cursor, _pipe, _tie = raw.partition("|")
|
| 562 |
+
walking, tie_id = bool(_pipe), int(_tie or 0)
|
| 563 |
+
walk_max = cursor
|
| 564 |
+
for _ in range(max_batches):
|
| 565 |
+
if walking:
|
| 566 |
+
dom = _base_domain(spec) + [("write_date", ">=", cursor), ("id", ">", tie_id)]
|
| 567 |
+
recs = O.search_read(spec["model"], dom, ["id"] + fields,
|
| 568 |
+
limit=batch, order="id asc")
|
| 569 |
+
if not recs: # walk complete β advance past the stamp
|
| 570 |
+
try:
|
| 571 |
+
wm = dt.datetime.strptime(walk_max, "%Y-%m-%d %H:%M:%S")
|
| 572 |
+
if wm < dt.datetime.utcnow() - dt.timedelta(minutes=2):
|
| 573 |
+
cursor = (wm + dt.timedelta(seconds=1)).strftime("%Y-%m-%d %H:%M:%S")
|
| 574 |
+
else:
|
| 575 |
+
cursor = walk_max
|
| 576 |
+
except ValueError:
|
| 577 |
+
cursor = walk_max
|
| 578 |
+
walking = False
|
| 579 |
+
log(f" {key}: id-walk complete -> cursor {cursor}")
|
| 580 |
+
break
|
| 581 |
+
_upsert(con, key, spec, recs)
|
| 582 |
+
pulled += len(recs)
|
| 583 |
+
tie_id = recs[-1]["id"]
|
| 584 |
+
walk_max = max(walk_max, max(str(r["write_date"]) for r in recs))
|
| 585 |
+
_save_state(con, key, "live", st["last_id"], f"{cursor}|{tie_id}")
|
| 586 |
+
continue
|
| 587 |
+
dom = _base_domain(spec) + [("write_date", ">=", cursor)]
|
| 588 |
+
recs = O.search_read(spec["model"], dom, ["id"] + fields,
|
| 589 |
+
limit=batch, order="write_date asc, id asc")
|
| 590 |
+
if not recs or (len(recs) == 1 and str(recs[0].get("write_date")) == cursor
|
| 591 |
+
and pulled == 0 and _already(con, key, recs[0])):
|
| 592 |
+
break
|
| 593 |
+
new_cursor = str(recs[-1]["write_date"])
|
| 594 |
+
if new_cursor == cursor and len(recs) == batch:
|
| 595 |
+
# full batch stuck on one second: enter the id-walk from id 0 (the write_date
|
| 596 |
+
# ordering shuffles ids within the stamp second, so the batch just pulled is
|
| 597 |
+
# NOT an id prefix β restart coverage by id; upserts make the overlap free)
|
| 598 |
+
walking, tie_id, walk_max = True, 0, cursor
|
| 599 |
+
_save_state(con, key, "live", st["last_id"], f"{cursor}|0")
|
| 600 |
+
log(f" {key}: mass-stamp second at {cursor} -> switching to id-walk")
|
| 601 |
+
continue
|
| 602 |
+
_upsert(con, key, spec, recs)
|
| 603 |
+
pulled += len(recs)
|
| 604 |
+
if new_cursor == cursor and len(recs) < batch:
|
| 605 |
+
cursor = new_cursor
|
| 606 |
+
break
|
| 607 |
+
cursor = new_cursor
|
| 608 |
+
# Checkpoint the cursor EVERY batch: a big delta (88k account_move rows after the
|
| 609 |
+
# 2026-07-15 recompute) means a killed pass otherwise re-pulls everything β
|
| 610 |
+
# upserts are idempotent, but the wasted RPC is real (learned 2026-07-17).
|
| 611 |
+
_save_state(con, key, "live", st["last_id"], cursor)
|
| 612 |
+
_save_state(con, key, "live", st["last_id"],
|
| 613 |
+
f"{cursor}|{tie_id}" if walking else cursor)
|
| 614 |
+
if pulled:
|
| 615 |
+
log(f" {key}: live sync +{pulled:,} rows (cursor {cursor}"
|
| 616 |
+
+ (f" walking id>{tie_id}" if walking else "") + ")")
|
| 617 |
+
return "live", pulled
|
| 618 |
+
finally:
|
| 619 |
+
con.close()
|
| 620 |
+
|
| 621 |
+
|
| 622 |
+
def _already(con, key, rec):
|
| 623 |
+
return bool(con.execute(f"SELECT 1 FROM {key} WHERE id=?", [rec["id"]]).fetchone())
|
| 624 |
+
|
| 625 |
+
|
| 626 |
+
def sync_all(entities=None, max_batches=200, log=print):
|
| 627 |
+
out = {}
|
| 628 |
+
for key in (entities or list(ENTITIES)):
|
| 629 |
+
t0 = time.time()
|
| 630 |
+
phase, pulled = sync_entity(key, max_batches=max_batches, log=log)
|
| 631 |
+
out[key] = {"phase": phase, "pulled": pulled, "secs": round(time.time() - t0, 1)}
|
| 632 |
+
# A full pass that lands every entity in live mode latches process readiness in-process β so
|
| 633 |
+
# readers never race the writer's lock to (re)discover the store is usable.
|
| 634 |
+
if entities is None and out and all(v["phase"] == "live" for v in out.values()):
|
| 635 |
+
mark_ready()
|
| 636 |
+
return out
|
| 637 |
+
|
| 638 |
+
|
| 639 |
+
def reconcile_deletes(entities=None, log=print):
|
| 640 |
+
"""Remove store rows hard-deleted in Odoo. unlink() doesn't move write_date, so the cursor
|
| 641 |
+
sync can never see a deletion β the count parity DETECTS the drift (by design); this repairs
|
| 642 |
+
it. Pulls the full live id set per entity in bounded id-ascending pages (id-only reads are
|
| 643 |
+
cheap) and deletes store ids not present live. Run from sync_runner / maintenance, not the
|
| 644 |
+
app's frequent passes."""
|
| 645 |
+
con = connect()
|
| 646 |
+
try:
|
| 647 |
+
out = {}
|
| 648 |
+
for key in (entities or list(ENTITIES)):
|
| 649 |
+
spec = ENTITIES[key]
|
| 650 |
+
live_ids, last = set(), 0
|
| 651 |
+
while True:
|
| 652 |
+
recs = O.search_read(spec["model"], _base_domain(spec) + [("id", ">", last)],
|
| 653 |
+
["id"], limit=50000, order="id asc")
|
| 654 |
+
if not recs:
|
| 655 |
+
break
|
| 656 |
+
live_ids.update(r["id"] for r in recs)
|
| 657 |
+
last = recs[-1]["id"]
|
| 658 |
+
store_ids = {r[0] for r in con.execute(f"SELECT id FROM {key}").fetchall()}
|
| 659 |
+
dead = store_ids - live_ids
|
| 660 |
+
if dead:
|
| 661 |
+
con.execute("CREATE TEMP TABLE IF NOT EXISTS _dead(id BIGINT)")
|
| 662 |
+
con.execute("DELETE FROM _dead")
|
| 663 |
+
con.executemany("INSERT INTO _dead VALUES (?)", [[i] for i in dead])
|
| 664 |
+
con.execute(f"DELETE FROM {key} WHERE id IN (SELECT id FROM _dead)")
|
| 665 |
+
con.execute("DELETE FROM _dead")
|
| 666 |
+
n = con.execute(f"SELECT count(*) FROM {key}").fetchone()[0]
|
| 667 |
+
con.execute("UPDATE _sync_state SET rows=? WHERE entity=?", [n, key])
|
| 668 |
+
out[key] = len(dead)
|
| 669 |
+
log(f" {key}: reconciled {len(dead):,} hard-deleted rows")
|
| 670 |
+
return out
|
| 671 |
+
finally:
|
| 672 |
+
con.close()
|
| 673 |
+
|
| 674 |
+
|
| 675 |
+
def status():
|
| 676 |
+
con = connect()
|
| 677 |
+
try:
|
| 678 |
+
return {r[0]: {"phase": r[1], "cursor": r[3], "rows": r[4], "updated": r[5]}
|
| 679 |
+
for r in con.execute("SELECT * FROM _sync_state").fetchall()}
|
| 680 |
+
finally:
|
| 681 |
+
con.close()
|
| 682 |
+
|
| 683 |
+
|
| 684 |
+
# ------------------------------------------------------------------ parity (store vs LIVE Odoo)
|
| 685 |
+
|
| 686 |
+
def _check(name, ours, live, tol=0.01):
|
| 687 |
+
gap = abs((ours or 0) - (live or 0))
|
| 688 |
+
return {"check": name, "ok": gap <= tol, "gap": round(gap, 4), "ours": ours, "live": live}
|
| 689 |
+
|
| 690 |
+
|
| 691 |
+
def validate(pre=None, months=("2026-01-01", "2026-07-01")):
|
| 692 |
+
"""Raw-fidelity parity: the store must mirror live Odoo β row counts per synced entity +
|
| 693 |
+
unscoped monetary sums over a window, to the cent. Only checks entities in LIVE phase."""
|
| 694 |
+
con = connect()
|
| 695 |
+
out = []
|
| 696 |
+
try:
|
| 697 |
+
live_state = status()
|
| 698 |
+
d0, d1 = months
|
| 699 |
+
for key, spec in ENTITIES.items():
|
| 700 |
+
if live_state.get(key, {}).get("phase") != "live":
|
| 701 |
+
continue
|
| 702 |
+
ours = con.execute(f"SELECT count(*) FROM {key}").fetchone()[0]
|
| 703 |
+
live = O.get_odoo().search_count(spec["model"], _base_domain(spec))
|
| 704 |
+
out.append(_check(f"store.{key} row count == live search_count", ours, live, tol=0))
|
| 705 |
+
if live_state.get("sale_order_line", {}).get("phase") == "live":
|
| 706 |
+
ours = con.execute(
|
| 707 |
+
"SELECT coalesce(sum(l.price_subtotal),0) FROM sale_order_line l "
|
| 708 |
+
"JOIN sale_order o ON o.id = l.order_id "
|
| 709 |
+
"WHERE o.date_order >= ? AND o.date_order < ?", [d0, d1]).fetchone()[0]
|
| 710 |
+
live = O.sum_field("sale.order.line",
|
| 711 |
+
[("order_id.date_order", ">=", f"{d0} 00:00:00"),
|
| 712 |
+
("order_id.date_order", "<", f"{d1} 00:00:00")],
|
| 713 |
+
"price_subtotal")
|
| 714 |
+
out.append(_check(f"store Ξ£ line revenue [{d0}..{d1}) == live (unscoped)", ours, live))
|
| 715 |
+
if live_state.get("sale_order", {}).get("phase") == "live":
|
| 716 |
+
ours = con.execute(
|
| 717 |
+
"SELECT coalesce(sum(amount_untaxed),0) FROM sale_order "
|
| 718 |
+
"WHERE date_order >= ? AND date_order < ?", [d0, d1]).fetchone()[0]
|
| 719 |
+
live = O.sum_field("sale.order",
|
| 720 |
+
[("date_order", ">=", f"{d0} 00:00:00"),
|
| 721 |
+
("date_order", "<", f"{d1} 00:00:00")], "amount_untaxed")
|
| 722 |
+
out.append(_check(f"store Ξ£ order amount_untaxed [{d0}..{d1}) == live (unscoped)",
|
| 723 |
+
ours, live))
|
| 724 |
+
finally:
|
| 725 |
+
con.close()
|
| 726 |
+
return out
|
| 727 |
+
|
| 728 |
+
|
| 729 |
+
if __name__ == "__main__":
|
| 730 |
+
args = sys.argv[1:]
|
| 731 |
+
if args and args[0] == "status":
|
| 732 |
+
for k, v in status().items():
|
| 733 |
+
print(f" {k:18s} {v['phase']:9s} rows={v['rows']:>9,} cursor={v['cursor']} ({v['updated']})")
|
| 734 |
+
elif args and args[0] == "reconcile":
|
| 735 |
+
ents = args[1].split(",") if len(args) > 1 else None
|
| 736 |
+
for k, n in reconcile_deletes(ents).items():
|
| 737 |
+
print(f" {k:18s} -{n:,} hard-deleted rows")
|
| 738 |
+
elif args and args[0] == "validate":
|
| 739 |
+
ok = True
|
| 740 |
+
for c in validate():
|
| 741 |
+
mark = "OK " if c["ok"] else "XX "
|
| 742 |
+
ok &= c["ok"]
|
| 743 |
+
print(f" {mark}{c['check'][:80]} gap={c['gap']}")
|
| 744 |
+
print("PARITY:", "GREEN" if ok else "RED")
|
| 745 |
+
sys.exit(0 if ok else 1)
|
| 746 |
+
else:
|
| 747 |
+
ents = args[0].split(",") if args else None
|
| 748 |
+
mb = int(args[1]) if len(args) > 1 else 200
|
| 749 |
+
for k, v in sync_all(ents, max_batches=mb).items():
|
| 750 |
+
print(f" {k:18s} {v['phase']:9s} +{v['pulled']:,} rows in {v['secs']}s")
|
royalimports_os/harness/evals.py
ADDED
|
@@ -0,0 +1,134 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""harness/evals.py β the Analyst eval gate (OM-4, 2026-07-11).
|
| 2 |
+
|
| 3 |
+
Runs the golden set (evals/analyst_golden.yml β truths from the parity-proven semantic layer over
|
| 4 |
+
CLOSED windows) against the Analyst and scores DETERMINISTICALLY: expected number found in the
|
| 5 |
+
answer (tolerance-aware, handles $/commas/% forms), required substrings, required tools in the
|
| 6 |
+
trace, required artifacts. No LLM judge in v1 β the validate() culture applied to AI answers.
|
| 7 |
+
|
| 8 |
+
Gate rule (from the plan): run before any deploy that touches model/ or the Analyst; ship only on
|
| 9 |
+
pass-rate >= the bar (start 80%, tighten as the model/skills improve).
|
| 10 |
+
|
| 11 |
+
Usage:
|
| 12 |
+
python -X utf8 -c "import ssl; ssl._create_default_https_context=ssl._create_unverified_context;
|
| 13 |
+
import harness.evals as E; E.run()" # live (needs OPENROUTER_API_KEY)
|
| 14 |
+
E.run(chat_fn=scripted) # offline harness test
|
| 15 |
+
"""
|
| 16 |
+
import re
|
| 17 |
+
from pathlib import Path
|
| 18 |
+
|
| 19 |
+
import yaml
|
| 20 |
+
|
| 21 |
+
GOLDEN = Path(__file__).resolve().parents[1] / "evals" / "analyst_golden.yml"
|
| 22 |
+
|
| 23 |
+
|
| 24 |
+
def _numbers(text):
|
| 25 |
+
"""Every number in the answer, normalized: $1,234.56 -> 1234.56; '59%'/'59.0 %' -> 0.59 too."""
|
| 26 |
+
out = []
|
| 27 |
+
for m in re.finditer(r"\$?([\d,]+(?:\.\d+)?)\s*(%?)", text):
|
| 28 |
+
try:
|
| 29 |
+
v = float(m.group(1).replace(",", ""))
|
| 30 |
+
except ValueError:
|
| 31 |
+
continue
|
| 32 |
+
out.append(v)
|
| 33 |
+
if m.group(2): # percent form: also offer the fraction
|
| 34 |
+
out.append(v / 100.0)
|
| 35 |
+
return out
|
| 36 |
+
|
| 37 |
+
|
| 38 |
+
def _norm_ws(s):
|
| 39 |
+
"""Collapse ALL unicode whitespace to single spaces. Models emit U+202F (narrow no-break
|
| 40 |
+
space) and friends inside names β 'Poppy\\u202fFlowers' failed a plain substring check on
|
| 41 |
+
two different models (the 'top_customer flake', diagnosed 2026-07-12)."""
|
| 42 |
+
return re.sub(r"\s+", " ", s)
|
| 43 |
+
|
| 44 |
+
|
| 45 |
+
def _check(item, result):
|
| 46 |
+
exp = item.get("expect") or {}
|
| 47 |
+
answer = _norm_ws(result.get("answer") or "")
|
| 48 |
+
reasons = []
|
| 49 |
+
if result.get("exhausted"):
|
| 50 |
+
reasons.append("tool budget exhausted")
|
| 51 |
+
if "value" in exp:
|
| 52 |
+
want = float(exp["value"])
|
| 53 |
+
tol = exp.get("abs", exp.get("rel", 0.001) * abs(want) or 0.01)
|
| 54 |
+
nums = _numbers(answer)
|
| 55 |
+
if exp.get("allow_pct_form"):
|
| 56 |
+
nums += [n for n in list(nums)]
|
| 57 |
+
if not any(abs(n - want) <= tol for n in nums):
|
| 58 |
+
reasons.append(f"expected {want:,.2f} (Β±{tol:,.2f}) not in answer")
|
| 59 |
+
for s in exp.get("contains", []):
|
| 60 |
+
if _norm_ws(s).lower() not in answer.lower():
|
| 61 |
+
reasons.append(f"missing substring {s!r}")
|
| 62 |
+
# WRONG-ANSWER guard (2026-07-28). `contains` can only prove an answer said the right thing;
|
| 63 |
+
# some failures are about saying a WRONG thing that a right answer never says β e.g. naming an
|
| 64 |
+
# internal salesperson as an agent ([[invoice-line-agent-commission]]). Those need a negative
|
| 65 |
+
# assertion or the eval passes on an answer that is confidently incorrect.
|
| 66 |
+
for s in exp.get("not_contains", []):
|
| 67 |
+
if _norm_ws(s).lower() in answer.lower():
|
| 68 |
+
reasons.append(f"forbidden substring {s!r} present")
|
| 69 |
+
used = [t["tool"] for t in result.get("tool_trace", [])]
|
| 70 |
+
for t in exp.get("tools", []):
|
| 71 |
+
if t not in used:
|
| 72 |
+
reasons.append(f"tool {t!r} not used (trace: {used})")
|
| 73 |
+
for t in exp.get("not_tools", []): # over-refusal guard: these must NOT appear
|
| 74 |
+
if t in used:
|
| 75 |
+
reasons.append(f"tool {t!r} used but forbidden here (trace: {used})")
|
| 76 |
+
if exp.get("artifact") == "chart":
|
| 77 |
+
if not any("chart" in a for a in result.get("artifacts", [])):
|
| 78 |
+
reasons.append("no chart artifact produced")
|
| 79 |
+
return (not reasons), reasons
|
| 80 |
+
|
| 81 |
+
|
| 82 |
+
def run(chat_fn=None, model=None, only=None, bar=0.8, verbose=True):
|
| 83 |
+
"""Run the golden set. Returns {passed, total, pass_rate, gate_ok, results}."""
|
| 84 |
+
import harness.analyst as A
|
| 85 |
+
items = yaml.safe_load(GOLDEN.read_text(encoding="utf-8"))["items"]
|
| 86 |
+
if only:
|
| 87 |
+
items = [i for i in items if i["id"] in only]
|
| 88 |
+
results, passed = [], 0
|
| 89 |
+
import os as _os
|
| 90 |
+
import time as _time
|
| 91 |
+
_os.environ["ANALYST_PATIENT"] = "1" # batch mode: wait out rate windows, don't fail fast
|
| 92 |
+
infra_fails = 0
|
| 93 |
+
aborted = False
|
| 94 |
+
for item in items:
|
| 95 |
+
_time.sleep(6) # pace the free-tier per-minute windows (Groq/Cerebras)
|
| 96 |
+
kw = {"chat_fn": chat_fn} if chat_fn else {}
|
| 97 |
+
if model:
|
| 98 |
+
kw["model"] = model
|
| 99 |
+
try:
|
| 100 |
+
res = A.ask(item["ask"], telemetry_kind="analyst_eval", **kw)
|
| 101 |
+
ok, reasons = _check(item, res)
|
| 102 |
+
infra_fails = 0
|
| 103 |
+
except Exception as e:
|
| 104 |
+
res, ok, reasons = {"answer": "", "tool_trace": []}, False, [f"RUN ERROR: {e}"]
|
| 105 |
+
# Provider exhaustion is NOT a model failure: two in a row means the ladder is out
|
| 106 |
+
# of quota β abort and report INCONCLUSIVE instead of burning the remaining items
|
| 107 |
+
# into a misleading FAIL (learned 2026-07-16: 14 wasted items, 4/18 "22%").
|
| 108 |
+
if "all LLM providers failed" in str(e):
|
| 109 |
+
infra_fails += 1
|
| 110 |
+
if infra_fails >= 2:
|
| 111 |
+
aborted = True
|
| 112 |
+
passed += ok
|
| 113 |
+
results.append({"id": item["id"], "ok": ok, "reasons": reasons,
|
| 114 |
+
"usage": res.get("usage"), "iterations": res.get("iterations")})
|
| 115 |
+
if verbose:
|
| 116 |
+
mark = "OK " if ok else "XX "
|
| 117 |
+
print(f" {mark}{item['id']:18s}" + ("" if ok else f" β {'; '.join(reasons)[:90]}"))
|
| 118 |
+
if aborted:
|
| 119 |
+
break
|
| 120 |
+
if aborted:
|
| 121 |
+
ran = [r for r in results if not str(r["reasons"])[:60].startswith("['RUN ERROR")]
|
| 122 |
+
if verbose:
|
| 123 |
+
print(f"\nEVAL ABORTED β LLM providers exhausted after {len(results)} item(s); "
|
| 124 |
+
f"{passed}/{len(ran) or 1} of the items that actually ran passed. "
|
| 125 |
+
"Gate is INCONCLUSIVE (not a model failure) β re-run when quota recovers.")
|
| 126 |
+
return {"passed": passed, "total": len(items), "pass_rate": None, "gate_ok": False,
|
| 127 |
+
"inconclusive": True, "results": results}
|
| 128 |
+
rate = passed / len(items) if items else 0.0
|
| 129 |
+
gate = rate >= bar
|
| 130 |
+
if verbose:
|
| 131 |
+
print(f"\nEVAL: {passed}/{len(items)} = {rate:.0%} GATE({bar:.0%}): "
|
| 132 |
+
f"{'PASS' if gate else 'FAIL'}")
|
| 133 |
+
return {"passed": passed, "total": len(items), "pass_rate": rate, "gate_ok": gate,
|
| 134 |
+
"results": results}
|
royalimports_os/harness/filter_sql.py
ADDED
|
@@ -0,0 +1,654 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""harness/filter_sql.py β the grid's filter/sort engine, compiled to SQL (CG-1).
|
| 2 |
+
|
| 3 |
+
THE PORT. `customer-grid/useVisibleRows.ts` evaluates the Airtable-parity filter TREE and the
|
| 4 |
+
multi-level sort in TypeScript, per row, in the browser. That works at customer grain (1,550 rows,
|
| 5 |
+
576 KB) and does NOT work at line grain, so a line-grain table has to filter server-side. This
|
| 6 |
+
module is the second implementation of that contract, emitting parameterized SQL instead of
|
| 7 |
+
booleans. `aios_grid.clean_filter_tree` (the validator) is the third. All three are held in
|
| 8 |
+
lock-step by `aios-web/verify_filter_engine.py`, which runs the SAME cases through the TS engine,
|
| 9 |
+
this compiler (executed on DuckDB), and the validator, and compares row sets.
|
| 10 |
+
|
| 11 |
+
Design: this file is PURE β no model, no store, no app imports. It takes a filter tree plus a
|
| 12 |
+
`columns` spec and returns SQL + params. `harness/semantic.store_columns` builds the spec from a
|
| 13 |
+
topic; nothing else here knows what a topic is.
|
| 14 |
+
|
| 15 |
+
columns = {colId: {"sql": <SQL expression>, "type": <field type>, "aggregate": bool}}
|
| 16 |
+
|
| 17 |
+
`sql` is a trusted IDENTIFIER/expression from the semantic model (never user input); every VALUE
|
| 18 |
+
is a bound parameter. That split is the whole injection story.
|
| 19 |
+
|
| 20 |
+
--------------------------------------------------------------------------------------------
|
| 21 |
+
TRI-STATE, AND WHY IT BECOMES COMPILE-TIME PRUNING
|
| 22 |
+
--------------------------------------------------------------------------------------------
|
| 23 |
+
`evalNode` in TS returns true | false | null, where null means "this node is INACTIVE and its
|
| 24 |
+
parent must IGNORE it". Get that wrong and a half-typed condition returns true, makes its `or`
|
| 25 |
+
group true, and shows every record.
|
| 26 |
+
|
| 27 |
+
Activeness reads ONLY the rule (op, value, value2, is-the-column-known) β never the row. So in
|
| 28 |
+
SQL it is not a third truth value at all: it is decided while COMPILING. An inactive node emits
|
| 29 |
+
no SQL, a group with no active children emits no SQL, and a root with nothing active emits NO
|
| 30 |
+
PREDICATE AT ALL (`compile_filter_tree` returns None) β which is exactly `matchFilterTree`
|
| 31 |
+
returning true for every row.
|
| 32 |
+
|
| 33 |
+
--------------------------------------------------------------------------------------------
|
| 34 |
+
THE FOUR PLACES A NAIVE PORT SILENTLY DIVERGES
|
| 35 |
+
--------------------------------------------------------------------------------------------
|
| 36 |
+
JavaScript coerces null to 0 / "" before comparing; SQL propagates NULL and drops the row.
|
| 37 |
+
|
| 38 |
+
1. `toNum(null)` is 0, so a NULL revenue PASSES `>= 0`. SQL: NULL >= 0 -> NULL -> dropped.
|
| 39 |
+
2. `String(null)` is "", so a NULL state PASSES `is not FL`. SQL: NULL <> 'fl' -> dropped.
|
| 40 |
+
3. `"" <= "2026-01-01"` is true, so a NULL date PASSES `is before`. SQL: dropped.
|
| 41 |
+
4. `contains` is a LITERAL substring test. `LIKE '%v%'` treats % and _ as wildcards, so a
|
| 42 |
+
value containing either matches the wrong rows. Hence `strpos`, not LIKE.
|
| 43 |
+
|
| 44 |
+
Every column is therefore read through `_value_sql`, which COALESCEs to the same zero-value
|
| 45 |
+
JavaScript would have coerced to. The one thing that must NOT be coalesced away is the
|
| 46 |
+
blank test itself β `isBlank` looks at the RAW value, and numeric 0 is NOT blank.
|
| 47 |
+
|
| 48 |
+
--------------------------------------------------------------------------------------------
|
| 49 |
+
AND THE FIFTH: `default: return true` IS ACTIVE, NOT INACTIVE
|
| 50 |
+
--------------------------------------------------------------------------------------------
|
| 51 |
+
`matchFilter`'s per-type switch ends in `default: return true`, reachable whenever an operator
|
| 52 |
+
does not apply to the field type β `contains` on a currency/date field, or `gt`/`between` on a
|
| 53 |
+
text field. A persisted tree can hold such a pair. That rule is ACTIVE and evaluates TRUE for
|
| 54 |
+
every row, which is NOT the same as inactive: under `or` it makes the whole group true and shows
|
| 55 |
+
everything. We emit `1=1` for those, and the gate asserts the difference is observable.
|
| 56 |
+
|
| 57 |
+
--------------------------------------------------------------------------------------------
|
| 58 |
+
INPUT CONTRACT
|
| 59 |
+
--------------------------------------------------------------------------------------------
|
| 60 |
+
Feed this CLEANED trees (`aios_grid.clean_filter_tree`). It is deliberately faithful to the TS
|
| 61 |
+
engine rather than fail-closed on garbage β an unknown OP compiles to `1=1` exactly as TS's
|
| 62 |
+
`default:` branch returns true β because the validator's job is to reject garbage and this
|
| 63 |
+
module's job is to mean the same thing as the client. The gate proves the composition.
|
| 64 |
+
|
| 65 |
+
Ported ahead of their consumer (CG-2) and gated at the same fidelity as the tree:
|
| 66 |
+
`compile_search` (`matchSearch`) and `member_ids` (the hand-picked-members union in
|
| 67 |
+
`useVisibleRows`).
|
| 68 |
+
"""
|
| 69 |
+
import math
|
| 70 |
+
import re
|
| 71 |
+
from collections import namedtuple
|
| 72 |
+
|
| 73 |
+
# Mirrors customer-grid/types.ts FilterOp. verify_filter_engine.py asserts this set is
|
| 74 |
+
# byte-identical to aios_grid.FILTER_OPS β three copies of a vocabulary need a drift gate.
|
| 75 |
+
FILTER_OPS = frozenset({
|
| 76 |
+
'contains', 'doesNotContain', 'eq', 'neq', 'isEmpty', 'isNotEmpty',
|
| 77 |
+
'gt', 'gte', 'lt', 'lte', 'between', 'within',
|
| 78 |
+
})
|
| 79 |
+
#: The RESERVED pseudo-column of a cohort-membership leaf. Mirrors types.ts COHORT_FIELD.
|
| 80 |
+
COHORT_FIELD = '__cohort__'
|
| 81 |
+
#: Set operators over a SET of cohorts. Mirrors types.ts COHORT_OPS; DISJOINT from FILTER_OPS.
|
| 82 |
+
COHORT_OPS = frozenset({'anyOf', 'allOf', 'noneOf'})
|
| 83 |
+
#: The single-cohort ops the leaf shipped with, kept as permanent aliases. types.ts mirrors this.
|
| 84 |
+
COHORT_OP_ALIASES = {'eq': 'anyOf', 'neq': 'noneOf'}
|
| 85 |
+
#: Mirrors types.ts MAX_COHORT_IDS.
|
| 86 |
+
MAX_COHORT_IDS = 20
|
| 87 |
+
|
| 88 |
+
|
| 89 |
+
def parse_cohort_ids(value):
|
| 90 |
+
"""The cohorts a leaf names. Mirrors types.ts `cohortIds()` and aios_grid.parse_cohort_ids."""
|
| 91 |
+
out = []
|
| 92 |
+
for raw in ('' if value is None else str(value)).split(','):
|
| 93 |
+
cid = raw.strip()[:120]
|
| 94 |
+
if not cid or cid in out:
|
| 95 |
+
continue
|
| 96 |
+
out.append(cid)
|
| 97 |
+
if len(out) >= MAX_COHORT_IDS:
|
| 98 |
+
break
|
| 99 |
+
return out
|
| 100 |
+
# types.ts VALUE_FREE_OPS: no value control is rendered, and they must never be judged
|
| 101 |
+
# "inactive because the value is blank" β that would turn them into no-ops.
|
| 102 |
+
VALUE_FREE_OPS = frozenset({'isEmpty', 'isNotEmpty'})
|
| 103 |
+
# types.ts isNumericType()
|
| 104 |
+
NUMERIC_TYPES = frozenset({'currency', 'int', 'pct'})
|
| 105 |
+
TEXTUAL_TYPES = frozenset({'text', 'status'})
|
| 106 |
+
|
| 107 |
+
SQL_TRUE = '1=1'
|
| 108 |
+
|
| 109 |
+
_NUM_CMP = {'gt': '>', 'gte': '>=', 'lt': '<', 'lte': '<=', 'eq': '=', 'neq': '<>'}
|
| 110 |
+
_DATE_CMP = dict(_NUM_CMP)
|
| 111 |
+
|
| 112 |
+
#: sql = the predicate (already parenthesised); params = bound values, positional;
|
| 113 |
+
#: uses_aggregate = the predicate references an aggregate column, so it belongs in HAVING
|
| 114 |
+
#: rather than WHERE; columns_used = colIds actually referenced in the emitted SQL (an
|
| 115 |
+
#: always-true `1=1` branch references nothing, and must not force HAVING routing).
|
| 116 |
+
Compiled = namedtuple('Compiled', 'sql params uses_aggregate columns_used')
|
| 117 |
+
|
| 118 |
+
|
| 119 |
+
# --------------------------------------------------------------------------- value coercion
|
| 120 |
+
|
| 121 |
+
# JS `Number(string)` trims exactly this set (WhiteSpace + LineTerminator). Python's float() and
|
| 122 |
+
# str.strip() use a DIFFERENT set -- notably U+0085 NEL, which Python treats as whitespace and JS
|
| 123 |
+
# does not -- so the set is spelled out rather than left to strip().
|
| 124 |
+
_JS_WS = "".join(chr(c) for c in (9, 10, 11, 12, 13, 32, 160, 5760, 8232, 8233,
|
| 125 |
+
8239, 8287, 12288, 65279)) + \
|
| 126 |
+
"".join(chr(c) for c in range(8192, 8203))
|
| 127 |
+
# ASCII-ONLY on purpose: `\d` in a Python str pattern matches every Unicode decimal digit, so
|
| 128 |
+
# `\d` here would parse the fullwidth "οΌοΌοΌ" that JS rejects -- reintroducing the exact bug
|
| 129 |
+
# this function exists to avoid.
|
| 130 |
+
_JS_DEC = re.compile(r"^[+-]?(?:[0-9]+\.?[0-9]*|\.[0-9]+)(?:[eE][+-]?[0-9]+)?$")
|
| 131 |
+
_JS_INF = re.compile(r"^[+-]?Infinity$")
|
| 132 |
+
_JS_RADIX = ((re.compile(r"^0[xX][0-9a-fA-F]+$"), 16),
|
| 133 |
+
(re.compile(r"^0[oO][0-7]+$"), 8),
|
| 134 |
+
(re.compile(r"^0[bB][01]+$"), 2))
|
| 135 |
+
|
| 136 |
+
|
| 137 |
+
def to_num(v):
|
| 138 |
+
"""Port of `toNum`: JS `Number(v)`, then `Number.isFinite` or 0.
|
| 139 |
+
|
| 140 |
+
NOT `float(v)`. Python's float() is MORE permissive than JS's Number() in ways that make SQL
|
| 141 |
+
narrow where the client shows everything -- the worse direction, because at line grain that
|
| 142 |
+
is a silent under-fetch sitting beside an honest-looking row_count:
|
| 143 |
+
|
| 144 |
+
"1_000" float() -> 1000.0 (PEP-515 underscores) Number() -> NaN -> 0
|
| 145 |
+
"οΌοΌοΌ" float() -> 250.0 (any Unicode Nd) Number() -> NaN -> 0
|
| 146 |
+
"\\u0085" + "12" float() -> 12.0 (NEL is Python whitespace) Number() -> NaN -> 0
|
| 147 |
+
|
| 148 |
+
and LESS permissive in one family, which errs the other way:
|
| 149 |
+
|
| 150 |
+
"0x10" float() -> raises -> 0 Number() -> 16 (also 0b, 0o)
|
| 151 |
+
|
| 152 |
+
A tree carrying any of these is reachable: the React builder renders <input type="number">,
|
| 153 |
+
but persisted / seeded / imported view JSON goes through `clean_filter_tree`, which passes
|
| 154 |
+
the value string through untouched by design (it validates structure, not numeric syntax).
|
| 155 |
+
|
| 156 |
+
Agreed by construction on the rest: ""/None -> 0, "abc"/"12,000" -> 0, " 250 " -> 250,
|
| 157 |
+
".5"/"5."/"+5", "1e400" -> Infinity -> 0, "-0" -> 0, and float64 truncation of
|
| 158 |
+
"9007199254740993" -> 9007199254740992.
|
| 159 |
+
"""
|
| 160 |
+
if isinstance(v, bool):
|
| 161 |
+
return 1.0 if v else 0.0 # JS Number(true) === 1
|
| 162 |
+
if isinstance(v, (int, float)):
|
| 163 |
+
n = float(v)
|
| 164 |
+
return n if math.isfinite(n) else 0.0
|
| 165 |
+
if v is None:
|
| 166 |
+
return 0.0 # JS Number(null) === 0
|
| 167 |
+
s = str(v).strip(_JS_WS)
|
| 168 |
+
if s == "":
|
| 169 |
+
return 0.0 # JS Number("") === 0
|
| 170 |
+
if _JS_INF.match(s):
|
| 171 |
+
return 0.0 # Infinity is not finite -> 0
|
| 172 |
+
for pattern, base in _JS_RADIX:
|
| 173 |
+
if pattern.match(s):
|
| 174 |
+
return float(int(s[2:], base))
|
| 175 |
+
if not _JS_DEC.match(s):
|
| 176 |
+
return 0.0 # JS NaN -> 0
|
| 177 |
+
n = float(s)
|
| 178 |
+
return n if math.isfinite(n) else 0.0
|
| 179 |
+
|
| 180 |
+
|
| 181 |
+
def _value_sql(spec):
|
| 182 |
+
"""The ONE expression for a column, used by BOTH the filter and the sort.
|
| 183 |
+
|
| 184 |
+
Two expressions would let two rows tie in the grid and not tie in SQL (dates truncated for
|
| 185 |
+
comparison but ordered by the raw timestamp is the obvious way to get that wrong), so the
|
| 186 |
+
next sort key would break the tie in one engine and not the other.
|
| 187 |
+
|
| 188 |
+
numeric: `round_even(x, 0)` reproduces `aios_grid._round`, which is Python's round() =
|
| 189 |
+
BANKER'S rounding. DuckDB's plain ROUND() rounds half AWAY from zero, so ROUND(1234.5)=1235
|
| 190 |
+
while the grid shows 1234 β `eq 1234` would then match on screen and miss in SQL. The grid
|
| 191 |
+
displays rounded values; filters must agree with what is on screen.
|
| 192 |
+
date: truncate to the 10-char ISO shape the payload carries, so lexical compare is
|
| 193 |
+
chronological and a timestamp column cannot smuggle ' 00:00:00' into the comparison.
|
| 194 |
+
text: lowercased, matching `String(raw ?? "").toLowerCase()`.
|
| 195 |
+
"""
|
| 196 |
+
e = spec['sql']
|
| 197 |
+
t = spec['type']
|
| 198 |
+
if t in NUMERIC_TYPES:
|
| 199 |
+
return f"COALESCE(round_even(TRY_CAST({e} AS DOUBLE), 0), 0)"
|
| 200 |
+
if t == 'date':
|
| 201 |
+
return f"COALESCE(SUBSTR(CAST({e} AS VARCHAR), 1, 10), '')"
|
| 202 |
+
return f"LOWER(COALESCE(CAST({e} AS VARCHAR), ''))"
|
| 203 |
+
|
| 204 |
+
|
| 205 |
+
def _blank_sql(spec):
|
| 206 |
+
"""Port of `isBlank`: null/undefined/"" are blank β numeric 0 is NOT.
|
| 207 |
+
|
| 208 |
+
Reads the RAW column, never `_value_sql` (which has already coalesced blanks into 0/"",
|
| 209 |
+
the very distinction this test exists to make).
|
| 210 |
+
"""
|
| 211 |
+
e = spec['sql']
|
| 212 |
+
if spec['type'] in NUMERIC_TYPES:
|
| 213 |
+
# 0 stays non-blank: CAST(0 AS VARCHAR) is '0'. The '' arm covers a text-typed column
|
| 214 |
+
# holding "", which `_round` passes through unchanged into the payload.
|
| 215 |
+
return f"(({e}) IS NULL OR CAST({e} AS VARCHAR) = '')"
|
| 216 |
+
if spec['type'] == 'date':
|
| 217 |
+
return f"(COALESCE(SUBSTR(CAST({e} AS VARCHAR), 1, 10), '') = '')"
|
| 218 |
+
return f"(COALESCE(CAST({e} AS VARCHAR), '') = '')"
|
| 219 |
+
|
| 220 |
+
|
| 221 |
+
# --------------------------------------------------------------------------- activeness
|
| 222 |
+
|
| 223 |
+
def is_rule_active(rule, columns):
|
| 224 |
+
"""Port of `isRuleActive` + `evalNode`'s two leaf guards. Row-independent by construction.
|
| 225 |
+
|
| 226 |
+
Unknown column -> inactive (evalNode returns null before matchFilter is ever called).
|
| 227 |
+
Value-free ops are always active. `between` needs BOTH bounds.
|
| 228 |
+
|
| 229 |
+
β The four VALUE-FREE ANCHOR MODES (`is before [today]`) carry no value, and the historical
|
| 230 |
+
"blank value = inactive" rule would IGNORE them β widening the result under a count nobody
|
| 231 |
+
would doubt. Same shape as the isEmpty/isNotEmpty no-op bug, arriving by a different door.
|
| 232 |
+
"""
|
| 233 |
+
if not isinstance(rule, dict):
|
| 234 |
+
return False
|
| 235 |
+
col = rule.get('colId')
|
| 236 |
+
if col == COHORT_FIELD: # a cohort leaf has no column to look up
|
| 237 |
+
# Driven by the SET it names, so a value of "," is INACTIVE rather than active-and-
|
| 238 |
+
# unanswerable. Mirrors the cohort branch at the top of types.ts isRuleActive.
|
| 239 |
+
return bool(parse_cohort_ids(rule.get('value')))
|
| 240 |
+
if col not in columns:
|
| 241 |
+
return False
|
| 242 |
+
op = rule.get('op')
|
| 243 |
+
if op in VALUE_FREE_OPS:
|
| 244 |
+
return True
|
| 245 |
+
if op == 'within':
|
| 246 |
+
return _window_bounds(rule.get('dateWindow'), None, probe=True)
|
| 247 |
+
if rule.get('rhs') is not None:
|
| 248 |
+
rhs = rule['rhs']
|
| 249 |
+
# An rhs naming a column this table does not have is IGNORED, by the same rule as an
|
| 250 |
+
# unknown left column two lines above β for EITHER kind. Ignoring one side and hiding
|
| 251 |
+
# every row for the other would make the comparison asymmetric in a way nobody could
|
| 252 |
+
# predict. Doing it here rather than in _compile_leaf keeps activeness the single place
|
| 253 |
+
# that decides, which is what makes the compiled branch below provably unreachable.
|
| 254 |
+
if not isinstance(rhs, dict) or rhs.get('colId') not in columns:
|
| 255 |
+
return False
|
| 256 |
+
return _rhs_complete(rhs)
|
| 257 |
+
if rule.get('dateMode') in _ANCHOR_VALUE_FREE:
|
| 258 |
+
return True
|
| 259 |
+
value = _as_str(rule.get('value'))
|
| 260 |
+
if op == 'between':
|
| 261 |
+
v2 = rule.get('value2')
|
| 262 |
+
return value != '' and v2 is not None and _as_str(v2) != ''
|
| 263 |
+
return value != ''
|
| 264 |
+
|
| 265 |
+
|
| 266 |
+
def _rhs_complete(rhs):
|
| 267 |
+
"""Port of types.ts isRhsComplete. An incomplete rhs makes the rule INACTIVE, exactly as a
|
| 268 |
+
half-typed value does."""
|
| 269 |
+
if not isinstance(rhs, dict) or not rhs.get('colId'):
|
| 270 |
+
return False
|
| 271 |
+
if rhs.get('kind') == 'measure':
|
| 272 |
+
return _windows().normalize(rhs.get('window')) is not None
|
| 273 |
+
return rhs.get('kind') == 'field'
|
| 274 |
+
|
| 275 |
+
|
| 276 |
+
def _windows():
|
| 277 |
+
"""`harness.windows` imported lazily, so this module stays importable on its own.
|
| 278 |
+
|
| 279 |
+
filter_sql is otherwise PURE (no model/store/app imports) and that property is what lets
|
| 280 |
+
aios_grid embed the grid anywhere. windows.py is the one exception and it is a deliberate
|
| 281 |
+
one: the date vocabulary is a two-ENGINE contract, and a third copy of "what the past month
|
| 282 |
+
means" is precisely the drift verify_windows.py exists to prevent. It imports nothing but
|
| 283 |
+
datetime, so purity is preserved in substance.
|
| 284 |
+
"""
|
| 285 |
+
from harness import windows as _w
|
| 286 |
+
return _w
|
| 287 |
+
|
| 288 |
+
|
| 289 |
+
#: Mirrors windows.ANCHOR_VALUE_FREE, resolved lazily on first use (see _windows).
|
| 290 |
+
class _AnchorValueFree:
|
| 291 |
+
def __contains__(self, mode):
|
| 292 |
+
return mode in _windows().ANCHOR_VALUE_FREE
|
| 293 |
+
|
| 294 |
+
|
| 295 |
+
_ANCHOR_VALUE_FREE = _AnchorValueFree()
|
| 296 |
+
|
| 297 |
+
|
| 298 |
+
def _window_bounds(spec, today, probe=False):
|
| 299 |
+
"""Resolve a date-condition RANGE, or refuse.
|
| 300 |
+
|
| 301 |
+
`probe=True` answers only "is this a resolvable window?" without needing `today` β which is
|
| 302 |
+
what activeness has to know while compiling, before any date is in hand.
|
| 303 |
+
|
| 304 |
+
β A missing `today` on the real path RAISES rather than widening. The client's answer to the
|
| 305 |
+
same gap is to match nothing; the server's must be to refuse, because a server that quietly
|
| 306 |
+
drops a condition reports an authoritative row_count for a query nobody asked for. Same rule
|
| 307 |
+
as the sibling/depth caps: truncate on the client, refuse on the server.
|
| 308 |
+
"""
|
| 309 |
+
w = _windows()
|
| 310 |
+
if probe:
|
| 311 |
+
return w.normalize(spec) is not None
|
| 312 |
+
if today is None:
|
| 313 |
+
raise ValueError(
|
| 314 |
+
'a relative-date condition needs `today` β compile_filter_tree(today=...). '
|
| 315 |
+
'Refusing rather than dropping the condition, which would widen the result while '
|
| 316 |
+
'the count beside it still looked authoritative.')
|
| 317 |
+
return w.resolve(spec, today)
|
| 318 |
+
|
| 319 |
+
|
| 320 |
+
def _as_str(v):
|
| 321 |
+
"""The builder stores values as raw input strings; be tolerant of a number that survived
|
| 322 |
+
an older payload. `str(500) != ''` is active and `to_num` reads it back, matching JS where
|
| 323 |
+
`500 === ""` is false."""
|
| 324 |
+
return '' if v is None else str(v)
|
| 325 |
+
|
| 326 |
+
|
| 327 |
+
# --------------------------------------------------------------------------- leaf
|
| 328 |
+
|
| 329 |
+
def _compile_leaf(rule, columns, state):
|
| 330 |
+
"""Port of `matchFilter`. Returns (sql, params) or None when the rule is INACTIVE."""
|
| 331 |
+
if not is_rule_active(rule, columns):
|
| 332 |
+
return None
|
| 333 |
+
col_id = rule['colId']
|
| 334 |
+
op = rule.get('op')
|
| 335 |
+
value = _as_str(rule.get('value'))
|
| 336 |
+
|
| 337 |
+
# --- owner item 5: a COHORT leaf, before any column lookup -------------------------------
|
| 338 |
+
# `__cohort__` is not a column. Handled first for the same reason evalNode handles it first:
|
| 339 |
+
# falling through to the column path would make it INACTIVE, and inactive means every row.
|
| 340 |
+
if col_id == COHORT_FIELD:
|
| 341 |
+
return _compile_cohort(rule, state)
|
| 342 |
+
|
| 343 |
+
spec = columns[col_id]
|
| 344 |
+
|
| 345 |
+
def used(sql, params=()):
|
| 346 |
+
state['used'].add(col_id)
|
| 347 |
+
if spec.get('aggregate'):
|
| 348 |
+
state['aggregate'] = True
|
| 349 |
+
return (sql, list(params))
|
| 350 |
+
|
| 351 |
+
# --- value-free ops, BEFORE any value-based short-circuit (the no-op bug) ---------------
|
| 352 |
+
if op == 'isEmpty':
|
| 353 |
+
return used(_blank_sql(spec))
|
| 354 |
+
if op == 'isNotEmpty':
|
| 355 |
+
return used(f"NOT {_blank_sql(spec)}")
|
| 356 |
+
|
| 357 |
+
v = _value_sql(spec)
|
| 358 |
+
t = spec['type']
|
| 359 |
+
|
| 360 |
+
# --- CG-9: the comparand is another COLUMN, coerced through the LEFT field's type --------
|
| 361 |
+
# `_value_sql` on a spec whose `sql` is the RIGHT column and whose `type` is the LEFT one β
|
| 362 |
+
# exactly what TS does (`toNum(other)` for a numeric LHS, `String(other ?? "")` for a date,
|
| 363 |
+
# `.toLowerCase()` for text). Reading the right column through its OWN type would compare a
|
| 364 |
+
# rounded number against an unrounded one and disagree on the ties.
|
| 365 |
+
rhs_v = None
|
| 366 |
+
if rule.get('rhs') is not None:
|
| 367 |
+
# No `rspec is None` branch: `is_rule_active` already rejected an rhs naming a column
|
| 368 |
+
# this table does not have, so it cannot be missing here. A defensive fallback would be
|
| 369 |
+
# unreachable, and the negative control showed exactly that β deleting it changed
|
| 370 |
+
# nothing, which is how dead code hides behind a careful-looking line.
|
| 371 |
+
rspec = columns[rule['rhs']['colId']]
|
| 372 |
+
rhs_v = _value_sql({'sql': rspec['sql'], 'type': t})
|
| 373 |
+
state['used'].add(rule['rhs']['colId'])
|
| 374 |
+
if rspec.get('aggregate'):
|
| 375 |
+
state['aggregate'] = True
|
| 376 |
+
|
| 377 |
+
if t in NUMERIC_TYPES:
|
| 378 |
+
if rhs_v is not None:
|
| 379 |
+
return used(f"({v} {_NUM_CMP[op]} {rhs_v})") if op in _NUM_CMP else (SQL_TRUE, [])
|
| 380 |
+
if op == 'between':
|
| 381 |
+
a, b = to_num(value), to_num(rule.get('value2'))
|
| 382 |
+
return used(f"({v} >= ? AND {v} <= ?)", [min(a, b), max(a, b)])
|
| 383 |
+
if op in _NUM_CMP:
|
| 384 |
+
return used(f"({v} {_NUM_CMP[op]} ?)", [to_num(value)])
|
| 385 |
+
return (SQL_TRUE, []) # contains/doesNotContain on a number -> default: true
|
| 386 |
+
|
| 387 |
+
if t == 'date':
|
| 388 |
+
if op == 'within':
|
| 389 |
+
bounds = _window_bounds(rule.get('dateWindow'), state.get('today'))
|
| 390 |
+
if bounds is None:
|
| 391 |
+
return ('1=0', []) # unreachable: activeness probed it. Never widen.
|
| 392 |
+
lo, hi = bounds
|
| 393 |
+
# `{v} <> ''` is NOT redundant. `_value_sql` COALESCEs a NULL date to '', and
|
| 394 |
+
# '' <= any upper bound is true β so an open-ended window would sweep in every
|
| 395 |
+
# never-ordered customer. TS guards it with `if (a === "") return false`.
|
| 396 |
+
parts, params = [f"{v} <> ''"], []
|
| 397 |
+
if lo is not None:
|
| 398 |
+
parts.append(f"{v} >= ?")
|
| 399 |
+
params.append(lo)
|
| 400 |
+
if hi is not None:
|
| 401 |
+
parts.append(f"{v} <= ?")
|
| 402 |
+
params.append(hi)
|
| 403 |
+
return used('(' + ' AND '.join(parts) + ')', params)
|
| 404 |
+
if rhs_v is not None:
|
| 405 |
+
return used(f"({v} {_DATE_CMP[op]} {rhs_v})") if op in _DATE_CMP else (SQL_TRUE, [])
|
| 406 |
+
if op == 'between':
|
| 407 |
+
v2 = _as_str(rule.get('value2'))
|
| 408 |
+
lo, hi = (value, v2) if value < v2 else (v2, value)
|
| 409 |
+
return used(f"({v} >= ? AND {v} <= ?)", [lo, hi])
|
| 410 |
+
if op in _DATE_CMP:
|
| 411 |
+
anchor = _resolve_anchor(rule.get('dateMode'), value, state.get('today'))
|
| 412 |
+
if anchor is None:
|
| 413 |
+
return ('1=0', []) # unresolvable anchor -> matches nothing (TS: false)
|
| 414 |
+
return used(f"({v} {_DATE_CMP[op]} ?)", [anchor])
|
| 415 |
+
return (SQL_TRUE, []) # contains/doesNotContain on a date -> default: true
|
| 416 |
+
|
| 417 |
+
# text / status β case-insensitive. strpos, not LIKE: `includes` is a LITERAL substring
|
| 418 |
+
# test, so a value holding % or _ must not act as a wildcard.
|
| 419 |
+
#
|
| 420 |
+
# BOTH SIDES fold in SQL (`LOWER(?)`, never Python's str.lower()). Python and JS agree on
|
| 421 |
+
# case folding; DuckDB does NOT β LOWER('ISTANBUL' with a dotted capital I) drops the dot
|
| 422 |
+
# while both languages keep it as i + combining dot. Folding the needle in Python and the
|
| 423 |
+
# haystack in DuckDB therefore produced two different strings, and a value could fail to
|
| 424 |
+
# match ITSELF. One folder for both operands is the only version that cannot do that.
|
| 425 |
+
if rhs_v is not None:
|
| 426 |
+
# Both operands are already folded by `_value_sql` (text branch), so no LOWER(?) here.
|
| 427 |
+
if op == 'contains':
|
| 428 |
+
return used(f"(strpos({v}, {rhs_v}) > 0)")
|
| 429 |
+
if op == 'doesNotContain':
|
| 430 |
+
return used(f"(strpos({v}, {rhs_v}) = 0)")
|
| 431 |
+
if op == 'eq':
|
| 432 |
+
return used(f"({v} = {rhs_v})")
|
| 433 |
+
if op == 'neq':
|
| 434 |
+
return used(f"({v} <> {rhs_v})")
|
| 435 |
+
return (SQL_TRUE, [])
|
| 436 |
+
if op == 'contains':
|
| 437 |
+
return used(f"(strpos({v}, LOWER(?)) > 0)", [value])
|
| 438 |
+
if op == 'doesNotContain':
|
| 439 |
+
return used(f"(strpos({v}, LOWER(?)) = 0)", [value])
|
| 440 |
+
if op == 'eq':
|
| 441 |
+
return used(f"({v} = LOWER(?))", [value])
|
| 442 |
+
if op == 'neq':
|
| 443 |
+
return used(f"({v} <> LOWER(?))", [value])
|
| 444 |
+
return (SQL_TRUE, []) # gt/gte/lt/lte/between/within on text -> default: true
|
| 445 |
+
|
| 446 |
+
|
| 447 |
+
def _resolve_anchor(mode, value, today):
|
| 448 |
+
"""A date condition's anchor -> one ISO date, or None. `today` may be absent ONLY for the
|
| 449 |
+
`exact` mode, which does not use it.
|
| 450 |
+
|
| 451 |
+
The sentinel is deliberate and load-bearing for BACK-COMPAT: every date filter that existed
|
| 452 |
+
before anchors is `exact`, and every caller that compiles one (store_query, store_rows)
|
| 453 |
+
passes no `today`. Requiring it unconditionally would turn a working filter into an
|
| 454 |
+
exception. A RELATIVE anchor still refuses β the server never widens.
|
| 455 |
+
"""
|
| 456 |
+
mode = mode or 'exact'
|
| 457 |
+
if today is None:
|
| 458 |
+
if mode != 'exact':
|
| 459 |
+
raise ValueError(
|
| 460 |
+
f"the date anchor {mode!r} is relative and needs `today` β "
|
| 461 |
+
f"compile_filter_tree(today=...). Refusing rather than dropping the condition, "
|
| 462 |
+
f"which would widen the result under an authoritative-looking count.")
|
| 463 |
+
# `exact` ignores `today` entirely (verify_windows resolves it across 11 probe dates and
|
| 464 |
+
# gets the same answer every time), so any valid date serves as the unused argument.
|
| 465 |
+
today = '1970-01-01'
|
| 466 |
+
return _windows().resolve_anchor(mode, value, today)
|
| 467 |
+
|
| 468 |
+
|
| 469 |
+
def _compile_cohort(rule, state):
|
| 470 |
+
"""Port of evalNode's cohort branch: membership in a SET of hand-curated sets of row ids.
|
| 471 |
+
|
| 472 |
+
Owner 2026-07-27: the leaf names one OR MORE cohorts and asks `anyOf` / `allOf` / `noneOf`.
|
| 473 |
+
A single id is a one-element set, so the shipped `is part of` compiles down this same path.
|
| 474 |
+
|
| 475 |
+
States, mirroring TS exactly:
|
| 476 |
+
every set present -> the union (anyOf / noneOf) or the intersection (allOf)
|
| 477 |
+
a set present but EMPTY -> that member contributes 1=0, because nobody is in an empty
|
| 478 |
+
cohort. So `allOf` collapses to 1=0 and `noneOf` to 1=1 β
|
| 479 |
+
which is what `set.has(pid)` on an empty Set does in TS.
|
| 480 |
+
ANY set ABSENT -> 1=0 for ALL THREE ops. The condition is unanswerable, and
|
| 481 |
+
unanswerable matches nothing: answering "everything" to
|
| 482 |
+
`is none of [a cohort we cannot see]` is the widening sin
|
| 483 |
+
wearing a plausible face, and answering it for the members we
|
| 484 |
+
DO have would silently ask a different question.
|
| 485 |
+
|
| 486 |
+
COALESCE(... , FALSE) because a NULL row id makes `id IN (...)` NULL, and `NOT NULL` is NULL
|
| 487 |
+
β SQL would drop the row where TS keeps it.
|
| 488 |
+
"""
|
| 489 |
+
sets = state.get('cohort_sets')
|
| 490 |
+
if sets is None:
|
| 491 |
+
raise ValueError(
|
| 492 |
+
'a cohort condition needs cohort_sets β compile_filter_tree(cohort_sets=...). '
|
| 493 |
+
'Refusing rather than dropping it: a dropped condition widens the result while the '
|
| 494 |
+
'count beside it still looks authoritative.')
|
| 495 |
+
id_sql = state.get('id_sql')
|
| 496 |
+
if not id_sql:
|
| 497 |
+
raise ValueError('a cohort condition needs id_sql (the row-identity column)')
|
| 498 |
+
op = COHORT_OP_ALIASES.get(rule.get('op'), rule.get('op'))
|
| 499 |
+
named = parse_cohort_ids(rule.get('value'))
|
| 500 |
+
members = [sets.get(cid) for cid in named]
|
| 501 |
+
if not named or any(m is None for m in members):
|
| 502 |
+
return ('1=0', [])
|
| 503 |
+
|
| 504 |
+
parts, params = [], []
|
| 505 |
+
for ids in members:
|
| 506 |
+
ids = [int(x) for x in ids]
|
| 507 |
+
if not ids:
|
| 508 |
+
parts.append('1=0')
|
| 509 |
+
continue
|
| 510 |
+
parts.append(f"COALESCE({id_sql} IN ({','.join('?' for _ in ids)}), FALSE)")
|
| 511 |
+
params.extend(ids)
|
| 512 |
+
if op == 'allOf':
|
| 513 |
+
return ('(' + ' AND '.join(parts) + ')', params)
|
| 514 |
+
union = '(' + ' OR '.join(parts) + ')'
|
| 515 |
+
return ((f"(NOT {union})" if op == 'noneOf' else union), params)
|
| 516 |
+
|
| 517 |
+
|
| 518 |
+
# --------------------------------------------------------------------------- tree
|
| 519 |
+
|
| 520 |
+
def _compile_node(node, columns, state):
|
| 521 |
+
"""Port of `evalNode`. Returns (sql, params), or None when the node is INACTIVE.
|
| 522 |
+
|
| 523 |
+
A group combines only its ACTIVE children with its OWN conjunction; a group with no active
|
| 524 |
+
children is itself inactive and disappears. That is what stops a half-built group from
|
| 525 |
+
either zeroing the grid (`and`) or disabling it (`or`).
|
| 526 |
+
"""
|
| 527 |
+
if not isinstance(node, dict):
|
| 528 |
+
return None
|
| 529 |
+
if isinstance(node.get('children'), list):
|
| 530 |
+
parts, params = [], []
|
| 531 |
+
for child in node['children']:
|
| 532 |
+
got = _compile_node(child, columns, state)
|
| 533 |
+
if got is None:
|
| 534 |
+
continue # inactive -> ignored entirely
|
| 535 |
+
parts.append(got[0])
|
| 536 |
+
params.extend(got[1])
|
| 537 |
+
if not parts:
|
| 538 |
+
return None
|
| 539 |
+
joiner = ' OR ' if node.get('conj') == 'or' else ' AND '
|
| 540 |
+
return ('(' + joiner.join(parts) + ')', params)
|
| 541 |
+
return _compile_leaf(node, columns, state)
|
| 542 |
+
|
| 543 |
+
|
| 544 |
+
def compile_filter_tree(nodes, conj='and', columns=None, member_ids=None, id_sql=None,
|
| 545 |
+
today=None, cohort_sets=None):
|
| 546 |
+
"""Port of `matchFilterTree` (+ the `memberPids` union in `useVisibleRows`).
|
| 547 |
+
|
| 548 |
+
Returns a `Compiled`, or **None when nothing is active** β meaning no predicate at all, not
|
| 549 |
+
a false one. `matchFilterTree` returns true when the tree is inactive, so emitting anything
|
| 550 |
+
here would narrow rows the client would have shown.
|
| 551 |
+
|
| 552 |
+
`member_ids` reproduces `members.has(r.pid) || matchFilterTree(...)`. Note the union lives
|
| 553 |
+
INSIDE `if (filters.length)` and is OR'd with a tree that is true for everything when
|
| 554 |
+
inactive β so an inactive tree must emit NOTHING, never `pid IN (...)`, which would narrow
|
| 555 |
+
to just the hand-picked members. Same family of bug as the tri-state one, inverted.
|
| 556 |
+
|
| 557 |
+
`today` is the TENANT'S today, required by any relative date condition (`is within β¦`,
|
| 558 |
+
`is before [one month ago]`) and by nothing else β an `exact` date needs none, which is what
|
| 559 |
+
keeps every pre-anchor caller working unchanged. `cohort_sets` is `{cohortId: [pid, ...]}`
|
| 560 |
+
for cohort-membership leaves. Both REFUSE when needed and missing; neither defaults.
|
| 561 |
+
"""
|
| 562 |
+
state = {'aggregate': False, 'used': set(),
|
| 563 |
+
'today': today, 'cohort_sets': cohort_sets, 'id_sql': id_sql}
|
| 564 |
+
got = _compile_node({'conj': conj, 'children': list(nodes or [])}, columns or {}, state)
|
| 565 |
+
if got is None:
|
| 566 |
+
return None
|
| 567 |
+
sql, params = got
|
| 568 |
+
if member_ids:
|
| 569 |
+
if not id_sql:
|
| 570 |
+
raise ValueError('member_ids requires id_sql (the row-identity column)')
|
| 571 |
+
ids = [int(x) for x in member_ids]
|
| 572 |
+
sql = f"({sql} OR {id_sql} IN ({','.join('?' for _ in ids)}))"
|
| 573 |
+
params = list(params) + ids
|
| 574 |
+
return Compiled(sql, list(params), state['aggregate'], frozenset(state['used']))
|
| 575 |
+
|
| 576 |
+
|
| 577 |
+
def compile_search(q, columns):
|
| 578 |
+
"""Port of `matchSearch`: the narrowing search box.
|
| 579 |
+
|
| 580 |
+
True if ANY human-readable (text/status) column contains `q`, case-insensitively. NUMERIC
|
| 581 |
+
columns are skipped on purpose so a query like "10" does not match revenue or order counts.
|
| 582 |
+
Dates are skipped too β `matchSearch` tests only text and status.
|
| 583 |
+
|
| 584 |
+
The caller ANDs this with the filter predicate, matching the TS pipeline order
|
| 585 |
+
(filter -> search). It is deliberately OUTSIDE the `member_ids` union: in `useVisibleRows`
|
| 586 |
+
the union applies to the filter step only, so a hand-picked member is still searchable away.
|
| 587 |
+
"""
|
| 588 |
+
# `.strip(_JS_WS)`, not `.strip()`: the client trims with JS String.trim(), whose whitespace
|
| 589 |
+
# set differs from Python's in BOTH directions (Python strips U+0085 NEL and U+001C..1F,
|
| 590 |
+
# JS does not; JS strips U+FEFF, Python does not). A query of just those characters is
|
| 591 |
+
# blank to one engine and a real search to the other. Folding is left to SQL β see the
|
| 592 |
+
# LOWER(?) note in _compile_leaf.
|
| 593 |
+
q = (q or '').strip(_JS_WS)
|
| 594 |
+
if not q:
|
| 595 |
+
return None
|
| 596 |
+
parts, params, used = [], [], set()
|
| 597 |
+
aggregate = False
|
| 598 |
+
for col_id, spec in (columns or {}).items():
|
| 599 |
+
if spec['type'] not in TEXTUAL_TYPES:
|
| 600 |
+
continue
|
| 601 |
+
parts.append(f"(strpos({_value_sql(spec)}, LOWER(?)) > 0)")
|
| 602 |
+
params.append(q)
|
| 603 |
+
used.add(col_id)
|
| 604 |
+
aggregate = aggregate or bool(spec.get('aggregate'))
|
| 605 |
+
if not parts:
|
| 606 |
+
# No searchable column: `matchSearch` returns false for every row, so the search
|
| 607 |
+
# narrows to nothing. That is a real predicate, not an absent one.
|
| 608 |
+
return Compiled('1=0', [], False, frozenset())
|
| 609 |
+
return Compiled('(' + ' OR '.join(parts) + ')', params, aggregate, frozenset(used))
|
| 610 |
+
|
| 611 |
+
|
| 612 |
+
def compile_order_by(sorts, columns, tiebreak_sql=None):
|
| 613 |
+
"""Port of `makeComparator`. Returns an ORDER BY body, or None when nothing sorts.
|
| 614 |
+
|
| 615 |
+
`tiebreak_sql` (the row-identity column) is appended as a final ASC key and is REQUIRED for
|
| 616 |
+
any windowed fetch. `Array.prototype.sort` is stable, so TS leaves tied rows in their
|
| 617 |
+
incoming order; SQL guarantees NOTHING for ties, so LIMIT/OFFSET paging over a non-total
|
| 618 |
+
order can repeat a row on one page and drop it from another. Appending row identity makes
|
| 619 |
+
the order total. It is a STRENGTHENING of the TS guarantee, not a divergence: the payload a
|
| 620 |
+
windowed fetch produces is itself in this order, so the client never re-sorts it differently.
|
| 621 |
+
|
| 622 |
+
BLANKS SORT LAST IN BOTH DIRECTIONS (Airtable's behavior, and the reason descending
|
| 623 |
+
"Last order" leads with the most recent rather than with never-ordered customers). Each key
|
| 624 |
+
contributes TWO terms: an always-ASC blank flag, then the value in the requested direction.
|
| 625 |
+
Two blanks tie on both terms, so the NEXT key decides β matching the comparator's
|
| 626 |
+
`if (aBlank && bBlank) continue`.
|
| 627 |
+
|
| 628 |
+
An unknown column is skipped: in TS its values are `undefined` on both sides, so both are
|
| 629 |
+
blank and the key is a no-op tiebreak.
|
| 630 |
+
|
| 631 |
+
Direction: `s.dir === "asc" ? c : -c` β only the exact string "asc" ascends, anything else
|
| 632 |
+
descends. Faithfully reproduced; do not "fix" it to default ASC.
|
| 633 |
+
|
| 634 |
+
Residual divergence, deliberate: TS uses `localeCompare`, we use LOWER() + DuckDB's binary
|
| 635 |
+
collation. They agree on case-insensitive ordering of ASCII and on ISO dates (unaffected by
|
| 636 |
+
LOWER), and differ on locale tie-breaks between case-variants of the same string ("a" vs
|
| 637 |
+
"A" sort adjacent under localeCompare, tie under LOWER) and on accent ordering.
|
| 638 |
+
"""
|
| 639 |
+
terms = []
|
| 640 |
+
for s in (sorts or []):
|
| 641 |
+
if not isinstance(s, dict):
|
| 642 |
+
continue
|
| 643 |
+
spec = (columns or {}).get(s.get('colId'))
|
| 644 |
+
if spec is None:
|
| 645 |
+
continue
|
| 646 |
+
terms.append(f"CASE WHEN {_blank_sql(spec)} THEN 1 ELSE 0 END ASC")
|
| 647 |
+
terms.append(f"{_value_sql(spec)} {'ASC' if s.get('dir') == 'asc' else 'DESC'}")
|
| 648 |
+
# Append the tiebreak even when NO sort key survived (all unknown, or none given). A
|
| 649 |
+
# windowed fetch needs a total order regardless of what the user sorted by β an unsorted
|
| 650 |
+
# LIMIT/OFFSET is exactly as unstable as a tied one. Callers wanting pure client parity
|
| 651 |
+
# pass no tiebreak_sql and still get None.
|
| 652 |
+
if tiebreak_sql:
|
| 653 |
+
terms.append(f"{tiebreak_sql} ASC")
|
| 654 |
+
return ', '.join(terms) or None
|
royalimports_os/harness/measure_filter.py
ADDED
|
@@ -0,0 +1,666 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""harness/measure_filter.py β answering `Where [Sales] [in the last 90 days] [>] [5,000]` (CG-8).
|
| 2 |
+
|
| 3 |
+
GATED 2026-07-26 by `royalimports_os/verify_measure_filter.py` β 119 assertions against the real
|
| 4 |
+
store (every resolved id set compared to a per-customer aggregate summed in Python from raw rows)
|
| 5 |
+
plus a synthetic store for the cases live data cannot produce. Negative control: 24 faithful
|
| 6 |
+
mutations, 24 caught. The gate found two things the reasoning below asserted and the code did not
|
| 7 |
+
do, both now fixed and both re-caught by the control:
|
| 8 |
+
|
| 9 |
+
- a rule with NO window resolved to ALL TIME. The error message four lines below says it
|
| 10 |
+
refuses to do exactly that; the line above it did it anyway.
|
| 11 |
+
- `sum()` over an all-NULL group is NULL, and `NULL > x` and `NULL <= x` are BOTH false, so
|
| 12 |
+
such a customer fell out of the two halves of the same question. Now COALESCEd to 0.
|
| 13 |
+
|
| 14 |
+
THE PROBLEM. `revenue_ytd` is a pre-computed column in `customer_data.pool()`. "Sales in the last
|
| 15 |
+
90 days" is a DIFFERENT aggregate over a DIFFERENT window, and no column holds it. So a measure
|
| 16 |
+
condition cannot be answered by looking at the row in hand β it has to be a query.
|
| 17 |
+
|
| 18 |
+
THE SHAPE. Resolve each measure condition server-side to a **set of customer ids**, and let the
|
| 19 |
+
existing client engine treat that set as a membership test. Per condition, one query:
|
| 20 |
+
|
| 21 |
+
SELECT rp_id FROM <topic scope> WHERE <window> GROUP BY rp_id
|
| 22 |
+
HAVING <the metric's OWN aggregate expression> <op> ?
|
| 23 |
+
|
| 24 |
+
Three properties that fall out, none incidental:
|
| 25 |
+
|
| 26 |
+
* **No second definition of a number.** The aggregate comes from `semantic._measure_sql` β the
|
| 27 |
+
same expression the Analyst and every metric already use. This cannot disagree with "revenue"
|
| 28 |
+
elsewhere because it IS that computation.
|
| 29 |
+
* **The displayed columns are untouched.** `revenue_ytd` still comes from `pool()`. We ADD a
|
| 30 |
+
predicate; we do not re-source the table. So this needs no `pool()`βstore migration, and does
|
| 31 |
+
not inherit that migration's parity problem.
|
| 32 |
+
* **Counts stay honest.** The client still holds every row, so the table stays whole-book and
|
| 33 |
+
`countLabel` keeps counting what it actually has (CG-3 `whole-book` mode).
|
| 34 |
+
|
| 35 |
+
β WHAT MAY BE OFFERED, AND WHY IT IS AN ALLOWLIST
|
| 36 |
+
`store_parity()` validates exactly five metrics against live Odoo. Every OTHER sum-metric on the
|
| 37 |
+
topic would get an expression with no independent check behind it β self-consistent with the rest
|
| 38 |
+
of the app and still wrong if the expression is wrong. Against "robust for EVERY numerically
|
| 39 |
+
driven object", that gap is load-bearing, so admission is gated on HAVING A CONTRACT and spelled
|
| 40 |
+
as a list. An allowlist fails CLOSED when a metric is added; iterating the topic's metrics would
|
| 41 |
+
fail open, silently offering an unvalidated number in the one place a number looks authoritative.
|
| 42 |
+
|
| 43 |
+
β AN UNRESOLVABLE CONDITION RAISES. It never returns "everything" or "nothing": both are answers,
|
| 44 |
+
and a wrong answer here is invisible β the grid just shows a different set of customers under a
|
| 45 |
+
count that looks authoritative. `ModelError` is the only safe failure.
|
| 46 |
+
"""
|
| 47 |
+
from harness import semantic as _sem
|
| 48 |
+
from harness import windows as _win
|
| 49 |
+
|
| 50 |
+
#: metric key -> how it reads in the condition builder. ONLY metrics with an independent
|
| 51 |
+
#: validation contract (semantic.store_parity / a metric `validate:` block) appear here.
|
| 52 |
+
#: `orders` lives on sales_orders and `customers` is a property of a SET, not of a customer β
|
| 53 |
+
#: neither is a per-customer measure, so neither is offered.
|
| 54 |
+
#: Each entry carries its TOPIC, because admission is no longer single-topic. `orders` lives on
|
| 55 |
+
#: sales_orders (order-header grain) while the other three live on sales_lines β and the topic
|
| 56 |
+
#: decides the FROM clause, the scope SQL and which dim identifies a customer. Letting the caller
|
| 57 |
+
#: pass the topic (the old shape) meant a caller could pass a wrong one; deriving it from the
|
| 58 |
+
#: measure means there is nothing to get wrong.
|
| 59 |
+
ADMITTED = {
|
| 60 |
+
'revenue': {'label': 'Sales', 'topic': 'sales_lines'},
|
| 61 |
+
'margin': {'label': 'Gross Margin $', 'topic': 'sales_lines'},
|
| 62 |
+
'units': {'label': 'Units', 'topic': 'sales_lines'},
|
| 63 |
+
# `orders` is admitted on the SAME rule as the other three and not by exception:
|
| 64 |
+
# `semantic.store_parity()` checks it store-vs-live per BU, and the metric carries an
|
| 65 |
+
# `order_count` validate contract. It is the replacement for the retired `Orders 24M`
|
| 66 |
+
# column β same question, with the period as a parameter instead of frozen at 24 months.
|
| 67 |
+
'orders': {'label': 'Orders', 'topic': 'sales_orders'},
|
| 68 |
+
# β The first COMPOSITE measure (2026-07-27, owner's follow-up): a RATIO, not an aggregate.
|
| 69 |
+
# `aov` is `revenue` (line grain, sales_lines) over `orders` (header grain, sales_orders), so
|
| 70 |
+
# it has no single query and `_measure_sql` REFUSES it β which is the fail-closed shape that
|
| 71 |
+
# makes a mis-wired ratio raise instead of quietly computing something else. It is resolved
|
| 72 |
+
# by computing each BASE per customer and combining them through `semantic._post_compute`,
|
| 73 |
+
# the same function every other surface uses, so AOV here cannot disagree with AOV anywhere.
|
| 74 |
+
# `topic` names where the metric is DECLARED; the bases each resolve on their own.
|
| 75 |
+
'aov': {'label': 'Avg order $', 'topic': 'sales_lines'},
|
| 76 |
+
# β `input_scale` SETTLES the question this measure was withheld over. The metric stays a
|
| 77 |
+
# FRACTION (margin/revenue, 0-1) β it is the semantic model's number and every other surface
|
| 78 |
+
# reads it β while the USER'S TYPED VALUE is divided by 100 on the way in, so
|
| 79 |
+
# `Gross Margin % > 20` compares against 0.20. The factor moves exactly once, at the input
|
| 80 |
+
# boundary, and never touches the metric. Scaling a COMPUTED comparand (a statistic, or the
|
| 81 |
+
# other side of a measure-vs-measure) would divide by 100 twice, so it is applied to the
|
| 82 |
+
# literal alone and the gate proves that by construction.
|
| 83 |
+
'margin_pct': {'label': 'Gross Margin %', 'topic': 'sales_lines', 'input_scale': 100.0},
|
| 84 |
+
# β COMPANY-LEVEL (2026-07-27, owner's ask). Credit notes carry team_id = 1 for everything,
|
| 85 |
+
# so a business-unit total is not a thing that exists β `_query_parts` refuses one and the
|
| 86 |
+
# live domain builder refuses one. `measure_fields(team_id)` therefore stops OFFERING these
|
| 87 |
+
# to a BU-scoped caller rather than letting them build a condition that can only error: a
|
| 88 |
+
# dead entry in the picker is worse than an absent one, and hiding what cannot be answered
|
| 89 |
+
# is what `filterable: false` already does for columns.
|
| 90 |
+
'returns': {'label': 'Returns $', 'topic': 'credit_notes'},
|
| 91 |
+
'returns_pct': {'label': 'Return %', 'topic': 'credit_notes', 'input_scale': 100.0},
|
| 92 |
+
}
|
| 93 |
+
|
| 94 |
+
#: The metrics `semantic.store_parity()` actually exercises against live Odoo. ADMITTED must be
|
| 95 |
+
#: a SUBSET of this β that is the whole admission rule ("does this measure carry an independent
|
| 96 |
+
#: validation contract"), and stating it as data lets the gate assert it instead of trusting the
|
| 97 |
+
#: comment above. Adding a measure here without adding it to store_parity's loop fails the gate.
|
| 98 |
+
PARITY_CHECKED = frozenset({'revenue', 'margin', 'units', 'orders', 'customers',
|
| 99 |
+
'returns', 'invoiced'})
|
| 100 |
+
|
| 101 |
+
#: numeric comparison ops, mirroring filter_sql._NUM_CMP. `between` is expressed as two
|
| 102 |
+
#: conditions by the caller rather than a third operator shape.
|
| 103 |
+
OPS = {'gt': '>', 'gte': '>=', 'lt': '<', 'lte': '<=', 'eq': '=', 'neq': '<>'}
|
| 104 |
+
|
| 105 |
+
#: CG-9 β the ops allowed when the right-hand side is ANOTHER MEASURE. `eq`/`neq` are withheld,
|
| 106 |
+
#: and not out of caution: both sides are sums of floats the user never sees, so `=` would match
|
| 107 |
+
#: nobody and `β ` would match everybody, and both would LOOK like they worked. CG-8 could offer
|
| 108 |
+
#: `=` honestly because the other side was a number a person typed. Mirrors types.ts
|
| 109 |
+
#: MEASURE_PAIR_OPS; the gate asserts the two lists are the same.
|
| 110 |
+
PAIR_OPS = ('gte', 'gt', 'lte', 'lt')
|
| 111 |
+
|
| 112 |
+
#: The statistics a condition may compare against β "who is in the top quartile" (owner,
|
| 113 |
+
#: 2026-07-27). Each is computed over the CALLER'S POOL for the same measure and window.
|
| 114 |
+
#: The labels are the sentence the user reads; the client mirrors them.
|
| 115 |
+
STATS = {
|
| 116 |
+
'avg': 'the average',
|
| 117 |
+
'median': 'the median',
|
| 118 |
+
'p10': 'the bottom 10%',
|
| 119 |
+
'p25': 'the bottom quartile',
|
| 120 |
+
'p75': 'the top quartile',
|
| 121 |
+
'p90': 'the top 10%',
|
| 122 |
+
}
|
| 123 |
+
# β The owner's OWN words, and short enough to fit: "the 75th percentile (top quartile)" clipped
|
| 124 |
+
# to "the 75th perce" in the value select, and a truncated parenthetical reads as a rendering
|
| 125 |
+
# bug. `> the top quartile` is also how a person says it out loud, so the shorter label is the
|
| 126 |
+
# better one twice over. Seen in a live screenshot of a run whose every assertion was green.
|
| 127 |
+
|
| 128 |
+
#: β THE POPULATION IS THE CALLER'S POOL, INCLUDING ZEROS. A customer who bought nothing has a
|
| 129 |
+
#: measure of 0, not of nothing β that is the convention the zero-group already establishes
|
| 130 |
+
#: everywhere else, and "the median customer" means the median of the customers you can see.
|
| 131 |
+
#: Two consequences worth knowing before reading a result:
|
| 132 |
+
#: Β· it is a PERMISSION boundary as well as a definition β a scoped user's median is theirs,
|
| 133 |
+
#: and a statistic over the whole book would leak the shape of data they cannot open;
|
| 134 |
+
#: Β· on a measure most customers score 0 on, the 75th percentile IS 0, so "top quartile"
|
| 135 |
+
#: degenerates to "anyone with any activity at all". True, and surprising.
|
| 136 |
+
|
| 137 |
+
|
| 138 |
+
def percentile(values, q):
|
| 139 |
+
"""The `q` quantile of `values` by LINEAR INTERPOLATION between closest ranks β the R-7
|
| 140 |
+
convention, which is what Excel's PERCENTILE.INC computes. `q` may also be a STATS key.
|
| 141 |
+
|
| 142 |
+
Pinned to one convention on purpose: nearest-rank and interpolation disagree on any even
|
| 143 |
+
population (the median of [1,2,3,4] is 2.5 here and 2 or 3 under nearest-rank), and the gate
|
| 144 |
+
checks it against hand-computed values rather than against `statistics.median` β a reference
|
| 145 |
+
borrowed from a library only proves we match that library.
|
| 146 |
+
|
| 147 |
+
Returns None for an empty population: there is no median of nobody, and 0 would be a number
|
| 148 |
+
the caller could act on.
|
| 149 |
+
"""
|
| 150 |
+
if isinstance(q, str):
|
| 151 |
+
if q == 'avg':
|
| 152 |
+
xs = list(values)
|
| 153 |
+
return (sum(xs) / len(xs)) if xs else None
|
| 154 |
+
if q not in STATS:
|
| 155 |
+
raise _sem.ModelError(
|
| 156 |
+
f"{q!r} is not a statistic. Offered: {sorted(STATS)}")
|
| 157 |
+
q = 0.5 if q == 'median' else int(q[1:]) / 100.0
|
| 158 |
+
xs = sorted(float(v) for v in values)
|
| 159 |
+
if not xs:
|
| 160 |
+
return None
|
| 161 |
+
if len(xs) == 1:
|
| 162 |
+
return xs[0]
|
| 163 |
+
pos = (len(xs) - 1) * float(q)
|
| 164 |
+
lo = int(pos)
|
| 165 |
+
hi = min(lo + 1, len(xs) - 1)
|
| 166 |
+
return xs[lo] + (xs[hi] - xs[lo]) * (pos - lo)
|
| 167 |
+
|
| 168 |
+
|
| 169 |
+
#: The same six comparisons as OPS, as Python callables β used wherever the comparison happens in
|
| 170 |
+
#: Python rather than in a HAVING (a composite measure, or a two-measure pair).
|
| 171 |
+
_CMP = {'gt': lambda a, b: a > b, 'gte': lambda a, b: a >= b,
|
| 172 |
+
'lt': lambda a, b: a < b, 'lte': lambda a, b: a <= b,
|
| 173 |
+
'eq': lambda a, b: a == b, 'neq': lambda a, b: a != b}
|
| 174 |
+
|
| 175 |
+
|
| 176 |
+
def is_company_level(key):
|
| 177 |
+
"""True when this measure cannot be scoped to a business unit.
|
| 178 |
+
|
| 179 |
+
Checked over every BASE the measure decomposes into, not just the declared topic: `Return %`
|
| 180 |
+
names credit_notes but also reads customer_invoices, and one un-scopable half is enough to
|
| 181 |
+
make the whole ratio un-scopable.
|
| 182 |
+
"""
|
| 183 |
+
bases, _ = _sem._expand_measures([key])
|
| 184 |
+
tps = _sem.topics()
|
| 185 |
+
for b in bases:
|
| 186 |
+
store = (tps.get(_sem.metrics()[b].get('topic')) or {}).get('store') or {}
|
| 187 |
+
if not store.get('team_col'):
|
| 188 |
+
return True
|
| 189 |
+
return False
|
| 190 |
+
|
| 191 |
+
|
| 192 |
+
def measure_fields(team_id=None):
|
| 193 |
+
"""The measures the condition builder may offer, in the payload's field shape.
|
| 194 |
+
|
| 195 |
+
These are FILTERABLE BUT NOT DISPLAYABLE β a measure is a question about a window, not a
|
| 196 |
+
column of the table, and rendering it as a column would silently pick a window for the user.
|
| 197 |
+
|
| 198 |
+
β `team_id` NARROWS THE OFFER. A COMPANY-LEVEL measure (returns, and anything else built on
|
| 199 |
+
documents that carry no trustworthy BU tag) cannot be answered for one business unit β
|
| 200 |
+
`_query_parts` refuses, and refusing is right. But a picker entry whose every use ends in an
|
| 201 |
+
error is worse than no entry, so a BU-scoped caller is not offered them at all. Same
|
| 202 |
+
principle as `filterable: false` on a column: do not offer what cannot be answered.
|
| 203 |
+
"""
|
| 204 |
+
mts = _sem.metrics()
|
| 205 |
+
out = []
|
| 206 |
+
for key, spec in ADMITTED.items():
|
| 207 |
+
m = mts.get(key)
|
| 208 |
+
if not m:
|
| 209 |
+
continue
|
| 210 |
+
if team_id and is_company_level(key):
|
| 211 |
+
continue
|
| 212 |
+
out.append({'key': key, 'label': spec['label'],
|
| 213 |
+
'type': _sem._FORMAT_TYPE.get(m.get('format'), 'currency')})
|
| 214 |
+
return out
|
| 215 |
+
|
| 216 |
+
|
| 217 |
+
def topic_for(key):
|
| 218 |
+
"""The topic a measure is answered on. Raises for anything not admitted."""
|
| 219 |
+
spec = ADMITTED.get(key)
|
| 220 |
+
if not spec:
|
| 221 |
+
raise _sem.ModelError(
|
| 222 |
+
f"{key!r} is not an admitted measure. Offered: {sorted(ADMITTED)} β a measure "
|
| 223 |
+
f"without an independent validation contract is not filterable (see this module's "
|
| 224 |
+
f"docstring).")
|
| 225 |
+
return spec['topic']
|
| 226 |
+
|
| 227 |
+
|
| 228 |
+
def _entity_sql(topic):
|
| 229 |
+
"""(from-clause, the id column customers are grouped by, the store binding) for a topic."""
|
| 230 |
+
t = _sem.topics().get(topic)
|
| 231 |
+
if not t or 'store' not in t:
|
| 232 |
+
raise _sem.ModelError(f"topic {topic!r} has no store binding")
|
| 233 |
+
s = t['store']
|
| 234 |
+
dims = s.get('dims') or {}
|
| 235 |
+
# sales_lines calls it `order_partner` (the line's parent order's partner); sales_orders
|
| 236 |
+
# calls the same thing `partner`. One name per topic, both meaning "the customer this row
|
| 237 |
+
# belongs to" β which is the only dim this module ever groups by.
|
| 238 |
+
dim = dims.get('order_partner') or dims.get('partner')
|
| 239 |
+
if not dim:
|
| 240 |
+
raise _sem.ModelError(f"topic {topic!r} has no customer dim to group by")
|
| 241 |
+
return f"{s['table']} {s['alias']} {s.get('join', '')}", dim['col'], s
|
| 242 |
+
|
| 243 |
+
|
| 244 |
+
def is_composite(key):
|
| 245 |
+
"""True for a measure with no single aggregate expression β a `ratio` or a `derived`.
|
| 246 |
+
|
| 247 |
+
These are answered by computing each BASE per customer and combining in Python; they can
|
| 248 |
+
never be a HAVING, which is why `_measure_sql` refusing them is a feature.
|
| 249 |
+
"""
|
| 250 |
+
return _sem.metrics().get(key, {}).get('agg') in ('ratio', 'derived')
|
| 251 |
+
|
| 252 |
+
|
| 253 |
+
def _base_topic(key):
|
| 254 |
+
"""The topic a BASE metric aggregates on β read from the METRIC, never from ADMITTED.
|
| 255 |
+
|
| 256 |
+
A composite decomposes into bases that need not themselves be offerable (and whose topics may
|
| 257 |
+
differ from each other: `aov`'s numerator is on sales_lines and its denominator on
|
| 258 |
+
sales_orders). Resolving the topic from the allowlist would make that undecidable.
|
| 259 |
+
"""
|
| 260 |
+
m = _sem.metrics().get(key)
|
| 261 |
+
if not m:
|
| 262 |
+
raise _sem.ModelError(f"unknown metric {key!r}")
|
| 263 |
+
topic = m.get('topic')
|
| 264 |
+
if not topic:
|
| 265 |
+
raise _sem.ModelError(f"metric {key!r} declares no topic to aggregate on")
|
| 266 |
+
return topic
|
| 267 |
+
|
| 268 |
+
|
| 269 |
+
def _query_parts(key, window, today, team_id):
|
| 270 |
+
"""The FROM / GROUP-BY id / aggregate / WHERE for one (measure, window), shared by the
|
| 271 |
+
one-sided and the two-sided resolver so there is one definition of the scope."""
|
| 272 |
+
topic = _base_topic(key)
|
| 273 |
+
# NO DEFAULT WINDOW. A rule that carries no window is not "all time" β it is a rule whose
|
| 274 |
+
# window we do not know, and answering it over all time WIDENS the condition while the count
|
| 275 |
+
# beside it still looks authoritative. `all_time` is a legitimate ANSWER the user can pick;
|
| 276 |
+
# it is not a legitimate GUESS. (This line shipped as `rule.get('window') or {...}` and the
|
| 277 |
+
# gate caught it contradicting the error message two lines below.)
|
| 278 |
+
bounds = _win.resolve(window, today)
|
| 279 |
+
if bounds is None:
|
| 280 |
+
raise _sem.ModelError(
|
| 281 |
+
f"unresolvable window {window!r} β refusing rather than defaulting to all "
|
| 282 |
+
f"time, which would widen the condition while the count still looked authoritative")
|
| 283 |
+
date_from, date_to = bounds
|
| 284 |
+
|
| 285 |
+
frm, id_col, s = _entity_sql(topic)
|
| 286 |
+
m = _sem.metrics()[key]
|
| 287 |
+
# COALESCE, because SQL's sum() over an all-NULL group is NULL, and `NULL > x` and
|
| 288 |
+
# `NULL <= x` are BOTH false: the customer would vanish from the two halves of the same
|
| 289 |
+
# question, so `margin <= 0` and `margin > 0` would not cover the book. The zero-group below
|
| 290 |
+
# cannot rescue them either β they DO have rows in the window, just no values. Python's sum
|
| 291 |
+
# of the same rows is 0.0, and 0 is what the grid would show, so 0 is the answer.
|
| 292 |
+
agg = f"COALESCE({_sem._measure_sql(m, s['alias'])}, 0)"
|
| 293 |
+
|
| 294 |
+
where, params = [_sem.render_scope(s['scope_sql'].strip())], []
|
| 295 |
+
if date_from:
|
| 296 |
+
where.append(f"CAST({s['date_col']} AS TIMESTAMP) >= ?")
|
| 297 |
+
params.append(f"{date_from} 00:00:00")
|
| 298 |
+
if date_to:
|
| 299 |
+
where.append(f"CAST({s['date_col']} AS TIMESTAMP) <= ?")
|
| 300 |
+
params.append(f"{date_to} 23:59:59")
|
| 301 |
+
if team_id:
|
| 302 |
+
if not s.get('team_col'):
|
| 303 |
+
raise _sem.ModelError(f"topic {topic!r} is company-level β it has no business-unit "
|
| 304 |
+
f"filter, so a BU-scoped caller cannot be answered safely")
|
| 305 |
+
where.append(f"{s['team_col']} = ?")
|
| 306 |
+
params.append(int(team_id))
|
| 307 |
+
return frm, id_col, agg, ' AND '.join(where), params
|
| 308 |
+
|
| 309 |
+
|
| 310 |
+
def _scaled(key, value):
|
| 311 |
+
"""A TYPED value in the units the metric actually holds.
|
| 312 |
+
|
| 313 |
+
`Gross Margin % > 20` is typed in percentage points and the metric is a fraction, so the 20
|
| 314 |
+
becomes 0.20. Applied to a LITERAL only β a statistic and the other side of a
|
| 315 |
+
measure-vs-measure are already on the metric's scale, and dividing them too would apply the
|
| 316 |
+
factor twice for a result that still looks plausible.
|
| 317 |
+
"""
|
| 318 |
+
scale = ADMITTED.get(key, {}).get('input_scale')
|
| 319 |
+
return (value / scale) if scale else value
|
| 320 |
+
|
| 321 |
+
|
| 322 |
+
def resolve_stat(rule, today, team_id=None, allowed_pids=None):
|
| 323 |
+
"""`Where [Sales] [this year] [>] [the 75th percentile]` β the top quartile of the pool.
|
| 324 |
+
|
| 325 |
+
One totals query for the measure, the statistic taken over the CALLER'S POOL (see the STATS
|
| 326 |
+
note on why the population is the pool and what that costs), then a comparison per customer.
|
| 327 |
+
"""
|
| 328 |
+
stat = (rule.get('rhs') or {}).get('stat')
|
| 329 |
+
if stat not in STATS:
|
| 330 |
+
raise _sem.ModelError(f"{stat!r} is not a statistic. Offered: {sorted(STATS)}")
|
| 331 |
+
key = rule.get('colId')
|
| 332 |
+
topic_for(key)
|
| 333 |
+
op = rule.get('op')
|
| 334 |
+
if op not in PAIR_OPS:
|
| 335 |
+
raise _sem.ModelError(
|
| 336 |
+
f"{op!r} cannot compare a measure with a statistic ({sorted(PAIR_OPS)}). Equality "
|
| 337 |
+
f"against a computed cut-off matches nobody except by coincidence.")
|
| 338 |
+
pool = _pool_of(allowed_pids)
|
| 339 |
+
if pool is None:
|
| 340 |
+
raise _sem.ModelError(
|
| 341 |
+
"a statistic needs a population to be computed over, and none was supplied β "
|
| 342 |
+
"refusing rather than taking it over whichever customers happen to have rows, "
|
| 343 |
+
"which would be a different question with an authoritative-looking answer")
|
| 344 |
+
con = _sem._store_con()
|
| 345 |
+
try:
|
| 346 |
+
totals = _totals(con, key, rule.get('window'), today, team_id)
|
| 347 |
+
finally:
|
| 348 |
+
con.close()
|
| 349 |
+
# β THE POPULATION IS THE CUSTOMERS WHO HAVE THIS MEASURE AT ALL β owner's decision,
|
| 350 |
+
# 2026-07-27: *"top quartile of customers who actually has returned something"*.
|
| 351 |
+
#
|
| 352 |
+
# It was the whole pool including zeros, which is the convention everywhere else and was
|
| 353 |
+
# wrong HERE. The pool carries lapsed customers, so ~30% of it sits at zero for any 12-month
|
| 354 |
+
# window β that dragged the 25th percentile to exactly 0.0, made `< the bottom quartile`
|
| 355 |
+
# match NOBODY, and would have been worse for Return %, where most customers never return
|
| 356 |
+
# anything and "the top quartile" would have degenerated to "anyone with a single refund".
|
| 357 |
+
#
|
| 358 |
+
# NON-ZERO, not has-rows: a zero Return % means nothing was returned, which is precisely the
|
| 359 |
+
# customer the owner is excluding. It also reads as one sentence β the question is asked
|
| 360 |
+
# among customers who have that measure β and it generalises: "the median Sales" means the
|
| 361 |
+
# median among customers who sold something.
|
| 362 |
+
#
|
| 363 |
+
# The ANSWER is restricted to the same population, so `< the bottom quartile` returns a
|
| 364 |
+
# quarter of the returners rather than every customer who never returned.
|
| 365 |
+
population = {pid: v for pid, v in totals.items() if pid in pool and v}
|
| 366 |
+
cut = percentile(list(population.values()), stat)
|
| 367 |
+
if cut is None: # nobody has this measure -> nobody qualifies
|
| 368 |
+
return set()
|
| 369 |
+
return {pid for pid, v in population.items() if _CMP[op](v, cut)}
|
| 370 |
+
|
| 371 |
+
|
| 372 |
+
def _pool_of(allowed_pids):
|
| 373 |
+
"""The caller's visible pool as a set of plain ints, or None.
|
| 374 |
+
|
| 375 |
+
β The pool arrives from `pool()`/a cohort and may hold numpy ints or id strings; the store
|
| 376 |
+
returns BIGINTs. A set difference across two int types silently finds nothing in common,
|
| 377 |
+
which here would look like "no lapsed customers" rather than an error.
|
| 378 |
+
"""
|
| 379 |
+
return None if allowed_pids is None else {int(p) for p in allowed_pids if p is not None}
|
| 380 |
+
|
| 381 |
+
|
| 382 |
+
def rule_complete(rule):
|
| 383 |
+
"""Is this rule's right-hand side FULLY present β a question that has actually been asked?
|
| 384 |
+
|
| 385 |
+
THE GAP THIS CLOSES (2026-07-27, the cold-container race). The component's view state
|
| 386 |
+
passes through the host while the user β or the client's own initialisation β is mid-edit,
|
| 387 |
+
and a rule can transiently arrive with its window, statistic or value missing. `to_num(None)`
|
| 388 |
+
is 0 by the engines' shared contract, so resolution used to ANSWER that half-asked question
|
| 389 |
+
as `<op> 0`: the id landed in `measureSets`, the client read the rule as answered and
|
| 390 |
+
cleared its pending marker, and the grid stood behind a count for a question nobody asked.
|
| 391 |
+
Refusing instead leaves the id OUT of the payload, which the client renders as PENDING
|
| 392 |
+
("Calculatingβ¦") β the honest state for a question still being typed. Exactly the class the
|
| 393 |
+
module docstring already names: an unresolvable condition never becomes an answer.
|
| 394 |
+
|
| 395 |
+
INCOMPLETE is distinct from MALFORMED, on purpose. An ABSENT window / stat / value is a
|
| 396 |
+
normal transient (the caller skips it silently β mid-edit is not a telemetry event); a
|
| 397 |
+
PRESENT-but-bogus one is a real defect and still RAISES downstream (`unresolvable window`,
|
| 398 |
+
`not a statistic`, `not admitted`). `value` presence is textual, not numeric: "0" is a
|
| 399 |
+
typed answer, while parseability stays `to_num`'s business so the two engines keep one
|
| 400 |
+
contract for garbage input.
|
| 401 |
+
"""
|
| 402 |
+
if not isinstance(rule, dict) or not rule.get('op'):
|
| 403 |
+
return False
|
| 404 |
+
if rule.get('window') is None:
|
| 405 |
+
return False
|
| 406 |
+
rhs = rule.get('rhs')
|
| 407 |
+
if isinstance(rhs, dict) and rhs.get('kind') == 'measure':
|
| 408 |
+
return bool(rhs.get('colId')) and rhs.get('window') is not None
|
| 409 |
+
if isinstance(rhs, dict) and rhs.get('kind') == 'stat':
|
| 410 |
+
return bool(rhs.get('stat'))
|
| 411 |
+
v = rule.get('value')
|
| 412 |
+
return v is not None and str(v).strip() != ''
|
| 413 |
+
|
| 414 |
+
|
| 415 |
+
def resolve_rule(rule, today, team_id=None, allowed_pids=None):
|
| 416 |
+
"""One measure condition -> the SET OF CUSTOMER IDS satisfying it.
|
| 417 |
+
|
| 418 |
+
`rule` = {colId: <measure key>, window: <window spec>, op: <numeric op>, value: <str|num>},
|
| 419 |
+
or β CG-9 β the same with `rhs: {kind:'measure', colId, window}` instead of a value, which
|
| 420 |
+
hands off to `resolve_pair`.
|
| 421 |
+
|
| 422 |
+
β Customers with NO rows in the window are absent from the GROUP BY, so a condition like
|
| 423 |
+
`revenue < 100` would miss them even though 0 < 100 is true. That is the single most
|
| 424 |
+
dangerous thing about this query shape: the missing rows are exactly the lapsed customers a
|
| 425 |
+
"who has gone quiet" filter is FOR. So the caller passes `allowed_pids` (the pool) and the
|
| 426 |
+
zero-group is reconstructed explicitly below rather than left to the SQL.
|
| 427 |
+
"""
|
| 428 |
+
if not rule_complete(rule):
|
| 429 |
+
# An absent value would coerce to 0 and resolve as a DIFFERENT question under an
|
| 430 |
+
# authoritative-looking count β the same failure class as defaulting a missing window
|
| 431 |
+
# to all time, caught by the same principle: refuse, never guess.
|
| 432 |
+
raise _sem.ModelError(
|
| 433 |
+
'incomplete measure condition β its window, statistic or value is not present; '
|
| 434 |
+
'refusing to answer a question that has not been fully asked')
|
| 435 |
+
rhs = rule.get('rhs')
|
| 436 |
+
if isinstance(rhs, dict) and rhs.get('kind') == 'measure':
|
| 437 |
+
return resolve_pair(rule, today, team_id, allowed_pids)
|
| 438 |
+
if isinstance(rhs, dict) and rhs.get('kind') == 'stat':
|
| 439 |
+
return resolve_stat(rule, today, team_id, allowed_pids)
|
| 440 |
+
key = rule.get('colId')
|
| 441 |
+
topic_for(key) # raises for anything not admitted
|
| 442 |
+
op = rule.get('op')
|
| 443 |
+
if op not in OPS:
|
| 444 |
+
raise _sem.ModelError(f"{op!r} is not a numeric comparison ({sorted(OPS)})")
|
| 445 |
+
|
| 446 |
+
from harness import filter_sql as _fs
|
| 447 |
+
# β THE SCALE IS APPLIED HERE, TO THE TYPED VALUE, AND NOWHERE ELSE.
|
| 448 |
+
#
|
| 449 |
+
# Its position relative to `_zero_satisfies` turns out NOT to matter, and the negative
|
| 450 |
+
# control is what established that rather than reasoning: dividing by a POSITIVE constant
|
| 451 |
+
# cannot change how a number compares to zero, so the zero-group is identical either way.
|
| 452 |
+
# (An earlier version of this comment claimed the opposite. It was wrong.) What DOES matter
|
| 453 |
+
# is that a COMPUTED comparand never passes through here β see `_scaled`.
|
| 454 |
+
value = _scaled(key, _fs.to_num(rule.get('value')))
|
| 455 |
+
|
| 456 |
+
pool = _pool_of(allowed_pids)
|
| 457 |
+
if is_composite(key):
|
| 458 |
+
# A ratio has no HAVING to push the comparison into: compute the value per customer, then
|
| 459 |
+
# compare in Python β the same shape `resolve_pair` uses, and for the same reason.
|
| 460 |
+
con = _sem._store_con()
|
| 461 |
+
try:
|
| 462 |
+
totals = _totals(con, key, rule.get('window'), today, team_id)
|
| 463 |
+
finally:
|
| 464 |
+
con.close()
|
| 465 |
+
# No pool means we can only speak about customers a base has seen. WITH a pool, everyone
|
| 466 |
+
# else scores 0 through the ratio's guard β which is what makes the zero-group work for a
|
| 467 |
+
# ratio exactly as it does for a sum ("AOV <= 250" includes those who bought nothing).
|
| 468 |
+
universe = pool if pool is not None else set(totals)
|
| 469 |
+
return {pid for pid in universe if _CMP[op](totals.get(pid, 0.0), value)}
|
| 470 |
+
|
| 471 |
+
frm, id_col, agg, where_sql, params = _query_parts(
|
| 472 |
+
key, rule.get('window'), today, team_id)
|
| 473 |
+
sql = (f"SELECT {id_col} AS pid, {agg} AS v FROM {frm} WHERE {where_sql} "
|
| 474 |
+
f"GROUP BY {id_col} HAVING {agg} {OPS[op]} ?")
|
| 475 |
+
|
| 476 |
+
con = _sem._store_con()
|
| 477 |
+
try:
|
| 478 |
+
matched = {int(r[0]) for r in con.execute(sql, params + [value]).fetchall()
|
| 479 |
+
if r[0] is not None}
|
| 480 |
+
# The zero-group: every customer in scope with NO rows in the window aggregates to 0, and
|
| 481 |
+
# SQL cannot return a group that has no rows. Reconstruct it from the pool rather than
|
| 482 |
+
# pretending those customers do not exist.
|
| 483 |
+
if pool is not None:
|
| 484 |
+
present = {int(r[0]) for r in con.execute(
|
| 485 |
+
f"SELECT DISTINCT {id_col} FROM {frm} WHERE {where_sql}",
|
| 486 |
+
params).fetchall() if r[0] is not None}
|
| 487 |
+
absent = pool - present
|
| 488 |
+
if absent and _zero_satisfies(op, value):
|
| 489 |
+
matched |= absent
|
| 490 |
+
finally:
|
| 491 |
+
con.close()
|
| 492 |
+
if pool is not None:
|
| 493 |
+
matched &= pool
|
| 494 |
+
return matched
|
| 495 |
+
|
| 496 |
+
|
| 497 |
+
def _totals(con, key, window, today, team_id):
|
| 498 |
+
"""`{pid: value}` for one (measure, window), COMPOSITES included.
|
| 499 |
+
|
| 500 |
+
For a plain aggregate this is resolve_rule's query minus the HAVING. For a ratio or a derived
|
| 501 |
+
metric it is one query per BASE β each on that base's own topic β combined per customer by
|
| 502 |
+
`semantic._post_compute`, the same function `store_query` uses for the un-grouped number. So
|
| 503 |
+
AOV per customer is the app's AOV, computed once and defined once.
|
| 504 |
+
|
| 505 |
+
β The pid keys come from DIFFERENT dims for a cross-topic ratio (`l.order_partner_id` and
|
| 506 |
+
`o.partner_id`). The union is taken rather than the numerator's keys, so a customer either
|
| 507 |
+
side has seen gets a number; the ratio's own `den == 0 -> 0.0` guard covers the rest.
|
| 508 |
+
"""
|
| 509 |
+
if is_composite(key):
|
| 510 |
+
bases, _ = _sem._expand_measures([key])
|
| 511 |
+
parts = {b: _totals(con, b, window, today, team_id) for b in bases}
|
| 512 |
+
pids = set()
|
| 513 |
+
for got in parts.values():
|
| 514 |
+
pids |= set(got)
|
| 515 |
+
return {pid: _sem._post_compute({b: parts[b].get(pid, 0.0) for b in bases}, key)
|
| 516 |
+
for pid in pids}
|
| 517 |
+
frm, id_col, agg, where_sql, params = _query_parts(key, window, today, team_id)
|
| 518 |
+
sql = (f"SELECT {id_col} AS pid, {agg} AS v FROM {frm} WHERE {where_sql} GROUP BY {id_col}")
|
| 519 |
+
# `float(r[1])`, not `float(r[1] or 0.0)`: `_query_parts` already wraps the aggregate in
|
| 520 |
+
# COALESCE(..., 0), so a NULL cannot arrive here β a second guard would be unreachable, and
|
| 521 |
+
# the negative control proved it (deleting it changed nothing). If one ever DID arrive, a
|
| 522 |
+
# TypeError is the right outcome: it would mean the SQL-level guard had been removed, which
|
| 523 |
+
# is the bug worth hearing about rather than papering over.
|
| 524 |
+
return {int(r[0]): float(r[1])
|
| 525 |
+
for r in con.execute(sql, params).fetchall() if r[0] is not None}
|
| 526 |
+
|
| 527 |
+
|
| 528 |
+
def resolve_pair(rule, today, team_id=None, allowed_pids=None):
|
| 529 |
+
"""CG-9 β `Where [Sales] [this year] [>] [Sales] [year to date, last year]`.
|
| 530 |
+
|
| 531 |
+
TWO QUERIES, compared in Python. The obvious alternative β one query with two conditional
|
| 532 |
+
aggregates and a HAVING comparing them β would have to reach inside `_sem._measure_sql` to
|
| 533 |
+
inject a per-side window into an expression that is already a `sum(CASE WHEN β¦ )` for
|
| 534 |
+
filtered measures. Two queries keep ONE definition of each number (the metric's own
|
| 535 |
+
aggregate, unmodified), let the two sides sit on DIFFERENT TOPICS, and make the zero-group
|
| 536 |
+
fall out for free: absent from a side's result means 0 on that side.
|
| 537 |
+
|
| 538 |
+
β The zero-group is LOUDER here than it is for a one-sided rule. Every customer with no rows
|
| 539 |
+
in EITHER window compares 0 against 0, so `<=` and `>=` match all of them β correct, and it
|
| 540 |
+
will look wrong on a narrow window. That is why `eq`/`neq` are not offered at all
|
| 541 |
+
(PAIR_OPS): two float sums are never exactly equal except at zero, where they always are.
|
| 542 |
+
"""
|
| 543 |
+
rhs = rule.get('rhs') or {}
|
| 544 |
+
lkey, rkey = rule.get('colId'), rhs.get('colId')
|
| 545 |
+
topic_for(lkey), topic_for(rkey) # raises for anything not admitted
|
| 546 |
+
op = rule.get('op')
|
| 547 |
+
if op not in PAIR_OPS:
|
| 548 |
+
raise _sem.ModelError(
|
| 549 |
+
f"{op!r} cannot compare two measures ({sorted(PAIR_OPS)}). Equality between two sums "
|
| 550 |
+
f"of floats matches nobody and inequality matches everybody; neither answers the "
|
| 551 |
+
f"question, and both look like they work.")
|
| 552 |
+
cmp = _CMP[op]
|
| 553 |
+
|
| 554 |
+
pool = _pool_of(allowed_pids)
|
| 555 |
+
con = _sem._store_con()
|
| 556 |
+
try:
|
| 557 |
+
left = _totals(con, lkey, rule.get('window'), today, team_id)
|
| 558 |
+
right = _totals(con, rkey, rhs.get('window'), today, team_id)
|
| 559 |
+
finally:
|
| 560 |
+
con.close()
|
| 561 |
+
# With no pool we can only speak about customers at least one side has seen β the zero-vs-
|
| 562 |
+
# zero group is unknowable without being told who exists. With a pool, it is every member.
|
| 563 |
+
universe = pool if pool is not None else (set(left) | set(right))
|
| 564 |
+
return {pid for pid in universe if cmp(left.get(pid, 0.0), right.get(pid, 0.0))}
|
| 565 |
+
|
| 566 |
+
|
| 567 |
+
def _zero_satisfies(op, value):
|
| 568 |
+
"""Does a customer with NO activity in the window (measure = 0) satisfy this condition?"""
|
| 569 |
+
return _CMP[op](0.0, value)
|
| 570 |
+
|
| 571 |
+
|
| 572 |
+
def resolve_values(key, window, today, team_id=None, allowed_pids=None):
|
| 573 |
+
"""`{pid: display value}` for one (measure, window) over the caller's pool β the COLUMN
|
| 574 |
+
version of `resolve_rule` (owner item 7, 2026-07-27: a formula-measure FIELD).
|
| 575 |
+
|
| 576 |
+
Same `_totals` the filter conditions and statistics use, so the column and the condition
|
| 577 |
+
cannot disagree. Three display decisions live here, not in the client:
|
| 578 |
+
|
| 579 |
+
* ZERO-FILLED over the pool. A customer with no rows in the window has a measure of 0 β
|
| 580 |
+
the zero-group convention everywhere else β and 0 is a real value the cell shows. Blank
|
| 581 |
+
is reserved for "could not compute" (the caller omits the column entirely on failure).
|
| 582 |
+
* `input_scale` applied OUTWARD. `margin_pct` is a fraction (0-1) internally; the user
|
| 583 |
+
types percentage points into the filter (input divides by 100), so the CELL shows points
|
| 584 |
+
too (value Γ 100) β one boundary, both directions, and `yoy_pct` beside it already reads
|
| 585 |
+
in points.
|
| 586 |
+
* Rounded like every other numeric column: whole numbers for money/counts (the pool's own
|
| 587 |
+
`_round` convention), one decimal for a percent.
|
| 588 |
+
|
| 589 |
+
Raises for anything not admitted / an unresolvable window β the caller degrades that column
|
| 590 |
+
to blank rather than to a number nobody computed.
|
| 591 |
+
"""
|
| 592 |
+
topic_for(key) # raises for anything not admitted
|
| 593 |
+
# The window must RESOLVE before any query runs β same refusal as `_query_parts`, checked
|
| 594 |
+
# here too because a composite's `_totals` path resolves per base and would otherwise raise
|
| 595 |
+
# deeper with a less useful message.
|
| 596 |
+
if _win.resolve(window, today) is None:
|
| 597 |
+
raise _sem.ModelError(
|
| 598 |
+
f"unresolvable window {window!r} for the {key!r} column β refusing rather than "
|
| 599 |
+
f"defaulting to all time")
|
| 600 |
+
pool = _pool_of(allowed_pids)
|
| 601 |
+
con = _sem._store_con()
|
| 602 |
+
try:
|
| 603 |
+
totals = _totals(con, key, window, today, team_id)
|
| 604 |
+
finally:
|
| 605 |
+
con.close()
|
| 606 |
+
scale = ADMITTED.get(key, {}).get('input_scale') or 1.0
|
| 607 |
+
mtype = _sem._FORMAT_TYPE.get(_sem.metrics().get(key, {}).get('format'), 'currency')
|
| 608 |
+
nd = 1 if mtype == 'pct' else None # money/counts whole, percents one decimal
|
| 609 |
+
universe = pool if pool is not None else set(totals)
|
| 610 |
+
# round(x, None) returns an int β exactly the pool's `_round` convention for money/counts.
|
| 611 |
+
return {int(p): round(float(totals.get(p, 0.0)) * scale, nd) for p in universe}
|
| 612 |
+
|
| 613 |
+
|
| 614 |
+
def resolve_tree(rules, today, team_id=None, allowed_pids=None):
|
| 615 |
+
"""{rule_id: sorted pids} for every measure condition in a tree.
|
| 616 |
+
|
| 617 |
+
Keyed by the rule's client-generated `id` so the browser can map an answer back to the
|
| 618 |
+
condition that asked it β positional matching would silently re-associate every answer the
|
| 619 |
+
moment a user deleted a condition.
|
| 620 |
+
"""
|
| 621 |
+
out = {}
|
| 622 |
+
for r in rules or []:
|
| 623 |
+
rid = str(r.get('id') or '')
|
| 624 |
+
if not rid:
|
| 625 |
+
raise _sem.ModelError('a measure condition needs a stable id to carry its answer back')
|
| 626 |
+
out[rid] = sorted(resolve_rule(r, today, team_id, allowed_pids))
|
| 627 |
+
return out
|
| 628 |
+
|
| 629 |
+
|
| 630 |
+
def signature(rule):
|
| 631 |
+
"""The identity of the QUESTION a rule asks, ignoring which condition row asked it.
|
| 632 |
+
|
| 633 |
+
Two conditions with the same measure, window, operator and value have the same answer, so
|
| 634 |
+
they cost one query between them β and a saved view re-rendered unchanged costs none, since
|
| 635 |
+
the caller can memoise on this. `id` is deliberately absent: it names the asker, not the
|
| 636 |
+
question.
|
| 637 |
+
|
| 638 |
+
β The RHS is part of the question. Without it, `Sales[ytd] > Sales[last year]` and
|
| 639 |
+
`Sales[ytd] > Sales[ytd_last_year]` would share one memo entry β same measure, same op, same
|
| 640 |
+
(empty) value β and the second would be served the first one's answer. Silent, and wrong in
|
| 641 |
+
exactly the comparison this was built for.
|
| 642 |
+
"""
|
| 643 |
+
w = _win.normalize(rule.get('window'))
|
| 644 |
+
rhs = rule.get('rhs')
|
| 645 |
+
rsig = None
|
| 646 |
+
if isinstance(rhs, dict):
|
| 647 |
+
rw = _win.normalize(rhs.get('window'))
|
| 648 |
+
# `stat` rides the same tuple: `> the median` and `> the 75th percentile` are different
|
| 649 |
+
# questions with the same measure, window, op and (empty) value.
|
| 650 |
+
rsig = (rhs.get('kind'), rhs.get('colId'), rhs.get('stat'),
|
| 651 |
+
None if rw is None else tuple(sorted(rw.items())))
|
| 652 |
+
return (rule.get('colId'), rule.get('op'), str(rule.get('value') or ''),
|
| 653 |
+
None if w is None else tuple(sorted(w.items())), rsig)
|
| 654 |
+
|
| 655 |
+
|
| 656 |
+
def collect(nodes):
|
| 657 |
+
"""Every MEASURE leaf in a filter tree, flattened (groups contribute their children)."""
|
| 658 |
+
found = []
|
| 659 |
+
for n in nodes or []:
|
| 660 |
+
if not isinstance(n, dict):
|
| 661 |
+
continue
|
| 662 |
+
if isinstance(n.get('children'), list):
|
| 663 |
+
found.extend(collect(n['children']))
|
| 664 |
+
elif n.get('colId') in ADMITTED:
|
| 665 |
+
found.append(n)
|
| 666 |
+
return found
|
royalimports_os/harness/routines.py
ADDED
|
@@ -0,0 +1,259 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""harness/routines.py β OM-5 v0: scheduled reports + KPI alerts over the semantic layer.
|
| 2 |
+
|
| 3 |
+
Two of the four productization user jobs (Scheduled Reporting, KPI monitoring). Definitions live
|
| 4 |
+
in data/store/routines.json (tenant data, git-ignored); evaluation happens ON READ against the
|
| 5 |
+
tenant store through the same governed queries as everything else β an alert is a metric key +
|
| 6 |
+
a mode, never a hand-rolled query.
|
| 7 |
+
|
| 8 |
+
v0 boundaries (deliberate):
|
| 9 |
+
- Channel is 'in-app' only. Email/Slack delivery is OUTBOUND and ships only with owner approval
|
| 10 |
+
(the Odoo mail client is whitelisted for Collections statements ONLY β nothing else sends).
|
| 11 |
+
- No background scheduler: reports record cadence + show due-state in-app; alerts evaluate live
|
| 12 |
+
whenever the Routines page (or a tool) asks. The enterprise scheduler lands with OM-5 delivery.
|
| 13 |
+
|
| 14 |
+
Alert modes:
|
| 15 |
+
- 'xmr' Wheeler individuals chart on complete monthly values (promoted from
|
| 16 |
+
modules/warehouse._xmr, generalized: no 0/100 clamps β metrics aren't percentages).
|
| 17 |
+
Three detection rules only (more = alarm fatigue, per the warehouse brief).
|
| 18 |
+
- 'threshold' last complete month above/below a fixed value.
|
| 19 |
+
"""
|
| 20 |
+
import json
|
| 21 |
+
import time
|
| 22 |
+
from datetime import date, timedelta
|
| 23 |
+
|
| 24 |
+
import harness.semantic as SEM
|
| 25 |
+
from harness.tools import VIEWS_PATH
|
| 26 |
+
|
| 27 |
+
ROUTINES_PATH = VIEWS_PATH.parent / "routines.json"
|
| 28 |
+
|
| 29 |
+
CADENCES = ("daily", "weekly", "monthly")
|
| 30 |
+
CHANNELS = ("in-app",) # email/slack gate on owner approval (outbound send)
|
| 31 |
+
ALERT_MODES = ("xmr", "threshold")
|
| 32 |
+
XMR_MONTHS = 18 # complete months fed to the individuals chart
|
| 33 |
+
|
| 34 |
+
|
| 35 |
+
def load():
|
| 36 |
+
if ROUTINES_PATH.exists():
|
| 37 |
+
return json.loads(ROUTINES_PATH.read_text(encoding="utf-8"))
|
| 38 |
+
return {"reports": {}, "alerts": {}}
|
| 39 |
+
|
| 40 |
+
|
| 41 |
+
def _save(d):
|
| 42 |
+
ROUTINES_PATH.parent.mkdir(parents=True, exist_ok=True)
|
| 43 |
+
ROUTINES_PATH.write_text(json.dumps(d, indent=1), encoding="utf-8")
|
| 44 |
+
|
| 45 |
+
|
| 46 |
+
# ------------------------------------------------------------------ scheduled reports
|
| 47 |
+
|
| 48 |
+
def schedule_report(view, cadence, channel="in-app"):
|
| 49 |
+
import harness.views as V
|
| 50 |
+
if view not in V.views() and view not in V.dashboards():
|
| 51 |
+
raise SEM.ModelError(f"unknown view/dashboard {view!r} β save_view/compose_dashboard first")
|
| 52 |
+
if cadence not in CADENCES:
|
| 53 |
+
raise SEM.ModelError(f"cadence must be one of {CADENCES}")
|
| 54 |
+
if channel not in CHANNELS:
|
| 55 |
+
raise SEM.ModelError(f"channel {channel!r} not available yet β v0 delivers in-app "
|
| 56 |
+
"(email/Slack ship with the delivery gate); use channel='in-app'")
|
| 57 |
+
d = load()
|
| 58 |
+
d["reports"][view] = {"cadence": cadence, "channel": channel,
|
| 59 |
+
"created_at": time.strftime("%Y-%m-%d %H:%M")}
|
| 60 |
+
_save(d)
|
| 61 |
+
return {"scheduled": view, "cadence": cadence, "channel": channel,
|
| 62 |
+
"note": "runs in-app (Workflows > Routines); external delivery ships with OM-5 email approval"}
|
| 63 |
+
|
| 64 |
+
|
| 65 |
+
def delete_report(view):
|
| 66 |
+
d = load()
|
| 67 |
+
d["reports"].pop(view, None)
|
| 68 |
+
_save(d)
|
| 69 |
+
|
| 70 |
+
|
| 71 |
+
_CADENCE_DAYS = {"daily": 1, "weekly": 7, "monthly": 30}
|
| 72 |
+
|
| 73 |
+
|
| 74 |
+
def mark_viewed(view):
|
| 75 |
+
"""Called when the scheduled view/dashboard is actually opened β resets its due-ness."""
|
| 76 |
+
d = load()
|
| 77 |
+
r = d.get("reports", {}).get(view)
|
| 78 |
+
if r is not None:
|
| 79 |
+
r["last_viewed"] = time.strftime("%Y-%m-%d %H:%M")
|
| 80 |
+
_save(d)
|
| 81 |
+
|
| 82 |
+
|
| 83 |
+
def due_reports():
|
| 84 |
+
"""Scheduled reports whose cadence has elapsed since last viewed (v0 in-app delivery via
|
| 85 |
+
the Routines page). Never-viewed ages from created_at."""
|
| 86 |
+
from datetime import datetime
|
| 87 |
+
out = []
|
| 88 |
+
for name, r in load().get("reports", {}).items():
|
| 89 |
+
days = _CADENCE_DAYS.get(r.get("cadence"), 7)
|
| 90 |
+
anchor = r.get("last_viewed") or r.get("created_at") or ""
|
| 91 |
+
try:
|
| 92 |
+
age = (datetime.now() - datetime.strptime(anchor, "%Y-%m-%d %H:%M")).days
|
| 93 |
+
except Exception:
|
| 94 |
+
age = days
|
| 95 |
+
if age >= days:
|
| 96 |
+
out.append({"name": name, "cadence": r.get("cadence"),
|
| 97 |
+
"last_viewed": r.get("last_viewed"), "age_days": age})
|
| 98 |
+
return out
|
| 99 |
+
|
| 100 |
+
|
| 101 |
+
# ------------------------------------------------------------------ insight routines (OM-5)
|
| 102 |
+
# A saved PROMPT on a cadence: the Analyst runs it in the background and the narrative (plus its
|
| 103 |
+
# governed artifacts) lands in-app on the Routines page. The narrative is a deliberate
|
| 104 |
+
# point-in-time SNAPSHOT (it narrates a moment); its charts still carry re-runnable queries.
|
| 105 |
+
|
| 106 |
+
def schedule_insight(name, prompt, cadence):
|
| 107 |
+
if cadence not in CADENCES:
|
| 108 |
+
raise SEM.ModelError(f"cadence must be one of {CADENCES}")
|
| 109 |
+
if not (prompt or "").strip():
|
| 110 |
+
raise SEM.ModelError("an insight needs a prompt β what should the Analyst look at?")
|
| 111 |
+
d = load()
|
| 112 |
+
d.setdefault("insights", {})[name] = {"prompt": prompt.strip(), "cadence": cadence,
|
| 113 |
+
"created_at": time.strftime("%Y-%m-%d %H:%M")}
|
| 114 |
+
_save(d)
|
| 115 |
+
return {"insight": name, "cadence": cadence,
|
| 116 |
+
"note": "runs in the background when due; the narrative lands on "
|
| 117 |
+
"Workflows > Routines"}
|
| 118 |
+
|
| 119 |
+
|
| 120 |
+
def delete_insight(name):
|
| 121 |
+
d = load()
|
| 122 |
+
(d.get("insights") or {}).pop(name, None)
|
| 123 |
+
_save(d)
|
| 124 |
+
|
| 125 |
+
|
| 126 |
+
def due_insights():
|
| 127 |
+
from datetime import datetime
|
| 128 |
+
out = []
|
| 129 |
+
for name, i in (load().get("insights") or {}).items():
|
| 130 |
+
days = _CADENCE_DAYS.get(i.get("cadence"), 7)
|
| 131 |
+
anchor = i.get("last_run_at") or ""
|
| 132 |
+
try:
|
| 133 |
+
age = (datetime.now() - datetime.strptime(anchor, "%Y-%m-%d %H:%M")).days
|
| 134 |
+
except Exception:
|
| 135 |
+
age = days # never run -> due now
|
| 136 |
+
if age >= days:
|
| 137 |
+
out.append(name)
|
| 138 |
+
return out
|
| 139 |
+
|
| 140 |
+
|
| 141 |
+
def run_insight(name):
|
| 142 |
+
"""Execute one insight through the Analyst and persist the narrative + artifacts."""
|
| 143 |
+
d = load()
|
| 144 |
+
ins = (d.get("insights") or {}).get(name)
|
| 145 |
+
if not ins:
|
| 146 |
+
raise SEM.ModelError(f"unknown insight {name!r}")
|
| 147 |
+
import harness.analyst as A
|
| 148 |
+
r = A.ask(ins["prompt"])
|
| 149 |
+
now = time.strftime("%Y-%m-%d %H:%M")
|
| 150 |
+
ins["last_run_at"] = now
|
| 151 |
+
ins["last"] = {"at": now, "answer": r["answer"], "artifacts": r.get("artifacts") or [],
|
| 152 |
+
"tokens": (r.get("usage") or {}).get("prompt_tokens", 0)
|
| 153 |
+
+ (r.get("usage") or {}).get("completion_tokens", 0)}
|
| 154 |
+
_save(d)
|
| 155 |
+
return ins["last"]
|
| 156 |
+
|
| 157 |
+
|
| 158 |
+
# ------------------------------------------------------------------ KPI alerts
|
| 159 |
+
|
| 160 |
+
def create_alert(metric, mode="xmr", team_id=None, threshold=None, direction=None):
|
| 161 |
+
m = SEM.metrics().get(metric)
|
| 162 |
+
if not m:
|
| 163 |
+
raise SEM.ModelError(f"unknown metric {metric!r} (use list_topics)")
|
| 164 |
+
if mode not in ALERT_MODES:
|
| 165 |
+
raise SEM.ModelError(f"mode must be one of {ALERT_MODES}")
|
| 166 |
+
if mode == "threshold":
|
| 167 |
+
if threshold is None or direction not in ("above", "below"):
|
| 168 |
+
raise SEM.ModelError("threshold mode needs threshold=<number> and "
|
| 169 |
+
"direction='above'|'below'")
|
| 170 |
+
d = load()
|
| 171 |
+
key = metric + (f"_team{team_id}" if team_id else "")
|
| 172 |
+
d["alerts"][key] = {"metric": metric, "mode": mode, "team_id": team_id,
|
| 173 |
+
"threshold": threshold, "direction": direction,
|
| 174 |
+
"created_at": time.strftime("%Y-%m-%d %H:%M")}
|
| 175 |
+
_save(d)
|
| 176 |
+
return {"alert": key, **evaluate_alert(d["alerts"][key])}
|
| 177 |
+
|
| 178 |
+
|
| 179 |
+
def delete_alert(key):
|
| 180 |
+
d = load()
|
| 181 |
+
d["alerts"].pop(key, None)
|
| 182 |
+
_save(d)
|
| 183 |
+
|
| 184 |
+
|
| 185 |
+
def xmr(series):
|
| 186 |
+
"""Wheeler XmR individuals limits + the three detection rules β the model-level primitive
|
| 187 |
+
(generalized from modules/warehouse._xmr; no percentage clamps). series=[(label, value)]."""
|
| 188 |
+
vals = [v for _, v in series]
|
| 189 |
+
if len(vals) < 8:
|
| 190 |
+
return {"center": None, "unpl": None, "lnpl": None,
|
| 191 |
+
"points": [{"label": l, "value": v, "signal": ""} for l, v in series]}
|
| 192 |
+
center = sum(vals) / len(vals)
|
| 193 |
+
mrs = [abs(vals[i] - vals[i - 1]) for i in range(1, len(vals))]
|
| 194 |
+
mr_bar = sum(mrs) / len(mrs) if mrs else 0.0
|
| 195 |
+
unpl = center + 2.66 * mr_bar
|
| 196 |
+
lnpl = center - 2.66 * mr_bar
|
| 197 |
+
pts = []
|
| 198 |
+
for i, (l, v) in enumerate(series):
|
| 199 |
+
sig = ""
|
| 200 |
+
if v > unpl or v < lnpl:
|
| 201 |
+
sig = "outside limits"
|
| 202 |
+
if not sig and i >= 7 and all((x - center) > 0 for _, x in series[i - 7:i + 1]):
|
| 203 |
+
sig = "sustained shift (8 above center)"
|
| 204 |
+
if not sig and i >= 7 and all((x - center) < 0 for _, x in series[i - 7:i + 1]):
|
| 205 |
+
sig = "sustained shift (8 below center)"
|
| 206 |
+
if not sig and i >= 3:
|
| 207 |
+
last4 = [x for _, x in series[i - 3:i + 1]]
|
| 208 |
+
near = sum(1 for x in last4
|
| 209 |
+
if abs(x - center) > 0.5 * ((unpl - center) if x >= center
|
| 210 |
+
else (center - lnpl)))
|
| 211 |
+
if near >= 3:
|
| 212 |
+
sig = "moderate shift emerging"
|
| 213 |
+
pts.append({"label": l, "value": v, "signal": sig})
|
| 214 |
+
return {"center": center, "unpl": unpl, "lnpl": lnpl, "points": pts}
|
| 215 |
+
|
| 216 |
+
|
| 217 |
+
def _month_window(n=XMR_MONTHS):
|
| 218 |
+
"""Last n COMPLETE months (the in-progress month would false-signal every alert)."""
|
| 219 |
+
first_cur = date.today().replace(day=1)
|
| 220 |
+
y, mo = first_cur.year, first_cur.month
|
| 221 |
+
for _ in range(n):
|
| 222 |
+
mo -= 1
|
| 223 |
+
if mo == 0:
|
| 224 |
+
y, mo = y - 1, 12
|
| 225 |
+
start = date(y, mo, 1)
|
| 226 |
+
return start.isoformat(), (first_cur - timedelta(days=1)).isoformat()
|
| 227 |
+
|
| 228 |
+
|
| 229 |
+
def evaluate_alert(a):
|
| 230 |
+
"""Live evaluation: monthly series of the metric (complete months only) -> mode verdict."""
|
| 231 |
+
metric = a["metric"]
|
| 232 |
+
topic = SEM.metrics()[metric]["topic"]
|
| 233 |
+
date_from, date_to = _month_window()
|
| 234 |
+
res = SEM.store_query(topic, [metric], grain="month", date_from=date_from, date_to=date_to,
|
| 235 |
+
team_id=a.get("team_id"))
|
| 236 |
+
series = [(r["period"][:7], r.get(metric) or 0) for r in res["rows"]]
|
| 237 |
+
out = {"metric": metric, "mode": a["mode"], "team_id": a.get("team_id"),
|
| 238 |
+
"window": f"{date_from}..{date_to}", "series": series}
|
| 239 |
+
if not series:
|
| 240 |
+
return {**out, "status": "no data", "fired": False}
|
| 241 |
+
last_label, last_val = series[-1]
|
| 242 |
+
if a["mode"] == "threshold":
|
| 243 |
+
thr, direc = a.get("threshold"), a.get("direction")
|
| 244 |
+
fired = (last_val > thr) if direc == "above" else (last_val < thr)
|
| 245 |
+
return {**out, "last": {"label": last_label, "value": last_val},
|
| 246 |
+
"threshold": thr, "direction": direc,
|
| 247 |
+
"status": (f"FIRED β {last_label} is {direc} {thr:,.0f}" if fired else "OK"),
|
| 248 |
+
"fired": bool(fired)}
|
| 249 |
+
lim = xmr(series)
|
| 250 |
+
sig = lim["points"][-1]["signal"] if lim["points"] else ""
|
| 251 |
+
return {**out, "last": {"label": last_label, "value": last_val},
|
| 252 |
+
"center": lim["center"], "unpl": lim["unpl"], "lnpl": lim["lnpl"],
|
| 253 |
+
"points": lim["points"],
|
| 254 |
+
"status": (f"SIGNAL β {sig} ({last_label})" if sig else "OK"),
|
| 255 |
+
"fired": bool(sig)}
|
| 256 |
+
|
| 257 |
+
|
| 258 |
+
def evaluate_all():
|
| 259 |
+
return {k: evaluate_alert(a) for k, a in load().get("alerts", {}).items()}
|
royalimports_os/harness/runtime.py
ADDED
|
@@ -0,0 +1,186 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""harness/runtime.py β X7: per-REQUEST tenant resolution for the shared stateless API (EXIT-4a).
|
| 2 |
+
|
| 3 |
+
THE COST ARGUMENT, in one rule: *no per-tenant state may be resident in the app process.* Today
|
| 4 |
+
each tenant carries its own interpreter, its own never-evicted pool, its own 9 prewarmed bundles
|
| 5 |
+
and its own DuckDB β EXIT-0 measured a **~0.6 GB commit floor per tenant**, duplicated exactly
|
| 6 |
+
(tenant B came within 2% of tenant A; nothing is shared). The target is ~30β80 MB. That is only
|
| 7 |
+
reachable if the API resolves the tenant from the REQUEST and holds nothing tenant-shaped between
|
| 8 |
+
requests except a bounded cache it is willing to throw away.
|
| 9 |
+
|
| 10 |
+
WHAT THIS REPLACES. `ui/cache.py`'s `st.cache_resource` singleton β one process, one tenant,
|
| 11 |
+
caches that are never evicted by design. That is correct for Streamlit and fatal for a shared
|
| 12 |
+
process. The Streamlit side keeps its singleton untouched this wave (two processes, two caches,
|
| 13 |
+
accepted for the strangler period; `ui/cache.py` dies with app.py at EXIT-6).
|
| 14 |
+
|
| 15 |
+
THREE RULES THIS FILE OBEYS, each of which is a way the old shape leaked:
|
| 16 |
+
1. **No module-level per-tenant globals.** Every tenant-shaped value hangs off a TenantRuntime
|
| 17 |
+
reached through `get_runtime(slug)`; nothing is stashed at import time. A module global is
|
| 18 |
+
exactly what makes "which tenant is this process serving?" unanswerable.
|
| 19 |
+
2. **LRU-BOUNDED.** `_MAX_RUNTIMES` runtimes live at once; the least recently used is evicted.
|
| 20 |
+
An unbounded registry cache is a memory leak with a tenant-count-shaped growth curve β the
|
| 21 |
+
thing this whole program exists to remove.
|
| 22 |
+
3. **The store handle is tenant-bound**, so a write cannot land in another tenant's namespace
|
| 23 |
+
even if a caller passes the wrong key downstream (EXIT-4b proof #3).
|
| 24 |
+
|
| 25 |
+
β THE DUCKDB CAVEAT, STATED NOT HIDDEN. `harness.datastore` holds ONE process-wide connection
|
| 26 |
+
(the fix for the 2026-07-16 "connection error unless I refresh" bug), so one PROCESS can serve
|
| 27 |
+
one analytical store at a time. This module therefore hands out `datastore_path` β the file a
|
| 28 |
+
tenant's analytical reads belong to β and does NOT switch the global underneath a live request.
|
| 29 |
+
Until the singleton is per-tenant (EXIT-5), an API process answering measure/Analyst queries must
|
| 30 |
+
be pinned to one tenant's file, and `assert_datastore_matches()` below is what turns a
|
| 31 |
+
mis-pinned deployment into a loud failure instead of a silent cross-tenant read.
|
| 32 |
+
"""
|
| 33 |
+
from __future__ import annotations
|
| 34 |
+
|
| 35 |
+
import os
|
| 36 |
+
import threading
|
| 37 |
+
from collections import OrderedDict
|
| 38 |
+
from dataclasses import dataclass, field
|
| 39 |
+
|
| 40 |
+
import core.store as store
|
| 41 |
+
from harness import datastore as _ds
|
| 42 |
+
from harness import tenants as _tenants
|
| 43 |
+
|
| 44 |
+
#: How many tenant runtimes stay resident. Small on purpose: a runtime is cheap to rebuild (a
|
| 45 |
+
#: Tenant literal + a couple of handles) and expensive to leak. Raise only with a measurement.
|
| 46 |
+
_MAX_RUNTIMES = int(os.environ.get("AIOS_MAX_RUNTIMES") or 32)
|
| 47 |
+
|
| 48 |
+
#: slug -> builder. The literal registry `harness/tenants.py` describes; per its own docstring
|
| 49 |
+
#: this becomes "a tenants store (DB / HF dataset)" β X4's control DB is where it goes, and
|
| 50 |
+
#: `register()` below is the seam that makes the swap a one-liner rather than a refactor.
|
| 51 |
+
_BUILDERS = {"royal-imports": _tenants.royal_imports}
|
| 52 |
+
|
| 53 |
+
# X8's isolation fixture, admitted ONLY on request. A stub tenant that is always present is a
|
| 54 |
+
# slug an attacker can mint a session against in production; gated, it exists exactly where it is
|
| 55 |
+
# needed (the TestClient isolation proofs) and nowhere else. Read at import so the flag cannot be
|
| 56 |
+
# flipped by a request.
|
| 57 |
+
if os.environ.get("AIOS_ENABLE_QA_TENANT") == "1":
|
| 58 |
+
_BUILDERS["qa-b"] = _tenants.qa_tenant_b
|
| 59 |
+
|
| 60 |
+
_CACHE: "OrderedDict[str, TenantRuntime]" = OrderedDict()
|
| 61 |
+
_LOCK = threading.RLock()
|
| 62 |
+
|
| 63 |
+
|
| 64 |
+
@dataclass
|
| 65 |
+
class TenantRuntime:
|
| 66 |
+
"""Everything a request needs to serve ONE tenant, and nothing about any other.
|
| 67 |
+
|
| 68 |
+
`store_namespace` is the prefix every product-data key is written under. Tenant #0 keeps the
|
| 69 |
+
EMPTY prefix so its existing keys (`users`, `customer_lists`, `customer_docs`, β¦) are found
|
| 70 |
+
exactly where they already live β a namespacing change that renamed tenant #0's keys would be
|
| 71 |
+
a silent data loss dressed as a refactor. Every OTHER tenant is prefixed, which is what makes
|
| 72 |
+
proof #3 hold.
|
| 73 |
+
"""
|
| 74 |
+
key: str
|
| 75 |
+
tenant: object # harness.base.Tenant
|
| 76 |
+
store_namespace: str = ""
|
| 77 |
+
pool_cache: dict = field(default_factory=dict)
|
| 78 |
+
_lock: threading.RLock = field(default_factory=threading.RLock, repr=False)
|
| 79 |
+
|
| 80 |
+
@property
|
| 81 |
+
def name(self):
|
| 82 |
+
return getattr(self.tenant, "name", self.key)
|
| 83 |
+
|
| 84 |
+
@property
|
| 85 |
+
def datastore_path(self):
|
| 86 |
+
"""The DuckDB file this tenant's analytical reads belong to (file-per-tenant IS the
|
| 87 |
+
isolation model β DuckDB has no RLS, so the OS boundary is the only one that holds)."""
|
| 88 |
+
return _ds.path_for(self.key)
|
| 89 |
+
|
| 90 |
+
# ---------------------------------------------------------------- the store handle
|
| 91 |
+
def store_key(self, name):
|
| 92 |
+
"""Namespace a product-data key for this tenant. The ONE place the prefix is applied."""
|
| 93 |
+
return f"{self.store_namespace}{name}" if self.store_namespace else str(name)
|
| 94 |
+
|
| 95 |
+
def get(self, name, fresh=False):
|
| 96 |
+
return store.get(self.store_key(name), fresh=fresh)
|
| 97 |
+
|
| 98 |
+
def put(self, name, data):
|
| 99 |
+
return store.put(self.store_key(name), data)
|
| 100 |
+
|
| 101 |
+
def update(self, name, fn, flush="sync"):
|
| 102 |
+
return store.update(self.store_key(name), fn, flush=flush)
|
| 103 |
+
|
| 104 |
+
def exists(self, name):
|
| 105 |
+
return store.exists(self.store_key(name))
|
| 106 |
+
|
| 107 |
+
def available(self):
|
| 108 |
+
return store.available()
|
| 109 |
+
|
| 110 |
+
def assert_datastore_matches(self):
|
| 111 |
+
"""RAISE if the process-wide DuckDB file is not this tenant's.
|
| 112 |
+
|
| 113 |
+
The alternative is the failure mode that matters: an analytical read served from another
|
| 114 |
+
tenant's file returns real, plausible, wrong rows β no exception, no telemetry, and the
|
| 115 |
+
number reconciles against the wrong book. A loud failure is strictly better, and this is
|
| 116 |
+
the assertion EXIT-5 deletes when the connection becomes per-tenant.
|
| 117 |
+
"""
|
| 118 |
+
want, have = self.datastore_path.resolve(), _ds.DB_PATH.resolve()
|
| 119 |
+
if want != have:
|
| 120 |
+
raise RuntimeError(
|
| 121 |
+
f"tenant {self.key!r} expects the analytical store at {want}, but this process "
|
| 122 |
+
f"has {have} open. Refusing the read β a cross-tenant answer would reconcile "
|
| 123 |
+
f"against the wrong book. (Pin the process with AIOS_DUCKDB_PATH, or call "
|
| 124 |
+
f"harness.datastore.use_path before serving this tenant.)")
|
| 125 |
+
|
| 126 |
+
|
| 127 |
+
def register(key, builder):
|
| 128 |
+
"""Add a tenant to the registry (a stub connector is a legitimate builder β see X8's qa-b).
|
| 129 |
+
Invalidates any cached runtime for the slug so a re-registration cannot be shadowed."""
|
| 130 |
+
_BUILDERS[str(key)] = builder
|
| 131 |
+
invalidate(key)
|
| 132 |
+
|
| 133 |
+
|
| 134 |
+
def known_tenants():
|
| 135 |
+
return sorted(_BUILDERS)
|
| 136 |
+
|
| 137 |
+
|
| 138 |
+
def get_runtime(tenant_key):
|
| 139 |
+
"""The runtime for `tenant_key`, built on first use and LRU-cached.
|
| 140 |
+
|
| 141 |
+
Raises KeyError for an unknown slug β the caller turns that into a 401 (fail-closed: an
|
| 142 |
+
unknown tenant is not a 404, because confirming which slugs exist answers a question the
|
| 143 |
+
request was not entitled to ask).
|
| 144 |
+
"""
|
| 145 |
+
key = str(tenant_key or "").strip().lower()
|
| 146 |
+
if not key:
|
| 147 |
+
raise KeyError("no tenant")
|
| 148 |
+
with _LOCK:
|
| 149 |
+
rt = _CACHE.get(key)
|
| 150 |
+
if rt is not None:
|
| 151 |
+
_CACHE.move_to_end(key) # LRU: touch on read
|
| 152 |
+
return rt
|
| 153 |
+
builder = _BUILDERS.get(key)
|
| 154 |
+
if builder is None:
|
| 155 |
+
raise KeyError(key)
|
| 156 |
+
# Built OUTSIDE the lock: a builder may do real work (a connector handshake), and holding the
|
| 157 |
+
# registry lock through it would serialise every other tenant's requests behind it.
|
| 158 |
+
tenant = builder()
|
| 159 |
+
with _LOCK:
|
| 160 |
+
existing = _CACHE.get(key)
|
| 161 |
+
if existing is not None: # another thread won the race β keep ONE runtime
|
| 162 |
+
_CACHE.move_to_end(key)
|
| 163 |
+
return existing
|
| 164 |
+
rt = TenantRuntime(key=key, tenant=tenant,
|
| 165 |
+
store_namespace="" if key == "royal-imports" else f"t/{key}/")
|
| 166 |
+
_CACHE[key] = rt
|
| 167 |
+
while len(_CACHE) > _MAX_RUNTIMES:
|
| 168 |
+
_CACHE.popitem(last=False) # evict least recently used
|
| 169 |
+
return rt
|
| 170 |
+
|
| 171 |
+
|
| 172 |
+
def invalidate(tenant_key=None):
|
| 173 |
+
"""Drop one tenant's runtime, or all of them. Explicit, per X7 β a cache with no documented
|
| 174 |
+
way to clear it becomes a restart."""
|
| 175 |
+
with _LOCK:
|
| 176 |
+
if tenant_key is None:
|
| 177 |
+
_CACHE.clear()
|
| 178 |
+
else:
|
| 179 |
+
_CACHE.pop(str(tenant_key or "").strip().lower(), None)
|
| 180 |
+
|
| 181 |
+
|
| 182 |
+
def resident():
|
| 183 |
+
"""The tenant slugs currently holding a runtime β LRU order, oldest first. For the EXIT-4
|
| 184 |
+
measurement and for asserting the bound actually binds."""
|
| 185 |
+
with _LOCK:
|
| 186 |
+
return list(_CACHE)
|
royalimports_os/harness/semantic.py
ADDED
|
@@ -0,0 +1,936 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""harness/semantic.py β the semantic seam (OM-0a, 2026-07-11).
|
| 2 |
+
|
| 3 |
+
The ONE governed registry of topics + metrics (royalimports_os/model/*.yml) and the resolver that
|
| 4 |
+
turns a metric key into a number through the SAME proven data layer the validated modules use
|
| 5 |
+
(core/odoo domain builders + aggregate helpers) β so scope parity is by construction, and every
|
| 6 |
+
metric carries its definition, ai_context, and an independent validate contract.
|
| 7 |
+
|
| 8 |
+
Consumers (the point of the seam): the app's pages, the metric dictionary (OM-0c), the saved-
|
| 9 |
+
dashboard viewer (OM-3; the Explore page was retired 2026-07-23), the AI Analyst's tools (OM-4:
|
| 10 |
+
list_topics / describe_topic / run_semantic_query), and the MCP server (OM-6). Plan:
|
| 11 |
+
.claude/wiki/research/omni-adoption.md (Part IV + the productization addendum).
|
| 12 |
+
|
| 13 |
+
Design constraints honored (from /odoo-api):
|
| 14 |
+
- read_group dot-path GROUPBY fails β scalar aggregates here use sum_field/distinct_count;
|
| 15 |
+
grouped queries (OM-3+) must group by direct m2o fields only.
|
| 16 |
+
- Scope is never hand-rolled: topics BIND to a named domain builder (sale_line_domain, ...);
|
| 17 |
+
the YAML `scope:` block is the declarative statement of what that builder bakes in.
|
| 18 |
+
- READ-ONLY on Odoo, always.
|
| 19 |
+
"""
|
| 20 |
+
import ast
|
| 21 |
+
import functools
|
| 22 |
+
import operator
|
| 23 |
+
from pathlib import Path
|
| 24 |
+
|
| 25 |
+
import yaml
|
| 26 |
+
|
| 27 |
+
import core.odoo as O
|
| 28 |
+
|
| 29 |
+
MODEL_DIR = Path(__file__).resolve().parents[1] / "model"
|
| 30 |
+
|
| 31 |
+
# Topic scope binds to a NAMED, reviewed domain builder β never a YAML-assembled domain (one
|
| 32 |
+
# source of truth for scope until OM-2 makes topics executable against the store).
|
| 33 |
+
def _order_domain(*a, **kw):
|
| 34 |
+
# lazy: order scope currently lives in modules/sales.py; OM-2 lifts it into core/the store
|
| 35 |
+
import modules.sales as S
|
| 36 |
+
return S.order_domain(*a, **kw)
|
| 37 |
+
|
| 38 |
+
|
| 39 |
+
def _customer_move_domain(move_type):
|
| 40 |
+
"""A posted customer document domain, matching `modules/returns.py`'s `_mdom` exactly.
|
| 41 |
+
|
| 42 |
+
β COMPANY-LEVEL, and it REFUSES a team rather than ignoring one. Credit notes all carry
|
| 43 |
+
team_id = 1, so a BU filter on the document is meaningless β silently dropping the argument
|
| 44 |
+
would hand a business-unit user the company's number under their own name, which is the
|
| 45 |
+
widening sin in a different currency.
|
| 46 |
+
"""
|
| 47 |
+
def build(date_from=None, date_to=None, team_id=None, **kw):
|
| 48 |
+
if team_id:
|
| 49 |
+
raise ModelError(
|
| 50 |
+
f"customer {move_type} documents are COMPANY-LEVEL β credit notes and invoices "
|
| 51 |
+
f"carry no trustworthy business-unit tag, so a team-scoped total cannot be "
|
| 52 |
+
f"produced. Refusing rather than returning the company number.")
|
| 53 |
+
dom = [("move_type", "=", move_type), ("state", "=", "posted")]
|
| 54 |
+
if date_from:
|
| 55 |
+
dom.append(("invoice_date", ">=", date_from))
|
| 56 |
+
if date_to:
|
| 57 |
+
dom.append(("invoice_date", "<=", date_to))
|
| 58 |
+
return dom
|
| 59 |
+
return build
|
| 60 |
+
|
| 61 |
+
|
| 62 |
+
_BUILDERS = {
|
| 63 |
+
"sale_line_domain": O.sale_line_domain,
|
| 64 |
+
"sale_order_domain": _order_domain,
|
| 65 |
+
"credit_note_domain": _customer_move_domain("out_refund"),
|
| 66 |
+
"customer_invoice_domain": _customer_move_domain("out_invoice"),
|
| 67 |
+
}
|
| 68 |
+
|
| 69 |
+
|
| 70 |
+
class ModelError(Exception):
|
| 71 |
+
"""A model-file problem (unknown key, bad reference, unparseable expr)."""
|
| 72 |
+
|
| 73 |
+
|
| 74 |
+
# ---------------------------------------------------------------- loading
|
| 75 |
+
|
| 76 |
+
@functools.lru_cache(maxsize=1)
|
| 77 |
+
def _tenant():
|
| 78 |
+
f = MODEL_DIR / "tenant.yml"
|
| 79 |
+
return yaml.safe_load(f.read_text(encoding="utf-8")) if f.exists() else {"params": {}}
|
| 80 |
+
|
| 81 |
+
|
| 82 |
+
def _sql_value(v):
|
| 83 |
+
"""Render a tenant param into SQL safely (params are versioned config, not user input β
|
| 84 |
+
but escape anyway): ints as-is, lists comma-joined, strings single-quoted + escaped."""
|
| 85 |
+
if isinstance(v, bool):
|
| 86 |
+
raise ModelError("boolean tenant params not supported in scope_sql")
|
| 87 |
+
if isinstance(v, (int, float)):
|
| 88 |
+
return str(v)
|
| 89 |
+
if isinstance(v, (list, tuple)):
|
| 90 |
+
return ", ".join(_sql_value(x) for x in v)
|
| 91 |
+
return "'" + str(v).replace("'", "''") + "'"
|
| 92 |
+
|
| 93 |
+
|
| 94 |
+
def render_scope(sql):
|
| 95 |
+
"""Substitute {param} placeholders in a store scope/filter SQL fragment from tenant.yml."""
|
| 96 |
+
params = _tenant().get("params") or {}
|
| 97 |
+
out = sql
|
| 98 |
+
for k, v in params.items():
|
| 99 |
+
out = out.replace("{" + k + "}", _sql_value(v))
|
| 100 |
+
if "{" in out and "}" in out:
|
| 101 |
+
import re as _re
|
| 102 |
+
missing = _re.findall(r"\{([a-z_]+)\}", out)
|
| 103 |
+
if missing:
|
| 104 |
+
raise ModelError(f"scope_sql references unknown tenant params: {missing}")
|
| 105 |
+
return out
|
| 106 |
+
|
| 107 |
+
|
| 108 |
+
@functools.lru_cache(maxsize=1)
|
| 109 |
+
def _model():
|
| 110 |
+
topics, metrics = {}, {}
|
| 111 |
+
for f in sorted((MODEL_DIR / "topics").glob("*.yml")):
|
| 112 |
+
d = yaml.safe_load(f.read_text(encoding="utf-8"))
|
| 113 |
+
if not d or "key" not in d:
|
| 114 |
+
raise ModelError(f"topic file {f.name} lacks a 'key'")
|
| 115 |
+
if d.get("domain_builder") and d["domain_builder"] not in _BUILDERS:
|
| 116 |
+
raise ModelError(f"topic {d['key']}: unknown domain_builder {d.get('domain_builder')!r}")
|
| 117 |
+
# a topic may be STORE-ONLY (no live-path builder yet, e.g. gl_lines) β metric() raises
|
| 118 |
+
# a clear error if a live resolution is attempted on it
|
| 119 |
+
topics[d["key"]] = d
|
| 120 |
+
for f in sorted((MODEL_DIR / "metrics").glob("*.yml")):
|
| 121 |
+
d = yaml.safe_load(f.read_text(encoding="utf-8"))
|
| 122 |
+
topic = d.get("topic")
|
| 123 |
+
if topic not in topics:
|
| 124 |
+
raise ModelError(f"metrics file {f.name}: unknown topic {topic!r}")
|
| 125 |
+
for m in d.get("metrics", []):
|
| 126 |
+
if "key" not in m:
|
| 127 |
+
raise ModelError(f"metrics file {f.name}: a metric lacks a 'key'")
|
| 128 |
+
if m["key"] in metrics:
|
| 129 |
+
raise ModelError(f"duplicate metric key {m['key']!r}")
|
| 130 |
+
m.setdefault("topic", topic) # a metric may name its own topic (rare)
|
| 131 |
+
if m["topic"] not in topics:
|
| 132 |
+
raise ModelError(f"metric {m['key']!r}: unknown topic {m['topic']!r}")
|
| 133 |
+
metrics[m["key"]] = m
|
| 134 |
+
return {"topics": topics, "metrics": metrics}
|
| 135 |
+
|
| 136 |
+
|
| 137 |
+
def reload_model():
|
| 138 |
+
_model.cache_clear()
|
| 139 |
+
_tenant.cache_clear()
|
| 140 |
+
return _model()
|
| 141 |
+
|
| 142 |
+
|
| 143 |
+
def topics():
|
| 144 |
+
return dict(_model()["topics"])
|
| 145 |
+
|
| 146 |
+
|
| 147 |
+
def metrics():
|
| 148 |
+
return dict(_model()["metrics"])
|
| 149 |
+
|
| 150 |
+
|
| 151 |
+
def describe(key):
|
| 152 |
+
"""Full metadata for a metric (the dictionary page / AI describe_topic feed)."""
|
| 153 |
+
m = _model()["metrics"].get(key)
|
| 154 |
+
if not m:
|
| 155 |
+
raise ModelError(f"unknown metric {key!r}")
|
| 156 |
+
return {**m, "topic_def": _model()["topics"][m["topic"]]}
|
| 157 |
+
|
| 158 |
+
|
| 159 |
+
# ---------------------------------------------------------------- resolving
|
| 160 |
+
|
| 161 |
+
def _domain(topic_def, date_from, date_to, team_id):
|
| 162 |
+
b = topic_def.get("domain_builder")
|
| 163 |
+
if not b:
|
| 164 |
+
raise ModelError(f"topic {topic_def['key']!r} is store-only β use store_query() "
|
| 165 |
+
f"(no live-path domain builder bound yet)")
|
| 166 |
+
return _BUILDERS[b](date_from, date_to, team_id=team_id)
|
| 167 |
+
|
| 168 |
+
|
| 169 |
+
_OPS = {ast.Add: operator.add, ast.Sub: operator.sub,
|
| 170 |
+
ast.Mult: operator.mul, ast.Div: operator.truediv}
|
| 171 |
+
|
| 172 |
+
|
| 173 |
+
def _eval_expr(expr, resolve):
|
| 174 |
+
"""Safely evaluate a derived-metric expression: metric keys, numbers, + - * / and parens."""
|
| 175 |
+
def ev(node):
|
| 176 |
+
if isinstance(node, ast.Expression):
|
| 177 |
+
return ev(node.body)
|
| 178 |
+
if isinstance(node, ast.BinOp) and type(node.op) in _OPS:
|
| 179 |
+
return _OPS[type(node.op)](ev(node.left), ev(node.right))
|
| 180 |
+
if isinstance(node, ast.UnaryOp) and isinstance(node.op, ast.USub):
|
| 181 |
+
return -ev(node.operand)
|
| 182 |
+
if isinstance(node, ast.Num): # py<3.8 compat name; Constant below
|
| 183 |
+
return node.n
|
| 184 |
+
if isinstance(node, ast.Constant) and isinstance(node.value, (int, float)):
|
| 185 |
+
return node.value
|
| 186 |
+
if isinstance(node, ast.Name):
|
| 187 |
+
return resolve(node.id)
|
| 188 |
+
raise ModelError(f"disallowed token in derived expr: {ast.dump(node)}")
|
| 189 |
+
try:
|
| 190 |
+
return ev(ast.parse(expr, mode="eval"))
|
| 191 |
+
except SyntaxError as e:
|
| 192 |
+
raise ModelError(f"unparseable derived expr {expr!r}: {e}")
|
| 193 |
+
|
| 194 |
+
|
| 195 |
+
def metric(key, date_from=None, date_to=None, team_id=None, _seen=None):
|
| 196 |
+
"""Resolve a registered metric to a number for a window (+optional BU). The only public
|
| 197 |
+
compute path β every surface that shows this metric goes through here."""
|
| 198 |
+
_seen = _seen or set()
|
| 199 |
+
if key in _seen:
|
| 200 |
+
raise ModelError(f"circular metric reference at {key!r}")
|
| 201 |
+
_seen = _seen | {key}
|
| 202 |
+
m = _model()["metrics"].get(key)
|
| 203 |
+
if not m:
|
| 204 |
+
raise ModelError(f"unknown metric {key!r}")
|
| 205 |
+
agg = m.get("agg")
|
| 206 |
+
if agg == "ratio":
|
| 207 |
+
num = metric(m["numerator"], date_from, date_to, team_id, _seen)
|
| 208 |
+
den = metric(m["denominator"], date_from, date_to, team_id, _seen)
|
| 209 |
+
return (num / den) if den else 0.0
|
| 210 |
+
if agg == "derived":
|
| 211 |
+
return _eval_expr(m["expr"], lambda k: metric(k, date_from, date_to, team_id, _seen))
|
| 212 |
+
t = _model()["topics"][m["topic"]]
|
| 213 |
+
dom = _domain(t, date_from, date_to, team_id)
|
| 214 |
+
if agg == "sum":
|
| 215 |
+
# β `negate` applies HERE too, not only in `_measure_sql`. It existed for GL income on a
|
| 216 |
+
# STORE-ONLY topic, so the live path never met it β the first metric that is both negated
|
| 217 |
+
# and parity-checked (`returns`) would otherwise compare a negative live number against a
|
| 218 |
+
# positive store one, and the "parity failure" would look like a data problem.
|
| 219 |
+
got = O.sum_field(t["entity"], dom, m["field"])
|
| 220 |
+
return -got if m.get("negate") else got
|
| 221 |
+
if agg == "count_distinct":
|
| 222 |
+
return O.distinct_count(t["entity"], dom, m["field"])
|
| 223 |
+
if agg == "count":
|
| 224 |
+
return O.get_odoo().search_count(t["entity"], dom)
|
| 225 |
+
raise ModelError(f"metric {key!r}: unknown agg {agg!r}")
|
| 226 |
+
|
| 227 |
+
|
| 228 |
+
# ---------------------------------------------------------------- validate contracts
|
| 229 |
+
|
| 230 |
+
def _v_order_level_revenue(date_from, date_to, team_id):
|
| 231 |
+
"""Independent check: line-level Ξ£ price_subtotal == order-level Ξ£ amount_untaxed."""
|
| 232 |
+
import modules.sales as S
|
| 233 |
+
line = metric("revenue", date_from, date_to, team_id)
|
| 234 |
+
order = O.sum_field("sale.order", S.order_domain(date_from, date_to, team_id), "amount_untaxed")
|
| 235 |
+
return line, order
|
| 236 |
+
|
| 237 |
+
|
| 238 |
+
def _v_order_count(date_from, date_to, team_id):
|
| 239 |
+
"""Exact reconciliation: order-level count == distinct line-parent orders + line-LESS orders.
|
| 240 |
+
(Confirmed orders with zero lines exist in the data β 2 found at first proof, 2026-07-11;
|
| 241 |
+
counting them explicitly makes the identity exact instead of hiding them in a tolerance,
|
| 242 |
+
and surfaces them as a data-health signal.)"""
|
| 243 |
+
import modules.sales as S
|
| 244 |
+
order_level = metric("orders", date_from, date_to, team_id)
|
| 245 |
+
dom = S.order_domain(date_from, date_to, team_id)
|
| 246 |
+
from_lines = O.distinct_count("sale.order.line",
|
| 247 |
+
O.sale_line_domain(date_from, date_to, team_id), "order_id")
|
| 248 |
+
empty = O.get_odoo().search_count("sale.order", dom + [("order_line", "=", False)])
|
| 249 |
+
return order_level, from_lines + empty
|
| 250 |
+
|
| 251 |
+
|
| 252 |
+
def _v_gl_opex(date_from, date_to, team_id):
|
| 253 |
+
"""Store-path opex == the live Odoo aggregate with the Expenses-module domain (posted,
|
| 254 |
+
expense-type accounts), to the cent. team_id N/A (company-level)."""
|
| 255 |
+
res = store_query("gl_lines", ["opex"], date_from=date_from, date_to=date_to)
|
| 256 |
+
ours = res["rows"][0]["opex"] if res["rows"] else 0
|
| 257 |
+
dom = [("move_id.state", "=", "posted"),
|
| 258 |
+
("account_id.account_type", "in", ["expense", "expense_depreciation"])]
|
| 259 |
+
if date_from:
|
| 260 |
+
dom.append(("date", ">=", date_from))
|
| 261 |
+
if date_to:
|
| 262 |
+
dom.append(("date", "<=", date_to))
|
| 263 |
+
live = O.sum_field("account.move.line", dom, "balance")
|
| 264 |
+
return ours, live
|
| 265 |
+
|
| 266 |
+
|
| 267 |
+
def _v_ar_outstanding(date_from, date_to, team_id):
|
| 268 |
+
"""Store-path AR outstanding == live Odoo aggregate (posted out_invoice/out_refund residuals).
|
| 269 |
+
No date window β outstanding is an as-of-now balance."""
|
| 270 |
+
res = store_query("receivables", ["ar_outstanding"])
|
| 271 |
+
ours = res["rows"][0]["ar_outstanding"] if res["rows"] else 0
|
| 272 |
+
live = O.sum_field("account.move",
|
| 273 |
+
[("state", "=", "posted"), ("move_type", "in", ["out_invoice", "out_refund"])],
|
| 274 |
+
"amount_residual_signed")
|
| 275 |
+
return ours, live
|
| 276 |
+
|
| 277 |
+
|
| 278 |
+
def _v_returns_sign(date_from, date_to, team_id):
|
| 279 |
+
"""Independent check on the NEGATION: `returns` must equal the ABSOLUTE untaxed total of the
|
| 280 |
+
same posted credit notes, read straight from Odoo without going through the metric.
|
| 281 |
+
|
| 282 |
+
A lost negation makes returns negative; a doubled one makes it negative again. Both read as
|
| 283 |
+
"no returns" rather than as an error, and neither is visible to any self-consistency check β
|
| 284 |
+
the store and the live path would agree with each other perfectly while both being wrong.
|
| 285 |
+
"""
|
| 286 |
+
ours = metric("returns", date_from, date_to, team_id)
|
| 287 |
+
raw = O.sum_field("account.move",
|
| 288 |
+
_BUILDERS["credit_note_domain"](date_from, date_to, team_id),
|
| 289 |
+
"amount_untaxed_signed")
|
| 290 |
+
return ours, abs(raw)
|
| 291 |
+
|
| 292 |
+
|
| 293 |
+
def _v_line_vs_move_grain(date_from, date_to, team_id):
|
| 294 |
+
"""Independent check on the invoice-LINE topic: the store's netted product-line total must
|
| 295 |
+
equal the same figure read straight from LIVE Odoo.
|
| 296 |
+
|
| 297 |
+
Two things this catches that self-consistency cannot. (1) A LOST CREDIT-NOTE NEGATION β
|
| 298 |
+
`price_subtotal` is stored POSITIVE on a refund line, so a dropped subtraction inflates
|
| 299 |
+
revenue while every internal check still agrees. (2) A WRONG `display_type` FILTER β the
|
| 300 |
+
invoice tables carry 'cogs' and 'payment_term' rows alongside 'product' ones, and including
|
| 301 |
+
them changes the total silently rather than erroring.
|
| 302 |
+
|
| 303 |
+
Company-level (team_id None) uses the strongest possible reference: the MOVE-grain
|
| 304 |
+
`amount_untaxed_signed`, a different table entirely. Per-BU there is no move-grain analogue
|
| 305 |
+
(account.move carries no business unit β every invoice sits on team 1), so the reference is
|
| 306 |
+
the live line-level aggregate reached through the sale-order dot path: still an independent
|
| 307 |
+
engine and query path from the DuckDB store, which is what the contract requires.
|
| 308 |
+
"""
|
| 309 |
+
ours = (store_query("invoice_lines", ["invoiced_line_sales"],
|
| 310 |
+
date_from=date_from, date_to=date_to, team_id=team_id)
|
| 311 |
+
.get("rows") or [{}])[0].get("invoiced_line_sales") or 0.0
|
| 312 |
+
if team_id is None:
|
| 313 |
+
raw = O.sum_field("account.move",
|
| 314 |
+
[("state", "=", "posted"),
|
| 315 |
+
("move_type", "in", ["out_invoice", "out_refund"]),
|
| 316 |
+
("invoice_date", ">=", date_from), ("invoice_date", "<=", date_to)],
|
| 317 |
+
"amount_untaxed_signed")
|
| 318 |
+
return ours, raw
|
| 319 |
+
dom = [("parent_state", "=", "posted"), ("display_type", "=", "product"),
|
| 320 |
+
("date", ">=", date_from), ("date", "<=", date_to),
|
| 321 |
+
("sale_line_ids.order_id.team_id", "=", team_id)]
|
| 322 |
+
gross = O.sum_field("account.move.line", dom + [("move_type", "=", "out_invoice")],
|
| 323 |
+
"price_subtotal")
|
| 324 |
+
refunds = O.sum_field("account.move.line", dom + [("move_type", "=", "out_refund")],
|
| 325 |
+
"price_subtotal")
|
| 326 |
+
return ours, (gross or 0) - (refunds or 0)
|
| 327 |
+
|
| 328 |
+
|
| 329 |
+
_VALIDATORS = {
|
| 330 |
+
"returns_sign": _v_returns_sign,
|
| 331 |
+
"line_vs_move_grain": _v_line_vs_move_grain,
|
| 332 |
+
"order_level_revenue": _v_order_level_revenue,
|
| 333 |
+
"order_count": _v_order_count,
|
| 334 |
+
"gl_opex": _v_gl_opex,
|
| 335 |
+
"ar_outstanding": _v_ar_outstanding,
|
| 336 |
+
}
|
| 337 |
+
|
| 338 |
+
|
| 339 |
+
def validate_metric(key, date_from=None, date_to=None, team_id=None, tol=0.01):
|
| 340 |
+
"""Run a metric's declared independent cross-check. Returns the standard check dict
|
| 341 |
+
(the validate.py convention: check/ok/gap)."""
|
| 342 |
+
m = _model()["metrics"].get(key)
|
| 343 |
+
if not m:
|
| 344 |
+
raise ModelError(f"unknown metric {key!r}")
|
| 345 |
+
contract = m.get("validate")
|
| 346 |
+
if not contract:
|
| 347 |
+
return {"check": f"semantic.{key}: no independent contract declared", "ok": True, "gap": 0.0,
|
| 348 |
+
"note": "declare one in model/metrics when an independent aggregate exists"}
|
| 349 |
+
fn = _VALIDATORS.get(contract["method"])
|
| 350 |
+
if not fn:
|
| 351 |
+
raise ModelError(f"metric {key!r}: unknown validate method {contract['method']!r}")
|
| 352 |
+
ours, independent = fn(date_from, date_to, team_id)
|
| 353 |
+
gap = abs((ours or 0) - (independent or 0))
|
| 354 |
+
return {"check": f"semantic.{key} == {contract['method']} ({date_from}..{date_to}, team={team_id})",
|
| 355 |
+
"ok": gap <= tol, "gap": round(gap, 4), "ours": ours, "independent": independent}
|
| 356 |
+
|
| 357 |
+
|
| 358 |
+
def validate(pre=None):
|
| 359 |
+
"""Module-convention validate(): run every declared contract over YTD, all-BU + per-BU."""
|
| 360 |
+
import core.periods as P
|
| 361 |
+
yf, yt = P.ytd()
|
| 362 |
+
out = []
|
| 363 |
+
for key, m in _model()["metrics"].items():
|
| 364 |
+
if not m.get("validate"):
|
| 365 |
+
continue
|
| 366 |
+
out.append(validate_metric(key, yf, yt, None))
|
| 367 |
+
topic = _model()["topics"][m["topic"]]
|
| 368 |
+
if (topic.get("store") or {}).get("team_col"): # company-level topics have no BU runs
|
| 369 |
+
for tid in O.TEAM_IDS:
|
| 370 |
+
out.append(validate_metric(key, yf, yt, tid))
|
| 371 |
+
return out
|
| 372 |
+
|
| 373 |
+
|
| 374 |
+
# ---------------------------------------------------------------- store path (OM-2)
|
| 375 |
+
# The compiler behind run_semantic_query: registered metrics + whitelisted dims β DuckDB SQL over
|
| 376 |
+
# the tenant store (harness/datastore.py). EVERY identifier comes from the versioned model files
|
| 377 |
+
# (trusted); every VALUE is parameterized β the small model only ever picks keys, so there is no
|
| 378 |
+
# injection surface. Live XML-RPC stays the validate() path (the store never validates itself).
|
| 379 |
+
|
| 380 |
+
def _store_con():
|
| 381 |
+
import harness.datastore as DS
|
| 382 |
+
if not DS.ready(): # a mid-backfill store returns PARTIAL totals silently
|
| 383 |
+
raise ModelError("the data cache is still warming up after a restart (a one-time first "
|
| 384 |
+
"sync) β the dashboards work now from live data; ask me again in a "
|
| 385 |
+
"moment and I'll have it")
|
| 386 |
+
return DS.ro_cursor() # cursor on the shared instance β no sync-writer contention
|
| 387 |
+
|
| 388 |
+
|
| 389 |
+
def _measure_sql(m, alias):
|
| 390 |
+
"""SQL for a base measure. Supports FILTERED measures (`store_filter_sql` β the Omni
|
| 391 |
+
filtered-measure concept: sum(CASE WHEN β¦ )) and `negate: true` (e.g. GL income, stored
|
| 392 |
+
credit-negative, reported positive)."""
|
| 393 |
+
agg, f = m.get("agg"), m.get("field")
|
| 394 |
+
flt = render_scope(m["store_filter_sql"]) if m.get("store_filter_sql") else None
|
| 395 |
+
if agg == "sum":
|
| 396 |
+
core = f"sum(CASE WHEN {flt} THEN {alias}.{f} ELSE 0 END)" if flt else f"sum({alias}.{f})"
|
| 397 |
+
elif agg == "count_distinct":
|
| 398 |
+
core = (f"count(DISTINCT CASE WHEN {flt} THEN {alias}.{f} END)" if flt
|
| 399 |
+
else f"count(DISTINCT {alias}.{f})")
|
| 400 |
+
elif agg == "count":
|
| 401 |
+
core = f"count(CASE WHEN {flt} THEN 1 END)" if flt else "count(*)"
|
| 402 |
+
else:
|
| 403 |
+
raise ModelError(f"metric {m['key']!r}: agg {agg!r} has no store expression")
|
| 404 |
+
return f"-({core})" if m.get("negate") else core
|
| 405 |
+
|
| 406 |
+
|
| 407 |
+
def _expand_measures(keys):
|
| 408 |
+
"""Expand ratio/derived metrics into the base measures SQL must aggregate, keeping the
|
| 409 |
+
requested keys for post-compute. Returns (base_keys, requested_keys)."""
|
| 410 |
+
mts = _model()["metrics"]
|
| 411 |
+
base, seen = [], set()
|
| 412 |
+
|
| 413 |
+
def need(k):
|
| 414 |
+
if k in seen:
|
| 415 |
+
return
|
| 416 |
+
seen.add(k)
|
| 417 |
+
m = mts.get(k)
|
| 418 |
+
if not m:
|
| 419 |
+
raise ModelError(f"unknown metric {k!r}")
|
| 420 |
+
if m["agg"] == "ratio":
|
| 421 |
+
need(m["numerator"]); need(m["denominator"])
|
| 422 |
+
elif m["agg"] == "derived":
|
| 423 |
+
import ast as _ast
|
| 424 |
+
for node in _ast.walk(_ast.parse(m["expr"], mode="eval")):
|
| 425 |
+
if isinstance(node, _ast.Name):
|
| 426 |
+
need(node.id)
|
| 427 |
+
else:
|
| 428 |
+
if k not in base:
|
| 429 |
+
base.append(k)
|
| 430 |
+
for k in keys:
|
| 431 |
+
need(k)
|
| 432 |
+
return base, list(keys)
|
| 433 |
+
|
| 434 |
+
|
| 435 |
+
def _post_compute(row, key):
|
| 436 |
+
m = _model()["metrics"][key]
|
| 437 |
+
if m["agg"] == "ratio":
|
| 438 |
+
num, den = _post_compute(row, m["numerator"]), _post_compute(row, m["denominator"])
|
| 439 |
+
return (num / den) if den else 0.0
|
| 440 |
+
if m["agg"] == "derived":
|
| 441 |
+
return _eval_expr(m["expr"], lambda k: _post_compute(row, k))
|
| 442 |
+
return row.get(key) or 0.0
|
| 443 |
+
|
| 444 |
+
|
| 445 |
+
# A metric's declared `format` -> the grid field type the compiler and the cells both read.
|
| 446 |
+
# One mapping, so a new metric cannot arrive as an untyped column.
|
| 447 |
+
_FORMAT_TYPE = {"usd": "currency", "int": "int", "pct": "pct"}
|
| 448 |
+
|
| 449 |
+
|
| 450 |
+
def _clean_tree(tree, cols):
|
| 451 |
+
"""Run a filter tree through the SAME validator the client's view state goes through.
|
| 452 |
+
|
| 453 |
+
`filter_sql`'s documented input contract is "feed me CLEANED trees" β it is faithful to the
|
| 454 |
+
TS engine rather than fail-closed on garbage, because rejecting garbage is the validator's
|
| 455 |
+
job. The UI path always cleans (view state is sanitised on the way into the store), but
|
| 456 |
+
store_query/store_rows are callable directly, and an uncleaned tree is where the two
|
| 457 |
+
diverge: `{"value": null}` is ACTIVE in TS (and throws on a text field) and INACTIVE here.
|
| 458 |
+
|
| 459 |
+
Normalising at the entry point makes that unreachable rather than merely unlikely, and
|
| 460 |
+
brings the depth / width / node-count caps to the server side too. `aios_grid` is a leaf
|
| 461 |
+
module with zero imports of its own, so this cannot cycle.
|
| 462 |
+
"""
|
| 463 |
+
from aios_grid import (clean_filter_tree, COHORT_FIELD as _COHORT_FIELD,
|
| 464 |
+
MAX_FILTER_DEPTH, MAX_FILTER_NODES, MAX_FILTER_SIBLINGS)
|
| 465 |
+
|
| 466 |
+
# REFUSE an over-large tree rather than TRUNCATE it. clean_filter_tree caps siblings per
|
| 467 |
+
# level, total nodes and depth by DROPPING the excess β correct on the client, where the
|
| 468 |
+
# cap is a DoS guard on a per-row render loop and a dropped condition is the lesser evil.
|
| 469 |
+
# On the server it is not: dropping AND-conditions WIDENS the result, and store_rows then
|
| 470 |
+
# reports a row_count and scope-wide totals that look authoritative for a query the caller
|
| 471 |
+
# never asked for. That is precisely the silent [:N] that [[no-unverifiable-aggregates]]
|
| 472 |
+
# forbids. A caller that exceeds the caps gets an error, not a quietly different answer.
|
| 473 |
+
total = 0
|
| 474 |
+
|
| 475 |
+
def _walk(nodes, depth):
|
| 476 |
+
nonlocal total
|
| 477 |
+
nodes = list(nodes or [])
|
| 478 |
+
if len(nodes) > MAX_FILTER_SIBLINGS:
|
| 479 |
+
raise ModelError(f"filter tree has {len(nodes)} conditions at one level; the limit "
|
| 480 |
+
f"is {MAX_FILTER_SIBLINGS}. Refusing rather than dropping the "
|
| 481 |
+
f"excess, which would silently widen the result.")
|
| 482 |
+
for node in nodes:
|
| 483 |
+
if not isinstance(node, dict):
|
| 484 |
+
continue
|
| 485 |
+
total += 1
|
| 486 |
+
if isinstance(node.get('children'), list):
|
| 487 |
+
if depth >= MAX_FILTER_DEPTH:
|
| 488 |
+
raise ModelError(f"filter tree nests deeper than {MAX_FILTER_DEPTH} levels; "
|
| 489 |
+
f"refusing rather than dropping the deepest group.")
|
| 490 |
+
_walk(node['children'], depth + 1)
|
| 491 |
+
|
| 492 |
+
_walk(tree, 1)
|
| 493 |
+
if total > MAX_FILTER_NODES:
|
| 494 |
+
raise ModelError(f"filter tree has {total} nodes; the limit is {MAX_FILTER_NODES}. "
|
| 495 |
+
f"Refusing rather than truncating, which would silently widen it.")
|
| 496 |
+
|
| 497 |
+
# CG-8. A MEASURE condition ("Sales in the last 90 days > 5,000") is answered by
|
| 498 |
+
# harness/measure_filter.py, which resolves it to a SET OF CUSTOMER IDS. It has no meaning
|
| 499 |
+
# on this path, and both ways it could arrive here are silent:
|
| 500 |
+
# - on a topic without that column, `clean_filter_tree` DROPS it as unknown β widening the
|
| 501 |
+
# query while store_rows reports an authoritative row_count for something nobody asked
|
| 502 |
+
# for (the same class as the sibling/depth caps above);
|
| 503 |
+
# - on sales_lines at row grain, `revenue` IS a real column, so it would compile as a
|
| 504 |
+
# per-LINE predicate with the window silently discarded β a different question answered
|
| 505 |
+
# confidently, which is worse than dropping it.
|
| 506 |
+
# The discriminator is the `window`, not the column name: no column condition has one.
|
| 507 |
+
windowed = []
|
| 508 |
+
cohorts = []
|
| 509 |
+
|
| 510 |
+
def _find(nodes):
|
| 511 |
+
for node in nodes or []:
|
| 512 |
+
if not isinstance(node, dict):
|
| 513 |
+
continue
|
| 514 |
+
if isinstance(node.get('children'), list):
|
| 515 |
+
_find(node['children'])
|
| 516 |
+
elif node.get('window') is not None:
|
| 517 |
+
windowed.append(str(node.get('colId')))
|
| 518 |
+
elif node.get('colId') == _COHORT_FIELD:
|
| 519 |
+
cohorts.append(str(node.get('value')))
|
| 520 |
+
|
| 521 |
+
_find(tree)
|
| 522 |
+
# Owner item 5, and the same refusal for the same reason. A cohort leaf names a
|
| 523 |
+
# hand-curated set of CUSTOMERS held per user in the tenant store β this path knows nothing
|
| 524 |
+
# about it, and `clean_filter_tree` below would drop it as an unknown key, widening the
|
| 525 |
+
# query while store_rows still reported an authoritative row_count. If a caller ever needs
|
| 526 |
+
# it here, the fix is to pass the membership in, not to let it fall through.
|
| 527 |
+
if cohorts:
|
| 528 |
+
raise ModelError(
|
| 529 |
+
f"filter tree carries cohort-membership condition(s) {sorted(set(cohorts))} β "
|
| 530 |
+
f"cohort membership is host state, not a column of this topic. Refusing rather than "
|
| 531 |
+
f"dropping it, which would silently widen the result under an authoritative count.")
|
| 532 |
+
if windowed:
|
| 533 |
+
raise ModelError(
|
| 534 |
+
f"filter tree carries measure condition(s) over a date window {sorted(set(windowed))}"
|
| 535 |
+
f" β those are resolved to a set of ids by harness.measure_filter, not compiled into "
|
| 536 |
+
f"this query. Refusing rather than dropping or mis-compiling them, either of which "
|
| 537 |
+
f"would silently answer a different question under an authoritative count.")
|
| 538 |
+
return clean_filter_tree(tree, set(cols))
|
| 539 |
+
|
| 540 |
+
|
| 541 |
+
def _measure_row_sql(m, alias):
|
| 542 |
+
"""A sum-metric's per-ROW value: the same expression `_measure_sql` aggregates, without the
|
| 543 |
+
aggregate wrapper. At line grain `revenue` IS `l.price_subtotal` for that line β deriving it
|
| 544 |
+
from the metric rather than hand-writing the column is what keeps ONE definition of the
|
| 545 |
+
number (the drift the semantic layer exists to prevent). Only `sum` has a row value; a
|
| 546 |
+
count/count_distinct metric is a property of a SET, not of a row."""
|
| 547 |
+
if m.get("agg") != "sum":
|
| 548 |
+
return None
|
| 549 |
+
f = m.get("field")
|
| 550 |
+
flt = render_scope(m["store_filter_sql"]) if m.get("store_filter_sql") else None
|
| 551 |
+
core = f"CASE WHEN {flt} THEN {alias}.{f} ELSE 0 END" if flt else f"{alias}.{f}"
|
| 552 |
+
return f"-({core})" if m.get("negate") else core
|
| 553 |
+
|
| 554 |
+
|
| 555 |
+
def store_columns(topic, include_measures=True, grain="aggregate"):
|
| 556 |
+
"""The `{colId: {sql, type, aggregate}}` spec `harness.filter_sql` compiles against (CG-1).
|
| 557 |
+
|
| 558 |
+
`grain="row"` is the LINE-grain projection (CG-2): sum-metrics resolve to their own raw
|
| 559 |
+
field with `aggregate=False`, so a line table filters and sorts on real row values in WHERE
|
| 560 |
+
rather than needing HAVING. count-style metrics are dropped β they have no row value.
|
| 561 |
+
|
| 562 |
+
This is the seam between the grid's field contract and the semantic model: the grid speaks
|
| 563 |
+
field KEYS, the compiler needs SQL expressions and a field TYPE, and only the model may say
|
| 564 |
+
what a key resolves to. Nothing about a topic leaks into filter_sql itself.
|
| 565 |
+
|
| 566 |
+
<dim> -> the dim's NAME column (what the grid displays and what a user means when
|
| 567 |
+
they type "contains floral"), filtered PRE-aggregation in WHERE. Correct
|
| 568 |
+
even in a grouped query: every row of a group shares its group's name.
|
| 569 |
+
<dim>_id -> the raw id, for exact machine-keyed filtering.
|
| 570 |
+
date -> the topic's date column, so a grid can filter/sort it like any other field.
|
| 571 |
+
<metric> -> the aggregate expression, marked `aggregate` so the caller routes it to
|
| 572 |
+
HAVING rather than WHERE (see store_query β that path is not built yet).
|
| 573 |
+
"""
|
| 574 |
+
t = _model()["topics"].get(topic)
|
| 575 |
+
if not t or "store" not in t:
|
| 576 |
+
raise ModelError(f"topic {topic!r} has no store binding")
|
| 577 |
+
s = t["store"]
|
| 578 |
+
cols = {}
|
| 579 |
+
for key, d in (s.get("dims") or {}).items():
|
| 580 |
+
cols[key] = {"sql": d.get("name_col") or d["col"], "type": "text", "aggregate": False}
|
| 581 |
+
# Only expose a separate id column when the dim actually HAS one. `city` is its own
|
| 582 |
+
# name (col == name_col), so a `city_id` would be a text column typed int β filtering
|
| 583 |
+
# it numerically would quietly compare 0 against 0 for every row.
|
| 584 |
+
if d.get("name_col") and d["name_col"] != d["col"]:
|
| 585 |
+
cols[f"{key}_id"] = {"sql": d["col"], "type": "int", "aggregate": False}
|
| 586 |
+
if s.get("date_col"):
|
| 587 |
+
cols["date"] = {"sql": s["date_col"], "type": "date", "aggregate": False}
|
| 588 |
+
if include_measures:
|
| 589 |
+
for k, m in _model()["metrics"].items():
|
| 590 |
+
if m.get("topic") != topic or m.get("agg") in ("ratio", "derived"):
|
| 591 |
+
continue # ratio/derived are post-computed, not SQL
|
| 592 |
+
# The field TYPE comes from the metric's declared format, never a guess: `units` is
|
| 593 |
+
# format:int, and typing it currency would render 791,960 units as dollars.
|
| 594 |
+
ftype = _FORMAT_TYPE.get(m.get("format"), "currency")
|
| 595 |
+
if grain == "row":
|
| 596 |
+
row_sql = _measure_row_sql(m, s["alias"])
|
| 597 |
+
if row_sql:
|
| 598 |
+
cols[k] = {"sql": row_sql, "type": ftype, "aggregate": False}
|
| 599 |
+
else:
|
| 600 |
+
cols[k] = {"sql": _measure_sql(m, s["alias"]),
|
| 601 |
+
"type": ftype, "aggregate": True}
|
| 602 |
+
return cols
|
| 603 |
+
|
| 604 |
+
|
| 605 |
+
def store_query(topic, measures, group_by=None, grain=None, date_from=None, date_to=None,
|
| 606 |
+
team_id=None, filters=None, sort=None, limit=1000, exclude_services=False,
|
| 607 |
+
filter_tree=None, filter_conj="and", today=None):
|
| 608 |
+
"""The semantic query over the tenant store β the engine behind the Analyst's
|
| 609 |
+
run_semantic_query tool and the saved-view re-runner. All keys whitelisted against the model."""
|
| 610 |
+
t = _model()["topics"].get(topic)
|
| 611 |
+
if not t or "store" not in t:
|
| 612 |
+
raise ModelError(f"topic {topic!r} has no store binding")
|
| 613 |
+
s = t["store"]
|
| 614 |
+
alias = s["alias"]
|
| 615 |
+
dims = s.get("dims") or {}
|
| 616 |
+
limit = max(1, min(int(limit or 1000), 5000))
|
| 617 |
+
|
| 618 |
+
base_keys, requested = _expand_measures(list(measures or []))
|
| 619 |
+
if not base_keys:
|
| 620 |
+
raise ModelError("at least one measure required")
|
| 621 |
+
# Cross-topic base metrics (e.g. aov = revenue Γ· orders, where orders lives on sales_orders):
|
| 622 |
+
# in a SCALAR query they resolve via a nested scalar query on their HOME topic (correct grain β
|
| 623 |
+
# never count(*) on the wrong table); in a GROUPED query they are refused (split the request).
|
| 624 |
+
foreign = [k for k in base_keys if _model()["metrics"][k]["topic"] != topic]
|
| 625 |
+
if foreign and (group_by or grain):
|
| 626 |
+
raise ModelError(f"metrics {foreign} belong to another topic β cross-topic measures are "
|
| 627 |
+
f"scalar-only; run them against their own topic when grouping")
|
| 628 |
+
base_keys = [k for k in base_keys if k not in foreign]
|
| 629 |
+
foreign_vals = {}
|
| 630 |
+
for k in foreign:
|
| 631 |
+
sub = store_query(_model()["metrics"][k]["topic"], [k], date_from=date_from,
|
| 632 |
+
date_to=date_to, team_id=team_id)
|
| 633 |
+
foreign_vals[k] = sub["rows"][0][k] if sub["rows"] else 0
|
| 634 |
+
if not base_keys: # purely foreign scalar request
|
| 635 |
+
return {"topic": topic, "rows": [foreign_vals], "row_count": 1, "sql": "(nested)",
|
| 636 |
+
"measures": requested, "group_by": [], "grain": None}
|
| 637 |
+
|
| 638 |
+
select, group_cols, params = [], [], []
|
| 639 |
+
gb = [g for g in (group_by or []) if g]
|
| 640 |
+
for g in gb:
|
| 641 |
+
if g not in dims:
|
| 642 |
+
raise ModelError(f"group_by {g!r} not a dim of {topic!r} (allowed: {list(dims)})")
|
| 643 |
+
d = dims[g]
|
| 644 |
+
select.append(f"{d['col']} AS {g}_id" if d.get("name_col") else f"{d['col']} AS {g}")
|
| 645 |
+
group_cols.append(d["col"])
|
| 646 |
+
if d.get("name_col"):
|
| 647 |
+
select.append(f"max({d['name_col']}) AS {g}")
|
| 648 |
+
if grain:
|
| 649 |
+
if grain not in ("month", "week", "day"):
|
| 650 |
+
raise ModelError("grain must be month|week|day")
|
| 651 |
+
select.insert(0, f"date_trunc('{grain}', CAST({s['date_col']} AS TIMESTAMP)) AS period")
|
| 652 |
+
group_cols.insert(0, f"date_trunc('{grain}', CAST({s['date_col']} AS TIMESTAMP))")
|
| 653 |
+
for k in base_keys:
|
| 654 |
+
select.append(f"{_measure_sql(_model()['metrics'][k], alias)} AS {k}")
|
| 655 |
+
|
| 656 |
+
where = [render_scope(s["scope_sql"].strip())]
|
| 657 |
+
# TYPE-CORRECT date bounds (the 2025-07-01 lesson): columns hold ISO strings in TWO shapes β
|
| 658 |
+
# datetimes ('YYYY-MM-DD HH:MM:SS', sale date_order) and bare dates ('YYYY-MM-DD', GL date).
|
| 659 |
+
# Lexical string comparison EXCLUDES the window's first day for bare dates ('2025-07-01' <
|
| 660 |
+
# '2025-07-01 00:00:00'), which silently dropped a whole day of GL ($49,467). Cast both sides.
|
| 661 |
+
if date_from:
|
| 662 |
+
where.append(f"CAST({s['date_col']} AS TIMESTAMP) >= ?")
|
| 663 |
+
params.append(f"{date_from} 00:00:00")
|
| 664 |
+
if date_to:
|
| 665 |
+
where.append(f"CAST({s['date_col']} AS TIMESTAMP) <= ?")
|
| 666 |
+
params.append(f"{date_to} 23:59:59")
|
| 667 |
+
if team_id:
|
| 668 |
+
if not s.get("team_col"):
|
| 669 |
+
raise ModelError(f"topic {topic!r} is company-level β it has no business-unit filter")
|
| 670 |
+
where.append(f"{s['team_col']} = ?"); params.append(int(team_id))
|
| 671 |
+
if exclude_services and s.get("service_filter"):
|
| 672 |
+
where.append(render_scope(s["service_filter"]))
|
| 673 |
+
for dim_key, vals in (filters or {}).items():
|
| 674 |
+
if dim_key not in dims:
|
| 675 |
+
raise ModelError(f"filter dim {dim_key!r} not allowed (use: {list(dims)})")
|
| 676 |
+
# id dims filter by int; TEXT dims (e.g. city) filter by the value itself β either way
|
| 677 |
+
# every VALUE stays parameterized (the whitelist covers identifiers only)
|
| 678 |
+
def _coerce(v):
|
| 679 |
+
try:
|
| 680 |
+
return int(v)
|
| 681 |
+
except (TypeError, ValueError):
|
| 682 |
+
return str(v)
|
| 683 |
+
ids = [_coerce(v) for v in (vals if isinstance(vals, (list, tuple)) else [vals])]
|
| 684 |
+
where.append(f"{dims[dim_key]['col']} IN ({','.join('?' for _ in ids)})")
|
| 685 |
+
params.extend(ids)
|
| 686 |
+
|
| 687 |
+
# The grid's filter TREE (CG-1). `filters` above stays the Analyst's flat dim=IN shape;
|
| 688 |
+
# this is the nested, 11-operator contract the table UI emits. Compiled by
|
| 689 |
+
# harness/filter_sql, which is held in lock-step with the TS engine and the validator by
|
| 690 |
+
# aios-web/verify_filter_engine.py. Appended AFTER the dim filters so params stay in
|
| 691 |
+
# positional order with `where`.
|
| 692 |
+
if filter_tree:
|
| 693 |
+
from harness import filter_sql as _fs
|
| 694 |
+
cols = store_columns(topic)
|
| 695 |
+
pred = _fs.compile_filter_tree(_clean_tree(filter_tree, cols), filter_conj, cols,
|
| 696 |
+
today=today)
|
| 697 |
+
if pred is not None:
|
| 698 |
+
if pred.uses_aggregate:
|
| 699 |
+
bad = sorted(c for c in pred.columns_used if cols[c].get("aggregate"))
|
| 700 |
+
raise ModelError(
|
| 701 |
+
f"filter_tree references measure(s) {bad} β a measure filter has to land in "
|
| 702 |
+
f"HAVING, and that path is deliberately NOT built: CG-1 ships the WHERE path "
|
| 703 |
+
f"because its consumer (CG-2, the line-grain sales table) does not aggregate. "
|
| 704 |
+
f"Filter on dims, or aggregate first and filter the result.")
|
| 705 |
+
where.append(pred.sql)
|
| 706 |
+
params.extend(pred.params)
|
| 707 |
+
|
| 708 |
+
sql = (f"SELECT {', '.join(select)} FROM {s['table']} {alias} {s.get('join','')} "
|
| 709 |
+
f"WHERE {' AND '.join(where)}")
|
| 710 |
+
if group_cols:
|
| 711 |
+
sql += f" GROUP BY {', '.join(group_cols)}"
|
| 712 |
+
if sort:
|
| 713 |
+
key = sort.lstrip("-")
|
| 714 |
+
if key not in base_keys + gb + (["period"] if grain else []):
|
| 715 |
+
raise ModelError(f"sort {sort!r} must reference a selected measure/dim")
|
| 716 |
+
sql += f" ORDER BY {key} {'DESC' if sort.startswith('-') else 'ASC'}"
|
| 717 |
+
elif grain:
|
| 718 |
+
sql += " ORDER BY period"
|
| 719 |
+
sql += f" LIMIT {limit}"
|
| 720 |
+
|
| 721 |
+
con = _store_con()
|
| 722 |
+
try:
|
| 723 |
+
cur = con.execute(sql, params)
|
| 724 |
+
cols = [d[0] for d in cur.description]
|
| 725 |
+
rows = [dict(zip(cols, r)) for r in cur.fetchall()]
|
| 726 |
+
finally:
|
| 727 |
+
con.close()
|
| 728 |
+
for row in rows: # ratio/derived post-compute per group
|
| 729 |
+
row.update(foreign_vals) # scalar cross-topic components
|
| 730 |
+
for k in requested:
|
| 731 |
+
if _model()["metrics"][k]["agg"] in ("ratio", "derived"):
|
| 732 |
+
row[k] = _post_compute(row, k)
|
| 733 |
+
if "period" in row and row["period"] is not None:
|
| 734 |
+
row["period"] = str(row["period"])[:10]
|
| 735 |
+
return {"topic": topic, "rows": rows, "row_count": len(rows), "sql": sql,
|
| 736 |
+
"measures": requested, "group_by": gb, "grain": grain}
|
| 737 |
+
|
| 738 |
+
|
| 739 |
+
def store_rows(topic, date_from=None, date_to=None, team_id=None, filter_tree=None,
|
| 740 |
+
filter_conj="and", search=None, sorts=None, limit=200, offset=0,
|
| 741 |
+
aggregates=None, exclude_services=False, id_sql=None, member_ids=None,
|
| 742 |
+
today=None):
|
| 743 |
+
"""ROW-grain fetch β the line-grain counterpart to store_query's aggregate path (CG-2).
|
| 744 |
+
|
| 745 |
+
store_query answers "what is the number"; this answers "which rows". At line grain the
|
| 746 |
+
payload is the binding constraint (254,837 sale_order_lines against 1,550 customers β the
|
| 747 |
+
whole book would be ~96 MB at the customer table's measured 377 B/row), so rows come back
|
| 748 |
+
WINDOWED.
|
| 749 |
+
|
| 750 |
+
A window is only honest if the counts and totals are not windowed with it
|
| 751 |
+
([[no-unverifiable-aggregates]]). So this returns THREE independent numbers, each from its
|
| 752 |
+
own query over the FULL scope, never from `len(rows)`:
|
| 753 |
+
|
| 754 |
+
row_count rows matching the filter across the whole scope -> the N of "N of M"
|
| 755 |
+
total_count rows in the scope with no filter at all -> the M
|
| 756 |
+
aggregates the topic's OWN metric definitions, summed over the whole FILTERED scope
|
| 757 |
+
|
| 758 |
+
That is the difference between a windowed fetch and a silent `[:N]` cap: the user is told
|
| 759 |
+
how many rows exist and shown totals for all of them, while receiving only the page they
|
| 760 |
+
can see.
|
| 761 |
+
|
| 762 |
+
Filters/sorts/search compile through harness.filter_sql, so a line table means EXACTLY what
|
| 763 |
+
the client engine means at customer grain β that equivalence is what verify_filter_engine.py
|
| 764 |
+
exists to hold.
|
| 765 |
+
|
| 766 |
+
Dates are returned already truncated to the 10-char ISO shape the filter compares against,
|
| 767 |
+
so what is displayed and what is filtered are the same string. Numerics come back RAW; the
|
| 768 |
+
payload layer rounds them (aios_grid._round), and the compiler mirrors that rounding.
|
| 769 |
+
"""
|
| 770 |
+
t = _model()["topics"].get(topic)
|
| 771 |
+
if not t or "store" not in t:
|
| 772 |
+
raise ModelError(f"topic {topic!r} has no store binding")
|
| 773 |
+
from harness import filter_sql as _fs
|
| 774 |
+
s = t["store"]
|
| 775 |
+
alias = s["alias"]
|
| 776 |
+
cols = store_columns(topic, grain="row")
|
| 777 |
+
id_sql = id_sql or f"{alias}.id"
|
| 778 |
+
limit = max(1, min(int(limit or 200), 5000))
|
| 779 |
+
offset = max(0, int(offset or 0))
|
| 780 |
+
|
| 781 |
+
# --- the scope every one of the three queries shares -------------------------------
|
| 782 |
+
scope, scope_params = [render_scope(s["scope_sql"].strip())], []
|
| 783 |
+
if date_from:
|
| 784 |
+
scope.append(f"CAST({s['date_col']} AS TIMESTAMP) >= ?")
|
| 785 |
+
scope_params.append(f"{date_from} 00:00:00")
|
| 786 |
+
if date_to:
|
| 787 |
+
scope.append(f"CAST({s['date_col']} AS TIMESTAMP) <= ?")
|
| 788 |
+
scope_params.append(f"{date_to} 23:59:59")
|
| 789 |
+
if team_id:
|
| 790 |
+
if not s.get("team_col"):
|
| 791 |
+
raise ModelError(f"topic {topic!r} is company-level β it has no business-unit filter")
|
| 792 |
+
scope.append(f"{s['team_col']} = ?")
|
| 793 |
+
scope_params.append(int(team_id))
|
| 794 |
+
if exclude_services and s.get("service_filter"):
|
| 795 |
+
scope.append(render_scope(s["service_filter"]))
|
| 796 |
+
|
| 797 |
+
# --- the user's narrowing, on top of the scope --------------------------------------
|
| 798 |
+
narrow, narrow_params = [], []
|
| 799 |
+
pred = _fs.compile_filter_tree(_clean_tree(filter_tree or [], cols), filter_conj, cols,
|
| 800 |
+
today=today,
|
| 801 |
+
member_ids=member_ids or None, id_sql=id_sql)
|
| 802 |
+
if pred is not None:
|
| 803 |
+
if pred.uses_aggregate: # cannot happen at grain="row"; guard anyway
|
| 804 |
+
raise ModelError(f"row-grain filter referenced an aggregate: {sorted(pred.columns_used)}")
|
| 805 |
+
narrow.append(pred.sql)
|
| 806 |
+
narrow_params.extend(pred.params)
|
| 807 |
+
spred = _fs.compile_search(search, cols)
|
| 808 |
+
if spred is not None:
|
| 809 |
+
narrow.append(spred.sql)
|
| 810 |
+
narrow_params.extend(spred.params)
|
| 811 |
+
|
| 812 |
+
frm = f"{s['table']} {alias} {s.get('join', '')}"
|
| 813 |
+
scope_where = " AND ".join(scope)
|
| 814 |
+
all_where = " AND ".join(scope + narrow)
|
| 815 |
+
con = _store_con()
|
| 816 |
+
try:
|
| 817 |
+
# 1. the WINDOW of rows
|
| 818 |
+
select = [f"{id_sql} AS _rid"] + [
|
| 819 |
+
(f"SUBSTR(CAST({c['sql']} AS VARCHAR), 1, 10) AS {k}" if c["type"] == "date"
|
| 820 |
+
else f"{c['sql']} AS {k}")
|
| 821 |
+
for k, c in cols.items()]
|
| 822 |
+
order = _fs.compile_order_by(sorts, cols, tiebreak_sql=id_sql) or f"{id_sql} ASC"
|
| 823 |
+
sql = (f"SELECT {', '.join(select)} FROM {frm} WHERE {all_where} "
|
| 824 |
+
f"ORDER BY {order} LIMIT {limit} OFFSET {offset}")
|
| 825 |
+
cur = con.execute(sql, scope_params + narrow_params)
|
| 826 |
+
names = [d[0] for d in cur.description]
|
| 827 |
+
rows = [dict(zip(names, r)) for r in cur.fetchall()]
|
| 828 |
+
|
| 829 |
+
# 2. the two counts β over the WHOLE scope, never len(rows)
|
| 830 |
+
count_sql = f"SELECT count(*) FROM {frm} WHERE {all_where}"
|
| 831 |
+
row_count = con.execute(count_sql, scope_params + narrow_params).fetchone()[0]
|
| 832 |
+
total_count = con.execute(f"SELECT count(*) FROM {frm} WHERE {scope_where}",
|
| 833 |
+
scope_params).fetchone()[0]
|
| 834 |
+
|
| 835 |
+
# 3. the aggregates β the topic's OWN metric definitions over the whole FILTERED scope
|
| 836 |
+
mts = _model()["metrics"]
|
| 837 |
+
keys = [k for k in (aggregates if aggregates is not None
|
| 838 |
+
else [k for k, m in mts.items()
|
| 839 |
+
if m.get("topic") == topic and m.get("agg") == "sum"])]
|
| 840 |
+
aggs = {}
|
| 841 |
+
if keys:
|
| 842 |
+
for k in keys:
|
| 843 |
+
if k not in mts or mts[k].get("topic") != topic:
|
| 844 |
+
raise ModelError(f"aggregate {k!r} is not a metric of topic {topic!r}")
|
| 845 |
+
agg_sql = ("SELECT " + ", ".join(f"{_measure_sql(mts[k], alias)} AS {k}" for k in keys)
|
| 846 |
+
+ f" FROM {frm} WHERE {all_where}")
|
| 847 |
+
cur = con.execute(agg_sql, scope_params + narrow_params)
|
| 848 |
+
aggs = dict(zip([d[0] for d in cur.description], cur.fetchone()))
|
| 849 |
+
finally:
|
| 850 |
+
con.close()
|
| 851 |
+
|
| 852 |
+
return {
|
| 853 |
+
"topic": topic,
|
| 854 |
+
"rows": rows,
|
| 855 |
+
"columns": {k: {"type": c["type"]} for k, c in cols.items()},
|
| 856 |
+
"window": {"offset": offset, "limit": limit, "returned": len(rows)},
|
| 857 |
+
"row_count": row_count, # N β matches the filter, across the WHOLE scope
|
| 858 |
+
"total_count": total_count, # M β the unfiltered scope
|
| 859 |
+
"aggregates": aggs, # over the whole FILTERED scope, not the window
|
| 860 |
+
"sql": sql,
|
| 861 |
+
"count_sql": count_sql,
|
| 862 |
+
}
|
| 863 |
+
|
| 864 |
+
|
| 865 |
+
def store_field_values(topic, dim, search=None, limit=25):
|
| 866 |
+
"""Resolve real filterable values for a dim (the Analyst's get_field_values tool β fixes
|
| 867 |
+
'NYC Florist' vs the actual record name BEFORE querying)."""
|
| 868 |
+
t = _model()["topics"].get(topic)
|
| 869 |
+
if not t or "store" not in t:
|
| 870 |
+
raise ModelError(f"topic {topic!r} has no store binding")
|
| 871 |
+
d = (t["store"].get("dims") or {}).get(dim)
|
| 872 |
+
if not d:
|
| 873 |
+
raise ModelError(f"unknown dim {dim!r} for topic {topic!r}")
|
| 874 |
+
if not d.get("name_col"):
|
| 875 |
+
raise ModelError(f"dim {dim!r} has no name column (filter by id)")
|
| 876 |
+
s = t["store"]
|
| 877 |
+
sql = (f"SELECT DISTINCT {d['col']} AS id, {d['name_col']} AS name "
|
| 878 |
+
f"FROM {s['table']} {s['alias']} {s.get('join','')} "
|
| 879 |
+
f"WHERE {render_scope(s['scope_sql'].strip())}")
|
| 880 |
+
params = []
|
| 881 |
+
if search:
|
| 882 |
+
sql += f" AND {d['name_col']} ILIKE ?"
|
| 883 |
+
params.append(f"%{search}%")
|
| 884 |
+
sql += f" ORDER BY name LIMIT {max(1, min(int(limit), 100))}"
|
| 885 |
+
con = _store_con()
|
| 886 |
+
try:
|
| 887 |
+
return [{"id": r[0], "name": r[1]} for r in con.execute(sql, params).fetchall()]
|
| 888 |
+
finally:
|
| 889 |
+
con.close()
|
| 890 |
+
|
| 891 |
+
|
| 892 |
+
def store_parity(date_from=None, date_to=None):
|
| 893 |
+
"""THE OM-2 gate: the store path must equal the live path to the cent, per metric per scope."""
|
| 894 |
+
import core.periods as P
|
| 895 |
+
if not date_from:
|
| 896 |
+
date_from, date_to = P.ytd()
|
| 897 |
+
out = []
|
| 898 |
+
for key in ("revenue", "margin", "units", "orders", "customers", "returns", "invoiced"):
|
| 899 |
+
m = _model()["metrics"][key]
|
| 900 |
+
# β A COMPANY-LEVEL topic is checked at company scope ONLY. `store_query` RAISES when a
|
| 901 |
+
# team is asked of a topic with no team_col, and the live builder refuses too β so
|
| 902 |
+
# looping the BUs here would fail the gate for a topic that is correct. Parity for a
|
| 903 |
+
# company-level number means company-level parity; pretending otherwise would either
|
| 904 |
+
# crash or, worse, compare two numbers that silently ignored the team.
|
| 905 |
+
_store = (_model()["topics"].get(m["topic"]) or {}).get("store") or {}
|
| 906 |
+
scopes = (None, *O.TEAM_IDS) if _store.get("team_col") else (None,)
|
| 907 |
+
for tid in scopes:
|
| 908 |
+
live = metric(key, date_from, date_to, tid)
|
| 909 |
+
res = store_query(m["topic"], [key], date_from=date_from, date_to=date_to, team_id=tid)
|
| 910 |
+
ours = res["rows"][0][key] if res["rows"] else 0
|
| 911 |
+
gap = abs((ours or 0) - (live or 0))
|
| 912 |
+
out.append({"check": f"store.{key} == live.{key} (team={tid})",
|
| 913 |
+
"ok": gap <= 0.01, "gap": round(gap, 4), "store": ours, "live": live})
|
| 914 |
+
# The agent dim (2026-07-17): store agent-filtered revenue must equal live revenue over the
|
| 915 |
+
# agent's FIRST-agent book β a fully independent path (live partner read + line-domain sum)
|
| 916 |
+
# against the store's res_partner.agent_id attribution. Largest book = the sharpest check.
|
| 917 |
+
recs = O.search_read("res.partner", [("active", "in", [True, False]),
|
| 918 |
+
("agent_ids", "!=", False)], ["agent_ids"], limit=100000)
|
| 919 |
+
books = {}
|
| 920 |
+
for r in recs:
|
| 921 |
+
a = (r.get("agent_ids") or [None])[0]
|
| 922 |
+
if a:
|
| 923 |
+
books.setdefault(a, []).append(r["id"])
|
| 924 |
+
if books:
|
| 925 |
+
top = max(books, key=lambda k: len(books[k]))
|
| 926 |
+
live = O.sum_field("sale.order.line",
|
| 927 |
+
O.sale_line_domain(date_from, date_to, partner_ids=books[top]),
|
| 928 |
+
"price_subtotal")
|
| 929 |
+
res = store_query("sales_lines", ["revenue"], date_from=date_from, date_to=date_to,
|
| 930 |
+
filters={"agent": [top]})
|
| 931 |
+
ours = res["rows"][0]["revenue"] if res["rows"] else 0
|
| 932 |
+
gap = abs((ours or 0) - (live or 0))
|
| 933 |
+
out.append({"check": f"store.revenue[agent={top}] == live book sum (first-agent, "
|
| 934 |
+
f"{len(books[top])} customers)",
|
| 935 |
+
"ok": gap <= 0.01, "gap": round(gap, 4), "store": ours, "live": live})
|
| 936 |
+
return out
|
royalimports_os/harness/skills/__init__.py
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
"""Portable skills β written once against the canonical model, run on any compatible connector."""
|
royalimports_os/harness/skills/sales.py
ADDED
|
@@ -0,0 +1,168 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Sales skill β PORTABLE. The de-Odoo'd replacement for modules/sales.py: every metric is expressed
|
| 2 |
+
against the canonical model and runs on any connector with the SALES capability (INVOICING for
|
| 3 |
+
returns). No model names, no `amount_untaxed`, no Odoo β the connector translates.
|
| 4 |
+
|
| 5 |
+
`validate(tenant)` parity-checks each metric against the current Odoo module, so a port only replaces
|
| 6 |
+
the original once it ties out to the cent (same discipline as the modules' own validate()).
|
| 7 |
+
|
| 8 |
+
Date windows come from core.periods (calendar math is source-agnostic).
|
| 9 |
+
"""
|
| 10 |
+
from __future__ import annotations
|
| 11 |
+
import sys
|
| 12 |
+
from pathlib import Path
|
| 13 |
+
sys.path.insert(0, str(Path(__file__).resolve().parents[2])) # royalimports_os/
|
| 14 |
+
import core.periods as P
|
| 15 |
+
from harness.canonical import Capability, Entity, Query, Filter
|
| 16 |
+
from harness.base import Tenant
|
| 17 |
+
|
| 18 |
+
REQUIRES = (Capability.SALES,)
|
| 19 |
+
|
| 20 |
+
|
| 21 |
+
# ------------------------------------------------------------------ tenant scope -> canonical filters
|
| 22 |
+
def _scope(tenant: Tenant, business_unit: str | None):
|
| 23 |
+
"""The client's scope CONFIG (was hard-coded Odoo constants) as canonical filters β valid on both
|
| 24 |
+
SALES_ORDER and SALES_ORDER_LINE (status/business_unit/customer exist on both)."""
|
| 25 |
+
cfg = tenant.config
|
| 26 |
+
fs = []
|
| 27 |
+
if cfg.get('confirmed_statuses'):
|
| 28 |
+
fs.append(Filter('status', 'in', list(cfg['confirmed_statuses'])))
|
| 29 |
+
bus = cfg.get('business_units') or {}
|
| 30 |
+
if business_unit and business_unit in bus:
|
| 31 |
+
fs.append(Filter('business_unit', 'in', [bus[business_unit]]))
|
| 32 |
+
elif bus:
|
| 33 |
+
fs.append(Filter('business_unit', 'in', list(bus.values())))
|
| 34 |
+
if cfg.get('excluded_customer_ids'):
|
| 35 |
+
fs.append(Filter('customer', 'not_in', list(cfg['excluded_customer_ids'])))
|
| 36 |
+
return tuple(fs)
|
| 37 |
+
|
| 38 |
+
|
| 39 |
+
def _sum(conn, entity, measure, df, dt, filters):
|
| 40 |
+
return sum(r[measure] for r in conn.aggregate(
|
| 41 |
+
Query(entity, (measure,), 'sum', date_from=df, date_to=dt, filters=filters)))
|
| 42 |
+
|
| 43 |
+
|
| 44 |
+
def _count(conn, entity, df, dt, filters):
|
| 45 |
+
return sum(r['count'] for r in conn.aggregate(
|
| 46 |
+
Query(entity, (), 'count', date_from=df, date_to=dt, filters=filters)))
|
| 47 |
+
|
| 48 |
+
|
| 49 |
+
# ------------------------------------------------------------------ metrics (portable)
|
| 50 |
+
def revenue(tenant, date_from, date_to, business_unit=None):
|
| 51 |
+
return _sum(tenant.source(Capability.SALES), Entity.SALES_ORDER, 'amount',
|
| 52 |
+
date_from, date_to, _scope(tenant, business_unit))
|
| 53 |
+
|
| 54 |
+
|
| 55 |
+
def headline(tenant, business_unit=None, t=None):
|
| 56 |
+
conn = tenant.source(Capability.SALES)
|
| 57 |
+
t = t or P.today()
|
| 58 |
+
yf, yt = P.ytd(t)
|
| 59 |
+
lf, lt = P.ytd_last_year(t)
|
| 60 |
+
sc = _scope(tenant, business_unit)
|
| 61 |
+
rev = _sum(conn, Entity.SALES_ORDER, 'amount', yf, yt, sc)
|
| 62 |
+
rev_ly = _sum(conn, Entity.SALES_ORDER, 'amount', lf, lt, sc)
|
| 63 |
+
orders = _count(conn, Entity.SALES_ORDER, yf, yt, sc)
|
| 64 |
+
custs = conn.distinct_count(Query(Entity.SALES_ORDER, date_from=yf, date_to=yt, filters=sc), 'customer')
|
| 65 |
+
return {'ytd_revenue': rev, 'ytd_revenue_ly': rev_ly, 'yoy_pct': P.yoy_pct(rev, rev_ly),
|
| 66 |
+
'ytd_orders': orders, 'ytd_customers': custs, 'aov': (rev / orders) if orders else 0.0}
|
| 67 |
+
|
| 68 |
+
|
| 69 |
+
def monthly_trend(tenant, n=13, business_unit=None, t=None):
|
| 70 |
+
conn = tenant.source(Capability.SALES)
|
| 71 |
+
t = t or P.today()
|
| 72 |
+
first = P.month_starts(n, t)[0][1]
|
| 73 |
+
span_from = f"{int(first[:4]) - 1:04d}{first[4:]}" # a year before the first shown month
|
| 74 |
+
q = Query(Entity.SALES_ORDER, ('amount',), 'sum', grain='month',
|
| 75 |
+
date_from=span_from, date_to=t.isoformat(), filters=_scope(tenant, business_unit))
|
| 76 |
+
m = {r['period']: r['amount'] for r in conn.aggregate(q)}
|
| 77 |
+
rows = []
|
| 78 |
+
for ym, _s, _e in P.month_starts(n, t):
|
| 79 |
+
y, mo = int(ym[:4]) - 1, int(ym[5:7])
|
| 80 |
+
this, last = m.get(ym, 0.0), m.get(f'{y:04d}-{mo:02d}', 0.0)
|
| 81 |
+
rows.append({'month': ym, 'revenue': this, 'revenue_ly': last, 'yoy_pct': P.yoy_pct(this, last)})
|
| 82 |
+
return rows
|
| 83 |
+
|
| 84 |
+
|
| 85 |
+
def top_customers(tenant, limit=25, business_unit=None, t=None):
|
| 86 |
+
conn = tenant.source(Capability.SALES)
|
| 87 |
+
yf, yt = P.ytd(t or P.today())
|
| 88 |
+
q = Query(Entity.SALES_ORDER, ('amount',), 'sum', group_by=('customer',),
|
| 89 |
+
date_from=yf, date_to=yt, filters=_scope(tenant, business_unit))
|
| 90 |
+
rows = [{'pid': r['customer']['id'], 'customer': r['customer']['name'],
|
| 91 |
+
'revenue': r['amount'], 'orders': r['count']}
|
| 92 |
+
for r in conn.aggregate(q) if r.get('customer')]
|
| 93 |
+
return sorted(rows, key=lambda x: -x['revenue'])[:limit]
|
| 94 |
+
|
| 95 |
+
|
| 96 |
+
def top_skus(tenant, limit=25, business_unit=None, t=None):
|
| 97 |
+
conn = tenant.source(Capability.SALES)
|
| 98 |
+
yf, yt = P.ytd(t or P.today())
|
| 99 |
+
q = Query(Entity.SALES_ORDER_LINE, ('amount', 'qty', 'margin'), 'sum', group_by=('product',),
|
| 100 |
+
date_from=yf, date_to=yt, filters=_scope(tenant, business_unit))
|
| 101 |
+
rows = []
|
| 102 |
+
for r in conn.aggregate(q):
|
| 103 |
+
if not r.get('product'):
|
| 104 |
+
continue
|
| 105 |
+
lines = r['count'] or 0
|
| 106 |
+
rows.append({'product': r['product']['name'], 'pid': r['product']['id'], 'revenue': r['amount'],
|
| 107 |
+
'qty': r['qty'], 'margin': r['margin'],
|
| 108 |
+
'profit_per_order': (r['margin'] / lines) if lines else 0.0})
|
| 109 |
+
return sorted(rows, key=lambda x: -x['revenue'])[:limit]
|
| 110 |
+
|
| 111 |
+
|
| 112 |
+
def by_category(tenant, limit=20, business_unit=None, t=None):
|
| 113 |
+
conn = tenant.source(Capability.SALES)
|
| 114 |
+
yf, yt = P.ytd(t or P.today())
|
| 115 |
+
q = Query(Entity.SALES_ORDER_LINE, ('amount',), 'sum', group_by=('category',),
|
| 116 |
+
date_from=yf, date_to=yt, filters=_scope(tenant, business_unit))
|
| 117 |
+
rows = [{'category': r['category'], 'revenue': r['amount']} for r in conn.aggregate(q)]
|
| 118 |
+
return sorted(rows, key=lambda x: -x['revenue'])[:limit]
|
| 119 |
+
|
| 120 |
+
|
| 121 |
+
def returns_monthly(tenant, n=13, t=None):
|
| 122 |
+
"""Returns (credit notes) per month β needs INVOICING, consolidated (no BU split)."""
|
| 123 |
+
conn = tenant.source(Capability.INVOICING)
|
| 124 |
+
t = t or P.today()
|
| 125 |
+
first = P.month_starts(n, t)[0][1]
|
| 126 |
+
span_from = f"{int(first[:4]) - 1:04d}{first[4:]}"
|
| 127 |
+
ex = tenant.config.get('excluded_customer_ids') or []
|
| 128 |
+
filters = [Filter('doc_type', 'eq', 'out_refund'), Filter('status', 'eq', 'posted')]
|
| 129 |
+
if ex:
|
| 130 |
+
filters.append(Filter('customer', 'not_in', list(ex)))
|
| 131 |
+
q = Query(Entity.INVOICE, ('amount',), 'sum', grain='month',
|
| 132 |
+
date_from=span_from, date_to=t.isoformat(), filters=tuple(filters))
|
| 133 |
+
m = {r['period']: -r['amount'] for r in conn.aggregate(q)} # out_refund amount is negative -> flip
|
| 134 |
+
rows = []
|
| 135 |
+
for ym, _s, _e in P.month_starts(n, t):
|
| 136 |
+
y, mo = int(ym[:4]) - 1, int(ym[5:7])
|
| 137 |
+
this, last = m.get(ym, 0.0), m.get(f'{y:04d}-{mo:02d}', 0.0)
|
| 138 |
+
rows.append({'month': ym, 'revenue': this, 'revenue_ly': last, 'yoy_pct': P.yoy_pct(this, last)})
|
| 139 |
+
return rows
|
| 140 |
+
|
| 141 |
+
|
| 142 |
+
# ------------------------------------------------------------------ parity validation (port gate)
|
| 143 |
+
def validate(tenant, t=None):
|
| 144 |
+
"""Reconcile the ported skill to the current Odoo module β a port ships only when it ties out."""
|
| 145 |
+
import modules.sales as M
|
| 146 |
+
t = t or P.today()
|
| 147 |
+
yf, yt = P.ytd(t)
|
| 148 |
+
checks = []
|
| 149 |
+
|
| 150 |
+
def chk(name, a, b, tol=1.0):
|
| 151 |
+
checks.append({'check': name, 'a': round(a, 2), 'b': round(b, 2),
|
| 152 |
+
'gap': round(a - b, 2), 'ok': abs(a - b) <= tol})
|
| 153 |
+
|
| 154 |
+
h, mh = headline(tenant), M.headline()
|
| 155 |
+
chk('headline YTD revenue == module', h['ytd_revenue'], mh['ytd_revenue'])
|
| 156 |
+
chk('headline YTD orders == module', h['ytd_orders'], mh['ytd_orders'])
|
| 157 |
+
chk('headline YTD customers == module', h['ytd_customers'], mh['ytd_customers'])
|
| 158 |
+
chk('Ξ£ top_customers == module', sum(r['revenue'] for r in top_customers(tenant, 10**9)),
|
| 159 |
+
sum(r['revenue'] for r in M.top_customers(limit=10**9)))
|
| 160 |
+
chk('Ξ£ by_category == module', sum(r['revenue'] for r in by_category(tenant, 10**9)),
|
| 161 |
+
sum(r['revenue'] for r in M.by_category(limit=10**9)))
|
| 162 |
+
chk('Ξ£ top_skus revenue == module', sum(r['revenue'] for r in top_skus(tenant, 10**9)),
|
| 163 |
+
sum(r['revenue'] for r in M.top_skus(limit=10**9)))
|
| 164 |
+
chk('monthly_trend Ξ£ this-yr == module', sum(r['revenue'] for r in monthly_trend(tenant)),
|
| 165 |
+
sum(r['revenue'] for r in M.monthly_trend()))
|
| 166 |
+
chk('returns_monthly Ξ£ this-yr == module', sum(r['revenue'] for r in returns_monthly(tenant)),
|
| 167 |
+
sum(r['revenue'] for r in M.returns_monthly()))
|
| 168 |
+
return checks
|
royalimports_os/harness/tables.py
ADDED
|
@@ -0,0 +1,129 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""harness/tables.py β a topic-backed TABLE for the grid (CG-2).
|
| 2 |
+
|
| 3 |
+
The grid's first table (customers) is built from `modules.customer_data.pool()`: one row per
|
| 4 |
+
customer, the whole book shipped as component args, filtered client-side. That works at 1,550
|
| 5 |
+
rows and does not work at line grain β `sales_lines` is 201,558 rows / ~89 MB.
|
| 6 |
+
|
| 7 |
+
This module builds the SECOND table: rows come from `semantic.store_rows`, so the fetch is
|
| 8 |
+
WINDOWED and the filtering is SQL. Its whole job is to hand the grid a payload in the same
|
| 9 |
+
host-neutral shape the React component already speaks, plus the one thing a windowed table must
|
| 10 |
+
carry that a whole-book table never needed:
|
| 11 |
+
|
| 12 |
+
counts = {matched, total, shown, windowed}
|
| 13 |
+
|
| 14 |
+
`matched` and `total` come from their own queries over the FULL scope; `shown` is the size of
|
| 15 |
+
this window. The component renders "showing SHOWN of MATCHED (filtered from TOTAL)" from those
|
| 16 |
+
numbers and NEVER from `rows.length` β which is the difference between an honest window and a
|
| 17 |
+
silent `[:N]` wearing a total ([[no-unverifiable-aggregates]]).
|
| 18 |
+
|
| 19 |
+
The field CONTRACT is derived from the topic, not hand-written: labels and types come from the
|
| 20 |
+
model's dims and metrics, so a new metric cannot arrive as an untyped, unlabelled column and the
|
| 21 |
+
grid can never disagree with the semantic layer about what a column means.
|
| 22 |
+
"""
|
| 23 |
+
from harness import semantic as _sem
|
| 24 |
+
|
| 25 |
+
#: Topic-backed tables the grid can open. `key` is the storage key its saved views live under β
|
| 26 |
+
#: separate from the customer table's, so the two tables' views can never collide.
|
| 27 |
+
TOPIC_TABLES = {
|
| 28 |
+
"sales_lines": {
|
| 29 |
+
"label": "Sales lines",
|
| 30 |
+
"storage_key": "sales_lines_table_workspace",
|
| 31 |
+
# the columns worth defaulting to visible, in order; everything else stays available
|
| 32 |
+
"default_visible": ["date", "order_partner", "product", "category", "state", "agent",
|
| 33 |
+
"team", "revenue", "units", "margin"],
|
| 34 |
+
# a line-grain table is unusable sorted by nothing in particular
|
| 35 |
+
"default_sorts": [{"colId": "date", "dir": "desc"}],
|
| 36 |
+
},
|
| 37 |
+
}
|
| 38 |
+
|
| 39 |
+
#: how many rows one window carries. Small enough that the payload stays a few hundred KB at
|
| 40 |
+
#: ~440 B/row, large enough to scroll. The honest counts make the window visible rather than
|
| 41 |
+
#: silent, so this is a performance knob and not a correctness one.
|
| 42 |
+
PAGE_ROWS = 200
|
| 43 |
+
|
| 44 |
+
|
| 45 |
+
def _labels(topic):
|
| 46 |
+
"""colId -> human label, taken from the MODEL (dim labels, metric labels)."""
|
| 47 |
+
t = _sem.topics()[topic]
|
| 48 |
+
out = {}
|
| 49 |
+
for key, d in ((t.get("store") or {}).get("dims") or {}).items():
|
| 50 |
+
out[key] = d.get("label") or key.replace("_", " ").title()
|
| 51 |
+
out[f"{key}_id"] = f"{out[key]} id"
|
| 52 |
+
out["date"] = "Date"
|
| 53 |
+
for k, m in _sem.metrics().items():
|
| 54 |
+
if m.get("topic") == topic:
|
| 55 |
+
out[k] = m.get("label") or k
|
| 56 |
+
return out
|
| 57 |
+
|
| 58 |
+
|
| 59 |
+
def table_fields(topic):
|
| 60 |
+
"""The grid field contract for a topic, derived from the semantic model.
|
| 61 |
+
|
| 62 |
+
Every column is `source='odoo'` β read-only. A line is a fact from the ERP; unlike the
|
| 63 |
+
customer table there is no overlay stratum here (notes/tags hang off a CUSTOMER, not off an
|
| 64 |
+
individual order line), so the two-strata split simply does not arise.
|
| 65 |
+
"""
|
| 66 |
+
cfg = TOPIC_TABLES[topic]
|
| 67 |
+
cols = _sem.store_columns(topic, grain="row")
|
| 68 |
+
labels = _labels(topic)
|
| 69 |
+
order = [k for k in cfg["default_visible"] if k in cols]
|
| 70 |
+
order += [k for k in cols if k not in order]
|
| 71 |
+
return [{
|
| 72 |
+
"key": k,
|
| 73 |
+
"label": labels.get(k, k.replace("_", " ").title()),
|
| 74 |
+
"type": cols[k]["type"],
|
| 75 |
+
"source": "odoo",
|
| 76 |
+
"default": k in cfg["default_visible"],
|
| 77 |
+
} for k in order]
|
| 78 |
+
|
| 79 |
+
|
| 80 |
+
def table_payload(topic, config=None, offset=0, limit=None, team_id=None,
|
| 81 |
+
date_from=None, date_to=None):
|
| 82 |
+
"""Fetch ONE window of a topic-backed table plus its scope-wide counts and totals.
|
| 83 |
+
|
| 84 |
+
`config` is a grid ViewConfig (filters / filterConj / sorts / search) β the same shape the
|
| 85 |
+
customer table persists, so a saved view means the same thing on either table.
|
| 86 |
+
|
| 87 |
+
Returns the component payload: {fields, rows, counts, aggregates, storageKey}.
|
| 88 |
+
"""
|
| 89 |
+
if topic not in TOPIC_TABLES:
|
| 90 |
+
raise _sem.ModelError(f"no topic-backed table registered for {topic!r}")
|
| 91 |
+
cfg = TOPIC_TABLES[topic]
|
| 92 |
+
config = config or {}
|
| 93 |
+
limit = PAGE_ROWS if limit is None else limit
|
| 94 |
+
|
| 95 |
+
got = _sem.store_rows(
|
| 96 |
+
topic,
|
| 97 |
+
filter_tree=config.get("filters") or [],
|
| 98 |
+
filter_conj=config.get("filterConj") or "and",
|
| 99 |
+
search=config.get("search") or None,
|
| 100 |
+
sorts=config.get("sorts") or cfg["default_sorts"],
|
| 101 |
+
member_ids=config.get("memberPids") or None,
|
| 102 |
+
limit=limit, offset=offset,
|
| 103 |
+
team_id=team_id, date_from=date_from, date_to=date_to,
|
| 104 |
+
)
|
| 105 |
+
|
| 106 |
+
rows = []
|
| 107 |
+
for i, r in enumerate(got["rows"]):
|
| 108 |
+
# `pid` is the grid's row identity contract (selection, detail, overlay all key on it).
|
| 109 |
+
# At line grain that identity is the order LINE id, which store_rows returns as _rid.
|
| 110 |
+
row = {k: v for k, v in r.items() if k != "_rid"}
|
| 111 |
+
row["pid"] = r["_rid"]
|
| 112 |
+
rows.append(row)
|
| 113 |
+
|
| 114 |
+
return {
|
| 115 |
+
"fields": table_fields(topic),
|
| 116 |
+
"rows": rows,
|
| 117 |
+
# THE honest window. `matched`/`total` are scope-wide queries; `shown` is this window.
|
| 118 |
+
# A caller must never recompute `matched` from len(rows).
|
| 119 |
+
"counts": {
|
| 120 |
+
"shown": len(rows),
|
| 121 |
+
"matched": got["row_count"],
|
| 122 |
+
"total": got["total_count"],
|
| 123 |
+
"windowed": True,
|
| 124 |
+
},
|
| 125 |
+
"aggregates": got["aggregates"], # over the whole FILTERED scope, not the window
|
| 126 |
+
"storageKey": cfg["storage_key"],
|
| 127 |
+
"label": cfg["label"],
|
| 128 |
+
"window": got["window"],
|
| 129 |
+
}
|
royalimports_os/harness/telemetry.py
ADDED
|
@@ -0,0 +1,94 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""harness/telemetry.py β usage telemetry v0 (OM-3; principle 7: the improvement loop is
|
| 2 |
+
structural). Append-only JSONL of what people actually do β pages opened, explore queries run,
|
| 3 |
+
Analyst questions asked β the observe-half of just-in-time modeling: what's used gets promoted
|
| 4 |
+
into the governed model; what's ignored gets pruned.
|
| 5 |
+
|
| 6 |
+
Fail-silent by design: telemetry must never break a page or a query. Local file v0
|
| 7 |
+
(data/store/usage.jsonl, tenant data, git-ignored); mined in-session for now, store-backed later.
|
| 8 |
+
"""
|
| 9 |
+
import json
|
| 10 |
+
import sys
|
| 11 |
+
import time
|
| 12 |
+
import traceback
|
| 13 |
+
from pathlib import Path
|
| 14 |
+
|
| 15 |
+
PATH = Path(__file__).resolve().parents[1] / "data" / "store" / "usage.jsonl"
|
| 16 |
+
ERR_PATH = PATH.parent / "errors.jsonl"
|
| 17 |
+
|
| 18 |
+
|
| 19 |
+
def log(kind, **fields):
|
| 20 |
+
try:
|
| 21 |
+
PATH.parent.mkdir(parents=True, exist_ok=True)
|
| 22 |
+
with PATH.open("a", encoding="utf-8") as f:
|
| 23 |
+
f.write(json.dumps({"ts": time.strftime("%Y-%m-%d %H:%M:%S"),
|
| 24 |
+
"kind": kind, **fields}, default=str) + "\n")
|
| 25 |
+
except Exception:
|
| 26 |
+
pass
|
| 27 |
+
|
| 28 |
+
|
| 29 |
+
def error(where, exc=None, **fields):
|
| 30 |
+
"""THE central error sink (owner rule 2026-07-23: ALL errors are logged). One JSONL line per
|
| 31 |
+
caught error (data/store/errors.jsonl β per-deployment, git-ignored) + a mirror line on
|
| 32 |
+
stderr so it also lands in the Space container logs. Call it from EVERY except block that
|
| 33 |
+
swallows or downgrades an exception β a failure with nowhere to land is how the app breaks
|
| 34 |
+
silently (the 2026-07-23 drawer fire). Never raises."""
|
| 35 |
+
try:
|
| 36 |
+
rec = {"ts": time.strftime("%Y-%m-%d %H:%M:%S"), "where": str(where)[:120], **fields}
|
| 37 |
+
if exc is not None:
|
| 38 |
+
rec["type"] = type(exc).__name__
|
| 39 |
+
rec["msg"] = str(exc)[:400]
|
| 40 |
+
try:
|
| 41 |
+
tb = "".join(traceback.format_exception(type(exc), exc, exc.__traceback__,
|
| 42 |
+
limit=8))
|
| 43 |
+
if tb.strip():
|
| 44 |
+
rec["trace"] = tb[-1800:]
|
| 45 |
+
except Exception:
|
| 46 |
+
pass
|
| 47 |
+
ERR_PATH.parent.mkdir(parents=True, exist_ok=True)
|
| 48 |
+
with ERR_PATH.open("a", encoding="utf-8") as f:
|
| 49 |
+
f.write(json.dumps(rec, default=str) + "\n")
|
| 50 |
+
print(f"[app-error] {rec['ts']} {rec['where']}: "
|
| 51 |
+
f"{rec.get('type', '')} {rec.get('msg', '')}", file=sys.stderr, flush=True)
|
| 52 |
+
except Exception:
|
| 53 |
+
pass
|
| 54 |
+
|
| 55 |
+
|
| 56 |
+
def read_errors(limit=500):
|
| 57 |
+
"""Tail of the error log (newest last) β the Data Health admin section reads this."""
|
| 58 |
+
if not ERR_PATH.exists():
|
| 59 |
+
return []
|
| 60 |
+
out = []
|
| 61 |
+
try:
|
| 62 |
+
with ERR_PATH.open(encoding="utf-8") as f:
|
| 63 |
+
for line in f:
|
| 64 |
+
try:
|
| 65 |
+
out.append(json.loads(line))
|
| 66 |
+
except json.JSONDecodeError:
|
| 67 |
+
continue
|
| 68 |
+
except Exception:
|
| 69 |
+
return out
|
| 70 |
+
return out[-limit:]
|
| 71 |
+
|
| 72 |
+
|
| 73 |
+
def read(limit=5000):
|
| 74 |
+
if not PATH.exists():
|
| 75 |
+
return []
|
| 76 |
+
out = []
|
| 77 |
+
with PATH.open(encoding="utf-8") as f:
|
| 78 |
+
for line in f:
|
| 79 |
+
try:
|
| 80 |
+
out.append(json.loads(line))
|
| 81 |
+
except json.JSONDecodeError:
|
| 82 |
+
continue
|
| 83 |
+
return out[-limit:]
|
| 84 |
+
|
| 85 |
+
|
| 86 |
+
def summary():
|
| 87 |
+
"""Counts by kind + the most-touched pages/topics β the promote-or-prune shortlist."""
|
| 88 |
+
from collections import Counter
|
| 89 |
+
ev = read()
|
| 90 |
+
kinds = Counter(e.get("kind") for e in ev)
|
| 91 |
+
pages = Counter(e.get("page") for e in ev if e.get("kind") == "page")
|
| 92 |
+
topics = Counter(e.get("topic") for e in ev if e.get("topic"))
|
| 93 |
+
return {"events": len(ev), "by_kind": dict(kinds),
|
| 94 |
+
"top_pages": pages.most_common(10), "top_topics": topics.most_common(10)}
|