Deploy AIOS web (React glide grid + FastAPI slice)
Browse filesThis view is limited to 50 files because it contains too many changes. Β See raw diff
- Dockerfile +68 -68
- README.md +1 -1
- api/deps.py +1 -1
- api/main.py +364 -364
- api/pages.py +1 -1
- api/routes_assets.py +165 -165
- platform/aios_grid.py +1870 -0
- platform/aios_grid_fields.json +413 -0
- platform/core/assets.py +73 -0
- platform/core/auth_google.py +91 -0
- platform/core/context.py +59 -0
- platform/core/grid_events.py +1334 -0
- platform/core/keychain.py +182 -0
- platform/core/links.py +59 -0
- platform/core/management_pl.py +161 -0
- platform/core/measure_resolve.py +190 -0
- platform/core/odoo.py +201 -0
- platform/core/periods.py +155 -0
- platform/core/perm_migrate.py +156 -0
- platform/core/perm_scope.py +286 -0
- platform/core/perms.py +215 -0
- platform/core/record_comments.py +155 -0
- platform/core/registry.py +199 -0
- platform/core/store.py +369 -0
- platform/core/store_backend.py +85 -0
- platform/core/store_pg.py +249 -0
- platform/core/table_store.py +479 -0
- platform/core/user_tables.py +292 -0
- platform/core/users.py +361 -0
- platform/evals/analyst_golden.yml +104 -0
- platform/harness/__init__.py +3 -0
- platform/harness/analyst.py +318 -0
- platform/harness/base.py +123 -0
- platform/harness/canonical.py +101 -0
- platform/harness/chatlog.py +102 -0
- platform/harness/connectors/__init__.py +1 -0
- platform/harness/connectors/odoo.py +147 -0
- platform/harness/connectors/quickbooks.py +37 -0
- platform/harness/datastore.py +750 -0
- platform/harness/evals.py +134 -0
- platform/harness/filter_eval.py +485 -0
- platform/harness/filter_sql.py +673 -0
- platform/harness/measure_filter.py +788 -0
- platform/harness/pg/schema.sql +180 -0
- platform/harness/routines.py +259 -0
- platform/harness/runtime.py +248 -0
- platform/harness/semantic.py +936 -0
- platform/harness/skills/__init__.py +1 -0
- platform/harness/skills/sales.py +168 -0
- platform/harness/tables.py +129 -0
Dockerfile
CHANGED
|
@@ -1,68 +1,68 @@
|
|
| 1 |
-
# AIOS web β the React shell + FastAPI over the
|
| 2 |
-
# this is the one that actually runs, and the one a host migration re-points (C1e).
|
| 3 |
-
#
|
| 4 |
-
# β WHY A SECOND DOCKERFILE EXISTS (ops/Dockerfile.api), stated here so nobody "tidies" the wrong
|
| 5 |
-
# one away. It is NOT a duplicated service β it is the SAME service built from a DIFFERENT BUILD
|
| 6 |
-
# CONTEXT:
|
| 7 |
-
# * HF Spaces: the context is the Space repo root, which `deploy_web.py` uploads FLATTENED β
|
| 8 |
-
# `api/`, `web/`, `
|
| 9 |
-
# * ops/compose.yml: the context is the local repo root, where the same files live at
|
| 10 |
-
# `aios-web/api/`, `aios-web/web/`, `
|
| 11 |
-
# β THE REAL FIX, and it is a follow-up rather than something to do the day of a first deploy:
|
| 12 |
-
# make `deploy_web.py` upload with the REPO layout so both contexts are identical, then this file
|
| 13 |
-
# serves both and `ops/Dockerfile.api` is deleted. Until then `ops/verify_portability.py` gates
|
| 14 |
-
# the two against each other on every property that matters (same app object, same launcher
|
| 15 |
-
# contract, non-root, healthchecked, no baked secrets), so they cannot drift silently.
|
| 16 |
-
#
|
| 17 |
-
# Two stages: node builds the React bundle, python serves it + the JSON API. Login is the app's own
|
| 18 |
-
# branded page over a signed session cookie (wave 1 removed the HTTP Basic gate).
|
| 19 |
-
|
| 20 |
-
# ---- stage 1: build the React bundle ----
|
| 21 |
-
FROM node:22-slim AS web
|
| 22 |
-
WORKDIR /web
|
| 23 |
-
# Resolve deps FRESH from the public registry (the local lockfile pinned a mirror-only lodash
|
| 24 |
-
# 4.18.1 that isn't on npmjs.org; a fresh install gets the public 4.17.x that glide accepts).
|
| 25 |
-
COPY web/package.json ./
|
| 26 |
-
RUN npm install --no-audit --no-fund --no-package-lock
|
| 27 |
-
COPY web/ ./
|
| 28 |
-
RUN rm -f package-lock.json && npm run build # -> /web/dist
|
| 29 |
-
|
| 30 |
-
# ---- stage 2: python API serving the bundle ----
|
| 31 |
-
FROM python:3.12-slim
|
| 32 |
-
WORKDIR /app
|
| 33 |
-
COPY requirements.txt ./
|
| 34 |
-
RUN pip install --no-cache-dir -r requirements.txt
|
| 35 |
-
COPY api/ ./api/
|
| 36 |
-
# the reused data layer (modules/core/harness)
|
| 37 |
-
# β COMMENT ON ITS OWN LINE. Docker does NOT support a trailing comment on an instruction:
|
| 38 |
-
# it parsed `# the reused data layer ...` as extra SOURCE PATHS and the first build ever run
|
| 39 |
-
# died with `failed to calculate checksum ... "/the": not found`. Only RUN tolerates it, and
|
| 40 |
-
# only because the shell eats the `#`. Gated in ops/verify_portability.py.
|
| 41 |
-
COPY
|
| 42 |
-
COPY --from=web /web/dist ./web/dist
|
| 43 |
-
|
| 44 |
-
# Run unprivileged β the C2 security baseline, and matched to ops/Dockerfile.api so the two images
|
| 45 |
-
# do not differ on a property that would only surface after a migration. uid 1000 is what HF
|
| 46 |
-
# Spaces expects.
|
| 47 |
-
RUN useradd --create-home --uid 1000 user && chown -R user:user /app
|
| 48 |
-
USER user
|
| 49 |
-
|
| 50 |
-
ENV RI_DIR=/app/
|
| 51 |
-
PORT=7860 \
|
| 52 |
-
PYTHONUNBUFFERED=1 \
|
| 53 |
-
AIOS_PREWARM=1
|
| 54 |
-
# β AIOS_INSECURE_SSL is deliberately NOT set. It disables TLS verification for every outbound
|
| 55 |
-
# call in the process and exists for one developer laptop. ops/verify_portability.py fails if any
|
| 56 |
-
# deploy config sets it.
|
| 57 |
-
EXPOSE 7860
|
| 58 |
-
|
| 59 |
-
# Liveness via stdlib rather than curl β it keeps the image free of an apt layer that exists only
|
| 60 |
-
# for the healthcheck.
|
| 61 |
-
HEALTHCHECK --interval=30s --timeout=5s --start-period=60s --retries=3 \
|
| 62 |
-
CMD python -c "import os,urllib.request,sys; sys.exit(0 if urllib.request.urlopen(f'http://127.0.0.1:{os.environ.get(\"PORT\",\"7860\")}/api/health',timeout=4).status==200 else 1)"
|
| 63 |
-
|
| 64 |
-
# β THE PORT COMES FROM THE ENVIRONMENT, ALWAYS. The application itself reads PORT nowhere β it is
|
| 65 |
-
# a launcher argument β which is what makes moving hosts a config change instead of a code change.
|
| 66 |
-
# Secrets (Odoo creds, APP_PASSWORD, AIOS_SESSION_SECRET, APP_BASE_URL) arrive as env at RUN time
|
| 67 |
-
# and are never baked; the repo-root .dockerignore drops **/.env from the build context.
|
| 68 |
-
CMD ["sh", "-c", "uvicorn api.main:app --host 0.0.0.0 --port ${PORT}"]
|
|
|
|
| 1 |
+
# AIOS web β the React shell + FastAPI over the platform data layer. THE CANONICAL IMAGE:
|
| 2 |
+
# this is the one that actually runs, and the one a host migration re-points (C1e).
|
| 3 |
+
#
|
| 4 |
+
# β WHY A SECOND DOCKERFILE EXISTS (ops/Dockerfile.api), stated here so nobody "tidies" the wrong
|
| 5 |
+
# one away. It is NOT a duplicated service β it is the SAME service built from a DIFFERENT BUILD
|
| 6 |
+
# CONTEXT:
|
| 7 |
+
# * HF Spaces: the context is the Space repo root, which `deploy_web.py` uploads FLATTENED β
|
| 8 |
+
# `api/`, `web/`, `platform/`, `requirements.txt` all at top level. Hence the paths here.
|
| 9 |
+
# * ops/compose.yml: the context is the local repo root, where the same files live at
|
| 10 |
+
# `aios-web/api/`, `aios-web/web/`, `platform/`. Different prefixes, same content.
|
| 11 |
+
# β THE REAL FIX, and it is a follow-up rather than something to do the day of a first deploy:
|
| 12 |
+
# make `deploy_web.py` upload with the REPO layout so both contexts are identical, then this file
|
| 13 |
+
# serves both and `ops/Dockerfile.api` is deleted. Until then `ops/verify_portability.py` gates
|
| 14 |
+
# the two against each other on every property that matters (same app object, same launcher
|
| 15 |
+
# contract, non-root, healthchecked, no baked secrets), so they cannot drift silently.
|
| 16 |
+
#
|
| 17 |
+
# Two stages: node builds the React bundle, python serves it + the JSON API. Login is the app's own
|
| 18 |
+
# branded page over a signed session cookie (wave 1 removed the HTTP Basic gate).
|
| 19 |
+
|
| 20 |
+
# ---- stage 1: build the React bundle ----
|
| 21 |
+
FROM node:22-slim AS web
|
| 22 |
+
WORKDIR /web
|
| 23 |
+
# Resolve deps FRESH from the public registry (the local lockfile pinned a mirror-only lodash
|
| 24 |
+
# 4.18.1 that isn't on npmjs.org; a fresh install gets the public 4.17.x that glide accepts).
|
| 25 |
+
COPY web/package.json ./
|
| 26 |
+
RUN npm install --no-audit --no-fund --no-package-lock
|
| 27 |
+
COPY web/ ./
|
| 28 |
+
RUN rm -f package-lock.json && npm run build # -> /web/dist
|
| 29 |
+
|
| 30 |
+
# ---- stage 2: python API serving the bundle ----
|
| 31 |
+
FROM python:3.12-slim
|
| 32 |
+
WORKDIR /app
|
| 33 |
+
COPY requirements.txt ./
|
| 34 |
+
RUN pip install --no-cache-dir -r requirements.txt
|
| 35 |
+
COPY api/ ./api/
|
| 36 |
+
# the reused data layer (modules/core/harness)
|
| 37 |
+
# β COMMENT ON ITS OWN LINE. Docker does NOT support a trailing comment on an instruction:
|
| 38 |
+
# it parsed `# the reused data layer ...` as extra SOURCE PATHS and the first build ever run
|
| 39 |
+
# died with `failed to calculate checksum ... "/the": not found`. Only RUN tolerates it, and
|
| 40 |
+
# only because the shell eats the `#`. Gated in ops/verify_portability.py.
|
| 41 |
+
COPY platform/ ./platform/
|
| 42 |
+
COPY --from=web /web/dist ./web/dist
|
| 43 |
+
|
| 44 |
+
# Run unprivileged β the C2 security baseline, and matched to ops/Dockerfile.api so the two images
|
| 45 |
+
# do not differ on a property that would only surface after a migration. uid 1000 is what HF
|
| 46 |
+
# Spaces expects.
|
| 47 |
+
RUN useradd --create-home --uid 1000 user && chown -R user:user /app
|
| 48 |
+
USER user
|
| 49 |
+
|
| 50 |
+
ENV RI_DIR=/app/platform \
|
| 51 |
+
PORT=7860 \
|
| 52 |
+
PYTHONUNBUFFERED=1 \
|
| 53 |
+
AIOS_PREWARM=1
|
| 54 |
+
# β AIOS_INSECURE_SSL is deliberately NOT set. It disables TLS verification for every outbound
|
| 55 |
+
# call in the process and exists for one developer laptop. ops/verify_portability.py fails if any
|
| 56 |
+
# deploy config sets it.
|
| 57 |
+
EXPOSE 7860
|
| 58 |
+
|
| 59 |
+
# Liveness via stdlib rather than curl β it keeps the image free of an apt layer that exists only
|
| 60 |
+
# for the healthcheck.
|
| 61 |
+
HEALTHCHECK --interval=30s --timeout=5s --start-period=60s --retries=3 \
|
| 62 |
+
CMD python -c "import os,urllib.request,sys; sys.exit(0 if urllib.request.urlopen(f'http://127.0.0.1:{os.environ.get(\"PORT\",\"7860\")}/api/health',timeout=4).status==200 else 1)"
|
| 63 |
+
|
| 64 |
+
# β THE PORT COMES FROM THE ENVIRONMENT, ALWAYS. The application itself reads PORT nowhere β it is
|
| 65 |
+
# a launcher argument β which is what makes moving hosts a config change instead of a code change.
|
| 66 |
+
# Secrets (Odoo creds, APP_PASSWORD, AIOS_SESSION_SECRET, APP_BASE_URL) arrive as env at RUN time
|
| 67 |
+
# and are never baked; the repo-root .dockerignore drops **/.env from the build context.
|
| 68 |
+
CMD ["sh", "-c", "uvicorn api.main:app --host 0.0.0.0 --port ${PORT}"]
|
README.md
CHANGED
|
@@ -9,5 +9,5 @@ pinned: false
|
|
| 9 |
---
|
| 10 |
|
| 11 |
Loopable β the AIOS platform shell: a React front end (Airtable-grade grid + dashboard pages)
|
| 12 |
-
over a FastAPI service reusing the
|
| 13 |
business-unit scope and module grants. Separate from the Streamlit app.
|
|
|
|
| 9 |
---
|
| 10 |
|
| 11 |
Loopable β the AIOS platform shell: a React front end (Airtable-grade grid + dashboard pages)
|
| 12 |
+
over a FastAPI service reusing the platform data layer. Login-gated: a branded login page over an HMAC-signed session cookie, with per-user
|
| 13 |
business-unit scope and module grants. Separate from the Streamlit app.
|
api/deps.py
CHANGED
|
@@ -19,7 +19,7 @@ from fastapi import Depends, HTTPException, Request, Response
|
|
| 19 |
|
| 20 |
import aios_session
|
| 21 |
|
| 22 |
-
_RI = Path(os.environ.get("RI_DIR") or Path(__file__).resolve().parents[2] / "
|
| 23 |
if str(_RI) not in sys.path:
|
| 24 |
sys.path.insert(0, str(_RI))
|
| 25 |
|
|
|
|
| 19 |
|
| 20 |
import aios_session
|
| 21 |
|
| 22 |
+
_RI = Path(os.environ.get("RI_DIR") or Path(__file__).resolve().parents[2] / "platform")
|
| 23 |
if str(_RI) not in sys.path:
|
| 24 |
sys.path.insert(0, str(_RI))
|
| 25 |
|
api/main.py
CHANGED
|
@@ -1,364 +1,364 @@
|
|
| 1 |
-
"""AIOS web API β the ONE shared, stateless process the Streamlit exit is aimed at.
|
| 2 |
-
|
| 3 |
-
It REUSES the `
|
| 4 |
-
reconciled Odoo model stays server-side and the browser only ever sees derived JSON. Serves the
|
| 5 |
-
JSON API under `/api/*` and the built React bundle (`aios-web/web/dist`) at `/`.
|
| 6 |
-
|
| 7 |
-
WHAT CHANGED IN EXIT WAVE 1 (2026-07-30) β three things, all of them load-bearing:
|
| 8 |
-
|
| 9 |
-
* **Real sessions replace HTTP Basic.** The whole app used to sit behind one shared
|
| 10 |
-
`APP_PASSWORD` with the browser's native Basic prompt, which means every caller was the same
|
| 11 |
-
anonymous principal and no route could scope anything. Now a signed stateless cookie carries
|
| 12 |
-
a real `core/users` identity (X3), and every v1 route resolves BU + own-book scope from the
|
| 13 |
-
user RECORD. `/api/health` stays unauthenticated (liveness only).
|
| 14 |
-
* **The overlay fork is deleted.** `aios-web/api/data/overlay.json` was a second writable home
|
| 15 |
-
for user-owned fields the Streamlit app keeps in the tenant store. ONE store now (C1c).
|
| 16 |
-
* **The SSL shim is env-gated.** It used to run at import, unconditionally.
|
| 17 |
-
|
| 18 |
-
β TENANT RESOLUTION IS PER REQUEST (X7). Nothing tenant-shaped is held at module level; the
|
| 19 |
-
session's tenant claim resolves through `harness.runtime.get_runtime`, which is LRU-bounded. This
|
| 20 |
-
is the rule the ~30β80 MB/tenant target depends on β EXIT-0 measured the alternative at a ~0.6 GB
|
| 21 |
-
commit FLOOR per tenant, duplicated within 2% per tenant, nothing shared.
|
| 22 |
-
"""
|
| 23 |
-
import os
|
| 24 |
-
import sys
|
| 25 |
-
from pathlib import Path
|
| 26 |
-
|
| 27 |
-
# --- the local Odoo SSL quirk, now BEHIND A GATE -------------------------------------------------
|
| 28 |
-
# The Windows trust store reports the (valid) Odoo cert as expired, so local runs need an
|
| 29 |
-
# unverified default context; the HF Space and the container are unaffected. This used to run
|
| 30 |
-
# unconditionally at import β i.e. the shipped container disabled TLS verification for every
|
| 31 |
-
# outbound HTTPS call it ever made, to Odoo and to everything else, forever. That is a
|
| 32 |
-
# man-in-the-middle away from being someone else's data. It is now opt-in, must be set
|
| 33 |
-
# deliberately, and is never on by default.
|
| 34 |
-
# β NEVER set AIOS_INSECURE_SSL in a deployed environment. It exists for one developer laptop.
|
| 35 |
-
if os.environ.get("AIOS_INSECURE_SSL") == "1":
|
| 36 |
-
import ssl
|
| 37 |
-
ssl._create_default_https_context = ssl._create_unverified_context # noqa: S323
|
| 38 |
-
|
| 39 |
-
# --- reuse the tenant #0 data layer verbatim. RI_DIR overrides the location in the container
|
| 40 |
-
# (where the layout differs from the local sibling-dir default). ---
|
| 41 |
-
_HERE = Path(__file__).resolve()
|
| 42 |
-
_RI = Path(os.environ.get("RI_DIR") or (_HERE.parents[2] / "
|
| 43 |
-
for _p in (str(_RI), str(_HERE.parent)):
|
| 44 |
-
if _p not in sys.path:
|
| 45 |
-
sys.path.insert(0, _p)
|
| 46 |
-
|
| 47 |
-
from dotenv import load_dotenv # noqa: E402
|
| 48 |
-
load_dotenv(_RI / ".env") # Odoo creds + APP_PASSWORD, git-ignored, never committed/printed
|
| 49 |
-
|
| 50 |
-
from fastapi import Body, Depends, FastAPI, Request # noqa: E402
|
| 51 |
-
from fastapi.middleware.gzip import GZipMiddleware # noqa: E402
|
| 52 |
-
from fastapi.responses import JSONResponse # noqa: E402
|
| 53 |
-
from fastapi.staticfiles import StaticFiles # noqa: E402
|
| 54 |
-
from starlette.exceptions import HTTPException as StarletteHTTPException # noqa: E402
|
| 55 |
-
|
| 56 |
-
import aios_session # noqa: E402
|
| 57 |
-
import routes_admin # noqa: E402
|
| 58 |
-
import routes_assets # noqa: E402 (wave 18 C2-ASSET β catalog product imagery)
|
| 59 |
-
import routes_auth # noqa: E402
|
| 60 |
-
import routes_automation # noqa: E402 (wave 18 C4-AUTO β SESSION D's router, mounted by A)
|
| 61 |
-
import routes_customers # noqa: E402
|
| 62 |
-
import routes_grid # noqa: E402
|
| 63 |
-
import routes_keychain # noqa: E402 (wave 18 C7 β keychain + connectors admin surfaces)
|
| 64 |
-
import routes_nav # noqa: E402
|
| 65 |
-
import routes_pages # noqa: E402
|
| 66 |
-
import routes_products # noqa: E402 (wave 15 C-TOPIC β gated, not yet in the nav)
|
| 67 |
-
import routes_records # noqa: E402
|
| 68 |
-
import routes_tables # noqa: E402 (wave 18 C3-UT β user-created databases over the wire)
|
| 69 |
-
from core import grid_events # noqa: E402
|
| 70 |
-
from deps import Session, module_gate # noqa: E402
|
| 71 |
-
|
| 72 |
-
_WEB_DIST = _HERE.parents[1] / "web" / "dist"
|
| 73 |
-
|
| 74 |
-
# --- THE CANONICAL FIELD CONTRACT: a startup assertion, and the referee gate's subject ----------
|
| 75 |
-
# Every field tags its semantic TYPE and its SOURCE. `source='odoo'` is READ-ONLY (Odoo is never
|
| 76 |
-
# written); `source='overlay'` is the editable stratum that lives OUTSIDE Odoo. Loaded from the
|
| 77 |
-
# ONE canonical file shared with the embedded host (`
|
| 78 |
-
# embed == standalone by construction.
|
| 79 |
-
#
|
| 80 |
-
# TWO REASONS THIS IS HERE and not folded into the routes:
|
| 81 |
-
# 1. It FAILS LOUDLY at import when the file is missing β which means the deploy or the RI_DIR
|
| 82 |
-
# layout is broken, and discovering that from a 500 on the first customer request instead of
|
| 83 |
-
# at startup costs a debugging session. (The pre-wave main.py had this guard; the EXIT-2a
|
| 84 |
-
# rewrite dropped it and this restores it.)
|
| 85 |
-
# 2. `aios-web/verify_fields_contract.py` β the cross-side REFEREE β reads `FIELDS` and
|
| 86 |
-
# `_PASSTHROUGH_KEYS` from this module to prove the canonical file, `aios_grid.py` and this
|
| 87 |
-
# API have not drifted. The rewrite removed them and the referee went red; retargeting the
|
| 88 |
-
# gate would have been the wrong repair ([[gate-can-report-green-on-nothing]]: retarget, do
|
| 89 |
-
# not delete β but only when the subject genuinely moved. Here it should not have moved).
|
| 90 |
-
#
|
| 91 |
-
# β THE ROUTES SERVE A SUPERSET OF THIS. `routes_customers._payload` derives its field list from
|
| 92 |
-
# `aios_grid.fields_from_workspace(ws)`, which is `FIELDS` PLUS the session user's own custom_ and
|
| 93 |
-
# measure_ columns β the same list the Streamlit host renders. That is the point: the standalone
|
| 94 |
-
# shell now sees the user's own columns instead of the bare base contract. `FIELDS` is the
|
| 95 |
-
# canonical FLOOR, asserted below to be exactly what `aios_grid` starts from.
|
| 96 |
-
import json as _json_contract # noqa: E402
|
| 97 |
-
_FIELDS_PATH = _RI / "aios_grid_fields.json"
|
| 98 |
-
if not _FIELDS_PATH.is_file():
|
| 99 |
-
raise FileNotFoundError(
|
| 100 |
-
f"AIOS web API: canonical field contract missing at {_FIELDS_PATH}. Set RI_DIR to the "
|
| 101 |
-
"
|
| 102 |
-
)
|
| 103 |
-
_contract = _json_contract.loads(_FIELDS_PATH.read_text(encoding="utf-8"))
|
| 104 |
-
FIELDS = _contract["fields"] if isinstance(_contract, dict) else _contract
|
| 105 |
-
# text/status/select/date pass through untouched; every OTHER odoo field is numeric -> rounded.
|
| 106 |
-
# Derived from field TYPE (not a hand-kept key list) so a new text/date field can never be
|
| 107 |
-
# wrongly rounded. `select` joined 2026-08-02 (dba) β a choice label rounded would be garbage.
|
| 108 |
-
_PASSTHROUGH_KEYS = {f["key"] for f in FIELDS
|
| 109 |
-
if f["source"] == "odoo" and f["type"] in ("text", "status", "select",
|
| 110 |
-
"date")}
|
| 111 |
-
|
| 112 |
-
app = FastAPI(title="AIOS web API")
|
| 113 |
-
|
| 114 |
-
# The customers payload measured 1.15 MB of JSON on the live Space, shipped UNCOMPRESSED β with
|
| 115 |
-
# the 754 KB bundle behind it, most of "the app is slow" was bytes on the wire. gzip takes the
|
| 116 |
-
# payload to ~10β15% of that. minimum_size spares the tiny acks the overhead.
|
| 117 |
-
app.add_middleware(GZipMiddleware, minimum_size=1024)
|
| 118 |
-
|
| 119 |
-
|
| 120 |
-
@app.exception_handler(StarletteHTTPException)
|
| 121 |
-
def _error_shape(request: Request, exc: StarletteHTTPException):
|
| 122 |
-
"""ONE error shape for every non-2xx (X2): `{"error": {"code", "message"}}`.
|
| 123 |
-
|
| 124 |
-
`deps.err` already raises detail in that shape; anything FastAPI raises on its own (a 404, a
|
| 125 |
-
422 from a malformed path param) is wrapped here so a client never has to branch on two
|
| 126 |
-
different error bodies.
|
| 127 |
-
"""
|
| 128 |
-
detail = exc.detail
|
| 129 |
-
if isinstance(detail, dict) and "error" in detail:
|
| 130 |
-
body = detail
|
| 131 |
-
else:
|
| 132 |
-
body = {"error": {"code": f"http_{exc.status_code}", "message": str(detail)}}
|
| 133 |
-
return JSONResponse(body, status_code=exc.status_code,
|
| 134 |
-
headers=getattr(exc, "headers", None))
|
| 135 |
-
|
| 136 |
-
|
| 137 |
-
@app.exception_handler(grid_events.StoreUnavailable)
|
| 138 |
-
def _store_unavailable(request: Request, exc: grid_events.StoreUnavailable):
|
| 139 |
-
"""THE SAFETY NET for "the store is down" β 503, from anywhere.
|
| 140 |
-
|
| 141 |
-
β WHY THIS IS APP-LEVEL AND NOT A `try` PER ROUTE. It was a try per route first, and a
|
| 142 |
-
`StoreUnavailable` raised while BUILDING THE PAYLOAD β before the route reached its own
|
| 143 |
-
try/except β surfaced as a 500. A store outage is a normal operational state and every route
|
| 144 |
-
here touches the store at least twice (the workspace read, then the write), so "remember to
|
| 145 |
-
wrap it" is a rule that gets forgotten once and then reports the wrong thing. One handler
|
| 146 |
-
means a store outage can only ever be a 503, whichever call raised it.
|
| 147 |
-
|
| 148 |
-
The seam raises this only when the caller passed no `fallback_ws` β i.e. exactly on this
|
| 149 |
-
adapter, which has no durable session dict to degrade into. A 200 over a write that
|
| 150 |
-
evaporated is the failure the whole rule exists to prevent.
|
| 151 |
-
"""
|
| 152 |
-
return JSONResponse(
|
| 153 |
-
{"error": {"code": "store_unavailable",
|
| 154 |
-
"message": "the tenant store is unavailable β no change was saved"}},
|
| 155 |
-
status_code=503)
|
| 156 |
-
|
| 157 |
-
|
| 158 |
-
@app.get("/api/health")
|
| 159 |
-
def health():
|
| 160 |
-
"""Unauthenticated LIVENESS only β it must answer before anyone can sign in, so it may not
|
| 161 |
-
reveal anything about the deployment beyond "the process is up". No version, no tenant list,
|
| 162 |
-
no config: a health endpoint is the one URL every scanner finds first."""
|
| 163 |
-
return {"ok": True}
|
| 164 |
-
|
| 165 |
-
|
| 166 |
-
app.include_router(routes_auth.router)
|
| 167 |
-
app.include_router(routes_nav.router)
|
| 168 |
-
app.include_router(routes_customers.router)
|
| 169 |
-
app.include_router(routes_products.router)
|
| 170 |
-
app.include_router(routes_assets.router)
|
| 171 |
-
app.include_router(routes_tables.router)
|
| 172 |
-
app.include_router(routes_grid.router)
|
| 173 |
-
app.include_router(routes_records.router)
|
| 174 |
-
# EXIT wave 2: the Y1 page-data envelope (one route for every ported dashboard) and Y4's user
|
| 175 |
-
# administration. `routes_pages` imports `pages`, which lazily imports each `pages_*` builder β so
|
| 176 |
-
# a new page is a builder module plus one registry line, and nothing here changes.
|
| 177 |
-
app.include_router(routes_pages.router)
|
| 178 |
-
app.include_router(routes_admin.router)
|
| 179 |
-
app.include_router(routes_automation.router)
|
| 180 |
-
app.include_router(routes_keychain.router)
|
| 181 |
-
|
| 182 |
-
|
| 183 |
-
# --- DEPRECATED ALIASES (removed when S2's shell flips; kept so the current bundle keeps working)
|
| 184 |
-
# They are the v1 handlers with the v1 session requirement β NOT the old unauthenticated Basic
|
| 185 |
-
# behavior. An alias that kept the old auth would be a bypass of everything above it.
|
| 186 |
-
_GATE = module_gate(routes_customers.MODULE)
|
| 187 |
-
|
| 188 |
-
|
| 189 |
-
@app.get("/api/customers", deprecated=True)
|
| 190 |
-
def _customers_alias(session: Session = Depends(_GATE)):
|
| 191 |
-
return routes_customers._payload(session)
|
| 192 |
-
|
| 193 |
-
|
| 194 |
-
@app.patch("/api/customers/{pid}", deprecated=True)
|
| 195 |
-
def _patch_alias(pid: int, body: dict = Body(default=None),
|
| 196 |
-
session: Session = Depends(_GATE)):
|
| 197 |
-
return routes_customers.patch_customer(pid, body, session)
|
| 198 |
-
|
| 199 |
-
|
| 200 |
-
def _assert_contract_floor():
|
| 201 |
-
"""The canonical FIELDS must be exactly what `aios_grid` starts an empty workspace from.
|
| 202 |
-
|
| 203 |
-
This is what stops `FIELDS` becoming a constant that exists only to satisfy a gate. If the
|
| 204 |
-
canonical JSON and `aios_grid.fields_from_workspace({})` ever disagree, the standalone API and
|
| 205 |
-
the embedded host are serving two different schemas and the grid's own contract has forked β
|
| 206 |
-
the exact drift `verify_fields_contract.py` was written to catch, now also caught at startup
|
| 207 |
-
on whatever machine is actually running.
|
| 208 |
-
"""
|
| 209 |
-
import aios_grid
|
| 210 |
-
base = [f for f in aios_grid.fields_from_workspace({}) if not f.get("custom")]
|
| 211 |
-
if [f["key"] for f in base] != [f["key"] for f in FIELDS]:
|
| 212 |
-
raise RuntimeError(
|
| 213 |
-
"AIOS web API: the canonical field contract and aios_grid.fields_from_workspace({}) "
|
| 214 |
-
"disagree on the base field set β embed and standalone would serve different schemas. "
|
| 215 |
-
"Run aios-web/verify_fields_contract.py.")
|
| 216 |
-
|
| 217 |
-
|
| 218 |
-
def _startup_notes():
|
| 219 |
-
"""Say the two things an operator must know, once, at import β never in a response body."""
|
| 220 |
-
if aios_session.EPHEMERAL_SECRET:
|
| 221 |
-
print("[aios-api] AIOS_SESSION_SECRET is not set β signing with a random per-process "
|
| 222 |
-
"key. Sessions will not survive a restart and will not work across workers. "
|
| 223 |
-
"Set it in production.")
|
| 224 |
-
if os.environ.get("AIOS_INSECURE_SSL") == "1":
|
| 225 |
-
print("[aios-api] AIOS_INSECURE_SSL=1 β TLS verification is DISABLED for outbound "
|
| 226 |
-
"requests. Local development only; never in a deployed environment.")
|
| 227 |
-
|
| 228 |
-
|
| 229 |
-
_assert_contract_floor()
|
| 230 |
-
_startup_notes()
|
| 231 |
-
|
| 232 |
-
|
| 233 |
-
# β THE STATIC MOUNT IS NOW UNAUTHENTICATED, and that is a change this wave made on purpose.
|
| 234 |
-
# Before EXIT-3a, `BasicAuth` middleware gated the WHOLE app including this mount. A branded login
|
| 235 |
-
# page cannot live behind a password prompt, so the shell's own assets must be public β which they
|
| 236 |
-
# are: `index.html`, the JS/CSS bundle and the favicon reveal nothing.
|
| 237 |
-
#
|
| 238 |
-
# β WHAT THAT SILENTLY DE-GATED, caught in review rather than in production. `web/dist/` also
|
| 239 |
-
# carries `sample_customers.json`, a DEV FIXTURE copied from `web/public/`. Its 8 customers are
|
| 240 |
-
# synthetic, but the `agent` column holds REAL EMPLOYEE NAMES, and it went from Basic-gated to
|
| 241 |
-
# publicly fetchable in this commit. Nothing needs it: both the API bridge and `useCustomerData`
|
| 242 |
-
# deleted their sample fallback on purpose ("NOTHING HERE FALLS BACK TO sample_customers.json"),
|
| 243 |
-
# and it survives only because it sits in `web/public/`. So it is refused here β 404, the same
|
| 244 |
-
# answer as any other path that is not part of the app.
|
| 245 |
-
#
|
| 246 |
-
# The right long-term fix is deleting it from `web/public/` (S2's lane β flagged in the mailbox);
|
| 247 |
-
# this guard is what makes the API safe regardless of what the bundle happens to contain.
|
| 248 |
-
_DEV_FIXTURES = {"sample_customers.json"}
|
| 249 |
-
|
| 250 |
-
|
| 251 |
-
class _AppStatic(StaticFiles):
|
| 252 |
-
async def get_response(self, path, scope):
|
| 253 |
-
if Path(path).name in _DEV_FIXTURES:
|
| 254 |
-
raise StarletteHTTPException(status_code=404, detail="Not Found")
|
| 255 |
-
resp = await super().get_response(path, scope)
|
| 256 |
-
# Vite content-hashes everything under assets/ (a change is a NEW url), so those are
|
| 257 |
-
# immutable β a repeat visit re-downloads zero bytes instead of the whole 750 KB bundle.
|
| 258 |
-
# index.html must stay revalidated or a deploy would strand returning browsers on the old
|
| 259 |
-
# bundle; ETag/304 makes that revalidation a header exchange, not a transfer.
|
| 260 |
-
# β Normalised first: on a Windows host StaticFiles hands this path with backslashes,
|
| 261 |
-
# and `startswith("assets/")` silently skipped every asset (measured on the local probe).
|
| 262 |
-
if path.replace("\\", "/").lstrip("/").startswith("assets/"):
|
| 263 |
-
resp.headers["Cache-Control"] = "public, max-age=31536000, immutable"
|
| 264 |
-
else:
|
| 265 |
-
resp.headers["Cache-Control"] = "no-cache"
|
| 266 |
-
return resp
|
| 267 |
-
|
| 268 |
-
|
| 269 |
-
# static bundle LAST so /api/* wins; html=True serves index.html at / plus the built assets
|
| 270 |
-
if _WEB_DIST.is_dir():
|
| 271 |
-
app.mount("/", _AppStatic(directory=str(_WEB_DIST), html=True), name="web")
|
| 272 |
-
|
| 273 |
-
|
| 274 |
-
# --- boot prewarm (AIOS_PREWARM=1 β set by the Dockerfile, never by tests) -----------------------
|
| 275 |
-
# Without this, the first visitor after every deploy/restart pays the full Odoo pool build in
|
| 276 |
-
# their request. The thread warms the CONSOLIDATED scope (None, None) + every registered page's
|
| 277 |
-
# default envelope; scoped users still pay their own scope's first build, once.
|
| 278 |
-
# Env-gated rather than a startup event so importing `api.main` in a gate (verify_api and friends
|
| 279 |
-
# run against fakes) can never fire a live Odoo pull.
|
| 280 |
-
def _seed_and_sync_store():
|
| 281 |
-
"""Bring the analytical store (harness.datastore) LIVE for this container β the app.py
|
| 282 |
-
bootstrap, mirrored (2026-07-31, owner item 1).
|
| 283 |
-
|
| 284 |
-
`harness.datastore` powers every measure column/condition, and ONLY app.py used to call
|
| 285 |
-
`ensure_seed()` β so on a fresh Space disk this container resolved measures against a store
|
| 286 |
-
that never existed and every measure cell served blank. Seeding alone was NOT enough either,
|
| 287 |
-
and that was measured live the same day: "datastore seeded in 1.7s" followed by an endless
|
| 288 |
-
`api:measure-column: ModelError the data cache is still warming up` β `ready()` demands
|
| 289 |
-
EVERY entity at phase 'live', and a seed that predates a newer entity leaves it un-synced
|
| 290 |
-
forever in a process with no sync loop. The SYNC SPRINT after the seed is what closes the
|
| 291 |
-
write_date gap and backfills anything the seed lacks (app.py:7779's exact pattern, bounded
|
| 292 |
-
passes). Fail-quiet throughout: no seed/token/Odoo β the columns stay blank, the rows still
|
| 293 |
-
serve.
|
| 294 |
-
"""
|
| 295 |
-
import time as _t
|
| 296 |
-
t0 = _t.time()
|
| 297 |
-
try:
|
| 298 |
-
from harness import datastore as _ds
|
| 299 |
-
if _ds.ensure_seed():
|
| 300 |
-
print(f"[aios-api] datastore seeded in {_t.time() - t0:.1f}s")
|
| 301 |
-
# DEDICATED Odoo connection for this thread (the W6 postmortem rule): the sync's
|
| 302 |
-
# search_reads must never interleave on the shared client's xmlrpc transport.
|
| 303 |
-
import core.odoo as _odoo
|
| 304 |
-
try:
|
| 305 |
-
_odoo._tlocal.client = _odoo.OdooClient()
|
| 306 |
-
except Exception:
|
| 307 |
-
pass
|
| 308 |
-
res = {}
|
| 309 |
-
for i in range(12):
|
| 310 |
-
res = _ds.sync_all(log=lambda *a, **k: None)
|
| 311 |
-
print(f"[aios-api] datastore sync pass {i + 1}: "
|
| 312 |
-
+ ", ".join(f"{k}={v.get('phase')}" for k, v in sorted(res.items())))
|
| 313 |
-
if all(v.get("phase") == "live" for v in res.values()):
|
| 314 |
-
break
|
| 315 |
-
print(f"[aios-api] datastore sync done in {_t.time() - t0:.1f}s "
|
| 316 |
-
f"(ready={_ds.ready()})")
|
| 317 |
-
except Exception as e: # noqa: BLE001
|
| 318 |
-
print(f"[aios-api] datastore seed/sync skipped: {e}")
|
| 319 |
-
|
| 320 |
-
|
| 321 |
-
def _store_resync_loop():
|
| 322 |
-
"""Keep the analytical mirror current β the API-process stand-in for the re-sync the
|
| 323 |
-
Streamlit app piggybacks on page renders. Measure memos key on the pool stamp, so a
|
| 324 |
-
refreshed pool re-reads the freshly synced store."""
|
| 325 |
-
import time as _t
|
| 326 |
-
while True:
|
| 327 |
-
_t.sleep(1800)
|
| 328 |
-
try:
|
| 329 |
-
from harness import datastore as _ds
|
| 330 |
-
import core.odoo as _odoo
|
| 331 |
-
try:
|
| 332 |
-
_odoo._tlocal.client = _odoo.OdooClient()
|
| 333 |
-
except Exception:
|
| 334 |
-
pass
|
| 335 |
-
_ds.sync_all(log=lambda *a, **k: None)
|
| 336 |
-
except Exception as e: # noqa: BLE001
|
| 337 |
-
print(f"[aios-api] store resync failed: {e}")
|
| 338 |
-
|
| 339 |
-
|
| 340 |
-
def _prewarm():
|
| 341 |
-
import time as _t
|
| 342 |
-
t0 = _t.time()
|
| 343 |
-
# β The store sync runs in its OWN thread, never ahead of the pool warm: a stale seed can
|
| 344 |
-
# take many minutes of XML-RPC to close, and the first live probe of this arrangement
|
| 345 |
-
# showed the pool warm (42s) silently queued behind it β the whole app cold for every
|
| 346 |
-
# visitor while a background column channel caught up. Measure cells retry via the
|
| 347 |
-
# transient rule until the sync lands; nothing else waits on it.
|
| 348 |
-
import threading as _th
|
| 349 |
-
_th.Thread(target=_seed_and_sync_store, daemon=True, name="store-seed-sync").start()
|
| 350 |
-
try:
|
| 351 |
-
from harness import runtime as _runtime
|
| 352 |
-
rt = _runtime.get_runtime("royal-imports")
|
| 353 |
-
routes_customers.warm_default(rt)
|
| 354 |
-
import pages as _pages
|
| 355 |
-
_pages.warm_default(rt)
|
| 356 |
-
print(f"[aios-api] prewarm done in {_t.time() - t0:.1f}s")
|
| 357 |
-
except Exception as e: # noqa: BLE001 β boot must not die on a warm-up
|
| 358 |
-
print(f"[aios-api] prewarm skipped: {e}")
|
| 359 |
-
|
| 360 |
-
|
| 361 |
-
if os.environ.get("AIOS_PREWARM") == "1":
|
| 362 |
-
import threading as _threading
|
| 363 |
-
_threading.Thread(target=_prewarm, daemon=True, name="prewarm").start()
|
| 364 |
-
_threading.Thread(target=_store_resync_loop, daemon=True, name="store-resync").start()
|
|
|
|
| 1 |
+
"""AIOS web API β the ONE shared, stateless process the Streamlit exit is aimed at.
|
| 2 |
+
|
| 3 |
+
It REUSES the `platform/` data layer verbatim (nothing re-implemented): the canonical,
|
| 4 |
+
reconciled Odoo model stays server-side and the browser only ever sees derived JSON. Serves the
|
| 5 |
+
JSON API under `/api/*` and the built React bundle (`aios-web/web/dist`) at `/`.
|
| 6 |
+
|
| 7 |
+
WHAT CHANGED IN EXIT WAVE 1 (2026-07-30) β three things, all of them load-bearing:
|
| 8 |
+
|
| 9 |
+
* **Real sessions replace HTTP Basic.** The whole app used to sit behind one shared
|
| 10 |
+
`APP_PASSWORD` with the browser's native Basic prompt, which means every caller was the same
|
| 11 |
+
anonymous principal and no route could scope anything. Now a signed stateless cookie carries
|
| 12 |
+
a real `core/users` identity (X3), and every v1 route resolves BU + own-book scope from the
|
| 13 |
+
user RECORD. `/api/health` stays unauthenticated (liveness only).
|
| 14 |
+
* **The overlay fork is deleted.** `aios-web/api/data/overlay.json` was a second writable home
|
| 15 |
+
for user-owned fields the Streamlit app keeps in the tenant store. ONE store now (C1c).
|
| 16 |
+
* **The SSL shim is env-gated.** It used to run at import, unconditionally.
|
| 17 |
+
|
| 18 |
+
β TENANT RESOLUTION IS PER REQUEST (X7). Nothing tenant-shaped is held at module level; the
|
| 19 |
+
session's tenant claim resolves through `harness.runtime.get_runtime`, which is LRU-bounded. This
|
| 20 |
+
is the rule the ~30β80 MB/tenant target depends on β EXIT-0 measured the alternative at a ~0.6 GB
|
| 21 |
+
commit FLOOR per tenant, duplicated within 2% per tenant, nothing shared.
|
| 22 |
+
"""
|
| 23 |
+
import os
|
| 24 |
+
import sys
|
| 25 |
+
from pathlib import Path
|
| 26 |
+
|
| 27 |
+
# --- the local Odoo SSL quirk, now BEHIND A GATE -------------------------------------------------
|
| 28 |
+
# The Windows trust store reports the (valid) Odoo cert as expired, so local runs need an
|
| 29 |
+
# unverified default context; the HF Space and the container are unaffected. This used to run
|
| 30 |
+
# unconditionally at import β i.e. the shipped container disabled TLS verification for every
|
| 31 |
+
# outbound HTTPS call it ever made, to Odoo and to everything else, forever. That is a
|
| 32 |
+
# man-in-the-middle away from being someone else's data. It is now opt-in, must be set
|
| 33 |
+
# deliberately, and is never on by default.
|
| 34 |
+
# β NEVER set AIOS_INSECURE_SSL in a deployed environment. It exists for one developer laptop.
|
| 35 |
+
if os.environ.get("AIOS_INSECURE_SSL") == "1":
|
| 36 |
+
import ssl
|
| 37 |
+
ssl._create_default_https_context = ssl._create_unverified_context # noqa: S323
|
| 38 |
+
|
| 39 |
+
# --- reuse the tenant #0 data layer verbatim. RI_DIR overrides the location in the container
|
| 40 |
+
# (where the layout differs from the local sibling-dir default). ---
|
| 41 |
+
_HERE = Path(__file__).resolve()
|
| 42 |
+
_RI = Path(os.environ.get("RI_DIR") or (_HERE.parents[2] / "platform"))
|
| 43 |
+
for _p in (str(_RI), str(_HERE.parent)):
|
| 44 |
+
if _p not in sys.path:
|
| 45 |
+
sys.path.insert(0, _p)
|
| 46 |
+
|
| 47 |
+
from dotenv import load_dotenv # noqa: E402
|
| 48 |
+
load_dotenv(_RI / ".env") # Odoo creds + APP_PASSWORD, git-ignored, never committed/printed
|
| 49 |
+
|
| 50 |
+
from fastapi import Body, Depends, FastAPI, Request # noqa: E402
|
| 51 |
+
from fastapi.middleware.gzip import GZipMiddleware # noqa: E402
|
| 52 |
+
from fastapi.responses import JSONResponse # noqa: E402
|
| 53 |
+
from fastapi.staticfiles import StaticFiles # noqa: E402
|
| 54 |
+
from starlette.exceptions import HTTPException as StarletteHTTPException # noqa: E402
|
| 55 |
+
|
| 56 |
+
import aios_session # noqa: E402
|
| 57 |
+
import routes_admin # noqa: E402
|
| 58 |
+
import routes_assets # noqa: E402 (wave 18 C2-ASSET β catalog product imagery)
|
| 59 |
+
import routes_auth # noqa: E402
|
| 60 |
+
import routes_automation # noqa: E402 (wave 18 C4-AUTO β SESSION D's router, mounted by A)
|
| 61 |
+
import routes_customers # noqa: E402
|
| 62 |
+
import routes_grid # noqa: E402
|
| 63 |
+
import routes_keychain # noqa: E402 (wave 18 C7 β keychain + connectors admin surfaces)
|
| 64 |
+
import routes_nav # noqa: E402
|
| 65 |
+
import routes_pages # noqa: E402
|
| 66 |
+
import routes_products # noqa: E402 (wave 15 C-TOPIC β gated, not yet in the nav)
|
| 67 |
+
import routes_records # noqa: E402
|
| 68 |
+
import routes_tables # noqa: E402 (wave 18 C3-UT β user-created databases over the wire)
|
| 69 |
+
from core import grid_events # noqa: E402
|
| 70 |
+
from deps import Session, module_gate # noqa: E402
|
| 71 |
+
|
| 72 |
+
_WEB_DIST = _HERE.parents[1] / "web" / "dist"
|
| 73 |
+
|
| 74 |
+
# --- THE CANONICAL FIELD CONTRACT: a startup assertion, and the referee gate's subject ----------
|
| 75 |
+
# Every field tags its semantic TYPE and its SOURCE. `source='odoo'` is READ-ONLY (Odoo is never
|
| 76 |
+
# written); `source='overlay'` is the editable stratum that lives OUTSIDE Odoo. Loaded from the
|
| 77 |
+
# ONE canonical file shared with the embedded host (`platform/aios_grid_fields.json`), so
|
| 78 |
+
# embed == standalone by construction.
|
| 79 |
+
#
|
| 80 |
+
# TWO REASONS THIS IS HERE and not folded into the routes:
|
| 81 |
+
# 1. It FAILS LOUDLY at import when the file is missing β which means the deploy or the RI_DIR
|
| 82 |
+
# layout is broken, and discovering that from a 500 on the first customer request instead of
|
| 83 |
+
# at startup costs a debugging session. (The pre-wave main.py had this guard; the EXIT-2a
|
| 84 |
+
# rewrite dropped it and this restores it.)
|
| 85 |
+
# 2. `aios-web/verify_fields_contract.py` β the cross-side REFEREE β reads `FIELDS` and
|
| 86 |
+
# `_PASSTHROUGH_KEYS` from this module to prove the canonical file, `aios_grid.py` and this
|
| 87 |
+
# API have not drifted. The rewrite removed them and the referee went red; retargeting the
|
| 88 |
+
# gate would have been the wrong repair ([[gate-can-report-green-on-nothing]]: retarget, do
|
| 89 |
+
# not delete β but only when the subject genuinely moved. Here it should not have moved).
|
| 90 |
+
#
|
| 91 |
+
# β THE ROUTES SERVE A SUPERSET OF THIS. `routes_customers._payload` derives its field list from
|
| 92 |
+
# `aios_grid.fields_from_workspace(ws)`, which is `FIELDS` PLUS the session user's own custom_ and
|
| 93 |
+
# measure_ columns β the same list the Streamlit host renders. That is the point: the standalone
|
| 94 |
+
# shell now sees the user's own columns instead of the bare base contract. `FIELDS` is the
|
| 95 |
+
# canonical FLOOR, asserted below to be exactly what `aios_grid` starts from.
|
| 96 |
+
import json as _json_contract # noqa: E402
|
| 97 |
+
_FIELDS_PATH = _RI / "aios_grid_fields.json"
|
| 98 |
+
if not _FIELDS_PATH.is_file():
|
| 99 |
+
raise FileNotFoundError(
|
| 100 |
+
f"AIOS web API: canonical field contract missing at {_FIELDS_PATH}. Set RI_DIR to the "
|
| 101 |
+
"platform root (it also carries the data layer this API imports)."
|
| 102 |
+
)
|
| 103 |
+
_contract = _json_contract.loads(_FIELDS_PATH.read_text(encoding="utf-8"))
|
| 104 |
+
FIELDS = _contract["fields"] if isinstance(_contract, dict) else _contract
|
| 105 |
+
# text/status/select/date pass through untouched; every OTHER odoo field is numeric -> rounded.
|
| 106 |
+
# Derived from field TYPE (not a hand-kept key list) so a new text/date field can never be
|
| 107 |
+
# wrongly rounded. `select` joined 2026-08-02 (dba) β a choice label rounded would be garbage.
|
| 108 |
+
_PASSTHROUGH_KEYS = {f["key"] for f in FIELDS
|
| 109 |
+
if f["source"] == "odoo" and f["type"] in ("text", "status", "select",
|
| 110 |
+
"date")}
|
| 111 |
+
|
| 112 |
+
app = FastAPI(title="AIOS web API")
|
| 113 |
+
|
| 114 |
+
# The customers payload measured 1.15 MB of JSON on the live Space, shipped UNCOMPRESSED β with
|
| 115 |
+
# the 754 KB bundle behind it, most of "the app is slow" was bytes on the wire. gzip takes the
|
| 116 |
+
# payload to ~10β15% of that. minimum_size spares the tiny acks the overhead.
|
| 117 |
+
app.add_middleware(GZipMiddleware, minimum_size=1024)
|
| 118 |
+
|
| 119 |
+
|
| 120 |
+
@app.exception_handler(StarletteHTTPException)
|
| 121 |
+
def _error_shape(request: Request, exc: StarletteHTTPException):
|
| 122 |
+
"""ONE error shape for every non-2xx (X2): `{"error": {"code", "message"}}`.
|
| 123 |
+
|
| 124 |
+
`deps.err` already raises detail in that shape; anything FastAPI raises on its own (a 404, a
|
| 125 |
+
422 from a malformed path param) is wrapped here so a client never has to branch on two
|
| 126 |
+
different error bodies.
|
| 127 |
+
"""
|
| 128 |
+
detail = exc.detail
|
| 129 |
+
if isinstance(detail, dict) and "error" in detail:
|
| 130 |
+
body = detail
|
| 131 |
+
else:
|
| 132 |
+
body = {"error": {"code": f"http_{exc.status_code}", "message": str(detail)}}
|
| 133 |
+
return JSONResponse(body, status_code=exc.status_code,
|
| 134 |
+
headers=getattr(exc, "headers", None))
|
| 135 |
+
|
| 136 |
+
|
| 137 |
+
@app.exception_handler(grid_events.StoreUnavailable)
|
| 138 |
+
def _store_unavailable(request: Request, exc: grid_events.StoreUnavailable):
|
| 139 |
+
"""THE SAFETY NET for "the store is down" β 503, from anywhere.
|
| 140 |
+
|
| 141 |
+
β WHY THIS IS APP-LEVEL AND NOT A `try` PER ROUTE. It was a try per route first, and a
|
| 142 |
+
`StoreUnavailable` raised while BUILDING THE PAYLOAD β before the route reached its own
|
| 143 |
+
try/except β surfaced as a 500. A store outage is a normal operational state and every route
|
| 144 |
+
here touches the store at least twice (the workspace read, then the write), so "remember to
|
| 145 |
+
wrap it" is a rule that gets forgotten once and then reports the wrong thing. One handler
|
| 146 |
+
means a store outage can only ever be a 503, whichever call raised it.
|
| 147 |
+
|
| 148 |
+
The seam raises this only when the caller passed no `fallback_ws` β i.e. exactly on this
|
| 149 |
+
adapter, which has no durable session dict to degrade into. A 200 over a write that
|
| 150 |
+
evaporated is the failure the whole rule exists to prevent.
|
| 151 |
+
"""
|
| 152 |
+
return JSONResponse(
|
| 153 |
+
{"error": {"code": "store_unavailable",
|
| 154 |
+
"message": "the tenant store is unavailable β no change was saved"}},
|
| 155 |
+
status_code=503)
|
| 156 |
+
|
| 157 |
+
|
| 158 |
+
@app.get("/api/health")
|
| 159 |
+
def health():
|
| 160 |
+
"""Unauthenticated LIVENESS only β it must answer before anyone can sign in, so it may not
|
| 161 |
+
reveal anything about the deployment beyond "the process is up". No version, no tenant list,
|
| 162 |
+
no config: a health endpoint is the one URL every scanner finds first."""
|
| 163 |
+
return {"ok": True}
|
| 164 |
+
|
| 165 |
+
|
| 166 |
+
app.include_router(routes_auth.router)
|
| 167 |
+
app.include_router(routes_nav.router)
|
| 168 |
+
app.include_router(routes_customers.router)
|
| 169 |
+
app.include_router(routes_products.router)
|
| 170 |
+
app.include_router(routes_assets.router)
|
| 171 |
+
app.include_router(routes_tables.router)
|
| 172 |
+
app.include_router(routes_grid.router)
|
| 173 |
+
app.include_router(routes_records.router)
|
| 174 |
+
# EXIT wave 2: the Y1 page-data envelope (one route for every ported dashboard) and Y4's user
|
| 175 |
+
# administration. `routes_pages` imports `pages`, which lazily imports each `pages_*` builder β so
|
| 176 |
+
# a new page is a builder module plus one registry line, and nothing here changes.
|
| 177 |
+
app.include_router(routes_pages.router)
|
| 178 |
+
app.include_router(routes_admin.router)
|
| 179 |
+
app.include_router(routes_automation.router)
|
| 180 |
+
app.include_router(routes_keychain.router)
|
| 181 |
+
|
| 182 |
+
|
| 183 |
+
# --- DEPRECATED ALIASES (removed when S2's shell flips; kept so the current bundle keeps working)
|
| 184 |
+
# They are the v1 handlers with the v1 session requirement β NOT the old unauthenticated Basic
|
| 185 |
+
# behavior. An alias that kept the old auth would be a bypass of everything above it.
|
| 186 |
+
_GATE = module_gate(routes_customers.MODULE)
|
| 187 |
+
|
| 188 |
+
|
| 189 |
+
@app.get("/api/customers", deprecated=True)
|
| 190 |
+
def _customers_alias(session: Session = Depends(_GATE)):
|
| 191 |
+
return routes_customers._payload(session)
|
| 192 |
+
|
| 193 |
+
|
| 194 |
+
@app.patch("/api/customers/{pid}", deprecated=True)
|
| 195 |
+
def _patch_alias(pid: int, body: dict = Body(default=None),
|
| 196 |
+
session: Session = Depends(_GATE)):
|
| 197 |
+
return routes_customers.patch_customer(pid, body, session)
|
| 198 |
+
|
| 199 |
+
|
| 200 |
+
def _assert_contract_floor():
|
| 201 |
+
"""The canonical FIELDS must be exactly what `aios_grid` starts an empty workspace from.
|
| 202 |
+
|
| 203 |
+
This is what stops `FIELDS` becoming a constant that exists only to satisfy a gate. If the
|
| 204 |
+
canonical JSON and `aios_grid.fields_from_workspace({})` ever disagree, the standalone API and
|
| 205 |
+
the embedded host are serving two different schemas and the grid's own contract has forked β
|
| 206 |
+
the exact drift `verify_fields_contract.py` was written to catch, now also caught at startup
|
| 207 |
+
on whatever machine is actually running.
|
| 208 |
+
"""
|
| 209 |
+
import aios_grid
|
| 210 |
+
base = [f for f in aios_grid.fields_from_workspace({}) if not f.get("custom")]
|
| 211 |
+
if [f["key"] for f in base] != [f["key"] for f in FIELDS]:
|
| 212 |
+
raise RuntimeError(
|
| 213 |
+
"AIOS web API: the canonical field contract and aios_grid.fields_from_workspace({}) "
|
| 214 |
+
"disagree on the base field set β embed and standalone would serve different schemas. "
|
| 215 |
+
"Run aios-web/verify_fields_contract.py.")
|
| 216 |
+
|
| 217 |
+
|
| 218 |
+
def _startup_notes():
|
| 219 |
+
"""Say the two things an operator must know, once, at import β never in a response body."""
|
| 220 |
+
if aios_session.EPHEMERAL_SECRET:
|
| 221 |
+
print("[aios-api] AIOS_SESSION_SECRET is not set β signing with a random per-process "
|
| 222 |
+
"key. Sessions will not survive a restart and will not work across workers. "
|
| 223 |
+
"Set it in production.")
|
| 224 |
+
if os.environ.get("AIOS_INSECURE_SSL") == "1":
|
| 225 |
+
print("[aios-api] AIOS_INSECURE_SSL=1 β TLS verification is DISABLED for outbound "
|
| 226 |
+
"requests. Local development only; never in a deployed environment.")
|
| 227 |
+
|
| 228 |
+
|
| 229 |
+
_assert_contract_floor()
|
| 230 |
+
_startup_notes()
|
| 231 |
+
|
| 232 |
+
|
| 233 |
+
# β THE STATIC MOUNT IS NOW UNAUTHENTICATED, and that is a change this wave made on purpose.
|
| 234 |
+
# Before EXIT-3a, `BasicAuth` middleware gated the WHOLE app including this mount. A branded login
|
| 235 |
+
# page cannot live behind a password prompt, so the shell's own assets must be public β which they
|
| 236 |
+
# are: `index.html`, the JS/CSS bundle and the favicon reveal nothing.
|
| 237 |
+
#
|
| 238 |
+
# β WHAT THAT SILENTLY DE-GATED, caught in review rather than in production. `web/dist/` also
|
| 239 |
+
# carries `sample_customers.json`, a DEV FIXTURE copied from `web/public/`. Its 8 customers are
|
| 240 |
+
# synthetic, but the `agent` column holds REAL EMPLOYEE NAMES, and it went from Basic-gated to
|
| 241 |
+
# publicly fetchable in this commit. Nothing needs it: both the API bridge and `useCustomerData`
|
| 242 |
+
# deleted their sample fallback on purpose ("NOTHING HERE FALLS BACK TO sample_customers.json"),
|
| 243 |
+
# and it survives only because it sits in `web/public/`. So it is refused here β 404, the same
|
| 244 |
+
# answer as any other path that is not part of the app.
|
| 245 |
+
#
|
| 246 |
+
# The right long-term fix is deleting it from `web/public/` (S2's lane β flagged in the mailbox);
|
| 247 |
+
# this guard is what makes the API safe regardless of what the bundle happens to contain.
|
| 248 |
+
_DEV_FIXTURES = {"sample_customers.json"}
|
| 249 |
+
|
| 250 |
+
|
| 251 |
+
class _AppStatic(StaticFiles):
|
| 252 |
+
async def get_response(self, path, scope):
|
| 253 |
+
if Path(path).name in _DEV_FIXTURES:
|
| 254 |
+
raise StarletteHTTPException(status_code=404, detail="Not Found")
|
| 255 |
+
resp = await super().get_response(path, scope)
|
| 256 |
+
# Vite content-hashes everything under assets/ (a change is a NEW url), so those are
|
| 257 |
+
# immutable β a repeat visit re-downloads zero bytes instead of the whole 750 KB bundle.
|
| 258 |
+
# index.html must stay revalidated or a deploy would strand returning browsers on the old
|
| 259 |
+
# bundle; ETag/304 makes that revalidation a header exchange, not a transfer.
|
| 260 |
+
# β Normalised first: on a Windows host StaticFiles hands this path with backslashes,
|
| 261 |
+
# and `startswith("assets/")` silently skipped every asset (measured on the local probe).
|
| 262 |
+
if path.replace("\\", "/").lstrip("/").startswith("assets/"):
|
| 263 |
+
resp.headers["Cache-Control"] = "public, max-age=31536000, immutable"
|
| 264 |
+
else:
|
| 265 |
+
resp.headers["Cache-Control"] = "no-cache"
|
| 266 |
+
return resp
|
| 267 |
+
|
| 268 |
+
|
| 269 |
+
# static bundle LAST so /api/* wins; html=True serves index.html at / plus the built assets
|
| 270 |
+
if _WEB_DIST.is_dir():
|
| 271 |
+
app.mount("/", _AppStatic(directory=str(_WEB_DIST), html=True), name="web")
|
| 272 |
+
|
| 273 |
+
|
| 274 |
+
# --- boot prewarm (AIOS_PREWARM=1 β set by the Dockerfile, never by tests) -----------------------
|
| 275 |
+
# Without this, the first visitor after every deploy/restart pays the full Odoo pool build in
|
| 276 |
+
# their request. The thread warms the CONSOLIDATED scope (None, None) + every registered page's
|
| 277 |
+
# default envelope; scoped users still pay their own scope's first build, once.
|
| 278 |
+
# Env-gated rather than a startup event so importing `api.main` in a gate (verify_api and friends
|
| 279 |
+
# run against fakes) can never fire a live Odoo pull.
|
| 280 |
+
def _seed_and_sync_store():
|
| 281 |
+
"""Bring the analytical store (harness.datastore) LIVE for this container β the app.py
|
| 282 |
+
bootstrap, mirrored (2026-07-31, owner item 1).
|
| 283 |
+
|
| 284 |
+
`harness.datastore` powers every measure column/condition, and ONLY app.py used to call
|
| 285 |
+
`ensure_seed()` β so on a fresh Space disk this container resolved measures against a store
|
| 286 |
+
that never existed and every measure cell served blank. Seeding alone was NOT enough either,
|
| 287 |
+
and that was measured live the same day: "datastore seeded in 1.7s" followed by an endless
|
| 288 |
+
`api:measure-column: ModelError the data cache is still warming up` β `ready()` demands
|
| 289 |
+
EVERY entity at phase 'live', and a seed that predates a newer entity leaves it un-synced
|
| 290 |
+
forever in a process with no sync loop. The SYNC SPRINT after the seed is what closes the
|
| 291 |
+
write_date gap and backfills anything the seed lacks (app.py:7779's exact pattern, bounded
|
| 292 |
+
passes). Fail-quiet throughout: no seed/token/Odoo β the columns stay blank, the rows still
|
| 293 |
+
serve.
|
| 294 |
+
"""
|
| 295 |
+
import time as _t
|
| 296 |
+
t0 = _t.time()
|
| 297 |
+
try:
|
| 298 |
+
from harness import datastore as _ds
|
| 299 |
+
if _ds.ensure_seed():
|
| 300 |
+
print(f"[aios-api] datastore seeded in {_t.time() - t0:.1f}s")
|
| 301 |
+
# DEDICATED Odoo connection for this thread (the W6 postmortem rule): the sync's
|
| 302 |
+
# search_reads must never interleave on the shared client's xmlrpc transport.
|
| 303 |
+
import core.odoo as _odoo
|
| 304 |
+
try:
|
| 305 |
+
_odoo._tlocal.client = _odoo.OdooClient()
|
| 306 |
+
except Exception:
|
| 307 |
+
pass
|
| 308 |
+
res = {}
|
| 309 |
+
for i in range(12):
|
| 310 |
+
res = _ds.sync_all(log=lambda *a, **k: None)
|
| 311 |
+
print(f"[aios-api] datastore sync pass {i + 1}: "
|
| 312 |
+
+ ", ".join(f"{k}={v.get('phase')}" for k, v in sorted(res.items())))
|
| 313 |
+
if all(v.get("phase") == "live" for v in res.values()):
|
| 314 |
+
break
|
| 315 |
+
print(f"[aios-api] datastore sync done in {_t.time() - t0:.1f}s "
|
| 316 |
+
f"(ready={_ds.ready()})")
|
| 317 |
+
except Exception as e: # noqa: BLE001
|
| 318 |
+
print(f"[aios-api] datastore seed/sync skipped: {e}")
|
| 319 |
+
|
| 320 |
+
|
| 321 |
+
def _store_resync_loop():
|
| 322 |
+
"""Keep the analytical mirror current β the API-process stand-in for the re-sync the
|
| 323 |
+
Streamlit app piggybacks on page renders. Measure memos key on the pool stamp, so a
|
| 324 |
+
refreshed pool re-reads the freshly synced store."""
|
| 325 |
+
import time as _t
|
| 326 |
+
while True:
|
| 327 |
+
_t.sleep(1800)
|
| 328 |
+
try:
|
| 329 |
+
from harness import datastore as _ds
|
| 330 |
+
import core.odoo as _odoo
|
| 331 |
+
try:
|
| 332 |
+
_odoo._tlocal.client = _odoo.OdooClient()
|
| 333 |
+
except Exception:
|
| 334 |
+
pass
|
| 335 |
+
_ds.sync_all(log=lambda *a, **k: None)
|
| 336 |
+
except Exception as e: # noqa: BLE001
|
| 337 |
+
print(f"[aios-api] store resync failed: {e}")
|
| 338 |
+
|
| 339 |
+
|
| 340 |
+
def _prewarm():
|
| 341 |
+
import time as _t
|
| 342 |
+
t0 = _t.time()
|
| 343 |
+
# β The store sync runs in its OWN thread, never ahead of the pool warm: a stale seed can
|
| 344 |
+
# take many minutes of XML-RPC to close, and the first live probe of this arrangement
|
| 345 |
+
# showed the pool warm (42s) silently queued behind it β the whole app cold for every
|
| 346 |
+
# visitor while a background column channel caught up. Measure cells retry via the
|
| 347 |
+
# transient rule until the sync lands; nothing else waits on it.
|
| 348 |
+
import threading as _th
|
| 349 |
+
_th.Thread(target=_seed_and_sync_store, daemon=True, name="store-seed-sync").start()
|
| 350 |
+
try:
|
| 351 |
+
from harness import runtime as _runtime
|
| 352 |
+
rt = _runtime.get_runtime("royal-imports")
|
| 353 |
+
routes_customers.warm_default(rt)
|
| 354 |
+
import pages as _pages
|
| 355 |
+
_pages.warm_default(rt)
|
| 356 |
+
print(f"[aios-api] prewarm done in {_t.time() - t0:.1f}s")
|
| 357 |
+
except Exception as e: # noqa: BLE001 β boot must not die on a warm-up
|
| 358 |
+
print(f"[aios-api] prewarm skipped: {e}")
|
| 359 |
+
|
| 360 |
+
|
| 361 |
+
if os.environ.get("AIOS_PREWARM") == "1":
|
| 362 |
+
import threading as _threading
|
| 363 |
+
_threading.Thread(target=_prewarm, daemon=True, name="prewarm").start()
|
| 364 |
+
_threading.Thread(target=_store_resync_loop, daemon=True, name="store-resync").start()
|
api/pages.py
CHANGED
|
@@ -30,7 +30,7 @@ not interchangeable: `FMTS` for kpi/table/validation, and viz `FieldType` for a
|
|
| 30 |
"""
|
| 31 |
import time
|
| 32 |
|
| 33 |
-
# `deps` FIRST, deliberately: it is what puts `
|
| 34 |
# ahead of any `core.*` makes this module importable in whatever order a caller reaches it.
|
| 35 |
from deps import Session, err, perms # noqa: I001
|
| 36 |
|
|
|
|
| 30 |
"""
|
| 31 |
import time
|
| 32 |
|
| 33 |
+
# `deps` FIRST, deliberately: it is what puts `platform/` on `sys.path`, so importing it
|
| 34 |
# ahead of any `core.*` makes this module importable in whatever order a caller reaches it.
|
| 35 |
from deps import Session, err, perms # noqa: I001
|
| 36 |
|
api/routes_assets.py
CHANGED
|
@@ -1,165 +1,165 @@
|
|
| 1 |
-
"""routes_assets.py β product image service (wave 18, contract C2-ASSET).
|
| 2 |
-
|
| 3 |
-
Serves the catalog's product imagery. Two sources, checked in order:
|
| 4 |
-
|
| 5 |
-
1. `AIOS_ASSET_DIR` β a local directory of master images whose FILENAME STEM is the product
|
| 6 |
-
`code` (Odoo default_code, case-insensitive). Dev mode: the owner's master folder on this
|
| 7 |
-
machine. Derivatives are generated on demand into a disk cache.
|
| 8 |
-
2. `AIOS_ASSET_REPO` β a private HF dataset (default `royal-imports/product-assets`) laid out
|
| 9 |
-
by `
|
| 10 |
-
assets/products/orig/<CODE><ext> originals, archived verbatim (R9)
|
| 11 |
-
assets/products/web/<CODE>.png ~800px alpha-preserving derivative
|
| 12 |
-
manifest.json {"codes": {"<CODE>": {"ext": ".png"}}, ...}
|
| 13 |
-
Files download once per container into the HF cache; responses carry immutable cache
|
| 14 |
-
headers so the browser re-fetches nothing.
|
| 15 |
-
|
| 16 |
-
QUALITY (R9): `q=web` = the low-res derivative the app displays; `q=print` and `q=orig` BOTH
|
| 17 |
-
serve the ORIGINAL bytes β v1's "high quality" IS the master file (dated amendment in the wave
|
| 18 |
-
doc). The gate is the PRODUCT grant: images decorate the product surface, so whoever may open
|
| 19 |
-
`product_data` may fetch them; nobody else may.
|
| 20 |
-
|
| 21 |
-
Codes are validated against a closed charset before they touch a filesystem or a repo path β
|
| 22 |
-
a path separator can never reach the lookup, so traversal is refused at the door (400), and an
|
| 23 |
-
unknown code is 404 `asset_not_found`, never an empty 200.
|
| 24 |
-
"""
|
| 25 |
-
import os
|
| 26 |
-
import re
|
| 27 |
-
import tempfile
|
| 28 |
-
import time
|
| 29 |
-
from pathlib import Path
|
| 30 |
-
|
| 31 |
-
from fastapi import Depends
|
| 32 |
-
from fastapi import APIRouter
|
| 33 |
-
from fastapi.responses import FileResponse
|
| 34 |
-
|
| 35 |
-
from deps import Session, err, module_gate
|
| 36 |
-
|
| 37 |
-
router = APIRouter(prefix="/api/v1")
|
| 38 |
-
|
| 39 |
-
#: Same grant as the product surface these images decorate (routes_products.MODULE, restated
|
| 40 |
-
#: here to avoid importing that module for one string).
|
| 41 |
-
MODULE = "product_data"
|
| 42 |
-
|
| 43 |
-
_WEB_EDGE = 800 # max edge (px) of the in-app derivative
|
| 44 |
-
_QUALITIES = ("web", "print", "orig")
|
| 45 |
-
_EXT_MEDIA = {".png": "image/png", ".jpg": "image/jpeg", ".jpeg": "image/jpeg"}
|
| 46 |
-
#: closed charset β no slashes, no backslashes, no colons, so a code can never be a path.
|
| 47 |
-
_CODE_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9 ._+&()\-]{0,63}$")
|
| 48 |
-
_IMMUTABLE = "public, max-age=31536000, immutable"
|
| 49 |
-
|
| 50 |
-
#: per-process index of the local master dir: rebuilt when the dir path or its mtime changes.
|
| 51 |
-
_DIR_INDEX = {"path": None, "mtime": None, "map": {}}
|
| 52 |
-
|
| 53 |
-
|
| 54 |
-
def _norm(code):
|
| 55 |
-
return str(code or "").strip().upper()
|
| 56 |
-
|
| 57 |
-
|
| 58 |
-
def _local_dir():
|
| 59 |
-
d = os.environ.get("AIOS_ASSET_DIR", "").strip()
|
| 60 |
-
if not d:
|
| 61 |
-
return None
|
| 62 |
-
p = Path(d)
|
| 63 |
-
return p if p.is_dir() else None
|
| 64 |
-
|
| 65 |
-
|
| 66 |
-
def _cache_root():
|
| 67 |
-
root = Path(os.environ.get("AIOS_ASSET_CACHE", "").strip()
|
| 68 |
-
or Path(tempfile.gettempdir()) / "aios_asset_cache")
|
| 69 |
-
root.mkdir(parents=True, exist_ok=True)
|
| 70 |
-
return root
|
| 71 |
-
|
| 72 |
-
|
| 73 |
-
def _dir_index(d):
|
| 74 |
-
"""{CODE: Path} over the master dir. Two files sharing an upper-cased stem keep the
|
| 75 |
-
lexicographically first β the same rule the ingest script applies, so dir mode and repo
|
| 76 |
-
mode resolve identically."""
|
| 77 |
-
try:
|
| 78 |
-
mtime = d.stat().st_mtime_ns
|
| 79 |
-
except OSError:
|
| 80 |
-
return {}
|
| 81 |
-
if _DIR_INDEX["path"] == str(d) and _DIR_INDEX["mtime"] == mtime:
|
| 82 |
-
return _DIR_INDEX["map"]
|
| 83 |
-
idx = {}
|
| 84 |
-
for p in sorted(d.iterdir()):
|
| 85 |
-
if not p.is_file() or p.name.startswith("."):
|
| 86 |
-
continue
|
| 87 |
-
if p.suffix.lower() not in _EXT_MEDIA:
|
| 88 |
-
continue
|
| 89 |
-
idx.setdefault(p.stem.upper(), p)
|
| 90 |
-
_DIR_INDEX.update(path=str(d), mtime=mtime, map=idx)
|
| 91 |
-
return idx
|
| 92 |
-
|
| 93 |
-
|
| 94 |
-
def _web_derivative(src, code):
|
| 95 |
-
"""The ~800px alpha-preserving PNG for one master file, built lazily into the disk cache.
|
| 96 |
-
Rebuilt when the master is newer than the cached copy."""
|
| 97 |
-
out = _cache_root() / "web"
|
| 98 |
-
out.mkdir(parents=True, exist_ok=True)
|
| 99 |
-
dst = out / f"{code}.png"
|
| 100 |
-
try:
|
| 101 |
-
if dst.is_file() and dst.stat().st_mtime >= src.stat().st_mtime:
|
| 102 |
-
return dst
|
| 103 |
-
except OSError:
|
| 104 |
-
pass
|
| 105 |
-
try:
|
| 106 |
-
from PIL import Image
|
| 107 |
-
except ImportError:
|
| 108 |
-
raise err(503, "pillow_missing",
|
| 109 |
-
"image derivatives need Pillow β `pip install pillow` on this host")
|
| 110 |
-
with Image.open(src) as im:
|
| 111 |
-
im.load()
|
| 112 |
-
if im.mode not in ("RGBA", "RGB", "LA", "L"):
|
| 113 |
-
im = im.convert("RGBA")
|
| 114 |
-
im.thumbnail((_WEB_EDGE, _WEB_EDGE))
|
| 115 |
-
im.save(dst, format="PNG", optimize=True)
|
| 116 |
-
return dst
|
| 117 |
-
|
| 118 |
-
|
| 119 |
-
def _assets():
|
| 120 |
-
"""The asset SOURCE lives in `
|
| 121 |
-
|
| 122 |
-
β NOT TIDINESS β a CONTRACT. `ops/verify_portability.py` B1/B2 assert the HuggingFace SDK
|
| 123 |
-
appears only in its declared homes and NEVER under `aios-web/api/`, because this process is
|
| 124 |
-
meant to lift to Hetzner/EC2 without a hub client. The first version of this file imported
|
| 125 |
-
`huggingface_hub` directly and the gate caught it."""
|
| 126 |
-
import core.assets as assets
|
| 127 |
-
return assets
|
| 128 |
-
|
| 129 |
-
|
| 130 |
-
@router.get("/assets/products")
|
| 131 |
-
def asset_manifest(session: Session = Depends(module_gate(MODULE))):
|
| 132 |
-
"""The codes that HAVE an image β so the catalog designer can mark gaps instead of
|
| 133 |
-
rendering broken tiles. `source` names which backend answered."""
|
| 134 |
-
d = _local_dir()
|
| 135 |
-
if d is not None:
|
| 136 |
-
return {"codes": sorted(_dir_index(d).keys()), "source": "dir"}
|
| 137 |
-
codes = _assets().manifest()
|
| 138 |
-
if codes:
|
| 139 |
-
return {"codes": sorted(codes.keys()), "source": "repo"}
|
| 140 |
-
return {"codes": [], "source": "none"}
|
| 141 |
-
|
| 142 |
-
|
| 143 |
-
@router.get("/assets/products/{code}")
|
| 144 |
-
def product_asset(code: str, q: str = "web",
|
| 145 |
-
session: Session = Depends(module_gate(MODULE))):
|
| 146 |
-
if q not in _QUALITIES:
|
| 147 |
-
raise err(400, "bad_quality", f"q must be one of {', '.join(_QUALITIES)}")
|
| 148 |
-
norm = _norm(code)
|
| 149 |
-
if not _CODE_RE.match(norm):
|
| 150 |
-
raise err(400, "bad_code", "that is not a product code")
|
| 151 |
-
|
| 152 |
-
path = None
|
| 153 |
-
d = _local_dir()
|
| 154 |
-
if d is not None:
|
| 155 |
-
src = _dir_index(d).get(norm)
|
| 156 |
-
if src is not None:
|
| 157 |
-
path = _web_derivative(src, norm) if q == "web" else src
|
| 158 |
-
if path is None:
|
| 159 |
-
path = _assets().fetch(norm, q)
|
| 160 |
-
if path is None or not Path(path).is_file():
|
| 161 |
-
raise err(404, "asset_not_found", f"no image for {norm}")
|
| 162 |
-
|
| 163 |
-
media = _EXT_MEDIA.get(Path(path).suffix.lower(), "application/octet-stream")
|
| 164 |
-
return FileResponse(str(path), media_type=media,
|
| 165 |
-
headers={"Cache-Control": _IMMUTABLE})
|
|
|
|
| 1 |
+
"""routes_assets.py β product image service (wave 18, contract C2-ASSET).
|
| 2 |
+
|
| 3 |
+
Serves the catalog's product imagery. Two sources, checked in order:
|
| 4 |
+
|
| 5 |
+
1. `AIOS_ASSET_DIR` β a local directory of master images whose FILENAME STEM is the product
|
| 6 |
+
`code` (Odoo default_code, case-insensitive). Dev mode: the owner's master folder on this
|
| 7 |
+
machine. Derivatives are generated on demand into a disk cache.
|
| 8 |
+
2. `AIOS_ASSET_REPO` β a private HF dataset (default `royal-imports/product-assets`) laid out
|
| 9 |
+
by `platform/ingest_product_assets.py`:
|
| 10 |
+
assets/products/orig/<CODE><ext> originals, archived verbatim (R9)
|
| 11 |
+
assets/products/web/<CODE>.png ~800px alpha-preserving derivative
|
| 12 |
+
manifest.json {"codes": {"<CODE>": {"ext": ".png"}}, ...}
|
| 13 |
+
Files download once per container into the HF cache; responses carry immutable cache
|
| 14 |
+
headers so the browser re-fetches nothing.
|
| 15 |
+
|
| 16 |
+
QUALITY (R9): `q=web` = the low-res derivative the app displays; `q=print` and `q=orig` BOTH
|
| 17 |
+
serve the ORIGINAL bytes β v1's "high quality" IS the master file (dated amendment in the wave
|
| 18 |
+
doc). The gate is the PRODUCT grant: images decorate the product surface, so whoever may open
|
| 19 |
+
`product_data` may fetch them; nobody else may.
|
| 20 |
+
|
| 21 |
+
Codes are validated against a closed charset before they touch a filesystem or a repo path β
|
| 22 |
+
a path separator can never reach the lookup, so traversal is refused at the door (400), and an
|
| 23 |
+
unknown code is 404 `asset_not_found`, never an empty 200.
|
| 24 |
+
"""
|
| 25 |
+
import os
|
| 26 |
+
import re
|
| 27 |
+
import tempfile
|
| 28 |
+
import time
|
| 29 |
+
from pathlib import Path
|
| 30 |
+
|
| 31 |
+
from fastapi import Depends
|
| 32 |
+
from fastapi import APIRouter
|
| 33 |
+
from fastapi.responses import FileResponse
|
| 34 |
+
|
| 35 |
+
from deps import Session, err, module_gate
|
| 36 |
+
|
| 37 |
+
router = APIRouter(prefix="/api/v1")
|
| 38 |
+
|
| 39 |
+
#: Same grant as the product surface these images decorate (routes_products.MODULE, restated
|
| 40 |
+
#: here to avoid importing that module for one string).
|
| 41 |
+
MODULE = "product_data"
|
| 42 |
+
|
| 43 |
+
_WEB_EDGE = 800 # max edge (px) of the in-app derivative
|
| 44 |
+
_QUALITIES = ("web", "print", "orig")
|
| 45 |
+
_EXT_MEDIA = {".png": "image/png", ".jpg": "image/jpeg", ".jpeg": "image/jpeg"}
|
| 46 |
+
#: closed charset β no slashes, no backslashes, no colons, so a code can never be a path.
|
| 47 |
+
_CODE_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9 ._+&()\-]{0,63}$")
|
| 48 |
+
_IMMUTABLE = "public, max-age=31536000, immutable"
|
| 49 |
+
|
| 50 |
+
#: per-process index of the local master dir: rebuilt when the dir path or its mtime changes.
|
| 51 |
+
_DIR_INDEX = {"path": None, "mtime": None, "map": {}}
|
| 52 |
+
|
| 53 |
+
|
| 54 |
+
def _norm(code):
|
| 55 |
+
return str(code or "").strip().upper()
|
| 56 |
+
|
| 57 |
+
|
| 58 |
+
def _local_dir():
|
| 59 |
+
d = os.environ.get("AIOS_ASSET_DIR", "").strip()
|
| 60 |
+
if not d:
|
| 61 |
+
return None
|
| 62 |
+
p = Path(d)
|
| 63 |
+
return p if p.is_dir() else None
|
| 64 |
+
|
| 65 |
+
|
| 66 |
+
def _cache_root():
|
| 67 |
+
root = Path(os.environ.get("AIOS_ASSET_CACHE", "").strip()
|
| 68 |
+
or Path(tempfile.gettempdir()) / "aios_asset_cache")
|
| 69 |
+
root.mkdir(parents=True, exist_ok=True)
|
| 70 |
+
return root
|
| 71 |
+
|
| 72 |
+
|
| 73 |
+
def _dir_index(d):
|
| 74 |
+
"""{CODE: Path} over the master dir. Two files sharing an upper-cased stem keep the
|
| 75 |
+
lexicographically first β the same rule the ingest script applies, so dir mode and repo
|
| 76 |
+
mode resolve identically."""
|
| 77 |
+
try:
|
| 78 |
+
mtime = d.stat().st_mtime_ns
|
| 79 |
+
except OSError:
|
| 80 |
+
return {}
|
| 81 |
+
if _DIR_INDEX["path"] == str(d) and _DIR_INDEX["mtime"] == mtime:
|
| 82 |
+
return _DIR_INDEX["map"]
|
| 83 |
+
idx = {}
|
| 84 |
+
for p in sorted(d.iterdir()):
|
| 85 |
+
if not p.is_file() or p.name.startswith("."):
|
| 86 |
+
continue
|
| 87 |
+
if p.suffix.lower() not in _EXT_MEDIA:
|
| 88 |
+
continue
|
| 89 |
+
idx.setdefault(p.stem.upper(), p)
|
| 90 |
+
_DIR_INDEX.update(path=str(d), mtime=mtime, map=idx)
|
| 91 |
+
return idx
|
| 92 |
+
|
| 93 |
+
|
| 94 |
+
def _web_derivative(src, code):
|
| 95 |
+
"""The ~800px alpha-preserving PNG for one master file, built lazily into the disk cache.
|
| 96 |
+
Rebuilt when the master is newer than the cached copy."""
|
| 97 |
+
out = _cache_root() / "web"
|
| 98 |
+
out.mkdir(parents=True, exist_ok=True)
|
| 99 |
+
dst = out / f"{code}.png"
|
| 100 |
+
try:
|
| 101 |
+
if dst.is_file() and dst.stat().st_mtime >= src.stat().st_mtime:
|
| 102 |
+
return dst
|
| 103 |
+
except OSError:
|
| 104 |
+
pass
|
| 105 |
+
try:
|
| 106 |
+
from PIL import Image
|
| 107 |
+
except ImportError:
|
| 108 |
+
raise err(503, "pillow_missing",
|
| 109 |
+
"image derivatives need Pillow β `pip install pillow` on this host")
|
| 110 |
+
with Image.open(src) as im:
|
| 111 |
+
im.load()
|
| 112 |
+
if im.mode not in ("RGBA", "RGB", "LA", "L"):
|
| 113 |
+
im = im.convert("RGBA")
|
| 114 |
+
im.thumbnail((_WEB_EDGE, _WEB_EDGE))
|
| 115 |
+
im.save(dst, format="PNG", optimize=True)
|
| 116 |
+
return dst
|
| 117 |
+
|
| 118 |
+
|
| 119 |
+
def _assets():
|
| 120 |
+
"""The asset SOURCE lives in `platform/core/assets.py`, not here.
|
| 121 |
+
|
| 122 |
+
β NOT TIDINESS β a CONTRACT. `ops/verify_portability.py` B1/B2 assert the HuggingFace SDK
|
| 123 |
+
appears only in its declared homes and NEVER under `aios-web/api/`, because this process is
|
| 124 |
+
meant to lift to Hetzner/EC2 without a hub client. The first version of this file imported
|
| 125 |
+
`huggingface_hub` directly and the gate caught it."""
|
| 126 |
+
import core.assets as assets
|
| 127 |
+
return assets
|
| 128 |
+
|
| 129 |
+
|
| 130 |
+
@router.get("/assets/products")
|
| 131 |
+
def asset_manifest(session: Session = Depends(module_gate(MODULE))):
|
| 132 |
+
"""The codes that HAVE an image β so the catalog designer can mark gaps instead of
|
| 133 |
+
rendering broken tiles. `source` names which backend answered."""
|
| 134 |
+
d = _local_dir()
|
| 135 |
+
if d is not None:
|
| 136 |
+
return {"codes": sorted(_dir_index(d).keys()), "source": "dir"}
|
| 137 |
+
codes = _assets().manifest()
|
| 138 |
+
if codes:
|
| 139 |
+
return {"codes": sorted(codes.keys()), "source": "repo"}
|
| 140 |
+
return {"codes": [], "source": "none"}
|
| 141 |
+
|
| 142 |
+
|
| 143 |
+
@router.get("/assets/products/{code}")
|
| 144 |
+
def product_asset(code: str, q: str = "web",
|
| 145 |
+
session: Session = Depends(module_gate(MODULE))):
|
| 146 |
+
if q not in _QUALITIES:
|
| 147 |
+
raise err(400, "bad_quality", f"q must be one of {', '.join(_QUALITIES)}")
|
| 148 |
+
norm = _norm(code)
|
| 149 |
+
if not _CODE_RE.match(norm):
|
| 150 |
+
raise err(400, "bad_code", "that is not a product code")
|
| 151 |
+
|
| 152 |
+
path = None
|
| 153 |
+
d = _local_dir()
|
| 154 |
+
if d is not None:
|
| 155 |
+
src = _dir_index(d).get(norm)
|
| 156 |
+
if src is not None:
|
| 157 |
+
path = _web_derivative(src, norm) if q == "web" else src
|
| 158 |
+
if path is None:
|
| 159 |
+
path = _assets().fetch(norm, q)
|
| 160 |
+
if path is None or not Path(path).is_file():
|
| 161 |
+
raise err(404, "asset_not_found", f"no image for {norm}")
|
| 162 |
+
|
| 163 |
+
media = _EXT_MEDIA.get(Path(path).suffix.lower(), "application/octet-stream")
|
| 164 |
+
return FileResponse(str(path), media_type=media,
|
| 165 |
+
headers={"Cache-Control": _IMMUTABLE})
|
platform/aios_grid.py
ADDED
|
@@ -0,0 +1,1870 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""aios_grid β embed the AIOS React/glide Airtable-style grid inside Streamlit.
|
| 2 |
+
|
| 3 |
+
This is REUSABLE MODULE INFRASTRUCTURE: the same self-contained grid that runs as the
|
| 4 |
+
standalone aios-web app is inlined into a single HTML file and hosted inside a Streamlit
|
| 5 |
+
component. The React bundle reads its data from `window.__AIOS_GRID__` (an object
|
| 6 |
+
`{fields, rows}`) that we inject into the page <head> before the app's module script runs
|
| 7 |
+
β so there is NO backend and NO /api call in the Streamlit container; the browser only ever
|
| 8 |
+
sees the derived JSON we hand it.
|
| 9 |
+
|
| 10 |
+
Usage (from any page):
|
| 11 |
+
import aios_grid
|
| 12 |
+
aios_grid.render(aios_grid.rows_from_pool(pool_rows), aios_grid.FIELDS)
|
| 13 |
+
|
| 14 |
+
Design notes:
|
| 15 |
+
* ZERO app-internal imports (no `modules.*` / `core.*`). FIELDS is a literal and
|
| 16 |
+
`rows_from_pool` takes already-built rows β so this helper is tenant/module-agnostic and
|
| 17 |
+
sidesteps deploy_hf.py's import guard entirely.
|
| 18 |
+
* The built HTML ships as `aios_grid_embed.html` in THIS directory (added to
|
| 19 |
+
deploy_hf.py INCLUDE). Build it with `npm run build:embed` in aios-web/web, then copy
|
| 20 |
+
dist-embed/index.html -> platform/aios_grid_embed.html (see build_embed.py).
|
| 21 |
+
For LOCAL dev before that copy, we fall back to reading dist-embed/index.html directly.
|
| 22 |
+
* The preferred host is a Streamlit Components v1 bridge. It sends data/view/schema args
|
| 23 |
+
into the React app and returns guarded events for persistence in the tenant store. The old
|
| 24 |
+
injected-HTML path remains a read/local-write fallback when component assets are absent.
|
| 25 |
+
"""
|
| 26 |
+
import json
|
| 27 |
+
import re
|
| 28 |
+
from pathlib import Path
|
| 29 |
+
|
| 30 |
+
_HERE = Path(__file__).resolve().parent
|
| 31 |
+
|
| 32 |
+
# Where the inlined single-file build lives. FIRST match wins:
|
| 33 |
+
# 1. the shipped copy in this dir (what deploy_hf.py uploads to the Space)
|
| 34 |
+
# 2. the raw build output in the sibling aios-web tree (local dev, pre-copy)
|
| 35 |
+
_EMBED_CANDIDATES = [
|
| 36 |
+
_HERE / "aios_grid_embed.html",
|
| 37 |
+
_HERE.parent / "aios-web" / "web" / "dist-embed" / "index.html",
|
| 38 |
+
]
|
| 39 |
+
_COMPONENT_CANDIDATES = [
|
| 40 |
+
_HERE / "aios_grid_component",
|
| 41 |
+
_HERE.parent / "aios-web" / "web" / "dist-embed",
|
| 42 |
+
]
|
| 43 |
+
_DECLARED_COMPONENTS = {}
|
| 44 |
+
|
| 45 |
+
# --- the FIELD CONTRACT β loaded from the CANONICAL source `aios_grid_fields.json` in THIS
|
| 46 |
+
# directory (the SINGLE source of truth, shared with aios-web/api/main.py). source='odoo' is
|
| 47 |
+
# READ-ONLY; source='overlay' is the editable stratum (notes/tags) that lives OUTSIDE Odoo.
|
| 48 |
+
#
|
| 49 |
+
# Why a sibling JSON and NOT an import: aios_grid.py's contract is ZERO app-internal imports so
|
| 50 |
+
# it stays tenant/module-agnostic and sidesteps deploy_hf.py's import guard. A JSON next to the
|
| 51 |
+
# module preserves that exactly β no import, no guard interaction β while still single-sourcing
|
| 52 |
+
# the values (a build-time copy from another tree would reintroduce the drift we're removing).
|
| 53 |
+
# The file ships to the Space via deploy_hf.py INCLUDE. Edit the JSON, then run
|
| 54 |
+
# aios-web/verify_fields_contract.py. ---
|
| 55 |
+
_FIELDS_PATH = _HERE / "aios_grid_fields.json"
|
| 56 |
+
|
| 57 |
+
|
| 58 |
+
def _load_fields():
|
| 59 |
+
if not _FIELDS_PATH.is_file():
|
| 60 |
+
raise FileNotFoundError(
|
| 61 |
+
f"aios_grid: canonical field contract missing at {_FIELDS_PATH}. It is the single "
|
| 62 |
+
"source of truth for the grid schema and MUST ship (deploy_hf.py INCLUDE lists it)."
|
| 63 |
+
)
|
| 64 |
+
doc = json.loads(_FIELDS_PATH.read_text(encoding="utf-8"))
|
| 65 |
+
fields = doc.get("fields") if isinstance(doc, dict) else doc
|
| 66 |
+
if not isinstance(fields, list) or not fields:
|
| 67 |
+
raise ValueError(f"aios_grid: {_FIELDS_PATH} has no 'fields' list.")
|
| 68 |
+
return fields
|
| 69 |
+
|
| 70 |
+
|
| 71 |
+
FIELDS = _load_fields()
|
| 72 |
+
|
| 73 |
+
|
| 74 |
+
def product_fields():
|
| 75 |
+
"""The PRODUCT table's canonical contract (wave 15 C-TOPIC, `product_data` key in the same
|
| 76 |
+
JSON). A separate accessor rather than a second module constant so the one file-read and the
|
| 77 |
+
one failure mode stay shared with `FIELDS`."""
|
| 78 |
+
doc = json.loads(_FIELDS_PATH.read_text(encoding="utf-8"))
|
| 79 |
+
fields = (doc.get("product_data") or {}).get("fields")
|
| 80 |
+
if not isinstance(fields, list) or not fields:
|
| 81 |
+
raise ValueError(f"aios_grid: {_FIELDS_PATH} has no product_data.fields list.")
|
| 82 |
+
return fields
|
| 83 |
+
|
| 84 |
+
# text/date fields pass through untouched; every OTHER odoo (numeric) field is rounded β
|
| 85 |
+
# mirrors aios-web/api/main.py _payload() exactly so embed == standalone byte-for-byte.
|
| 86 |
+
|
| 87 |
+
|
| 88 |
+
def _round(v):
|
| 89 |
+
return round(v) if isinstance(v, (int, float)) and not isinstance(v, bool) else v
|
| 90 |
+
|
| 91 |
+
|
| 92 |
+
# Types a USER may create from the column menu (owner item 7, 2026-07-26). Mirrors
|
| 93 |
+
# customer-grid/types.ts CREATABLE_TYPES; verify_fields_contract.py holds the two in step.
|
| 94 |
+
# select β a single-select with its own `options` (the "Status" a user wants to add; distinct
|
| 95 |
+
# from the Odoo-sourced `status` lifecycle, which is not user-defined)
|
| 96 |
+
# user β an assignee, whose choices come from the HOST's real user list, never from here
|
| 97 |
+
#: `multiselect` (wave-2 item 5, 2026-07-27): the Airtable-style MULTI select β declared
|
| 98 |
+
#: options like `select`, but the cell holds a comma-joined SET and the row belongs to every
|
| 99 |
+
#: member of it (the `multi` grouping/cell contract the Cohorts column established).
|
| 100 |
+
#: Wave-5 item 11 (2026-07-27): `checkbox` (cell = bool; the overlay stores '1' or '') Β·
|
| 101 |
+
#: `phone` / `email` / `url` (text-family with per-type rendering) Β· `rating` (top-level
|
| 102 |
+
#: `max`, SVG stars client-side β never emoji) Β· `created_time` (READ-ONLY, renders the row's
|
| 103 |
+
#: `_created`) Β· `formula` (READ-ONLY, client-computed from the row's other cells).
|
| 104 |
+
CUSTOM_FIELD_TYPES = {"text", "select", "multiselect", "user", "int", "currency", "pct", "date",
|
| 105 |
+
"checkbox", "phone", "email", "url", "rating", "created_time", "formula",
|
| 106 |
+
"automation"}
|
| 107 |
+
#: User-created types whose CELLS are read-only: their values are computed (formula β client
|
| 108 |
+
#: side, any error degrades to BLANK) or system-owned (created_time = the row's `_created`).
|
| 109 |
+
#: Emitted with the cohort column's read-only mechanism β `source: 'odoo'` + `derived` β so
|
| 110 |
+
#: the client never offers an editor and the host never accepts a cell write for them.
|
| 111 |
+
READONLY_CUSTOM_TYPES = {"created_time", "formula"}
|
| 112 |
+
MAX_FIELD_OPTIONS = 50
|
| 113 |
+
MAX_FORMULA_LEN = 500
|
| 114 |
+
#: `rating` bounds. Airtable caps at 10; below 2 a rating is a checkbox.
|
| 115 |
+
RATING_MAX_DEFAULT, RATING_MAX_MIN, RATING_MAX_MAX = 5, 2, 10
|
| 116 |
+
|
| 117 |
+
#: Key prefix of a FORMULA-MEASURE column (owner item 7, 2026-07-27): a user-created field that
|
| 118 |
+
#: IS a measure over a window β `Sales Β· the last 90 days` as a column. Mirrors the client's
|
| 119 |
+
#: `measure_` keys in CustomerGrid.createField. Distinct from `custom_` because the two strata
|
| 120 |
+
#: could not be more different: `custom_` is the EDITABLE overlay (user-typed values), while a
|
| 121 |
+
#: measure field is READ-ONLY and its values are computed by the host per render.
|
| 122 |
+
MEASURE_FIELD_PREFIX = "measure_"
|
| 123 |
+
#: The numeric types a measure can render as (semantic._FORMAT_TYPE's range).
|
| 124 |
+
MEASURE_FIELD_TYPES = {"currency", "int", "pct"}
|
| 125 |
+
|
| 126 |
+
|
| 127 |
+
def _clean_options(raw):
|
| 128 |
+
"""Choices for a `select`: strings, trimmed, de-duplicated case-insensitively, capped.
|
| 129 |
+
Mirrors types.ts parseOptions β a choice list that means one thing in the picker and
|
| 130 |
+
another in the filter dropdown is a column with two vocabularies."""
|
| 131 |
+
out, seen = [], set()
|
| 132 |
+
for v in list(raw or [])[:MAX_FIELD_OPTIONS * 2]:
|
| 133 |
+
if not isinstance(v, (str, int, float)) or isinstance(v, bool):
|
| 134 |
+
continue
|
| 135 |
+
s = str(v).strip()[:120]
|
| 136 |
+
if not s or s.lower() in seen:
|
| 137 |
+
continue
|
| 138 |
+
seen.add(s.lower())
|
| 139 |
+
out.append(s)
|
| 140 |
+
return out[:MAX_FIELD_OPTIONS]
|
| 141 |
+
|
| 142 |
+
|
| 143 |
+
def _clean_option_colors(raw, options):
|
| 144 |
+
"""Choice-label -> #RRGGBB, limited to the field's canonical option vocabulary."""
|
| 145 |
+
if not isinstance(raw, dict):
|
| 146 |
+
return {}
|
| 147 |
+
supplied = {}
|
| 148 |
+
for label, color in raw.items():
|
| 149 |
+
if not isinstance(label, str) or not isinstance(color, str):
|
| 150 |
+
continue
|
| 151 |
+
clean = color.strip().upper()
|
| 152 |
+
if re.fullmatch(r"#[0-9A-F]{6}", clean):
|
| 153 |
+
supplied[label.strip().lower()] = clean
|
| 154 |
+
out = {}
|
| 155 |
+
for option in options or []:
|
| 156 |
+
color = supplied.get(str(option).strip().lower())
|
| 157 |
+
if color:
|
| 158 |
+
out[str(option)] = color
|
| 159 |
+
return out
|
| 160 |
+
|
| 161 |
+
|
| 162 |
+
def _choice_appearance(raw, options):
|
| 163 |
+
"""Validated select-family appearance. Absent colour toggle means legacy-on."""
|
| 164 |
+
if not isinstance(raw, dict):
|
| 165 |
+
return {}
|
| 166 |
+
out = {}
|
| 167 |
+
if isinstance(raw.get("colorCodeOptions"), bool):
|
| 168 |
+
out["colorCodeOptions"] = raw["colorCodeOptions"]
|
| 169 |
+
colors = _clean_option_colors(raw.get("optionColors"), options)
|
| 170 |
+
if colors:
|
| 171 |
+
out["optionColors"] = colors
|
| 172 |
+
return out
|
| 173 |
+
|
| 174 |
+
|
| 175 |
+
def _clean_rating_max(raw):
|
| 176 |
+
"""A rating's star count, bounded. Anything unusable is the default, not a refusal β the
|
| 177 |
+
field still holds its 1..max integers either way."""
|
| 178 |
+
try:
|
| 179 |
+
return max(RATING_MAX_MIN, min(int(raw), RATING_MAX_MAX))
|
| 180 |
+
except (TypeError, ValueError):
|
| 181 |
+
return RATING_MAX_DEFAULT
|
| 182 |
+
|
| 183 |
+
|
| 184 |
+
#: The field types a number-style display format may apply to. `formula` is here because its
|
| 185 |
+
#: RESULT is a number the client renders; `pct` already renders in points and takes decimals.
|
| 186 |
+
_NUMBER_FORMAT_TYPES = {"int", "currency", "pct", "formula"}
|
| 187 |
+
|
| 188 |
+
|
| 189 |
+
def _clean_format(raw, ftype):
|
| 190 |
+
"""Per-type DISPLAY format (wave-5 item 10), fail-closed: unknown keys are DROPPED, wrong
|
| 191 |
+
types return None (the property is simply absent). Rendering-only β a format can change how
|
| 192 |
+
a value reads, never what it is, which is why this needs no parity gate of its own."""
|
| 193 |
+
if not isinstance(raw, dict):
|
| 194 |
+
return None
|
| 195 |
+
out = {}
|
| 196 |
+
if ftype in _NUMBER_FORMAT_TYPES:
|
| 197 |
+
if isinstance(raw.get("thousands"), bool):
|
| 198 |
+
out["thousands"] = raw["thousands"]
|
| 199 |
+
if raw.get("decimals") is not None:
|
| 200 |
+
try:
|
| 201 |
+
d = int(raw["decimals"])
|
| 202 |
+
except (TypeError, ValueError):
|
| 203 |
+
d = None
|
| 204 |
+
if d is not None and 0 <= d <= 4:
|
| 205 |
+
out["decimals"] = d
|
| 206 |
+
if isinstance(raw.get("abbrev"), bool):
|
| 207 |
+
out["abbrev"] = raw["abbrev"]
|
| 208 |
+
elif ftype in ("date", "created_time"):
|
| 209 |
+
if isinstance(raw.get("time"), bool):
|
| 210 |
+
out["time"] = raw["time"]
|
| 211 |
+
if raw.get("tz") in ("local", "utc"):
|
| 212 |
+
out["tz"] = raw["tz"]
|
| 213 |
+
return out or None
|
| 214 |
+
|
| 215 |
+
|
| 216 |
+
def _clean_permissions(raw):
|
| 217 |
+
"""`{edit: 'everyone' | 'creator' | 'admins'}` or None (wave-5 item 1). WHO may set it is
|
| 218 |
+
the host handler's business (creator/admin, enforced fail-closed there); this validates
|
| 219 |
+
only the shape, like every other property here."""
|
| 220 |
+
if isinstance(raw, dict) and raw.get("edit") in ("everyone", "creator", "admins"):
|
| 221 |
+
return {"edit": raw["edit"]}
|
| 222 |
+
return None
|
| 223 |
+
|
| 224 |
+
|
| 225 |
+
#: β THE CHARSET MUST ADMIT EVERY TOKEN THE CLIENT ENGINE PARSES, or a legal formula is
|
| 226 |
+
#: refused by a wall that is supposed to be structural (2026-08-03).
|
| 227 |
+
#:
|
| 228 |
+
#: This regex was written when a formula was arithmetic over refs. On 2026-07-31 the client
|
| 229 |
+
#: engine (owner item 2) gained STRING literals, `&` concatenation and `^` β CONCATENATE, TEXT,
|
| 230 |
+
#: LEFT/RIGHT/MID, and any `IF(cond, "yes", "no")`. This list was never widened to match, so
|
| 231 |
+
#: every such formula died here: `field_upsert` refused the create, and `fields_from_workspace`
|
| 232 |
+
#: dropped the column on read. Nothing went red β a refused create looks like a quiet failure
|
| 233 |
+
#: and a dropped column looks like a column nobody made.
|
| 234 |
+
#:
|
| 235 |
+
#: Found by trying to ship the owner's own Buy signal formula, which is `IF(..., "Buy now",
|
| 236 |
+
#: "OK")` and could not be created through the product UI at all.
|
| 237 |
+
#:
|
| 238 |
+
#: β IT IS STILL STRUCTURAL, and deliberately not a second grammar β that is the filter_sql-class
|
| 239 |
+
#: drift risk this file's docstring names. The charset says which characters may appear; the
|
| 240 |
+
#: engine says what they mean. `_quotes_balanced` below is the one structural rule the quote
|
| 241 |
+
#: character brings with it.
|
| 242 |
+
_FORMULA_CHARS = re.compile(r"^[\w\s{}()+\-*/.,<>=!^&\"]*$")
|
| 243 |
+
_FORMULA_REF = re.compile(r"\{([^{}]*)\}")
|
| 244 |
+
|
| 245 |
+
|
| 246 |
+
def _quotes_balanced(s):
|
| 247 |
+
"""An even number of `"` β the structural half of string support.
|
| 248 |
+
|
| 249 |
+
Sound because the engine's own escape is Excel's: `""` inside a string is one quote, and it
|
| 250 |
+
contributes TWO characters. So a well-formed expression always has an even count and an
|
| 251 |
+
unterminated string always has an odd one. What a balanced pair MEANS is the engine's
|
| 252 |
+
business, exactly as with parentheses.
|
| 253 |
+
"""
|
| 254 |
+
return s.count('"') % 2 == 0
|
| 255 |
+
|
| 256 |
+
|
| 257 |
+
#: Wave-18 C5-AUTOFIELD. `kind` is a whitelist because an unknown kind would be a column that
|
| 258 |
+
#: silently never runs; `source` names where the run's subject comes from.
|
| 259 |
+
AUTOMATION_KINDS = {"instagram_profile"}
|
| 260 |
+
AUTOMATION_SOURCES = {"record_url_field", "self"}
|
| 261 |
+
MAX_AUTOMATION_SETTINGS = 12
|
| 262 |
+
|
| 263 |
+
|
| 264 |
+
def _clean_automation(raw, valid_keys=None):
|
| 265 |
+
"""Validate an `automation` config bag. Returns the clean dict, or None when there is
|
| 266 |
+
nothing valid to store (the column then renders as unconfigured β never invented)."""
|
| 267 |
+
if not isinstance(raw, dict):
|
| 268 |
+
return None
|
| 269 |
+
kind = str(raw.get('kind') or '').strip()
|
| 270 |
+
if kind not in AUTOMATION_KINDS:
|
| 271 |
+
return None
|
| 272 |
+
source = str(raw.get('source') or 'record_url_field').strip()
|
| 273 |
+
if source not in AUTOMATION_SOURCES:
|
| 274 |
+
source = 'record_url_field'
|
| 275 |
+
out = {'kind': kind, 'source': source}
|
| 276 |
+
url_field = str(raw.get('urlField') or '').strip()[:80]
|
| 277 |
+
# fail closed on a ref that does not exist, exactly as _clean_formula does at WRITE time
|
| 278 |
+
if url_field and (valid_keys is None or url_field in valid_keys):
|
| 279 |
+
out['urlField'] = url_field
|
| 280 |
+
settings = {}
|
| 281 |
+
for k, v in list((raw.get('settings') or {}).items())[:MAX_AUTOMATION_SETTINGS]:
|
| 282 |
+
if isinstance(v, bool) or isinstance(v, (int, float)):
|
| 283 |
+
settings[str(k)[:40]] = v
|
| 284 |
+
elif isinstance(v, str):
|
| 285 |
+
settings[str(k)[:40]] = v[:200]
|
| 286 |
+
if settings:
|
| 287 |
+
out['settings'] = settings
|
| 288 |
+
return out
|
| 289 |
+
|
| 290 |
+
|
| 291 |
+
def _clean_formula(raw, valid_keys=None):
|
| 292 |
+
"""STRUCTURAL passthrough for a formula field's expression (wave-5 item 9).
|
| 293 |
+
|
| 294 |
+
Meaning is NOT checked here: the CLIENT engine owns the grammar (arithmetic over `{field}`
|
| 295 |
+
refs, ABS/ROUND/MIN/MAX/IF), and any evaluation error degrades to a BLANK cell β never a
|
| 296 |
+
wrong number. That is the `_clean_window` split one stratum up, and deliberately NOT a
|
| 297 |
+
Python mirror of the grammar: a second engine is the filter_sql-class drift risk.
|
| 298 |
+
Structure IS checked β charset, length, balanced parens, BALANCED QUOTES, well-formed
|
| 299 |
+
non-empty `{refs}` β and at WRITE time (`valid_keys` given) every ref must name a field this
|
| 300 |
+
table has, fail closed. At READ time refs are left alone: a referenced field deleted later
|
| 301 |
+
must blank the CELLS, not vaporise the column.
|
| 302 |
+
|
| 303 |
+
β A `{ref}` INSIDE A STRING LITERAL is still checked against `valid_keys` at write time, so
|
| 304 |
+
`IF(x, "see {notafield}", "")` is refused. That is a false rejection and it is the
|
| 305 |
+
fail-closed direction: the alternative is teaching this function where strings begin and
|
| 306 |
+
end, which is the second grammar the paragraph above refuses to write.
|
| 307 |
+
"""
|
| 308 |
+
if not isinstance(raw, str):
|
| 309 |
+
return None
|
| 310 |
+
s = raw.strip()
|
| 311 |
+
if not s or len(s) > MAX_FORMULA_LEN or not _FORMULA_CHARS.match(s):
|
| 312 |
+
return None
|
| 313 |
+
depth = 0
|
| 314 |
+
for ch in s:
|
| 315 |
+
if ch == "(":
|
| 316 |
+
depth += 1
|
| 317 |
+
elif ch == ")":
|
| 318 |
+
depth -= 1
|
| 319 |
+
if depth < 0:
|
| 320 |
+
return None
|
| 321 |
+
if depth:
|
| 322 |
+
return None
|
| 323 |
+
if not _quotes_balanced(s):
|
| 324 |
+
return None
|
| 325 |
+
refs = _FORMULA_REF.findall(s)
|
| 326 |
+
leftover = _FORMULA_REF.sub("", s)
|
| 327 |
+
if "{" in leftover or "}" in leftover: # unbalanced / nested braces
|
| 328 |
+
return None
|
| 329 |
+
if any(not r.strip() for r in refs): # a `{}` ref names nothing
|
| 330 |
+
return None
|
| 331 |
+
if valid_keys is not None and any(r not in valid_keys for r in refs):
|
| 332 |
+
return None
|
| 333 |
+
return s
|
| 334 |
+
|
| 335 |
+
|
| 336 |
+
def _field_extras(saved, ftype):
|
| 337 |
+
"""createdBy / permissions / format / scope β the validated passthrough the created strata
|
| 338 |
+
share (wave 5). `createdBy` is only ever WRITTEN host-side (the handler stamps it); here it
|
| 339 |
+
is carried so the client can gate its menus and the handler can enforce against it.
|
| 340 |
+
`scope` (wave-6 item 9): 'cohort' marks a field created as cohort-specific β the handler
|
| 341 |
+
stamps it at create (cohort page only) and preserves it like createdBy; carried here so the
|
| 342 |
+
client can label the field, filtered OUT of other pages by fields_from_workspace."""
|
| 343 |
+
out = {}
|
| 344 |
+
who = saved.get("createdBy")
|
| 345 |
+
if isinstance(who, str) and who.strip():
|
| 346 |
+
out["createdBy"] = who.strip()[:80]
|
| 347 |
+
perms = _clean_permissions(saved.get("permissions"))
|
| 348 |
+
if perms:
|
| 349 |
+
out["permissions"] = perms
|
| 350 |
+
fmt = _clean_format(saved.get("format"), ftype)
|
| 351 |
+
if fmt:
|
| 352 |
+
out["format"] = fmt
|
| 353 |
+
if saved.get("scope") == "cohort":
|
| 354 |
+
out["scope"] = "cohort"
|
| 355 |
+
corrected_from = saved.get("labelCorrectedFrom")
|
| 356 |
+
correction_id = saved.get("labelCorrectionId")
|
| 357 |
+
if (isinstance(corrected_from, str) and corrected_from.strip()
|
| 358 |
+
and isinstance(correction_id, str) and correction_id.strip()):
|
| 359 |
+
out["labelCorrectedFrom"] = corrected_from.strip()[:120]
|
| 360 |
+
out["labelCorrectionId"] = correction_id.strip()[:180]
|
| 361 |
+
return out
|
| 362 |
+
|
| 363 |
+
|
| 364 |
+
#: The DERIVED column listing the cohorts a customer is in (owner, 2026-07-27).
|
| 365 |
+
#:
|
| 366 |
+
#: NOT in `aios_grid_fields.json`, deliberately. That contract is per-TABLE and shared with the
|
| 367 |
+
#: standalone API and the dev sample; a cohort is per-USER, so the column exists exactly when the
|
| 368 |
+
#: caller has cohorts β the same condition under which the `__cohort__` FILTER field is offered.
|
| 369 |
+
#: Putting it in the canonical contract would mean an always-present column that is empty for
|
| 370 |
+
#: everyone else, plus three consumers to keep in step for a value none of them can produce.
|
| 371 |
+
COHORT_COLUMN = 'cohorts'
|
| 372 |
+
|
| 373 |
+
|
| 374 |
+
def cohort_field(label='Locked views'):
|
| 375 |
+
"""The derived membership column's descriptor.
|
| 376 |
+
|
| 377 |
+
β WAVE 17 item 14 / C-STR β THE LABEL AND THE NOTE SPEAK THE NEW VOCABULARY; THE KEY DOES
|
| 378 |
+
NOT. `COHORT_COLUMN` is still `'cohorts'` and the function is still `cohort_field`, because
|
| 379 |
+
every stored view that shows or groups by this column names it by KEY, and every gate in
|
| 380 |
+
two runtimes names the function. The owner renamed a CONCEPT ("we should stop calling it
|
| 381 |
+
Cohort, but locked instead"), which is a change to what a reader sees β renaming the
|
| 382 |
+
identifiers would break saved views to change a word nobody reads.
|
| 383 |
+
|
| 384 |
+
`source: 'odoo'` is doing ONE job here and it is not provenance: the client keys editability
|
| 385 |
+
off `source == 'overlay'`, and `_cl_handle_grid_event` accepts cell writes only for overlay
|
| 386 |
+
keys. So 'odoo' is what makes this column READ-ONLY at both ends. `derived: True` is what
|
| 387 |
+
stops the column menu calling it a "source field" on that basis.
|
| 388 |
+
|
| 389 |
+
β `filterable: False`, and the replacement is the `Where [Cohort] [β¦]` CONDITION, not another
|
| 390 |
+
column. Text ops over a joined string would ALMOST work and disagree at the edges β `contains
|
| 391 |
+
"VIP"` also matches a cohort called "VIP club" β and a filter that is nearly right is worse
|
| 392 |
+
than one that is absent.
|
| 393 |
+
"""
|
| 394 |
+
return {
|
| 395 |
+
'key': COHORT_COLUMN, 'label': label, 'type': 'text', 'source': 'odoo',
|
| 396 |
+
'default': False, 'filterable': False, 'derived': True, 'multi': True,
|
| 397 |
+
'note': 'The locked views this customer is in, newest first. A locked view holds a SET, '
|
| 398 |
+
'so grouping by this column puts a customer under EVERY view they are in β the '
|
| 399 |
+
'group counts therefore add up to more than the record count, which stays the '
|
| 400 |
+
'number of distinct customers. Read-only: membership changes only by adding or '
|
| 401 |
+
'removing customers on the locked view itself.',
|
| 402 |
+
}
|
| 403 |
+
|
| 404 |
+
|
| 405 |
+
def cohort_cells(cohorts, allowed_pids=None):
|
| 406 |
+
"""`{pid: {'cohorts': 'Q3 calls, Lost'}}` from `[{id,name,pids}, ...]`.
|
| 407 |
+
|
| 408 |
+
Built per render from the caller's OWN cohorts and handed to `rows_from_pool` as `derived`,
|
| 409 |
+
never merged into the cached pool rows β those are shared across users, and stamping one
|
| 410 |
+
user's cohorts onto them would leak the membership to everybody else on the next render.
|
| 411 |
+
"""
|
| 412 |
+
cells = {}
|
| 413 |
+
for c in cohorts or []:
|
| 414 |
+
# β The COMMA is the separator the client splits on to group a customer into EVERY
|
| 415 |
+
# cohort they are in, so it cannot also occur inside a name. Cohort names are free text
|
| 416 |
+
# ("Q3 calls, west" is a name somebody will type), so a comma is replaced here rather
|
| 417 |
+
# than left to break the split silently β one group called "Q3 calls" and another called
|
| 418 |
+
# "west" would be two lists that do not exist. The cost is cosmetic and confined to the
|
| 419 |
+
# cell; the Cohort page still shows the name the user typed.
|
| 420 |
+
name = str(c.get('name') or c.get('id') or '').replace(',', ' ').strip()
|
| 421 |
+
for pid in c.get('pids') or ():
|
| 422 |
+
if allowed_pids is not None and pid not in allowed_pids:
|
| 423 |
+
continue
|
| 424 |
+
cells.setdefault(pid, []).append(name)
|
| 425 |
+
return {pid: {COHORT_COLUMN: ', '.join(names)} for pid, names in cells.items()}
|
| 426 |
+
|
| 427 |
+
|
| 428 |
+
def clean_measure_field(raw, offered):
|
| 429 |
+
"""Validate one UNTRUSTED formula-measure field (owner item 7) against the caller's OFFER.
|
| 430 |
+
|
| 431 |
+
`offered` is `{measure key: {label, type}}` from `measure_filter.measure_fields(team_id)` β
|
| 432 |
+
the same admission the condition builder uses, so a field can only name a measure this
|
| 433 |
+
caller could also filter by. Returns the canonical stored shape, or None (fail closed).
|
| 434 |
+
|
| 435 |
+
`source:'odoo'` + `derived:True` is the cohort column's read-only mechanism, reused:
|
| 436 |
+
the client keys editability off `source == 'overlay'` and the host accepts cell writes only
|
| 437 |
+
for overlay keys, so a measure column cannot be typed into at either end. `filterable:False`
|
| 438 |
+
because the REPLACEMENT is the measure CONDITION with the same measure and window β the
|
| 439 |
+
governed, gate-proved path (CG-8/CG-12) β not text ops over a derived cell.
|
| 440 |
+
"""
|
| 441 |
+
if not isinstance(raw, dict):
|
| 442 |
+
return None
|
| 443 |
+
key = str(raw.get("key") or "")
|
| 444 |
+
if not key.startswith(MEASURE_FIELD_PREFIX) or len(key) > 80:
|
| 445 |
+
return None
|
| 446 |
+
spec = raw.get("measure")
|
| 447 |
+
if not isinstance(spec, dict):
|
| 448 |
+
return None
|
| 449 |
+
m = (offered or {}).get(spec.get("key"))
|
| 450 |
+
if not m:
|
| 451 |
+
return None # not admitted for this caller -> fail closed
|
| 452 |
+
window = _clean_window(spec.get("window"))
|
| 453 |
+
if window is None:
|
| 454 |
+
return None # a measure column with no period is not a column
|
| 455 |
+
mtype = m.get("type") if m.get("type") in MEASURE_FIELD_TYPES else "currency"
|
| 456 |
+
return {
|
| 457 |
+
"key": key,
|
| 458 |
+
"label": str(raw.get("label") or m.get("label") or "Measure")[:120],
|
| 459 |
+
"type": mtype,
|
| 460 |
+
"source": "odoo",
|
| 461 |
+
"default": True,
|
| 462 |
+
"custom": True,
|
| 463 |
+
"derived": True,
|
| 464 |
+
"filterable": False,
|
| 465 |
+
"agg": "sum" if mtype in ("currency", "int") else None,
|
| 466 |
+
"note": str(raw.get("note") or "")[:2000],
|
| 467 |
+
"measure": {"key": str(spec.get("key"))[:80], "window": window},
|
| 468 |
+
}
|
| 469 |
+
|
| 470 |
+
|
| 471 |
+
def measure_fields_of(fields):
|
| 472 |
+
"""The formula-measure columns among `fields` β the ones whose values the host must compute
|
| 473 |
+
per render (see `rows_from_pool`'s `derived`)."""
|
| 474 |
+
return [f for f in fields or [] if isinstance(f.get("measure"), dict)]
|
| 475 |
+
|
| 476 |
+
|
| 477 |
+
def fields_from_workspace(workspace=None, cohorts=False, scope_key=None, fields_base=None):
|
| 478 |
+
"""Overlay persisted notes/custom fields onto the immutable source-field contract.
|
| 479 |
+
|
| 480 |
+
`cohorts=True` appends the derived cohort column β see `cohort_field`. Appended LAST and
|
| 481 |
+
`default: False`, so it never displaces a column somebody already reads; the Fields menu is
|
| 482 |
+
where you turn it on.
|
| 483 |
+
|
| 484 |
+
Three saved strata pass through: notes on base fields, `custom_` overlay fields (editable),
|
| 485 |
+
and `measure_` formula-measure fields (read-only, host-computed β owner item 7). A measure
|
| 486 |
+
field was validated against the caller's measure OFFER when it was written
|
| 487 |
+
(`clean_measure_field`); here only its SHAPE is re-checked, because this module has zero
|
| 488 |
+
app-internal imports and cannot know the offer. A measure that has since become
|
| 489 |
+
unanswerable (a BU scope on a company-level measure) degrades to a BLANK column at value
|
| 490 |
+
time, never to an error.
|
| 491 |
+
|
| 492 |
+
`scope_key` (wave-6 item 9) names the PAGE doing the asking ('customer' / 'cohort'). A
|
| 493 |
+
saved def carrying `scope` is emitted only when it matches β so a cohort-specific field
|
| 494 |
+
never appears on the Customer table. FAIL-CLOSED: a caller that passes no scope_key sees
|
| 495 |
+
only unscoped (global) fields; base contract fields are never scoped.
|
| 496 |
+
|
| 497 |
+
`fields_base` (wave 16 C-TOPIC) β the canonical contract to overlay onto. Absent = the
|
| 498 |
+
CUSTOMER contract (`FIELDS`), byte-identical to before the parameter existed; the product
|
| 499 |
+
surface passes `product_fields()`. The workspace dict a caller hands in must already be the
|
| 500 |
+
matching table object's bucket β this function cannot tell a customer overlay from a
|
| 501 |
+
product one, which is exactly why the buckets are separate stores.
|
| 502 |
+
"""
|
| 503 |
+
saved = dict((workspace or {}).get("fields") or {})
|
| 504 |
+
out = []
|
| 505 |
+
base_fields = fields_base if fields_base is not None else FIELDS
|
| 506 |
+
base_keys = {field["key"] for field in base_fields}
|
| 507 |
+
for base in base_fields:
|
| 508 |
+
meta = saved.get(base["key"]) or {}
|
| 509 |
+
field = dict(base)
|
| 510 |
+
if isinstance(meta.get("note"), str):
|
| 511 |
+
field["note"] = meta["note"][:2000]
|
| 512 |
+
# Wave-5 item 10: a saved DISPLAY format on any base field (a preset included) β how a
|
| 513 |
+
# number or date READS, per user. Rendering-only, so this is the whole acceptance.
|
| 514 |
+
fmt = _clean_format(meta.get("format"), base.get("type"))
|
| 515 |
+
if fmt:
|
| 516 |
+
field["format"] = fmt
|
| 517 |
+
# PRESET measure fields (wave-2 item 8): a preset+measure base may take a saved
|
| 518 |
+
# window/label override. Wave 6 deleted every preset member (the owner's
|
| 519 |
+
# no-buildable-presets rule) so this branch is currently MEMBERLESS β kept as the
|
| 520 |
+
# measure_ path's twin for any future preset-carrying contract, and because deleting
|
| 521 |
+
# it would silently change what a re-added preset means.
|
| 522 |
+
if base.get("preset") and isinstance(base.get("measure"), dict):
|
| 523 |
+
saved_measure = meta.get("measure") if isinstance(meta.get("measure"), dict) else {}
|
| 524 |
+
window = _clean_window(saved_measure.get("window"))
|
| 525 |
+
if window is not None:
|
| 526 |
+
field["measure"] = {"key": base["measure"]["key"], "window": window}
|
| 527 |
+
if isinstance(meta.get("label"), str) and meta["label"].strip():
|
| 528 |
+
field["label"] = meta["label"][:120]
|
| 529 |
+
out.append(field)
|
| 530 |
+
for key, field in saved.items():
|
| 531 |
+
if key in base_keys or not isinstance(field, dict):
|
| 532 |
+
continue
|
| 533 |
+
if field.get("scope") and field.get("scope") != scope_key:
|
| 534 |
+
continue # a cohort-specific field on another page (wave-6 item 9)
|
| 535 |
+
if (str(key).startswith(MEASURE_FIELD_PREFIX)
|
| 536 |
+
and isinstance(field.get("measure"), dict)):
|
| 537 |
+
window = _clean_window(field["measure"].get("window"))
|
| 538 |
+
mkey = str(field["measure"].get("key") or "")
|
| 539 |
+
if window is None or not mkey:
|
| 540 |
+
continue
|
| 541 |
+
mtype = field.get("type") if field.get("type") in MEASURE_FIELD_TYPES else "currency"
|
| 542 |
+
out.append({
|
| 543 |
+
"key": str(key)[:80],
|
| 544 |
+
"label": str(field.get("label") or "Measure")[:120],
|
| 545 |
+
"type": mtype,
|
| 546 |
+
"source": "odoo",
|
| 547 |
+
"default": bool(field.get("default", True)),
|
| 548 |
+
"custom": True,
|
| 549 |
+
"derived": True,
|
| 550 |
+
"filterable": False,
|
| 551 |
+
"agg": "sum" if mtype in ("currency", "int") else None,
|
| 552 |
+
"note": str(field.get("note") or "")[:2000],
|
| 553 |
+
"measure": {"key": mkey[:80], "window": window},
|
| 554 |
+
**_field_extras(field, mtype),
|
| 555 |
+
})
|
| 556 |
+
continue
|
| 557 |
+
if not str(key).startswith("custom_"):
|
| 558 |
+
continue
|
| 559 |
+
ftype = field.get("type")
|
| 560 |
+
if ftype not in CUSTOM_FIELD_TYPES:
|
| 561 |
+
continue
|
| 562 |
+
if ftype in READONLY_CUSTOM_TYPES:
|
| 563 |
+
# Wave-5 items 9/11: the read-only user-created pair. Emitted with the cohort
|
| 564 |
+
# column's mechanism (source 'odoo' + derived) so the client never offers an
|
| 565 |
+
# editor and the host's overlay-write guard excludes them by construction.
|
| 566 |
+
# FILTERABLE since wave 6 (owner item 6): their values live client-side
|
| 567 |
+
# (formula computes over the row, created_time renders `_created`), this
|
| 568 |
+
# table's counts are client-mode, and the windowed count path never sees
|
| 569 |
+
# these tables β so the client engine answers them soundly.
|
| 570 |
+
entry = {
|
| 571 |
+
"key": str(key)[:80],
|
| 572 |
+
"label": str(field.get("label") or "Untitled")[:120],
|
| 573 |
+
"type": ftype,
|
| 574 |
+
"source": "odoo",
|
| 575 |
+
"derived": True,
|
| 576 |
+
"filterable": True,
|
| 577 |
+
"default": bool(field.get("default", True)),
|
| 578 |
+
"custom": True,
|
| 579 |
+
"note": str(field.get("note") or "")[:2000],
|
| 580 |
+
**_field_extras(field, ftype),
|
| 581 |
+
}
|
| 582 |
+
if ftype == "formula":
|
| 583 |
+
formula = _clean_formula(field.get("formula"))
|
| 584 |
+
if formula is None:
|
| 585 |
+
continue # a formula field without a formula is nothing
|
| 586 |
+
entry["formula"] = formula
|
| 587 |
+
out.append(entry)
|
| 588 |
+
continue
|
| 589 |
+
if field.get("source") != "overlay":
|
| 590 |
+
continue
|
| 591 |
+
options = (_clean_options(field.get("options"))
|
| 592 |
+
if ftype in ("select", "multiselect") else [])
|
| 593 |
+
if ftype in ("select", "multiselect") and not options:
|
| 594 |
+
# A select with no surviving choices can never hold a value. Dropping the COLUMN
|
| 595 |
+
# would lose the user's data; degrading it to text keeps every stored value
|
| 596 |
+
# readable and lets them re-add choices.
|
| 597 |
+
ftype = "text"
|
| 598 |
+
out.append({
|
| 599 |
+
"key": str(key)[:80],
|
| 600 |
+
"label": str(field.get("label") or "Untitled")[:120],
|
| 601 |
+
"type": ftype,
|
| 602 |
+
"source": "overlay",
|
| 603 |
+
"default": bool(field.get("default", True)),
|
| 604 |
+
"custom": True,
|
| 605 |
+
"agg": field.get("agg") if field.get("agg") in {"sum"} else None,
|
| 606 |
+
"note": str(field.get("note") or "")[:2000],
|
| 607 |
+
**({"options": options} if ftype in ("select", "multiselect") else {}),
|
| 608 |
+
**(_choice_appearance(field, options)
|
| 609 |
+
if ftype in ("select", "multiselect") else {}),
|
| 610 |
+
# comma-joined SET semantics (the Cohorts column's contract): the row belongs to
|
| 611 |
+
# every member, groups count it under each, the toolbar count stays distinct.
|
| 612 |
+
**({"multi": True} if ftype == "multiselect" else {}),
|
| 613 |
+
**({"max": _clean_rating_max(field.get("max"))} if ftype == "rating" else {}),
|
| 614 |
+
**({"automation": _clean_automation(field.get("automation"))}
|
| 615 |
+
if ftype == "automation" and _clean_automation(field.get("automation")) else {}),
|
| 616 |
+
**_field_extras(field, ftype),
|
| 617 |
+
})
|
| 618 |
+
if cohorts:
|
| 619 |
+
out.append(cohort_field())
|
| 620 |
+
return out
|
| 621 |
+
|
| 622 |
+
|
| 623 |
+
def rows_from_pool(pool_rows, fields=None, overlays=None, derived=None):
|
| 624 |
+
"""Map customer_data.pool() dicts -> the API row shape the grid expects:
|
| 625 |
+
pid + each Odoo field (numeric fields rounded, text/date passed through) + the
|
| 626 |
+
persisted external overlay. Mirrors the standalone API payload contract.
|
| 627 |
+
|
| 628 |
+
`derived` is `{pid: {key: value}}` for columns the HOST computes per render rather than
|
| 629 |
+
reads off the pool row β today just the cohort column. A separate argument from `overlays`
|
| 630 |
+
on purpose: `overlays` is the PERSISTED user stratum, and putting a value there that is
|
| 631 |
+
never written back would make the dict mean two things.
|
| 632 |
+
"""
|
| 633 |
+
fields = fields or FIELDS
|
| 634 |
+
odoo_fields = [field for field in fields if field["source"] == "odoo"]
|
| 635 |
+
overlay_fields = [field for field in fields if field["source"] == "overlay"]
|
| 636 |
+
derived_keys = [field["key"] for field in fields if field.get("derived")]
|
| 637 |
+
overlays = overlays or {}
|
| 638 |
+
derived = derived or {}
|
| 639 |
+
out = []
|
| 640 |
+
for r in pool_rows:
|
| 641 |
+
pid = r.get("pid")
|
| 642 |
+
# `_created` (wave-5 item 11) rides every row like `pid` does β the datum the
|
| 643 |
+
# `created_time` field type renders, regardless of that field's own key. Not a Field:
|
| 644 |
+
# it has no column of its own until a user creates one. `lat`/`lon` (wave-7 W11) ride
|
| 645 |
+
# the same way: the Map VIEW's data, nullable, deliberately not a column.
|
| 646 |
+
row = {"pid": pid, "_created": r.get("_created") or "",
|
| 647 |
+
"lat": r.get("lat"), "lon": r.get("lon")}
|
| 648 |
+
for field in odoo_fields:
|
| 649 |
+
k = field["key"]
|
| 650 |
+
if field.get("derived"):
|
| 651 |
+
continue # not on the pool row β filled from `derived` below
|
| 652 |
+
v = r.get(k)
|
| 653 |
+
row[k] = v if field["type"] in {"text", "status", "date"} else _round(v)
|
| 654 |
+
saved = overlays.get(str(pid), {}) or {}
|
| 655 |
+
for field in overlay_fields:
|
| 656 |
+
row[field["key"]] = saved.get(field["key"], "")
|
| 657 |
+
got = derived.get(pid) or {}
|
| 658 |
+
for k in derived_keys:
|
| 659 |
+
# '' not None: a customer in no cohort has an EMPTY cohort list, and `is empty` on a
|
| 660 |
+
# text column is the question somebody will ask of it.
|
| 661 |
+
row[k] = got.get(k, "")
|
| 662 |
+
out.append(row)
|
| 663 |
+
return out
|
| 664 |
+
|
| 665 |
+
|
| 666 |
+
# --- the FILTER-TREE contract (mirrors customer-grid/types.ts) ---------------
|
| 667 |
+
# Ops the Airtable-parity condition builder can emit. isEmpty/isNotEmpty are
|
| 668 |
+
# VALUE-FREE (they legitimately carry no value and must never be dropped for it).
|
| 669 |
+
FILTER_OPS = {'contains', 'doesNotContain', 'eq', 'neq', 'isEmpty', 'isNotEmpty',
|
| 670 |
+
'gt', 'gte', 'lt', 'lte', 'between', 'within',
|
| 671 |
+
# Wave 2026-08-02 (C-OPS): RANK operators β evaluated as a SET pass over the
|
| 672 |
+
# sibling-filtered domain by the client engine (useVisibleRows). The validator
|
| 673 |
+
# accepts them like any op (structural, not semantic); filter_sql REFUSES to
|
| 674 |
+
# compile them to row SQL (a per-row WHERE cannot express Top-N). aboveAvg /
|
| 675 |
+
# belowAvg are VALUE-FREE; the rest encode their argument in `value` as a
|
| 676 |
+
# string int (topN/bottomN 1..10000, inTopPct/inBottomPct 1..100,
|
| 677 |
+
# inQuartile 1..4, inDecile 1..10). Deliberately NOT in MEASURE_OPS: on a
|
| 678 |
+
# measure-carrying column they rank the field's own derived values.
|
| 679 |
+
'topN', 'bottomN', 'inTopPct', 'inBottomPct',
|
| 680 |
+
'aboveAvg', 'belowAvg', 'inQuartile', 'inDecile'}
|
| 681 |
+
#: The RESERVED pseudo-column of a cohort-membership leaf (owner item 5, 2026-07-26). It is not
|
| 682 |
+
#: a Field and never will be β a cohort is a hand-curated SET, so making it a column would mean
|
| 683 |
+
#: a cell per row per cohort. Mirrors customer-grid/types.ts COHORT_FIELD.
|
| 684 |
+
COHORT_FIELD = '__cohort__'
|
| 685 |
+
#: Ops a cohort leaf may carry (owner, 2026-07-27): set operators over a SET of cohorts.
|
| 686 |
+
#: Anything else is dropped. Deliberately DISJOINT from FILTER_OPS β see types.ts COHORT_OPS:
|
| 687 |
+
#: a set op reaching a column leaf would fall through the client engine's switch to "no
|
| 688 |
+
#: narrowing", so keeping the vocabularies apart makes the existing fail-closed drop do the work.
|
| 689 |
+
COHORT_OPS = {'anyOf', 'allOf', 'noneOf'}
|
| 690 |
+
#: The single-cohort ops this leaf shipped with, kept as PERMANENT aliases and REWRITTEN here:
|
| 691 |
+
#: `is part of [one]` is `is any of [that one]`, so a saved view keeps answering and upgrades the
|
| 692 |
+
#: next time it is written. Mirrors types.ts COHORT_OP_ALIASES.
|
| 693 |
+
COHORT_OP_ALIASES = {'eq': 'anyOf', 'neq': 'noneOf'}
|
| 694 |
+
#: How many cohorts one condition may name. Mirrors types.ts MAX_COHORT_IDS.
|
| 695 |
+
MAX_COHORT_IDS = 20
|
| 696 |
+
|
| 697 |
+
|
| 698 |
+
def parse_cohort_ids(value):
|
| 699 |
+
"""The cohorts a leaf names, parsed out of `value`. Mirrors types.ts `cohortIds()`.
|
| 700 |
+
|
| 701 |
+
Comma-separated in one string because `FilterRule.value` is what all four layers persist and
|
| 702 |
+
round-trip, and a one-element list is byte-identical to what the single-cohort leaf already
|
| 703 |
+
stored β so every shipped view parses with no migration. Safe because a cohort id is built
|
| 704 |
+
from `[a-z0-9_]` only (modules/cohort.new_id), so a comma cannot occur inside one.
|
| 705 |
+
"""
|
| 706 |
+
out = []
|
| 707 |
+
for raw in ('' if value is None else str(value)).split(','):
|
| 708 |
+
cid = raw.strip()[:120]
|
| 709 |
+
if not cid or cid in out:
|
| 710 |
+
continue
|
| 711 |
+
out.append(cid)
|
| 712 |
+
if len(out) >= MAX_COHORT_IDS:
|
| 713 |
+
break
|
| 714 |
+
return out
|
| 715 |
+
# Airtable allows 3 nesting levels (root conditions -> group -> group), then grays
|
| 716 |
+
# the button out. MAX_FILTER_DEPTH in types.ts must stay in lock-step with this.
|
| 717 |
+
MAX_FILTER_DEPTH = 3
|
| 718 |
+
MAX_FILTER_NODES = 100 # total nodes across the whole tree
|
| 719 |
+
MAX_FILTER_SIBLINGS = 50 # per level
|
| 720 |
+
|
| 721 |
+
|
| 722 |
+
#: Shape of a measure condition's date window. aios_grid has ZERO app-internal imports by
|
| 723 |
+
#: design, so it does NOT know the window VOCABULARY β `harness/windows.py` owns that, mirrored
|
| 724 |
+
#: in `customer-grid/windows.ts`, and a third copy here is exactly the drift those two already
|
| 725 |
+
#: need a gate to prevent. This validates SHAPE only.
|
| 726 |
+
WINDOW_MAX_N = 3650
|
| 727 |
+
|
| 728 |
+
|
| 729 |
+
def _clean_window(raw):
|
| 730 |
+
"""Structural passthrough for a measure condition's `{kind, n?, from?, to?}` window.
|
| 731 |
+
|
| 732 |
+
Meaning is NOT checked here: an unrecognised `kind` survives this function and is REFUSED by
|
| 733 |
+
`harness.measure_filter.resolve_rule`, the layer that owns the vocabulary. Splitting it this
|
| 734 |
+
way keeps the grid module reusable and keeps one definition of what "last quarter" means.
|
| 735 |
+
"""
|
| 736 |
+
if not isinstance(raw, dict):
|
| 737 |
+
return None
|
| 738 |
+
kind = raw.get('kind')
|
| 739 |
+
if not isinstance(kind, str) or not kind or len(kind) > 40:
|
| 740 |
+
return None
|
| 741 |
+
out = {'kind': kind}
|
| 742 |
+
if raw.get('n') is not None:
|
| 743 |
+
try:
|
| 744 |
+
out['n'] = max(1, min(int(raw['n']), WINDOW_MAX_N))
|
| 745 |
+
except (TypeError, ValueError):
|
| 746 |
+
return None
|
| 747 |
+
for side in ('from', 'to'):
|
| 748 |
+
if raw.get(side) not in (None, ''):
|
| 749 |
+
out[side] = str(raw[side])[:32]
|
| 750 |
+
return out
|
| 751 |
+
|
| 752 |
+
|
| 753 |
+
def _clean_rhs(raw, valid_keys):
|
| 754 |
+
"""CG-9 β validate `{kind, colId, window?}`, the "compare against another attribute" side.
|
| 755 |
+
|
| 756 |
+
SHAPE and KEY only: `colId` must be something this table has (the caller widens `valid_keys`
|
| 757 |
+
with the measure keys, exactly as it does for the left side), and a measure rhs must carry a
|
| 758 |
+
window. What the window MEANS is `harness/windows.py`'s business, same split as `_clean_window`.
|
| 759 |
+
"""
|
| 760 |
+
if not isinstance(raw, dict):
|
| 761 |
+
return None
|
| 762 |
+
kind = raw.get('kind')
|
| 763 |
+
if kind not in ('field', 'measure', 'stat'):
|
| 764 |
+
return None
|
| 765 |
+
if kind == 'stat':
|
| 766 |
+
# A STATISTIC carries no column: the population is the comparand. Shape only β which
|
| 767 |
+
# statistics exist is `harness/measure_filter.STATS`'s business, and an unrecognised one
|
| 768 |
+
# is REFUSED there rather than guessed at, exactly like an unrecognised window kind.
|
| 769 |
+
stat = raw.get('stat')
|
| 770 |
+
if not isinstance(stat, str) or not stat or len(stat) > 24:
|
| 771 |
+
return None
|
| 772 |
+
return {'kind': 'stat', 'stat': stat}
|
| 773 |
+
col = raw.get('colId')
|
| 774 |
+
if col not in valid_keys:
|
| 775 |
+
return None
|
| 776 |
+
out = {'kind': kind, 'colId': col}
|
| 777 |
+
if kind == 'measure':
|
| 778 |
+
window = _clean_window(raw.get('window'))
|
| 779 |
+
if window is None:
|
| 780 |
+
return None # a measure comparand with no period is not a question
|
| 781 |
+
out['window'] = window
|
| 782 |
+
return out
|
| 783 |
+
|
| 784 |
+
|
| 785 |
+
#: View DISPLAY MODES beside the grid (wave-6 item 10; 'map' wave-7 W11; 'dashboard' wave-8
|
| 786 |
+
#: I19). Mirrors customer-grid/types.ts DISPLAY_MODES; 'grid' is what an absent/unknown
|
| 787 |
+
#: display means, so it is never stored.
|
| 788 |
+
#: β 'dashboard' is RETAINED FOREVER (wave-9 I10, contract C2). The owner renamed the mode to
|
| 789 |
+
#: "Chart" (a Dashboard MODULE is coming and the two would collide), but this set is the
|
| 790 |
+
#: gatekeeper for a STORED value: `_clean_display` DROPS an unknown mode, so removing
|
| 791 |
+
#: 'dashboard' here would silently downgrade every already-saved chart view to grid β and live
|
| 792 |
+
#: views are sitting in mode:'dashboard' right now (wave 8's own close-out records one). The
|
| 793 |
+
#: rename is therefore a stored-value MIGRATION, not a constant rename: accept 'dashboard' on
|
| 794 |
+
#: READ forever, only ever WRITE 'chart'.
|
| 795 |
+
DISPLAY_MODES = {'grid', 'list', 'calendar', 'kanban', 'map', 'dashboard', 'chart',
|
| 796 |
+
'timeseries', 'catalog'}
|
| 797 |
+
|
| 798 |
+
#: C-DISP (wave 2026-08-02): the time-series view's bucket vocabulary and caps, plus the
|
| 799 |
+
#: calendar-summary metric cap. types.ts mirrors these as TS_BUCKETS / TS_MAX_LAST_N /
|
| 800 |
+
#: TS_MAX_FIELDS / MAX_CALENDAR_METRICS, and cleanDisplay applies the same per-entry drops,
|
| 801 |
+
#: so an accepted save reads back byte-identically on both engines.
|
| 802 |
+
TS_BUCKETS = {'week', 'month', 'quarter', 'year'}
|
| 803 |
+
TS_MAX_LAST_N = 120
|
| 804 |
+
TS_MAX_FIELDS = 12
|
| 805 |
+
MAX_CALENDAR_METRICS = 4
|
| 806 |
+
_ISO_DAY = re.compile(r'^\d{4}-\d{2}-\d{2}$')
|
| 807 |
+
|
| 808 |
+
#: C6-CATALOG (wave 18) β the catalog view's vocabulary and caps. types.ts mirrors every name
|
| 809 |
+
#: below, and `cleanDisplay` applies the same drops in the SAME ORDER, so an accepted save reads
|
| 810 |
+
#: back identically on both engines. The code budget is the order-sensitive one β see
|
| 811 |
+
#: `_clean_catalogs`.
|
| 812 |
+
MAX_CATALOGS = 12
|
| 813 |
+
MAX_CATALOG_PAGES = 40
|
| 814 |
+
MAX_CATALOG_CODES = 500 # cumulative across ONE catalog's pages, spent in PAGE ORDER
|
| 815 |
+
CATALOG_PAPERS = {'letter', 'a4', 'tabloid'}
|
| 816 |
+
CATALOG_ORIENTATIONS = {'portrait', 'landscape'}
|
| 817 |
+
CATALOG_QUALITIES = {'web', 'print'}
|
| 818 |
+
CATALOG_PAGE_KINDS = {'cover', 'intro', 'section', 'gallery'}
|
| 819 |
+
CATALOG_COLS = (2, 3, 4)
|
| 820 |
+
CATALOG_ID_MAX, CATALOG_NAME_MAX = 40, 80
|
| 821 |
+
CATALOG_TITLE_MAX, CATALOG_BODY_MAX, CATALOG_CODE_MAX = 120, 2000, 60
|
| 822 |
+
_HEX6 = re.compile(r'^#[0-9A-Fa-f]{6}$')
|
| 823 |
+
|
| 824 |
+
#: Wave 14 C-ACC ([[loopable-wave14-split]]; rulings R2/R3). Mirrored by types.ts
|
| 825 |
+
#: TS_DELTA_KINDS / TS_MAX_CUSTOM_ROWS / TS_MAX_STYLES β the C-DISP byte-identical law.
|
| 826 |
+
TS_DELTA_KINDS = ('abs', 'pct', 'yoy', 'ytd')
|
| 827 |
+
TS_MAX_CUSTOM_ROWS = 12
|
| 828 |
+
TS_MAX_STYLES = 200
|
| 829 |
+
#: R2 β a formula row's `expr` is stored VERBATIM and NEVER parsed here (evaluation is client
|
| 830 |
+
#: law; the client refuses unknown refs/cycles/div-zero itself). The charset wall is the whole
|
| 831 |
+
#: server-side contract: row refs `[...]`, arithmetic, numbers β no markup, no control chars.
|
| 832 |
+
_TS_EXPR_OK = re.compile(r'^[A-Za-z0-9_ .+\-*/()\[\]]+$')
|
| 833 |
+
|
| 834 |
+
#: Old wire value -> the value we store today. Applied AFTER the membership test so an unknown
|
| 835 |
+
#: mode is still rejected rather than accidentally aliased.
|
| 836 |
+
_LEGACY_MODES = {'dashboard': 'chart'}
|
| 837 |
+
|
| 838 |
+
#: Chart kinds a chart-mode view may hold (wave-8 I19, contract C2). Mirrors the client's
|
| 839 |
+
#: union. Deliberately small: the owner asked to "start with simple charts" and expand, and a
|
| 840 |
+
#: kind the client cannot draw is worse than one that does not exist yet.
|
| 841 |
+
#: Wave-16 C-CHARTCAP: + 'table' β the group-by aggregate table (by-rep / by-BU /
|
| 842 |
+
#: top-customers, the third Sales block shape). Client renderer: DashboardView's
|
| 843 |
+
#: GroupTableView over salesParity.tableFromSpec.
|
| 844 |
+
CHART_KINDS = {'bar', 'line', 'area', 'donut', 'kpi', 'table'}
|
| 845 |
+
CHART_AGGS = {'sum', 'avg', 'count', 'min', 'max'}
|
| 846 |
+
MAX_CHARTS = 12 #: per view β a dashboard, not an unbounded render loop
|
| 847 |
+
MAX_CHART_TITLE = 60
|
| 848 |
+
|
| 849 |
+
#: Wave-9 I11 (contract C2) β chart customisation, host-validated.
|
| 850 |
+
#:
|
| 851 |
+
#: `palette` names a colour JOB, never a colour. A browser must not be able to post a raw hex:
|
| 852 |
+
#: the four names below map to the four jobs a palette can do (identity / magnitude / polarity)
|
| 853 |
+
#: and resolve to brand ramps client-side, so a tenant restyle cannot be defeated by a stored
|
| 854 |
+
#: literal. STATUS colours (good/warning/serious/critical) are deliberately NOT selectable β
|
| 855 |
+
#: they are reserved signal, and reusing them as "series 4" is how a chart starts lying.
|
| 856 |
+
CHART_PALETTES = {'brand', 'categorical', 'sequential', 'diverging'}
|
| 857 |
+
CHART_FORMATS = {'auto', 'number', 'currency', 'percent', 'compact'}
|
| 858 |
+
MAX_AXIS_LABEL = 40
|
| 859 |
+
#: `size` is the I10 drag. Width is in GRID COLUMNS (a 12-column board), height in px.
|
| 860 |
+
CHART_W_RANGE = (1, 12)
|
| 861 |
+
CHART_H_RANGE = (120, 800)
|
| 862 |
+
|
| 863 |
+
|
| 864 |
+
def _clean_chart(raw, valid_keys):
|
| 865 |
+
"""One dashboard chart, fail-closed. Returns None if the chart cannot be drawn.
|
| 866 |
+
|
| 867 |
+
A chart's `y` is optional (absent = count of rows, which is what "how many customers per
|
| 868 |
+
state" means). `x` is NOT: a chart with no category axis has nothing to plot against, and
|
| 869 |
+
silently keeping it would put an empty card on the dashboard with no way to tell why.
|
| 870 |
+
"""
|
| 871 |
+
if not isinstance(raw, dict):
|
| 872 |
+
return None
|
| 873 |
+
kind = raw.get('kind')
|
| 874 |
+
if kind not in CHART_KINDS:
|
| 875 |
+
return None
|
| 876 |
+
x = raw.get('x')
|
| 877 |
+
if x not in valid_keys:
|
| 878 |
+
return None # dead category ref -> the chart goes, not the board
|
| 879 |
+
cid = raw.get('id')
|
| 880 |
+
if not isinstance(cid, str) or not cid.strip():
|
| 881 |
+
return None # the client owns chart ids; an unidentified card
|
| 882 |
+
# cannot be edited or removed, so it must not persist
|
| 883 |
+
out = {'id': cid.strip()[:64], 'kind': kind, 'x': x,
|
| 884 |
+
'agg': raw.get('agg') if raw.get('agg') in CHART_AGGS else 'sum'}
|
| 885 |
+
if raw.get('y') in valid_keys:
|
| 886 |
+
out['y'] = raw['y']
|
| 887 |
+
else:
|
| 888 |
+
# no measurable column -> the only honest aggregation left is "how many rows"
|
| 889 |
+
out['agg'] = 'count'
|
| 890 |
+
title = raw.get('title')
|
| 891 |
+
if isinstance(title, str) and title.strip():
|
| 892 |
+
out['title'] = title.strip()[:MAX_CHART_TITLE]
|
| 893 |
+
|
| 894 |
+
# ββ wave-9 I11 (contract C2): customisation ββββββββββββββββββββββββββββββββββββββββββββ
|
| 895 |
+
# `splitBy` is the field whose values become the SERIES. It is deliberately not called
|
| 896 |
+
# `colorBy`: that name already means two other things here (`config.colorBy` = row
|
| 897 |
+
# colouring, `display.colorField` = map pin colour) and a third sense would be unreadable.
|
| 898 |
+
if raw.get('splitBy') in valid_keys and raw['splitBy'] != x:
|
| 899 |
+
out['splitBy'] = raw['splitBy']
|
| 900 |
+
# Stacking is only a question once there are series to stack, and only for the two kinds
|
| 901 |
+
# that can express it. Anywhere else it is dropped rather than stored as a lie the client
|
| 902 |
+
# would have to re-decide.
|
| 903 |
+
if out.get('splitBy') and kind in ('bar', 'area') and raw.get('stacked') is True:
|
| 904 |
+
out['stacked'] = True
|
| 905 |
+
if raw.get('palette') in CHART_PALETTES:
|
| 906 |
+
out['palette'] = raw['palette']
|
| 907 |
+
|
| 908 |
+
axis = raw.get('axis')
|
| 909 |
+
if isinstance(axis, dict):
|
| 910 |
+
# β ONE y-scale, always. There is no second-axis key here and there must never be:
|
| 911 |
+
# two y-scales on one frame can manufacture any correlation you like by rescaling, and
|
| 912 |
+
# the honest alternatives are two charts, small multiples, or indexing to a common base.
|
| 913 |
+
# Ruled explicitly in contract C2 against the "Tableau versatility" brief.
|
| 914 |
+
clean_axis = {}
|
| 915 |
+
for side in ('x', 'y'):
|
| 916 |
+
spec = axis.get(side)
|
| 917 |
+
if not isinstance(spec, dict):
|
| 918 |
+
continue
|
| 919 |
+
one = {}
|
| 920 |
+
lab = spec.get('label')
|
| 921 |
+
if isinstance(lab, str) and lab.strip():
|
| 922 |
+
one['label'] = lab.strip()[:MAX_AXIS_LABEL]
|
| 923 |
+
if spec.get('format') in CHART_FORMATS:
|
| 924 |
+
one['format'] = spec['format']
|
| 925 |
+
if one:
|
| 926 |
+
clean_axis[side] = one
|
| 927 |
+
if clean_axis:
|
| 928 |
+
out['axis'] = clean_axis
|
| 929 |
+
|
| 930 |
+
size = raw.get('size')
|
| 931 |
+
if isinstance(size, dict):
|
| 932 |
+
one = {}
|
| 933 |
+
for key, (lo, hi) in (('w', CHART_W_RANGE), ('h', CHART_H_RANGE)):
|
| 934 |
+
try:
|
| 935 |
+
one[key] = max(lo, min(hi, int(size[key])))
|
| 936 |
+
except (KeyError, TypeError, ValueError):
|
| 937 |
+
pass # a partial size is fine: the client defaults the missing axis
|
| 938 |
+
if one:
|
| 939 |
+
out['size'] = one
|
| 940 |
+
|
| 941 |
+
# ββ Wave 14 R3 ([[loopable-wave14-split]]): a METRIC chart may carry a PERIOD β the
|
| 942 |
+
# trend-over-buckets encoding. Kept only when the chart's value field is measure-backed:
|
| 943 |
+
# a category column has no time dimension, and a stored period on it would promise a
|
| 944 |
+
# trend the TS channel must refuse. `span` is meaningful only beside `bucket`.
|
| 945 |
+
if isinstance(out.get('y'), str) and out['y'].startswith('measure_'):
|
| 946 |
+
if raw.get('bucket') in TS_BUCKETS:
|
| 947 |
+
out['bucket'] = raw['bucket']
|
| 948 |
+
sp = raw.get('span')
|
| 949 |
+
if isinstance(sp, dict):
|
| 950 |
+
n = sp.get('lastN')
|
| 951 |
+
if (isinstance(n, int) and not isinstance(n, bool)
|
| 952 |
+
and 1 <= n <= TS_MAX_LAST_N):
|
| 953 |
+
out['span'] = {'lastN': n}
|
| 954 |
+
# ββ Wave-16 C-CHARTCAP: the YoY companion. Kept ONLY where it can mean something β
|
| 955 |
+
# beside a kept bucket (the compare series) or on a sum-of-metric KPI (the delta
|
| 956 |
+
# line). Anything else is a stored claim the renderer would have to re-refuse.
|
| 957 |
+
# Mirrors the client's cleanCharts rule key for key.
|
| 958 |
+
if raw.get('compare') == 'prior_year' and (
|
| 959 |
+
out.get('bucket') or (kind == 'kpi' and out.get('agg') == 'sum')):
|
| 960 |
+
out['compare'] = 'prior_year'
|
| 961 |
+
return out
|
| 962 |
+
|
| 963 |
+
|
| 964 |
+
def _clean_catalog_page(raw, budget):
|
| 965 |
+
"""C6-CATALOG β one page of a catalog. Returns `(page | None, codes_spent)`.
|
| 966 |
+
|
| 967 |
+
`budget` is what is LEFT of the catalog's 500-code allowance. Codes are deduped WITHIN a
|
| 968 |
+
page and not across the catalog: a product legitimately appears on a gallery page and again
|
| 969 |
+
in its section listing, and de-duplicating globally would silently delete the second
|
| 970 |
+
appearance. The budget is spent in page order, so a catalog that runs out loses the TAIL of
|
| 971 |
+
its last pages β never a random scatter, and never a page (a page with no products is a
|
| 972 |
+
heading the user can still see and fix).
|
| 973 |
+
"""
|
| 974 |
+
if not isinstance(raw, dict):
|
| 975 |
+
return None, 0
|
| 976 |
+
page_id = str(raw.get('id') or '')[:CATALOG_ID_MAX]
|
| 977 |
+
kind = raw.get('kind')
|
| 978 |
+
if not page_id or kind not in CATALOG_PAGE_KINDS:
|
| 979 |
+
return None, 0
|
| 980 |
+
out = {'id': page_id, 'kind': kind}
|
| 981 |
+
for key, cap in (('title', CATALOG_TITLE_MAX), ('body', CATALOG_BODY_MAX),
|
| 982 |
+
('imageCode', CATALOG_CODE_MAX)):
|
| 983 |
+
v = raw.get(key)
|
| 984 |
+
if isinstance(v, str) and v:
|
| 985 |
+
out[key] = v[:cap]
|
| 986 |
+
products = raw.get('products')
|
| 987 |
+
if isinstance(products, list) and budget > 0:
|
| 988 |
+
clean_p, seen_p = [], set()
|
| 989 |
+
for c in products:
|
| 990 |
+
if not isinstance(c, str) or not c:
|
| 991 |
+
continue
|
| 992 |
+
c = c[:CATALOG_CODE_MAX]
|
| 993 |
+
if c in seen_p:
|
| 994 |
+
continue
|
| 995 |
+
seen_p.add(c)
|
| 996 |
+
clean_p.append(c)
|
| 997 |
+
if len(clean_p) >= budget:
|
| 998 |
+
break
|
| 999 |
+
if clean_p:
|
| 1000 |
+
out['products'] = clean_p
|
| 1001 |
+
layout = raw.get('layout')
|
| 1002 |
+
if isinstance(layout, dict):
|
| 1003 |
+
clean_l = {}
|
| 1004 |
+
cols = layout.get('cols')
|
| 1005 |
+
if isinstance(cols, int) and not isinstance(cols, bool) and cols in CATALOG_COLS:
|
| 1006 |
+
clean_l['cols'] = cols
|
| 1007 |
+
# The kanbanClamp/tsSparkline asymmetry, one per direction: pack and colour SHOW by
|
| 1008 |
+
# default (the 2027 catalogue shows both), price does NOT (it shows no prices at all).
|
| 1009 |
+
# So only the opt-OUT is storable for the first two and only the opt-IN for the third β
|
| 1010 |
+
# a second spelling of a default is how a round trip starts churning.
|
| 1011 |
+
if layout.get('showPack') is False:
|
| 1012 |
+
clean_l['showPack'] = False
|
| 1013 |
+
if layout.get('showColor') is False:
|
| 1014 |
+
clean_l['showColor'] = False
|
| 1015 |
+
if layout.get('showPrice') is True:
|
| 1016 |
+
clean_l['showPrice'] = True
|
| 1017 |
+
if clean_l:
|
| 1018 |
+
out['layout'] = clean_l
|
| 1019 |
+
return out, len(out.get('products') or ())
|
| 1020 |
+
|
| 1021 |
+
|
| 1022 |
+
def _clean_catalogs(raw, valid_keys):
|
| 1023 |
+
"""C6-CATALOG (wave 18) β `display.catalogs`, fail-closed. Returns a list or None.
|
| 1024 |
+
|
| 1025 |
+
A catalog is a PRINT artifact, so the two structural keys that decide how it paginates
|
| 1026 |
+
(`paper`, `orientation`) are NORMALISED WITH A DEFAULT rather than dropped: a page box with
|
| 1027 |
+
no size is not a smaller catalog, it is an unrenderable one. Everything else follows the
|
| 1028 |
+
house rules β unknown keys dropped, per-entry drops never cost the neighbours, empty
|
| 1029 |
+
sub-objects omitted entirely (`brand`, `fields`, `layout`, `products`) so an absent key and
|
| 1030 |
+
an empty one are not two spellings of the same nothing.
|
| 1031 |
+
|
| 1032 |
+
`fields` binds the listing lines to real columns (the 2027 listing prints description / SKU /
|
| 1033 |
+
pack / colour, and `product_data` carries no pack or colour of its own β the user binds
|
| 1034 |
+
custom fields). Refs are checked against `valid_keys` HERE and not on the client, the same
|
| 1035 |
+
split `dateField`/`stackField` already run.
|
| 1036 |
+
"""
|
| 1037 |
+
if not isinstance(raw, list):
|
| 1038 |
+
return None
|
| 1039 |
+
out = []
|
| 1040 |
+
for c in raw:
|
| 1041 |
+
if len(out) >= MAX_CATALOGS:
|
| 1042 |
+
break
|
| 1043 |
+
if not isinstance(c, dict):
|
| 1044 |
+
continue
|
| 1045 |
+
cat_id = str(c.get('id') or '')[:CATALOG_ID_MAX]
|
| 1046 |
+
name = c.get('name')
|
| 1047 |
+
# An EMPTY name is legal (the user cleared the box and will type again) β an ABSENT one
|
| 1048 |
+
# is a malformed record. The `tsRows` label rule, same reasoning.
|
| 1049 |
+
if not cat_id or not isinstance(name, str):
|
| 1050 |
+
continue
|
| 1051 |
+
cat = {'id': cat_id, 'name': name[:CATALOG_NAME_MAX]}
|
| 1052 |
+
cat['paper'] = c['paper'] if c.get('paper') in CATALOG_PAPERS else 'letter'
|
| 1053 |
+
cat['orientation'] = (c['orientation']
|
| 1054 |
+
if c.get('orientation') in CATALOG_ORIENTATIONS else 'portrait')
|
| 1055 |
+
if c.get('quality') in CATALOG_QUALITIES:
|
| 1056 |
+
cat['quality'] = c['quality']
|
| 1057 |
+
brand = c.get('brand')
|
| 1058 |
+
if isinstance(brand, dict):
|
| 1059 |
+
clean_b = {}
|
| 1060 |
+
for k in ('primary', 'accent'):
|
| 1061 |
+
v = brand.get(k)
|
| 1062 |
+
if isinstance(v, str) and _HEX6.match(v):
|
| 1063 |
+
clean_b[k] = v
|
| 1064 |
+
company = brand.get('company')
|
| 1065 |
+
if isinstance(company, str) and company:
|
| 1066 |
+
clean_b['company'] = company[:CATALOG_NAME_MAX]
|
| 1067 |
+
# An asset CODE (resolved through C2-ASSET), never a URL: an arbitrary host inside
|
| 1068 |
+
# print CSS is exactly the tokens-not-values rule this contract carries.
|
| 1069 |
+
logo = brand.get('logo')
|
| 1070 |
+
if isinstance(logo, str) and logo:
|
| 1071 |
+
clean_b['logo'] = logo[:CATALOG_CODE_MAX]
|
| 1072 |
+
if clean_b:
|
| 1073 |
+
cat['brand'] = clean_b
|
| 1074 |
+
binds = c.get('fields')
|
| 1075 |
+
if isinstance(binds, dict):
|
| 1076 |
+
clean_bind = {k: binds[k] for k in ('name', 'pack', 'color', 'price')
|
| 1077 |
+
if binds.get(k) in valid_keys}
|
| 1078 |
+
if clean_bind:
|
| 1079 |
+
cat['fields'] = clean_bind
|
| 1080 |
+
pages, budget = [], MAX_CATALOG_CODES
|
| 1081 |
+
raw_pages = c.get('pages')
|
| 1082 |
+
if isinstance(raw_pages, list):
|
| 1083 |
+
for p in raw_pages:
|
| 1084 |
+
if len(pages) >= MAX_CATALOG_PAGES:
|
| 1085 |
+
break
|
| 1086 |
+
page, spent = _clean_catalog_page(p, budget)
|
| 1087 |
+
if page is None:
|
| 1088 |
+
continue
|
| 1089 |
+
budget -= spent
|
| 1090 |
+
pages.append(page)
|
| 1091 |
+
# ALWAYS emitted, even empty: a catalog with no pages yet is the state every catalog
|
| 1092 |
+
# starts in, and dropping the key would make "new" and "corrupt" the same wire value.
|
| 1093 |
+
cat['pages'] = pages
|
| 1094 |
+
out.append(cat)
|
| 1095 |
+
return out or None
|
| 1096 |
+
|
| 1097 |
+
|
| 1098 |
+
def _clean_display(raw, valid_keys):
|
| 1099 |
+
"""Structural passthrough for a view's `config.display` (wave-6 item 10), fail-closed.
|
| 1100 |
+
|
| 1101 |
+
`{mode, dateField?, stackField?, titleField?, colorField?, sizeField?, charts?}` β mode
|
| 1102 |
+
must be a known non-grid mode (grid is the absent default, so storing it would be a second
|
| 1103 |
+
way to say nothing); every field ref must name a field this table has (a ref to a deleted
|
| 1104 |
+
field is DROPPED and the client falls back to its per-mode default); unknown keys are
|
| 1105 |
+
dropped. What each mode MEANS β calendar wants a date-family field, kanban a select-family
|
| 1106 |
+
stack, map a single-select to colour by and a numeric to size by β is the client's
|
| 1107 |
+
business: it is the only layer that renders them, and a wrong-typed ref degrades to that
|
| 1108 |
+
surface's default rather than to an error (the `_clean_window` split).
|
| 1109 |
+
|
| 1110 |
+
Wave-8 (contract C2) adds the map encodings (`colorField` I3, `sizeField` I5) and
|
| 1111 |
+
dashboard `charts` (I19). A chart whose x/y names a deleted field is dropped INDIVIDUALLY β
|
| 1112 |
+
never the whole array, because losing one column should not cost the user a dashboard they
|
| 1113 |
+
spent time building.
|
| 1114 |
+
"""
|
| 1115 |
+
if not isinstance(raw, dict):
|
| 1116 |
+
return None
|
| 1117 |
+
mode = raw.get('mode')
|
| 1118 |
+
if mode not in DISPLAY_MODES or mode == 'grid':
|
| 1119 |
+
return None
|
| 1120 |
+
# Wave-9 I10 (C2): normalise the legacy wire value AFTER the membership test, so an unknown
|
| 1121 |
+
# mode is still rejected rather than accidentally aliased into a real one. Every already
|
| 1122 |
+
# saved 'dashboard' view reads back as 'chart' from here on; nothing writes 'dashboard'.
|
| 1123 |
+
mode = _LEGACY_MODES.get(mode, mode)
|
| 1124 |
+
out = {'mode': mode}
|
| 1125 |
+
for ref in ('dateField', 'stackField', 'titleField', 'colorField', 'sizeField'):
|
| 1126 |
+
if raw.get(ref) in valid_keys:
|
| 1127 |
+
out[ref] = raw[ref]
|
| 1128 |
+
# ββ C-DISP (wave 2026-08-02) βββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 1129 |
+
# kanbanClamp: stored ONLY as the literal opt-OUT. Absent means clamped β the new
|
| 1130 |
+
# standardized default β so storing True would be a second way to say nothing (the same
|
| 1131 |
+
# rule that keeps mode:'grid' out of the store).
|
| 1132 |
+
if raw.get('kanbanClamp') is False:
|
| 1133 |
+
out['kanbanClamp'] = False
|
| 1134 |
+
if raw.get('calendarMode') in ('records', 'summary'):
|
| 1135 |
+
out['calendarMode'] = raw['calendarMode']
|
| 1136 |
+
metrics = raw.get('calendarMetrics')
|
| 1137 |
+
if isinstance(metrics, list):
|
| 1138 |
+
clean_m, seen_m = [], set()
|
| 1139 |
+
for m in metrics[:MAX_CALENDAR_METRICS]:
|
| 1140 |
+
# Dropped INDIVIDUALLY (the charts precedent): one dead metric must not cost the
|
| 1141 |
+
# user the summary card they configured around it.
|
| 1142 |
+
if not isinstance(m, dict):
|
| 1143 |
+
continue
|
| 1144 |
+
mid = str(m.get('id') or '')[:40]
|
| 1145 |
+
if (not mid or mid in seen_m or m.get('field') not in valid_keys
|
| 1146 |
+
or m.get('agg') not in CHART_AGGS):
|
| 1147 |
+
continue
|
| 1148 |
+
seen_m.add(mid)
|
| 1149 |
+
clean_m.append({'id': mid, 'field': m['field'], 'agg': m['agg']})
|
| 1150 |
+
if clean_m:
|
| 1151 |
+
out['calendarMetrics'] = clean_m
|
| 1152 |
+
if raw.get('tsBucket') in TS_BUCKETS:
|
| 1153 |
+
out['tsBucket'] = raw['tsBucket']
|
| 1154 |
+
span = raw.get('tsSpan')
|
| 1155 |
+
if isinstance(span, dict):
|
| 1156 |
+
clean_span = {}
|
| 1157 |
+
n = span.get('lastN')
|
| 1158 |
+
if isinstance(n, int) and not isinstance(n, bool) and 1 <= n <= TS_MAX_LAST_N:
|
| 1159 |
+
clean_span['lastN'] = n
|
| 1160 |
+
else:
|
| 1161 |
+
f, t = span.get('from'), span.get('to')
|
| 1162 |
+
f = f if isinstance(f, str) and _ISO_DAY.match(f) else None
|
| 1163 |
+
t = t if isinstance(t, str) and _ISO_DAY.match(t) else None
|
| 1164 |
+
if f and t and f > t:
|
| 1165 |
+
f, t = t, f
|
| 1166 |
+
if f:
|
| 1167 |
+
clean_span['from'] = f
|
| 1168 |
+
if t:
|
| 1169 |
+
clean_span['to'] = t
|
| 1170 |
+
if clean_span:
|
| 1171 |
+
out['tsSpan'] = clean_span
|
| 1172 |
+
ts_fields = raw.get('tsFields')
|
| 1173 |
+
if isinstance(ts_fields, list):
|
| 1174 |
+
clean_f, seen_f = [], set()
|
| 1175 |
+
for k in ts_fields[:TS_MAX_FIELDS]:
|
| 1176 |
+
if k in valid_keys and k not in seen_f:
|
| 1177 |
+
seen_f.add(k)
|
| 1178 |
+
clean_f.append(k)
|
| 1179 |
+
if clean_f:
|
| 1180 |
+
out['tsFields'] = clean_f
|
| 1181 |
+
# ββ Wave 14 C-ACC ([[loopable-wave14-split]] R2; items 17/18) ββββββββββββββββββββββββ
|
| 1182 |
+
deltas = raw.get('tsDeltas')
|
| 1183 |
+
if isinstance(deltas, list):
|
| 1184 |
+
clean_d, seen_d = [], set()
|
| 1185 |
+
for d in deltas:
|
| 1186 |
+
if d in TS_DELTA_KINDS and d not in seen_d:
|
| 1187 |
+
seen_d.add(d)
|
| 1188 |
+
clean_d.append(d)
|
| 1189 |
+
if clean_d:
|
| 1190 |
+
out['tsDeltas'] = clean_d
|
| 1191 |
+
# Gridlines: stored ONLY as the literal opt-OUT (absent = shown), sparkline ONLY as the
|
| 1192 |
+
# literal opt-IN (absent = off) β the kanbanClamp asymmetry, one per direction.
|
| 1193 |
+
if raw.get('tsGridlines') is False:
|
| 1194 |
+
out['tsGridlines'] = False
|
| 1195 |
+
if raw.get('tsSparkline') is True:
|
| 1196 |
+
out['tsSparkline'] = True
|
| 1197 |
+
rows = raw.get('tsRows')
|
| 1198 |
+
if isinstance(rows, list):
|
| 1199 |
+
clean_r, seen_r = [], set()
|
| 1200 |
+
for r in rows[:TS_MAX_CUSTOM_ROWS]:
|
| 1201 |
+
if not isinstance(r, dict):
|
| 1202 |
+
continue
|
| 1203 |
+
rid = str(r.get('id') or '')[:40]
|
| 1204 |
+
r_kind = r.get('kind')
|
| 1205 |
+
if not rid or rid in seen_r or r_kind not in ('note', 'formula'):
|
| 1206 |
+
continue
|
| 1207 |
+
label = r.get('label')
|
| 1208 |
+
if not isinstance(label, str):
|
| 1209 |
+
continue # ABSENT label = malformed; an EMPTY one is a legal spacer row
|
| 1210 |
+
# (GRID's dated asymmetry amendments, 2026-08-02)
|
| 1211 |
+
one = {'id': rid, 'kind': r_kind, 'label': label.strip()[:120]}
|
| 1212 |
+
if r_kind == 'formula':
|
| 1213 |
+
expr = r.get('expr')
|
| 1214 |
+
if (isinstance(expr, str) and expr.strip()
|
| 1215 |
+
and len(expr) <= 200 and _TS_EXPR_OK.match(expr)):
|
| 1216 |
+
one['expr'] = expr
|
| 1217 |
+
# else: keep the ROW, drop the EXPR β it renders "β". Vanishing the row
|
| 1218 |
+
# would delete the user's label to punish their arithmetic (GRID's dated
|
| 1219 |
+
# asymmetry amendment; the calendarMetrics per-entry-drop precedent).
|
| 1220 |
+
seen_r.add(rid)
|
| 1221 |
+
clean_r.append(one)
|
| 1222 |
+
if clean_r:
|
| 1223 |
+
out['tsRows'] = clean_r
|
| 1224 |
+
styles = raw.get('tsStyles')
|
| 1225 |
+
if isinstance(styles, dict):
|
| 1226 |
+
clean_s = {}
|
| 1227 |
+
for s_key, s_val in styles.items():
|
| 1228 |
+
if len(clean_s) >= TS_MAX_STYLES:
|
| 1229 |
+
break # capped, not truncated silently: the gate names this
|
| 1230 |
+
if not isinstance(s_key, str) or not s_key or len(s_key) > 96:
|
| 1231 |
+
continue # key = rowId or "rowId:colKey" β the client's grammar
|
| 1232 |
+
if not isinstance(s_val, dict):
|
| 1233 |
+
continue
|
| 1234 |
+
one = {}
|
| 1235 |
+
if s_val.get('bold') is True:
|
| 1236 |
+
one['bold'] = True
|
| 1237 |
+
if s_val.get('line') is True:
|
| 1238 |
+
one['line'] = True
|
| 1239 |
+
if one:
|
| 1240 |
+
clean_s[s_key] = one
|
| 1241 |
+
if clean_s:
|
| 1242 |
+
out['tsStyles'] = clean_s
|
| 1243 |
+
charts = raw.get('charts')
|
| 1244 |
+
if isinstance(charts, list):
|
| 1245 |
+
clean = [c for c in (_clean_chart(x, valid_keys) for x in charts[:MAX_CHARTS]) if c]
|
| 1246 |
+
# de-dupe by id: two cards sharing an id are one card as far as the client's keyed
|
| 1247 |
+
# render is concerned, and the second would silently shadow the first
|
| 1248 |
+
seen, uniq = set(), []
|
| 1249 |
+
for c in clean:
|
| 1250 |
+
if c['id'] in seen:
|
| 1251 |
+
continue
|
| 1252 |
+
seen.add(c['id'])
|
| 1253 |
+
uniq.append(c)
|
| 1254 |
+
if uniq:
|
| 1255 |
+
out['charts'] = uniq
|
| 1256 |
+
# ββ C6-CATALOG (wave 18) βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 1257 |
+
catalogs = _clean_catalogs(raw.get('catalogs'), valid_keys)
|
| 1258 |
+
if catalogs:
|
| 1259 |
+
out['catalogs'] = catalogs
|
| 1260 |
+
return out
|
| 1261 |
+
|
| 1262 |
+
|
| 1263 |
+
#: FOLDERS over the saved views / cohorts sidebars (wave-8 I11, contract C4).
|
| 1264 |
+
#:
|
| 1265 |
+
#: β Folder membership is stored as a SIDE MAP (`itemFolders`), not as a `folderId` ON each
|
| 1266 |
+
#: view or cohort β a deliberate amendment to C4's first wording, recorded in the split doc.
|
| 1267 |
+
#: Two reasons. (1) A cohort lives in a DIFFERENT store (`customer_cohorts`, keyed by cohort
|
| 1268 |
+
#: id) and adding a `folders` key beside those ids would collide with a cohort whose generated
|
| 1269 |
+
#: id happened to be 'folders'. (2) Folder placement is a per-user ORGANISING act, not part of
|
| 1270 |
+
#: what a view IS: keeping it out of the view config means duplicating or exporting a view does
|
| 1271 |
+
#: not drag a folder reference along with it. One map, one home, both surfaces.
|
| 1272 |
+
FOLDER_SURFACES = {"views", "cohorts"}
|
| 1273 |
+
MAX_FOLDERS = 60
|
| 1274 |
+
MAX_FOLDER_NAME = 80
|
| 1275 |
+
|
| 1276 |
+
#: Wave-9 I15 (contract C5) β a user-chosen folder icon, as {shape, tone}.
|
| 1277 |
+
#:
|
| 1278 |
+
#: Both halves are WHITELISTS, never free values: `shape` names geometry the client already
|
| 1279 |
+
#: draws (one source, `iconShapes.ts`, read by both painters) and `tone` names a palette token,
|
| 1280 |
+
#: not a colour β so a browser cannot post a hex and defeat a tenant restyle, and a shape the
|
| 1281 |
+
#: client cannot render can never reach the store.
|
| 1282 |
+
#: β MIRRORS CLIENT'S `iconShapes.ts` ENUMERATION EXACTLY (C5: CLIENT enumerates, HOST mirrors β
|
| 1283 |
+
#: posted in the split doc 2026-07-29, HOST adopted it the same day, replacing a provisional
|
| 1284 |
+
#: 12-shape guess of mine that contained shapes the client cannot draw). Do not extend this set
|
| 1285 |
+
#: without the matching client geometry: an unknown shape falls back to the default folder mark,
|
| 1286 |
+
#: which is also I14's "existing folders get the folder icon" for every pre-wave-9 folder.
|
| 1287 |
+
FOLDER_ICON_SHAPES = {"folder", "star", "flag", "tag", "bookmark", "box", "circle", "square"}
|
| 1288 |
+
#: Tones are the C1 pastels β FILLS ONLY, never text (the standing palette rule). 'grey' is the
|
| 1289 |
+
#: default, and is CLIENT's key name: not 'neutral', which is what HOST first guessed.
|
| 1290 |
+
FOLDER_ICON_TONES = {"blue", "green", "yellow", "red", "grey"}
|
| 1291 |
+
FOLDER_ICON_DEFAULT_TONE = "grey"
|
| 1292 |
+
|
| 1293 |
+
|
| 1294 |
+
#: Wave-9 I17 (contract C4) β who may EDIT a saved view.
|
| 1295 |
+
#:
|
| 1296 |
+
#: β READ THIS BEFORE BUILDING ON IT. Views are stored PER USER today
|
| 1297 |
+
#: (`core/table_store.TableStore.workspace` reads `store.get(table_key)[username]`), so one
|
| 1298 |
+
#: user's views are invisible to every other user and "collaborative" has nothing to act on
|
| 1299 |
+
#: yet. This validator is therefore CORRECT-BUT-INERT plumbing: it makes the setting durable
|
| 1300 |
+
#: and fail-closed now, so that when shared views land the permission does not need a data
|
| 1301 |
+
#: migration and no saved view is retro-restricted. It does NOT make anything shared, and
|
| 1302 |
+
#: nothing in the app currently reads it to grant or deny cross-user access.
|
| 1303 |
+
#: Recorded as the C4 amendment in .claude/wiki/research/grid-wave9-split.md.
|
| 1304 |
+
VIEW_EDIT_MODES = {'personal', 'collaborative', 'users'}
|
| 1305 |
+
MAX_VIEW_USERS = 50
|
| 1306 |
+
|
| 1307 |
+
|
| 1308 |
+
def clean_view_permissions(raw, default, known_users=None):
|
| 1309 |
+
"""{edit, users?} β fail-closed on both halves.
|
| 1310 |
+
|
| 1311 |
+
`default` is supplied by the CALLER because it splits by path, and that split is a
|
| 1312 |
+
permission rule rather than a formatting one: absent on a view that already exists means a
|
| 1313 |
+
pre-wave-9 view and must stay 'collaborative' (retro-restricting somebody's saved view is a
|
| 1314 |
+
silent takeaway), while absent on CREATE must be 'personal' (a new view must never be
|
| 1315 |
+
anyone-can-edit purely by omission).
|
| 1316 |
+
|
| 1317 |
+
`known_users` (when given) is the real account list: an unknown name is DROPPED, and an
|
| 1318 |
+
'users' grant left with nobody in it collapses to 'personal' rather than to everyone.
|
| 1319 |
+
"""
|
| 1320 |
+
mode = (raw or {}).get('edit') if isinstance(raw, dict) else None
|
| 1321 |
+
if mode not in VIEW_EDIT_MODES:
|
| 1322 |
+
mode = default if default in VIEW_EDIT_MODES else 'personal'
|
| 1323 |
+
if mode != 'users':
|
| 1324 |
+
return {'edit': mode}
|
| 1325 |
+
names, seen = [], set()
|
| 1326 |
+
for u in list((raw or {}).get('users') or [])[:MAX_VIEW_USERS]:
|
| 1327 |
+
u = str(u or '').strip()
|
| 1328 |
+
if not u or u.lower() in seen:
|
| 1329 |
+
continue
|
| 1330 |
+
if known_users is not None and u not in known_users:
|
| 1331 |
+
continue # fail-closed: a name we cannot resolve grants nothing
|
| 1332 |
+
seen.add(u.lower())
|
| 1333 |
+
names.append(u)
|
| 1334 |
+
if not names:
|
| 1335 |
+
return {'edit': 'personal'} # an empty grant is NOT "everyone"
|
| 1336 |
+
return {'edit': 'users', 'users': names}
|
| 1337 |
+
|
| 1338 |
+
|
| 1339 |
+
def clean_folder_icon(raw):
|
| 1340 |
+
"""{shape, tone} or None. Fail-closed on both halves, independently.
|
| 1341 |
+
|
| 1342 |
+
A folder with a valid shape but a junk tone keeps the shape and defaults the tone rather
|
| 1343 |
+
than losing the icon entirely β losing a user's pick because one half was wrong is the kind
|
| 1344 |
+
of silent data loss the folder events already avoid elsewhere.
|
| 1345 |
+
"""
|
| 1346 |
+
if not isinstance(raw, dict):
|
| 1347 |
+
return None
|
| 1348 |
+
shape = raw.get("shape")
|
| 1349 |
+
if shape not in FOLDER_ICON_SHAPES:
|
| 1350 |
+
return None
|
| 1351 |
+
tone = raw.get("tone")
|
| 1352 |
+
return {"shape": shape,
|
| 1353 |
+
"tone": tone if tone in FOLDER_ICON_TONES else FOLDER_ICON_DEFAULT_TONE}
|
| 1354 |
+
|
| 1355 |
+
|
| 1356 |
+
def clean_folders(raw):
|
| 1357 |
+
"""Validate the per-surface folder lists, fail-closed. {surface: [{id, name, order}]}."""
|
| 1358 |
+
out = {}
|
| 1359 |
+
for surface in FOLDER_SURFACES:
|
| 1360 |
+
items, seen = [], set()
|
| 1361 |
+
for f in list((raw or {}).get(surface) or [])[:MAX_FOLDERS]:
|
| 1362 |
+
if not isinstance(f, dict):
|
| 1363 |
+
continue
|
| 1364 |
+
fid = str(f.get("id") or "").strip()[:80]
|
| 1365 |
+
name = str(f.get("name") or "").strip()[:MAX_FOLDER_NAME]
|
| 1366 |
+
if not fid or not name or fid in seen:
|
| 1367 |
+
continue # an unidentified or unnamed folder cannot be shown or edited
|
| 1368 |
+
seen.add(fid)
|
| 1369 |
+
try:
|
| 1370 |
+
order = int(f.get("order", len(items)))
|
| 1371 |
+
except (TypeError, ValueError):
|
| 1372 |
+
order = len(items)
|
| 1373 |
+
row = {"id": fid, "name": name, "order": order}
|
| 1374 |
+
icon = clean_folder_icon(f.get("icon")) # wave-9 I15 (C5); absent = default mark
|
| 1375 |
+
if icon:
|
| 1376 |
+
row["icon"] = icon
|
| 1377 |
+
items.append(row)
|
| 1378 |
+
items.sort(key=lambda x: x["order"])
|
| 1379 |
+
for i, f in enumerate(items):
|
| 1380 |
+
f["order"] = i # re-index so `order` is always dense and total
|
| 1381 |
+
if items:
|
| 1382 |
+
out[surface] = items
|
| 1383 |
+
return out
|
| 1384 |
+
|
| 1385 |
+
|
| 1386 |
+
def clean_item_folders(raw, folders, valid_ids):
|
| 1387 |
+
"""{surface: {itemId: folderId}} β dropping any placement whose ITEM or FOLDER is gone.
|
| 1388 |
+
|
| 1389 |
+
This is what makes a deleted folder's contents fall back to the root rather than vanish:
|
| 1390 |
+
nothing stores "this item is in no folder", so an unresolvable placement simply disappears
|
| 1391 |
+
and the item renders at the top level. Same for an item that was deleted elsewhere β its
|
| 1392 |
+
stale placement can never resurrect it, because the sidebars render ITEMS and consult this
|
| 1393 |
+
map, never the other way round.
|
| 1394 |
+
"""
|
| 1395 |
+
out = {}
|
| 1396 |
+
for surface in FOLDER_SURFACES:
|
| 1397 |
+
fids = {f["id"] for f in (folders or {}).get(surface, [])}
|
| 1398 |
+
ok = {}
|
| 1399 |
+
for item_id, fid in ((raw or {}).get(surface) or {}).items():
|
| 1400 |
+
if not isinstance(item_id, str) or not isinstance(fid, str):
|
| 1401 |
+
continue
|
| 1402 |
+
if item_id in (valid_ids or {}).get(surface, ()) and fid in fids:
|
| 1403 |
+
ok[item_id[:120]] = fid
|
| 1404 |
+
if ok:
|
| 1405 |
+
out[surface] = ok
|
| 1406 |
+
return out
|
| 1407 |
+
|
| 1408 |
+
|
| 1409 |
+
def clean_filter_tree(raw, valid_keys, depth=1, budget=None, cohort_ids=None):
|
| 1410 |
+
"""Recursively validate an UNTRUSTED filter tree (conditions + nested groups).
|
| 1411 |
+
|
| 1412 |
+
Returns a clean tree of leaf conditions ({colId, op, value, value2}) and
|
| 1413 |
+
groups ({conj, children}). Module-agnostic on purpose: any module embedding
|
| 1414 |
+
the grid validates its own view state through this one function.
|
| 1415 |
+
|
| 1416 |
+
Fail-closed PER NODE: anything unrecognised is DROPPED rather than raised β
|
| 1417 |
+
the same contract the rest of the view sanitiser follows, so one bad rule can
|
| 1418 |
+
never cost a user their whole saved view. Depth, per-level width and total
|
| 1419 |
+
node count are all capped: the tree is re-evaluated for every row on every
|
| 1420 |
+
render, so an unbounded structure would be a persistent client-side DoS.
|
| 1421 |
+
Empty groups are dropped (they carry no meaning once persisted).
|
| 1422 |
+
|
| 1423 |
+
`cohort_ids` is the set of cohorts the CALLER may see. A cohort leaf naming anything else is
|
| 1424 |
+
DROPPED here rather than left for the engine β a deleted cohort would otherwise leave a
|
| 1425 |
+
condition that can only match nothing, so `List is not [deleted]` would show an empty table
|
| 1426 |
+
forever with no way to tell why. `None` means this host has no cohorts, and then every
|
| 1427 |
+
cohort leaf is dropped: fail-closed, like every other unknown key.
|
| 1428 |
+
"""
|
| 1429 |
+
if budget is None:
|
| 1430 |
+
budget = [MAX_FILTER_NODES]
|
| 1431 |
+
out = []
|
| 1432 |
+
for node in list(raw or [])[:MAX_FILTER_SIBLINGS]:
|
| 1433 |
+
if budget[0] <= 0:
|
| 1434 |
+
break
|
| 1435 |
+
if not isinstance(node, dict):
|
| 1436 |
+
continue
|
| 1437 |
+
if isinstance(node.get('children'), list): # a condition GROUP
|
| 1438 |
+
if depth >= MAX_FILTER_DEPTH:
|
| 1439 |
+
continue # too deep -> drop
|
| 1440 |
+
budget[0] -= 1
|
| 1441 |
+
children = clean_filter_tree(node['children'], valid_keys,
|
| 1442 |
+
depth + 1, budget, cohort_ids)
|
| 1443 |
+
if children:
|
| 1444 |
+
out.append({'conj': 'or' if node.get('conj') == 'or' else 'and',
|
| 1445 |
+
'children': children})
|
| 1446 |
+
continue
|
| 1447 |
+
if node.get('colId') == COHORT_FIELD: # a cohort-membership leaf
|
| 1448 |
+
op = COHORT_OP_ALIASES.get(node.get('op'), node.get('op'))
|
| 1449 |
+
named = parse_cohort_ids(node.get('value'))
|
| 1450 |
+
# ALL of them, or the leaf goes. A set that quietly lost a member asks a DIFFERENT
|
| 1451 |
+
# question, and for `noneOf` a strictly wider one: `is none of [A, B]` degrading to
|
| 1452 |
+
# `is none of [A]` would show every row in B under a count nobody would doubt. This
|
| 1453 |
+
# is the same all-or-nothing the single-cohort leaf already had, extended to a set.
|
| 1454 |
+
if op in COHORT_OPS and named and all(c in (cohort_ids or ()) for c in named):
|
| 1455 |
+
budget[0] -= 1
|
| 1456 |
+
out.append({'colId': COHORT_FIELD, 'op': op,
|
| 1457 |
+
'value': ','.join(named), 'value2': ''})
|
| 1458 |
+
continue
|
| 1459 |
+
if node.get('colId') in valid_keys and node.get('op') in FILTER_OPS:
|
| 1460 |
+
budget[0] -= 1
|
| 1461 |
+
# `or ''` would be wrong here: it maps every FALSY value to '', and '' is the
|
| 1462 |
+
# signal for "inactive". A numeric 0 (or 0.0, or False) is a real value the client
|
| 1463 |
+
# treats as active β `0 === ""` is false in TS β so `revenue = 0` would silently
|
| 1464 |
+
# stop filtering and show every row instead of the zero-revenue ones.
|
| 1465 |
+
val, val2 = node.get('value'), node.get('value2')
|
| 1466 |
+
leaf = {'colId': node['colId'], 'op': node['op'],
|
| 1467 |
+
'value': ('' if val is None else str(val))[:500],
|
| 1468 |
+
'value2': ('' if val2 is None else str(val2))[:500]}
|
| 1469 |
+
# CG-8. A MEASURE condition ("Sales, in the last 90 days, > 5,000") carries two
|
| 1470 |
+
# extra members: a stable client-generated `id`, which is how the server's answer
|
| 1471 |
+
# finds its way back to the condition that asked (positional matching silently
|
| 1472 |
+
# re-associates every answer the moment a user deletes a condition), and the
|
| 1473 |
+
# `window`. Emitted ONLY when the input has them β a column condition's cleaned
|
| 1474 |
+
# shape is unchanged, so every persisted view deserialises byte-identically and
|
| 1475 |
+
# `clean_filter_tree` stays idempotent (verify_filter_engine.py asserts that by
|
| 1476 |
+
# exact structural comparison).
|
| 1477 |
+
rid = node.get('id')
|
| 1478 |
+
if rid not in (None, ''):
|
| 1479 |
+
leaf['id'] = str(rid)[:64]
|
| 1480 |
+
window = _clean_window(node.get('window'))
|
| 1481 |
+
if window is not None:
|
| 1482 |
+
leaf['window'] = window
|
| 1483 |
+
# Owner items 3 + 4, carried under the SAME rule as CG-8's `id`/`window`: emitted
|
| 1484 |
+
# only when the input has them, so a plain column condition's cleaned shape is
|
| 1485 |
+
# byte-identical to what it always was and `clean_filter_tree` stays idempotent
|
| 1486 |
+
# (verify_filter_engine.py asserts that by exact structural comparison). Drop the
|
| 1487 |
+
# carry-through and the next autosave silently strips a date condition back to a
|
| 1488 |
+
# bare comparison against an empty value β i.e. back to INACTIVE.
|
| 1489 |
+
date_window = _clean_window(node.get('dateWindow'))
|
| 1490 |
+
if date_window is not None:
|
| 1491 |
+
leaf['dateWindow'] = date_window
|
| 1492 |
+
mode = node.get('dateMode')
|
| 1493 |
+
# SHAPE only. An unrecognised mode survives here and is refused by
|
| 1494 |
+
# `windows.resolve_anchor`, which returns None and makes the condition match
|
| 1495 |
+
# NOTHING β the same split as `_clean_window`, and the reason this module can stay
|
| 1496 |
+
# free of the date vocabulary it would otherwise have to keep in step.
|
| 1497 |
+
if isinstance(mode, str) and 0 < len(mode) <= 40:
|
| 1498 |
+
leaf['dateMode'] = mode
|
| 1499 |
+
rhs = _clean_rhs(node.get('rhs'), valid_keys)
|
| 1500 |
+
if rhs is not None:
|
| 1501 |
+
leaf['rhs'] = rhs
|
| 1502 |
+
out.append(leaf)
|
| 1503 |
+
return out
|
| 1504 |
+
|
| 1505 |
+
|
| 1506 |
+
def _default_view_config(fields):
|
| 1507 |
+
shown = [field["key"] for field in fields
|
| 1508 |
+
if field.get("default") is not False or field["source"] == "overlay"]
|
| 1509 |
+
hidden = [field["key"] for field in fields if field["key"] not in shown]
|
| 1510 |
+
return {
|
| 1511 |
+
# `filters` is the ROOT of the filter tree: leaf conditions and/or nested
|
| 1512 |
+
# condition groups ({conj, children}); `filterConj` joins the root level.
|
| 1513 |
+
"filters": [], "filterConj": "and",
|
| 1514 |
+
"sorts": [], "groupBy": None, "colorBy": None,
|
| 1515 |
+
"rowHeightMode": "short", "order": shown + hidden, "visible": shown,
|
| 1516 |
+
"widths": {}, "memberPids": [],
|
| 1517 |
+
}
|
| 1518 |
+
|
| 1519 |
+
|
| 1520 |
+
#: Wave 17 R1 / C-LOCKV β the `kind` a PROJECTED locked view wears. A cohort is not a separate
|
| 1521 |
+
#: kind of object any more: it is a saved view whose rows are a hand-curated set.
|
| 1522 |
+
LOCKED_VIEW_KIND = 'locked'
|
| 1523 |
+
|
| 1524 |
+
|
| 1525 |
+
def locked_view_projection(entry, base_config):
|
| 1526 |
+
"""One cohort -> the saved-view row that IS it (wave 17 R1, contract C-LOCKV).
|
| 1527 |
+
|
| 1528 |
+
β THE LOCK IS THE VIEW'S IDENTITY, NOT ITS CONFIGURATION. `config.cohortLock` names the
|
| 1529 |
+
view's OWN id, which is what makes the shipped engine law (`useVisibleRows`: intersect the
|
| 1530 |
+
named set FIRST, unconditionally, and match NOTHING when the membership is unresolvable) do
|
| 1531 |
+
all the work with no second mechanism. Membership itself is NEVER copied in here β it stays
|
| 1532 |
+
in `customer_cohorts` and travels as `workspace.lists`, because a per-reader-scoped
|
| 1533 |
+
collection inside a client-writable `config` is deleted by the next autosave (see the
|
| 1534 |
+
contract's reason 2).
|
| 1535 |
+
|
| 1536 |
+
β `locked: True` is the LEGACY "undeletable/mode-frozen" flag and is deliberately NOT set:
|
| 1537 |
+
these views are ordinary in every respect the owner asked for β reorder, folder, sort,
|
| 1538 |
+
filter, change display mode. The lock mark in the rail is driven by `kind`.
|
| 1539 |
+
"""
|
| 1540 |
+
return {
|
| 1541 |
+
'id': entry['id'],
|
| 1542 |
+
'name': entry.get('name') or entry['id'],
|
| 1543 |
+
'kind': LOCKED_VIEW_KIND,
|
| 1544 |
+
'config': {**base_config, 'cohortLock': entry['id']},
|
| 1545 |
+
}
|
| 1546 |
+
|
| 1547 |
+
|
| 1548 |
+
def views_from_defs(defs, saved_views, fields, system_name="All customers", locked_lists=None):
|
| 1549 |
+
"""Convert legacy list formulas into the shared serializable SavedView contract.
|
| 1550 |
+
|
| 1551 |
+
`system_name` (wave 16 C-TOPIC) labels the system view per TOPIC ("All products" on the
|
| 1552 |
+
product surface). The ID stays "all-customers" on every topic β the client pins it
|
| 1553 |
+
(UNDELETABLE_VIEW_IDS, the landing default), and an id that varies by surface would fork
|
| 1554 |
+
that contract for a label's sake.
|
| 1555 |
+
|
| 1556 |
+
`locked_lists` (wave 17 R1) are the caller's cohorts, each PROJECTED as a saved view whose
|
| 1557 |
+
id IS the cohort id β so every stored reference to that id (a `cohortLock` on another view,
|
| 1558 |
+
an `is part of` condition, a folder placement) keeps pointing at the same thing and no
|
| 1559 |
+
rewrite map is needed. Saved config OVERLAYS the projection through the same mechanism the
|
| 1560 |
+
`list:` views have always used, which is what gives a locked view its own sort, filter,
|
| 1561 |
+
columns and display mode with no new storage."""
|
| 1562 |
+
base = _default_view_config(fields)
|
| 1563 |
+
views = [{
|
| 1564 |
+
"id": "all-customers", "name": system_name, "kind": "system",
|
| 1565 |
+
"locked": True, "config": dict(base),
|
| 1566 |
+
}]
|
| 1567 |
+
op_map = {">=": "gte", ">": "gt", "<=": "lte", "<": "lt", "=": "eq",
|
| 1568 |
+
"contains": "contains"}
|
| 1569 |
+
for name, definition in (defs or {}).items():
|
| 1570 |
+
filters = []
|
| 1571 |
+
for rule in definition.get("rules") or []:
|
| 1572 |
+
if rule.get("field") not in {field["key"] for field in fields}:
|
| 1573 |
+
continue
|
| 1574 |
+
filters.append({
|
| 1575 |
+
"colId": rule["field"],
|
| 1576 |
+
"op": op_map.get(rule.get("op"), "eq"),
|
| 1577 |
+
"value": str(rule.get("value") if rule.get("value") is not None else ""),
|
| 1578 |
+
})
|
| 1579 |
+
sort = str(definition.get("sort") or "")
|
| 1580 |
+
sorts = ([{"colId": sort.lstrip("-"),
|
| 1581 |
+
"dir": "desc" if sort.startswith("-") else "asc"}]
|
| 1582 |
+
if sort.lstrip("-") in {field["key"] for field in fields} else [])
|
| 1583 |
+
views.append({
|
| 1584 |
+
"id": "list:" + str(name),
|
| 1585 |
+
"name": str(name),
|
| 1586 |
+
"kind": "list",
|
| 1587 |
+
"note": str(definition.get("note") or ""),
|
| 1588 |
+
"config": {
|
| 1589 |
+
**base, "filters": filters, "sorts": sorts,
|
| 1590 |
+
"memberPids": [int(pid) for pid in definition.get("members") or []
|
| 1591 |
+
if isinstance(pid, int) or str(pid).isdigit()],
|
| 1592 |
+
},
|
| 1593 |
+
})
|
| 1594 |
+
# Wave 17 R1 β the cohorts, as ordinary views. Appended BEFORE the saved-config overlay
|
| 1595 |
+
# below so a user's own edits to a locked view (its sort, its columns, its display mode)
|
| 1596 |
+
# land on the projection instead of creating a second row with the same id.
|
| 1597 |
+
for _entry in (locked_lists or []):
|
| 1598 |
+
if isinstance(_entry, dict) and _entry.get('id'):
|
| 1599 |
+
views.append(locked_view_projection(_entry, base))
|
| 1600 |
+
index = {view["id"]: i for i, view in enumerate(views)}
|
| 1601 |
+
for view_id, saved in (saved_views or {}).items():
|
| 1602 |
+
if not isinstance(saved, dict) or not isinstance(saved.get("config"), dict):
|
| 1603 |
+
continue
|
| 1604 |
+
clean = dict(saved)
|
| 1605 |
+
clean["id"] = str(view_id)
|
| 1606 |
+
if view_id in index:
|
| 1607 |
+
views[index[view_id]] = clean
|
| 1608 |
+
else:
|
| 1609 |
+
views.append(clean)
|
| 1610 |
+
# β WAVE 17 R1 β RE-STAMP THE LOCK AFTER THE OVERLAY. The loop above REPLACES a projected
|
| 1611 |
+
# view with its saved record, and a saved record that omits `cohortLock` would hand back a
|
| 1612 |
+
# view that shows the WHOLE BOOK under a locked view's name. That is not hypothetical: the
|
| 1613 |
+
# client rebuilds `config` on every autosave (a column resize is enough), and the lock is
|
| 1614 |
+
# identity here, not something the browser is the source of truth for. Read-side rather than
|
| 1615 |
+
# write-side-only on purpose β this also repairs any record already written by another path.
|
| 1616 |
+
_locked_ids = {e['id']: e for e in (locked_lists or [])
|
| 1617 |
+
if isinstance(e, dict) and e.get('id')}
|
| 1618 |
+
if _locked_ids:
|
| 1619 |
+
for _v in views:
|
| 1620 |
+
_entry = _locked_ids.get(_v.get('id'))
|
| 1621 |
+
if not _entry:
|
| 1622 |
+
continue
|
| 1623 |
+
_v['kind'] = LOCKED_VIEW_KIND
|
| 1624 |
+
_v['config'] = {**(_v.get('config') or {}), 'cohortLock': _v['id']}
|
| 1625 |
+
# One thing, one name: the cohort store owns it (the rename event routes there), so
|
| 1626 |
+
# a stale `name` on the saved record can never fork into a second title.
|
| 1627 |
+
_v['name'] = _entry.get('name') or _v['id']
|
| 1628 |
+
return views
|
| 1629 |
+
|
| 1630 |
+
|
| 1631 |
+
def workspace_wire(ws, uname, pool_pids, defs=None, scope_key='customer', storage_key=None,
|
| 1632 |
+
fields_base=None, with_cohorts=True):
|
| 1633 |
+
"""The client's `GridWorkspace` WIRE SHAPE from the stored table workspace β the ONE
|
| 1634 |
+
projection, shared by both servers (app.py's `_table_grid` and the API's `/workspace`).
|
| 1635 |
+
|
| 1636 |
+
β WHY THIS EXISTS (2026-07-30). The API route used to return the STORE shape with no
|
| 1637 |
+
`storageKey` β and the client validator (`fetchWorkspace`) requires one, so the standalone
|
| 1638 |
+
shell silently discarded the whole workspace: saved views never rendered and `cohortMode`
|
| 1639 |
+
never arrived (the Cohort route drew the Customer surface). Duplicating the host's inline
|
| 1640 |
+
projection into the route would have re-created the same drift one wave later; extracting it
|
| 1641 |
+
means the wire can only be one thing.
|
| 1642 |
+
|
| 1643 |
+
Returns `(workspace, fields, views, cohort_lists)` β the extra three because the host
|
| 1644 |
+
interleaves further work (docs, derived cells, measure sets) that consumes them.
|
| 1645 |
+
|
| 1646 |
+
HOST-ONLY extras stay with the host: `docs`/`docPayload`, `pool`, `hideViews`,
|
| 1647 |
+
`cohortMode`/`scopeChoice` (the API stamps its own from `?scope=`).
|
| 1648 |
+
|
| 1649 |
+
Wave 16 C-TOPIC: `fields_base` selects the canonical contract (absent = customer,
|
| 1650 |
+
byte-identical); `with_cohorts=False` is the PRODUCT surface β cohorts are customer
|
| 1651 |
+
machinery (`modules/cohort` stores partner pids), and resolving them against product pids
|
| 1652 |
+
would intersect two unrelated id spaces and render a plausible, meaningless member count.
|
| 1653 |
+
"""
|
| 1654 |
+
import modules.cohort as cohort_mod
|
| 1655 |
+
|
| 1656 |
+
cohort_lists = []
|
| 1657 |
+
if with_cohorts:
|
| 1658 |
+
for cid, c in sorted(cohort_mod.visible(uname, pool_pids).items(),
|
| 1659 |
+
key=lambda kv: (kv[1].get('name') or '').lower()):
|
| 1660 |
+
members = [p for p in (c.get('members') or []) if p in pool_pids]
|
| 1661 |
+
entry = {'id': cid, 'name': c.get('name') or cid, 'pids': members}
|
| 1662 |
+
# Rule 8b: a member can drop out of the 24-month pool without the cohort being
|
| 1663 |
+
# wrong, and a silently smaller cohort is exactly what the unverifiable-count rule
|
| 1664 |
+
# forbids.
|
| 1665 |
+
missing = len(c.get('members') or []) - len(members)
|
| 1666 |
+
if missing:
|
| 1667 |
+
entry['missing'] = missing
|
| 1668 |
+
cohort_lists.append(entry)
|
| 1669 |
+
|
| 1670 |
+
fields = fields_from_workspace(ws, cohorts=bool(cohort_lists), scope_key=scope_key,
|
| 1671 |
+
fields_base=fields_base)
|
| 1672 |
+
views = views_from_defs(defs or {}, ws.get('views'), fields,
|
| 1673 |
+
system_name=("All products" if scope_key == 'product'
|
| 1674 |
+
else "All customers"),
|
| 1675 |
+
locked_lists=cohort_lists)
|
| 1676 |
+
workspace = {'storageKey': storage_key, 'views': views, 'lists': cohort_lists}
|
| 1677 |
+
# Owner item 3 (2026-07-31): where this user left off. The client's own localStorage copy
|
| 1678 |
+
# wins when present; this is the server's answer for a FRESH browser, which used to fall
|
| 1679 |
+
# all the way to the system default view (and whatever display mode was stored on it).
|
| 1680 |
+
if ws.get('activeViewId'):
|
| 1681 |
+
workspace['activeViewId'] = str(ws['activeViewId'])
|
| 1682 |
+
|
| 1683 |
+
# FOLDERS (owner item 11, contract C4), re-validated at SERVE time: a view or cohort can be
|
| 1684 |
+
# deleted by a path that knows nothing about folders, and the placement map must not outlive
|
| 1685 |
+
# the thing it points at.
|
| 1686 |
+
_folders = clean_folders(ws.get('folders'))
|
| 1687 |
+
_view_ids = {v['id'] for v in (views or []) if isinstance(v, dict) and v.get('id')}
|
| 1688 |
+
# ββ WAVE 17 R1 (C-LOCKV amendment 2026-08-03): the two folder surfaces become ONE, AT
|
| 1689 |
+
# SERVE TIME rather than by a store migration. A locked view is an ordinary view now, so
|
| 1690 |
+
# its folder has to be an ordinary view folder β but rewriting the stored map would be a
|
| 1691 |
+
# one-shot write that has to be got right once, while this is a projection that is right
|
| 1692 |
+
# every time it runs. New drags write to `views` anyway (the client only knows that
|
| 1693 |
+
# surface), so `cohorts` drains on its own and never needs a second pass.
|
| 1694 |
+
# β A cohorts-surface folder whose id ALREADY names a views folder is DROPPED, not merged:
|
| 1695 |
+
# re-parenting somebody's list into a folder that merely shares an id is a worse outcome
|
| 1696 |
+
# than the list appearing at the root, where it is visible and one drag from home.
|
| 1697 |
+
_cf = list(_folders.get('cohorts') or [])
|
| 1698 |
+
if _cf:
|
| 1699 |
+
_vf = list(_folders.get('views') or [])
|
| 1700 |
+
_taken = {f['id'] for f in _vf}
|
| 1701 |
+
_order = len(_vf)
|
| 1702 |
+
for _f in _cf:
|
| 1703 |
+
if _f['id'] in _taken:
|
| 1704 |
+
continue
|
| 1705 |
+
_vf.append({**_f, 'order': _order})
|
| 1706 |
+
_order += 1
|
| 1707 |
+
_folders['views'] = _vf
|
| 1708 |
+
_raw_item_folders = dict(ws.get('itemFolders') or {})
|
| 1709 |
+
if _raw_item_folders.get('cohorts'):
|
| 1710 |
+
# Cohort placements now describe VIEWS (same ids β that is the point of preserving them).
|
| 1711 |
+
# A placement already stored on the views surface WINS: it is the more recent act.
|
| 1712 |
+
_raw_item_folders['views'] = {**dict(_raw_item_folders.get('cohorts') or {}),
|
| 1713 |
+
**dict(_raw_item_folders.get('views') or {})}
|
| 1714 |
+
_placed = clean_item_folders(
|
| 1715 |
+
_raw_item_folders, _folders,
|
| 1716 |
+
{'views': _view_ids, 'cohorts': {c['id'] for c in cohort_lists}})
|
| 1717 |
+
if _folders.get('views'):
|
| 1718 |
+
workspace['folders'] = _folders['views']
|
| 1719 |
+
# β `cohortFolders` IS NO LONGER EMITTED. The rail has no cohorts section to fold, and a
|
| 1720 |
+
# wire that still described one would invite a second rendering of rows that are now views.
|
| 1721 |
+
_vplaced = _placed.get('views') or {}
|
| 1722 |
+
for _v in (views or []):
|
| 1723 |
+
if isinstance(_v, dict) and _v.get('id') in _vplaced:
|
| 1724 |
+
_v['folderId'] = _vplaced[_v['id']]
|
| 1725 |
+
_cplaced = _placed.get('cohorts') or {}
|
| 1726 |
+
for _c in cohort_lists:
|
| 1727 |
+
if _c['id'] in _cplaced:
|
| 1728 |
+
_c['folderId'] = _cplaced[_c['id']]
|
| 1729 |
+
|
| 1730 |
+
# RECORD LAYOUT (wave 2026-08-02, C-LAYOUT): the per-user record-detail field order,
|
| 1731 |
+
# re-validated at SERVE time exactly like folders β a field can be deleted by a path
|
| 1732 |
+
# that knows nothing about this stratum, and a stale key must not outlive its field.
|
| 1733 |
+
_rl = ws.get('recordLayout')
|
| 1734 |
+
if isinstance(_rl, dict) and isinstance(_rl.get('order'), list):
|
| 1735 |
+
_fkeys = {f['key'] for f in fields if isinstance(f, dict) and f.get('key')}
|
| 1736 |
+
_order, _seen = [], set()
|
| 1737 |
+
for _k in _rl['order'][:200]:
|
| 1738 |
+
_k = str(_k or '')
|
| 1739 |
+
if _k and _k in _fkeys and _k not in _seen:
|
| 1740 |
+
_seen.add(_k)
|
| 1741 |
+
_order.append(_k)
|
| 1742 |
+
if _order:
|
| 1743 |
+
workspace['recordLayout'] = {'order': _order}
|
| 1744 |
+
|
| 1745 |
+
return workspace, fields, views, cohort_lists
|
| 1746 |
+
|
| 1747 |
+
|
| 1748 |
+
def embed_html_path():
|
| 1749 |
+
"""The first existing candidate path for the inlined single-file build, or None."""
|
| 1750 |
+
for p in _EMBED_CANDIDATES:
|
| 1751 |
+
if p.is_file():
|
| 1752 |
+
return p
|
| 1753 |
+
return None
|
| 1754 |
+
|
| 1755 |
+
|
| 1756 |
+
def scope_counts(shown, matched, total):
|
| 1757 |
+
"""The honest 'N of M' a SERVER-WINDOWED table must carry (CG-2).
|
| 1758 |
+
|
| 1759 |
+
`matched` and `total` MUST come from their own queries over the whole scope. Never pass
|
| 1760 |
+
`len(rows)` as `matched` β that is the silent [:N] this exists to prevent: the page would
|
| 1761 |
+
report the window size as though it were the result size.
|
| 1762 |
+
|
| 1763 |
+
Refuses the shapes that could only be a mistake, because a wrong count here is invisible on
|
| 1764 |
+
screen (it looks like a smaller dataset, not like an error).
|
| 1765 |
+
"""
|
| 1766 |
+
shown, matched, total = int(shown), int(matched), int(total)
|
| 1767 |
+
if matched > total:
|
| 1768 |
+
raise ValueError(f"matched ({matched}) exceeds total ({total}) β a filter cannot match "
|
| 1769 |
+
f"more rows than the scope holds")
|
| 1770 |
+
if shown > matched:
|
| 1771 |
+
raise ValueError(f"shown ({shown}) exceeds matched ({matched}) β the window cannot hold "
|
| 1772 |
+
f"more rows than the filter matched")
|
| 1773 |
+
return {"shown": shown, "matched": matched, "total": total, "windowed": True}
|
| 1774 |
+
|
| 1775 |
+
|
| 1776 |
+
def build_html(rows, fields, *, workspace=None, path=None, counts=None):
|
| 1777 |
+
"""PURE: read the inlined single-file build and inject the data handshake.
|
| 1778 |
+
|
| 1779 |
+
Returns a self-contained HTML string with
|
| 1780 |
+
<script>window.__AIOS_GRID__ = {"fields": ..., "rows": ...};</script>
|
| 1781 |
+
inserted immediately after the opening <head> tag β a CLASSIC (non-module) script, so it
|
| 1782 |
+
executes during parse BEFORE the app's deferred `type="module"` script reads the global.
|
| 1783 |
+
Raises FileNotFoundError if no build is present (callers should try/except and fall back).
|
| 1784 |
+
"""
|
| 1785 |
+
p = Path(path) if path else embed_html_path()
|
| 1786 |
+
if p is None or not Path(p).is_file():
|
| 1787 |
+
raise FileNotFoundError(
|
| 1788 |
+
"aios_grid: no embed build found. Run `npm run build:embed` in aios-web/web and "
|
| 1789 |
+
"copy dist-embed/index.html -> platform/aios_grid_embed.html "
|
| 1790 |
+
"(python build_embed.py). Looked in: "
|
| 1791 |
+
+ " ; ".join(str(c) for c in _EMBED_CANDIDATES))
|
| 1792 |
+
html = Path(p).read_text(encoding="utf-8")
|
| 1793 |
+
# ensure_ascii=False keeps names readable; default=str is a safety net for stray
|
| 1794 |
+
# date/Decimal; the </ escape prevents a value containing "</script>" from breaking out.
|
| 1795 |
+
data = {"fields": fields, "rows": rows, "workspace": workspace or {}}
|
| 1796 |
+
# Only a WINDOWED table carries counts. Omitting the key leaves the client counting the rows
|
| 1797 |
+
# it holds, which is correct when it holds all of them.
|
| 1798 |
+
if counts:
|
| 1799 |
+
data["counts"] = counts
|
| 1800 |
+
payload = json.dumps(data, ensure_ascii=False, default=str).replace("</", "<\\/")
|
| 1801 |
+
tag = "<script>window.__AIOS_GRID__ = " + payload + ";</script>"
|
| 1802 |
+
new_html, n = re.subn(r"(<head\b[^>]*>)", lambda m: m.group(1) + tag, html, count=1)
|
| 1803 |
+
if n == 0: # no <head> (shouldn't happen) β still run it first
|
| 1804 |
+
new_html = tag + html
|
| 1805 |
+
return new_html
|
| 1806 |
+
|
| 1807 |
+
|
| 1808 |
+
def component_dir():
|
| 1809 |
+
for path in _COMPONENT_CANDIDATES:
|
| 1810 |
+
if (path / "index.html").is_file():
|
| 1811 |
+
return path
|
| 1812 |
+
return None
|
| 1813 |
+
|
| 1814 |
+
|
| 1815 |
+
def render(rows, fields, *, workspace=None, height=680, key="aios_grid",
|
| 1816 |
+
user_options=None, user_avatars=None, measures=None, measure_sets=None,
|
| 1817 |
+
today=None, viewer=None):
|
| 1818 |
+
"""Render the Airtable-style grid and return its latest guarded host event.
|
| 1819 |
+
|
| 1820 |
+
The preferred custom-component path forwards `key` and supports two-way persistence.
|
| 1821 |
+
If only the legacy HTML build exists, it renders safely with no return event.
|
| 1822 |
+
|
| 1823 |
+
`user_options` are the assignable people for `user`-typed overlay columns. They come from
|
| 1824 |
+
the HOST (the tenant's real accounts) rather than from anything the grid could invent, so
|
| 1825 |
+
an assignee is always someone who can actually log in and see the row.
|
| 1826 |
+
|
| 1827 |
+
`measures` / `measure_sets` are CG-8. A measure is FILTERABLE BUT NOT DISPLAYABLE: "Sales in
|
| 1828 |
+
the last 90 days" is a question about a window, not a column of this table, and rendering it
|
| 1829 |
+
as a column would silently pick a window for the user. The HOST resolves each condition to a
|
| 1830 |
+
set of row ids (`{ruleId: [pid, ...]}`) and the client treats that set as a membership test β
|
| 1831 |
+
which is why this module can offer measure filtering while keeping ZERO app-internal imports:
|
| 1832 |
+
it never computes a measure, it only carries one.
|
| 1833 |
+
|
| 1834 |
+
`today` is the TENANT'S current date as an ISO string, and every relative date condition
|
| 1835 |
+
("is within the past month", "is before one month ago") resolves against it. It comes from
|
| 1836 |
+
the host rather than from `new Date()` in the browser for the same three reasons
|
| 1837 |
+
harness/windows.py takes it as a parameter: a viewer a day ahead of the server would resolve
|
| 1838 |
+
a different month than the server summed, a window must not shift between the count and the
|
| 1839 |
+
rows, and the tenant's day boundary is the tenant's. Omit it and those conditions match
|
| 1840 |
+
NOTHING with a visible marker β never everything.
|
| 1841 |
+
"""
|
| 1842 |
+
import streamlit.components.v1 as _c
|
| 1843 |
+
directory = component_dir()
|
| 1844 |
+
payload = {"fields": fields, "rows": rows, "workspace": workspace or {},
|
| 1845 |
+
"userOptions": [str(u)[:120] for u in (user_options or [])][:500],
|
| 1846 |
+
# Wave 14 C-AVATAR β display name -> data URL, the userOptions vocabulary's
|
| 1847 |
+
# companion. Same caps; a value that is not a data: URL is dropped here too
|
| 1848 |
+
# (defence in depth beside the API route's write-side wall).
|
| 1849 |
+
"userAvatars": {str(k)[:120]: str(v)
|
| 1850 |
+
for k, v in list((user_avatars or {}).items())[:500]
|
| 1851 |
+
if str(v).startswith("data:image/")},
|
| 1852 |
+
"measures": list(measures or []),
|
| 1853 |
+
"today": str(today) if today else None,
|
| 1854 |
+
# Wave-5 item 1: who is looking, host-computed β what gates the client's
|
| 1855 |
+
# "Edit field permissions" menu entry. The HOST-side enforcement is the wall;
|
| 1856 |
+
# this is the courtesy that keeps the menu honest.
|
| 1857 |
+
"viewer": ({"name": str(viewer.get("name") or "")[:80],
|
| 1858 |
+
"isAdmin": bool(viewer.get("isAdmin"))}
|
| 1859 |
+
if isinstance(viewer, dict) else None),
|
| 1860 |
+
"measureSets": {str(k): list(v) for k, v in (measure_sets or {}).items()}}
|
| 1861 |
+
if directory is not None:
|
| 1862 |
+
cache_key = str(directory)
|
| 1863 |
+
component = _DECLARED_COMPONENTS.get(cache_key)
|
| 1864 |
+
if component is None:
|
| 1865 |
+
component = _c.declare_component("aios_customer_grid", path=cache_key)
|
| 1866 |
+
_DECLARED_COMPONENTS[cache_key] = component
|
| 1867 |
+
return component(payload=payload, height=height, key=key, default=None)
|
| 1868 |
+
html = build_html(rows, fields, workspace=workspace)
|
| 1869 |
+
_c.html(html, height=height, scrolling=False)
|
| 1870 |
+
return None
|
platform/aios_grid_fields.json
ADDED
|
@@ -0,0 +1,413 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"_comment": "CANONICAL field contract for the AIOS Airtable-style grid \u2014 the SINGLE source of truth. Consumed by platform/aios_grid.py (embed/Space host) and aios-web/api/main.py (standalone API), and regenerated into aios-web/web/public/sample_customers.json. Edit HERE only, then run aios-web/verify_fields_contract.py. source=odoo is READ-ONLY; source=overlay is the editable stratum (notes/tags) outside Odoo. type in {text,status,select,currency,int,date,pct} (select = a fixed-choice READ-ONLY brand attribute; dba is the first, wave 2026-08-02). `description` (wave 5) is the CANONICAL per-field description \u2014 every field must carry one, and since wave 7 (owner W8, 2026-07-28) every description is ONE SHORT PLAIN sentence (two only when a fact would otherwise mislead): what the field IS, nothing else \u2014 no filter tips, no '(none)' coaching, no rationale; the user's workspace NOTE overrides it in the (i) hover, never in this file. BUILDER FACT (documented here, deliberately NOT in user-facing text): blank text attributes display as '(none)', so `is '(none)'` \u2014 not `is empty` \u2014 finds the blanks on agent/city/state/country/zip/payment_terms/pricelist/tags. filterable:false = the CONDITION BUILDER does not offer it (still displayed, still sortable); every such field must have a replacement declared in aios-web/verify_fields_contract.py. 2026-07-27 partner attributes: country/zip/payment_terms/pricelist/tags/customer_since all ship default:false. zip is TEXT because a postal code has leading zeros. Odoo's credit_limit (1% populated) and user_id salesperson (2%) are deliberately ABSENT; agent_ids is the salesperson field and AR is where credit exposure comes from. Wave-5 item 8 (2026-07-27): ltm_rev and at_risk are DELETED \u2014 LTM's replacement is a creatable Sales measure column (the demo column IS Sales \u00b7 the last 12 months), at_risk's replacement is a formula field, e.g. MAX(0, {revenue_ly} - {revenue_ytd}). Wave-6 item 8 (2026-07-27, the no-buildable-presets rule): revenue_ytd, revenue_ly, orders_24m, aov and yoy_pct are DELETED \u2014 every one is self-buildable, so a frozen pre-set beside the builder was two ways to ask one question. Replacements (recorded in verify_fields_contract.py): creatable measure columns for Sales / Orders / Avg order $ over any period (harness/measure_filter.py ADMITTED carries revenue, orders and the composite aov), and a formula over two measure columns for YoY, e.g. ({sales_ytd} - {sales_ly}) / {sales_ly}. Stale view colIds naming the five self-heal on the next autosave (the established rule).",
|
| 3 |
+
"fields": [
|
| 4 |
+
{
|
| 5 |
+
"key": "customer",
|
| 6 |
+
"label": "Customer",
|
| 7 |
+
"type": "text",
|
| 8 |
+
"source": "odoo",
|
| 9 |
+
"pinned": true,
|
| 10 |
+
"default": true,
|
| 11 |
+
"description": "The customer's name in Odoo. One row per customer who ordered in the last 24 months."
|
| 12 |
+
},
|
| 13 |
+
{
|
| 14 |
+
"key": "odoo_status",
|
| 15 |
+
"label": "Odoo record",
|
| 16 |
+
"type": "status",
|
| 17 |
+
"source": "odoo",
|
| 18 |
+
"default": false,
|
| 19 |
+
"description": "Whether this customer still exists in Odoo. Archived means deleted there."
|
| 20 |
+
},
|
| 21 |
+
{
|
| 22 |
+
"key": "agent",
|
| 23 |
+
"label": "Agent",
|
| 24 |
+
"type": "text",
|
| 25 |
+
"source": "odoo",
|
| 26 |
+
"default": true,
|
| 27 |
+
"description": "The sales agent who owns this account."
|
| 28 |
+
},
|
| 29 |
+
{
|
| 30 |
+
"key": "dba",
|
| 31 |
+
"label": "DBA",
|
| 32 |
+
"type": "select",
|
| 33 |
+
"source": "odoo",
|
| 34 |
+
"default": false,
|
| 35 |
+
"options": [
|
| 36 |
+
"Fisch",
|
| 37 |
+
"Royal",
|
| 38 |
+
"Both"
|
| 39 |
+
],
|
| 40 |
+
"description": "The brand this customer buys from - Fisch, Royal, or both. Amazon-channel orders are not a DBA."
|
| 41 |
+
},
|
| 42 |
+
{
|
| 43 |
+
"key": "salesperson",
|
| 44 |
+
"label": "Salesperson",
|
| 45 |
+
"type": "text",
|
| 46 |
+
"source": "odoo",
|
| 47 |
+
"default": false,
|
| 48 |
+
"description": "Who keyed in most of this customer's orders \u2014 not the Agent, who owns the account."
|
| 49 |
+
},
|
| 50 |
+
{
|
| 51 |
+
"key": "city",
|
| 52 |
+
"label": "City",
|
| 53 |
+
"type": "text",
|
| 54 |
+
"source": "odoo",
|
| 55 |
+
"default": true,
|
| 56 |
+
"description": "City on the customer's Odoo address."
|
| 57 |
+
},
|
| 58 |
+
{
|
| 59 |
+
"key": "state",
|
| 60 |
+
"label": "State",
|
| 61 |
+
"type": "text",
|
| 62 |
+
"source": "odoo",
|
| 63 |
+
"default": true,
|
| 64 |
+
"description": "State or province on the customer's Odoo address."
|
| 65 |
+
},
|
| 66 |
+
{
|
| 67 |
+
"key": "country",
|
| 68 |
+
"label": "Country",
|
| 69 |
+
"type": "text",
|
| 70 |
+
"source": "odoo",
|
| 71 |
+
"default": false,
|
| 72 |
+
"description": "Country on the customer's Odoo address."
|
| 73 |
+
},
|
| 74 |
+
{
|
| 75 |
+
"key": "zip",
|
| 76 |
+
"label": "ZIP",
|
| 77 |
+
"type": "text",
|
| 78 |
+
"source": "odoo",
|
| 79 |
+
"default": false,
|
| 80 |
+
"description": "Postal code on the customer's Odoo address."
|
| 81 |
+
},
|
| 82 |
+
{
|
| 83 |
+
"key": "customer_since",
|
| 84 |
+
"label": "Customer since",
|
| 85 |
+
"type": "date",
|
| 86 |
+
"source": "odoo",
|
| 87 |
+
"default": false,
|
| 88 |
+
"description": "When this customer was first set up in Odoo."
|
| 89 |
+
},
|
| 90 |
+
{
|
| 91 |
+
"key": "tags",
|
| 92 |
+
"label": "Tags",
|
| 93 |
+
"type": "text",
|
| 94 |
+
"source": "odoo",
|
| 95 |
+
"default": false,
|
| 96 |
+
"description": "Odoo labels on this customer, comma-separated."
|
| 97 |
+
},
|
| 98 |
+
{
|
| 99 |
+
"key": "pricelist",
|
| 100 |
+
"label": "Price list",
|
| 101 |
+
"type": "text",
|
| 102 |
+
"source": "odoo",
|
| 103 |
+
"default": false,
|
| 104 |
+
"description": "The price list this customer buys on."
|
| 105 |
+
},
|
| 106 |
+
{
|
| 107 |
+
"key": "payment_terms",
|
| 108 |
+
"label": "Payment terms",
|
| 109 |
+
"type": "text",
|
| 110 |
+
"source": "odoo",
|
| 111 |
+
"default": false,
|
| 112 |
+
"description": "Payment terms on this customer's account \u2014 Net 30, for example."
|
| 113 |
+
},
|
| 114 |
+
{
|
| 115 |
+
"key": "last_order",
|
| 116 |
+
"label": "Last order",
|
| 117 |
+
"type": "date",
|
| 118 |
+
"source": "odoo",
|
| 119 |
+
"default": true,
|
| 120 |
+
"description": "Date of the most recent confirmed order."
|
| 121 |
+
},
|
| 122 |
+
{
|
| 123 |
+
"key": "overdue_days",
|
| 124 |
+
"label": "Overdue days",
|
| 125 |
+
"type": "int",
|
| 126 |
+
"source": "odoo",
|
| 127 |
+
"default": true,
|
| 128 |
+
"description": "How many days late this customer is running against their own usual ordering rhythm."
|
| 129 |
+
},
|
| 130 |
+
{
|
| 131 |
+
"_note": "filterable:false \u2014 DERIVED ANALYTIC: est_missed is min(cycles missed, 3) x AOV, a score we compute rather than an object the business has, so a condition on it would read as a fact about the customer when it is a fact about our arithmetic. It still displays and still sorts. Until wave 6 this flag also covered the frozen-window presets (revenue_ytd / revenue_ly / orders_24m / aov / yoy_pct); those are now DELETED outright under the owner's no-buildable-presets rule \u2014 see _comment. est_missed itself STAYS: no creatable measure or formula reproduces the cadence model behind it.",
|
| 132 |
+
"key": "est_missed",
|
| 133 |
+
"label": "Est. missed $",
|
| 134 |
+
"type": "currency",
|
| 135 |
+
"source": "odoo",
|
| 136 |
+
"default": true,
|
| 137 |
+
"agg": "sum",
|
| 138 |
+
"filterable": false,
|
| 139 |
+
"description": "Estimated sales missed while quiet: missed orders (capped at 3) times average order value. An estimate, not money owed."
|
| 140 |
+
},
|
| 141 |
+
{
|
| 142 |
+
"key": "ar_open",
|
| 143 |
+
"label": "AR open $",
|
| 144 |
+
"type": "currency",
|
| 145 |
+
"source": "odoo",
|
| 146 |
+
"default": false,
|
| 147 |
+
"description": "Invoiced money owed, not yet overdue."
|
| 148 |
+
},
|
| 149 |
+
{
|
| 150 |
+
"key": "ar_overdue",
|
| 151 |
+
"label": "AR overdue $",
|
| 152 |
+
"type": "currency",
|
| 153 |
+
"source": "odoo",
|
| 154 |
+
"default": false,
|
| 155 |
+
"description": "Invoiced money past due \u2014 same basis as the Collections page."
|
| 156 |
+
},
|
| 157 |
+
{
|
| 158 |
+
"key": "ar_exposure",
|
| 159 |
+
"label": "Credit exposure $",
|
| 160 |
+
"type": "currency",
|
| 161 |
+
"source": "odoo",
|
| 162 |
+
"default": false,
|
| 163 |
+
"description": "The most you could be out if they stopped paying today: open, overdue, draft and not-yet-invoiced."
|
| 164 |
+
},
|
| 165 |
+
{
|
| 166 |
+
"key": "ar_aged_1_30",
|
| 167 |
+
"label": "1-30 days $",
|
| 168 |
+
"type": "currency",
|
| 169 |
+
"source": "odoo",
|
| 170 |
+
"default": false,
|
| 171 |
+
"description": "Overdue between 1 and 30 days. The four aging buckets sum to AR overdue $."
|
| 172 |
+
},
|
| 173 |
+
{
|
| 174 |
+
"key": "ar_aged_31_60",
|
| 175 |
+
"label": "31-60 days $",
|
| 176 |
+
"type": "currency",
|
| 177 |
+
"source": "odoo",
|
| 178 |
+
"default": false,
|
| 179 |
+
"description": "Overdue between 31 and 60 days. The four aging buckets sum to AR overdue $."
|
| 180 |
+
},
|
| 181 |
+
{
|
| 182 |
+
"key": "ar_aged_61_90",
|
| 183 |
+
"label": "61-90 days $",
|
| 184 |
+
"type": "currency",
|
| 185 |
+
"source": "odoo",
|
| 186 |
+
"default": false,
|
| 187 |
+
"description": "Overdue between 61 and 90 days. The four aging buckets sum to AR overdue $."
|
| 188 |
+
},
|
| 189 |
+
{
|
| 190 |
+
"key": "ar_aged_90_plus",
|
| 191 |
+
"label": "90+ days $",
|
| 192 |
+
"type": "currency",
|
| 193 |
+
"source": "odoo",
|
| 194 |
+
"default": false,
|
| 195 |
+
"description": "Overdue by more than 90 days. The four aging buckets sum to AR overdue $."
|
| 196 |
+
},
|
| 197 |
+
{
|
| 198 |
+
"key": "days_to_pay",
|
| 199 |
+
"label": "Days to pay",
|
| 200 |
+
"type": "int",
|
| 201 |
+
"source": "odoo",
|
| 202 |
+
"default": false,
|
| 203 |
+
"description": "Average days to pay an invoice in full. Blank means no fully paid invoice yet."
|
| 204 |
+
},
|
| 205 |
+
{
|
| 206 |
+
"key": "top_category",
|
| 207 |
+
"label": "Top category",
|
| 208 |
+
"type": "text",
|
| 209 |
+
"source": "odoo",
|
| 210 |
+
"default": false,
|
| 211 |
+
"description": "The category this customer spent the most on in the last 12 months."
|
| 212 |
+
},
|
| 213 |
+
{
|
| 214 |
+
"key": "top_category_pct",
|
| 215 |
+
"label": "Top category %",
|
| 216 |
+
"type": "pct",
|
| 217 |
+
"source": "odoo",
|
| 218 |
+
"default": false,
|
| 219 |
+
"description": "Share of last-12-months spend that went to the top category."
|
| 220 |
+
},
|
| 221 |
+
{
|
| 222 |
+
"key": "sku_count",
|
| 223 |
+
"label": "SKUs bought",
|
| 224 |
+
"type": "int",
|
| 225 |
+
"source": "odoo",
|
| 226 |
+
"default": false,
|
| 227 |
+
"description": "Distinct products bought in the last 12 months."
|
| 228 |
+
},
|
| 229 |
+
{
|
| 230 |
+
"key": "top_sku",
|
| 231 |
+
"label": "Top SKU",
|
| 232 |
+
"type": "text",
|
| 233 |
+
"source": "odoo",
|
| 234 |
+
"default": false,
|
| 235 |
+
"description": "The product this customer spent the most on in the last 12 months."
|
| 236 |
+
},
|
| 237 |
+
{
|
| 238 |
+
"key": "days_since",
|
| 239 |
+
"label": "Days since order",
|
| 240 |
+
"type": "int",
|
| 241 |
+
"source": "odoo",
|
| 242 |
+
"default": false,
|
| 243 |
+
"description": "Days since the last confirmed order."
|
| 244 |
+
},
|
| 245 |
+
{
|
| 246 |
+
"key": "typical_gap_days",
|
| 247 |
+
"label": "Typical gap days",
|
| 248 |
+
"type": "int",
|
| 249 |
+
"source": "odoo",
|
| 250 |
+
"default": false,
|
| 251 |
+
"description": "Days this customer usually goes between orders, from their own history."
|
| 252 |
+
},
|
| 253 |
+
{
|
| 254 |
+
"key": "notes",
|
| 255 |
+
"label": "Notes",
|
| 256 |
+
"type": "text",
|
| 257 |
+
"source": "overlay",
|
| 258 |
+
"default": false,
|
| 259 |
+
"description": "Your notes on this customer. Saved in this app only, visible only to you."
|
| 260 |
+
}
|
| 261 |
+
],
|
| 262 |
+
"_product_comment": "ADDITIVE, wave 15 C-TOPIC. The PRODUCT table's field contract. Kept as a SEPARATE top-level key rather than restructuring `fields` into {customer_data, product_data}: both existing readers (aios_grid._load_fields, aios-web/api/main.py) index doc['fields'] directly, and reshaping that mid-wave would break the embed for a cosmetic gain. The keyed shape can arrive when both readers move in ONE commit; until then this is the product half and `fields` is the customer half.",
|
| 263 |
+
"_product_removed_buy_now": "OWNER, 2026-08-03: 'Buy signal' (key buy_now, a select of Buy now / OK) is NO LONGER A PRESET FIELD. It never earned one: it is a formula over two columns that are both still right here, and the platform has a formula field type for exactly that. THE FORMULA, which reproduces the retired column row for row (modules/product_data.validate proves the equivalence, and goes red if it ever stops holding): IF({lead_days} > 0, IF({dos} < {lead_days}, \"Buy now\", \"OK\"), \"\") . Every branch matches the old server rule, including the blanks - the formula engine refuses a comparison against a blank rather than coercing it to 0, so a SKU with no days-of-supply or no lead time comes out empty, which is 'we do not know' and not 'you are fine'. NOTE the column is still COMPUTED in product_data.pool(): it ships nowhere (rows_from_pool projects strictly through this contract, so no Field means no cell on the wire) and exists only as validate()'s oracle. A formula field is PER-USER, so nothing shared may filter on it - the Buy list view filters on dos/lead_days directly (_seed_wave17).",
|
| 264 |
+
"product_data": {
|
| 265 |
+
"identity": "pid",
|
| 266 |
+
"business_key": "code",
|
| 267 |
+
"fields": [
|
| 268 |
+
{
|
| 269 |
+
"key": "code",
|
| 270 |
+
"label": "SKU",
|
| 271 |
+
"type": "text",
|
| 272 |
+
"source": "odoo",
|
| 273 |
+
"pinned": true,
|
| 274 |
+
"default": true,
|
| 275 |
+
"description": "The SKU code \u2014 the product's real business key. `pid` is a stable CRC32 of it because the grid keys on an integer."
|
| 276 |
+
},
|
| 277 |
+
{
|
| 278 |
+
"key": "product",
|
| 279 |
+
"label": "Product",
|
| 280 |
+
"type": "text",
|
| 281 |
+
"source": "odoo",
|
| 282 |
+
"default": true,
|
| 283 |
+
"description": "Product name as it appears in Odoo."
|
| 284 |
+
},
|
| 285 |
+
{
|
| 286 |
+
"key": "category",
|
| 287 |
+
"label": "Category",
|
| 288 |
+
"type": "select",
|
| 289 |
+
"source": "odoo",
|
| 290 |
+
"default": true,
|
| 291 |
+
"description": "Product category; '(uncategorized)' when Odoo carries none."
|
| 292 |
+
},
|
| 293 |
+
{
|
| 294 |
+
"key": "supplier",
|
| 295 |
+
"label": "Supplier",
|
| 296 |
+
"type": "text",
|
| 297 |
+
"source": "odoo",
|
| 298 |
+
"default": true,
|
| 299 |
+
"description": "Who makes it, from the inventory mastersheet. Not an Odoo field."
|
| 300 |
+
},
|
| 301 |
+
{
|
| 302 |
+
"key": "origin_country",
|
| 303 |
+
"label": "Country",
|
| 304 |
+
"type": "text",
|
| 305 |
+
"source": "odoo",
|
| 306 |
+
"default": false,
|
| 307 |
+
"description": "Country of origin, from the inventory mastersheet."
|
| 308 |
+
},
|
| 309 |
+
{
|
| 310 |
+
"key": "lead_days",
|
| 311 |
+
"label": "Lead time (days)",
|
| 312 |
+
"type": "int",
|
| 313 |
+
"source": "odoo",
|
| 314 |
+
"default": true,
|
| 315 |
+
"description": "Order-to-arrival days for this supplier. Drives the buy signal."
|
| 316 |
+
},
|
| 317 |
+
{
|
| 318 |
+
"key": "first_cost",
|
| 319 |
+
"label": "First cost",
|
| 320 |
+
"type": "currency",
|
| 321 |
+
"source": "odoo",
|
| 322 |
+
"default": false,
|
| 323 |
+
"description": "Quoted unit cost at origin, before freight and duty."
|
| 324 |
+
},
|
| 325 |
+
{
|
| 326 |
+
"key": "rev_ytd",
|
| 327 |
+
"label": "Revenue YTD",
|
| 328 |
+
"type": "currency",
|
| 329 |
+
"source": "odoo",
|
| 330 |
+
"default": true,
|
| 331 |
+
"description": "Year-to-date revenue for this SKU, BU-scoped when the caller is."
|
| 332 |
+
},
|
| 333 |
+
{
|
| 334 |
+
"key": "rev_ly",
|
| 335 |
+
"label": "Revenue LY",
|
| 336 |
+
"type": "currency",
|
| 337 |
+
"source": "odoo",
|
| 338 |
+
"description": "Same period last year \u2014 seasonal wholesale compares like for like."
|
| 339 |
+
},
|
| 340 |
+
{
|
| 341 |
+
"key": "yoy_pct",
|
| 342 |
+
"label": "YoY %",
|
| 343 |
+
"type": "pct",
|
| 344 |
+
"source": "odoo",
|
| 345 |
+
"description": "Year-over-year change; null when last year was zero (a ratio to zero is not a number)."
|
| 346 |
+
},
|
| 347 |
+
{
|
| 348 |
+
"key": "qty_ytd",
|
| 349 |
+
"label": "Units YTD",
|
| 350 |
+
"type": "int",
|
| 351 |
+
"source": "odoo",
|
| 352 |
+
"description": "Units sold year to date."
|
| 353 |
+
},
|
| 354 |
+
{
|
| 355 |
+
"key": "orders_ytd",
|
| 356 |
+
"label": "Orders YTD",
|
| 357 |
+
"type": "int",
|
| 358 |
+
"source": "odoo",
|
| 359 |
+
"description": "Distinct orders containing this SKU, year to date."
|
| 360 |
+
},
|
| 361 |
+
{
|
| 362 |
+
"key": "on_hand",
|
| 363 |
+
"label": "On hand",
|
| 364 |
+
"type": "int",
|
| 365 |
+
"source": "odoo",
|
| 366 |
+
"description": "Units in stock. CONSOLIDATED \u2014 one physical warehouse, not brand-tagged, so this column is ABSENT for a BU-scoped caller rather than silently company-wide."
|
| 367 |
+
},
|
| 368 |
+
{
|
| 369 |
+
"key": "unit_cost",
|
| 370 |
+
"label": "Unit cost",
|
| 371 |
+
"type": "currency",
|
| 372 |
+
"source": "odoo",
|
| 373 |
+
"description": "Inventory unit cost. Consolidated; absent for a BU-scoped caller."
|
| 374 |
+
},
|
| 375 |
+
{
|
| 376 |
+
"key": "inv_value",
|
| 377 |
+
"label": "Stock value",
|
| 378 |
+
"type": "currency",
|
| 379 |
+
"source": "odoo",
|
| 380 |
+
"description": "On-hand value at cost. Consolidated; absent for a BU-scoped caller."
|
| 381 |
+
},
|
| 382 |
+
{
|
| 383 |
+
"key": "qty_ltm",
|
| 384 |
+
"label": "Units LTM",
|
| 385 |
+
"type": "int",
|
| 386 |
+
"source": "odoo",
|
| 387 |
+
"description": "Units sold in the last twelve months. Consolidated; absent for a BU-scoped caller."
|
| 388 |
+
},
|
| 389 |
+
{
|
| 390 |
+
"key": "dos",
|
| 391 |
+
"label": "Days of supply",
|
| 392 |
+
"type": "int",
|
| 393 |
+
"source": "odoo",
|
| 394 |
+
"description": "Days of supply at the LTM rate; null means it never sells through. Consolidated; absent for a BU-scoped caller."
|
| 395 |
+
},
|
| 396 |
+
{
|
| 397 |
+
"key": "cover_gap_d",
|
| 398 |
+
"label": "Cover gap (days)",
|
| 399 |
+
"type": "int",
|
| 400 |
+
"source": "odoo",
|
| 401 |
+
"default": false,
|
| 402 |
+
"description": "Days of supply minus lead time. Negative means it runs out before a reorder lands."
|
| 403 |
+
},
|
| 404 |
+
{
|
| 405 |
+
"key": "stock_bucket",
|
| 406 |
+
"label": "Stock status",
|
| 407 |
+
"type": "select",
|
| 408 |
+
"source": "odoo",
|
| 409 |
+
"description": "Dead / excess / healthy bucket from the inventory module. Consolidated; absent for a BU-scoped caller."
|
| 410 |
+
}
|
| 411 |
+
]
|
| 412 |
+
}
|
| 413 |
+
}
|
platform/core/assets.py
ADDED
|
@@ -0,0 +1,73 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""core/assets.py β the product-image ASSET SOURCE (wave 18, C2-ASSET).
|
| 2 |
+
|
| 3 |
+
WHY THIS FILE EXISTS AT ALL, given the route is three functions long: the API process is
|
| 4 |
+
HOST-AGNOSTIC by contract (`ops/verify_portability.py` checks B1/B2 β the HuggingFace SDK may
|
| 5 |
+
appear ONLY in its declared homes, all of which live under `platform/`, so the FastAPI
|
| 6 |
+
service can be lifted to Hetzner/EC2 without dragging a hub client with it). The first version
|
| 7 |
+
of `routes_assets.py` imported `huggingface_hub` directly and the gate caught it. The fetch is
|
| 8 |
+
a STORE-shaped concern anyway, so it belongs beside `core/store.py`.
|
| 9 |
+
|
| 10 |
+
Two sources, checked by the caller in order:
|
| 11 |
+
* a local master directory (`AIOS_ASSET_DIR`) β dev, and the fastest path on the owner's box;
|
| 12 |
+
* the private HF dataset `AIOS_ASSET_REPO` (default `royal-imports/product-assets`), laid out
|
| 13 |
+
by `ingest_product_assets.py`:
|
| 14 |
+
assets/products/orig/<CODE><ext> masters, archived verbatim (owner ruling R9)
|
| 15 |
+
assets/products/web/<CODE>.png ~800px derivatives (what the app displays)
|
| 16 |
+
manifest.json {"codes": {"<CODE>": {"ext": ".png"}}, ...}
|
| 17 |
+
|
| 18 |
+
Every failure returns None/{} rather than raising: a missing image is "no picture for that SKU",
|
| 19 |
+
never a broken catalog page.
|
| 20 |
+
"""
|
| 21 |
+
import os
|
| 22 |
+
import time
|
| 23 |
+
from pathlib import Path
|
| 24 |
+
|
| 25 |
+
DEFAULT_REPO = 'royal-imports/product-assets'
|
| 26 |
+
_MANIFEST_TTL = 300
|
| 27 |
+
#: (fetched_at, {"CODE": {"ext": ".png"}}) β one manifest read per process per TTL.
|
| 28 |
+
_MANIFEST = {'at': 0.0, 'codes': None}
|
| 29 |
+
|
| 30 |
+
|
| 31 |
+
def repo_id():
|
| 32 |
+
return os.environ.get('AIOS_ASSET_REPO', '').strip() or DEFAULT_REPO
|
| 33 |
+
|
| 34 |
+
|
| 35 |
+
def _token():
|
| 36 |
+
return os.environ.get('HF_TOKEN') or None
|
| 37 |
+
|
| 38 |
+
|
| 39 |
+
def manifest():
|
| 40 |
+
"""{CODE: {'ext': '.png'}} for the configured asset repo, or {} when unreachable."""
|
| 41 |
+
now = time.time()
|
| 42 |
+
if _MANIFEST['codes'] is not None and now - _MANIFEST['at'] < _MANIFEST_TTL:
|
| 43 |
+
return _MANIFEST['codes']
|
| 44 |
+
try:
|
| 45 |
+
import json
|
| 46 |
+
|
| 47 |
+
from huggingface_hub import hf_hub_download
|
| 48 |
+
p = hf_hub_download(repo_id(), 'manifest.json', repo_type='dataset', token=_token())
|
| 49 |
+
codes = (json.loads(Path(p).read_text(encoding='utf-8')) or {}).get('codes') or {}
|
| 50 |
+
except Exception:
|
| 51 |
+
codes = {}
|
| 52 |
+
_MANIFEST.update(at=now, codes=codes)
|
| 53 |
+
return codes
|
| 54 |
+
|
| 55 |
+
|
| 56 |
+
def fetch(code, quality='web'):
|
| 57 |
+
"""A local Path to one asset, or None. The hub cache dedupes across requests, so a container
|
| 58 |
+
downloads each file at most once."""
|
| 59 |
+
meta = manifest().get(code)
|
| 60 |
+
if not meta:
|
| 61 |
+
return None
|
| 62 |
+
rel = (f'assets/products/web/{code}.png' if quality == 'web'
|
| 63 |
+
else f"assets/products/orig/{code}{meta.get('ext', '.png')}")
|
| 64 |
+
try:
|
| 65 |
+
from huggingface_hub import hf_hub_download
|
| 66 |
+
return Path(hf_hub_download(repo_id(), rel, repo_type='dataset', token=_token()))
|
| 67 |
+
except Exception:
|
| 68 |
+
return None
|
| 69 |
+
|
| 70 |
+
|
| 71 |
+
def reset_cache():
|
| 72 |
+
"""Drop the memoized manifest β for gates and for a post-ingest refresh."""
|
| 73 |
+
_MANIFEST.update(at=0.0, codes=None)
|
platform/core/auth_google.py
ADDED
|
@@ -0,0 +1,91 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Sign in with Google β TEMPLATE ONLY (wave-7 W10, 2026-07-28). Nothing here runs in
|
| 2 |
+
production yet: every flow function raises until the wiring checklist in
|
| 3 |
+
docs/GOOGLE_LOGIN_TODO.md is completed and GOOGLE_LOGIN_ENABLED=1 is set.
|
| 4 |
+
|
| 5 |
+
DESIGN (recorded now so the full build has a contract to meet):
|
| 6 |
+
- OAuth 2.0 authorization-code flow with PKCE against Google Identity
|
| 7 |
+
(https://accounts.google.com/.well-known/openid-configuration). No SDK dependency β
|
| 8 |
+
three HTTPS calls (authorize redirect, token exchange, JWKS fetch) keep the surface
|
| 9 |
+
auditable and the requirements.txt unchanged until we commit.
|
| 10 |
+
- Google is an IDENTITY, not an ACCOUNT SOURCE. A Google sign-in maps to an EXISTING
|
| 11 |
+
core/users.py account via its `email` attribute (set_access(..., email=...)); an
|
| 12 |
+
unknown email FAILS CLOSED with "no account for this Google identity" β Google login
|
| 13 |
+
never creates users, so the per-BU/per-module grant model stays the only door.
|
| 14 |
+
- The password gate stays byte-for-byte intact beside it. APP_PASSWORD remains the
|
| 15 |
+
bootstrap master; removing password auth is a separate, owner-approved step.
|
| 16 |
+
|
| 17 |
+
ENV (placeholders β see .env.example additions in docs/GOOGLE_LOGIN_TODO.md):
|
| 18 |
+
GOOGLE_OAUTH_CLIENT_ID OAuth client id from Google Cloud console
|
| 19 |
+
GOOGLE_OAUTH_CLIENT_SECRET its secret (Space secret / .env β never committed)
|
| 20 |
+
GOOGLE_OAUTH_REDIRECT_URI e.g. https://royal-imports-cfo-os.hf.space/ (must be
|
| 21 |
+
registered VERBATIM in the console; localhost:8501 for dev)
|
| 22 |
+
GOOGLE_LOGIN_ENABLED '1' arms the flow; anything else keeps this a template
|
| 23 |
+
"""
|
| 24 |
+
from __future__ import annotations
|
| 25 |
+
|
| 26 |
+
import os
|
| 27 |
+
|
| 28 |
+
AUTH_ENDPOINT = "https://accounts.google.com/o/oauth2/v2/auth"
|
| 29 |
+
TOKEN_ENDPOINT = "https://oauth2.googleapis.com/token"
|
| 30 |
+
JWKS_URI = "https://www.googleapis.com/oauth2/v3/certs"
|
| 31 |
+
SCOPES = "openid email profile"
|
| 32 |
+
|
| 33 |
+
|
| 34 |
+
def configured() -> bool:
|
| 35 |
+
"""True when the console credentials exist β the LOGIN BUTTON renders iff this is True.
|
| 36 |
+
(Separate from enabled(): a configured-but-disabled deploy shows nothing either, so a
|
| 37 |
+
half-finished setup never paints a dead button.)"""
|
| 38 |
+
return bool(os.environ.get("GOOGLE_OAUTH_CLIENT_ID")) and enabled()
|
| 39 |
+
|
| 40 |
+
|
| 41 |
+
def enabled() -> bool:
|
| 42 |
+
return os.environ.get("GOOGLE_LOGIN_ENABLED", "") == "1"
|
| 43 |
+
|
| 44 |
+
|
| 45 |
+
def auth_url(state: str, code_challenge: str) -> str:
|
| 46 |
+
"""The Google consent-screen URL for the authorization-code + PKCE flow.
|
| 47 |
+
|
| 48 |
+
`state` must be an unguessable per-session token the callback VERIFIES (CSRF);
|
| 49 |
+
`code_challenge` is BASE64URL(SHA256(code_verifier)) with the verifier held in the
|
| 50 |
+
session. Template guard: raises until the TODO checklist lands.
|
| 51 |
+
"""
|
| 52 |
+
raise NotImplementedError(
|
| 53 |
+
"Google login is a template β complete docs/GOOGLE_LOGIN_TODO.md, then implement "
|
| 54 |
+
"auth_url() (urlencode client_id, redirect_uri, response_type=code, scope, state, "
|
| 55 |
+
"code_challenge, code_challenge_method=S256 onto AUTH_ENDPOINT)."
|
| 56 |
+
)
|
| 57 |
+
|
| 58 |
+
|
| 59 |
+
def exchange_code(code: str, code_verifier: str) -> dict:
|
| 60 |
+
"""POST the authorization code to TOKEN_ENDPOINT β {'id_token', 'access_token', ...}.
|
| 61 |
+
Template guard: raises until implemented (requests.post with client_id/secret,
|
| 62 |
+
redirect_uri, grant_type=authorization_code, code, code_verifier)."""
|
| 63 |
+
raise NotImplementedError(
|
| 64 |
+
"Google login is a template β implement exchange_code() per docs/GOOGLE_LOGIN_TODO.md."
|
| 65 |
+
)
|
| 66 |
+
|
| 67 |
+
|
| 68 |
+
def verify_id_token(id_token: str) -> dict:
|
| 69 |
+
"""Validate the JWT against Google's JWKS (signature, iss, aud=client_id, exp) and
|
| 70 |
+
return its claims ({'email', 'email_verified', 'name', ...}). MUST reject
|
| 71 |
+
email_verified=False. Template guard: raises until implemented."""
|
| 72 |
+
raise NotImplementedError(
|
| 73 |
+
"Google login is a template β implement verify_id_token() per docs/GOOGLE_LOGIN_TODO.md."
|
| 74 |
+
)
|
| 75 |
+
|
| 76 |
+
|
| 77 |
+
def account_for_claims(claims: dict):
|
| 78 |
+
"""Map verified Google claims β the core/users.py account, FAIL-CLOSED.
|
| 79 |
+
|
| 80 |
+
The one function already real: the mapping rule is the design's heart and testable
|
| 81 |
+
today. Returns the users.registry() record whose `email` equals the verified claim
|
| 82 |
+
(case-insensitive), only if the account is active; None otherwise β never a new user.
|
| 83 |
+
"""
|
| 84 |
+
email = str(claims.get("email") or "").strip().lower()
|
| 85 |
+
if not email or claims.get("email_verified") is not True:
|
| 86 |
+
return None
|
| 87 |
+
from core import users
|
| 88 |
+
for username, u in (users.registry() or {}).items():
|
| 89 |
+
if str(u.get("email") or "").strip().lower() == email and u.get("active", True):
|
| 90 |
+
return users._public(username, u)
|
| 91 |
+
return None
|
platform/core/context.py
ADDED
|
@@ -0,0 +1,59 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Request context β the single object that carries the GLOBAL filters through a page render.
|
| 2 |
+
|
| 3 |
+
Instead of threading date_from / date_to / team_id / ... through every function, a page builds
|
| 4 |
+
one `Ctx` from the sidebar controls and passes it (or its primitives) down. Modules read only
|
| 5 |
+
what they need. Add a new global filter once here and every module can opt into it.
|
| 6 |
+
|
| 7 |
+
The DBA (doing-business-as) brand filter lives here as `team_id`:
|
| 8 |
+
None -> All / consolidated (Fisch + Royal)
|
| 9 |
+
5 -> Fisch
|
| 10 |
+
6 -> Royal
|
| 11 |
+
HQ / company-level modules (cash flow, balance sheet, AR/AP, consolidated P&L) ignore brand and
|
| 12 |
+
call `ctx.consolidated()`; operational modules (sales, customers, products, margins) honour it.
|
| 13 |
+
"""
|
| 14 |
+
from dataclasses import dataclass, replace
|
| 15 |
+
import core.odoo as O
|
| 16 |
+
import core.periods as P
|
| 17 |
+
|
| 18 |
+
# label -> team_id used by the sidebar Brand (DBA) selector
|
| 19 |
+
BRANDS = {'All': None, 'Fisch': 5, 'Royal': 6}
|
| 20 |
+
|
| 21 |
+
|
| 22 |
+
@dataclass(frozen=True)
|
| 23 |
+
class Ctx:
|
| 24 |
+
date_from: str
|
| 25 |
+
date_to: str
|
| 26 |
+
period_label: str = 'Year to date'
|
| 27 |
+
team_id: int | None = None # None = All DBAs (consolidated); 5 = Fisch; 6 = Royal
|
| 28 |
+
doc_mode: str = 'order' # reserved: 'order' vs 'invoice' recognition basis
|
| 29 |
+
|
| 30 |
+
@property
|
| 31 |
+
def brand(self) -> str:
|
| 32 |
+
return O.TEAM_NAMES.get(self.team_id, 'All') if self.team_id else 'All'
|
| 33 |
+
|
| 34 |
+
@property
|
| 35 |
+
def teams(self) -> list:
|
| 36 |
+
"""Team ids in scope (single brand, or both when consolidated)."""
|
| 37 |
+
return [self.team_id] if self.team_id else list(O.TEAM_IDS)
|
| 38 |
+
|
| 39 |
+
@property
|
| 40 |
+
def window(self) -> tuple:
|
| 41 |
+
return self.date_from, self.date_to
|
| 42 |
+
|
| 43 |
+
def consolidated(self) -> 'Ctx':
|
| 44 |
+
"""All-DBA copy β HQ modules use this so the brand filter never scopes company numbers."""
|
| 45 |
+
return replace(self, team_id=None)
|
| 46 |
+
|
| 47 |
+
def with_period(self, date_from, date_to, label=None) -> 'Ctx':
|
| 48 |
+
return replace(self, date_from=date_from, date_to=date_to,
|
| 49 |
+
period_label=label or self.period_label)
|
| 50 |
+
|
| 51 |
+
|
| 52 |
+
def from_ui(period_label, period_window, brand_label='All', doc_mode='order') -> Ctx:
|
| 53 |
+
df, dt = period_window
|
| 54 |
+
return Ctx(df, dt, period_label, BRANDS.get(brand_label), doc_mode)
|
| 55 |
+
|
| 56 |
+
|
| 57 |
+
def default(t=None) -> Ctx:
|
| 58 |
+
df, dt = P.ytd(t)
|
| 59 |
+
return Ctx(df, dt, 'Year to date', None)
|
platform/core/grid_events.py
ADDED
|
@@ -0,0 +1,1334 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""core/grid_events.py β THE WRITE SEAM, lifted out of Streamlit (EXIT-1a, X1, 2026-07-30).
|
| 2 |
+
|
| 3 |
+
This is the grid's event handler: the code that takes an UNTRUSTED event object from the
|
| 4 |
+
browser component and decides what may be persisted. It was `app.py:_cl_handle_grid_event` /
|
| 5 |
+
`_cl_handle_one` and moved here as a PURE MOVE β no behavior change inside a move (MA principle
|
| 6 |
+
5). Every body below is the app.py body verbatim; the ONLY mechanical rewrites are the
|
| 7 |
+
Streamlit couplings the contract names:
|
| 8 |
+
|
| 9 |
+
st.session_state['_cl_grid_seen_ids'] -> ctx.seen_ids (adapter-owned dedup state)
|
| 10 |
+
st.session_state['_cl_doc_payload'] -> ctx.out.doc (EventResult.doc)
|
| 11 |
+
st.session_state['_cohort_toast'] -> ctx.out.toast (EventResult.toast)
|
| 12 |
+
st.session_state['cl_table_workspace'] -> ctx.fallback_ws (store-unavailable fallback)
|
| 13 |
+
is_admin() (the bare call inside doc_delete, and _cl_table_workspace's admin=None default)
|
| 14 |
+
-> ctx.admin
|
| 15 |
+
|
| 16 |
+
There is NO `import streamlit` in this file and there must never be one β `verify_seam.py`
|
| 17 |
+
enforces that with a `sys.meta_path` block, which is the only proof that survives refactoring.
|
| 18 |
+
|
| 19 |
+
TWO ADAPTERS, ONE HANDLER:
|
| 20 |
+
* the Streamlit adapter β app.py keeps `_cl_handle_grid_event(event, uname, allowed_pids, β¦)`
|
| 21 |
+
with its old name and signature, builds an EventCtx whose `seen_ids` and `fallback_ws` are
|
| 22 |
+
session-state-backed dicts, and applies the Result to session state.
|
| 23 |
+
* the HTTP adapter β `aios-web/api` builds an EventCtx with `fallback_ws=None`. A None
|
| 24 |
+
fallback with the store unavailable is a **503**, never a silent in-memory workspace: an
|
| 25 |
+
API request has no durable session dict to fall back to, so pretending to persist would
|
| 26 |
+
lose the write with a 200 on it.
|
| 27 |
+
|
| 28 |
+
β LAYERING, stated honestly. `core/` is not supposed to import `modules/` (ARCHITECTURE Β§1's
|
| 29 |
+
one-directional rule). This handler genuinely straddles: it is the WRITE half of the customer
|
| 30 |
+
table and its persistence verbs live in `modules.customer_data` / `modules.cohort`. The
|
| 31 |
+
contracted path for the strangler is `core/grid_events.py` (both adapters import it from here),
|
| 32 |
+
so the import stays and the violation is RECORDED rather than hidden. It is the one file in
|
| 33 |
+
`core/` that points up; when EXIT-5 ports the surfaces, the persistence verbs are what move
|
| 34 |
+
down, and this note is the marker for that work.
|
| 35 |
+
"""
|
| 36 |
+
import datetime as dt
|
| 37 |
+
from dataclasses import dataclass, field
|
| 38 |
+
from typing import MutableMapping, Optional
|
| 39 |
+
|
| 40 |
+
import core.store as store
|
| 41 |
+
import core.table_store as _tstore # wave-9 I17: the shared-view authorisation predicates
|
| 42 |
+
import core.users as users
|
| 43 |
+
import harness.telemetry as _tel # _tel.error() = THE error sink (all caught errors log)
|
| 44 |
+
import modules.cohort as cohort_mod
|
| 45 |
+
import modules.customer_data as cl_mod
|
| 46 |
+
|
| 47 |
+
#: C5 (owner item 12): customer documents. The cap is enforced HOST-side on the DECODED byte
|
| 48 |
+
#: count β the client checks the same number first, but a client check only spares the user a
|
| 49 |
+
#: pointless upload; this is what bounds what the tenant's dataset can be made to hold.
|
| 50 |
+
#: MOVED from app.py:1410β1412 with the handler (app.py re-imports these during the strangler).
|
| 51 |
+
_DOCS_KEY = 'customer_docs'
|
| 52 |
+
_DOC_MAX_BYTES = 5 * 1024 * 1024
|
| 53 |
+
_DOC_MAX_PER_ROW = 50
|
| 54 |
+
|
| 55 |
+
#: The props of a field def the CLIENT actually draws β the wave-6 no-blip comparison set.
|
| 56 |
+
#: Host-only stamps (createdBy, permissions) are deliberately absent: the client renders
|
| 57 |
+
#: nothing from its own copy of those, and a permissions refusal is answered explicitly.
|
| 58 |
+
_FIELD_ECHO_PROPS = ('label', 'type', 'note', 'options', 'colorCodeOptions',
|
| 59 |
+
'optionColors', 'max', 'formula', 'agg', 'scope', 'format', 'measure')
|
| 60 |
+
|
| 61 |
+
|
| 62 |
+
class StoreUnavailable(RuntimeError):
|
| 63 |
+
"""The store is down AND the caller offered no durable fallback (HTTP adapter β 503).
|
| 64 |
+
|
| 65 |
+
The Streamlit adapter never sees this: it passes a session-state-backed `fallback_ws`, so
|
| 66 |
+
its store-down path degrades to the same captioned in-memory workspace it always had. An
|
| 67 |
+
API caller has no such dict, and the honest answer to "persist this" with nowhere to
|
| 68 |
+
persist it is an error β not a 200 over a write that evaporates.
|
| 69 |
+
"""
|
| 70 |
+
|
| 71 |
+
|
| 72 |
+
@dataclass
|
| 73 |
+
class EventResult:
|
| 74 |
+
"""What the HANDLER needs to tell its adapter β the three things it cannot do itself.
|
| 75 |
+
|
| 76 |
+
`rerender` is the old return value with its old meaning: "this page must re-render from
|
| 77 |
+
server state", NOT "handled" (see `handle_one`'s docstring β the distinction is the
|
| 78 |
+
difference between a snappy grid and a sluggish one).
|
| 79 |
+
"""
|
| 80 |
+
rerender: bool = False
|
| 81 |
+
doc: Optional[dict] = None # doc_fetch's answer β the bridge has no response channel
|
| 82 |
+
toast: Optional[str] = None # add_to_list's confirmation
|
| 83 |
+
|
| 84 |
+
|
| 85 |
+
@dataclass
|
| 86 |
+
class EventCtx:
|
| 87 |
+
"""Everything the handler is allowed to know, and nothing about how it is being served.
|
| 88 |
+
|
| 89 |
+
β `allowed_pids` is a SECURITY parameter: it is the pool this caller may touch at all, and
|
| 90 |
+
every row-scoped event (overlay_patch, doc_*, cohort membership, add_to_list, memberPids)
|
| 91 |
+
is intersected with it. `admin` and `uname` are likewise the HOST's statement of identity β
|
| 92 |
+
never anything the browser said about itself.
|
| 93 |
+
|
| 94 |
+
`seen_ids` is ADAPTER-OWNED dedup state (the event log resends its recent window on every
|
| 95 |
+
emit, so consumed ids must stay consumed): Streamlit passes a session-state dict, HTTP
|
| 96 |
+
passes the caller's per-session mapping.
|
| 97 |
+
|
| 98 |
+
`fallback_ws` is the store-unavailable workspace home, the OUTER dict keyed by username
|
| 99 |
+
(exactly what `st.session_state.setdefault('cl_table_workspace', {})` is). HTTP passes
|
| 100 |
+
None, and None + store down raises StoreUnavailable (β 503).
|
| 101 |
+
|
| 102 |
+
`out` is the sink for the two parked answers, so `handle_one` can keep returning a bare
|
| 103 |
+
bool exactly as it did in app.py while doc/toast still escape. (Additive amendment to X1's
|
| 104 |
+
field list, 2026-07-30 β defaulted; recorded in the S1 mailbox.)
|
| 105 |
+
"""
|
| 106 |
+
uname: str
|
| 107 |
+
allowed_pids: frozenset
|
| 108 |
+
fields: list
|
| 109 |
+
measure_keys: frozenset = frozenset()
|
| 110 |
+
resolved_ids: frozenset = frozenset()
|
| 111 |
+
cohort_ids: frozenset = frozenset()
|
| 112 |
+
measure_offer: tuple = ()
|
| 113 |
+
visible_views: tuple = ()
|
| 114 |
+
admin: bool = False
|
| 115 |
+
#: β WAVE 15 C-PERM β the fields this caller's PERMANENT PERMISSIONS hide, transitive
|
| 116 |
+
#: closure included (`core.perm_scope.hidden_keys`). A SECURITY parameter in the same class
|
| 117 |
+
#: as `allowed_pids`, and here for the same reason: the read paths strip a hidden field from
|
| 118 |
+
#: both wires, so a caller cannot SEE it β but without this they could still WRITE it by
|
| 119 |
+
#: naming the key, because `overlay_keys` only asks whether the column exists. Empty means
|
| 120 |
+
#: "nothing hidden", which is every legacy record and therefore every caller today.
|
| 121 |
+
hidden_keys: frozenset = frozenset()
|
| 122 |
+
scope_key: str = 'customer'
|
| 123 |
+
#: Wave 16 C-TOPIC β the TABLE OPS this ctx reads and writes (a `core.table_store.make`
|
| 124 |
+
#: object). None = the CUSTOMER table (`modules.customer_data.TABLE_OPS`), which keeps
|
| 125 |
+
#: every existing caller byte-identical. The PRODUCT surface passes
|
| 126 |
+
#: `modules.product_data.TABLE_OPS` so its views/fields/overlays land in the product
|
| 127 |
+
#: bucket β never the customer one, whose pids are a different id space entirely.
|
| 128 |
+
table: Optional[object] = None
|
| 129 |
+
seen_ids: Optional[MutableMapping] = None
|
| 130 |
+
fallback_ws: Optional[MutableMapping] = None
|
| 131 |
+
out: EventResult = field(default_factory=EventResult)
|
| 132 |
+
|
| 133 |
+
def __post_init__(self):
|
| 134 |
+
# A caller that forgets `seen_ids` gets per-call dedup rather than a crash β the SET is
|
| 135 |
+
# what makes re-delivery idempotent, and its absence must not be the thing that decides
|
| 136 |
+
# whether an event is processed at all.
|
| 137 |
+
if self.seen_ids is None:
|
| 138 |
+
self.seen_ids = {}
|
| 139 |
+
|
| 140 |
+
|
| 141 |
+
def _tops(ctx):
|
| 142 |
+
"""The table-ops object this ctx operates on β the customer table unless the caller chose
|
| 143 |
+
another topic. ONE resolution point so 'which bucket does this write land in' has exactly
|
| 144 |
+
one answer per event."""
|
| 145 |
+
return ctx.table if ctx.table is not None else cl_mod.TABLE_OPS
|
| 146 |
+
|
| 147 |
+
|
| 148 |
+
# ------------------------------------------------------------------ the workspace family (X1 amd.)
|
| 149 |
+
def table_workspace(ctx, allowed_pids=None, consume_corrections=True):
|
| 150 |
+
"""Durable Airtable-style view/schema/overlay state, with an honest session fallback.
|
| 151 |
+
|
| 152 |
+
Wave-9 I17: the returned `views` are this user's OWN views MERGED with every view SHARED
|
| 153 |
+
with them. Own views win a same-id collision β a personal view is the user's own object and
|
| 154 |
+
must never be shadowed by somebody else's share.
|
| 155 |
+
|
| 156 |
+
β `allowed_pids` is a SECURITY parameter, not a convenience. A shared view carries
|
| 157 |
+
`memberPids`, which the writer's scope validated on the way IN β but the READER may have a
|
| 158 |
+
narrower scope. Handing a Fisch-scoped user a view whose member list was built by a
|
| 159 |
+
full-access user would leak the other BU's customers straight through the strict-isolation
|
| 160 |
+
rule. So membership is re-filtered against the READER on the way OUT. Pass it wherever the
|
| 161 |
+
result reaches a renderer or an event handler; omitting it drops memberPids entirely rather
|
| 162 |
+
than trusting the stored list (fail-closed: a smaller view, never a wider one).
|
| 163 |
+
|
| 164 |
+
MOVED from app.py:5323 (`_cl_table_workspace`). `admin` was a parameter defaulting to a live
|
| 165 |
+
`is_admin()` call; it now always comes from `ctx.admin`, which is identical at the one call
|
| 166 |
+
site because app.py computes `admin = bool(is_admin())` once per render. `allowed_pids`
|
| 167 |
+
stays an EXPLICIT argument because the folder leg deliberately omits it.
|
| 168 |
+
"""
|
| 169 |
+
uname = ctx.uname
|
| 170 |
+
if store.available():
|
| 171 |
+
ws = _tops(ctx).workspace(
|
| 172 |
+
uname, consume_corrections=bool(consume_corrections))
|
| 173 |
+
try:
|
| 174 |
+
shared = _tops(ctx).shared_views(uname, bool(ctx.admin))
|
| 175 |
+
except Exception as _se:
|
| 176 |
+
_tel.error('sharedviews:read', _se)
|
| 177 |
+
shared = {}
|
| 178 |
+
if shared:
|
| 179 |
+
merged = dict(shared)
|
| 180 |
+
merged.update(ws.get('views') or {}) # own views win the id collision
|
| 181 |
+
ws['views'] = {vid: scope_view(v, uname, allowed_pids)
|
| 182 |
+
for vid, v in merged.items()}
|
| 183 |
+
return ws
|
| 184 |
+
# Store down. Streamlit degrades to its captioned session workspace exactly as before; an
|
| 185 |
+
# API caller has no durable dict and must be told, not quietly served memory.
|
| 186 |
+
if ctx.fallback_ws is None:
|
| 187 |
+
raise StoreUnavailable('table workspace unavailable: no store and no session fallback')
|
| 188 |
+
return ctx.fallback_ws.setdefault(
|
| 189 |
+
uname, {'views': {}, 'fields': {}, 'overlays': {}})
|
| 190 |
+
|
| 191 |
+
|
| 192 |
+
def scope_view(view, uname, allowed_pids):
|
| 193 |
+
"""Re-scope a view to the READER. A no-op for a view the reader created.
|
| 194 |
+
|
| 195 |
+
Two things travel inside a shared view that are SCOPE, not presentation:
|
| 196 |
+
1. `memberPids` β a customer list. Re-filtered against the reader's permitted pool.
|
| 197 |
+
2. cohort conditions β a cohort id is a permission handle owned by ONE user. For a reader
|
| 198 |
+
who does not own it the leaf is UNANSWERABLE, and partial evaluation of an unanswerable
|
| 199 |
+
leaf WIDENS `is none of` ([[cg-cohort-set-conditions]]) β i.e. it would show MORE rows,
|
| 200 |
+
not fewer. So the whole filter tree is dropped for a foreign reader rather than
|
| 201 |
+
evaluated with a hole in it. A visibly unfiltered shared view is recoverable; a
|
| 202 |
+
silently widened one is a leak.
|
| 203 |
+
"""
|
| 204 |
+
if not isinstance(view, dict) or view.get('createdBy') == uname:
|
| 205 |
+
return view
|
| 206 |
+
out = dict(view)
|
| 207 |
+
cfg = dict(out.get('config') or {})
|
| 208 |
+
mp = cfg.get('memberPids')
|
| 209 |
+
if isinstance(mp, list):
|
| 210 |
+
cfg['memberPids'] = ([p for p in mp if p in (allowed_pids or ())]
|
| 211 |
+
if allowed_pids is not None else [])
|
| 212 |
+
if tree_has_cohort(cfg.get('filters')):
|
| 213 |
+
cfg['filters'] = []
|
| 214 |
+
cfg['_scopedOut'] = 'cohort' # the client says so rather than pretending
|
| 215 |
+
out['config'] = cfg
|
| 216 |
+
return out
|
| 217 |
+
|
| 218 |
+
|
| 219 |
+
def tree_has_cohort(node):
|
| 220 |
+
"""True when a filter tree references the Cohorts column anywhere (any depth)."""
|
| 221 |
+
import aios_grid as _agc
|
| 222 |
+
col = _agc.COHORT_COLUMN
|
| 223 |
+
if isinstance(node, list):
|
| 224 |
+
return any(tree_has_cohort(n) for n in node)
|
| 225 |
+
if not isinstance(node, dict):
|
| 226 |
+
return False
|
| 227 |
+
if node.get('colId') == col or node.get('field') == col:
|
| 228 |
+
return True
|
| 229 |
+
return tree_has_cohort(node.get('conditions') or node.get('children'))
|
| 230 |
+
|
| 231 |
+
|
| 232 |
+
def measure_leaves(nodes):
|
| 233 |
+
"""Every leaf of a filter tree that carries a date WINDOW β i.e. every measure condition.
|
| 234 |
+
|
| 235 |
+
Keyed off the window rather than the column name on purpose: a window is the one thing no
|
| 236 |
+
column condition has, so this cannot mistake `revenue_ytd > 5000` for a measure.
|
| 237 |
+
"""
|
| 238 |
+
out = []
|
| 239 |
+
for node in nodes or []:
|
| 240 |
+
if not isinstance(node, dict):
|
| 241 |
+
continue
|
| 242 |
+
if isinstance(node.get('children'), list):
|
| 243 |
+
out.extend(measure_leaves(node['children']))
|
| 244 |
+
elif node.get('window') is not None:
|
| 245 |
+
out.append(node)
|
| 246 |
+
return out
|
| 247 |
+
|
| 248 |
+
|
| 249 |
+
# ------------------------------------------------------------------ private handler helpers
|
| 250 |
+
def _field_echo_equal(accepted, raw):
|
| 251 |
+
"""Does the ACCEPTED def match what the client already renders? (wave-6 item 3)
|
| 252 |
+
|
| 253 |
+
The no-blip law: a field_upsert whose acceptance changed NOTHING the client draws needs no
|
| 254 |
+
re-render β the client is the ORIGIN of the change, its optimistic copy is already right,
|
| 255 |
+
and the rerun it would buy costs a full payload re-serialisation plus an iframe remount
|
| 256 |
+
(the "blip" the owner named). STRICT per-prop equality after collapsing the empty forms
|
| 257 |
+
('' / [] / {} / absent all render identically): any normalisation a validator applied is a
|
| 258 |
+
real difference the client must be SHOWN. Fail-closed β unsure means not equal means rerun.
|
| 259 |
+
"""
|
| 260 |
+
empty = ('', [], {})
|
| 261 |
+
for k in _FIELD_ECHO_PROPS:
|
| 262 |
+
a, b = accepted.get(k), raw.get(k) if isinstance(raw, dict) else None
|
| 263 |
+
if k == 'scope':
|
| 264 |
+
# 'global' and absent are the SAME rendering (an unscoped field) β the event may
|
| 265 |
+
# say 'global' explicitly while the stored def says it by omission.
|
| 266 |
+
a = None if a == 'global' else a
|
| 267 |
+
b = None if b == 'global' else b
|
| 268 |
+
if (None if a in empty else a) != (None if b in empty else b):
|
| 269 |
+
return False
|
| 270 |
+
return True
|
| 271 |
+
|
| 272 |
+
|
| 273 |
+
def _unique_name(wanted, existing, *, fallback='Untitled', max_len=120):
|
| 274 |
+
"""Return a durable, case-insensitively unique field/view display name."""
|
| 275 |
+
limit = max(1, int(max_len))
|
| 276 |
+
|
| 277 |
+
def _clean(value):
|
| 278 |
+
return ' '.join(str(value or '').split())
|
| 279 |
+
|
| 280 |
+
base = (_clean(wanted) or _clean(fallback) or 'Untitled')[:limit].rstrip()
|
| 281 |
+
taken = {_clean(value).casefold() for value in existing if _clean(value)}
|
| 282 |
+
if base.casefold() not in taken:
|
| 283 |
+
return base
|
| 284 |
+
index = 2
|
| 285 |
+
while True:
|
| 286 |
+
suffix = f' {index}'
|
| 287 |
+
stem = base[:max(0, limit - len(suffix))].rstrip()
|
| 288 |
+
candidate = f'{stem}{suffix}' if stem else str(index)[-limit:]
|
| 289 |
+
if candidate.casefold() not in taken:
|
| 290 |
+
return candidate
|
| 291 |
+
index += 1
|
| 292 |
+
|
| 293 |
+
|
| 294 |
+
def _known_usernames():
|
| 295 |
+
"""Real account names, for C4's fail-closed `users` grant. None = "could not resolve".
|
| 296 |
+
|
| 297 |
+
None means DO NOT FILTER, deliberately. Returning an empty set on a transient registry
|
| 298 |
+
error would silently collapse a user's stored 'specific users' grant to 'personal' on the
|
| 299 |
+
very next autosave β destroying a setting because a read blipped. The grant is inert today
|
| 300 |
+
anyway (views are per-user; nothing reads it to allow cross-user access), so the honest
|
| 301 |
+
trade is: never invent access, never destroy a stored intent either.
|
| 302 |
+
"""
|
| 303 |
+
try:
|
| 304 |
+
reg = set(users.registry() or {})
|
| 305 |
+
except Exception as _ue:
|
| 306 |
+
_tel.error('viewperms:users', _ue)
|
| 307 |
+
return None
|
| 308 |
+
# β EMPTY IS "UNRESOLVED", NOT "NOBODY EXISTS". registry() returns {} rather than raising
|
| 309 |
+
# when the user store cannot be read (a store blip logs `store:get:users` and hands back an
|
| 310 |
+
# empty dict) β and a working app ALWAYS has at least the bootstrap admin. Treating {} as a
|
| 311 |
+
# real answer would filter every name out of a stored grant and collapse it to 'personal'
|
| 312 |
+
# on the next autosave. Caught by testing this in BARE mode with no store token.
|
| 313 |
+
return reg or None
|
| 314 |
+
|
| 315 |
+
|
| 316 |
+
# ------------------------------------------------------------------ THE SEAM
|
| 317 |
+
def handle_events(events, ctx):
|
| 318 |
+
"""The event LOG entry point β one EventResult. The `[-24:]` window lives here.
|
| 319 |
+
|
| 320 |
+
β THE VALUE IS AN EVENT LOG `{'events': [...]}`, NOT ONE EVENT (2026-07-27). A component
|
| 321 |
+
has exactly one value slot, and two quick emits share it: creating a field emits
|
| 322 |
+
`field_upsert` and the insertColumn autosave emits `view_upsert` 420ms later β with the
|
| 323 |
+
server still inside the first rerun, the second write clobbered the slot before any run
|
| 324 |
+
read it, and the field creation VANISHED with no error anywhere (measured live; the same
|
| 325 |
+
race sat latent under overlay-field creation, masked by localStorage). The client now
|
| 326 |
+
sends its recent window of events; each is processed exactly once, keyed by its id. The
|
| 327 |
+
single-event shape is still accepted β it is what an older client in a stale tab sends.
|
| 328 |
+
|
| 329 |
+
Accepts everything the component value can be: the log dict, a bare list of events (what
|
| 330 |
+
the HTTP body's `events` array unwraps to), or one legacy single-event object.
|
| 331 |
+
|
| 332 |
+
`EventResult.rerender` is True when ANY processed event demands a re-render from server
|
| 333 |
+
state β the same OR-fold `_cl_handle_grid_event` did.
|
| 334 |
+
"""
|
| 335 |
+
if isinstance(events, dict) and isinstance(events.get('events'), list):
|
| 336 |
+
seq = events['events'][-24:]
|
| 337 |
+
elif isinstance(events, list):
|
| 338 |
+
seq = events[-24:]
|
| 339 |
+
else:
|
| 340 |
+
seq = [events]
|
| 341 |
+
rerun = False
|
| 342 |
+
for one in seq:
|
| 343 |
+
rerun = handle_one(one, ctx) or rerun
|
| 344 |
+
ctx.out.rerender = rerun
|
| 345 |
+
return ctx.out
|
| 346 |
+
|
| 347 |
+
|
| 348 |
+
def handle_one(event, ctx):
|
| 349 |
+
"""Validate ONE untrusted component event.
|
| 350 |
+
|
| 351 |
+
Odoo-source fields are never accepted as cell updates. View and schema objects are
|
| 352 |
+
user-scoped; row patches additionally require the pid to be in the currently permitted
|
| 353 |
+
Customer List pool.
|
| 354 |
+
|
| 355 |
+
RETURN VALUE = "this page must re-render from server state", NOT "handled".
|
| 356 |
+
The distinction is the difference between a snappy grid and a sluggish one.
|
| 357 |
+
Persisting is not by itself a reason to rerun: Streamlit has ALREADY re-run the
|
| 358 |
+
script once simply to deliver the component value, and the client is the ORIGIN
|
| 359 |
+
of the change, so its own state is correct before the server hears about it. An
|
| 360 |
+
extra st.rerun() therefore buys nothing and costs a second full page run.
|
| 361 |
+
Measured on the 1,550-row book (2026-07-25): one filter tweak produced TWO
|
| 362 |
+
page_customer_data runs ~2.8s apart, each re-serialising the whole ~576 KB row
|
| 363 |
+
payload β and the handler on the second run is a guaranteed no-op, because the
|
| 364 |
+
event-id dedupe two lines below has already consumed that id.
|
| 365 |
+
|
| 366 |
+
THE WAVE-6 NO-BLIP LAW (item 3, 2026-07-27) β True ONLY when the next render
|
| 367 |
+
genuinely differs from what the client already draws:
|
| 368 |
+
field_upsert β False when the accepted def ECHOES the client's copy
|
| 369 |
+
(_field_echo_equal); True on any validator mutation, any
|
| 370 |
+
permissions refusal, and ALWAYS when a measure window changed
|
| 371 |
+
(values must be re-resolved server-side).
|
| 372 |
+
overlay_patch β False when every requested update was accepted verbatim;
|
| 373 |
+
True the moment anything was refused or truncated.
|
| 374 |
+
field_delete β False: the client removed the column optimistically and the
|
| 375 |
+
store scrub is invisible to the next render.
|
| 376 |
+
field_duplicate β True only when host-held VALUES must flow back (custom_ cell
|
| 377 |
+
copies, measure_ recomputes); formula/created_time clones
|
| 378 |
+
compute client-side.
|
| 379 |
+
view_delete β False (the deleted view simply stops being served).
|
| 380 |
+
view_upsert β unchanged: False, except an unresolved measure condition.
|
| 381 |
+
cohort_* / add_to_list β True: the lists payload is server truth, and these are
|
| 382 |
+
not the hot path.
|
| 383 |
+
|
| 384 |
+
The id dedupe is a SET (not a last-id slot): the event log resends the whole recent
|
| 385 |
+
window on every emit, so every id must stay consumed, not just the latest one.
|
| 386 |
+
"""
|
| 387 |
+
# The ctx fields the moved body reads as plain locals β bound once here so every branch
|
| 388 |
+
# below is textually the app.py body (which is what makes this move reviewable).
|
| 389 |
+
uname = ctx.uname
|
| 390 |
+
allowed_pids = ctx.allowed_pids
|
| 391 |
+
fields = ctx.fields
|
| 392 |
+
measure_keys = ctx.measure_keys
|
| 393 |
+
resolved_ids = ctx.resolved_ids
|
| 394 |
+
cohort_ids = ctx.cohort_ids
|
| 395 |
+
measure_offer = ctx.measure_offer
|
| 396 |
+
visible_views = ctx.visible_views
|
| 397 |
+
admin = ctx.admin
|
| 398 |
+
scope_key = ctx.scope_key
|
| 399 |
+
|
| 400 |
+
if not isinstance(event, dict):
|
| 401 |
+
return False
|
| 402 |
+
event_id = str(event.get('id') or '')[:180]
|
| 403 |
+
seen = ctx.seen_ids
|
| 404 |
+
if not event_id or event_id in seen:
|
| 405 |
+
return False
|
| 406 |
+
seen[event_id] = True
|
| 407 |
+
if len(seen) > 400: # a session-lifetime set, kept bounded
|
| 408 |
+
for stale in list(seen)[:200]:
|
| 409 |
+
seen.pop(stale, None)
|
| 410 |
+
kind = event.get('type')
|
| 411 |
+
field_by_key = {f['key']: f for f in fields}
|
| 412 |
+
valid_keys = set(field_by_key)
|
| 413 |
+
overlay_keys = {f['key'] for f in fields if f.get('source') == 'overlay'}
|
| 414 |
+
# Event validation is not a render payload. It must not consume a one-shot label
|
| 415 |
+
# correction before the browser has had a chance to receive it.
|
| 416 |
+
ws = table_workspace(
|
| 417 |
+
ctx, allowed_pids=allowed_pids, consume_corrections=False)
|
| 418 |
+
|
| 419 |
+
def _session_ready():
|
| 420 |
+
ws.setdefault('views', {})
|
| 421 |
+
ws.setdefault('fields', {})
|
| 422 |
+
ws.setdefault('overlays', {})
|
| 423 |
+
|
| 424 |
+
if kind == 'view_select':
|
| 425 |
+
# Owner item 3 (2026-07-31): remember WHERE THE USER IS so a fresh browser resumes
|
| 426 |
+
# there instead of the system default view. Presentation state only β the read side
|
| 427 |
+
# re-validates the id against what the caller may see, so nothing here needs an
|
| 428 |
+
# authorisation wall beyond the length cap. Never a re-render: the client is already
|
| 429 |
+
# on that view.
|
| 430 |
+
vid = str(event.get('viewId') or '').strip()[:120]
|
| 431 |
+
if vid and store.available():
|
| 432 |
+
try:
|
| 433 |
+
_tops(ctx).save_active_view(uname, vid)
|
| 434 |
+
except Exception as _ae:
|
| 435 |
+
_tel.error('grid:view-select', _ae)
|
| 436 |
+
return False
|
| 437 |
+
|
| 438 |
+
if kind == 'record_layout':
|
| 439 |
+
# Wave 2026-08-02 (C-LAYOUT): the per-user record-detail field order. Presentation
|
| 440 |
+
# state for one surface; grid column order is untouched. Unknown keys are pruned
|
| 441 |
+
# against the LIVE field set, dupes collapse, and the wire re-validates at serve
|
| 442 |
+
# time β so a client racing a field delete degrades to default order, never an error.
|
| 443 |
+
raw_order = event.get('order')
|
| 444 |
+
if not isinstance(raw_order, list):
|
| 445 |
+
return False
|
| 446 |
+
order, seen = [], set()
|
| 447 |
+
for k in raw_order[:200]:
|
| 448 |
+
k = str(k or '').strip()[:80]
|
| 449 |
+
if k and k in valid_keys and k not in seen:
|
| 450 |
+
seen.add(k)
|
| 451 |
+
order.append(k)
|
| 452 |
+
if store.available():
|
| 453 |
+
_tops(ctx).save_record_layout(uname, order)
|
| 454 |
+
else:
|
| 455 |
+
_session_ready()
|
| 456 |
+
if order:
|
| 457 |
+
ws['recordLayout'] = {'order': order}
|
| 458 |
+
else:
|
| 459 |
+
ws.pop('recordLayout', None)
|
| 460 |
+
# Autosave hot path, like view_select: the client already renders the order it asked
|
| 461 |
+
# for, and the wire echoes it back on the next workspace read.
|
| 462 |
+
return False
|
| 463 |
+
|
| 464 |
+
if kind == 'view_upsert':
|
| 465 |
+
raw = event.get('view')
|
| 466 |
+
if not isinstance(raw, dict) or not isinstance(raw.get('config'), dict):
|
| 467 |
+
return False
|
| 468 |
+
view_id = str(raw.get('id') or '').strip()[:120]
|
| 469 |
+
name = str(raw.get('name') or '').strip()[:120]
|
| 470 |
+
requested_name = name
|
| 471 |
+
if not view_id or not name:
|
| 472 |
+
return False
|
| 473 |
+
cfg = raw['config']
|
| 474 |
+
# The filter TREE (leaf conditions and/or nested condition groups) is
|
| 475 |
+
# validated by the module-agnostic grid contract helper β imported lazily
|
| 476 |
+
# here for the same reason the page does it: a missing embed build must
|
| 477 |
+
# never break page import.
|
| 478 |
+
import aios_grid as _ag
|
| 479 |
+
# A MEASURE key is valid in a FILTER and nowhere else (CG-8): it may not be ordered,
|
| 480 |
+
# shown, grouped or coloured by, because it is not a column of this table. Hence the
|
| 481 |
+
# widened key set here and the plain `valid_keys` everywhere below.
|
| 482 |
+
# `cohort_ids` is a PERMISSION list, not a convenience: a cohort rule naming anything
|
| 483 |
+
# else is dropped here. Leaving it in would persist a condition the engine can only
|
| 484 |
+
# answer with "nothing", so `Cohort is not [a list I deleted]` would show an empty table
|
| 485 |
+
# forever with nothing on screen explaining why.
|
| 486 |
+
filters = _ag.clean_filter_tree(cfg.get('filters'),
|
| 487 |
+
valid_keys | set(measure_keys),
|
| 488 |
+
cohort_ids=set(cohort_ids))
|
| 489 |
+
sorts = []
|
| 490 |
+
for rule in list(cfg.get('sorts') or [])[:20]:
|
| 491 |
+
if (isinstance(rule, dict) and rule.get('colId') in valid_keys
|
| 492 |
+
and rule.get('dir') in ('asc', 'desc')):
|
| 493 |
+
sorts.append({'colId': rule['colId'], 'dir': rule['dir']})
|
| 494 |
+
order = list(dict.fromkeys(
|
| 495 |
+
key for key in list(cfg.get('order') or []) if key in valid_keys))
|
| 496 |
+
order += [field['key'] for field in fields if field['key'] not in order]
|
| 497 |
+
visible = list(dict.fromkeys(
|
| 498 |
+
key for key in list(cfg.get('visible') or []) if key in valid_keys))
|
| 499 |
+
widths = {}
|
| 500 |
+
raw_widths = cfg.get('widths') if isinstance(cfg.get('widths'), dict) else {}
|
| 501 |
+
for key, value in raw_widths.items():
|
| 502 |
+
if key in valid_keys:
|
| 503 |
+
try:
|
| 504 |
+
widths[key] = max(60, min(600, round(float(value))))
|
| 505 |
+
except (TypeError, ValueError):
|
| 506 |
+
pass
|
| 507 |
+
# C-LOCK (wave 2026-08-02, item 12's mechanism): a view may be LOCKED to one cohort.
|
| 508 |
+
# `cohort_ids` is the same permission list the cohort filter leaf uses β a lock naming
|
| 509 |
+
# a cohort this session cannot see is DROPPED at write, exactly like a cohort rule.
|
| 510 |
+
# (Readers of a SHARED locked view resolve the id against their OWN visible sets; an
|
| 511 |
+
# unresolvable lock matches NOTHING client-side β the cohort-leaf law, never a widen.)
|
| 512 |
+
cohort_lock = str(cfg.get('cohortLock') or '').strip()[:120]
|
| 513 |
+
# β WAVE 17 R1 (C-LOCKV guard a) β A PROJECTED LOCKED VIEW RE-STAMPS ITS OWN LOCK.
|
| 514 |
+
# A locked view IS its set: its id and its lock are the same fact. The client rebuilds
|
| 515 |
+
# `config` from scratch on every autosave (a column resize is enough), so taking the
|
| 516 |
+
# lock from the browser would mean one omitted key turns "the 40 accounts we agreed to
|
| 517 |
+
# call" into the whole book, under the same name, with nothing going red. The read side
|
| 518 |
+
# re-stamps too (aios_grid.views_from_defs) β this keeps the STORE from accumulating the
|
| 519 |
+
# wrong record in the first place.
|
| 520 |
+
_is_locked_view = view_id in set(cohort_ids)
|
| 521 |
+
if _is_locked_view:
|
| 522 |
+
cohort_lock = view_id
|
| 523 |
+
config = {
|
| 524 |
+
'filters': filters,
|
| 525 |
+
# conjunction joining the ROOT-level conditions; anything but an
|
| 526 |
+
# explicit 'or' means 'and' (legacy views had an implicit AND).
|
| 527 |
+
'filterConj': 'or' if cfg.get('filterConj') == 'or' else 'and',
|
| 528 |
+
**({'cohortLock': cohort_lock}
|
| 529 |
+
if cohort_lock and cohort_lock in set(cohort_ids) else {}),
|
| 530 |
+
'sorts': sorts,
|
| 531 |
+
'groupBy': cfg.get('groupBy') if cfg.get('groupBy') in valid_keys else None,
|
| 532 |
+
'colorBy': cfg.get('colorBy') if cfg.get('colorBy') in valid_keys else None,
|
| 533 |
+
'rowHeightMode': (cfg.get('rowHeightMode')
|
| 534 |
+
if cfg.get('rowHeightMode') in ('short', 'medium', 'tall')
|
| 535 |
+
else 'short'),
|
| 536 |
+
'order': order,
|
| 537 |
+
'visible': visible,
|
| 538 |
+
'widths': widths,
|
| 539 |
+
'memberPids': [pid for pid in list(cfg.get('memberPids') or [])
|
| 540 |
+
if isinstance(pid, int) and pid in allowed_pids],
|
| 541 |
+
}
|
| 542 |
+
# Wave-6 item 10: a non-grid display mode rides the view config, validated
|
| 543 |
+
# structurally (mode whitelist, refs must be this table's fields).
|
| 544 |
+
display = _ag._clean_display(cfg.get('display'), valid_keys)
|
| 545 |
+
if display:
|
| 546 |
+
config['display'] = display
|
| 547 |
+
# Wave-6 item 11 (pin): how many leading columns stay frozen while scrolling
|
| 548 |
+
# sideways. Clamped 1..8 β 0 would unfreeze the identity column the whole
|
| 549 |
+
# surface pivots on; absent = the legacy single frozen column.
|
| 550 |
+
if cfg.get('frozenCount') is not None:
|
| 551 |
+
try:
|
| 552 |
+
config['frozenCount'] = max(1, min(8, int(cfg['frozenCount'])))
|
| 553 |
+
except (TypeError, ValueError):
|
| 554 |
+
pass
|
| 555 |
+
# ββ wave-9 C3 (lock) + C4 (permissions) ββββββββββββββββββββββββββββββββββββββββββββ
|
| 556 |
+
# The PRIOR stored view decides both: whether this is a create or an update (C4's
|
| 557 |
+
# default splits on exactly that) and who is allowed to change the lock (C3).
|
| 558 |
+
prior = (ws.get('views') or {}).get(view_id) or {}
|
| 559 |
+
# β THE SHARED WALL (wave-9 I17). `ws` is already visibility-filtered, so a view the
|
| 560 |
+
# caller may not SEE reads as absent β which would look like a CREATE and let anybody
|
| 561 |
+
# overwrite somebody else's shared view by guessing its id. Ask the store for the raw
|
| 562 |
+
# record and refuse before deciding anything else. Fail-closed on both branches.
|
| 563 |
+
_raw_shared = _tops(ctx).shared_view(view_id) if store.available() else None
|
| 564 |
+
if _raw_shared is not None and not prior:
|
| 565 |
+
return False # exists, not visible to this caller: refuse
|
| 566 |
+
if _raw_shared is not None and not _tstore._may_edit(_raw_shared, uname, admin):
|
| 567 |
+
return False # visible but not editable by this caller
|
| 568 |
+
is_new = not prior
|
| 569 |
+
# C4: the creator is HOST-STAMPED and never read from the browser β the same rule as
|
| 570 |
+
# the field-level createdBy. An existing view keeps whoever made it.
|
| 571 |
+
creator = prior.get('createdBy') or uname
|
| 572 |
+
# Only the creator or an admin may change the PERMISSIONS themselves. A collaborator who
|
| 573 |
+
# could rewrite them could grant themselves sole ownership of someone else's view, or
|
| 574 |
+
# quietly widen a users-scoped view to everyone β privilege escalation by edit.
|
| 575 |
+
if not is_new and not _tstore._may_administer(prior, uname, admin):
|
| 576 |
+
raw = dict(raw)
|
| 577 |
+
raw['permissions'] = prior.get('permissions')
|
| 578 |
+
raw['locked'] = prior.get('locked')
|
| 579 |
+
# ββ I12/C3 BELT (2026-07-31): a LOCKED view's DISPLAY MODE is frozen HOST-SIDE too.
|
| 580 |
+
# The client's switcher already refuses (setDisplayMode), and its comment has claimed
|
| 581 |
+
# "the host refuses too" since wave 9 β this makes that claim true for direct API
|
| 582 |
+
# callers. β MIRRORS types.ts `isModeFrozen` EXACTLY: SYSTEM views and the undeletable
|
| 583 |
+
# list:/cohort: projections are EXEMPT β their `locked` is legacy "undeletable", not
|
| 584 |
+
# mode-frozen, and freezing them would trap the default view in whatever mode it last
|
| 585 |
+
# saved (the owner's stuck-in-Map report is that trap, client-made). An upsert that
|
| 586 |
+
# UNLOCKS in the same write (authorised above) may change display freely.
|
| 587 |
+
_mode_frozen = (prior.get('locked') and raw.get('locked') is not False
|
| 588 |
+
and prior.get('kind') != 'system'
|
| 589 |
+
and view_id != 'all-customers')
|
| 590 |
+
if _mode_frozen:
|
| 591 |
+
if prior.get('config', {}).get('display'):
|
| 592 |
+
config['display'] = dict(prior['config']['display'])
|
| 593 |
+
else:
|
| 594 |
+
config.pop('display', None)
|
| 595 |
+
# β A SYSTEM VIEW IS PER-USER STATE AND MUST NEVER BE SHARED. Caught in the wave-9
|
| 596 |
+
# integration pass, before deploy: `all-customers` is every user's DEFAULT view and it
|
| 597 |
+
# carries their own column widths, visible fields and row height. C4's read-default
|
| 598 |
+
# ('collaborative' when permissions are absent, so nobody's saved view is
|
| 599 |
+
# retro-restricted) is right for a view somebody AUTHORED β but applied to the system
|
| 600 |
+
# view it routed `all-customers` into the shared bucket, and the first user to resize a
|
| 601 |
+
# column would have published their layout to everyone. Verified in the store:
|
| 602 |
+
# shared_table_views() was returning ['all-customers'].
|
| 603 |
+
# System views are pinned personal, so they stay in each user's own workspace exactly as
|
| 604 |
+
# before this wave. `kind` is host-validated two lines below; read it here from `raw` and
|
| 605 |
+
# treat the stored kind as authoritative for an existing view.
|
| 606 |
+
_kind = (prior.get('kind') if not is_new
|
| 607 |
+
else (raw.get('kind') if raw.get('kind') in ('system', 'list', 'custom')
|
| 608 |
+
else 'custom'))
|
| 609 |
+
if _kind == 'system' or view_id == 'all-customers':
|
| 610 |
+
perms = {'edit': 'personal'}
|
| 611 |
+
else:
|
| 612 |
+
perms = _ag.clean_view_permissions(
|
| 613 |
+
raw.get('permissions'),
|
| 614 |
+
default='personal' if is_new else 'collaborative',
|
| 615 |
+
known_users=_known_usernames())
|
| 616 |
+
# C3: only the creator or an admin may SET or CLEAR the lock; anyone else keeps whatever
|
| 617 |
+
# is already stored. Enforced here rather than hidden in the client. This composes with
|
| 618 |
+
# C4 and does not override it: C4 governs who may edit the CONTENTS, C3 who may freeze
|
| 619 |
+
# the display MODE.
|
| 620 |
+
# β Inert in practice TODAY β a user can only ever write their own per-user workspace,
|
| 621 |
+
# so `uname == creator` always holds. Correct plumbing for shared views, not a live gate.
|
| 622 |
+
may_lock = admin or uname == creator
|
| 623 |
+
locked = bool(raw.get('locked')) if may_lock else bool(prior.get('locked'))
|
| 624 |
+
# C3: the LOCK's actual job β a locked view's display mode is frozen, so a locked Kanban
|
| 625 |
+
# stays Kanban. This half IS live per-user and is the part the owner asked for. Filters,
|
| 626 |
+
# sorts and columns stay editable by design.
|
| 627 |
+
if _mode_frozen:
|
| 628 |
+
was = (prior.get('config') or {}).get('display')
|
| 629 |
+
now = config.get('display')
|
| 630 |
+
if (was or {}).get('mode') != (now or {}).get('mode'):
|
| 631 |
+
if was:
|
| 632 |
+
config['display'] = was
|
| 633 |
+
else:
|
| 634 |
+
config.pop('display', None)
|
| 635 |
+
# WAVE 17 R1 (C-LOCKV guard b) β ONE THING, ONE NAME. Renaming a locked view renames
|
| 636 |
+
# the SET it is, in the store that owns it; the view record must not grow a second title
|
| 637 |
+
# that the read side would then have to arbitrate. Routed rather than duplicated.
|
| 638 |
+
if _is_locked_view and store.available():
|
| 639 |
+
_cur = (cohort_mod.all_for(uname).get(view_id) or {}).get('name')
|
| 640 |
+
if name and name != _cur:
|
| 641 |
+
try:
|
| 642 |
+
cohort_mod.rename(uname, view_id, name)
|
| 643 |
+
except ValueError as _e:
|
| 644 |
+
_tel.error('cohort:rename-via-view', _e)
|
| 645 |
+
view = {'id': view_id, 'name': name,
|
| 646 |
+
# β DERIVED FROM THE ID, never taken from the browser: `kind` is what paints the
|
| 647 |
+
# lock mark, so a client claiming it for an ordinary view would be claiming a
|
| 648 |
+
# guarantee the engine is not making about that view's rows.
|
| 649 |
+
'kind': _ag.LOCKED_VIEW_KIND if _is_locked_view
|
| 650 |
+
else raw.get('kind') if raw.get('kind') in ('system', 'list', 'custom')
|
| 651 |
+
else 'custom',
|
| 652 |
+
'locked': locked,
|
| 653 |
+
'createdBy': creator,
|
| 654 |
+
'permissions': perms,
|
| 655 |
+
'note': str(raw.get('note') or '')[:2000],
|
| 656 |
+
'config': config}
|
| 657 |
+
# System/list projections are not all persisted in this table store, but custom views
|
| 658 |
+
# may not impersonate their visible names. Persisted view names are allocated
|
| 659 |
+
# tenant-globally by TableStore inside the write transaction.
|
| 660 |
+
_reserved_view_names = [
|
| 661 |
+
value.get('name') for value in visible_views
|
| 662 |
+
if (isinstance(value, dict) and value.get('id') != view_id
|
| 663 |
+
and value.get('kind') in ('system', 'list'))
|
| 664 |
+
]
|
| 665 |
+
if view_id != 'all-customers':
|
| 666 |
+
_reserved_view_names.append('All customers')
|
| 667 |
+
if store.available():
|
| 668 |
+
view = (_tops(ctx).save_view(
|
| 669 |
+
uname, view, reserved_names=_reserved_view_names, is_admin=admin) or view)
|
| 670 |
+
else:
|
| 671 |
+
_session_ready()
|
| 672 |
+
_view_names = list(_reserved_view_names)
|
| 673 |
+
_view_names.extend(
|
| 674 |
+
value.get('name')
|
| 675 |
+
for candidate_id, value in ws['views'].items()
|
| 676 |
+
if candidate_id != view_id and isinstance(value, dict)
|
| 677 |
+
)
|
| 678 |
+
view['name'] = _unique_name(view.get('name'), _view_names)
|
| 679 |
+
ws['views'][view_id] = view
|
| 680 |
+
name_changed = view.get('name') != requested_name
|
| 681 |
+
# PERSISTED, but deliberately NO rerun β see the return-value contract in
|
| 682 |
+
# this function's docstring. This is the hot path: every filter, sort,
|
| 683 |
+
# group, colour, column-resize and column-toggle autosaves through here.
|
| 684 |
+
#
|
| 685 |
+
# CG-8 is the ONE exception, and it is self-limiting. A measure condition is answered
|
| 686 |
+
# server-side, so if this view carries one whose answer was NOT in the payload we just
|
| 687 |
+
# shipped, the client is showing a pending marker with nothing coming. Ask for exactly
|
| 688 |
+
# one more run β and only then, so the common case (a column filter) still costs zero.
|
| 689 |
+
# If the top-of-run read of the component's own value already covered it, this is False
|
| 690 |
+
# and nothing extra happens; if that read is ever stale, this is what stops the
|
| 691 |
+
# condition hanging unresolved.
|
| 692 |
+
#
|
| 693 |
+
# β COMPLETE rules only (wave-6 item 3, found by the live roundtrip on this rev):
|
| 694 |
+
# picking a measure FIELD mints a valueless rule whose autosave used to land here as
|
| 695 |
+
# "unresolved" and trigger a rerun that could resolve NOTHING β `rule_complete` refuses
|
| 696 |
+
# half-asked questions β so the only effect was a remount that closed the builder
|
| 697 |
+
# popover mid-build (the wave-5 "gate flake" runs, now explained). A rule still being
|
| 698 |
+
# typed stays client-side PENDING by design; the run that matters fires when its VALUE
|
| 699 |
+
# arrives and the question becomes answerable.
|
| 700 |
+
from harness import measure_filter as _mfc
|
| 701 |
+
unresolved = {str(r.get('id') or '') for r in measure_leaves(filters)
|
| 702 |
+
if _mfc.rule_complete(r)} - set(resolved_ids)
|
| 703 |
+
return name_changed or bool(unresolved - {''})
|
| 704 |
+
|
| 705 |
+
if kind == 'view_delete':
|
| 706 |
+
view_id = str(event.get('viewId') or '')[:120]
|
| 707 |
+
if not view_id or view_id == 'all-customers':
|
| 708 |
+
return False
|
| 709 |
+
# WAVE 17 R1 (C-LOCKV guard c) β deleting a LOCKED view deletes the set it is. Without
|
| 710 |
+
# this the store would drop the saved config and the projection would rebuild the view
|
| 711 |
+
# on the very next render: the row would come back, minus the user's sort and columns,
|
| 712 |
+
# which reads as a corruption rather than as a refusal. Own sets only (`all_for`), the
|
| 713 |
+
# same wall `cohort_delete` rides.
|
| 714 |
+
if store.available() and view_id in cohort_mod.all_for(uname):
|
| 715 |
+
cohort_mod.delete(uname, view_id)
|
| 716 |
+
_tops(ctx).delete_view(uname, view_id)
|
| 717 |
+
# True, unlike an ordinary view_delete: the lists payload IS server truth and the
|
| 718 |
+
# client cannot know the membership channel changed.
|
| 719 |
+
return True
|
| 720 |
+
# Wave-9 I17: deleting a SHARED view is narrower than editing it β creator or admin
|
| 721 |
+
# only. A collaborator can change what a shared view shows; they cannot destroy
|
| 722 |
+
# everybody else's copy of it, because there is only one copy.
|
| 723 |
+
if store.available():
|
| 724 |
+
_raw_shared = _tops(ctx).shared_view(view_id)
|
| 725 |
+
if _raw_shared is not None and not _tstore._may_administer(
|
| 726 |
+
_raw_shared, uname, admin):
|
| 727 |
+
return False
|
| 728 |
+
_tops(ctx).delete_view(uname, view_id)
|
| 729 |
+
else:
|
| 730 |
+
_session_ready()
|
| 731 |
+
ws['views'].pop(view_id, None)
|
| 732 |
+
# Wave-6 item 3: the client removed the view optimistically; a deleted view simply
|
| 733 |
+
# stops being served, so the next render differs in nothing the client still shows.
|
| 734 |
+
return False
|
| 735 |
+
|
| 736 |
+
if kind == 'field_upsert':
|
| 737 |
+
raw = event.get('field')
|
| 738 |
+
if not isinstance(raw, dict):
|
| 739 |
+
return False
|
| 740 |
+
key = str(raw.get('key') or '')[:80]
|
| 741 |
+
import aios_grid as _ag2
|
| 742 |
+
if key.startswith(_ag2.MEASURE_FIELD_PREFIX):
|
| 743 |
+
# A FORMULA-MEASURE column (owner item 7). Validated against THIS caller's measure
|
| 744 |
+
# offer β the same admission the condition builder uses β so a field can only name a
|
| 745 |
+
# measure this user could also filter by. Fail closed on anything else.
|
| 746 |
+
field = _ag2.clean_measure_field(raw, {m['key']: m for m in measure_offer})
|
| 747 |
+
if field is None:
|
| 748 |
+
return False
|
| 749 |
+
fmt = _ag2._clean_format(raw.get('format'), field.get('type'))
|
| 750 |
+
if fmt:
|
| 751 |
+
field['format'] = fmt
|
| 752 |
+
elif key in field_by_key and not key.startswith('custom_'):
|
| 753 |
+
base = field_by_key[key]
|
| 754 |
+
field = {**base, 'note': str(raw.get('note') or '')[:2000]}
|
| 755 |
+
# Wave-5 item 10: a DISPLAY format on any base field (presets included) β how a
|
| 756 |
+
# number/date reads, per user. Rendering-only; validated fail-closed per type.
|
| 757 |
+
fmt = _ag2._clean_format(raw.get('format'), base.get('type'))
|
| 758 |
+
if fmt:
|
| 759 |
+
field['format'] = fmt
|
| 760 |
+
else:
|
| 761 |
+
field.pop('format', None) # cleared -> back to the type's default render
|
| 762 |
+
# PRESET measure fields (wave-2 item 8): the browser may change the PERIOD (and the
|
| 763 |
+
# auto-renamed label) of revenue_ytd / revenue_ly / orders_24m β nothing else. The
|
| 764 |
+
# whitelist IS the base contract: only a field that shipped with preset+measure can
|
| 765 |
+
# take a window here, and the measure KEY is pinned to the base's (fail closed).
|
| 766 |
+
if (base.get('preset') and isinstance(base.get('measure'), dict)
|
| 767 |
+
and isinstance(raw.get('measure'), dict)):
|
| 768 |
+
window = _ag2._clean_window(raw['measure'].get('window'))
|
| 769 |
+
if window is not None:
|
| 770 |
+
field['measure'] = {'key': base['measure']['key'], 'window': window}
|
| 771 |
+
if isinstance(raw.get('label'), str) and raw['label'].strip():
|
| 772 |
+
field['label'] = str(raw['label'])[:120]
|
| 773 |
+
elif key.startswith('custom_') and raw.get('type') in _ag2.CUSTOM_FIELD_TYPES:
|
| 774 |
+
ftype = raw['type']
|
| 775 |
+
readonly = ftype in _ag2.READONLY_CUSTOM_TYPES
|
| 776 |
+
# The editable family must SAY it is overlay (the client's own contract); the
|
| 777 |
+
# read-only pair (formula / created_time) is stored under the cohort column's
|
| 778 |
+
# mechanism instead, so no cell write can ever be accepted for it.
|
| 779 |
+
if not readonly and raw.get('source') != 'overlay':
|
| 780 |
+
return False
|
| 781 |
+
field = {'key': key, 'label': str(raw.get('label') or 'Untitled')[:120],
|
| 782 |
+
'type': ftype, 'source': 'odoo' if readonly else 'overlay',
|
| 783 |
+
'default': True, 'custom': True,
|
| 784 |
+
'note': str(raw.get('note') or '')[:2000]}
|
| 785 |
+
if readonly:
|
| 786 |
+
field['derived'] = True
|
| 787 |
+
# FILTERABLE since wave 6 (owner item 6): formula/created_time values live
|
| 788 |
+
# client-side, and this table's counts are client-mode β the client engine
|
| 789 |
+
# answers conditions on them soundly. Mirrors fields_from_workspace.
|
| 790 |
+
field['filterable'] = True
|
| 791 |
+
if raw.get('agg') == 'sum' and not readonly:
|
| 792 |
+
field['agg'] = 'sum'
|
| 793 |
+
# A `select`/`multiselect` carries its own vocabulary; a `user` does NOT β its
|
| 794 |
+
# choices are the tenant's accounts, resolved fresh on every render, so persisting
|
| 795 |
+
# a snapshot here would go stale the moment somebody joins or leaves.
|
| 796 |
+
if ftype in ('select', 'multiselect'):
|
| 797 |
+
field['options'] = _ag2._clean_options(raw.get('options'))
|
| 798 |
+
if not field['options']:
|
| 799 |
+
return False # a select with no choices can never hold a value
|
| 800 |
+
field.update(_ag2._choice_appearance(raw, field['options']))
|
| 801 |
+
if ftype == 'rating':
|
| 802 |
+
field['max'] = _ag2._clean_rating_max(raw.get('max'))
|
| 803 |
+
if ftype == 'formula':
|
| 804 |
+
# WRITE-time is where refs must exist (fail closed); read-time leaves them
|
| 805 |
+
# alone so a later-deleted ref blanks the CELLS, not the column.
|
| 806 |
+
formula = _ag2._clean_formula(raw.get('formula'), valid_keys)
|
| 807 |
+
if formula is None:
|
| 808 |
+
return False
|
| 809 |
+
field['formula'] = formula
|
| 810 |
+
if ftype == 'automation':
|
| 811 |
+
# C5-AUTOFIELD (wave 18): persist the validated config bag. No `return False`
|
| 812 |
+
# when absent β an unconfigured automation column is a legitimate intermediate
|
| 813 |
+
# state (create the column, then configure it in the gear).
|
| 814 |
+
auto = _ag2._clean_automation(raw.get('automation'), valid_keys)
|
| 815 |
+
if auto:
|
| 816 |
+
field['automation'] = auto
|
| 817 |
+
fmt = _ag2._clean_format(raw.get('format'), ftype)
|
| 818 |
+
if fmt:
|
| 819 |
+
field['format'] = fmt
|
| 820 |
+
else:
|
| 821 |
+
return False
|
| 822 |
+
# Wave-5 item 1 β createdBy + permissions on the CREATED strata (custom_/measure_).
|
| 823 |
+
# `createdBy` is stamped HOST-SIDE at create and preserved on every later write: the
|
| 824 |
+
# client never supplies it, so a browser cannot claim someone else's field. A
|
| 825 |
+
# permissions CHANGE is accepted only from the creator or an admin; a legacy field
|
| 826 |
+
# with no createdBy is admin-only (fail closed, both directions). The table workspace
|
| 827 |
+
# is per-user today, so this wall becomes load-bearing when sharing arrives β but the
|
| 828 |
+
# admin path and the legacy path are enforceable (and NC'd) right now.
|
| 829 |
+
needs_values = False
|
| 830 |
+
refused_perms = False
|
| 831 |
+
if key.startswith('custom_') or key.startswith(_ag2.MEASURE_FIELD_PREFIX):
|
| 832 |
+
prior = ws.get('fields', {}).get(key)
|
| 833 |
+
prior = prior if isinstance(prior, dict) else None
|
| 834 |
+
if prior is None:
|
| 835 |
+
field['createdBy'] = uname
|
| 836 |
+
# Wave-6 item 9: scope is chosen at CREATE, on the cohort page only β
|
| 837 |
+
# 'cohort' marks the field cohort-specific; 'global' or absent stays
|
| 838 |
+
# unscoped, so an old bundle that sends no scope keeps today's behavior.
|
| 839 |
+
if scope_key == 'cohort' and raw.get('scope') == 'cohort':
|
| 840 |
+
field['scope'] = 'cohort'
|
| 841 |
+
else:
|
| 842 |
+
if isinstance(prior.get('createdBy'), str) and prior['createdBy'].strip():
|
| 843 |
+
field['createdBy'] = prior['createdBy']
|
| 844 |
+
# Scope is PRESERVED like createdBy: the client copy never moves an
|
| 845 |
+
# existing field between scopes.
|
| 846 |
+
if prior.get('scope') == 'cohort':
|
| 847 |
+
field['scope'] = 'cohort'
|
| 848 |
+
want = _ag2._clean_permissions(raw.get('permissions'))
|
| 849 |
+
have = _ag2._clean_permissions((prior or {}).get('permissions'))
|
| 850 |
+
owner = field.get('createdBy')
|
| 851 |
+
if want is not None and (admin or (owner and owner == uname)):
|
| 852 |
+
field['permissions'] = want
|
| 853 |
+
else:
|
| 854 |
+
if have is not None:
|
| 855 |
+
field['permissions'] = have
|
| 856 |
+
# Asked for a permissions change and did not get it: the client's
|
| 857 |
+
# optimistic copy is a lie that must be repainted (wave-6 item 3).
|
| 858 |
+
refused_perms = want is not None and want != have
|
| 859 |
+
# A NEW measure column, or an existing one whose window moved: the VALUES
|
| 860 |
+
# are host-computed, so the next render genuinely differs.
|
| 861 |
+
if key.startswith(_ag2.MEASURE_FIELD_PREFIX):
|
| 862 |
+
needs_values = prior is None or prior.get('measure') != field.get('measure')
|
| 863 |
+
# Canonical names are reserved outside the durable workspace. Persisted field names are
|
| 864 |
+
# allocated by TableStore from the transaction's current state, not this stale snapshot.
|
| 865 |
+
_workspace_field_keys = set((ws.get('fields') or {}))
|
| 866 |
+
_reserved_field_names = [
|
| 867 |
+
value.get('label') for candidate_key, value in field_by_key.items()
|
| 868 |
+
if (candidate_key != key and candidate_key not in _workspace_field_keys
|
| 869 |
+
and isinstance(value, dict))
|
| 870 |
+
]
|
| 871 |
+
if store.available():
|
| 872 |
+
field = (_tops(ctx).save_field(
|
| 873 |
+
uname, field, reserved_names=_reserved_field_names,
|
| 874 |
+
correction_id=event_id) or field)
|
| 875 |
+
else:
|
| 876 |
+
_session_ready()
|
| 877 |
+
_field_names = list(_reserved_field_names)
|
| 878 |
+
_field_names.extend(
|
| 879 |
+
value.get('label')
|
| 880 |
+
for candidate_key, value in ws['fields'].items()
|
| 881 |
+
if candidate_key != key and isinstance(value, dict)
|
| 882 |
+
)
|
| 883 |
+
requested_label = ' '.join(
|
| 884 |
+
str(field.get('label') or 'Untitled').split())[:120].rstrip()
|
| 885 |
+
field.pop('labelCorrectedFrom', None)
|
| 886 |
+
field.pop('labelCorrectionId', None)
|
| 887 |
+
field['label'] = _unique_name(requested_label, _field_names)
|
| 888 |
+
if field['label'] != requested_label and event_id:
|
| 889 |
+
field['labelCorrectedFrom'] = requested_label
|
| 890 |
+
field['labelCorrectionId'] = str(event_id)[:180]
|
| 891 |
+
if not field.get('custom') and not str(field.get('note') or '').strip():
|
| 892 |
+
ws['fields'].pop(key, None)
|
| 893 |
+
else:
|
| 894 |
+
ws['fields'][key] = field
|
| 895 |
+
if needs_values or refused_perms:
|
| 896 |
+
return True
|
| 897 |
+
# THE NO-BLIP RETURN (wave-6 item 3): an accepted def that echoes what the client
|
| 898 |
+
# already renders needs no rerun β see _field_echo_equal and the docstring's law.
|
| 899 |
+
return not _field_echo_equal(field, raw)
|
| 900 |
+
|
| 901 |
+
if kind in ('cohort_add', 'cohort_remove', 'cohort_rename', 'cohort_delete'):
|
| 902 |
+
# Cohort membership/lifecycle edits from the grid's cohort panel (wave-2 item 2 β
|
| 903 |
+
# replaces the retired Manage popover, so rename/delete keep a surface). Fail-closed:
|
| 904 |
+
# only the caller's OWN cohorts (all_for), and member pids are intersected with the
|
| 905 |
+
# caller's permitted pool β the same guard add_to_list rides.
|
| 906 |
+
if not store.available():
|
| 907 |
+
return False
|
| 908 |
+
cohort_id = str(event.get('cohortId') or '').strip()[:120]
|
| 909 |
+
if not cohort_id or cohort_id not in cohort_mod.all_for(uname):
|
| 910 |
+
return False
|
| 911 |
+
try:
|
| 912 |
+
if kind == 'cohort_rename':
|
| 913 |
+
name = str(event.get('name') or '').strip()[:120]
|
| 914 |
+
if not name:
|
| 915 |
+
return False
|
| 916 |
+
cohort_mod.rename(uname, cohort_id, name)
|
| 917 |
+
elif kind == 'cohort_delete':
|
| 918 |
+
cohort_mod.delete(uname, cohort_id)
|
| 919 |
+
else:
|
| 920 |
+
raw_pids = event.get('pids')
|
| 921 |
+
if not isinstance(raw_pids, list):
|
| 922 |
+
return False
|
| 923 |
+
pids = [p for p in raw_pids if isinstance(p, int) and p in allowed_pids]
|
| 924 |
+
if not pids:
|
| 925 |
+
return False
|
| 926 |
+
if kind == 'cohort_add':
|
| 927 |
+
cohort_mod.add_members(uname, cohort_id, pids)
|
| 928 |
+
else:
|
| 929 |
+
cohort_mod.remove_members(uname, cohort_id, pids)
|
| 930 |
+
except ValueError as e:
|
| 931 |
+
_tel.error('cohort:panel-edit', e)
|
| 932 |
+
return False
|
| 933 |
+
# True: the lists payload (names, membership, counts) must reflect this on the next
|
| 934 |
+
# render β the panel is showing server truth, not local optimism.
|
| 935 |
+
return True
|
| 936 |
+
|
| 937 |
+
if kind in ('doc_add', 'doc_delete', 'doc_fetch'):
|
| 938 |
+
# DOCUMENTS on a customer row (owner item 12, contract C5). Metadata in the store key
|
| 939 |
+
# `customer_docs`; the BYTES live at their own path in the tenant dataset, never in a
|
| 940 |
+
# JSON blob β see core.store.upload_bytes for why that distinction is load-bearing.
|
| 941 |
+
import base64 as _b64c
|
| 942 |
+
import re as _re_mod
|
| 943 |
+
if not store.available():
|
| 944 |
+
return False
|
| 945 |
+
pid = event.get('pid')
|
| 946 |
+
doc_id = str(event.get('docId') or '').strip()[:80]
|
| 947 |
+
# The pid wall is the SAME one every other row-scoped event rides: a caller may only
|
| 948 |
+
# touch customers inside their own permitted pool. Without it, documents would be the
|
| 949 |
+
# one surface where a scoped agent could read another book's rows.
|
| 950 |
+
if not isinstance(pid, int) or pid not in allowed_pids or not doc_id:
|
| 951 |
+
return False
|
| 952 |
+
meta_all = store.get(_DOCS_KEY) or {}
|
| 953 |
+
row = list((meta_all.get(str(pid)) or []))
|
| 954 |
+
|
| 955 |
+
if kind == 'doc_add':
|
| 956 |
+
raw_b64 = event.get('data_b64')
|
| 957 |
+
name = str(event.get('name') or 'document')[:200]
|
| 958 |
+
if not isinstance(raw_b64, str) or not raw_b64:
|
| 959 |
+
return False
|
| 960 |
+
try:
|
| 961 |
+
blob = _b64c.b64decode(raw_b64, validate=True)
|
| 962 |
+
except Exception:
|
| 963 |
+
return False # not base64 -> not a file we will store
|
| 964 |
+
# Re-check the DECODED size host-side. The client checks it too, but a client check
|
| 965 |
+
# is a courtesy: this is the number that bounds what the tenant's repo can be made
|
| 966 |
+
# to hold by anyone who can reach the endpoint.
|
| 967 |
+
if not blob or len(blob) > _DOC_MAX_BYTES:
|
| 968 |
+
return False
|
| 969 |
+
if len(row) >= _DOC_MAX_PER_ROW or any(d.get('id') == doc_id for d in row):
|
| 970 |
+
return False
|
| 971 |
+
safe = _re_mod.sub(r'[^A-Za-z0-9._-]', '_', name)[:80] or 'file'
|
| 972 |
+
path = f'docs/{pid}/{doc_id}_{safe}'
|
| 973 |
+
try:
|
| 974 |
+
store.upload_bytes(path, blob, message=f'doc {doc_id} for customer {pid}')
|
| 975 |
+
except Exception as _ue:
|
| 976 |
+
_tel.error('docs:upload', _ue)
|
| 977 |
+
return False
|
| 978 |
+
row.append({'id': doc_id, 'name': name,
|
| 979 |
+
'mime': str(event.get('mime') or 'application/octet-stream')[:120],
|
| 980 |
+
'size': len(blob), 'by': uname,
|
| 981 |
+
'ts': dt.datetime.now().strftime('%Y-%m-%dT%H:%M:%S'),
|
| 982 |
+
'path': path})
|
| 983 |
+
elif kind == 'doc_delete':
|
| 984 |
+
hit = next((d for d in row if d.get('id') == doc_id), None)
|
| 985 |
+
if not hit:
|
| 986 |
+
return False
|
| 987 |
+
# THE WALL: the uploader or an admin. Enforced here regardless of what the client
|
| 988 |
+
# chose to render β `canDelete` in the payload only hides a button.
|
| 989 |
+
if hit.get('by') != uname and not admin:
|
| 990 |
+
return False
|
| 991 |
+
store.delete_path(hit.get('path') or '')
|
| 992 |
+
row = [d for d in row if d.get('id') != doc_id]
|
| 993 |
+
else: # doc_fetch
|
| 994 |
+
hit = next((d for d in row if d.get('id') == doc_id), None)
|
| 995 |
+
if not hit:
|
| 996 |
+
return False
|
| 997 |
+
blob = store.download_bytes(hit.get('path') or '')
|
| 998 |
+
if blob is None:
|
| 999 |
+
return False
|
| 1000 |
+
# The bridge is a fire-and-forget event LOG with no response channel, so the answer
|
| 1001 |
+
# is PARKED on the Result and rides the next render's payload (C5).
|
| 1002 |
+
ctx.out.doc = {
|
| 1003 |
+
'pid': pid, 'docId': doc_id, 'name': hit.get('name') or 'document',
|
| 1004 |
+
'mime': hit.get('mime') or 'application/octet-stream',
|
| 1005 |
+
'data_b64': _b64c.b64encode(blob).decode('ascii')}
|
| 1006 |
+
return True # re-render to deliver it
|
| 1007 |
+
|
| 1008 |
+
def _write(cur):
|
| 1009 |
+
cur[str(pid)] = row
|
| 1010 |
+
return cur
|
| 1011 |
+
store.update(_DOCS_KEY, _write, flush='async')
|
| 1012 |
+
return True
|
| 1013 |
+
|
| 1014 |
+
if kind in ('folder_create', 'folder_rename', 'folder_delete', 'folder_duplicate',
|
| 1015 |
+
'item_move'):
|
| 1016 |
+
# FOLDERS over the Views / Cohorts sidebars (owner item 11, contract C4). All five
|
| 1017 |
+
# events touch ONE home β the table workspace's `folders` + `itemFolders` β so a cohort
|
| 1018 |
+
# can be filed without the cohort STORE learning about folders at all.
|
| 1019 |
+
import aios_grid as _agf
|
| 1020 |
+
if not store.available():
|
| 1021 |
+
return False
|
| 1022 |
+
surface = str(event.get('surface') or '')
|
| 1023 |
+
if surface not in _agf.FOLDER_SURFACES:
|
| 1024 |
+
return False
|
| 1025 |
+
# The ids this user may actually file: their own views, their own cohorts. Fail-closed β
|
| 1026 |
+
# an item_move naming somebody else's cohort is dropped by clean_item_folders anyway,
|
| 1027 |
+
# but refusing here means we never write it in the first place.
|
| 1028 |
+
def _own_ids(sfc, cur_ws):
|
| 1029 |
+
if sfc == 'views':
|
| 1030 |
+
return set((cur_ws.get('views') or {}))
|
| 1031 |
+
return set(cohort_mod.all_for(uname))
|
| 1032 |
+
|
| 1033 |
+
def _mutate(cur_ws):
|
| 1034 |
+
folders = _agf.clean_folders(cur_ws.get('folders'))
|
| 1035 |
+
placed = dict(cur_ws.get('itemFolders') or {})
|
| 1036 |
+
lst = list(folders.get(surface) or [])
|
| 1037 |
+
# β the wire field is `folderId` on EVERY folder event (CLIENT's types.ts is the
|
| 1038 |
+
# built contract; the split doc's first draft said `id`). Converged here rather
|
| 1039 |
+
# than asking the client to re-emit β see the C4 reconciliation note in the doc.
|
| 1040 |
+
fid = str(event.get('folderId') or '').strip()[:80]
|
| 1041 |
+
if kind == 'folder_create':
|
| 1042 |
+
name = str(event.get('name') or '').strip()[:_agf.MAX_FOLDER_NAME]
|
| 1043 |
+
if not fid or not name or len(lst) >= _agf.MAX_FOLDERS:
|
| 1044 |
+
return None
|
| 1045 |
+
if any(f['id'] == fid for f in lst):
|
| 1046 |
+
return None
|
| 1047 |
+
row = {'id': fid, 'name': name, 'order': len(lst)}
|
| 1048 |
+
icon = _agf.clean_folder_icon(event.get('icon')) # wave-9 I15 (C5)
|
| 1049 |
+
if icon:
|
| 1050 |
+
row['icon'] = icon
|
| 1051 |
+
lst.append(row)
|
| 1052 |
+
elif kind == 'folder_rename':
|
| 1053 |
+
name = str(event.get('name') or '').strip()[:_agf.MAX_FOLDER_NAME]
|
| 1054 |
+
hit = next((f for f in lst if f['id'] == fid), None)
|
| 1055 |
+
if not hit or not name:
|
| 1056 |
+
return None
|
| 1057 |
+
hit['name'] = name
|
| 1058 |
+
# wave-9 I15 (C5): the rename pane is also where the icon is changed. An event
|
| 1059 |
+
# that carries NO icon key leaves the existing one alone (a rename must not
|
| 1060 |
+
# silently strip a chosen icon); an explicit null clears it back to default.
|
| 1061 |
+
if 'icon' in event:
|
| 1062 |
+
icon = _agf.clean_folder_icon(event.get('icon'))
|
| 1063 |
+
if icon:
|
| 1064 |
+
hit['icon'] = icon
|
| 1065 |
+
else:
|
| 1066 |
+
hit.pop('icon', None)
|
| 1067 |
+
elif kind == 'folder_delete':
|
| 1068 |
+
if not any(f['id'] == fid for f in lst):
|
| 1069 |
+
return None
|
| 1070 |
+
lst = [f for f in lst if f['id'] != fid]
|
| 1071 |
+
# CONTENTS MOVE TO ROOT, never delete. Dropping the placements is exactly that:
|
| 1072 |
+
# an item with no placement renders at the top level.
|
| 1073 |
+
placed[surface] = {k: v for k, v in (placed.get(surface) or {}).items()
|
| 1074 |
+
if v != fid}
|
| 1075 |
+
elif kind == 'folder_duplicate':
|
| 1076 |
+
new_id = str(event.get('newId') or '').strip()[:80]
|
| 1077 |
+
src = next((f for f in lst if f['id'] == fid), None)
|
| 1078 |
+
if not src or not new_id or len(lst) >= _agf.MAX_FOLDERS:
|
| 1079 |
+
return None
|
| 1080 |
+
if any(f['id'] == new_id for f in lst):
|
| 1081 |
+
return None
|
| 1082 |
+
lst.append({'id': new_id, 'name': f"{src['name']} copy"[:_agf.MAX_FOLDER_NAME],
|
| 1083 |
+
'order': len(lst)})
|
| 1084 |
+
# β The folder's CONTENTS are duplicated by the client emitting the ordinary
|
| 1085 |
+
# per-item duplicate events it already owns (view_upsert with a fresh id, the
|
| 1086 |
+
# cohort copy path) and then item_move-ing them here. Copying views/cohorts
|
| 1087 |
+
# inside this handler would be a SECOND implementation of "duplicate an item",
|
| 1088 |
+
# and the two would drift the first time either one changed.
|
| 1089 |
+
else: # item_move
|
| 1090 |
+
item_id = str(event.get('itemId') or '').strip()[:120]
|
| 1091 |
+
target = event.get('folderId')
|
| 1092 |
+
if not item_id or item_id not in _own_ids(surface, cur_ws):
|
| 1093 |
+
return None
|
| 1094 |
+
cur = dict(placed.get(surface) or {})
|
| 1095 |
+
if target is None:
|
| 1096 |
+
cur.pop(item_id, None) # back to the root
|
| 1097 |
+
else:
|
| 1098 |
+
tid = str(target)[:80]
|
| 1099 |
+
if not any(f['id'] == tid for f in lst):
|
| 1100 |
+
return None
|
| 1101 |
+
cur[item_id] = tid
|
| 1102 |
+
placed[surface] = cur
|
| 1103 |
+
folders[surface] = lst
|
| 1104 |
+
clean = _agf.clean_folders(folders)
|
| 1105 |
+
return clean, _agf.clean_item_folders(
|
| 1106 |
+
placed, clean,
|
| 1107 |
+
{sfc: _own_ids(sfc, cur_ws) for sfc in _agf.FOLDER_SURFACES})
|
| 1108 |
+
|
| 1109 |
+
# β NO allowed_pids ON PURPOSE β this is app.py's bare `_cl_table_workspace(uname)`
|
| 1110 |
+
# call at the old :6268, preserved exactly. Omitting the pool drops memberPids from
|
| 1111 |
+
# any foreign shared view (fail-closed), which is harmless here because the folder
|
| 1112 |
+
# legs read only view IDS and the folder maps. Passing the pool would be a behavior
|
| 1113 |
+
# change inside a pure move.
|
| 1114 |
+
_res = _mutate(table_workspace(ctx, consume_corrections=False))
|
| 1115 |
+
if _res is None:
|
| 1116 |
+
return False # refused: nothing is written
|
| 1117 |
+
_tops(ctx).save_folders(uname, _res[0], _res[1])
|
| 1118 |
+
# True: the sidebars render from server state (which folder holds what), not from local
|
| 1119 |
+
# optimism β the same contract the cohort panel's membership edits ride.
|
| 1120 |
+
return True
|
| 1121 |
+
|
| 1122 |
+
if kind == 'field_delete':
|
| 1123 |
+
# Delete a USER-CREATED column outright (owner gap closed 2026-07-27). The prefix IS
|
| 1124 |
+
# the permission: only the created strata (`custom_` overlay fields, `measure_` formula
|
| 1125 |
+
# columns) are deletable β a base field's key fails the prefix test no matter what the
|
| 1126 |
+
# browser claims, so the table's contract cannot be deleted from the client. The store
|
| 1127 |
+
# scrubs the field's stored cell values with it (delete_table_field); views naming the
|
| 1128 |
+
# key self-heal on their next autosave, the rule every stale colId already rides.
|
| 1129 |
+
key = str(event.get('key') or '')[:80]
|
| 1130 |
+
import aios_grid as _ag3
|
| 1131 |
+
if not (key.startswith('custom_') or key.startswith(_ag3.MEASURE_FIELD_PREFIX)):
|
| 1132 |
+
return False
|
| 1133 |
+
if store.available():
|
| 1134 |
+
_tops(ctx).delete_field(uname, key)
|
| 1135 |
+
else:
|
| 1136 |
+
_session_ready()
|
| 1137 |
+
ws['fields'].pop(key, None)
|
| 1138 |
+
for _row in ws['overlays'].values():
|
| 1139 |
+
if isinstance(_row, dict):
|
| 1140 |
+
_row.pop(key, None)
|
| 1141 |
+
# Wave-6 item 3: the client dropped the column optimistically; the store scrub is
|
| 1142 |
+
# invisible to the next render (the values leave WITH the column).
|
| 1143 |
+
return False
|
| 1144 |
+
|
| 1145 |
+
if kind == 'field_duplicate':
|
| 1146 |
+
# Wave-5 item 1. The DESTINATION key is client-generated (the established pattern β
|
| 1147 |
+
# the client must know it synchronously to place the clone right of the source), and
|
| 1148 |
+
# both keys must sit in the SAME created stratum: base odoo fields offer no Duplicate,
|
| 1149 |
+
# because duplicating the source-of-truth into an editable copy would need a
|
| 1150 |
+
# value-snapshot mechanism nobody asked for. Overlay VALUES are copied only for
|
| 1151 |
+
# `custom_` sources (a measure clone recomputes; a formula clone re-evaluates).
|
| 1152 |
+
# The duplicator becomes the clone's creator β it is a NEW field they made.
|
| 1153 |
+
src_key = str(event.get('sourceKey') or '')[:80]
|
| 1154 |
+
new_key = str(event.get('key') or '')[:80]
|
| 1155 |
+
import aios_grid as _ag5
|
| 1156 |
+
prefix = ('custom_' if src_key.startswith('custom_')
|
| 1157 |
+
else _ag5.MEASURE_FIELD_PREFIX
|
| 1158 |
+
if src_key.startswith(_ag5.MEASURE_FIELD_PREFIX) else None)
|
| 1159 |
+
if (prefix is None or not new_key.startswith(prefix) or new_key == src_key
|
| 1160 |
+
or new_key in field_by_key):
|
| 1161 |
+
return False
|
| 1162 |
+
src = ws.get('fields', {}).get(src_key)
|
| 1163 |
+
if not isinstance(src, dict):
|
| 1164 |
+
return False
|
| 1165 |
+
clone = dict(src)
|
| 1166 |
+
clone['key'] = new_key
|
| 1167 |
+
label = str(event.get('label') or '').strip()[:120]
|
| 1168 |
+
requested_label = label or (str(src.get('label') or 'Untitled')[:110] + ' copy')
|
| 1169 |
+
_workspace_field_keys = set((ws.get('fields') or {}))
|
| 1170 |
+
_reserved_field_names = [
|
| 1171 |
+
value.get('label') for candidate_key, value in field_by_key.items()
|
| 1172 |
+
if (candidate_key != new_key and candidate_key not in _workspace_field_keys
|
| 1173 |
+
and isinstance(value, dict))
|
| 1174 |
+
]
|
| 1175 |
+
clone['label'] = requested_label
|
| 1176 |
+
clone['createdBy'] = uname
|
| 1177 |
+
# Wave-6 item 9: the clone inherits the source's scope via dict(src); on the cohort
|
| 1178 |
+
# page the duplicate surface may explicitly place it instead (same create-time rule
|
| 1179 |
+
# as field_upsert β a page that is not the cohort page cannot mint a cohort field).
|
| 1180 |
+
if scope_key == 'cohort' and event.get('scope') in ('cohort', 'global'):
|
| 1181 |
+
if event['scope'] == 'cohort':
|
| 1182 |
+
clone['scope'] = 'cohort'
|
| 1183 |
+
else:
|
| 1184 |
+
clone.pop('scope', None)
|
| 1185 |
+
if store.available():
|
| 1186 |
+
clone = (_tops(ctx).duplicate_field(
|
| 1187 |
+
uname, src_key, new_key, clone,
|
| 1188 |
+
reserved_names=_reserved_field_names,
|
| 1189 |
+
correction_id=event_id) or clone)
|
| 1190 |
+
else:
|
| 1191 |
+
_session_ready()
|
| 1192 |
+
_field_names = list(_reserved_field_names)
|
| 1193 |
+
_field_names.extend(
|
| 1194 |
+
value.get('label')
|
| 1195 |
+
for candidate_key, value in ws['fields'].items()
|
| 1196 |
+
if candidate_key != new_key and isinstance(value, dict)
|
| 1197 |
+
)
|
| 1198 |
+
clone.pop('labelCorrectedFrom', None)
|
| 1199 |
+
clone.pop('labelCorrectionId', None)
|
| 1200 |
+
clone['label'] = _unique_name(requested_label, _field_names)
|
| 1201 |
+
if clone['label'] != requested_label and event_id:
|
| 1202 |
+
clone['labelCorrectedFrom'] = requested_label
|
| 1203 |
+
clone['labelCorrectionId'] = str(event_id)[:180]
|
| 1204 |
+
ws['fields'][new_key] = clone
|
| 1205 |
+
if src_key.startswith('custom_'):
|
| 1206 |
+
for _row in ws['overlays'].values():
|
| 1207 |
+
if isinstance(_row, dict) and src_key in _row:
|
| 1208 |
+
_row[new_key] = _row[src_key]
|
| 1209 |
+
# Wave-6 item 3: True only when host-held VALUES must flow back β copied overlay
|
| 1210 |
+
# cells (custom_ sources) or host-computed measure values. formula/created_time
|
| 1211 |
+
# clones compute client-side, so the optimistic insert is already complete.
|
| 1212 |
+
label_changed = clone['label'] != requested_label
|
| 1213 |
+
return label_changed or src.get('type') not in ('formula', 'created_time')
|
| 1214 |
+
|
| 1215 |
+
if kind == 'add_to_list':
|
| 1216 |
+
# "Add to list" (owner item 6): a view's current matches become fixed cohort members.
|
| 1217 |
+
#
|
| 1218 |
+
# β `pids` arrives from the BROWSER. It is intersected with `allowed_pids` β the pool this
|
| 1219 |
+
# user may actually see β before anything is stored. The client computing the set is a
|
| 1220 |
+
# correctness decision (it is the only engine that already knows the answer, and asking a
|
| 1221 |
+
# second engine the same question is the drift verify_filter_engine.py exists to prevent);
|
| 1222 |
+
# it is NOT a trust decision.
|
| 1223 |
+
raw_pids = event.get('pids')
|
| 1224 |
+
if not isinstance(raw_pids, list):
|
| 1225 |
+
return False
|
| 1226 |
+
pids = [p for p in raw_pids if isinstance(p, int) and p in allowed_pids]
|
| 1227 |
+
if not pids:
|
| 1228 |
+
return False
|
| 1229 |
+
cohort_id = str(event.get('cohortId') or '').strip()[:120]
|
| 1230 |
+
name = str(event.get('name') or '').strip()[:120]
|
| 1231 |
+
if not store.available():
|
| 1232 |
+
return False
|
| 1233 |
+
try:
|
| 1234 |
+
if not cohort_id:
|
| 1235 |
+
if not name:
|
| 1236 |
+
return False
|
| 1237 |
+
cohort_id = cohort_mod.create(uname, name, pids)
|
| 1238 |
+
else:
|
| 1239 |
+
if cohort_id not in cohort_mod.all_for(uname):
|
| 1240 |
+
return False # not this user's cohort β fail closed, do not create
|
| 1241 |
+
cohort_mod.add_members(uname, cohort_id, pids)
|
| 1242 |
+
except ValueError as e:
|
| 1243 |
+
_tel.error('cohort:add_to_list', e)
|
| 1244 |
+
return False
|
| 1245 |
+
# True: the Cohort page's list and counts must reflect this, and the toast below only
|
| 1246 |
+
# renders on a fresh run.
|
| 1247 |
+
ctx.out.toast = (
|
| 1248 |
+
f"Added {len(pids):,} customer{'' if len(pids) == 1 else 's'} to "
|
| 1249 |
+
f"{name or 'the list'}.")
|
| 1250 |
+
return True
|
| 1251 |
+
|
| 1252 |
+
if kind == 'overlay_patch':
|
| 1253 |
+
pid = event.get('pid')
|
| 1254 |
+
if not isinstance(pid, int) or pid not in allowed_pids:
|
| 1255 |
+
return False
|
| 1256 |
+
updates = {}
|
| 1257 |
+
refused = False
|
| 1258 |
+
raw_updates = event.get('updates') if isinstance(event.get('updates'), dict) else {}
|
| 1259 |
+
for key, value in raw_updates.items():
|
| 1260 |
+
if key not in overlay_keys or not isinstance(value, (str, int, float)):
|
| 1261 |
+
refused = True
|
| 1262 |
+
continue
|
| 1263 |
+
# C-PERM: a field this caller's permissions HIDE cannot be written, only refused.
|
| 1264 |
+
# `overlay_keys` asks whether the column exists, not whether this person may touch
|
| 1265 |
+
# it β so without this line a restricted user who knows a hidden key could PATCH a
|
| 1266 |
+
# value they are not allowed to read back. Refused LOUDLY (repaint) so the client
|
| 1267 |
+
# reverts rather than showing a value the store never took.
|
| 1268 |
+
if key in (ctx.hidden_keys or frozenset()):
|
| 1269 |
+
refused = True
|
| 1270 |
+
continue
|
| 1271 |
+
# Wave-5 item 1: `permissions.edit` is enforced HERE, per key β the client hiding
|
| 1272 |
+
# its editor is courtesy, this is the wall. Fail-closed: a restricted field with
|
| 1273 |
+
# no readable createdBy admits only admins.
|
| 1274 |
+
perms = (field_by_key.get(key) or {}).get('permissions') or {}
|
| 1275 |
+
edit = perms.get('edit')
|
| 1276 |
+
if edit == 'admins' and not admin:
|
| 1277 |
+
refused = True
|
| 1278 |
+
continue
|
| 1279 |
+
if edit == 'creator' and not (
|
| 1280 |
+
admin or (field_by_key.get(key) or {}).get('createdBy') == uname):
|
| 1281 |
+
refused = True
|
| 1282 |
+
continue
|
| 1283 |
+
# C5-AUTOFIELD (wave 18): an automation cell is MACHINE-WRITTEN. The client marks
|
| 1284 |
+
# the type read-only (READONLY_CELL_TYPES) but that is courtesy β this is the wall.
|
| 1285 |
+
# Without it the UI hides the editor while the API still takes a PATCH, which is
|
| 1286 |
+
# exactly the shape a "read-only" type must not have.
|
| 1287 |
+
if (field_by_key.get(key) or {}).get('type') == 'automation':
|
| 1288 |
+
refused = True
|
| 1289 |
+
continue
|
| 1290 |
+
clean = str(value)[:10000]
|
| 1291 |
+
if isinstance(value, str) and clean != value:
|
| 1292 |
+
# Truncated: the stored truth is shorter than the client's copy. (An
|
| 1293 |
+
# int/float arriving as its string form renders identically β not a refusal.)
|
| 1294 |
+
refused = True
|
| 1295 |
+
definition = field_by_key.get(key) or {}
|
| 1296 |
+
if definition.get('type') in ('select', 'multiselect') and clean:
|
| 1297 |
+
options = [str(v) for v in definition.get('options') or []]
|
| 1298 |
+
canonical = {v.strip().lower(): v for v in options}
|
| 1299 |
+
if definition.get('type') == 'select':
|
| 1300 |
+
normalized = canonical.get(clean.strip().lower())
|
| 1301 |
+
if normalized is None:
|
| 1302 |
+
refused = True
|
| 1303 |
+
continue
|
| 1304 |
+
else:
|
| 1305 |
+
normalized_parts = []
|
| 1306 |
+
seen = set()
|
| 1307 |
+
invalid = False
|
| 1308 |
+
for part in clean.split(','):
|
| 1309 |
+
match = canonical.get(part.strip().lower())
|
| 1310 |
+
if match is None:
|
| 1311 |
+
invalid = True
|
| 1312 |
+
break
|
| 1313 |
+
if match.lower() not in seen:
|
| 1314 |
+
seen.add(match.lower())
|
| 1315 |
+
normalized_parts.append(match)
|
| 1316 |
+
if invalid:
|
| 1317 |
+
refused = True
|
| 1318 |
+
continue
|
| 1319 |
+
normalized = ','.join(normalized_parts)
|
| 1320 |
+
if normalized != clean:
|
| 1321 |
+
refused = True
|
| 1322 |
+
clean = normalized
|
| 1323 |
+
updates[key] = clean
|
| 1324 |
+
if updates:
|
| 1325 |
+
if store.available():
|
| 1326 |
+
_tops(ctx).patch_overlay(uname, pid, updates)
|
| 1327 |
+
else:
|
| 1328 |
+
_session_ready()
|
| 1329 |
+
ws['overlays'].setdefault(str(pid), {}).update(updates)
|
| 1330 |
+
# Wave-6 item 3: the client's cell already shows the typed value β rerun ONLY when
|
| 1331 |
+
# some of what it shows was refused or clamped and must be repainted to the truth.
|
| 1332 |
+
return refused
|
| 1333 |
+
|
| 1334 |
+
return False
|
platform/core/keychain.py
ADDED
|
@@ -0,0 +1,182 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""core/keychain.py β per-tenant CREDENTIAL STORE (wave 18, contract C7 / ruling R3).
|
| 2 |
+
|
| 3 |
+
An admin stores a data source's secrets (Royal's Odoo API key today; any tenant's sources
|
| 4 |
+
tomorrow) encrypted at rest in the TENANT's store bucket. The runtime posture is R3's
|
| 5 |
+
"live with env fallback": the connector layer asks `odoo_creds(rt)` FIRST and falls back to
|
| 6 |
+
the process environment (tenant #0's `.env` / Space secrets), so Royal keeps working with an
|
| 7 |
+
empty keychain and a new tenant needs no code to bring keys. The FULL cutover (env retired)
|
| 8 |
+
is staged for a later wave β R3, verbatim.
|
| 9 |
+
|
| 10 |
+
CRYPTO: Fernet (AES128-CBC + HMAC, the `cryptography` package) under ONE platform key,
|
| 11 |
+
`AIOS_KEYCHAIN_KEY` (a `Fernet.generate_key()` value; a Space secret, never in the store).
|
| 12 |
+
No key β the keychain is LOCKED: metadata still lists (labels are not secrets), but nothing
|
| 13 |
+
decrypts and writes are refused β fail closed, never a plaintext fallback.
|
| 14 |
+
|
| 15 |
+
SHAPE (bucket `keychain`, per-tenant via the runtime handle):
|
| 16 |
+
|
| 17 |
+
{entries: {id: {id, label, type, enc, preview, created, createdBy}}}
|
| 18 |
+
|
| 19 |
+
`enc` = Fernet token over the JSON fields dict. `preview` = a masked hint ("Β·Β·Β·Β·1234")
|
| 20 |
+
computed ONCE at write from the entry's most secret-looking field β the ONLY part of a
|
| 21 |
+
secret that ever leaves this module through a route. `read_fields` exists for the CONNECTOR
|
| 22 |
+
LAYER; no route returns its output.
|
| 23 |
+
"""
|
| 24 |
+
import datetime as _dt
|
| 25 |
+
import json
|
| 26 |
+
import os
|
| 27 |
+
import secrets as _secrets
|
| 28 |
+
|
| 29 |
+
KEY = 'keychain'
|
| 30 |
+
ENTRY_TYPES = {'odoo', 'generic'}
|
| 31 |
+
#: which stored field feeds the masked preview, per type (first present wins)
|
| 32 |
+
_PREVIEW_FIELDS = ('api_key', 'value', 'password', 'token')
|
| 33 |
+
MAX_ENTRIES = 40
|
| 34 |
+
MAX_FIELD_LEN = 500
|
| 35 |
+
|
| 36 |
+
|
| 37 |
+
class KeychainLocked(Exception):
|
| 38 |
+
"""Raised when AIOS_KEYCHAIN_KEY is absent/wrong β callers answer 503, never {}."""
|
| 39 |
+
|
| 40 |
+
|
| 41 |
+
def _fernet():
|
| 42 |
+
key = (os.environ.get('AIOS_KEYCHAIN_KEY') or '').strip()
|
| 43 |
+
if not key:
|
| 44 |
+
raise KeychainLocked('AIOS_KEYCHAIN_KEY is not configured')
|
| 45 |
+
try:
|
| 46 |
+
from cryptography.fernet import Fernet
|
| 47 |
+
return Fernet(key.encode('ascii'))
|
| 48 |
+
except Exception as e:
|
| 49 |
+
raise KeychainLocked(f'keychain key unusable: {type(e).__name__}')
|
| 50 |
+
|
| 51 |
+
|
| 52 |
+
def unlocked():
|
| 53 |
+
try:
|
| 54 |
+
_fernet()
|
| 55 |
+
return True
|
| 56 |
+
except KeychainLocked:
|
| 57 |
+
return False
|
| 58 |
+
|
| 59 |
+
|
| 60 |
+
def _bucket(rt):
|
| 61 |
+
return rt.get(KEY) or {}
|
| 62 |
+
|
| 63 |
+
|
| 64 |
+
def list_entries(rt):
|
| 65 |
+
"""Metadata ONLY β never a decrypted field, never `enc` itself."""
|
| 66 |
+
out = []
|
| 67 |
+
for eid, e in sorted((_bucket(rt).get('entries') or {}).items()):
|
| 68 |
+
if not isinstance(e, dict):
|
| 69 |
+
continue
|
| 70 |
+
out.append({'id': eid, 'label': e.get('label') or eid,
|
| 71 |
+
'type': e.get('type') or 'generic',
|
| 72 |
+
'preview': e.get('preview') or '',
|
| 73 |
+
'created': e.get('created') or '',
|
| 74 |
+
'createdBy': e.get('createdBy') or ''})
|
| 75 |
+
return out
|
| 76 |
+
|
| 77 |
+
|
| 78 |
+
def add_entry(rt, label, etype, fields, username):
|
| 79 |
+
"""Encrypt + store. Returns the metadata row, or raises ValueError on shape problems and
|
| 80 |
+
KeychainLocked when there is no key (a secret must never be stored in the clear)."""
|
| 81 |
+
label = ' '.join(str(label or '').split())[:80]
|
| 82 |
+
etype = str(etype or '').strip().lower()
|
| 83 |
+
if not label:
|
| 84 |
+
raise ValueError('give the key a label')
|
| 85 |
+
if etype not in ENTRY_TYPES:
|
| 86 |
+
raise ValueError(f"type must be one of {', '.join(sorted(ENTRY_TYPES))}")
|
| 87 |
+
if not isinstance(fields, dict) or not fields:
|
| 88 |
+
raise ValueError('fields must be a non-empty object')
|
| 89 |
+
clean = {}
|
| 90 |
+
for k, v in list(fields.items())[:12]:
|
| 91 |
+
k = str(k).strip()[:40]
|
| 92 |
+
if k and isinstance(v, (str, int, float, bool)):
|
| 93 |
+
clean[k] = str(v)[:MAX_FIELD_LEN]
|
| 94 |
+
if not clean:
|
| 95 |
+
raise ValueError('no usable fields')
|
| 96 |
+
token = _fernet().encrypt(json.dumps(clean, ensure_ascii=False).encode('utf-8'))
|
| 97 |
+
secretish = next((clean[k] for k in _PREVIEW_FIELDS if clean.get(k)), '')
|
| 98 |
+
preview = ('Β·Β·Β·Β·' + secretish[-4:]) if len(secretish) >= 4 else ('Β·Β·Β·Β·' if secretish else '')
|
| 99 |
+
eid = f'k_{_secrets.token_hex(6)}'
|
| 100 |
+
row = {'id': eid, 'label': label, 'type': etype, 'enc': token.decode('ascii'),
|
| 101 |
+
'preview': preview,
|
| 102 |
+
'created': _dt.datetime.now().strftime('%Y-%m-%dT%H:%M:%S'),
|
| 103 |
+
'createdBy': str(username or '')}
|
| 104 |
+
|
| 105 |
+
def _up(cur):
|
| 106 |
+
entries = cur.setdefault('entries', {})
|
| 107 |
+
if len(entries) >= MAX_ENTRIES:
|
| 108 |
+
raise ValueError(f'this keychain is at its {MAX_ENTRIES}-entry cap')
|
| 109 |
+
entries[eid] = row
|
| 110 |
+
return cur
|
| 111 |
+
|
| 112 |
+
rt.update(KEY, _up, flush='sync')
|
| 113 |
+
return {'id': eid, 'label': label, 'type': etype, 'preview': preview,
|
| 114 |
+
'created': row['created'], 'createdBy': row['createdBy']}
|
| 115 |
+
|
| 116 |
+
|
| 117 |
+
def delete_entry(rt, entry_id):
|
| 118 |
+
def _drop(cur):
|
| 119 |
+
(cur.get('entries') or {}).pop(str(entry_id), None)
|
| 120 |
+
return cur
|
| 121 |
+
rt.update(KEY, _drop, flush='sync')
|
| 122 |
+
return True
|
| 123 |
+
|
| 124 |
+
|
| 125 |
+
def read_fields(rt, entry_id):
|
| 126 |
+
"""Decrypt ONE entry's fields β for the CONNECTOR LAYER only; no route returns this.
|
| 127 |
+
None when the entry does not exist; KeychainLocked when the key is absent/wrong (a wrong
|
| 128 |
+
key must be loud β silently empty creds would read as 'not configured')."""
|
| 129 |
+
e = (_bucket(rt).get('entries') or {}).get(str(entry_id))
|
| 130 |
+
if not isinstance(e, dict) or not e.get('enc'):
|
| 131 |
+
return None
|
| 132 |
+
try:
|
| 133 |
+
raw = _fernet().decrypt(str(e['enc']).encode('ascii'))
|
| 134 |
+
except KeychainLocked:
|
| 135 |
+
raise
|
| 136 |
+
except Exception:
|
| 137 |
+
raise KeychainLocked('this entry does not decrypt under the configured key')
|
| 138 |
+
try:
|
| 139 |
+
data = json.loads(raw.decode('utf-8'))
|
| 140 |
+
return data if isinstance(data, dict) else None
|
| 141 |
+
except Exception:
|
| 142 |
+
return None
|
| 143 |
+
|
| 144 |
+
|
| 145 |
+
def odoo_creds(rt):
|
| 146 |
+
"""R3's resolver seam: the FIRST odoo-type entry's fields, else None (the caller falls
|
| 147 |
+
back to the environment). Deterministic order = insertion-id sort, so 'first' is stable."""
|
| 148 |
+
for row in list_entries(rt):
|
| 149 |
+
if row['type'] == 'odoo':
|
| 150 |
+
return read_fields(rt, row['id'])
|
| 151 |
+
return None
|
| 152 |
+
|
| 153 |
+
|
| 154 |
+
def test_entry(rt, entry_id, timeout=8):
|
| 155 |
+
"""A cheap, READ-ONLY liveness probe. odoo: xmlrpc `common.version()` (no auth, no data);
|
| 156 |
+
generic: shape-only. Always answers {ok, message} β an exception here is a result, not a
|
| 157 |
+
crash."""
|
| 158 |
+
try:
|
| 159 |
+
e = (_bucket(rt).get('entries') or {}).get(str(entry_id))
|
| 160 |
+
if not isinstance(e, dict):
|
| 161 |
+
return {'ok': False, 'message': 'no such entry'}
|
| 162 |
+
fields = read_fields(rt, entry_id) or {}
|
| 163 |
+
if e.get('type') != 'odoo':
|
| 164 |
+
return {'ok': bool(fields), 'message': f'{len(fields)} field(s) stored'}
|
| 165 |
+
url = str(fields.get('url') or '').rstrip('/')
|
| 166 |
+
if not url.startswith(('http://', 'https://')):
|
| 167 |
+
return {'ok': False, 'message': 'url must start with http(s)://'}
|
| 168 |
+
import socket
|
| 169 |
+
import xmlrpc.client
|
| 170 |
+
old = socket.getdefaulttimeout()
|
| 171 |
+
socket.setdefaulttimeout(timeout)
|
| 172 |
+
try:
|
| 173 |
+
common = xmlrpc.client.ServerProxy(f'{url}/xmlrpc/2/common', allow_none=True)
|
| 174 |
+
ver = common.version()
|
| 175 |
+
finally:
|
| 176 |
+
socket.setdefaulttimeout(old)
|
| 177 |
+
sv = (ver or {}).get('server_version') if isinstance(ver, dict) else None
|
| 178 |
+
return {'ok': True, 'message': f'Odoo answered (server {sv or "unknown"})'}
|
| 179 |
+
except KeychainLocked as e:
|
| 180 |
+
return {'ok': False, 'message': str(e)}
|
| 181 |
+
except Exception as e:
|
| 182 |
+
return {'ok': False, 'message': f'{type(e).__name__}: {str(e)[:120]}'}
|
platform/core/links.py
ADDED
|
@@ -0,0 +1,59 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Deep links into the app β ?page=<key>&entity=<kind>:<id>&src=<source>.
|
| 2 |
+
|
| 3 |
+
The app (app.py) consumes these params once per session right after login and routes to the page,
|
| 4 |
+
opening the entity drawer when an entity is present. Digest/alert/mention emails build their links
|
| 5 |
+
here so every emailed number lands the reader on the exact row it came from. `src` tags where the
|
| 6 |
+
click came from (digest / alert / mention) β the app logs it as the click-through read receipt.
|
| 7 |
+
"""
|
| 8 |
+
import os
|
| 9 |
+
import sys
|
| 10 |
+
from urllib.parse import quote
|
| 11 |
+
|
| 12 |
+
#: β THE ONE HOST LITERAL LEFT IN THE RUNTIME PATH, and it is a MIGRATION HAZARD β named here
|
| 13 |
+
#: rather than buried in a default argument, because the day we change hosts this is what silently
|
| 14 |
+
#: keeps mailing customers a link to the DEAD one. Found by `ops/verify_portability.py` on its
|
| 15 |
+
#: first run (2026-07-30).
|
| 16 |
+
#:
|
| 17 |
+
#: Why it is still here instead of raising: `deeplink()` has exactly one family of callers
|
| 18 |
+
#: (`modules/digest.py`, 7 sites, all building email hrefs), `DIGEST_ENABLED` defaults to ON, and
|
| 19 |
+
#: no deploy sets `APP_BASE_URL` today β so refusing to guess would break a LIVE feature to fix a
|
| 20 |
+
#: latent one. Instead: both deploy scripts now push `APP_BASE_URL`, which makes the fallback
|
| 21 |
+
#: unreachable in a deployed environment, and reaching it prints an operator warning.
|
| 22 |
+
#:
|
| 23 |
+
#: β EXIT CONDITION for this constant: once a deploy has set `APP_BASE_URL` and it is confirmed
|
| 24 |
+
#: present in the running environment, delete the fallback and let `deeplink()` raise instead. It
|
| 25 |
+
#: is declared in the portability gate's RESIDUALS so the gate reports it honestly rather than
|
| 26 |
+
#: reporting green over it.
|
| 27 |
+
_LEGACY_HF_FALLBACK = 'https://royal-imports-cfo-os.hf.space'
|
| 28 |
+
|
| 29 |
+
BASE_URL = (os.environ.get('APP_BASE_URL') or _LEGACY_HF_FALLBACK).rstrip('/')
|
| 30 |
+
|
| 31 |
+
#: Warned LAZILY and ONCE β at the first link actually built, not at import. An import-time print
|
| 32 |
+
#: fires in every gate, every subprocess and every unit test that so much as touches `core`, which
|
| 33 |
+
#: is how a real warning becomes noise nobody reads.
|
| 34 |
+
_WARNED = [False]
|
| 35 |
+
|
| 36 |
+
|
| 37 |
+
def _warn_once_if_defaulted():
|
| 38 |
+
if _WARNED[0] or os.environ.get('APP_BASE_URL'):
|
| 39 |
+
return
|
| 40 |
+
_WARNED[0] = True
|
| 41 |
+
print(f"[links] APP_BASE_URL is not set - emailed deep links will point at "
|
| 42 |
+
f"{_LEGACY_HF_FALLBACK}, which is correct ONLY while that is where the app lives. "
|
| 43 |
+
f"Set APP_BASE_URL in the deployed environment.", file=sys.stderr)
|
| 44 |
+
|
| 45 |
+
# entity kinds the URL scheme supports (must stay in sync with app.py's _desc_from_param)
|
| 46 |
+
LINK_KINDS = ('customer', 'sku', 'agent', 'account')
|
| 47 |
+
|
| 48 |
+
|
| 49 |
+
def deeplink(page=None, kind=None, ident=None, src=None):
|
| 50 |
+
"""An absolute app URL. deeplink('customer_data', 'customer', 1234, src='digest')."""
|
| 51 |
+
_warn_once_if_defaulted()
|
| 52 |
+
parts = []
|
| 53 |
+
if page:
|
| 54 |
+
parts.append(f"page={quote(str(page), safe='')}")
|
| 55 |
+
if kind in LINK_KINDS and ident not in (None, ''):
|
| 56 |
+
parts.append(f"entity={quote(f'{kind}:{ident}', safe=':')}")
|
| 57 |
+
if src:
|
| 58 |
+
parts.append(f"src={quote(str(src), safe='')}")
|
| 59 |
+
return BASE_URL + ('/?' + '&'.join(parts) if parts else '/')
|
platform/core/management_pl.py
ADDED
|
@@ -0,0 +1,161 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""core/management_pl.py β reproduce the operating model's MANAGEMENT P&Ls
|
| 2 |
+
(FFS-M / RI-M / GD-M / BU-M) monthly from RAW ODOO, Sep-2024 cutover onward. READ-ONLY.
|
| 3 |
+
|
| 4 |
+
Decoded from v42 (see 17. Application development/financial_analysis/MODEL_LOGIC_MAP.md Β§10):
|
| 5 |
+
|
| 6 |
+
-M[entity][account] = clean_entity[account] # live from Odoo (PROVEN to the cent)
|
| 7 |
+
+ hq_overlay[entity][account] # HQ cost allocated to the entity
|
| 8 |
+
+ adj_overrides[entity][account] # tiny, transition-month only
|
| 9 |
+
|
| 10 |
+
clean_entity = account.analytic.line grouped by general_account_id (GL row) x account_id
|
| 11 |
+
(analytic = ENTITY column). Analytic ids: Fisch=1 Royal=2 GiftwareDeals=3
|
| 12 |
+
HQ=4 Internal=5. Sections by account_type. This is build_income_statements.py
|
| 13 |
+
(verified exact). Proven live: Oct-2025 acct 51000 -> Fisch 327,531.85 etc.
|
| 14 |
+
|
| 15 |
+
hq_overlay / adj_overrides come from data/mgmt_overlay.json (extracted read-only from v42 by
|
| 16 |
+
build_mgmt_overlay.py). The HQ allocation is HARDCODED in the model (PNL (HQ-*) tabs are literal
|
| 17 |
+
values, not formulas), so storing it is faithful to how the model is actually maintained.
|
| 18 |
+
|
| 19 |
+
Consolidation: BU-M = FFS-M + RI-M + GD-M (the Internal analytic is eliminated; HQ is allocated out).
|
| 20 |
+
|
| 21 |
+
Pure: data in -> plain dicts out. No Streamlit, no writes.
|
| 22 |
+
"""
|
| 23 |
+
from __future__ import annotations
|
| 24 |
+
import json
|
| 25 |
+
import os
|
| 26 |
+
from functools import lru_cache
|
| 27 |
+
|
| 28 |
+
from core import odoo as O
|
| 29 |
+
|
| 30 |
+
# ---- entity <-> Odoo analytic account ids (confirmed live) --------------------------------------
|
| 31 |
+
ENTITY_ANALYTIC = {'FFS': 1, 'RI': 2, 'GD': 3, 'HQ': 4, 'Internal': 5}
|
| 32 |
+
ANALYTIC_ENTITY = {v: k for k, v in ENTITY_ANALYTIC.items()}
|
| 33 |
+
BUS = ['FFS', 'RI', 'GD'] # the business units that get a -M statement
|
| 34 |
+
ENTITY_LABEL = {'FFS': 'Fisch (FFS)', 'RI': 'Royal (RI)', 'GD': 'Giftware Deals (GD)', 'BU': 'Consolidated (BU)'}
|
| 35 |
+
|
| 36 |
+
# P&L sections by Odoo account_type (mirrors build_income_statements.py)
|
| 37 |
+
INCOME_TYPES = {'income'}
|
| 38 |
+
OTHER_INCOME_TYPES = {'income_other'}
|
| 39 |
+
COGS_TYPES = {'expense_direct_cost'}
|
| 40 |
+
EXPENSE_TYPES = {'expense'}
|
| 41 |
+
DEPREC_TYPES = {'expense_depreciation'}
|
| 42 |
+
PL_TYPES = INCOME_TYPES | OTHER_INCOME_TYPES | COGS_TYPES | EXPENSE_TYPES | DEPREC_TYPES
|
| 43 |
+
|
| 44 |
+
_OVERLAY_PATH = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))),
|
| 45 |
+
'data', 'mgmt_overlay.json')
|
| 46 |
+
|
| 47 |
+
|
| 48 |
+
def month_bounds(year: int, month: int) -> tuple[str, str]:
|
| 49 |
+
import calendar
|
| 50 |
+
last = calendar.monthrange(year, month)[1]
|
| 51 |
+
return f'{year}-{month:02d}-01', f'{year}-{month:02d}-{last:02d}'
|
| 52 |
+
|
| 53 |
+
|
| 54 |
+
@lru_cache(maxsize=1)
|
| 55 |
+
def account_master() -> dict:
|
| 56 |
+
"""{gl_account_id: {'code','name','type'}} for every account."""
|
| 57 |
+
rows = O.search_read('account.account', [], ['id', 'code', 'name', 'account_type'])
|
| 58 |
+
return {r['id']: {'code': r.get('code') or '', 'name': r.get('name') or '',
|
| 59 |
+
'type': r.get('account_type')} for r in rows}
|
| 60 |
+
|
| 61 |
+
|
| 62 |
+
def _display(acc_type: str, raw: float) -> float:
|
| 63 |
+
"""Income shown raw; costs/expenses/depreciation flipped positive (model convention)."""
|
| 64 |
+
if acc_type in INCOME_TYPES or acc_type in OTHER_INCOME_TYPES:
|
| 65 |
+
return raw
|
| 66 |
+
return -raw
|
| 67 |
+
|
| 68 |
+
|
| 69 |
+
@lru_cache(maxsize=64)
|
| 70 |
+
def clean_entities(year: int, month: int) -> dict:
|
| 71 |
+
"""The clean per-entity P&L straight from Odoo analytic lines, for one month.
|
| 72 |
+
|
| 73 |
+
Returns {entity_key: {account_code: display_amount}} for FFS/RI/GD/HQ/Internal, restricted to
|
| 74 |
+
P&L account types. This is the LIVE, proven-exact foundation (= build_income_statements.py).
|
| 75 |
+
"""
|
| 76 |
+
start, end = month_bounds(year, month)
|
| 77 |
+
rows = O.read_group('account.analytic.line',
|
| 78 |
+
domain=[('date', '>=', start), ('date', '<=', end)],
|
| 79 |
+
fields=['amount:sum'], groupby=['general_account_id', 'account_id'], lazy=False)
|
| 80 |
+
am = account_master()
|
| 81 |
+
out = {k: {} for k in ENTITY_ANALYTIC}
|
| 82 |
+
for r in rows:
|
| 83 |
+
gen = r.get('general_account_id'); ana = r.get('account_id')
|
| 84 |
+
if not gen or not ana:
|
| 85 |
+
continue
|
| 86 |
+
gid, aid = gen[0], ana[0]
|
| 87 |
+
ent = ANALYTIC_ENTITY.get(aid)
|
| 88 |
+
acc = am.get(gid)
|
| 89 |
+
if ent is None or not acc or acc['type'] not in PL_TYPES:
|
| 90 |
+
continue
|
| 91 |
+
amt = _display(acc['type'], r.get('amount') or 0.0)
|
| 92 |
+
if abs(amt) < 0.005:
|
| 93 |
+
continue
|
| 94 |
+
out[ent][acc['code']] = out[ent].get(acc['code'], 0.0) + amt
|
| 95 |
+
return out
|
| 96 |
+
|
| 97 |
+
|
| 98 |
+
@lru_cache(maxsize=1)
|
| 99 |
+
def _overlay() -> dict:
|
| 100 |
+
"""The stored HQ-allocation + transition-reconciliation overlay (build_mgmt_overlay.py).
|
| 101 |
+
Shape: {'YYYY-MM': {'FFS': {acct: amt}, ...}}. These are financial figures, so they live in the
|
| 102 |
+
PRIVATE HF Dataset store on the Space (never in the public Space repo); locally they load from the
|
| 103 |
+
committed JSON. File first, so local runs need no token. Empty -> -M = clean entity only."""
|
| 104 |
+
try:
|
| 105 |
+
with open(_OVERLAY_PATH, encoding='utf-8') as f:
|
| 106 |
+
return json.load(f)
|
| 107 |
+
except (FileNotFoundError, json.JSONDecodeError):
|
| 108 |
+
pass
|
| 109 |
+
try:
|
| 110 |
+
from core import store
|
| 111 |
+
return store.get('mgmt_overlay') or {}
|
| 112 |
+
except Exception:
|
| 113 |
+
return {}
|
| 114 |
+
|
| 115 |
+
|
| 116 |
+
def management_pl(year: int, month: int, entity: str) -> dict:
|
| 117 |
+
"""The management P&L for one entity-month: {account_code: amount} = clean + overlay.
|
| 118 |
+
|
| 119 |
+
entity in {'FFS','RI','GD'}; 'BU' returns the consolidation (sum of the three).
|
| 120 |
+
"""
|
| 121 |
+
if entity == 'BU':
|
| 122 |
+
agg: dict = {}
|
| 123 |
+
for bu in BUS:
|
| 124 |
+
for code, amt in management_pl(year, month, bu).items():
|
| 125 |
+
agg[code] = agg.get(code, 0.0) + amt
|
| 126 |
+
return agg
|
| 127 |
+
clean = dict(clean_entities(year, month).get(entity, {}))
|
| 128 |
+
ov = _overlay().get(f'{year}-{month:02d}', {}).get(entity, {})
|
| 129 |
+
for code, amt in ov.items():
|
| 130 |
+
clean[code] = clean.get(code, 0.0) + amt
|
| 131 |
+
return clean
|
| 132 |
+
|
| 133 |
+
|
| 134 |
+
# ---- section roll-ups (for the statement view) --------------------------------------------------
|
| 135 |
+
def _bucket(code: str) -> str | None:
|
| 136 |
+
acc = next((a for a in account_master().values() if a['code'] == code), None)
|
| 137 |
+
if not acc:
|
| 138 |
+
return None
|
| 139 |
+
t = acc['type']
|
| 140 |
+
if t in INCOME_TYPES: return 'income'
|
| 141 |
+
if t in OTHER_INCOME_TYPES: return 'other_income'
|
| 142 |
+
if t in COGS_TYPES: return 'cogs'
|
| 143 |
+
if t in EXPENSE_TYPES: return 'expense'
|
| 144 |
+
if t in DEPREC_TYPES: return 'deprec'
|
| 145 |
+
return None
|
| 146 |
+
|
| 147 |
+
|
| 148 |
+
def statement(year: int, month: int, entity: str) -> dict:
|
| 149 |
+
"""Section subtotals + net for one entity-month (income / cogs / gross / opex / deprec / net)."""
|
| 150 |
+
lines = management_pl(year, month, entity)
|
| 151 |
+
sec = {'income': 0.0, 'other_income': 0.0, 'cogs': 0.0, 'expense': 0.0, 'deprec': 0.0}
|
| 152 |
+
for code, amt in lines.items():
|
| 153 |
+
b = _bucket(code)
|
| 154 |
+
if b:
|
| 155 |
+
sec[b] += amt
|
| 156 |
+
income = sec['income'] + sec['other_income']
|
| 157 |
+
gross = income - sec['cogs']
|
| 158 |
+
net = gross - sec['expense'] - sec['deprec']
|
| 159 |
+
return {'income': income, 'cogs': sec['cogs'], 'gross_profit': gross,
|
| 160 |
+
'operating_expense': sec['expense'], 'depreciation': sec['deprec'], 'net_profit': net,
|
| 161 |
+
'lines': lines}
|
platform/core/measure_resolve.py
ADDED
|
@@ -0,0 +1,190 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""core/measure_resolve.py β HOST-NEUTRAL measure resolution (EXIT-5, SHELL.md ladder step 3).
|
| 2 |
+
|
| 3 |
+
The `_cl_measure_offer` / `_cl_resolve_measures` / `_cl_measure_columns` family, OUT of app.py:
|
| 4 |
+
one implementation, two hosts. The Streamlit adapter passes its `st.session_state` memos and its
|
| 5 |
+
pending-view read; the API passes runtime-level dicts and no pending view (over HTTP the
|
| 6 |
+
`view_upsert` lands via `/grid/events` BEFORE the workspace re-read, so the saved views already
|
| 7 |
+
carry the freshest rule β there is no mid-run component value to read).
|
| 8 |
+
|
| 9 |
+
No streamlit import, no app import β harness only, so both adapters can call it.
|
| 10 |
+
|
| 11 |
+
Memo discipline (unchanged from the app.py originals):
|
| 12 |
+
* keys carry (stamp, scope, POOL IDENTITY, question) β the pool's identity, never its size,
|
| 13 |
+
because Cohort passes a different pool per cohort and two same-sized cohorts must not be
|
| 14 |
+
handed each other's answers.
|
| 15 |
+
* `column_values` memoises permanent failures as None but NEVER a transient one (datastore
|
| 16 |
+
still warming after a restart) β a memoised transient means blank measure cells that stay
|
| 17 |
+
blank for the whole session long after the data landed.
|
| 18 |
+
* both memos are bounded by clearing wholesale past a cap β session/runtime-lifetime caches,
|
| 19 |
+
not leaks.
|
| 20 |
+
"""
|
| 21 |
+
import hashlib
|
| 22 |
+
|
| 23 |
+
|
| 24 |
+
def offer(team_id=None, on_error=None):
|
| 25 |
+
"""The measures the condition builder / measure columns may use, or [] if the semantic
|
| 26 |
+
store is unavailable. The caller's BU narrows the offer (a company-level measure cannot be
|
| 27 |
+
answered for one business unit)."""
|
| 28 |
+
try:
|
| 29 |
+
from harness import measure_filter as _mf
|
| 30 |
+
return _mf.measure_fields(team_id)
|
| 31 |
+
except Exception as e: # no store / model problem -> offer nothing
|
| 32 |
+
if on_error:
|
| 33 |
+
on_error('measures', e)
|
| 34 |
+
return []
|
| 35 |
+
|
| 36 |
+
|
| 37 |
+
def _pool_key(allowed_pids):
|
| 38 |
+
"""The pool's identity, hashed. Part of every ANSWER, not just the question: the zero-group
|
| 39 |
+
is reconstructed from it (a customer with no orders in the window has no row for SQL to
|
| 40 |
+
group, so `Sales < 100` would miss exactly the lapsed customers the filter is for)."""
|
| 41 |
+
return hashlib.blake2b(
|
| 42 |
+
repr(sorted(allowed_pids)).encode(), digest_size=8).hexdigest()
|
| 43 |
+
|
| 44 |
+
|
| 45 |
+
def condition_sets(view_configs, pending_cfg, team_id, allowed_pids, today, stamp, memo,
|
| 46 |
+
on_error=None):
|
| 47 |
+
"""Every measure condition across `view_configs` (+ the optional pending config, which goes
|
| 48 |
+
LAST and overwrites β the freshest statement of what the user wants) -> `{ruleId: [pid,β¦]}`.
|
| 49 |
+
|
| 50 |
+
A rule that is incomplete (mid-edit), unresolvable, or errors is left OUT of the answer β
|
| 51 |
+
the client reads a missing id as PENDING and says "Calculatingβ¦" rather than showing a count
|
| 52 |
+
it cannot stand behind. Never resolve an incomplete rule: `to_num(None)` is 0 by the
|
| 53 |
+
engines' shared contract, so a valueless `Sales > β¦` would become `Sales > 0` β a wrong set
|
| 54 |
+
under a confident count.
|
| 55 |
+
"""
|
| 56 |
+
from harness import measure_filter as _mf
|
| 57 |
+
by_id = {}
|
| 58 |
+
for cfg in list(view_configs or []) + [pending_cfg or {}]:
|
| 59 |
+
for rule in _mf.collect((cfg or {}).get('filters')):
|
| 60 |
+
rid = str(rule.get('id') or '')
|
| 61 |
+
if rid:
|
| 62 |
+
by_id[rid] = rule
|
| 63 |
+
rules = list(by_id.values())
|
| 64 |
+
if not rules:
|
| 65 |
+
return {}
|
| 66 |
+
pk = _pool_key(allowed_pids)
|
| 67 |
+
out = {}
|
| 68 |
+
for rule in rules:
|
| 69 |
+
if not _mf.rule_complete(rule):
|
| 70 |
+
continue
|
| 71 |
+
key = (stamp, team_id, pk, _mf.signature(rule))
|
| 72 |
+
if key not in memo:
|
| 73 |
+
try:
|
| 74 |
+
memo[key] = _mf.resolve_rule(rule, today, team_id, allowed_pids)
|
| 75 |
+
except Exception as e:
|
| 76 |
+
# An unresolvable condition must NOT become "everything" or "nothing" silently.
|
| 77 |
+
if on_error:
|
| 78 |
+
on_error('measure-resolve', e)
|
| 79 |
+
continue
|
| 80 |
+
out[str(rule['id'])] = sorted(memo[key])
|
| 81 |
+
if len(memo) > 200: # a lifetime cache, not a leak
|
| 82 |
+
memo.clear()
|
| 83 |
+
return out
|
| 84 |
+
|
| 85 |
+
|
| 86 |
+
def series_values(fields, bucket, buckets, team_id, allowed_pids, today, stamp,
|
| 87 |
+
memo, on_error=None):
|
| 88 |
+
"""`{field key: {'values': {bucket_start: v}, 'agg': kind}}` for every measure column
|
| 89 |
+
among `fields`, over `buckets` = [(start_iso, end_iso), β¦] β the C-TS channel, C-TSWIN
|
| 90 |
+
semantics since wave 14: each field's OWN window slides across the buckets
|
| 91 |
+
(`today := bucket end`), so two fields on the same measure with different windows are
|
| 92 |
+
DIFFERENT questions β the memo key carries the window signature, which the wave-13 key
|
| 93 |
+
(measure-only) silently conflated. `custom` fixed-range windows are the CALLER's to drop
|
| 94 |
+
(`window_fixed`) before calling; one that slips through memoises as a permanent failure.
|
| 95 |
+
|
| 96 |
+
Same memo discipline as `column_values`: keys carry (stamp, scope, POOL IDENTITY, the
|
| 97 |
+
question); a permanent failure memoises as None (the caller reports that field as
|
| 98 |
+
dropped), a TRANSIENT one (datastore still warming) never memoises, so the next call
|
| 99 |
+
retries instead of freezing a blank panel for the session's lifetime.
|
| 100 |
+
"""
|
| 101 |
+
out = {}
|
| 102 |
+
mfields = [f for f in fields or [] if isinstance(f.get('measure'), dict)]
|
| 103 |
+
if not mfields or not buckets:
|
| 104 |
+
return out
|
| 105 |
+
from harness import measure_filter as _mf
|
| 106 |
+
from harness import windows as _wn
|
| 107 |
+
pk = _pool_key(allowed_pids)
|
| 108 |
+
span_from, span_to = str(buckets[0][0]), str(buckets[-1][1])
|
| 109 |
+
for f in mfields:
|
| 110 |
+
spec = f.get('measure') or {}
|
| 111 |
+
mkey = spec.get('key')
|
| 112 |
+
w = _wn.normalize(spec.get('window'))
|
| 113 |
+
wsig = None if w is None else tuple(sorted(w.items()))
|
| 114 |
+
key = (stamp, team_id, pk, mkey, bucket, span_from, span_to, wsig)
|
| 115 |
+
if key not in memo:
|
| 116 |
+
try:
|
| 117 |
+
memo[key] = _mf.resolve_series_slid(mkey, spec.get('window'), buckets,
|
| 118 |
+
today, team_id, allowed_pids)
|
| 119 |
+
except Exception as e:
|
| 120 |
+
if on_error:
|
| 121 |
+
on_error('measure-series', e)
|
| 122 |
+
transient = False
|
| 123 |
+
try:
|
| 124 |
+
from harness import datastore as _ds
|
| 125 |
+
transient = not _ds.ready()
|
| 126 |
+
except Exception:
|
| 127 |
+
transient = False
|
| 128 |
+
if transient:
|
| 129 |
+
continue # no memo entry -> the next call tries again
|
| 130 |
+
memo[key] = None
|
| 131 |
+
ans = memo[key]
|
| 132 |
+
if ans is None:
|
| 133 |
+
continue
|
| 134 |
+
out[f['key']] = ans
|
| 135 |
+
if len(memo) > 200:
|
| 136 |
+
memo.clear()
|
| 137 |
+
return out
|
| 138 |
+
|
| 139 |
+
|
| 140 |
+
def column_values(fields, team_id, allowed_pids, today, stamp, memo, on_error=None):
|
| 141 |
+
"""Values for every FORMULA-MEASURE column among `fields` -> `{pid: {field key: value}}`,
|
| 142 |
+
handed to `rows_from_pool(derived=β¦)` beside the cohort cells.
|
| 143 |
+
|
| 144 |
+
A column that cannot be computed (store down, BU scope on a company-level measure,
|
| 145 |
+
unresolvable window) degrades to BLANK β the value dict simply omits that field key, and
|
| 146 |
+
the client renders empty cells, never $0: blank is "could not compute", 0 is a real zero.
|
| 147 |
+
"""
|
| 148 |
+
mfields = [f for f in fields or [] if isinstance(f.get('measure'), dict)]
|
| 149 |
+
if not mfields:
|
| 150 |
+
return {}
|
| 151 |
+
from harness import measure_filter as _mf
|
| 152 |
+
from harness import windows as _wn
|
| 153 |
+
pk = _pool_key(allowed_pids)
|
| 154 |
+
out = {}
|
| 155 |
+
for f in mfields:
|
| 156 |
+
spec = f['measure']
|
| 157 |
+
w = _wn.normalize(spec.get('window'))
|
| 158 |
+
key = (stamp, team_id, pk, spec.get('key'),
|
| 159 |
+
None if w is None else tuple(sorted(w.items())))
|
| 160 |
+
if key not in memo:
|
| 161 |
+
try:
|
| 162 |
+
memo[key] = _mf.resolve_values(spec.get('key'), spec.get('window'),
|
| 163 |
+
today, team_id, allowed_pids)
|
| 164 |
+
except Exception as e:
|
| 165 |
+
if on_error:
|
| 166 |
+
on_error('measure-column', e)
|
| 167 |
+
# β ONLY A PERMANENT FAILURE MAY BE MEMOISED. "The data cache is still warming
|
| 168 |
+
# up after a restart" is TRANSIENT and self-healing β memoising it means a user
|
| 169 |
+
# who opened the page during that window gets blank measure cells that STAY
|
| 170 |
+
# blank for the whole session. A permanent failure (an unadmitted measure, an
|
| 171 |
+
# unresolvable window) still memoises, so the retry-storm this guard was
|
| 172 |
+
# written for cannot come back.
|
| 173 |
+
transient = False
|
| 174 |
+
try:
|
| 175 |
+
from harness import datastore as _ds
|
| 176 |
+
transient = not _ds.ready()
|
| 177 |
+
except Exception:
|
| 178 |
+
transient = False
|
| 179 |
+
if transient:
|
| 180 |
+
continue # no memo entry -> the next call tries again
|
| 181 |
+
memo[key] = None
|
| 182 |
+
vals = memo[key]
|
| 183 |
+
if vals is None:
|
| 184 |
+
continue
|
| 185 |
+
fkey = f['key']
|
| 186 |
+
for pid, v in vals.items():
|
| 187 |
+
out.setdefault(pid, {})[fkey] = v
|
| 188 |
+
if len(memo) > 100:
|
| 189 |
+
memo.clear()
|
| 190 |
+
return out
|
platform/core/odoo.py
ADDED
|
@@ -0,0 +1,201 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""CFO-OS shared Odoo data layer (READ-ONLY).
|
| 2 |
+
|
| 3 |
+
Reuses the existing write-guarded client (ffs_dashboard/odoo_client.py) so there is
|
| 4 |
+
a single source of truth for the connection. Adds:
|
| 5 |
+
- cached singleton client
|
| 6 |
+
- read_group / search_read passthroughs with light cleanup
|
| 7 |
+
- business constants (teams, excluded partners)
|
| 8 |
+
- the "sales universe" base domain (confirmed orders, RI+FFS only)
|
| 9 |
+
|
| 10 |
+
NOTHING here writes to Odoo β the underlying client hard-blocks writes.
|
| 11 |
+
"""
|
| 12 |
+
import os
|
| 13 |
+
import sys
|
| 14 |
+
import queue as _queue
|
| 15 |
+
import threading
|
| 16 |
+
from concurrent.futures import ThreadPoolExecutor
|
| 17 |
+
from functools import lru_cache
|
| 18 |
+
from pathlib import Path
|
| 19 |
+
|
| 20 |
+
# Self-contained read-only client lives at the app root (platform/odoo_client.py) with its
|
| 21 |
+
# own .env β so this whole folder can be moved/hosted anywhere without external paths.
|
| 22 |
+
_ROOT = Path(__file__).resolve().parents[1] # platform/
|
| 23 |
+
if str(_ROOT) not in sys.path:
|
| 24 |
+
sys.path.insert(0, str(_ROOT))
|
| 25 |
+
from odoo_client import OdooClient # noqa: E402 (read-only, write-guarded)
|
| 26 |
+
|
| 27 |
+
# ---- business constants ----
|
| 28 |
+
TEAM_IDS = [5, 6]
|
| 29 |
+
TEAM_NAMES = {5: 'Fisch', 6: 'Royal'}
|
| 30 |
+
# Partners to exclude from analytics (not part of RI/FFS). Sourced from env so no
|
| 31 |
+
# customer name is ever hardcoded in the (potentially public) repo; set as a Space secret.
|
| 32 |
+
EXCLUDE_PARTNER_NAMES = {n.strip() for n in os.environ.get('EXCLUDE_PARTNER_NAMES', '').split(',') if n.strip()}
|
| 33 |
+
|
| 34 |
+
# Confirmed sale.order.line scope for RI+FFS (the analytics "sales universe").
|
| 35 |
+
SALE_LINE_BASE = [
|
| 36 |
+
('state', 'in', ['sale', 'done']),
|
| 37 |
+
('order_id.team_id', 'in', TEAM_IDS),
|
| 38 |
+
('product_id', '!=', False),
|
| 39 |
+
]
|
| 40 |
+
# Posted customer invoices/credit notes (AR + revenue recognised).
|
| 41 |
+
INVOICE_BASE = [
|
| 42 |
+
('move_type', 'in', ['out_invoice', 'out_refund']),
|
| 43 |
+
('state', '=', 'posted'),
|
| 44 |
+
]
|
| 45 |
+
|
| 46 |
+
|
| 47 |
+
@lru_cache(maxsize=1)
|
| 48 |
+
def _global_client():
|
| 49 |
+
"""The single read-only client used for normal (main-thread) calls."""
|
| 50 |
+
return OdooClient()
|
| 51 |
+
|
| 52 |
+
|
| 53 |
+
# ---- concurrent read pool -------------------------------------------------------------------
|
| 54 |
+
# Each Odoo XML-RPC call is a round-trip; a heavy drawer fires a dozen of them and they add up to
|
| 55 |
+
# many seconds when run one-by-one. `parallel()` runs independent READ tasks across a small pool of
|
| 56 |
+
# extra connections, with each task transparently using its own connection via a thread-local β so
|
| 57 |
+
# the existing read_group/search_read/sum_field helpers need no change. READ-ONLY only.
|
| 58 |
+
_POOL_MAX = 8
|
| 59 |
+
_tlocal = threading.local()
|
| 60 |
+
_pool_q = _queue.Queue()
|
| 61 |
+
_pool_n = 0
|
| 62 |
+
_pool_lock = threading.Lock()
|
| 63 |
+
|
| 64 |
+
|
| 65 |
+
def _checkout():
|
| 66 |
+
global _pool_n
|
| 67 |
+
try:
|
| 68 |
+
return _pool_q.get_nowait()
|
| 69 |
+
except _queue.Empty:
|
| 70 |
+
with _pool_lock:
|
| 71 |
+
grow = _pool_n < _POOL_MAX
|
| 72 |
+
if grow:
|
| 73 |
+
_pool_n += 1
|
| 74 |
+
return OdooClient() if grow else _pool_q.get() # else block until one frees
|
| 75 |
+
|
| 76 |
+
|
| 77 |
+
def _checkin(c):
|
| 78 |
+
_pool_q.put(c)
|
| 79 |
+
|
| 80 |
+
|
| 81 |
+
def get_odoo():
|
| 82 |
+
"""The active read-only client. Inside a parallel() task this is that task's pooled connection;
|
| 83 |
+
otherwise it's the shared singleton. Transparent to every read_group/search_read/... caller."""
|
| 84 |
+
c = getattr(_tlocal, 'client', None)
|
| 85 |
+
return c if c is not None else _global_client()
|
| 86 |
+
|
| 87 |
+
|
| 88 |
+
def set_doc_mode(mode):
|
| 89 |
+
"""Recognition basis for THIS thread's domain builds: 'order' (all confirmed orders) or 'invoice'
|
| 90 |
+
(fully-invoiced orders only). Cached bundles call this before pulling; parallel() propagates it to
|
| 91 |
+
each worker. Thread-local, so concurrent user sessions never race on each other's basis."""
|
| 92 |
+
_tlocal.doc_mode = 'invoice' if mode == 'invoice' else 'order'
|
| 93 |
+
|
| 94 |
+
|
| 95 |
+
def doc_mode():
|
| 96 |
+
return getattr(_tlocal, 'doc_mode', 'order')
|
| 97 |
+
|
| 98 |
+
|
| 99 |
+
def parallel(tasks):
|
| 100 |
+
"""Run independent zero-arg READ callables concurrently across the connection pool; returns
|
| 101 |
+
results in order. Inside each task, get_odoo() (hence read_group/search_read/sum_field/
|
| 102 |
+
search_count) uses that task's own connection. Never use for writes."""
|
| 103 |
+
tasks = list(tasks)
|
| 104 |
+
if not tasks:
|
| 105 |
+
return []
|
| 106 |
+
if len(tasks) == 1:
|
| 107 |
+
return [tasks[0]()]
|
| 108 |
+
|
| 109 |
+
caller_mode = doc_mode() # propagate the submitter's basis to each worker
|
| 110 |
+
def _run(fn):
|
| 111 |
+
c = _checkout()
|
| 112 |
+
_tlocal.client = c
|
| 113 |
+
_tlocal.doc_mode = caller_mode
|
| 114 |
+
try:
|
| 115 |
+
return fn()
|
| 116 |
+
finally:
|
| 117 |
+
_tlocal.client = None
|
| 118 |
+
_tlocal.doc_mode = 'order'
|
| 119 |
+
_checkin(c)
|
| 120 |
+
with ThreadPoolExecutor(max_workers=min(len(tasks), _POOL_MAX)) as ex:
|
| 121 |
+
return list(ex.map(_run, tasks))
|
| 122 |
+
|
| 123 |
+
|
| 124 |
+
def parallel_map(task_dict):
|
| 125 |
+
"""parallel() over a {name: callable} mapping -> {name: result}."""
|
| 126 |
+
keys = list(task_dict)
|
| 127 |
+
return dict(zip(keys, parallel([task_dict[k] for k in keys])))
|
| 128 |
+
|
| 129 |
+
|
| 130 |
+
def m2o_id(v):
|
| 131 |
+
return v[0] if isinstance(v, list) and v else None
|
| 132 |
+
|
| 133 |
+
|
| 134 |
+
def m2o_name(v):
|
| 135 |
+
return v[1] if isinstance(v, list) and len(v) > 1 else ''
|
| 136 |
+
|
| 137 |
+
|
| 138 |
+
@lru_cache(maxsize=1)
|
| 139 |
+
def excluded_partner_ids():
|
| 140 |
+
"""Resolve the excluded partner names (from EXCLUDE_PARTNER_NAMES) to ids, for fast
|
| 141 |
+
domain filtering. Cached for the process."""
|
| 142 |
+
o = get_odoo()
|
| 143 |
+
if not EXCLUDE_PARTNER_NAMES:
|
| 144 |
+
return tuple()
|
| 145 |
+
rows = o.search_read('res.partner',
|
| 146 |
+
[('name', 'in', list(EXCLUDE_PARTNER_NAMES))], ['id'])
|
| 147 |
+
return tuple(r['id'] for r in rows)
|
| 148 |
+
|
| 149 |
+
|
| 150 |
+
def sale_line_domain(date_from=None, date_to=None, team_id=None, extra=None, partner_ids=None):
|
| 151 |
+
"""Build a sale.order.line domain in the RI+FFS confirmed-order scope, excluding
|
| 152 |
+
the configured house accounts, with optional date window / single team / extra clauses.
|
| 153 |
+
partner_ids (a collection, possibly empty) restricts to those customers β the carrier for
|
| 154 |
+
the Customer module's Agent filter; None = no partner restriction (an empty set matches none)."""
|
| 155 |
+
dom = list(SALE_LINE_BASE)
|
| 156 |
+
if team_id is not None:
|
| 157 |
+
dom = [d for d in dom if not (isinstance(d, tuple) and d[0] == 'order_id.team_id')]
|
| 158 |
+
dom.append(('order_id.team_id', '=', team_id))
|
| 159 |
+
if date_from:
|
| 160 |
+
dom.append(('order_id.date_order', '>=', f'{date_from} 00:00:00'))
|
| 161 |
+
if date_to:
|
| 162 |
+
dom.append(('order_id.date_order', '<=', f'{date_to} 23:59:59'))
|
| 163 |
+
ex = excluded_partner_ids()
|
| 164 |
+
if ex:
|
| 165 |
+
dom.append(('order_partner_id', 'not in', list(ex)))
|
| 166 |
+
if partner_ids is not None:
|
| 167 |
+
dom.append(('order_partner_id', 'in', list(partner_ids)))
|
| 168 |
+
if doc_mode() == 'invoice':
|
| 169 |
+
dom.append(('order_id.invoice_status', '=', 'invoiced'))
|
| 170 |
+
if extra:
|
| 171 |
+
dom.extend(extra)
|
| 172 |
+
return dom
|
| 173 |
+
|
| 174 |
+
|
| 175 |
+
def read_group(model, domain, fields, groupby, **kw):
|
| 176 |
+
return get_odoo().read_group(model, domain=domain, fields=fields, groupby=groupby, **kw)
|
| 177 |
+
|
| 178 |
+
|
| 179 |
+
def search_read(model, domain=None, fields=None, **kw):
|
| 180 |
+
return get_odoo().search_read(model, domain=domain or [], fields=fields or [], **kw)
|
| 181 |
+
|
| 182 |
+
|
| 183 |
+
def sum_field(model, domain, field):
|
| 184 |
+
"""One-line aggregate sum of `field` over a domain (uses read_group, no row fetch).
|
| 185 |
+
|
| 186 |
+
Robust to the empty match set: a groupby=[] read_group over zero rows makes Odoo return a
|
| 187 |
+
None aggregate, which the server's XML-RPC layer cannot marshal ("cannot marshal None").
|
| 188 |
+
An empty set simply means the sum is 0.0, so we treat that specific Fault as zero."""
|
| 189 |
+
try:
|
| 190 |
+
g = get_odoo().read_group(model, domain=domain, fields=[f'{field}:sum'], groupby=[], lazy=False)
|
| 191 |
+
except Exception as e: # noqa: BLE001 β narrow check below
|
| 192 |
+
if 'cannot marshal None' in str(e):
|
| 193 |
+
return 0.0
|
| 194 |
+
raise
|
| 195 |
+
return (g[0].get(field) or 0.0) if g else 0.0
|
| 196 |
+
|
| 197 |
+
|
| 198 |
+
def distinct_count(model, domain, field):
|
| 199 |
+
"""Distinct count of a (m2o) field over a domain via grouped read_group."""
|
| 200 |
+
g = get_odoo().read_group(model, domain=domain, fields=[field], groupby=[field], lazy=False)
|
| 201 |
+
return sum(1 for r in g if r.get(field))
|
platform/core/periods.py
ADDED
|
@@ -0,0 +1,155 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Period helpers β floral wholesale is highly seasonal, so YoY / same-period
|
| 2 |
+
comparisons are first-class. Everything returns ISO date strings (yyyy-mm-dd).
|
| 3 |
+
|
| 4 |
+
`today` is injectable for testing/reproducibility.
|
| 5 |
+
"""
|
| 6 |
+
import datetime as dt
|
| 7 |
+
|
| 8 |
+
|
| 9 |
+
def _d(x):
|
| 10 |
+
return x.isoformat()
|
| 11 |
+
|
| 12 |
+
|
| 13 |
+
def today():
|
| 14 |
+
return dt.date.today()
|
| 15 |
+
|
| 16 |
+
|
| 17 |
+
def ytd(t=None):
|
| 18 |
+
"""(start, end) for year-to-date through `t`."""
|
| 19 |
+
t = t or today()
|
| 20 |
+
return _d(dt.date(t.year, 1, 1)), _d(t)
|
| 21 |
+
|
| 22 |
+
|
| 23 |
+
def ytd_last_year(t=None):
|
| 24 |
+
"""Same calendar window one year earlier (Jan 1 LY β same month/day LY).
|
| 25 |
+
Clamps Feb 29 β Feb 28 in non-leap years."""
|
| 26 |
+
t = t or today()
|
| 27 |
+
try:
|
| 28 |
+
end = dt.date(t.year - 1, t.month, t.day)
|
| 29 |
+
except ValueError:
|
| 30 |
+
end = dt.date(t.year - 1, t.month, t.day - 1)
|
| 31 |
+
return _d(dt.date(t.year - 1, 1, 1)), end if isinstance(end, str) else _d(end)
|
| 32 |
+
|
| 33 |
+
|
| 34 |
+
def ltm(t=None):
|
| 35 |
+
"""Trailing 12 months ending `t`."""
|
| 36 |
+
t = t or today()
|
| 37 |
+
return _d(t - dt.timedelta(days=365)), _d(t)
|
| 38 |
+
|
| 39 |
+
|
| 40 |
+
def prior_ltm(t=None):
|
| 41 |
+
t = t or today()
|
| 42 |
+
return _d(t - dt.timedelta(days=730)), _d(t - dt.timedelta(days=365))
|
| 43 |
+
|
| 44 |
+
|
| 45 |
+
def mtd(t=None):
|
| 46 |
+
t = t or today()
|
| 47 |
+
return _d(dt.date(t.year, t.month, 1)), _d(t)
|
| 48 |
+
|
| 49 |
+
|
| 50 |
+
def wtd(t=None):
|
| 51 |
+
"""(Monday of this ISO week, t) β week-to-date."""
|
| 52 |
+
t = t or today()
|
| 53 |
+
return _d(t - dt.timedelta(days=t.weekday())), _d(t)
|
| 54 |
+
|
| 55 |
+
|
| 56 |
+
def qtd(t=None):
|
| 57 |
+
"""(first day of this calendar quarter, t) β quarter-to-date."""
|
| 58 |
+
t = t or today()
|
| 59 |
+
q_start_month = ((t.month - 1) // 3) * 3 + 1
|
| 60 |
+
return _d(dt.date(t.year, q_start_month, 1)), _d(t)
|
| 61 |
+
|
| 62 |
+
|
| 63 |
+
def quarter_range(t=None):
|
| 64 |
+
"""(first day, last day) of the calendar quarter containing t."""
|
| 65 |
+
t = t or today()
|
| 66 |
+
qs = ((t.month - 1) // 3) * 3 + 1
|
| 67 |
+
first = dt.date(t.year, qs, 1)
|
| 68 |
+
nxt = dt.date(t.year + (1 if qs + 3 > 12 else 0), ((qs + 2) % 12) + 1, 1)
|
| 69 |
+
return _d(first), _d(nxt - dt.timedelta(days=1))
|
| 70 |
+
|
| 71 |
+
|
| 72 |
+
def shift_year(date_from, date_to, weeks=False):
|
| 73 |
+
"""Same window one year earlier. For week-aligned comparisons (weeks=True) shift by exactly 52
|
| 74 |
+
weeks (364 days) so the weekday lines up; otherwise shift by calendar year (clamping Feb 29)."""
|
| 75 |
+
f = dt.date.fromisoformat(date_from)
|
| 76 |
+
t = dt.date.fromisoformat(date_to)
|
| 77 |
+
if weeks:
|
| 78 |
+
return _d(f - dt.timedelta(days=364)), _d(t - dt.timedelta(days=364))
|
| 79 |
+
|
| 80 |
+
def back(d):
|
| 81 |
+
try:
|
| 82 |
+
return dt.date(d.year - 1, d.month, d.day)
|
| 83 |
+
except ValueError:
|
| 84 |
+
return dt.date(d.year - 1, d.month, d.day - 1)
|
| 85 |
+
return _d(back(f)), _d(back(t))
|
| 86 |
+
|
| 87 |
+
|
| 88 |
+
def last_n_days(n, t=None):
|
| 89 |
+
t = t or today()
|
| 90 |
+
return _d(t - dt.timedelta(days=n)), _d(t)
|
| 91 |
+
|
| 92 |
+
|
| 93 |
+
def year_range(year):
|
| 94 |
+
return f'{year}-01-01', f'{year}-12-31'
|
| 95 |
+
|
| 96 |
+
|
| 97 |
+
def month_range(year, month):
|
| 98 |
+
"""(first day, last day) of a given calendar month, ISO strings."""
|
| 99 |
+
first = dt.date(year, month, 1)
|
| 100 |
+
nxt = dt.date(year + (1 if month == 12 else 0), 1 if month == 12 else month + 1, 1)
|
| 101 |
+
return _d(first), _d(nxt - dt.timedelta(days=1))
|
| 102 |
+
|
| 103 |
+
|
| 104 |
+
def period_options(t=None, n_months=15):
|
| 105 |
+
"""Ordered {label: (date_from, date_to)} for the close/reconciliation period picker."""
|
| 106 |
+
t = t or today()
|
| 107 |
+
opts = {}
|
| 108 |
+
opts['This month (to date)'] = (_d(dt.date(t.year, t.month, 1)), _d(t))
|
| 109 |
+
lm_year, lm_month = (t.year, t.month - 1) if t.month > 1 else (t.year - 1, 12)
|
| 110 |
+
lf, lt = month_range(lm_year, lm_month)
|
| 111 |
+
opts[f'Last month ({lf[:7]})'] = (lf, lt)
|
| 112 |
+
opts['Quarter to date'] = (_d(dt.date(t.year, ((t.month - 1) // 3) * 3 + 1, 1)), _d(t))
|
| 113 |
+
opts['Year to date'] = (_d(dt.date(t.year, 1, 1)), _d(t))
|
| 114 |
+
for label, mf, mt in reversed(month_starts(n_months, t)):
|
| 115 |
+
if label == f'{t.year}-{t.month:02d}' or label == lf[:7]:
|
| 116 |
+
continue
|
| 117 |
+
opts[label] = (mf, mt)
|
| 118 |
+
return opts
|
| 119 |
+
|
| 120 |
+
|
| 121 |
+
def month_starts(n_back=13, t=None):
|
| 122 |
+
"""List of (yyyy-mm, start, end) for the last n_back months including current."""
|
| 123 |
+
t = t or today()
|
| 124 |
+
out = []
|
| 125 |
+
y, m = t.year, t.month
|
| 126 |
+
for _ in range(n_back):
|
| 127 |
+
start = dt.date(y, m, 1)
|
| 128 |
+
end = (dt.date(y + (m // 12), (m % 12) + 1, 1) - dt.timedelta(days=1))
|
| 129 |
+
out.append((f'{y}-{m:02d}', _d(start), _d(end)))
|
| 130 |
+
m -= 1
|
| 131 |
+
if m == 0:
|
| 132 |
+
m = 12
|
| 133 |
+
y -= 1
|
| 134 |
+
return list(reversed(out))
|
| 135 |
+
|
| 136 |
+
|
| 137 |
+
def week_starts(n_back=12, t=None):
|
| 138 |
+
"""List of (label, start, end) for the last n_back ISO weeks (MonβSun) including the current
|
| 139 |
+
(partial) week. label = the Monday as 'MM-DD' (compact, chart-friendly); end is clamped to t
|
| 140 |
+
for the current week so weekly totals never include future days."""
|
| 141 |
+
t = t or today()
|
| 142 |
+
this_mon = t - dt.timedelta(days=t.weekday())
|
| 143 |
+
out = []
|
| 144 |
+
for i in range(n_back):
|
| 145 |
+
mon = this_mon - dt.timedelta(weeks=i)
|
| 146 |
+
sun = mon + dt.timedelta(days=6)
|
| 147 |
+
end = min(sun, t)
|
| 148 |
+
out.append((f'{mon.month:02d}-{mon.day:02d}', _d(mon), _d(end)))
|
| 149 |
+
return list(reversed(out))
|
| 150 |
+
|
| 151 |
+
|
| 152 |
+
def yoy_pct(this_v, last_v):
|
| 153 |
+
if last_v:
|
| 154 |
+
return (this_v - last_v) / last_v * 100.0
|
| 155 |
+
return None
|
platform/core/perm_migrate.py
ADDED
|
@@ -0,0 +1,156 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""core/perm_migrate.py β R1's one-time migration: `bus`/`agent` become a PERMANENT FILTER.
|
| 2 |
+
|
| 3 |
+
Owner ruling R1: BU access stops being its own field and becomes an ordinary permanent filter
|
| 4 |
+
under the permissioning engine, declared and edited exactly like "Agent is Tara". This module
|
| 5 |
+
builds that filter from the legacy record and β the part that matters β PROVES the swap is a
|
| 6 |
+
no-op before anything relies on it.
|
| 7 |
+
|
| 8 |
+
β WHY THE PROOF IS ABOUT VALUES AND NOT ABOUT ROWS (C-PERM amendment 3).
|
| 9 |
+
`modules/customer_data._pool_build` passes `team_id` into `cust._cust_rev` three times (YTD, LY,
|
| 10 |
+
LTM) and into `cust._cadence_bulk`. It therefore decides what `rev`, `ly`, `ltm`, `aov`,
|
| 11 |
+
`est_missed` and the derived `status` MEAN β it is not a row filter. Swap it for a post-filter
|
| 12 |
+
and a Fisch-only user keeps a plausible row LIST while every number on it quietly becomes
|
| 13 |
+
Fisch+Royal, worst for `dba = Both` customers, who are precisely the ones a BU filter admits.
|
| 14 |
+
A pid-set reconciliation reports GREEN on that. So `reconcile()` compares VALUES.
|
| 15 |
+
|
| 16 |
+
The pushdown survives as `perm_scope.derive_pool_scope` β a DERIVATION OF the filter rather than
|
| 17 |
+
a wall beside it β so a correctly migrated record produces the IDENTICAL pool call, and the
|
| 18 |
+
reconciliation below should be exact rather than merely close. If it is not exact, that is a
|
| 19 |
+
finding for the owner, not a tolerance to widen.
|
| 20 |
+
"""
|
| 21 |
+
import core.perm_scope as perm_scope
|
| 22 |
+
import core.perms as perms
|
| 23 |
+
|
| 24 |
+
MODULE = 'customer_data'
|
| 25 |
+
|
| 26 |
+
#: `_dba_attrs` emits exactly {'Fisch','Royal','Both'} (blank = no brand attributable in 24
|
| 27 |
+
#: months). A BU grant admits its own brand AND the dual-brand customers: a Both customer IS a
|
| 28 |
+
#: Fisch customer. Mirrors `perm_scope._DBA_TEAM`, inverted.
|
| 29 |
+
#:
|
| 30 |
+
#: β TWO VALUES, AND THE COLUMN VOCABULARY HAS NO "IS ANY OF". `FILTER_OPS` is single-valued by
|
| 31 |
+
#: design (`eq`, `neq`, `contains`, β¦) β `anyOf` belongs to the COHORT vocabulary and is
|
| 32 |
+
#: deliberately DISJOINT from it, so a column leaf carrying `anyOf` is refused by every engine
|
| 33 |
+
#: (`filter_eval` returns False, `clean_filter_tree` drops it). This migration's first draft
|
| 34 |
+
#: emitted exactly that and `reconcile()` caught it as two lost customers. So the BU condition
|
| 35 |
+
#: is an OR GROUP of `eq` leaves, which is what the condition builder itself produces.
|
| 36 |
+
_TEAM_DBA = {5: ('Fisch', 'Both'), 6: ('Royal', 'Both')}
|
| 37 |
+
|
| 38 |
+
|
| 39 |
+
def filter_for(rec):
|
| 40 |
+
"""The permanent filter equivalent to this record's legacy `bus` + `agent`, or None when the
|
| 41 |
+
record was unrestricted (an admin, or `bus:'all'` with no agent link).
|
| 42 |
+
|
| 43 |
+
AND-conjoined at the root, and top-level by construction β which is also what makes it
|
| 44 |
+
pushdown-derivable (`derive_pool_scope` reads top-level AND leaves only).
|
| 45 |
+
"""
|
| 46 |
+
nodes = []
|
| 47 |
+
team_id = perms.scope_team_id(rec)
|
| 48 |
+
if team_id in _TEAM_DBA:
|
| 49 |
+
nodes.append({'conj': 'or',
|
| 50 |
+
'children': [{'colId': 'dba', 'op': 'eq', 'value': v, 'value2': ''}
|
| 51 |
+
for v in _TEAM_DBA[team_id]]})
|
| 52 |
+
agent = perms.scope_agent(rec)
|
| 53 |
+
if agent:
|
| 54 |
+
nodes.append({'colId': 'agent', 'op': 'eq', 'value': str(agent), 'value2': ''})
|
| 55 |
+
if not nodes:
|
| 56 |
+
return None
|
| 57 |
+
return {'conj': 'and', 'nodes': nodes}
|
| 58 |
+
|
| 59 |
+
|
| 60 |
+
def perms_for(rec, modules=(MODULE,)):
|
| 61 |
+
"""The `perms` block this legacy record becomes.
|
| 62 |
+
|
| 63 |
+
β ACCESS IS COPIED FROM THE OLD GRANT, NOT ASSUMED. `may_open` is what decided reachability
|
| 64 |
+
before this wave, so migrating a record must not widen it β and on a migrated record an
|
| 65 |
+
undeclared module DENIES (amendment 4), so every module the user could reach has to be
|
| 66 |
+
declared here or the migration is a lockout.
|
| 67 |
+
"""
|
| 68 |
+
out = {}
|
| 69 |
+
for key in modules:
|
| 70 |
+
out[key] = {'access': bool(perms.may_open(rec, key)),
|
| 71 |
+
'filter': filter_for(rec),
|
| 72 |
+
'hiddenFields': []}
|
| 73 |
+
return out
|
| 74 |
+
|
| 75 |
+
|
| 76 |
+
def plan(registry_dict, modules=(MODULE,)):
|
| 77 |
+
"""`{username: perms_block}` for every record that is not already migrated.
|
| 78 |
+
|
| 79 |
+
Admins are included and get an all-access block with no filter: they bypass `perm_scope`
|
| 80 |
+
anyway, but leaving them un-migrated would keep the legacy reader alive for the one account
|
| 81 |
+
most likely to be inspected when something looks wrong.
|
| 82 |
+
"""
|
| 83 |
+
out = {}
|
| 84 |
+
for uname, rec in (registry_dict or {}).items():
|
| 85 |
+
if not isinstance(rec, dict) or perm_scope.is_migrated(rec):
|
| 86 |
+
continue
|
| 87 |
+
out[uname] = perms_for(rec, modules)
|
| 88 |
+
return out
|
| 89 |
+
|
| 90 |
+
|
| 91 |
+
#: The row members whose meaning depends on `team_id` β the ones amendment 3 is about. A
|
| 92 |
+
#: reconciliation that skipped these would be the pid-only gate that cannot see the leak.
|
| 93 |
+
VALUE_KEYS = ('rev', 'ly', 'ltm', 'aov', 'est_missed', 'status', 'orders', 'last_order')
|
| 94 |
+
|
| 95 |
+
|
| 96 |
+
def reconcile(rec, pool_fn, modules=(MODULE,)):
|
| 97 |
+
"""Prove the migrated record sees EXACTLY what the legacy record saw.
|
| 98 |
+
|
| 99 |
+
`pool_fn(agent, team_id) -> rows` is injected so this is testable without Odoo and so the
|
| 100 |
+
caller decides whether to hit the live builder or a fixture.
|
| 101 |
+
|
| 102 |
+
Returns `{ok, legacy_scope, derived_scope, lost, gained, value_diffs}`:
|
| 103 |
+
* `lost` β pids the legacy wall admitted and the migrated one drops. Expected to be
|
| 104 |
+
EMPTY. One narrow class is known and must be REPORTED rather than tolerated:
|
| 105 |
+
`_dba_attrs` uses a 731-day window while `_cust_rev`'s LY window can reach a
|
| 106 |
+
little further back, so a customer whose only in-scope revenue predates that
|
| 107 |
+
window carries a BLANK `dba` and fails a `dba` leaf. Blank is the honest "no
|
| 108 |
+
brand attributable" state, so such a customer is NAMED here, never rounded
|
| 109 |
+
away.
|
| 110 |
+
* `gained` β pids the migrated wall admits and the legacy one did not. Must be empty: it
|
| 111 |
+
is the widening direction, and there is no acceptable non-empty value.
|
| 112 |
+
* `value_diffs` β per-pid disagreements on VALUE_KEYS. The assertion amendment 3 exists
|
| 113 |
+
for; a non-empty list means the pushdown was not applied and the numbers are
|
| 114 |
+
consolidated.
|
| 115 |
+
"""
|
| 116 |
+
legacy_scope = (perms.scope_agent(rec), perms.scope_team_id(rec))
|
| 117 |
+
migrated = dict(rec)
|
| 118 |
+
migrated['perms'] = perms_for(rec, modules)
|
| 119 |
+
migrated['perms_v'] = perm_scope.PERMS_VERSION
|
| 120 |
+
team_id, agent = perm_scope.derive_pool_scope(migrated, MODULE)
|
| 121 |
+
derived_scope = (agent, team_id)
|
| 122 |
+
|
| 123 |
+
legacy_rows = pool_fn(legacy_scope[0], legacy_scope[1])
|
| 124 |
+
derived_rows = pool_fn(agent, team_id)
|
| 125 |
+
fields = _fields_of(legacy_rows)
|
| 126 |
+
kept = perm_scope.apply_row_scope(derived_rows, migrated, MODULE, fields)
|
| 127 |
+
|
| 128 |
+
legacy_by = {r.get('pid'): r for r in legacy_rows}
|
| 129 |
+
kept_by = {r.get('pid'): r for r in kept}
|
| 130 |
+
lost = sorted(set(legacy_by) - set(kept_by))
|
| 131 |
+
gained = sorted(set(kept_by) - set(legacy_by))
|
| 132 |
+
|
| 133 |
+
value_diffs = []
|
| 134 |
+
for pid in sorted(set(legacy_by) & set(kept_by)):
|
| 135 |
+
a, b = legacy_by[pid], kept_by[pid]
|
| 136 |
+
for k in VALUE_KEYS:
|
| 137 |
+
if k in a and a.get(k) != b.get(k):
|
| 138 |
+
value_diffs.append({'pid': pid, 'key': k,
|
| 139 |
+
'legacy': a.get(k), 'migrated': b.get(k)})
|
| 140 |
+
return {'ok': not lost and not gained and not value_diffs,
|
| 141 |
+
'legacy_scope': legacy_scope, 'derived_scope': derived_scope,
|
| 142 |
+
'lost': lost, 'gained': gained, 'value_diffs': value_diffs,
|
| 143 |
+
'names': {r.get('pid'): r.get('customer') for r in legacy_rows
|
| 144 |
+
if r.get('pid') in set(lost)}}
|
| 145 |
+
|
| 146 |
+
|
| 147 |
+
def _fields_of(rows):
|
| 148 |
+
"""A field list good enough to EVALUATE with, inferred from the assembled rows.
|
| 149 |
+
|
| 150 |
+
The permanent filters this module builds name `dba` (text) and `agent` (text) only, so the
|
| 151 |
+
inference is exact for them. It is deliberately not a schema: `permits()` needs types to
|
| 152 |
+
dispatch comparisons, and inferring `text` for an unknown key would be a guess with
|
| 153 |
+
consequences β so anything not recognised is left out, which makes its leaf unanswerable and
|
| 154 |
+
therefore DENYING rather than silently mistyped.
|
| 155 |
+
"""
|
| 156 |
+
return [{'key': 'dba', 'type': 'text'}, {'key': 'agent', 'type': 'text'}]
|
platform/core/perm_scope.py
ADDED
|
@@ -0,0 +1,286 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""core/perm_scope.py β the permission WALL for table modules (wave 15, C-PERM).
|
| 2 |
+
|
| 3 |
+
ONE place answers the three questions a restricted account raises on a grid surface:
|
| 4 |
+
|
| 5 |
+
may_access(user, module) may they open it at all?
|
| 6 |
+
visible_fields(fields, u, mod) which COLUMNS may they receive?
|
| 7 |
+
apply_row_scope(rows, u, mod, β¦) which ROWS may they receive?
|
| 8 |
+
|
| 9 |
+
plus one that exists only because of how the pool is built:
|
| 10 |
+
|
| 11 |
+
derive_pool_scope(user, module) which (team_id, agent) must the pool be BUILT with?
|
| 12 |
+
|
| 13 |
+
Both hosts call these β `aios-web/api` (`grid_assembly`) and `app.py` (`_table_grid`) β because
|
| 14 |
+
a wall that exists on one runtime and not the other is not a wall. `core/perms.py` stays what it
|
| 15 |
+
is (module GRANTS + the legacy BU derivation); this module is the row/field/pushdown layer that
|
| 16 |
+
sits on top, and it is deliberately a separate file so the legacy readers can keep their
|
| 17 |
+
semantics untouched while this one fails closed.
|
| 18 |
+
|
| 19 |
+
ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 20 |
+
THE RECORD
|
| 21 |
+
|
| 22 |
+
user['perms'] = {'<module_key>': {'access': bool,
|
| 23 |
+
'filter': {'conj'?: 'and'|'or', 'nodes': [...]} | None,
|
| 24 |
+
'hiddenFields': ['<field key>', ...]}}
|
| 25 |
+
user['perms_v'] = 1 stamped by the migration and by every write
|
| 26 |
+
|
| 27 |
+
`perms_v` is the EXPLICIT-RESOLUTION marker, and it is here because of `permissioning.md`
|
| 28 |
+
Part II gap #5: "`allowed_modules() is None` is fail-open by default β¦ make 'resolved:
|
| 29 |
+
unrestricted' an explicit value so absence/uncertainty DENIES." The same class already shipped
|
| 30 |
+
twice in this codebase (`modules: []` and `bus: [<unknown id>]` both read as UNRESTRICTED β
|
| 31 |
+
`routes_admin.py`'s own docstring documents both). So:
|
| 32 |
+
|
| 33 |
+
* `perms_v` ABSENT β the record is UN-MIGRATED, and the LEGACY wall applies unchanged
|
| 34 |
+
(`core.perms` module grants + the `bus`/`agent` query scope). That is not fail-open: it is
|
| 35 |
+
today's real wall, and it bounds the rollout window to "until the migration runs".
|
| 36 |
+
* `perms_v == 1` and the module has NO entry β **DENY**. Absence now means what it says.
|
| 37 |
+
* `role == 'admin'` bypasses all of it β which is also what keeps BREAK-GLASS alive.
|
| 38 |
+
`deps._user_for` hands back a hardcoded master dict on a store outage
|
| 39 |
+
(`{'username':'admin','role':'admin','bus':'all','modules':'all'}`) that will never carry a
|
| 40 |
+
perms block; without this clause an explicit-marker scheme locks the owner out of their own
|
| 41 |
+
product at exactly the moment the store is broken.
|
| 42 |
+
"""
|
| 43 |
+
import re
|
| 44 |
+
|
| 45 |
+
import core.perms as perms
|
| 46 |
+
|
| 47 |
+
#: `aios_grid._FORMULA_REF`'s pattern, restated rather than imported: this module is imported by
|
| 48 |
+
#: the API's request path and `aios_grid` pulls in the whole grid stack. Same regex, one line,
|
| 49 |
+
#: and `verify_api` asserts the two agree so it cannot drift into a different grammar.
|
| 50 |
+
_FORMULA_REF = re.compile(r"\{([^{}]*)\}")
|
| 51 |
+
|
| 52 |
+
PERMS_VERSION = 1
|
| 53 |
+
|
| 54 |
+
|
| 55 |
+
def _rec(user):
|
| 56 |
+
return user if isinstance(user, dict) else {}
|
| 57 |
+
|
| 58 |
+
|
| 59 |
+
def is_migrated(user):
|
| 60 |
+
"""True once this record carries an explicit resolution. See the module docstring."""
|
| 61 |
+
return int(_rec(user).get('perms_v') or 0) >= PERMS_VERSION
|
| 62 |
+
|
| 63 |
+
|
| 64 |
+
def entry(user, module):
|
| 65 |
+
"""This user's declared permissions for `module`, or None if nothing is declared.
|
| 66 |
+
|
| 67 |
+
None is AMBIGUOUS on purpose and every caller must resolve it against `is_migrated`:
|
| 68 |
+
on a migrated record it means DENY, on a legacy one it means "ask the old wall".
|
| 69 |
+
"""
|
| 70 |
+
p = _rec(user).get('perms')
|
| 71 |
+
if not isinstance(p, dict):
|
| 72 |
+
return None
|
| 73 |
+
e = p.get(module)
|
| 74 |
+
return e if isinstance(e, dict) else None
|
| 75 |
+
|
| 76 |
+
|
| 77 |
+
def may_access(user, module):
|
| 78 |
+
"""May this account open `module` at all? Fail-closed on a migrated record."""
|
| 79 |
+
if perms.is_admin(user):
|
| 80 |
+
return True
|
| 81 |
+
e = entry(user, module)
|
| 82 |
+
if e is not None:
|
| 83 |
+
return bool(e.get('access', True))
|
| 84 |
+
if is_migrated(user):
|
| 85 |
+
# Migrated and undeclared = denied. This is the whole point of the marker.
|
| 86 |
+
return False
|
| 87 |
+
return perms.may_open(user, module) # legacy record: the old grant wall
|
| 88 |
+
|
| 89 |
+
|
| 90 |
+
# ββ FIELDS βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 91 |
+
def hidden_keys(user, module, fields):
|
| 92 |
+
"""The TRANSITIVE closure of hidden field keys (C-PERM amendment 5).
|
| 93 |
+
|
| 94 |
+
β WHY A CLOSURE AND NOT A SET DIFFERENCE. A formula field is computed in the BROWSER
|
| 95 |
+
(`formulaEngine.ts`, injected by `computedRows`) from `{ref}`s into other columns, and a
|
| 96 |
+
measure column's value arrives precomputed in `derived`. So hiding field X has exactly three
|
| 97 |
+
possible outcomes and only one of them is coherent:
|
| 98 |
+
|
| 99 |
+
strip X, keep formulas β every formula over X computes blank or wrong, silently
|
| 100 |
+
keep X's value for them β X has leaked, wearing a formula's name
|
| 101 |
+
strip X AND its dependentsβ the only honest answer
|
| 102 |
+
|
| 103 |
+
So a hidden field drags every formula that references it β and every formula that references
|
| 104 |
+
THAT formula, hence the fixpoint loop β out of the payload with it.
|
| 105 |
+
|
| 106 |
+
β This runs on every assembly, so it is a fixpoint over a handful of custom fields, not a
|
| 107 |
+
graph library. `MAX_PASSES` bounds a reference cycle the client would refuse to evaluate
|
| 108 |
+
anyway; without it a self-referential pair would spin here.
|
| 109 |
+
"""
|
| 110 |
+
e = entry(user, module)
|
| 111 |
+
if perms.is_admin(user) or not e:
|
| 112 |
+
return frozenset()
|
| 113 |
+
hidden = {str(k) for k in (e.get('hiddenFields') or ()) if k}
|
| 114 |
+
if not hidden:
|
| 115 |
+
return frozenset()
|
| 116 |
+
|
| 117 |
+
refs = {}
|
| 118 |
+
for f in fields or ():
|
| 119 |
+
if not isinstance(f, dict) or not f.get('key'):
|
| 120 |
+
continue
|
| 121 |
+
expr = f.get('formula')
|
| 122 |
+
if isinstance(expr, str) and expr:
|
| 123 |
+
refs[f['key']] = {m.strip() for m in _FORMULA_REF.findall(expr) if m.strip()}
|
| 124 |
+
|
| 125 |
+
MAX_PASSES = 12
|
| 126 |
+
for _ in range(MAX_PASSES):
|
| 127 |
+
grew = False
|
| 128 |
+
for key, deps in refs.items():
|
| 129 |
+
if key not in hidden and deps & hidden:
|
| 130 |
+
hidden.add(key)
|
| 131 |
+
grew = True
|
| 132 |
+
if not grew:
|
| 133 |
+
break
|
| 134 |
+
return frozenset(hidden)
|
| 135 |
+
|
| 136 |
+
|
| 137 |
+
def visible_fields(fields, user, module):
|
| 138 |
+
"""`fields` minus the hidden closure. Order preserved β the column order is the user's."""
|
| 139 |
+
hide = hidden_keys(user, module, fields)
|
| 140 |
+
if not hide:
|
| 141 |
+
return list(fields or ())
|
| 142 |
+
return [f for f in (fields or ())
|
| 143 |
+
if not (isinstance(f, dict) and f.get('key') in hide)]
|
| 144 |
+
|
| 145 |
+
|
| 146 |
+
def strip_row(row, hide):
|
| 147 |
+
"""Drop hidden keys from ONE assembled row. Cheap enough to run per row, and it must run
|
| 148 |
+
per row: the field list and the row payload are two different wires, and stripping only the
|
| 149 |
+
first would leave the value sitting in the second where anyone can read it."""
|
| 150 |
+
if not hide or not isinstance(row, dict):
|
| 151 |
+
return row
|
| 152 |
+
return {k: v for k, v in row.items() if k not in hide}
|
| 153 |
+
|
| 154 |
+
|
| 155 |
+
# ββ ROWS βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 156 |
+
def apply_row_scope(rows, user, module, fields, ctx=None):
|
| 157 |
+
"""The rows this account may receive: `filter_eval.permits` over the permanent filter.
|
| 158 |
+
|
| 159 |
+
`permits`, never `matches` β an unanswerable permanent filter DENIES rather than being
|
| 160 |
+
ignored. See `harness/filter_eval`'s docstring for the field-rename walkthrough that makes
|
| 161 |
+
the difference a leak rather than a preference.
|
| 162 |
+
"""
|
| 163 |
+
if perms.is_admin(user):
|
| 164 |
+
return list(rows or ())
|
| 165 |
+
e = entry(user, module)
|
| 166 |
+
tree = (e or {}).get('filter')
|
| 167 |
+
if not tree:
|
| 168 |
+
return list(rows or ())
|
| 169 |
+
from harness import filter_eval as fe
|
| 170 |
+
return [r for r in (rows or ()) if fe.permits(tree, r, fields, ctx)]
|
| 171 |
+
|
| 172 |
+
|
| 173 |
+
# ββ THE PUSHDOWN βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 174 |
+
#: `dba` value sets that pin a BU, and the Odoo team they pin. `_dba_attrs` emits exactly
|
| 175 |
+
#: {'Fisch','Royal','Both'} (blank for "no brand attributable in 24 months"), so a permission
|
| 176 |
+
#: filter admitting Fisch admits {Fisch, Both} β a Both customer IS a Fisch customer.
|
| 177 |
+
_DBA_TEAM = ((frozenset({'fisch', 'both'}), 5), (frozenset({'royal', 'both'}), 6))
|
| 178 |
+
|
| 179 |
+
#: Only equality pushes down. `FILTER_OPS` is single-valued (there is no column-level "is any
|
| 180 |
+
#: of" β that vocabulary belongs to cohort leaves and is disjoint), so a multi-value BU
|
| 181 |
+
#: condition arrives as an OR GROUP of `eq` leaves, handled by `_group_dba_team` below.
|
| 182 |
+
_PUSHDOWN_OPS = frozenset({'eq'})
|
| 183 |
+
|
| 184 |
+
|
| 185 |
+
def derive_pool_scope(user, module):
|
| 186 |
+
"""`(team_id, agent)` the POOL must be BUILT with, derived from the permanent filter.
|
| 187 |
+
|
| 188 |
+
β THIS EXISTS BECAUSE `team_id` SHAPES VALUES, NOT ROW MEMBERSHIP (C-PERM amendment 3).
|
| 189 |
+
`modules/customer_data._pool_build` passes `team_id` into `cust._cust_rev` three times (YTD,
|
| 190 |
+
LY, LTM) and into `cust._cadence_bulk`, so it decides what `rev`, `ly`, `ltm`, `aov`,
|
| 191 |
+
`est_missed` and the derived `status` MEAN. Enforce a BU purely as a post-filter and a
|
| 192 |
+
Fisch-only user keeps a correct-looking row LIST while every number on it silently becomes
|
| 193 |
+
Fisch+Royal β worst for `dba = Both` customers, who are exactly the ones a BU filter admits.
|
| 194 |
+
A pid-level reconciliation cannot see that; the values one can, and does.
|
| 195 |
+
|
| 196 |
+
So the query-level pushdown SURVIVES β but as a DERIVATION OF the permanent filter rather
|
| 197 |
+
than a second wall beside it, which is what keeps R1's "BU access is just a filter" true at
|
| 198 |
+
the level the owner asked for it (one declaration, one UI, one engine).
|
| 199 |
+
|
| 200 |
+
Pure, and recomputed per request rather than stored: a stored derivation drifts from the
|
| 201 |
+
filter it came from, and then two things disagree about what an account may see.
|
| 202 |
+
|
| 203 |
+
Reads TOP-LEVEL AND-conjunction leaves ONLY. A leaf under `or` guarantees no narrowing β
|
| 204 |
+
`dba is Fisch OR revenue > 10` must not pin the pool to Fisch β so it never pushes down.
|
| 205 |
+
Anything not recognised here simply is not pushed down; `apply_row_scope` still applies the
|
| 206 |
+
whole tree, so the wall is unchanged either way. Belt AND braces, deliberately: the pushdown
|
| 207 |
+
is what makes the VALUES right, `permits()` is what makes the ROWS right.
|
| 208 |
+
"""
|
| 209 |
+
if perms.is_admin(user):
|
| 210 |
+
return None, None
|
| 211 |
+
e = entry(user, module)
|
| 212 |
+
tree = (e or {}).get('filter')
|
| 213 |
+
if not tree:
|
| 214 |
+
# Un-migrated records still answer through the legacy derivation, so the old wall keeps
|
| 215 |
+
# working until the migration has run.
|
| 216 |
+
if not is_migrated(user):
|
| 217 |
+
return perms.scope_team_id(user), perms.scope_agent(user)
|
| 218 |
+
return None, None
|
| 219 |
+
|
| 220 |
+
from harness import filter_eval as fe
|
| 221 |
+
nodes, conj = fe.tree_parts(tree)
|
| 222 |
+
if conj == 'or':
|
| 223 |
+
return None, None
|
| 224 |
+
|
| 225 |
+
team_id, agent = None, None
|
| 226 |
+
for n in nodes:
|
| 227 |
+
if not isinstance(n, dict):
|
| 228 |
+
continue
|
| 229 |
+
if isinstance(n.get('children'), list):
|
| 230 |
+
# A top-level OR GROUP under an AND root IS a guaranteed narrowing β every row must
|
| 231 |
+
# satisfy it β so it may push down, unlike a leaf under an OR ROOT (refused above).
|
| 232 |
+
# This is the shape a multi-value BU condition actually takes; see `_group_dba_team`.
|
| 233 |
+
tid = _group_dba_team(n)
|
| 234 |
+
if tid is not None:
|
| 235 |
+
team_id = tid if team_id in (None, tid) else None
|
| 236 |
+
continue
|
| 237 |
+
if n.get('op') not in _PUSHDOWN_OPS:
|
| 238 |
+
continue
|
| 239 |
+
col = n.get('colId')
|
| 240 |
+
raw = n.get('value')
|
| 241 |
+
if col == 'dba':
|
| 242 |
+
vals = {v.strip().lower() for v in str(raw or '').split(',') if v.strip()}
|
| 243 |
+
if not vals:
|
| 244 |
+
continue
|
| 245 |
+
for allowed, tid in _DBA_TEAM:
|
| 246 |
+
if vals <= allowed:
|
| 247 |
+
# Both BUs named = no narrowing to push; leave it to the post-filter.
|
| 248 |
+
team_id = tid if team_id in (None, tid) else None
|
| 249 |
+
break
|
| 250 |
+
elif col == 'agent':
|
| 251 |
+
v = str(raw or '').strip()
|
| 252 |
+
# A SET of agents cannot become the pool's single `agent_name`; the post-filter
|
| 253 |
+
# handles it. Only an unambiguous single value pushes down.
|
| 254 |
+
if v and ',' not in v:
|
| 255 |
+
agent = v
|
| 256 |
+
return team_id, agent
|
| 257 |
+
|
| 258 |
+
|
| 259 |
+
def _group_dba_team(group):
|
| 260 |
+
"""The team a top-level `or` group pins, or None.
|
| 261 |
+
|
| 262 |
+
Recognises ONLY the exact shape "every child is a `dba eq <brand>` leaf" β the group the
|
| 263 |
+
condition builder emits for a multi-value BU condition, and the one `perm_migrate` writes.
|
| 264 |
+
Every OTHER group returns None and is left entirely to the post-filter: a group mixing `dba`
|
| 265 |
+
with another column, or containing a nested group, does not pin a BU on its own, and
|
| 266 |
+
guessing that it does would build the pool from the wrong book. Narrow by construction β
|
| 267 |
+
the pushdown may only ever be an OPTIMISATION of a constraint the filter already expresses.
|
| 268 |
+
"""
|
| 269 |
+
if group.get('conj') != 'or':
|
| 270 |
+
return None
|
| 271 |
+
children = group.get('children') or []
|
| 272 |
+
if not children:
|
| 273 |
+
return None
|
| 274 |
+
vals = set()
|
| 275 |
+
for c in children:
|
| 276 |
+
if (not isinstance(c, dict) or isinstance(c.get('children'), list)
|
| 277 |
+
or c.get('colId') != 'dba' or c.get('op') != 'eq'):
|
| 278 |
+
return None
|
| 279 |
+
v = str(c.get('value') or '').strip().lower()
|
| 280 |
+
if not v:
|
| 281 |
+
return None
|
| 282 |
+
vals.add(v)
|
| 283 |
+
for allowed, tid in _DBA_TEAM:
|
| 284 |
+
if vals <= allowed:
|
| 285 |
+
return tid
|
| 286 |
+
return None
|
platform/core/perms.py
ADDED
|
@@ -0,0 +1,215 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""core/perms.py β the permission + scope predicates as PURE functions of a user record.
|
| 2 |
+
|
| 3 |
+
EXIT-3b (2026-07-30). `ui/session.py` holds the same predicates keyed off `st.session_state`;
|
| 4 |
+
these take the user record explicitly, so the API can answer "what may this session see?" without
|
| 5 |
+
a Streamlit session. ONE implementation of each rule, two callers β and `verify_api.py` asserts
|
| 6 |
+
lock-step against `ui/session.py`'s source (the [[date-window-vocabulary]] discipline: a mirrored
|
| 7 |
+
vocabulary that is not gated has already drifted).
|
| 8 |
+
|
| 9 |
+
β THE TRAP THIS FILE EXISTS TO AVOID. `ui/session.brand_team()` reads the sidebar's BU selector
|
| 10 |
+
and defaults to `'All'` β `team_id=None` β CONSOLIDATED. Mirroring it literally on the API would
|
| 11 |
+
be a fail-OPEN: an API session has no selector, so a Royal-only user would be handed Fisch rows β
|
| 12 |
+
the exact leak strict isolation exists to prevent. The scope must be derived from the USER RECORD
|
| 13 |
+
(`bus`), never from a UI default. `scope_team_id()` below is that derivation.
|
| 14 |
+
|
| 15 |
+
Fail-closed everywhere: an unknown module key is denied, an unreadable `bus` narrows rather than
|
| 16 |
+
widens, and a user with no grant sees nothing rather than everything.
|
| 17 |
+
"""
|
| 18 |
+
import core.context as ctxlib
|
| 19 |
+
import core.registry as registry
|
| 20 |
+
import core.users as users
|
| 21 |
+
|
| 22 |
+
# Old module keys migrate ON READ β grants, Library prefs and ?page= deep links keep working
|
| 23 |
+
# without ever editing a stored user record. MUST stay identical to ui/session._LEGACY_KEYS
|
| 24 |
+
# (gated in verify_api.py): 'myday' shipped before the Customer List rename, 'customer_list'
|
| 25 |
+
# before the 2026-07-26 rename to 'customer_data', 'map' joined in wave 8 when the Map PAGE
|
| 26 |
+
# became a VIEW of the customer table. A chain is fine only if every hop resolves to the CURRENT
|
| 27 |
+
# key, so 'myday' points at the END of the chain, not at the middle.
|
| 28 |
+
# β 'customers' is deliberately NOT here. It is a real registry key β the parent of
|
| 29 |
+
# customer_data + cohort β and may_open() gives a parent grant to its children. Mapping it away
|
| 30 |
+
# would silently narrow every "customers" grant to one child.
|
| 31 |
+
_LEGACY_KEYS = {'myday': 'customer_data', 'customer_list': 'customer_data',
|
| 32 |
+
'map': 'customer_data', 'library': 'settings',
|
| 33 |
+
# wave 16 (item 5, R10): Cohort folded into the Customer rail as locked
|
| 34 |
+
# views β a cohort grant or ?page=cohort deep link lands on the grid that
|
| 35 |
+
# now hosts them.
|
| 36 |
+
'cohort': 'customer_data',
|
| 37 |
+
# wave 16 (item 7, R11): the Agents PAGE is deleted; an 'agent'-granted
|
| 38 |
+
# login's surviving surface is the Customer grid over their OWN m2m book β
|
| 39 |
+
# the legacy query-scope (scope_agent reads the record's agent LINK, not
|
| 40 |
+
# this list) keeps that book exact, shared customers included. The R1
|
| 41 |
+
# migration deliberately SKIPS agent-linked records: `agent is X` cannot
|
| 42 |
+
# express m2m membership (a shared customer displays its primary agent),
|
| 43 |
+
# so their wall stays legacy until a membership predicate exists.
|
| 44 |
+
'agent': 'customer_data',
|
| 45 |
+
# wave 17 (item 15, R2): the Collections PAGE is gone β its worklist is the
|
| 46 |
+
# shared "Collections" view on the Customer grid. An `ar` grant (or an
|
| 47 |
+
# ?page=ar deep link) lands on the surface that now does the job. The registry
|
| 48 |
+
# row survives as `validate_only`, so this is a grant migration, not a deletion:
|
| 49 |
+
# nobody loses access, they arrive somewhere that still exists.
|
| 50 |
+
'ar': 'customer_data',
|
| 51 |
+
# wave 17 (item 13, R3): Procurement folded into the Product grid β the buy
|
| 52 |
+
# list is a view over formula fields there.
|
| 53 |
+
'procurement': 'product_data'}
|
| 54 |
+
|
| 55 |
+
|
| 56 |
+
def is_admin(user):
|
| 57 |
+
return (user or {}).get('role') == 'admin'
|
| 58 |
+
|
| 59 |
+
|
| 60 |
+
def allowed_modules(user):
|
| 61 |
+
"""Registry keys this user may open β None means ALL (admins / unrestricted users)."""
|
| 62 |
+
m = (user or {}).get('modules', 'all')
|
| 63 |
+
if not m or m == 'all':
|
| 64 |
+
return None
|
| 65 |
+
return {_LEGACY_KEYS.get(k, k) for k in m}
|
| 66 |
+
|
| 67 |
+
|
| 68 |
+
def may_open(user, key):
|
| 69 |
+
"""True iff `user` may OPEN module `key`. A PARENT grant covers its sub-modules (granting
|
| 70 |
+
'customers' grants 'customer_data'). Fail-closed β an unknown key stays denied."""
|
| 71 |
+
am = allowed_modules(user)
|
| 72 |
+
if am is None or key in am:
|
| 73 |
+
return True
|
| 74 |
+
parent = registry.BY_KEY.get(key, {}).get('parent')
|
| 75 |
+
return parent in am if parent else False
|
| 76 |
+
|
| 77 |
+
|
| 78 |
+
# ------------------------------------------------------------------ BU (business unit) scope
|
| 79 |
+
def allowed_bu_labels(user):
|
| 80 |
+
"""The BU labels this user may see β `core.users.allowed_bus_labels`, which is already a
|
| 81 |
+
pure function of the record. Named here so callers have one import for the whole rule set."""
|
| 82 |
+
return users.allowed_bus_labels(user)
|
| 83 |
+
|
| 84 |
+
|
| 85 |
+
def allowed_team_ids(user):
|
| 86 |
+
"""The concrete Odoo team_ids this user may see, as a frozenset. `All` expands to every BU."""
|
| 87 |
+
labels = allowed_bu_labels(user)
|
| 88 |
+
if 'All' in labels:
|
| 89 |
+
return frozenset(t for t in ctxlib.BRANDS.values() if t is not None)
|
| 90 |
+
return frozenset(ctxlib.BRANDS[l] for l in labels
|
| 91 |
+
if l in ctxlib.BRANDS and ctxlib.BRANDS[l] is not None)
|
| 92 |
+
|
| 93 |
+
|
| 94 |
+
def scope_team_id(user):
|
| 95 |
+
"""The `team_id` an API request must query with β the ONE number that enforces BU isolation.
|
| 96 |
+
|
| 97 |
+
None means CONSOLIDATED, and it is returned ONLY when the user may genuinely see every BU
|
| 98 |
+
(`bus='all'`, or a multi-BU grant, both of which put 'All' in the label list). A user
|
| 99 |
+
permitted exactly one BU gets that BU's id PINNED, never None.
|
| 100 |
+
|
| 101 |
+
β This is the opposite default from `ui/session.brand_team()`, which falls back to 'All'
|
| 102 |
+
because a UI selector has to start somewhere. Here there is no selector to read, so the
|
| 103 |
+
fallback would BE the leak. A single-BU user's rows are pinned at the query, not filtered
|
| 104 |
+
afterwards, so there is no window in which the other BU's rows exist in the response.
|
| 105 |
+
"""
|
| 106 |
+
labels = allowed_bu_labels(user)
|
| 107 |
+
if 'All' in labels:
|
| 108 |
+
return None
|
| 109 |
+
ids = sorted(allowed_team_ids(user))
|
| 110 |
+
return ids[0] if len(ids) == 1 else None
|
| 111 |
+
|
| 112 |
+
|
| 113 |
+
def scope_agent(user):
|
| 114 |
+
"""The sales agent whose OWN BOOK this session is confined to, or None for the whole book.
|
| 115 |
+
|
| 116 |
+
β Mirrors `page_customer_data` (app.py), which reads `u.get('agent')` with NO admin
|
| 117 |
+
exemption β deliberately different from `page_agent`'s `None if is_admin() else β¦`. The
|
| 118 |
+
customer TABLE is the surface `/api/v1/customers` serves, so its rule is the one that
|
| 119 |
+
travels: an account linked to an agent sees that agent's book, admin or not. Getting this
|
| 120 |
+
wrong in the permissive direction would hand a restricted sales-agent login the whole book.
|
| 121 |
+
"""
|
| 122 |
+
return (user or {}).get('agent') or None
|
| 123 |
+
|
| 124 |
+
|
| 125 |
+
# ------------------------------------------------------------------ nav projection
|
| 126 |
+
def nav_pages(user):
|
| 127 |
+
"""The nav this user may see: registry order, `may_open`-filtered, archived EXCLUDED.
|
| 128 |
+
|
| 129 |
+
Shape (X2's `{key,label,source?,parent?}` plus ONE additive field, `chrome`):
|
| 130 |
+
[{key, label, source?, chrome: 'main'|'utility'}, β¦]
|
| 131 |
+
|
| 132 |
+
Four rules, each of which has bitten before:
|
| 133 |
+
|
| 134 |
+
* ARCHIVED IS INVISIBLE TO EVERYONE ([[module-archive-semantics]]) β not "hidden unless
|
| 135 |
+
admin". 18 of the 28 registry rows are archived and none may appear here.
|
| 136 |
+
|
| 137 |
+
* `group_only` ROWS ARE EXCLUDED ENTIRELY, and therefore **no `parent` is ever emitted**.
|
| 138 |
+
Wave-9 I8 flipped the host nav to a FLAT list: children became top-level and the folder
|
| 139 |
+
key was dropped, because a `group_only` key names a FAMILY and has no `PAGE_FUNCS` entry
|
| 140 |
+
β rendering a row for it is a white screen one click away (app.py:8182-8187 says exactly
|
| 141 |
+
this). `customers` is the only such row. Emitting `parent: 'customers'` on its children
|
| 142 |
+
while the parent itself is filtered out would hand the client a DANGLING reference to a
|
| 143 |
+
row that is not in the payload, which is worse than a flat list β so the flat list is
|
| 144 |
+
what ships, matching the surface a user actually sees today.
|
| 145 |
+
|
| 146 |
+
* `nav: False` IS NOT `archived`. `settings`, `analyst` and `dictionary` are LIVE surfaces
|
| 147 |
+
the host renders outside the module list (Analyst is the unrestricted landing page,
|
| 148 |
+
Settings sits in the chrome). Dropping them would leave the shell unable to reach two of
|
| 149 |
+
the nine live surfaces, so they ship flagged `chrome: 'utility'` and the client places
|
| 150 |
+
them where the host does. `chrome: 'main'` is the module list.
|
| 151 |
+
|
| 152 |
+
* NOT filtered by `ui/session.user_enabled_modules()`. That is a PREFERENCE read from
|
| 153 |
+
`platform/data/store/prefs.json` β a local file on one box. Consuming it here
|
| 154 |
+
would put per-user FILESYSTEM state inside a process EXIT-4 requires to be stateless,
|
| 155 |
+
and the same user's nav would differ per container. It is a Library setting, not a
|
| 156 |
+
permission; the permission gate is `may_open`, and that is what this applies.
|
| 157 |
+
"""
|
| 158 |
+
out = []
|
| 159 |
+
for m in registry.REGISTRY:
|
| 160 |
+
if m.get('archived') or m.get('group_only'):
|
| 161 |
+
continue
|
| 162 |
+
# WAVE 17 (R2 + R8) β `validate_only` rows have NO SURFACE ANYWHERE. Not the module
|
| 163 |
+
# list, not the utility group behind the account menu, not a page. They exist so
|
| 164 |
+
# `validate.py` keeps proving contracts that a LIVE surface depends on:
|
| 165 |
+
# Β· `ar` β the Collections page is gone (its worklist is a saved view on the
|
| 166 |
+
# Customer grid now), but `ar.validate()` is what reconciles the
|
| 167 |
+
# AR columns that view is built from. Archiving the row would have
|
| 168 |
+
# skipped it (validate.py skips archived), deleting the proof while
|
| 169 |
+
# keeping the numbers β the worst of both.
|
| 170 |
+
# Β· `dictionary` β the semantic-layer contracts, the Analyst's grounding proof.
|
| 171 |
+
# `api_surface: False` is the NARROWER neighbour: the row still has a Streamlit page,
|
| 172 |
+
# but it is not a surface of the PRODUCT. `ar` is the case β its Collections worklist
|
| 173 |
+
# is a saved view on the Customer grid now, while the admin-only Statements sender
|
| 174 |
+
# (the one sanctioned Odoo writer) keeps its host page.
|
| 175 |
+
# β This is NOT `nav: False`. That flag means "a live surface the HOST renders outside
|
| 176 |
+
# the module list" and ships as `chrome: 'utility'`, which is exactly the account-menu
|
| 177 |
+
# row R8 says to delete.
|
| 178 |
+
if m.get('validate_only') or m.get('api_surface') is False:
|
| 179 |
+
continue
|
| 180 |
+
# WAVE 17 R8, the other half β `settings` STOPS being offered to the API's clients.
|
| 181 |
+
# It shipped as `chrome: 'utility'`, which the React shell rendered as "App settings":
|
| 182 |
+
# a hand-off that opens the STREAMLIT settings page in a new tab, beside the shell's
|
| 183 |
+
# own Profile modal that does the same job better. Two doors to one room, one of them
|
| 184 |
+
# leading out of the application. The owner's words were "I don't even know what it
|
| 185 |
+
# does β delete them".
|
| 186 |
+
# β The ROW STAYS, and so does the Streamlit page behind it: this host's own account
|
| 187 |
+
# menu still opens it, and the dispatch guard uses `settings` as its always-renderable
|
| 188 |
+
# last resort for a user whose whole grant has been retired. What changes is that the
|
| 189 |
+
# API no longer advertises it as a surface of the PRODUCT.
|
| 190 |
+
if m['key'] == 'settings':
|
| 191 |
+
continue
|
| 192 |
+
if not may_open(user, m['key']):
|
| 193 |
+
continue
|
| 194 |
+
row = {'key': m['key'], 'label': m['label'],
|
| 195 |
+
'chrome': 'main' if m.get('nav', True) else 'utility'}
|
| 196 |
+
if m.get('source'):
|
| 197 |
+
row['source'] = m['source']
|
| 198 |
+
out.append(row)
|
| 199 |
+
return out
|
| 200 |
+
|
| 201 |
+
|
| 202 |
+
def landing_page(user):
|
| 203 |
+
"""Where a restricted user lands: the first MAIN-chrome page in their grant. None for
|
| 204 |
+
unrestricted users (the client lands on the Analyst, as the host does).
|
| 205 |
+
|
| 206 |
+
Mirrors `ui/session.first_allowed_module` minus its archived last-resort leg β that leg
|
| 207 |
+
exists so a misconfigured Streamlit login is not bricked; an API caller gets an explicit
|
| 208 |
+
answer instead, and serving an archived key would contradict the rule above.
|
| 209 |
+
"""
|
| 210 |
+
if allowed_modules(user) is None:
|
| 211 |
+
return None
|
| 212 |
+
for row in nav_pages(user):
|
| 213 |
+
if row['chrome'] == 'main':
|
| 214 |
+
return row['key']
|
| 215 |
+
return None
|
platform/core/record_comments.py
ADDED
|
@@ -0,0 +1,155 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Durable comments attached to customer records.
|
| 2 |
+
|
| 3 |
+
The service is deliberately unaware of HTTP and of the concrete store backend.
|
| 4 |
+
It receives a tenant-bound ``TenantRuntime`` handle, so the same logical key can
|
| 5 |
+
never cross a tenant boundary. Odoo remains read-only: comments live only in
|
| 6 |
+
AIOS's writable tenant store.
|
| 7 |
+
"""
|
| 8 |
+
from __future__ import annotations
|
| 9 |
+
|
| 10 |
+
import datetime as dt
|
| 11 |
+
import uuid
|
| 12 |
+
|
| 13 |
+
STORE_KEY = "customer_record_comments"
|
| 14 |
+
MAX_BODY = 4000
|
| 15 |
+
MAX_PER_RECORD = 500
|
| 16 |
+
|
| 17 |
+
|
| 18 |
+
class CommentsUnavailable(RuntimeError):
|
| 19 |
+
"""The tenant store cannot currently provide durable comments."""
|
| 20 |
+
|
| 21 |
+
|
| 22 |
+
class CommentForbidden(RuntimeError):
|
| 23 |
+
"""The caller tried to remove another user's comment."""
|
| 24 |
+
|
| 25 |
+
|
| 26 |
+
def _now_iso():
|
| 27 |
+
return dt.datetime.now(dt.timezone.utc).replace(microsecond=0).isoformat().replace("+00:00", "Z")
|
| 28 |
+
|
| 29 |
+
|
| 30 |
+
def _record_key(pid):
|
| 31 |
+
return str(int(pid))
|
| 32 |
+
|
| 33 |
+
|
| 34 |
+
def _clean_comment(raw):
|
| 35 |
+
if not isinstance(raw, dict):
|
| 36 |
+
return None
|
| 37 |
+
cid = str(raw.get("id") or "").strip()
|
| 38 |
+
body = str(raw.get("body") or "").strip()
|
| 39 |
+
author_key = str(raw.get("authorKey") or "").strip().lower()
|
| 40 |
+
author = str(raw.get("author") or author_key).strip()
|
| 41 |
+
created_at = str(raw.get("createdAt") or "").strip()
|
| 42 |
+
if not cid or not body or not author_key or not created_at:
|
| 43 |
+
return None
|
| 44 |
+
return {
|
| 45 |
+
"id": cid,
|
| 46 |
+
"body": body[:MAX_BODY],
|
| 47 |
+
"authorKey": author_key,
|
| 48 |
+
"author": author[:120] or author_key,
|
| 49 |
+
"createdAt": created_at,
|
| 50 |
+
}
|
| 51 |
+
|
| 52 |
+
|
| 53 |
+
def _read(runtime):
|
| 54 |
+
if not runtime.available():
|
| 55 |
+
raise CommentsUnavailable("the tenant store is unavailable")
|
| 56 |
+
try:
|
| 57 |
+
data = runtime.get(STORE_KEY)
|
| 58 |
+
except Exception as exc:
|
| 59 |
+
raise CommentsUnavailable("comments could not be read") from exc
|
| 60 |
+
return data if isinstance(data, dict) else {}
|
| 61 |
+
|
| 62 |
+
|
| 63 |
+
def list_comments(runtime, pid):
|
| 64 |
+
"""Return one record's comments oldest-first, dropping malformed legacy entries."""
|
| 65 |
+
rows = _read(runtime).get(_record_key(pid), [])
|
| 66 |
+
if not isinstance(rows, list):
|
| 67 |
+
return []
|
| 68 |
+
cleaned = [comment for raw in rows if (comment := _clean_comment(raw)) is not None]
|
| 69 |
+
cleaned.sort(key=lambda comment: (comment["createdAt"], comment["id"]))
|
| 70 |
+
return cleaned
|
| 71 |
+
|
| 72 |
+
|
| 73 |
+
def add_comment(runtime, pid, body, author_key, author_name="", now=None, comment_id=None):
|
| 74 |
+
"""Append one server-authored comment atomically and synchronously."""
|
| 75 |
+
text = str(body or "").strip()
|
| 76 |
+
if not text:
|
| 77 |
+
raise ValueError("write a comment before posting")
|
| 78 |
+
if len(text) > MAX_BODY:
|
| 79 |
+
raise ValueError(f"comments may be at most {MAX_BODY} characters")
|
| 80 |
+
uname = str(author_key or "").strip().lower()
|
| 81 |
+
if not uname:
|
| 82 |
+
raise ValueError("a signed-in author is required")
|
| 83 |
+
comment = {
|
| 84 |
+
"id": str(comment_id or uuid.uuid4().hex),
|
| 85 |
+
"body": text,
|
| 86 |
+
"authorKey": uname,
|
| 87 |
+
"author": str(author_name or uname).strip()[:120] or uname,
|
| 88 |
+
"createdAt": str(now or _now_iso()),
|
| 89 |
+
}
|
| 90 |
+
if not runtime.available():
|
| 91 |
+
raise CommentsUnavailable("the tenant store is unavailable")
|
| 92 |
+
|
| 93 |
+
def _append(data):
|
| 94 |
+
if not isinstance(data, dict):
|
| 95 |
+
data = {}
|
| 96 |
+
key = _record_key(pid)
|
| 97 |
+
rows = data.get(key)
|
| 98 |
+
if not isinstance(rows, list):
|
| 99 |
+
rows = []
|
| 100 |
+
if len(rows) >= MAX_PER_RECORD:
|
| 101 |
+
raise ValueError(f"a record may hold at most {MAX_PER_RECORD} comments")
|
| 102 |
+
rows.append(comment)
|
| 103 |
+
data[key] = rows
|
| 104 |
+
return data
|
| 105 |
+
|
| 106 |
+
try:
|
| 107 |
+
runtime.update(STORE_KEY, _append, flush="sync")
|
| 108 |
+
except ValueError:
|
| 109 |
+
raise
|
| 110 |
+
except Exception as exc:
|
| 111 |
+
raise CommentsUnavailable("the comment was not saved") from exc
|
| 112 |
+
return dict(comment)
|
| 113 |
+
|
| 114 |
+
|
| 115 |
+
def delete_comment(runtime, pid, comment_id, actor_key, admin=False):
|
| 116 |
+
"""Delete a comment owned by ``actor_key`` (or by an administrator)."""
|
| 117 |
+
cid = str(comment_id or "").strip()
|
| 118 |
+
actor = str(actor_key or "").strip().lower()
|
| 119 |
+
if not cid:
|
| 120 |
+
return False
|
| 121 |
+
if not runtime.available():
|
| 122 |
+
raise CommentsUnavailable("the tenant store is unavailable")
|
| 123 |
+
state = {"found": False, "forbidden": False}
|
| 124 |
+
|
| 125 |
+
def _delete(data):
|
| 126 |
+
if not isinstance(data, dict):
|
| 127 |
+
return {}
|
| 128 |
+
key = _record_key(pid)
|
| 129 |
+
rows = data.get(key)
|
| 130 |
+
if not isinstance(rows, list):
|
| 131 |
+
return data
|
| 132 |
+
kept = []
|
| 133 |
+
for raw in rows:
|
| 134 |
+
comment = _clean_comment(raw)
|
| 135 |
+
if comment is None or comment["id"] != cid:
|
| 136 |
+
kept.append(raw)
|
| 137 |
+
continue
|
| 138 |
+
state["found"] = True
|
| 139 |
+
if not admin and comment["authorKey"] != actor:
|
| 140 |
+
state["forbidden"] = True
|
| 141 |
+
kept.append(raw)
|
| 142 |
+
if kept:
|
| 143 |
+
data[key] = kept
|
| 144 |
+
else:
|
| 145 |
+
data.pop(key, None)
|
| 146 |
+
return data
|
| 147 |
+
|
| 148 |
+
try:
|
| 149 |
+
runtime.update(STORE_KEY, _delete, flush="sync")
|
| 150 |
+
except Exception as exc:
|
| 151 |
+
raise CommentsUnavailable("the comment was not deleted") from exc
|
| 152 |
+
if state["forbidden"]:
|
| 153 |
+
raise CommentForbidden("only the author or an administrator may delete this comment")
|
| 154 |
+
return bool(state["found"])
|
| 155 |
+
|
platform/core/registry.py
ADDED
|
@@ -0,0 +1,199 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Module registry β the single source of truth for the dashboards.
|
| 2 |
+
|
| 3 |
+
`app.py` builds the sidebar nav from this, and decides whether the global Brand (DBA) filter
|
| 4 |
+
applies to a page. `validate.py` runs each module's validate() from this. Adding a module =
|
| 5 |
+
add one row here + a page function in app.py + a module in modules/.
|
| 6 |
+
|
| 7 |
+
Fields:
|
| 8 |
+
key stable id (matches the modules/<key>.py file and the app page_<key> function)
|
| 9 |
+
label sidebar/title text (no emojis)
|
| 10 |
+
brand True -> DBA-filterable (honours the Fisch/Royal selector)
|
| 11 |
+
False -> consolidated/company-level (ignores the brand selector)
|
| 12 |
+
hq True -> HQ / company-level (cash flow, balance sheet, AR/AP, consolidated)
|
| 13 |
+
validate True -> module exposes validate() and is included in validate.py
|
| 14 |
+
parent set -> this row is a SUB-MODULE of <parent key>: it leaves the Workflows dropdown and
|
| 15 |
+
renders as a sub-nav button when its parent (or a sibling) is the active module; a
|
| 16 |
+
parent grant covers its sub-modules (owner nav redesign 2026-07-23)
|
| 17 |
+
archived True -> RETIRED from the UI for EVERYONE (owner 2026-07-27, supersedes the
|
| 18 |
+
2026-07-23 per-user-restore semantics): no nav entry, no Settings>Modules checkbox,
|
| 19 |
+
prewarm and validate.py skip it. Code and data layer stay in the repo but are NOT
|
| 20 |
+
maintained β spend no effort on archived modules until the owner un-archives.
|
| 21 |
+
2026-07-23 pass: backorders, pricecomp, o2c, bookings, spend, expenses.
|
| 22 |
+
2026-07-27 pass (keep = Sales + Customers family + Agents + Procurement +
|
| 23 |
+
Collections + Analyst/plumbing): products, assortment, financial, management,
|
| 24 |
+
pricing, inventory, vendor, warehouse, returns, outreach, complexity, health.
|
| 25 |
+
note one-line scope note shown in the UI / docs
|
| 26 |
+
"""
|
| 27 |
+
# WAVE-9 I8 β `'source'`: which connected system a module's data comes from.
|
| 28 |
+
#
|
| 29 |
+
# The owner's model: every nav entry is "a piece of database from a source", so the nav shows the
|
| 30 |
+
# source beside the name ("Sales Β· Odoo"). It is a REGISTRY FACT rather than a string in the nav
|
| 31 |
+
# code precisely so the nav needs no edit when a second connector lands β a user-created blank
|
| 32 |
+
# table simply carries a different source, and a module with no `source` (Metric Dictionary,
|
| 33 |
+
# Settings, the Analyst) renders no badge because it is not a database at all.
|
| 34 |
+
# See [[connector-onboarding]]: the plug-in seam is datastore.py, and Odoo has no delegated
|
| 35 |
+
# OAuth, so the "+ New" connect flow is the credential-form shape, not an OAuth redirect.
|
| 36 |
+
REGISTRY = [
|
| 37 |
+
# ARCHIVED wave 16 (owner item 7, R3/R11, 2026-08-02): the PAGE went on the strength of
|
| 38 |
+
# C-CHARTCAP β every Sales block shape (YoY compare series, KPI delta tiles, group-by
|
| 39 |
+
# tables) is now assemblable BY HAND from the grid's chart/dashboard views, which is what
|
| 40 |
+
# R3 required before deletion. `modules/sales.py` (the data layer) STAYS β drawers and the
|
| 41 |
+
# briefing still read it, and pages_sales.py survives UNREGISTERED as the Y1 envelope's
|
| 42 |
+
# template + verify_api's fixture.
|
| 43 |
+
{'key': 'sales', 'label': 'Sales', 'brand': True, 'hq': False, 'validate': True, 'source': 'Odoo', 'archived': True,
|
| 44 |
+
'note': 'Revenue, YoY, seasonality, reps, customers, SKUs β by BU. Retired wave 16: '
|
| 45 |
+
'rebuild as grid chart/dashboard views (compare series, KPI deltas, tables).'},
|
| 46 |
+
# ARCHIVED AS A PAGE (owner item 14, wave 8): "archive the current form of the Customer
|
| 47 |
+
# dashboard completely, i want to redesign it, exactly with the backend we have". So the
|
| 48 |
+
# PAGE goes and the DATA LAYER stays β modules/customers.py still powers the Customer table's
|
| 49 |
+
# metrics and still validates against Odoo, which is what the redesign will be built on.
|
| 50 |
+
#
|
| 51 |
+
# β NOT 'archived': True. Archived means invisible to everyone, and this row is the parent of
|
| 52 |
+
# customer_data + cohort β archiving it would take the whole family out of the nav and strand
|
| 53 |
+
# its children. 'group_only' says exactly what is true: this key names a FOLDER, never a
|
| 54 |
+
# destination. It has no PAGE_FUNCS entry, the nav never renders it as a leaf, and a stale
|
| 55 |
+
# deep link to it redirects to the family's first visible member.
|
| 56 |
+
{'key': 'customers', 'label': 'Customers', 'brand': True, 'hq': False, 'validate': True,
|
| 57 |
+
'group_only': True,
|
| 58 |
+
'note': 'Folder for the customer surfaces (Customer table, Cohort). The old Customers '
|
| 59 |
+
'dashboard was retired for redesign (owner item 14, 2026-07-28); its metric layer '
|
| 60 |
+
'(modules/customers.py) still backs the table and still validates against Odoo.'},
|
| 61 |
+
# LABEL renamed 'Customer List' -> 'Data' (owner 2026-07-25) -> 'Customer' (owner item 5,
|
| 62 |
+
# 2026-07-27); KEY renamed 'customer_list' -> 'customer_data' (owner 2026-07-26). The key
|
| 63 |
+
# rename is safe ONLY because _LEGACY_KEYS (ui/session.py) migrates grants, Library prefs
|
| 64 |
+
# and ?page= deep links ON READ β stored user records are never edited, exactly as
|
| 65 |
+
# myday->customer_list was done. The STORE keys are deliberately NOT renamed:
|
| 66 |
+
# 'customer_table_workspace' and 'customer_lists' hold real user data, and renaming them
|
| 67 |
+
# would orphan every saved view. Label-only changes need none of that machinery.
|
| 68 |
+
{'key': 'customer_data', 'label': 'Customer', 'brand': True, 'hq': False, 'validate': False, 'source': 'Odoo', 'parent': 'customers',
|
| 69 |
+
'note': 'Build, filter and save customer lists: the whole scoped book with per-customer '
|
| 70 |
+
'metrics; saved lists are visible, editable formulas (Call list and Win-back ship '
|
| 71 |
+
'as templates) plus hand-picked members. Sub-module of Customers (owner nav redesign 2026-07-23; replaced My Day).'},
|
| 72 |
+
# Cohort (owner item 6, 2026-07-26): the SAME table as Data over a FIXED set. The
|
| 73 |
+
# difference is membership semantics, not layout β a saved view re-populates as the
|
| 74 |
+
# data moves, a cohort only changes when a person edits it. validate:False because
|
| 75 |
+
# there is no Odoo aggregate to reconcile a hand-picked list against; its metrics are
|
| 76 |
+
# the customers module's, already validated there.
|
| 77 |
+
# ARCHIVED wave 16 (owner item 5, R10 β the fold-in): every cohort now projects as a
|
| 78 |
+
# LOCKED VIEW in the Customer rail's "Cohorts" section (wave-15 C-LOCK), so the separate
|
| 79 |
+
# nav destination is gone. ALL cohort machinery stays: `scope=cohort` still works
|
| 80 |
+
# (routes_grid._SCOPES, storage keys, events), page_cohort remains in app.py, and
|
| 81 |
+
# _LEGACY_KEYS maps cohortβcustomer_data so grants + ?page= deep links land on the grid
|
| 82 |
+
# that now hosts the cohorts.
|
| 83 |
+
{'key': 'cohort', 'label': 'Cohort', 'brand': True, 'hq': False, 'validate': False, 'source': 'Odoo', 'parent': 'customers', 'archived': True,
|
| 84 |
+
'note': 'Hand-curated, unchanging customer lists β folded into the Customer rail as '
|
| 85 |
+
'locked views (wave 16). The set is fixed: it changes only when someone adds '
|
| 86 |
+
'or removes a member. Open them from Customer > Views > Cohorts.'},
|
| 87 |
+
# ARCHIVED wave 16 (owner item 7, R11, 2026-08-02) beside Sales. The agent DRAWER and the
|
| 88 |
+
# data layer (modules/agent.py, agent_* caches) STAY β agent entities still open from
|
| 89 |
+
# customer surfaces; an own-book agent login's home is the Customer grid narrowed by the
|
| 90 |
+
# R1 permanent filter ("Agent is X"), not this page.
|
| 91 |
+
{'key': 'agent', 'label': 'Agents', 'brand': True, 'hq': False, 'validate': True, 'source': 'Odoo', 'archived': True,
|
| 92 |
+
'note': 'Per-agent book: sales (custom dates), returns, top SKUs, full customer list incl. inactive. '
|
| 93 |
+
'Retired wave 16: agent drawers + the R1 own-book filter carry the use case.'},
|
| 94 |
+
# Wave 16 C-TOPIC (owner items 9+10, R4): the PRODUCT table family β the second object on
|
| 95 |
+
# the table-page factory. `products_family` is the folder head (the `customers` pattern:
|
| 96 |
+
# group_only, never a destination); `product_data` is the grid over the SKU dataset, with
|
| 97 |
+
# its OWN workspace bucket ('product_table_workspace') and the /products + scope=product
|
| 98 |
+
# seams. The archived Streamlit `products` (SKU) row below stays archived and untouched,
|
| 99 |
+
# exactly as R4 rules.
|
| 100 |
+
{'key': 'products_family', 'label': 'Products', 'brand': True, 'hq': False, 'validate': False,
|
| 101 |
+
'group_only': True,
|
| 102 |
+
'note': 'Folder for the product surfaces (Product table). The archived SKU dashboard is '
|
| 103 |
+
'not part of this family.'},
|
| 104 |
+
{'key': 'product_data', 'label': 'Product', 'brand': True, 'hq': False, 'validate': True, 'source': 'Odoo', 'parent': 'products_family',
|
| 105 |
+
'note': 'The SKU catalogue as a grid: per-product revenue, units and (consolidated) '
|
| 106 |
+
'stock columns, with saved views and custom fields. Identity = the SKU code.'},
|
| 107 |
+
{'key': 'products', 'label': 'SKU', 'brand': True, 'hq': False, 'validate': True, 'archived': True,
|
| 108 |
+
'note': 'SKU movers, zombies, velocity, coverage, drawers β by BU (sales-derived).'},
|
| 109 |
+
{'key': 'assortment', 'label': 'Assortment', 'brand': True, 'hq': False, 'validate': True, 'archived': True,
|
| 110 |
+
'note': 'Facet-level performance (category/color/occasion/collection/season) + season readiness β by BU.'},
|
| 111 |
+
{'key': 'financial', 'label': 'Financial', 'brand': True, 'hq': False, 'validate': True, 'archived': True,
|
| 112 |
+
'note': 'Gross margin by BU/category/SKU. Cash-conversion cycle is HQ-consolidated.'},
|
| 113 |
+
{'key': 'pricing', 'label': 'Pricing', 'brand': True, 'hq': False, 'validate': True, 'archived': True,
|
| 114 |
+
'note': 'Per-SKU margin/markup/LTM sales + allocated net P&L (channel-rate cost-to-SKU) for pricing decisions.'},
|
| 115 |
+
{'key': 'management', 'label': 'Management P&L', 'brand': False, 'hq': True, 'validate': True, 'archived': True,
|
| 116 |
+
'nav': False, # rendered INSIDE the Financial module (view toggle) β archived WITH financial
|
| 117 |
+
'note': 'Operating-model management income statements (FFS/RI/GD/BU-M) reproduced from raw Odoo, Sep-2024+.'},
|
| 118 |
+
{'key': 'inventory', 'label': 'Inventory', 'brand': False, 'hq': False, 'validate': True, 'archived': True,
|
| 119 |
+
'note': 'On-hand stock is physically consolidated (shared warehouse); not DBA-split.'},
|
| 120 |
+
{'key': 'procurement', 'label': 'Procurement', 'brand': False, 'hq': False, 'validate': True, 'source': 'Odoo',
|
| 121 |
+
'validate_only': True,
|
| 122 |
+
'note': 'PAGE RETIRED 2026-08-03 (owner wave-17 item 13: "We should be able to replace '
|
| 123 |
+
'Procurement completely, and add it as part of the Product database"). The buy '
|
| 124 |
+
'list is now a saved VIEW on the Product grid, filtered on a FORMULA field over '
|
| 125 |
+
'the supplier/lead-time columns and the demand measure β the owner\'s ruling R3: '
|
| 126 |
+
'"Buy list is just a View, with a Filter from a Formula field that taps into '
|
| 127 |
+
'Metrics Fields... the 8-month demand baseline etc. is just math in a Formula '
|
| 128 |
+
'field." The ROW STAYS validate_only: `modules/procurement.py` still owns the '
|
| 129 |
+
'supplier master map (procurement_suppliers.json, 3,963 SKUs) that seeds those '
|
| 130 |
+
'columns, and its validate() is the proof they reconcile.'},
|
| 131 |
+
{'key': 'vendor', 'label': 'Vendors', 'brand': False, 'hq': False, 'validate': False, 'archived': True,
|
| 132 |
+
'note': 'Vendor master data + the SKUs each supplies; multiple-vendor / cheapest-price view.'},
|
| 133 |
+
{'key': 'warehouse', 'label': 'Warehouse', 'brand': False, 'hq': False, 'validate': True, 'archived': True,
|
| 134 |
+
'note': 'Movement & efficiency: on-time ship, Pick-Pack-Ship cycle times, throughput, backlog age, shrinkage. Physical ops (consolidated).'},
|
| 135 |
+
{'key': 'ar', 'label': 'Collections', 'brand': False, 'hq': True, 'validate': True,
|
| 136 |
+
'source': 'Odoo', 'api_surface': False, 'admin_only': True,
|
| 137 |
+
'note': 'PAGE RETIRED 2026-08-03 (owner wave-17 item 15: "Collections should also be '
|
| 138 |
+
'entirely replicable as just a View under Customer"). The worklist is the shared '
|
| 139 |
+
'"Collections" view on the Customer grid β same numbers, from this module\'s own '
|
| 140 |
+
'reconciled blocks (ar_open/ar_overdue/ar_exposure/days_to_pay + the four aging '
|
| 141 |
+
'buckets). The ROW STAYS validate_only so validate.py keeps running ar.validate(), '
|
| 142 |
+
'which is the proof those columns rest on; archiving it would have skipped the '
|
| 143 |
+
'reconciliation while the numbers kept shipping. β NOT `validate_only`, and the '
|
| 144 |
+
'difference is a LIVE WORKFLOW: this page also mounts the admin-gated STATEMENTS '
|
| 145 |
+
'sender (app._collections_statements -> modules/collections_send), the ONE '
|
| 146 |
+
'sanctioned Odoo writer, which the same ruling says stays untouched. So the row '
|
| 147 |
+
'keeps a Streamlit page for ADMINS ONLY (`admin_only`) and leaves the API payload '
|
| 148 |
+
'entirely (`api_surface: False`) β no "Collections" in the React nav, no second '
|
| 149 |
+
'worklist, and the biweekly send keeps its door.'},
|
| 150 |
+
{'key': 'returns', 'label': 'Returns', 'brand': False, 'hq': True, 'validate': True, 'archived': True,
|
| 151 |
+
'note': 'Credit-note lens: refund concentration by SKU (quality) and customer (behavior). Company-level.'},
|
| 152 |
+
{'key': 'outreach', 'label': 'Outreach', 'brand': True, 'hq': False, 'validate': False, 'archived': True,
|
| 153 |
+
'note': 'Campaign emails to customer segments: templates, suppression, send log, revenue attribution. Sending is admin-gated behind SAFE_MODE.'},
|
| 154 |
+
{'key': 'backorders', 'label': 'Backorders', 'brand': True, 'hq': False, 'validate': True, 'archived': True,
|
| 155 |
+
'note': 'Confirmed-undelivered order lines aged vs promise date, valued, with a supply-aware next action per row (ship / expedite / call). Wholesale scope.'},
|
| 156 |
+
{'key': 'pricecomp', 'label': 'Price Compliance', 'brand': True, 'hq': False, 'validate': True, 'archived': True,
|
| 157 |
+
'note': 'Selling below the customer pricelist tier (LTM, per customer x SKU): the pocket-price floor worklist with annualized leak $. Sub-30% ratios usually mean a stale or pack-basis RULE β fix the rule, not the rep.'},
|
| 158 |
+
{'key': 'o2c', 'label': 'Cash Timing', 'brand': True, 'hq': False, 'validate': True, 'archived': True,
|
| 159 |
+
'note': 'Order-to-cash stage decomposition (order-to-ship / ship-to-invoice / invoice-to-paid, each with its owner) + the terms-gap rollup: contractual vs actual days per payment term with the free-credit $ it strands.'},
|
| 160 |
+
{'key': 'bookings', 'label': 'Order Book', 'brand': True, 'hq': False, 'validate': True, 'archived': True,
|
| 161 |
+
'note': 'Pre-season booking coverage: cumulative booked $ for Aug-Dec delivery vs last year same-week; category TY-vs-LY-as-of-date. The container-program decision, months ahead.'},
|
| 162 |
+
{'key': 'spend', 'label': 'Spend & Payables', 'brand': False, 'hq': True, 'validate': True, 'archived': True,
|
| 163 |
+
'note': 'Vendor-bill side: payment-terms capture (early-pay cash surrendered), duplicate-bill review, the opex spend cube (fragmentation, consolidate+rebid) and freight recovery. Company-level.'},
|
| 164 |
+
{'key': 'complexity', 'label': 'SKU Complexity', 'brand': False, 'hq': False, 'validate': True, 'archived': True,
|
| 165 |
+
'note': 'BCG tail rationalization: every SKU re-costed by ACTIVITY (lines, picks, returns) + 25%/yr carrying; KILL candidates rest on hard math (GM minus carrying), REVIEW on the pooled activity estimate. All-channel.'},
|
| 166 |
+
{'key': 'expenses', 'label': 'Expenses', 'brand': False, 'hq': True, 'validate': True, 'archived': True,
|
| 167 |
+
'note': 'Operating expense from the GL (all expense-type accounts; COGS excluded): trend, operating leverage (opex % of revenue), the YoY cost bridge, XmR control-limit spike watch list, fixed/variable split and a drill-to-ledger category directory. Company-level.'},
|
| 168 |
+
{'key': 'health', 'label': 'Data Health', 'brand': False, 'hq': True, 'validate': True, 'archived': True,
|
| 169 |
+
'note': 'Close/reconciliation scan; period-filtered, mostly company-level. ARCHITECTURE '
|
| 170 |
+
'(owner 2026-07-23): every discrepancy / potential-error marker lives here β '
|
| 171 |
+
'procurement mapping gaps, count-trust (unverified on-hand counts), untracked-on-'
|
| 172 |
+
'order β so operational workflows stay clean for doing the work.'},
|
| 173 |
+
{'key': 'dictionary', 'label': 'Metric Dictionary', 'brand': False, 'hq': True, 'validate': True,
|
| 174 |
+
'nav': False, 'validate_only': True,
|
| 175 |
+
'note': 'PAGE RETIRED 2026-07-23 (owner: broken/not customer-facing) β row kept ONLY so '
|
| 176 |
+
'validate.py keeps running the semantic-layer contracts (the Analyst grounding '
|
| 177 |
+
'proof). No nav, no page. Wave 17 R8 ("I don\'t even know what it does β delete '
|
| 178 |
+
'them") made that literal: `validate_only` takes it out of the account menu too, '
|
| 179 |
+
'which is the last place it was still visible. The proof survives; the door does not.'},
|
| 180 |
+
{'key': 'automation', 'label': 'Automation', 'brand': False, 'hq': True, 'validate': False,
|
| 181 |
+
'nav': True,
|
| 182 |
+
'note': 'Wave 18 (C-AUTONAV): the Automation surface β scheduled jobs that create and '
|
| 183 |
+
'refresh user databases (website scrape-to-DB, the Instagram field). React-only '
|
| 184 |
+
'surface (no PAGE_FUNCS entry, the no-new-Streamlit rule); admins hold it via '
|
| 185 |
+
'"all", other users need the explicit grant β fail-closed default.'},
|
| 186 |
+
{'key': 'settings', 'label': 'Settings', 'brand': False, 'hq': True, 'validate': False,
|
| 187 |
+
'nav': False, # sidebar sentinel (account group) β owner IA 2026-07-12
|
| 188 |
+
'note': 'User scope settings: the Business Unit toggle (strict isolation) and the Data basis '
|
| 189 |
+
'toggle (Orders vs Invoiced) moved here from the sidebar.'},
|
| 190 |
+
{'key': 'analyst', 'label': 'AIOS Analyst', 'brand': False, 'hq': True, 'validate': False,
|
| 191 |
+
'nav': False, # sidebar sentinel button; eval gate = harness/evals.py (run pre-ship, NOT in validate.py β live LLM cost)
|
| 192 |
+
'note': 'Ask the business a question in plain language: a small AI model calls governed tools over the semantic layer β answers carry their query trace and drill links. AI-generated output (Art. 50 labeled).'},
|
| 193 |
+
]
|
| 194 |
+
|
| 195 |
+
BY_KEY = {m['key']: m for m in REGISTRY}
|
| 196 |
+
|
| 197 |
+
|
| 198 |
+
def brand_filterable(key) -> bool:
|
| 199 |
+
return bool(BY_KEY.get(key, {}).get('brand'))
|
platform/core/store.py
ADDED
|
@@ -0,0 +1,369 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Tiny JSON persistence on a PRIVATE Hugging Face Dataset repo β the platform's ONLY writable store
|
| 2 |
+
(Odoo stays strictly read-only). Holds the user registry, customer notes and per-user tags.
|
| 3 |
+
|
| 4 |
+
Auth uses HF_TOKEN (a Space secret / .env value with write access to the royal-imports org). The
|
| 5 |
+
repo is created private on first write. Reads use a short in-process cache that writes refresh, so
|
| 6 |
+
within a running instance reads are consistent with writes; a fresh instance re-fetches the latest.
|
| 7 |
+
|
| 8 |
+
WAVE 18 (C1-TENANT, R2): the store is now a CLASS, one instance per repo. The module-level
|
| 9 |
+
functions below delegate to the default instance (`OS_DATA_REPO`, tenant #0) β every existing
|
| 10 |
+
`import core.store as store` caller is byte-for-byte unaffected, which the gate battery proves.
|
| 11 |
+
`for_repo(repo_id)` hands back the bound instance for a tenant that owns its OWN dataset repo
|
| 12 |
+
(nurilab, gtmlab, β¦); `harness.runtime.TenantRuntime` binds it from the tenant record. Each
|
| 13 |
+
instance carries its own cache / dirty-marks / flush workers / commit-gap clock, so one tenant's
|
| 14 |
+
write burst can never starve another's flush window.
|
| 15 |
+
"""
|
| 16 |
+
import io
|
| 17 |
+
import os
|
| 18 |
+
import json
|
| 19 |
+
import threading
|
| 20 |
+
import time as _time
|
| 21 |
+
from pathlib import Path
|
| 22 |
+
|
| 23 |
+
try:
|
| 24 |
+
from dotenv import load_dotenv
|
| 25 |
+
load_dotenv(Path(__file__).resolve().parents[1] / '.env')
|
| 26 |
+
except Exception:
|
| 27 |
+
pass
|
| 28 |
+
|
| 29 |
+
from huggingface_hub import HfApi, hf_hub_download
|
| 30 |
+
from huggingface_hub.utils import EntryNotFoundError, RepositoryNotFoundError
|
| 31 |
+
|
| 32 |
+
REPO = os.environ.get('OS_DATA_REPO', 'royal-imports/cfo-os-data')
|
| 33 |
+
_FLUSH_DELAY = 2.0 # coalesce a burst of edits into one commit
|
| 34 |
+
_FLUSH_MIN_GAP = 20.0 # floor between commits of one key (the 256-commits/hr budget)
|
| 35 |
+
|
| 36 |
+
|
| 37 |
+
def _token():
|
| 38 |
+
return os.environ.get('HF_TOKEN') or None
|
| 39 |
+
|
| 40 |
+
|
| 41 |
+
class Store:
|
| 42 |
+
"""One repo's persistence: JSON-per-key + raw byte paths, with the wave-7 async flush.
|
| 43 |
+
|
| 44 |
+
Every method body is the pre-wave-18 module function, verbatim except globalsβfields.
|
| 45 |
+
The behaviour contracts that matter are stated on each method and are UNCHANGED:
|
| 46 |
+
strict-read-guards-the-write, exists()-is-True-on-uncertainty, lenient display reads.
|
| 47 |
+
"""
|
| 48 |
+
|
| 49 |
+
def __init__(self, repo):
|
| 50 |
+
self.repo = str(repo)
|
| 51 |
+
self._lock = threading.RLock()
|
| 52 |
+
self._cache = {}
|
| 53 |
+
# --- async flush state (wave-7 W3, 2026-07-28) ------------------------------------
|
| 54 |
+
# One typed character in a Notes cell used to cost THREE synchronous network calls
|
| 55 |
+
# inside the component round-trip. update(..., flush='async') applies the mutation to
|
| 56 |
+
# the in-process cache synchronously (read-your-writes: every reader consults the
|
| 57 |
+
# cache first) and a background worker coalesces uploads. The restart-wipe protection
|
| 58 |
+
# is UNCHANGED where it matters: the FIRST update of a key in a fresh process still
|
| 59 |
+
# strict-reads. Known, accepted: with a second concurrent writer the stale window
|
| 60 |
+
# widens from ~one round-trip to ~the flush gap.
|
| 61 |
+
self._owned = {} # name -> True once cache reflects our own last accepted write
|
| 62 |
+
self._dirty = {} # name -> cache is ahead of the hub
|
| 63 |
+
self._flushers = {} # name -> live worker thread
|
| 64 |
+
self._repo_ok = False # create_repo(exist_ok=True) confirmed once per instance
|
| 65 |
+
self._last_flush = {} # name -> monotonic time of the last completed upload
|
| 66 |
+
|
| 67 |
+
def available(self):
|
| 68 |
+
"""True when a token is configured β callers degrade gracefully without one."""
|
| 69 |
+
return bool(_token())
|
| 70 |
+
|
| 71 |
+
def _download(self, name):
|
| 72 |
+
"""Parse {name}.json into a dict. Returns {} ONLY when the file is genuinely absent.
|
| 73 |
+
Any other failure (network, auth, rate-limit, corrupt JSON) RAISES β a transient read
|
| 74 |
+
error must never be mistaken for an empty store, or a caller could overwrite real data
|
| 75 |
+
with an empty dict (this is what wiped the user registry on restart)."""
|
| 76 |
+
try:
|
| 77 |
+
path = hf_hub_download(self.repo, f'{name}.json', repo_type='dataset',
|
| 78 |
+
token=_token(), force_download=True)
|
| 79 |
+
with open(path, encoding='utf-8') as f:
|
| 80 |
+
return json.load(f)
|
| 81 |
+
except (EntryNotFoundError, RepositoryNotFoundError, FileNotFoundError):
|
| 82 |
+
return {}
|
| 83 |
+
except ValueError as e:
|
| 84 |
+
# Wave 18 (R2, found by SESSION D): current huggingface_hub wraps a missing REPO in
|
| 85 |
+
# ValueError("Force download failedβ¦") with the 404 as __cause__ β so a brand-new
|
| 86 |
+
# tenant repo's very first strict read raised here and aborted its very first write.
|
| 87 |
+
# A genuinely-absent repo is the same fact as a genuinely-absent file: {}.
|
| 88 |
+
cause = getattr(e, '__cause__', None)
|
| 89 |
+
if isinstance(cause, (EntryNotFoundError, RepositoryNotFoundError)):
|
| 90 |
+
return {}
|
| 91 |
+
raise
|
| 92 |
+
|
| 93 |
+
def get(self, name, fresh=False):
|
| 94 |
+
"""Load {name}.json as a dict. LENIENT (for display reads): on a transient fetch error,
|
| 95 |
+
hand back the last known-good cached value if we have one, else {}. Mutating callers go
|
| 96 |
+
through update()/_read_strict, which RAISE instead."""
|
| 97 |
+
with self._lock:
|
| 98 |
+
if not fresh and name in self._cache:
|
| 99 |
+
return json.loads(json.dumps(self._cache[name])) # hand back a copy
|
| 100 |
+
try:
|
| 101 |
+
data = self._download(name)
|
| 102 |
+
except Exception as e:
|
| 103 |
+
try: # lazy import β core stays layer-clean
|
| 104 |
+
import harness.telemetry as _tel # noqa: PLC0415
|
| 105 |
+
_tel.error(f'store:get:{name}', e,
|
| 106 |
+
fallback='cached' if name in self._cache else 'empty')
|
| 107 |
+
except Exception:
|
| 108 |
+
pass
|
| 109 |
+
if name in self._cache:
|
| 110 |
+
return json.loads(json.dumps(self._cache[name]))
|
| 111 |
+
return {}
|
| 112 |
+
self._cache[name] = data
|
| 113 |
+
return json.loads(json.dumps(data))
|
| 114 |
+
|
| 115 |
+
def _read_strict(self, name):
|
| 116 |
+
"""Like get(fresh=True) but RAISES on any transient read error ({} only for a genuinely
|
| 117 |
+
absent file), so a failed read ABORTS a read-modify-write instead of merging into {}."""
|
| 118 |
+
with self._lock:
|
| 119 |
+
data = self._download(name)
|
| 120 |
+
self._cache[name] = data
|
| 121 |
+
return json.loads(json.dumps(data))
|
| 122 |
+
|
| 123 |
+
def exists(self, name):
|
| 124 |
+
"""True if {name}.json is confirmed present, False if confirmed absent, and True on ANY
|
| 125 |
+
uncertainty β a seeder must never clobber a registry we merely failed to reach."""
|
| 126 |
+
try:
|
| 127 |
+
files = HfApi(token=_token()).list_repo_files(self.repo, repo_type='dataset')
|
| 128 |
+
return f'{name}.json' in files
|
| 129 |
+
except RepositoryNotFoundError:
|
| 130 |
+
return False
|
| 131 |
+
except Exception:
|
| 132 |
+
return True
|
| 133 |
+
|
| 134 |
+
def _upload(self, name, data):
|
| 135 |
+
"""The raw hub write. create_repo is confirmed ONCE per instance β it was a full extra
|
| 136 |
+
API round-trip on every put(), pure latency after the first success."""
|
| 137 |
+
api = HfApi(token=_token())
|
| 138 |
+
if not self._repo_ok:
|
| 139 |
+
api.create_repo(self.repo, repo_type='dataset', private=True, exist_ok=True)
|
| 140 |
+
self._repo_ok = True
|
| 141 |
+
buf = io.BytesIO(json.dumps(data, ensure_ascii=False, indent=2).encode('utf-8'))
|
| 142 |
+
api.upload_file(path_or_fileobj=buf, path_in_repo=f'{name}.json', repo_id=self.repo,
|
| 143 |
+
repo_type='dataset', commit_message=f'update {name}')
|
| 144 |
+
|
| 145 |
+
def upload_bytes(self, path_in_repo, data, message=None):
|
| 146 |
+
"""Persist RAW BYTES at an arbitrary path in the tenant dataset (wave-8 C5, documents).
|
| 147 |
+
Bytes are never cached in-process: large, cold, fetched on explicit user action."""
|
| 148 |
+
if not self.available():
|
| 149 |
+
raise RuntimeError('No HF_TOKEN configured β persistence is unavailable.')
|
| 150 |
+
with self._lock:
|
| 151 |
+
api = HfApi(token=_token())
|
| 152 |
+
if not self._repo_ok:
|
| 153 |
+
api.create_repo(self.repo, repo_type='dataset', private=True, exist_ok=True)
|
| 154 |
+
self._repo_ok = True
|
| 155 |
+
api.upload_file(path_or_fileobj=io.BytesIO(data), path_in_repo=path_in_repo,
|
| 156 |
+
repo_id=self.repo, repo_type='dataset',
|
| 157 |
+
commit_message=message or f'add {path_in_repo}')
|
| 158 |
+
|
| 159 |
+
def download_bytes(self, path_in_repo):
|
| 160 |
+
"""Raw bytes back, or None when absent/unreachable β a missing attachment degrades to
|
| 161 |
+
"that file is gone", never to a broken page."""
|
| 162 |
+
if not self.available():
|
| 163 |
+
return None
|
| 164 |
+
try:
|
| 165 |
+
local = hf_hub_download(self.repo, path_in_repo, repo_type='dataset',
|
| 166 |
+
token=_token())
|
| 167 |
+
return Path(local).read_bytes()
|
| 168 |
+
except (EntryNotFoundError, RepositoryNotFoundError):
|
| 169 |
+
return None
|
| 170 |
+
except Exception:
|
| 171 |
+
return None
|
| 172 |
+
|
| 173 |
+
def delete_path(self, path_in_repo):
|
| 174 |
+
"""Remove a stored file. Absent is SUCCESS β deleting what is already gone is the goal."""
|
| 175 |
+
if not self.available():
|
| 176 |
+
return False
|
| 177 |
+
try:
|
| 178 |
+
HfApi(token=_token()).delete_file(path_in_repo=path_in_repo, repo_id=self.repo,
|
| 179 |
+
repo_type='dataset',
|
| 180 |
+
commit_message=f'delete {path_in_repo}')
|
| 181 |
+
return True
|
| 182 |
+
except (EntryNotFoundError, RepositoryNotFoundError):
|
| 183 |
+
return True
|
| 184 |
+
except Exception:
|
| 185 |
+
return False
|
| 186 |
+
|
| 187 |
+
def put(self, name, data):
|
| 188 |
+
"""Persist {name}.json (creates the private repo on first write). Updates the cache."""
|
| 189 |
+
if not self.available():
|
| 190 |
+
raise RuntimeError('No HF_TOKEN configured β persistence is unavailable.')
|
| 191 |
+
with self._lock:
|
| 192 |
+
self._upload(name, data)
|
| 193 |
+
self._cache[name] = json.loads(json.dumps(data))
|
| 194 |
+
self._owned[name] = True
|
| 195 |
+
self._last_flush[name] = _time.monotonic()
|
| 196 |
+
return self._cache[name]
|
| 197 |
+
|
| 198 |
+
def _flush_worker(self, name):
|
| 199 |
+
"""Coalescing uploader for one key. Runs while dirty; retries with backoff on failure β
|
| 200 |
+
the cache stays the truth and the dirty mark survives, so nothing is lost silently."""
|
| 201 |
+
backoff = 2.0
|
| 202 |
+
while True:
|
| 203 |
+
_time.sleep(_FLUSH_DELAY)
|
| 204 |
+
with self._lock:
|
| 205 |
+
gap = _time.monotonic() - self._last_flush.get(name, 0.0)
|
| 206 |
+
if gap < _FLUSH_MIN_GAP:
|
| 207 |
+
wait = _FLUSH_MIN_GAP - gap
|
| 208 |
+
elif not self._dirty.get(name):
|
| 209 |
+
self._flushers.pop(name, None)
|
| 210 |
+
return
|
| 211 |
+
else:
|
| 212 |
+
self._dirty[name] = False
|
| 213 |
+
snapshot = json.loads(json.dumps(self._cache.get(name, {})))
|
| 214 |
+
wait = None
|
| 215 |
+
if wait is not None:
|
| 216 |
+
_time.sleep(wait)
|
| 217 |
+
continue
|
| 218 |
+
try:
|
| 219 |
+
self._upload(name, snapshot)
|
| 220 |
+
with self._lock:
|
| 221 |
+
self._last_flush[name] = _time.monotonic()
|
| 222 |
+
backoff = 2.0
|
| 223 |
+
except Exception as e:
|
| 224 |
+
with self._lock:
|
| 225 |
+
self._dirty[name] = True # keep it queued; cache is still the truth
|
| 226 |
+
try:
|
| 227 |
+
import harness.telemetry as _tel # noqa: PLC0415
|
| 228 |
+
_tel.error(f'store:flush:{name}', e)
|
| 229 |
+
except Exception:
|
| 230 |
+
pass
|
| 231 |
+
_time.sleep(backoff)
|
| 232 |
+
backoff = min(backoff * 2, 60.0)
|
| 233 |
+
|
| 234 |
+
def _schedule_flush(self, name):
|
| 235 |
+
"""Mark dirty + ensure exactly one live worker for the key. Caller holds the lock."""
|
| 236 |
+
self._dirty[name] = True
|
| 237 |
+
t = self._flushers.get(name)
|
| 238 |
+
if t is not None and t.is_alive():
|
| 239 |
+
return
|
| 240 |
+
t = threading.Thread(target=self._flush_worker, args=(name,), daemon=True,
|
| 241 |
+
name=f'store-flush-{self.repo}:{name}')
|
| 242 |
+
self._flushers[name] = t
|
| 243 |
+
t.start()
|
| 244 |
+
|
| 245 |
+
def flush(self, name=None, timeout=30.0):
|
| 246 |
+
"""Block until pending async writes for `name` (or ALL keys) reach the hub β QA gates
|
| 247 |
+
and shutdown hooks use this; the app itself never needs to."""
|
| 248 |
+
deadline = _time.monotonic() + timeout
|
| 249 |
+
names = [name] if name else None
|
| 250 |
+
while _time.monotonic() < deadline:
|
| 251 |
+
with self._lock:
|
| 252 |
+
pending = {k for k, v in self._dirty.items() if v} | {
|
| 253 |
+
k for k, t in self._flushers.items() if t.is_alive()}
|
| 254 |
+
if names is not None:
|
| 255 |
+
pending &= set(names)
|
| 256 |
+
if not pending:
|
| 257 |
+
return True
|
| 258 |
+
_time.sleep(0.2)
|
| 259 |
+
return False
|
| 260 |
+
|
| 261 |
+
def _flush_now_at_exit(self):
|
| 262 |
+
"""Process-exit safety net: push any dirty keys synchronously, bypassing the coalescing
|
| 263 |
+
gap β a Space restart must not eat the last seconds of the async window."""
|
| 264 |
+
with self._lock:
|
| 265 |
+
dirty = [k for k, v in self._dirty.items() if v]
|
| 266 |
+
snaps = {k: json.loads(json.dumps(self._cache.get(k, {}))) for k in dirty}
|
| 267 |
+
for k in dirty:
|
| 268 |
+
self._dirty[k] = False
|
| 269 |
+
for k, d in snaps.items():
|
| 270 |
+
try:
|
| 271 |
+
self._upload(k, d)
|
| 272 |
+
except Exception:
|
| 273 |
+
pass # exiting anyway β nothing left to reschedule
|
| 274 |
+
|
| 275 |
+
def update(self, name, fn, flush='sync'):
|
| 276 |
+
"""Read-modify-write. STRICT first read (raises on a transient error rather than
|
| 277 |
+
starting from an empty dict), apply fn(data)->data, persist.
|
| 278 |
+
|
| 279 |
+
flush='sync' (default) = fresh strict read + blocking upload. flush='async' (the
|
| 280 |
+
table-workspace hot path, wave-7 W3) = the mutation applies to the in-process cache and
|
| 281 |
+
returns immediately; a coalescing worker commits. The strict read still guards the
|
| 282 |
+
FIRST async update of a key per process."""
|
| 283 |
+
with self._lock:
|
| 284 |
+
if flush == 'async' and self._owned.get(name) and name in self._cache:
|
| 285 |
+
data = json.loads(json.dumps(self._cache[name]))
|
| 286 |
+
else:
|
| 287 |
+
data = self._read_strict(name)
|
| 288 |
+
result = fn(data)
|
| 289 |
+
data = result if result is not None else data
|
| 290 |
+
if flush == 'async':
|
| 291 |
+
if not self.available():
|
| 292 |
+
raise RuntimeError('No HF_TOKEN configured β persistence is unavailable.')
|
| 293 |
+
self._cache[name] = json.loads(json.dumps(data))
|
| 294 |
+
self._owned[name] = True
|
| 295 |
+
self._schedule_flush(name)
|
| 296 |
+
return data
|
| 297 |
+
self.put(name, data)
|
| 298 |
+
return data
|
| 299 |
+
|
| 300 |
+
|
| 301 |
+
# ------------------------------------------------------------------- instances + module API
|
| 302 |
+
_INSTANCES = {}
|
| 303 |
+
_INSTANCES_LOCK = threading.RLock()
|
| 304 |
+
|
| 305 |
+
|
| 306 |
+
def for_repo(repo_id):
|
| 307 |
+
"""The Store bound to `repo_id` β one instance per repo per process (each with its own
|
| 308 |
+
cache and flush state). The default instance IS `for_repo(REPO)`."""
|
| 309 |
+
rid = str(repo_id or '').strip() or REPO
|
| 310 |
+
with _INSTANCES_LOCK:
|
| 311 |
+
inst = _INSTANCES.get(rid)
|
| 312 |
+
if inst is None:
|
| 313 |
+
inst = Store(rid)
|
| 314 |
+
_INSTANCES[rid] = inst
|
| 315 |
+
return inst
|
| 316 |
+
|
| 317 |
+
|
| 318 |
+
_DEFAULT = for_repo(REPO)
|
| 319 |
+
|
| 320 |
+
|
| 321 |
+
def available():
|
| 322 |
+
return _DEFAULT.available()
|
| 323 |
+
|
| 324 |
+
|
| 325 |
+
def get(name, fresh=False):
|
| 326 |
+
return _DEFAULT.get(name, fresh=fresh)
|
| 327 |
+
|
| 328 |
+
|
| 329 |
+
def _read_strict(name):
|
| 330 |
+
return _DEFAULT._read_strict(name)
|
| 331 |
+
|
| 332 |
+
|
| 333 |
+
def exists(name):
|
| 334 |
+
return _DEFAULT.exists(name)
|
| 335 |
+
|
| 336 |
+
|
| 337 |
+
def upload_bytes(path_in_repo, data, message=None):
|
| 338 |
+
return _DEFAULT.upload_bytes(path_in_repo, data, message=message)
|
| 339 |
+
|
| 340 |
+
|
| 341 |
+
def download_bytes(path_in_repo):
|
| 342 |
+
return _DEFAULT.download_bytes(path_in_repo)
|
| 343 |
+
|
| 344 |
+
|
| 345 |
+
def delete_path(path_in_repo):
|
| 346 |
+
return _DEFAULT.delete_path(path_in_repo)
|
| 347 |
+
|
| 348 |
+
|
| 349 |
+
def put(name, data):
|
| 350 |
+
return _DEFAULT.put(name, data)
|
| 351 |
+
|
| 352 |
+
|
| 353 |
+
def flush(name=None, timeout=30.0):
|
| 354 |
+
return _DEFAULT.flush(name=name, timeout=timeout)
|
| 355 |
+
|
| 356 |
+
|
| 357 |
+
def update(name, fn, flush='sync'):
|
| 358 |
+
return _DEFAULT.update(name, fn, flush=flush)
|
| 359 |
+
|
| 360 |
+
|
| 361 |
+
def _flush_all_at_exit():
|
| 362 |
+
with _INSTANCES_LOCK:
|
| 363 |
+
instances = list(_INSTANCES.values())
|
| 364 |
+
for inst in instances:
|
| 365 |
+
inst._flush_now_at_exit()
|
| 366 |
+
|
| 367 |
+
|
| 368 |
+
import atexit # noqa: E402 (registered after the class it needs)
|
| 369 |
+
atexit.register(_flush_all_at_exit)
|
platform/core/store_backend.py
ADDED
|
@@ -0,0 +1,85 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""core/store_backend.py β which store is active: `STORE_BACKEND=hf|pg` (X4 / EXIT-2b).
|
| 2 |
+
|
| 3 |
+
import core.store_backend as store # new code
|
| 4 |
+
store.get('users') # goes to whichever backend is selected
|
| 5 |
+
|
| 6 |
+
β WHAT THIS DOES **NOT** DO, stated first because assuming otherwise would be a silent data
|
| 7 |
+
split. Flipping `STORE_BACKEND=pg` does NOT redirect the ~40 existing callers that say
|
| 8 |
+
`import core.store as store` β they are bound to the HF module directly. Rewiring them is task
|
| 9 |
+
**C-4** (the HFβPostgres cutover with its dual-read window), and C-4 cannot start until the
|
| 10 |
+
database exists (owner blocker **B-3**). Until then this selector is the SEAM, exercised by new
|
| 11 |
+
code and by the gate, and the default is `hf`, which is exactly today's behaviour.
|
| 12 |
+
|
| 13 |
+
Doing it any other way would be worse. The tempting shortcut β have `core/store.py` itself
|
| 14 |
+
delegate β would mean every existing caller silently changes backend the moment an env var is set
|
| 15 |
+
in some shell, with no dual-read window and no way to compare the two stores' contents first.
|
| 16 |
+
"One store" is the goal; getting there through a flag nobody audited would produce two.
|
| 17 |
+
|
| 18 |
+
THE DEFAULT IS `hf` AND STAYS `hf` until C-4 says otherwise. An unrecognised value is a
|
| 19 |
+
configuration error and RAISES rather than falling back β a typo'd backend name that quietly
|
| 20 |
+
served the old store would be discovered by data going missing.
|
| 21 |
+
"""
|
| 22 |
+
import os
|
| 23 |
+
|
| 24 |
+
import core.store as _hf
|
| 25 |
+
|
| 26 |
+
_NAMES = ('available', 'get', 'exists', 'put', 'update', 'upload_bytes', 'download_bytes',
|
| 27 |
+
'delete_path', 'flush')
|
| 28 |
+
|
| 29 |
+
|
| 30 |
+
def name():
|
| 31 |
+
"""The selected backend name, validated. `hf` unless explicitly told otherwise."""
|
| 32 |
+
raw = (os.environ.get('STORE_BACKEND') or 'hf').strip().lower()
|
| 33 |
+
if raw not in ('hf', 'pg'):
|
| 34 |
+
raise RuntimeError(
|
| 35 |
+
f"STORE_BACKEND={raw!r} is not a backend. Use 'hf' (the HF Dataset store, the "
|
| 36 |
+
f"default) or 'pg' (Postgres, needs DATABASE_URL). Refusing to guess: a typo that "
|
| 37 |
+
f"silently served the other store is how data ends up in two places.")
|
| 38 |
+
return raw
|
| 39 |
+
|
| 40 |
+
|
| 41 |
+
def active():
|
| 42 |
+
"""The backend MODULE. Resolved per call, so a test can flip the env var without a reimport."""
|
| 43 |
+
if name() == 'pg':
|
| 44 |
+
import core.store_pg as _pg
|
| 45 |
+
return _pg
|
| 46 |
+
return _hf
|
| 47 |
+
|
| 48 |
+
|
| 49 |
+
# --- the nine delegates. Written out rather than generated with setattr so that reading this file
|
| 50 |
+
# tells you the whole interface, and so a new store operation cannot be added to core/store.py
|
| 51 |
+
# without something here failing to match (verify_store_pg.py compares the signatures).
|
| 52 |
+
def available():
|
| 53 |
+
return active().available()
|
| 54 |
+
|
| 55 |
+
|
| 56 |
+
def get(name_, fresh=False):
|
| 57 |
+
return active().get(name_, fresh=fresh)
|
| 58 |
+
|
| 59 |
+
|
| 60 |
+
def exists(name_):
|
| 61 |
+
return active().exists(name_)
|
| 62 |
+
|
| 63 |
+
|
| 64 |
+
def put(name_, data):
|
| 65 |
+
return active().put(name_, data)
|
| 66 |
+
|
| 67 |
+
|
| 68 |
+
def update(name_, fn, flush='sync'):
|
| 69 |
+
return active().update(name_, fn, flush=flush)
|
| 70 |
+
|
| 71 |
+
|
| 72 |
+
def upload_bytes(path_in_repo, data, message=None):
|
| 73 |
+
return active().upload_bytes(path_in_repo, data, message=message)
|
| 74 |
+
|
| 75 |
+
|
| 76 |
+
def download_bytes(path_in_repo):
|
| 77 |
+
return active().download_bytes(path_in_repo)
|
| 78 |
+
|
| 79 |
+
|
| 80 |
+
def delete_path(path_in_repo):
|
| 81 |
+
return active().delete_path(path_in_repo)
|
| 82 |
+
|
| 83 |
+
|
| 84 |
+
def flush(name_=None, timeout=30.0):
|
| 85 |
+
return active().flush(name_, timeout=timeout)
|
platform/core/store_pg.py
ADDED
|
@@ -0,0 +1,249 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""core/store_pg.py β `core/store.py`'s interface, backed by Postgres (X4 / EXIT-2b, 2026-07-30).
|
| 2 |
+
|
| 3 |
+
THE INTERFACE IS THE CONTRACT, and it is `core/store.py`'s nine functions:
|
| 4 |
+
|
| 5 |
+
available() Β· get(name, fresh=False) Β· exists(name) Β· put(name, data)
|
| 6 |
+
update(name, fn, flush='sync') Β· upload_bytes(path, data, message=None)
|
| 7 |
+
download_bytes(path) Β· delete_path(path) Β· flush(name=None, timeout=30.0)
|
| 8 |
+
|
| 9 |
+
Same names, same signatures, same return types, same failure semantics β so switching backends is
|
| 10 |
+
an env var (`STORE_BACKEND=hf|pg`) and not a rewrite of every caller. `core/store_backend.py`
|
| 11 |
+
does the selection; nothing above it needs to know which store it is talking to.
|
| 12 |
+
|
| 13 |
+
β NOT VERIFIED AGAINST A REAL SERVER β owner blocker **B-3** (choose + provision managed
|
| 14 |
+
Postgres, US-region, co-located with Ashburn). `verify_store_pg.py` is written and it SKIPS LOUDLY
|
| 15 |
+
without `DATABASE_URL`; it is not reported as passing, and this module is not the default backend.
|
| 16 |
+
The schema is `harness/pg/schema.sql`. Sequenced around the blocker, never stubbed past it.
|
| 17 |
+
|
| 18 |
+
WHAT POSTGRES FIXES, precisely (C1c β these are the reasons, not a preference):
|
| 19 |
+
* **`get()` is cache-first on the HF store**, so a write from another process is invisible to a
|
| 20 |
+
running app until restart. Observed live in wave 11. Here a read is a SELECT: cross-process
|
| 21 |
+
coherence is the default rather than a thing to remember.
|
| 22 |
+
* **`update()` was read-modify-write with no concurrency control** β two writers raced and the
|
| 23 |
+
loser vanished silently. Here it runs inside ONE transaction with `SELECT β¦ FOR UPDATE`, so
|
| 24 |
+
the second writer waits and then sees the first writer's value.
|
| 25 |
+
* **A write was an HTTP commit** against a 256-commits/hour repo budget, which is why the hot
|
| 26 |
+
path needed a coalescing async flusher at all. A Postgres write is a write; `flush='async'`
|
| 27 |
+
stays in the signature and becomes a no-op, because there is nothing to coalesce.
|
| 28 |
+
|
| 29 |
+
β ONE DELIBERATE DIFFERENCE FROM THE HF BACKEND, and it is a fix. `store.get()` swallows transient
|
| 30 |
+
read failures and hands back a cached-or-empty dict, because a display read must not break a page.
|
| 31 |
+
That leniency is exactly what wiped the user registry once (an empty read merged into a write), and
|
| 32 |
+
`_read_strict` exists to opt out of it. Postgres has no equivalent "the network blipped" state
|
| 33 |
+
worth hiding: a connection failure RAISES from `get()` here. Callers that must not break on a read
|
| 34 |
+
should catch it β silently returning `{}` from a database that is simply unreachable is how an
|
| 35 |
+
empty dict gets written back over real data.
|
| 36 |
+
"""
|
| 37 |
+
import json
|
| 38 |
+
import os
|
| 39 |
+
import threading
|
| 40 |
+
|
| 41 |
+
_URL_ENV = 'DATABASE_URL'
|
| 42 |
+
_LOCK = threading.RLock()
|
| 43 |
+
_POOL = {'pool': None, 'url': None, 'schema': None}
|
| 44 |
+
|
| 45 |
+
|
| 46 |
+
def _url():
|
| 47 |
+
return os.environ.get(_URL_ENV) or None
|
| 48 |
+
|
| 49 |
+
|
| 50 |
+
def _schema_for(tenant_slug=None):
|
| 51 |
+
"""`t_<slug>` with `-` normalised to `_` (a hyphen is not legal in an unquoted identifier).
|
| 52 |
+
|
| 53 |
+
Mirrors `control.provision_tenant_schema` exactly β the two MUST agree or a write lands in a
|
| 54 |
+
schema the DDL never created. `AIOS_TENANT` names the tenant this process serves; it defaults
|
| 55 |
+
to tenant #0 so the default behaviour is unchanged, the same rule `harness.datastore.path_for`
|
| 56 |
+
follows for the DuckDB filename.
|
| 57 |
+
"""
|
| 58 |
+
slug = (tenant_slug or os.environ.get('AIOS_TENANT') or 'royal-imports').strip().lower()
|
| 59 |
+
return 't_' + slug.replace('-', '_')
|
| 60 |
+
|
| 61 |
+
|
| 62 |
+
def _pool():
|
| 63 |
+
"""The psycopg connection pool, opened once per process.
|
| 64 |
+
|
| 65 |
+
psycopg is imported LAZILY and only here: `STORE_BACKEND=hf` must not require the dependency
|
| 66 |
+
to be installed at all, which is what keeps this file safe to commit before B-3 unblocks.
|
| 67 |
+
"""
|
| 68 |
+
with _LOCK:
|
| 69 |
+
url = _url()
|
| 70 |
+
if not url:
|
| 71 |
+
raise RuntimeError(
|
| 72 |
+
f'{_URL_ENV} is not set β the Postgres store backend has nothing to connect to.')
|
| 73 |
+
if _POOL['pool'] is not None and _POOL['url'] == url:
|
| 74 |
+
return _POOL['pool']
|
| 75 |
+
try:
|
| 76 |
+
from psycopg_pool import ConnectionPool
|
| 77 |
+
except ImportError as e: # pragma: no cover - depends on B-3
|
| 78 |
+
raise RuntimeError(
|
| 79 |
+
'STORE_BACKEND=pg needs psycopg[binary,pool]. It is deliberately NOT in '
|
| 80 |
+
'requirements.txt until the database exists (owner blocker B-3), so that the '
|
| 81 |
+
'default hf backend installs nothing it does not use.') from e
|
| 82 |
+
if _POOL['pool'] is not None:
|
| 83 |
+
try:
|
| 84 |
+
_POOL['pool'].close()
|
| 85 |
+
except Exception:
|
| 86 |
+
pass
|
| 87 |
+
# min_size=0 so a process that never touches the store opens no connection at all β the
|
| 88 |
+
# shared API serves plenty of requests (health, static, a cached payload) that never read.
|
| 89 |
+
_POOL['pool'] = ConnectionPool(url, min_size=0, max_size=8, open=True, timeout=10.0)
|
| 90 |
+
_POOL['url'] = url
|
| 91 |
+
return _POOL['pool']
|
| 92 |
+
|
| 93 |
+
|
| 94 |
+
def available():
|
| 95 |
+
"""True when a connection can actually be MADE, not merely when a URL is configured.
|
| 96 |
+
|
| 97 |
+
`core.store.available()` only checks for a token, which is all it can cheaply do. Here the
|
| 98 |
+
honest answer needs a round-trip, so the result is memoised per process: callers use this to
|
| 99 |
+
decide whether to degrade the UI, and a `SELECT 1` per render would be absurd.
|
| 100 |
+
"""
|
| 101 |
+
if not _url():
|
| 102 |
+
return False
|
| 103 |
+
if _POOL.get('ok'):
|
| 104 |
+
return True
|
| 105 |
+
try:
|
| 106 |
+
with _pool().connection() as con:
|
| 107 |
+
con.execute('SELECT 1')
|
| 108 |
+
_POOL['ok'] = True
|
| 109 |
+
return True
|
| 110 |
+
except Exception:
|
| 111 |
+
return False
|
| 112 |
+
|
| 113 |
+
|
| 114 |
+
def _table(kind, tenant_slug=None):
|
| 115 |
+
"""`"t_slug"."store_kv"` β the qualified table, safely quoted."""
|
| 116 |
+
from psycopg import sql
|
| 117 |
+
return sql.SQL('{}.{}').format(sql.Identifier(_schema_for(tenant_slug)),
|
| 118 |
+
sql.Identifier(kind))
|
| 119 |
+
|
| 120 |
+
|
| 121 |
+
def _q(template, kind, tenant_slug=None):
|
| 122 |
+
"""Compose `template` with `{tbl}` = the QUALIFIED table and `{bare}` = the table name alone.
|
| 123 |
+
|
| 124 |
+
β THE TWO ARE NOT INTERCHANGEABLE, and getting it wrong is a runtime syntax error nothing
|
| 125 |
+
local would catch (no database here β owner blocker B-3). Inside `ON CONFLICT β¦ DO UPDATE`,
|
| 126 |
+
Postgres refers to the conflicting row by the table's ALIAS, which defaults to the bare
|
| 127 |
+
table name: `store_kv.rev` is valid, `t_royal_imports.store_kv.rev` is not. So the FROM
|
| 128 |
+
position takes `{tbl}` and the DO-UPDATE position takes `{bare}`.
|
| 129 |
+
|
| 130 |
+
Identifiers go through `psycopg.sql.Identifier`, so the schema name β derived from a tenant
|
| 131 |
+
slug β cannot be an injection vector even though it is interpolated. The slug is ALSO
|
| 132 |
+
regex-constrained at its source (`control.tenants`), which is the belt to this brace.
|
| 133 |
+
"""
|
| 134 |
+
from psycopg import sql
|
| 135 |
+
return sql.SQL(template).format(tbl=_table(kind, tenant_slug),
|
| 136 |
+
bare=sql.Identifier(kind))
|
| 137 |
+
|
| 138 |
+
|
| 139 |
+
def get(name, fresh=False, tenant_slug=None):
|
| 140 |
+
"""The value stored under `name`, or `{}` when the key is genuinely absent.
|
| 141 |
+
|
| 142 |
+
`fresh` is accepted and IGNORED: it exists in the HF signature to bypass a process cache, and
|
| 143 |
+
there is no cache here β every read is a SELECT. Keeping the parameter means callers do not
|
| 144 |
+
branch on the backend.
|
| 145 |
+
"""
|
| 146 |
+
with _pool().connection() as con:
|
| 147 |
+
row = con.execute(
|
| 148 |
+
_q('SELECT value FROM {tbl} WHERE key = %s', 'store_kv', tenant_slug),
|
| 149 |
+
(str(name),)).fetchone()
|
| 150 |
+
return dict(row[0]) if row and isinstance(row[0], dict) else (row[0] if row else {})
|
| 151 |
+
|
| 152 |
+
|
| 153 |
+
def exists(name, tenant_slug=None):
|
| 154 |
+
"""True/False β and unlike the HF backend this is NEVER "True on uncertainty".
|
| 155 |
+
|
| 156 |
+
`core.store.exists` returns True on any error so a caller that would seed the store cannot
|
| 157 |
+
clobber a registry it merely failed to reach. That guard exists because a FILE HOST cannot
|
| 158 |
+
distinguish "absent" from "unreachable". Postgres can: an absent row is a fact, and an
|
| 159 |
+
unreachable server raises instead of answering.
|
| 160 |
+
"""
|
| 161 |
+
with _pool().connection() as con:
|
| 162 |
+
row = con.execute(
|
| 163 |
+
_q('SELECT 1 FROM {tbl} WHERE key = %s', 'store_kv', tenant_slug),
|
| 164 |
+
(str(name),)).fetchone()
|
| 165 |
+
return row is not None
|
| 166 |
+
|
| 167 |
+
|
| 168 |
+
def put(name, data, tenant_slug=None):
|
| 169 |
+
with _pool().connection() as con:
|
| 170 |
+
con.execute(
|
| 171 |
+
_q('INSERT INTO {tbl} (key, value) VALUES (%s, %s::jsonb) '
|
| 172 |
+
'ON CONFLICT (key) DO UPDATE SET value = EXCLUDED.value, '
|
| 173 |
+
'updated_at = now(), rev = {bare}.rev + 1', 'store_kv', tenant_slug),
|
| 174 |
+
(str(name), json.dumps(data)))
|
| 175 |
+
return data
|
| 176 |
+
|
| 177 |
+
|
| 178 |
+
def update(name, fn, flush='sync', tenant_slug=None):
|
| 179 |
+
"""Read-modify-write inside ONE transaction, with the row LOCKED for the duration.
|
| 180 |
+
|
| 181 |
+
This is the operation the HF backend could not make safe. There, `update` read over HTTP,
|
| 182 |
+
applied `fn`, and uploaded β so two concurrent writers both read the old value and the second
|
| 183 |
+
upload silently discarded the first writer's change. `SELECT β¦ FOR UPDATE` makes the second
|
| 184 |
+
writer WAIT and then apply `fn` to the first writer's result, which is what "read-modify-write"
|
| 185 |
+
was always supposed to mean.
|
| 186 |
+
|
| 187 |
+
`flush` is accepted and ignored (see the module docstring): there is no commit budget to
|
| 188 |
+
coalesce against, so 'async' has nothing to defer.
|
| 189 |
+
"""
|
| 190 |
+
with _pool().connection() as con:
|
| 191 |
+
with con.transaction():
|
| 192 |
+
row = con.execute(
|
| 193 |
+
_q('SELECT value FROM {tbl} WHERE key = %s FOR UPDATE',
|
| 194 |
+
'store_kv', tenant_slug),
|
| 195 |
+
(str(name),)).fetchone()
|
| 196 |
+
data = dict(row[0]) if row and isinstance(row[0], dict) else (row[0] if row else {})
|
| 197 |
+
result = fn(data)
|
| 198 |
+
data = result if result is not None else data
|
| 199 |
+
con.execute(
|
| 200 |
+
_q('INSERT INTO {tbl} (key, value) VALUES (%s, %s::jsonb) '
|
| 201 |
+
'ON CONFLICT (key) DO UPDATE SET value = EXCLUDED.value, '
|
| 202 |
+
'updated_at = now(), rev = {bare}.rev + 1', 'store_kv', tenant_slug),
|
| 203 |
+
(str(name), json.dumps(data)))
|
| 204 |
+
return data
|
| 205 |
+
|
| 206 |
+
|
| 207 |
+
def upload_bytes(path_in_repo, data, message=None, tenant_slug=None):
|
| 208 |
+
with _pool().connection() as con:
|
| 209 |
+
con.execute(
|
| 210 |
+
_q('INSERT INTO {tbl} (path, bytes, message) VALUES (%s, %s, %s) '
|
| 211 |
+
'ON CONFLICT (path) DO UPDATE SET bytes = EXCLUDED.bytes, '
|
| 212 |
+
'message = EXCLUDED.message, updated_at = now()', 'store_blobs', tenant_slug),
|
| 213 |
+
(str(path_in_repo), bytes(data), message))
|
| 214 |
+
|
| 215 |
+
|
| 216 |
+
def download_bytes(path_in_repo, tenant_slug=None):
|
| 217 |
+
"""Raw bytes, or None when absent β the same contract as the HF backend: a missing attachment
|
| 218 |
+
degrades to "that file is gone", never to a broken page."""
|
| 219 |
+
with _pool().connection() as con:
|
| 220 |
+
row = con.execute(
|
| 221 |
+
_q('SELECT bytes FROM {tbl} WHERE path = %s', 'store_blobs', tenant_slug),
|
| 222 |
+
(str(path_in_repo),)).fetchone()
|
| 223 |
+
return bytes(row[0]) if row else None
|
| 224 |
+
|
| 225 |
+
|
| 226 |
+
def delete_path(path_in_repo, tenant_slug=None):
|
| 227 |
+
"""Absent is SUCCESS β deleting what is already gone is the goal (HF backend's contract)."""
|
| 228 |
+
with _pool().connection() as con:
|
| 229 |
+
con.execute(_q('DELETE FROM {tbl} WHERE path = %s', 'store_blobs', tenant_slug),
|
| 230 |
+
(str(path_in_repo),))
|
| 231 |
+
return True
|
| 232 |
+
|
| 233 |
+
|
| 234 |
+
def flush(name=None, timeout=30.0):
|
| 235 |
+
"""A no-op that returns True: every write above is already committed when it returns. Kept in
|
| 236 |
+
the interface because QA gates and shutdown hooks call it and must not branch on the backend."""
|
| 237 |
+
return True
|
| 238 |
+
|
| 239 |
+
|
| 240 |
+
def close():
|
| 241 |
+
"""Release the pool (tests, and a process that is shutting down cleanly)."""
|
| 242 |
+
with _LOCK:
|
| 243 |
+
if _POOL['pool'] is not None:
|
| 244 |
+
try:
|
| 245 |
+
_POOL['pool'].close()
|
| 246 |
+
finally:
|
| 247 |
+
_POOL['pool'] = None
|
| 248 |
+
_POOL['url'] = None
|
| 249 |
+
_POOL.pop('ok', None)
|
platform/core/table_store.py
ADDED
|
@@ -0,0 +1,479 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""The generic per-user TABLE WORKSPACE store β the persistence half of the table-page factory.
|
| 2 |
+
|
| 3 |
+
One durable store key holds one table OBJECT's per-user Airtable-style state:
|
| 4 |
+
|
| 5 |
+
{username: {'views': {view_id: SavedView},
|
| 6 |
+
'fields': {field_key: Field}, # notes + custom_ + measure_ strata
|
| 7 |
+
'overlays': {str(pid): {field_key: value}}}}
|
| 8 |
+
|
| 9 |
+
`make(table_key)` returns the six operations a table page's host loop needs, closed over that
|
| 10 |
+
key. The Customer table's ops (`modules/customer_data.py`, key 'customer_table_workspace') are
|
| 11 |
+
these exact functions β the logic MOVED here 2026-07-27 so that duplicating the Customer table
|
| 12 |
+
pattern to a new object is a registry row + a config, not a copy of the store plumbing
|
| 13 |
+
(owner directive: the table-page factory).
|
| 14 |
+
|
| 15 |
+
A LIST (membership/formula semantics) is deliberately a different store from a VIEW
|
| 16 |
+
(presentation/query state) β see modules/customer_data.py's customer_lists key.
|
| 17 |
+
"""
|
| 18 |
+
import core.store as store
|
| 19 |
+
|
| 20 |
+
#: Wave-9 I17 β the SHARED bucket. Views whose permissions make them visible to anyone but
|
| 21 |
+
#: their creator live here instead of in a personal workspace, under a key that cannot collide
|
| 22 |
+
#: with a username (usernames come from core/users.py and are never dunder-wrapped; `is_shared`
|
| 23 |
+
#: guards it anyway). ONE HOME PER VIEW, never both: a view moved to personal is REMOVED from
|
| 24 |
+
#: here, and a view shared is removed from its creator's workspace. Two homes would mean two
|
| 25 |
+
#: divergent copies the moment either was edited.
|
| 26 |
+
SHARED_KEY = '__shared__'
|
| 27 |
+
|
| 28 |
+
|
| 29 |
+
def _may_see(view, viewer, is_admin=False):
|
| 30 |
+
"""Visibility for ONE shared view, fail-closed.
|
| 31 |
+
|
| 32 |
+
'collaborative' = everyone who can already open the module (the caller has gated that).
|
| 33 |
+
'users' = the named users, plus the creator, plus admins β an admin who could not
|
| 34 |
+
see a view could not administer it either.
|
| 35 |
+
Anything unrecognised returns False rather than defaulting open: an unreadable permission
|
| 36 |
+
must never widen access ([[aios-permissioning]] β no fail-open defaults).
|
| 37 |
+
"""
|
| 38 |
+
if not isinstance(view, dict):
|
| 39 |
+
return False
|
| 40 |
+
if view.get('createdBy') == viewer or is_admin:
|
| 41 |
+
return True
|
| 42 |
+
perms = view.get('permissions') or {}
|
| 43 |
+
edit = perms.get('edit')
|
| 44 |
+
if edit == 'collaborative':
|
| 45 |
+
return True
|
| 46 |
+
if edit == 'users':
|
| 47 |
+
return viewer in set(perms.get('users') or ())
|
| 48 |
+
return False # 'personal', absent, or junk
|
| 49 |
+
|
| 50 |
+
|
| 51 |
+
def _may_edit(view, viewer, is_admin=False):
|
| 52 |
+
"""Who may WRITE a shared view. Same set as visibility today β the owner's item asks 'who
|
| 53 |
+
can edit' and lists who 'can have access', i.e. seeing and editing are one grant. Kept as a
|
| 54 |
+
separate function so they can diverge (a future read-only share) without hunting callers."""
|
| 55 |
+
return _may_see(view, viewer, is_admin)
|
| 56 |
+
|
| 57 |
+
|
| 58 |
+
def _may_administer(view, viewer, is_admin=False):
|
| 59 |
+
"""Who may change a view's PERMISSIONS, or delete it: the creator or an admin ONLY.
|
| 60 |
+
|
| 61 |
+
Deliberately narrower than _may_edit. If a collaborator could rewrite `permissions` they
|
| 62 |
+
could grant themselves sole ownership of somebody else's view, or quietly widen a
|
| 63 |
+
users-scoped view to everyone β the classic privilege-escalation-by-edit hole.
|
| 64 |
+
"""
|
| 65 |
+
if not isinstance(view, dict):
|
| 66 |
+
return False
|
| 67 |
+
return bool(is_admin) or view.get('createdBy') == viewer
|
| 68 |
+
|
| 69 |
+
|
| 70 |
+
def is_shared(view):
|
| 71 |
+
"""A view belongs in the shared bucket when its permissions reach beyond its creator."""
|
| 72 |
+
return ((view or {}).get('permissions') or {}).get('edit') in ('collaborative', 'users')
|
| 73 |
+
|
| 74 |
+
|
| 75 |
+
def _unique_name(wanted, existing, *, fallback='Untitled', max_len=120):
|
| 76 |
+
"""Allocate one human-facing name inside a store.update transaction.
|
| 77 |
+
|
| 78 |
+
Keys/ids remain structural identity. Names compare case-insensitively after collapsing
|
| 79 |
+
whitespace, because those variants are indistinguishable in the UI. This helper belongs
|
| 80 |
+
in the store layer: allocating from a pre-write snapshot lets two concurrent requests both
|
| 81 |
+
choose the same free name before either write lands.
|
| 82 |
+
"""
|
| 83 |
+
limit = max(1, int(max_len))
|
| 84 |
+
|
| 85 |
+
def _clean(value):
|
| 86 |
+
return ' '.join(str(value or '').split())
|
| 87 |
+
|
| 88 |
+
base = (_clean(wanted) or _clean(fallback) or 'Untitled')[:limit].rstrip()
|
| 89 |
+
taken = {_clean(value).casefold() for value in existing if _clean(value)}
|
| 90 |
+
if base.casefold() not in taken:
|
| 91 |
+
return base
|
| 92 |
+
index = 2
|
| 93 |
+
while True:
|
| 94 |
+
suffix = f' {index}'
|
| 95 |
+
stem = base[:max(0, limit - len(suffix))].rstrip()
|
| 96 |
+
candidate = f'{stem}{suffix}' if stem else str(index)[-limit:]
|
| 97 |
+
if candidate.casefold() not in taken:
|
| 98 |
+
return candidate
|
| 99 |
+
index += 1
|
| 100 |
+
|
| 101 |
+
|
| 102 |
+
class TableStore:
|
| 103 |
+
"""The six store operations for one table object's workspace, closed over its store key.
|
| 104 |
+
|
| 105 |
+
`st` (wave 18, C3-UT) is the STORE HANDLE β anything exposing `get(name)` /
|
| 106 |
+
`update(name, fn, flush=)`. Default = `core.store` (tenant #0, every existing caller,
|
| 107 |
+
zero behaviour change). The API passes the session's `TenantRuntime`, whose accessors
|
| 108 |
+
apply the tenant prefix / repo binding β which is what makes a user table created by a
|
| 109 |
+
Nurilab admin land in Nurilab's store instead of Royal's.
|
| 110 |
+
"""
|
| 111 |
+
|
| 112 |
+
def __init__(self, table_key, st=None):
|
| 113 |
+
self.table_key = table_key
|
| 114 |
+
self._st = st if st is not None else store
|
| 115 |
+
|
| 116 |
+
# ---------------------------------------------------------------- read
|
| 117 |
+
def workspace(self, username, consume_corrections=True):
|
| 118 |
+
"""One user's durable workspace: always the full three-strata shape."""
|
| 119 |
+
try:
|
| 120 |
+
data = self._st.get(self.table_key) or {}
|
| 121 |
+
ws = data.get(username, {}) or {}
|
| 122 |
+
# A collision acknowledgement is protocol state, not part of a field definition.
|
| 123 |
+
# Consume it with the first fresh workspace payload after the correcting write,
|
| 124 |
+
# then splice a bounded copy into that payload only. Keeping it out of `fields`
|
| 125 |
+
# prevents an old request id surviving forever and overriding a later rename.
|
| 126 |
+
corrections = {}
|
| 127 |
+
if consume_corrections and ws.get('fieldCorrections'):
|
| 128 |
+
def _take(current):
|
| 129 |
+
current_ws = current.get(username) or {}
|
| 130 |
+
pending = current_ws.get('fieldCorrections') or {}
|
| 131 |
+
corrections.update({
|
| 132 |
+
str(key)[:80]: dict(value)
|
| 133 |
+
for key, value in pending.items()
|
| 134 |
+
if isinstance(value, dict)
|
| 135 |
+
})
|
| 136 |
+
current_ws.pop('fieldCorrections', None)
|
| 137 |
+
return current
|
| 138 |
+
|
| 139 |
+
data = self._st.update(self.table_key, _take, flush='async')
|
| 140 |
+
ws = (data or {}).get(username, {}) or {}
|
| 141 |
+
except Exception:
|
| 142 |
+
ws = {}
|
| 143 |
+
corrections = {}
|
| 144 |
+
fields = {
|
| 145 |
+
key: dict(value) if isinstance(value, dict) else value
|
| 146 |
+
for key, value in (ws.get('fields') or {}).items()
|
| 147 |
+
}
|
| 148 |
+
for key, ack in corrections.items():
|
| 149 |
+
field = fields.get(key)
|
| 150 |
+
accepted_label = str(ack.get('label') or '')[:120]
|
| 151 |
+
requested_label = str(ack.get('labelCorrectedFrom') or '')[:120]
|
| 152 |
+
correction_id = str(ack.get('labelCorrectionId') or '')[:180]
|
| 153 |
+
# A newer field write clears/replaces the pending ack in the SAME transaction.
|
| 154 |
+
# The label check is an extra belt against ever attaching a stale ack to a newer
|
| 155 |
+
# definition if a future store implementation weakens that ordering.
|
| 156 |
+
if (isinstance(field, dict) and accepted_label
|
| 157 |
+
and str(field.get('label') or '') == accepted_label
|
| 158 |
+
and requested_label and correction_id):
|
| 159 |
+
field['labelCorrectedFrom'] = requested_label
|
| 160 |
+
field['labelCorrectionId'] = correction_id
|
| 161 |
+
out = {
|
| 162 |
+
'views': dict(ws.get('views') or {}),
|
| 163 |
+
'fields': fields,
|
| 164 |
+
'overlays': dict(ws.get('overlays') or {}),
|
| 165 |
+
# wave-8 I11 (C4): folders over the saved views / cohorts sidebars. A FOURTH
|
| 166 |
+
# stratum rather than a key on each item β see aios_grid.clean_folders for why
|
| 167 |
+
# (a cohort lives in another store, and filing is an organising act, not part of
|
| 168 |
+
# what a view is). Absent for every workspace saved before this wave, which is
|
| 169 |
+
# exactly "no folders yet".
|
| 170 |
+
'folders': dict(ws.get('folders') or {}),
|
| 171 |
+
'itemFolders': dict(ws.get('itemFolders') or {}),
|
| 172 |
+
}
|
| 173 |
+
# 2026-07-31 (owner item 3): WHERE THE USER LEFT OFF survives a new browser. The
|
| 174 |
+
# client's localStorage copy wins when present; this is the server's answer for a
|
| 175 |
+
# fresh profile, which used to fall all the way to the system default view.
|
| 176 |
+
if ws.get('activeViewId'):
|
| 177 |
+
out['activeViewId'] = str(ws['activeViewId'])
|
| 178 |
+
# Wave 2026-08-02 (C-LAYOUT): the per-user record-detail field order. A fifth
|
| 179 |
+
# stratum, absent until the user first reorders β exactly "default order".
|
| 180 |
+
if isinstance(ws.get('recordLayout'), dict):
|
| 181 |
+
out['recordLayout'] = dict(ws['recordLayout'])
|
| 182 |
+
return out
|
| 183 |
+
|
| 184 |
+
# ---------------------------------------------------------------- write
|
| 185 |
+
def _update(self, username, change):
|
| 186 |
+
def _up(data):
|
| 187 |
+
ws = data.setdefault(username, {})
|
| 188 |
+
ws.setdefault('views', {})
|
| 189 |
+
ws.setdefault('fields', {})
|
| 190 |
+
ws.setdefault('overlays', {})
|
| 191 |
+
ws.setdefault('folders', {})
|
| 192 |
+
ws.setdefault('itemFolders', {})
|
| 193 |
+
change(ws)
|
| 194 |
+
return data
|
| 195 |
+
# flush='async' (wave-7 W3): this is THE hot path β every autosaved filter tweak,
|
| 196 |
+
# column note and typed overlay cell lands here inside the component round-trip, and
|
| 197 |
+
# the historical synchronous hub commit cost seconds per edit. The mutation applies to
|
| 198 |
+
# the in-process cache (read-your-writes for every subsequent render); the hub write
|
| 199 |
+
# coalesces in the background. Registry/auth writes elsewhere stay flush='sync'.
|
| 200 |
+
return self._st.update(self.table_key, _up, flush='async')
|
| 201 |
+
|
| 202 |
+
def save_active_view(self, username, view_id):
|
| 203 |
+
"""Remember which view this user last opened (owner item 3, 2026-07-31).
|
| 204 |
+
|
| 205 |
+
Presentation state, not authorisation: the READ side re-validates the id against what
|
| 206 |
+
the caller may actually see, so a stale or foreign id degrades to the default view
|
| 207 |
+
rather than granting anything. Stored per user like every other stratum.
|
| 208 |
+
"""
|
| 209 |
+
vid = str(view_id or '').strip()[:120]
|
| 210 |
+
if not vid or username == SHARED_KEY:
|
| 211 |
+
return
|
| 212 |
+
|
| 213 |
+
def _set(ws):
|
| 214 |
+
ws['activeViewId'] = vid
|
| 215 |
+
self._update(username, _set)
|
| 216 |
+
|
| 217 |
+
def save_record_layout(self, username, order):
|
| 218 |
+
"""The per-user RECORD-DETAIL field order (wave 2026-08-02, C-LAYOUT).
|
| 219 |
+
|
| 220 |
+
Presentation state for ONE surface β the record modal. Deliberately not view config:
|
| 221 |
+
the owner's ask is per-user, not per-view, and it must never reorder grid columns.
|
| 222 |
+
The event handler validated keys against the live field set; the wire re-validates at
|
| 223 |
+
serve time (aios_grid.workspace_wire), so a deleted field cannot outlive itself here.
|
| 224 |
+
An empty order clears the stratum back to "default order".
|
| 225 |
+
"""
|
| 226 |
+
if username == SHARED_KEY:
|
| 227 |
+
return
|
| 228 |
+
clean, seen = [], set()
|
| 229 |
+
for key in (order or [])[:200]:
|
| 230 |
+
key = str(key or '').strip()[:80]
|
| 231 |
+
if key and key not in seen:
|
| 232 |
+
seen.add(key)
|
| 233 |
+
clean.append(key)
|
| 234 |
+
|
| 235 |
+
def _set(ws):
|
| 236 |
+
if clean:
|
| 237 |
+
ws['recordLayout'] = {'order': clean}
|
| 238 |
+
else:
|
| 239 |
+
ws.pop('recordLayout', None)
|
| 240 |
+
|
| 241 |
+
self._update(username, _set)
|
| 242 |
+
|
| 243 |
+
def save_folders(self, username, folders, item_folders):
|
| 244 |
+
"""Replace the folder stratum wholesale (wave-8 I11).
|
| 245 |
+
|
| 246 |
+
Wholesale rather than per-folder because the caller has ALREADY validated the complete
|
| 247 |
+
picture through aios_grid.clean_folders / clean_item_folders, and those two are
|
| 248 |
+
interdependent: a placement is only legal while its folder exists, so committing them
|
| 249 |
+
separately would leave a window where a reader sees an item filed into a folder that is
|
| 250 |
+
not there yet. One write, one consistent state.
|
| 251 |
+
"""
|
| 252 |
+
def _set(ws):
|
| 253 |
+
ws['folders'] = dict(folders or {})
|
| 254 |
+
ws['itemFolders'] = dict(item_folders or {})
|
| 255 |
+
self._update(username, _set)
|
| 256 |
+
|
| 257 |
+
def shared_views(self, viewer, is_admin=False):
|
| 258 |
+
"""Every SHARED view this viewer may see, by id (wave-9 I17).
|
| 259 |
+
|
| 260 |
+
Read-only and independent of the viewer's own workspace: the caller merges. Returns
|
| 261 |
+
only what `_may_see` allows, so a caller cannot accidentally render somebody else's
|
| 262 |
+
personal view by forgetting to filter.
|
| 263 |
+
"""
|
| 264 |
+
try:
|
| 265 |
+
bucket = ((self._st.get(self.table_key) or {}).get(SHARED_KEY) or {}).get('views') or {}
|
| 266 |
+
except Exception:
|
| 267 |
+
return {}
|
| 268 |
+
return {vid: dict(v) for vid, v in bucket.items()
|
| 269 |
+
if _may_see(v, viewer, is_admin)}
|
| 270 |
+
|
| 271 |
+
def shared_view(self, view_id):
|
| 272 |
+
"""One shared view RAW β no visibility filter. For authorisation decisions only: a
|
| 273 |
+
caller must know a view exists and who owns it before it can decide whether the actor
|
| 274 |
+
may touch it. Never hand the result to a renderer without checking `_may_see`."""
|
| 275 |
+
try:
|
| 276 |
+
return ((self._st.get(self.table_key) or {}).get(SHARED_KEY) or {}
|
| 277 |
+
).get('views', {}).get(str(view_id))
|
| 278 |
+
except Exception:
|
| 279 |
+
return None
|
| 280 |
+
|
| 281 |
+
def save_view(self, username, view, shared=None, reserved_names=(), is_admin=False):
|
| 282 |
+
"""Upsert a SavedView into its ONE home β personal workspace or the shared bucket.
|
| 283 |
+
|
| 284 |
+
`shared` defaults to reading the view's own permissions (`is_shared`). Whichever home
|
| 285 |
+
it lands in, the view is REMOVED from the other, so a view can never exist as two
|
| 286 |
+
copies that diverge on the next edit.
|
| 287 |
+
|
| 288 |
+
β AUTHORISATION IS THE CALLER'S JOB and must happen BEFORE this is called β this layer
|
| 289 |
+
moves data and does not know who is asking. `_cl_handle_one` is the wall.
|
| 290 |
+
"""
|
| 291 |
+
view_id = str((view or {}).get('id') or '').strip()
|
| 292 |
+
if not view_id:
|
| 293 |
+
raise ValueError('view id is required')
|
| 294 |
+
if username == SHARED_KEY:
|
| 295 |
+
raise ValueError('reserved username')
|
| 296 |
+
to_shared = is_shared(view) if shared is None else bool(shared)
|
| 297 |
+
requested = dict(view)
|
| 298 |
+
accepted = {}
|
| 299 |
+
|
| 300 |
+
def _up(data):
|
| 301 |
+
# View names are tenant-global: every personal workspace plus the shared bucket.
|
| 302 |
+
# This deliberately includes views the actor cannot see. The only disclosed fact
|
| 303 |
+
# is that a display name is already taken, while the categorical "no duplicate
|
| 304 |
+
# view names" contract remains true when a personal view is later shared.
|
| 305 |
+
names = list(reserved_names or ())
|
| 306 |
+
for workspace in data.values():
|
| 307 |
+
if not isinstance(workspace, dict):
|
| 308 |
+
continue
|
| 309 |
+
names.extend(
|
| 310 |
+
value.get('name')
|
| 311 |
+
for candidate_id, value in (workspace.get('views') or {}).items()
|
| 312 |
+
if candidate_id != view_id and isinstance(value, dict)
|
| 313 |
+
)
|
| 314 |
+
payload = dict(requested)
|
| 315 |
+
payload['name'] = _unique_name(payload.get('name'), names)
|
| 316 |
+
accepted.clear()
|
| 317 |
+
accepted.update(payload)
|
| 318 |
+
if to_shared:
|
| 319 |
+
bucket = data.setdefault(SHARED_KEY, {})
|
| 320 |
+
bucket.setdefault('views', {})[view_id] = payload
|
| 321 |
+
# it may have lived in the creator's workspace before being shared
|
| 322 |
+
owner = data.get(payload.get('createdBy') or username) or {}
|
| 323 |
+
(owner.get('views') or {}).pop(view_id, None)
|
| 324 |
+
else:
|
| 325 |
+
ws = data.setdefault(username, {})
|
| 326 |
+
ws.setdefault('views', {})[view_id] = payload
|
| 327 |
+
(data.get(SHARED_KEY, {}).get('views') or {}).pop(view_id, None)
|
| 328 |
+
return data
|
| 329 |
+
|
| 330 |
+
self._st.update(self.table_key, _up, flush='async')
|
| 331 |
+
return dict(accepted)
|
| 332 |
+
|
| 333 |
+
def delete_view(self, username, view_id):
|
| 334 |
+
"""Delete a custom/list view override. The system all-rows view is guarded by caller.
|
| 335 |
+
|
| 336 |
+
Removes from BOTH homes: the caller has already authorised the delete, and leaving a
|
| 337 |
+
stale copy in the other bucket would resurrect the view on the next read.
|
| 338 |
+
"""
|
| 339 |
+
vid = str(view_id)
|
| 340 |
+
|
| 341 |
+
def _up(data):
|
| 342 |
+
(data.get(username, {}).get('views') or {}).pop(vid, None)
|
| 343 |
+
(data.get(SHARED_KEY, {}).get('views') or {}).pop(vid, None)
|
| 344 |
+
return data
|
| 345 |
+
|
| 346 |
+
self._st.update(self.table_key, _up, flush='async')
|
| 347 |
+
|
| 348 |
+
def save_field(self, username, field, reserved_names=(), correction_id=None):
|
| 349 |
+
"""Persist a column note or a user-created (custom_/measure_) field definition."""
|
| 350 |
+
key = str((field or {}).get('key') or '').strip()
|
| 351 |
+
if not key:
|
| 352 |
+
raise ValueError('field key is required')
|
| 353 |
+
requested = dict(field)
|
| 354 |
+
accepted = {}
|
| 355 |
+
|
| 356 |
+
def _save(ws):
|
| 357 |
+
names = list(reserved_names or ())
|
| 358 |
+
names.extend(
|
| 359 |
+
value.get('label')
|
| 360 |
+
for candidate_key, value in (ws.get('fields') or {}).items()
|
| 361 |
+
if candidate_key != key and isinstance(value, dict)
|
| 362 |
+
)
|
| 363 |
+
payload = dict(requested)
|
| 364 |
+
payload.pop('labelCorrectedFrom', None)
|
| 365 |
+
payload.pop('labelCorrectionId', None)
|
| 366 |
+
requested_label = ' '.join(
|
| 367 |
+
str(payload.get('label') or 'Untitled').split())[:120].rstrip()
|
| 368 |
+
payload['label'] = _unique_name(requested_label, names)
|
| 369 |
+
corrections = ws.setdefault('fieldCorrections', {})
|
| 370 |
+
corrections.pop(key, None)
|
| 371 |
+
if payload['label'] != requested_label and correction_id:
|
| 372 |
+
corrections[key] = {
|
| 373 |
+
'label': payload['label'],
|
| 374 |
+
'labelCorrectedFrom': requested_label,
|
| 375 |
+
'labelCorrectionId': str(correction_id)[:180],
|
| 376 |
+
}
|
| 377 |
+
if not corrections:
|
| 378 |
+
ws.pop('fieldCorrections', None)
|
| 379 |
+
accepted.clear()
|
| 380 |
+
accepted.update(payload)
|
| 381 |
+
# A cleared note on an immutable source field returns to the canonical schema
|
| 382 |
+
# instead of leaving a meaningless override row. Custom fields remain even with
|
| 383 |
+
# an empty note β and so does a PRESET field carrying a measure-window override
|
| 384 |
+
# (wave-2 item 8) or a DISPLAY-format override (wave-5 item 10): the note may be
|
| 385 |
+
# empty but the period/format is real user state.
|
| 386 |
+
if (not payload.get('custom') and not str(payload.get('note') or '').strip()
|
| 387 |
+
and not isinstance(payload.get('measure'), dict)
|
| 388 |
+
and not isinstance(payload.get('format'), dict)):
|
| 389 |
+
ws['fields'].pop(key, None)
|
| 390 |
+
else:
|
| 391 |
+
ws['fields'][key] = payload
|
| 392 |
+
|
| 393 |
+
self._update(username, _save)
|
| 394 |
+
return dict(accepted)
|
| 395 |
+
|
| 396 |
+
def duplicate_field(self, username, source_key, new_key, field,
|
| 397 |
+
reserved_names=(), correction_id=None):
|
| 398 |
+
"""Clone a user-created field in ONE store transaction (wave-5 item 1): the new
|
| 399 |
+
definition plus β for `custom_` overlay sources only β every stored cell value under
|
| 400 |
+
the source key. One transaction, because a def without its values (or values without a
|
| 401 |
+
def) is exactly the orphan state delete_field exists to prevent, in reverse.
|
| 402 |
+
The caller validated both keys (same created stratum) and stamped the clone's
|
| 403 |
+
createdBy; this layer only moves data."""
|
| 404 |
+
source_key = str(source_key or '').strip()
|
| 405 |
+
new_key = str(new_key or '').strip()
|
| 406 |
+
if not source_key or not new_key or source_key == new_key:
|
| 407 |
+
raise ValueError('duplicate_field needs two distinct keys')
|
| 408 |
+
requested = dict(field)
|
| 409 |
+
accepted = {}
|
| 410 |
+
|
| 411 |
+
def _dup(ws):
|
| 412 |
+
names = list(reserved_names or ())
|
| 413 |
+
names.extend(
|
| 414 |
+
value.get('label')
|
| 415 |
+
for candidate_key, value in (ws.get('fields') or {}).items()
|
| 416 |
+
if candidate_key != new_key and isinstance(value, dict)
|
| 417 |
+
)
|
| 418 |
+
payload = dict(requested)
|
| 419 |
+
payload.pop('labelCorrectedFrom', None)
|
| 420 |
+
payload.pop('labelCorrectionId', None)
|
| 421 |
+
requested_label = ' '.join(
|
| 422 |
+
str(payload.get('label') or 'Untitled').split())[:120].rstrip()
|
| 423 |
+
payload['label'] = _unique_name(requested_label, names)
|
| 424 |
+
corrections = ws.setdefault('fieldCorrections', {})
|
| 425 |
+
corrections.pop(new_key, None)
|
| 426 |
+
if payload['label'] != requested_label and correction_id:
|
| 427 |
+
corrections[new_key] = {
|
| 428 |
+
'label': payload['label'],
|
| 429 |
+
'labelCorrectedFrom': requested_label,
|
| 430 |
+
'labelCorrectionId': str(correction_id)[:180],
|
| 431 |
+
}
|
| 432 |
+
if not corrections:
|
| 433 |
+
ws.pop('fieldCorrections', None)
|
| 434 |
+
accepted.clear()
|
| 435 |
+
accepted.update(payload)
|
| 436 |
+
ws['fields'][new_key] = payload
|
| 437 |
+
if source_key.startswith('custom_'):
|
| 438 |
+
for row in ws['overlays'].values():
|
| 439 |
+
if isinstance(row, dict) and source_key in row:
|
| 440 |
+
row[new_key] = row[source_key]
|
| 441 |
+
|
| 442 |
+
self._update(username, _dup)
|
| 443 |
+
return dict(accepted)
|
| 444 |
+
|
| 445 |
+
def delete_field(self, username, key):
|
| 446 |
+
"""Delete a USER-CREATED field definition outright (owner gap closed 2026-07-27).
|
| 447 |
+
|
| 448 |
+
Only the created strata ever reach here (`custom_` overlay fields, `measure_` formula
|
| 449 |
+
columns β the caller enforces the prefix). The stored overlay VALUES for the key are
|
| 450 |
+
scrubbed with it: a deleted column's cells must not linger as orphan data that would
|
| 451 |
+
silently resurface if the key were ever reused. Views referencing the key self-heal on
|
| 452 |
+
their next autosave (an unknown colId is dropped) β the rule every stale key rides.
|
| 453 |
+
"""
|
| 454 |
+
key = str(key or '').strip()
|
| 455 |
+
if not key:
|
| 456 |
+
return
|
| 457 |
+
|
| 458 |
+
def _drop(ws):
|
| 459 |
+
ws['fields'].pop(key, None)
|
| 460 |
+
for row in ws['overlays'].values():
|
| 461 |
+
if isinstance(row, dict):
|
| 462 |
+
row.pop(key, None)
|
| 463 |
+
|
| 464 |
+
self._update(username, _drop)
|
| 465 |
+
|
| 466 |
+
def patch_overlay(self, username, pid, updates):
|
| 467 |
+
"""Patch only the external editable stratum; never writes to the source system."""
|
| 468 |
+
clean = dict(updates or {})
|
| 469 |
+
if not clean:
|
| 470 |
+
return
|
| 471 |
+
|
| 472 |
+
def _patch(ws):
|
| 473 |
+
ws['overlays'].setdefault(str(int(pid)), {}).update(clean)
|
| 474 |
+
|
| 475 |
+
self._update(username, _patch)
|
| 476 |
+
|
| 477 |
+
|
| 478 |
+
def make(table_key, st=None):
|
| 479 |
+
return TableStore(table_key, st=st)
|
platform/core/user_tables.py
ADDED
|
@@ -0,0 +1,292 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""USER-CREATED TABLES (wave-9 I8 / contract C6) β databases that are not a connector.
|
| 2 |
+
|
| 3 |
+
The owner's "+ New" flow has two branches: connect a SOURCE (Odoo today) or *"create a completely
|
| 4 |
+
new database blank, where this time the user can actually insert row that they want."* The first
|
| 5 |
+
branch is the connector seam and lives in `harness/datastore.py`; THIS module is the second.
|
| 6 |
+
|
| 7 |
+
WHY A SEPARATE STORE AND NOT A REGISTRY ROW. `core/registry.py` is a static Python list read at
|
| 8 |
+
import time β it is the catalogue of modules the PRODUCT ships, and a tenant cannot append to it
|
| 9 |
+
at runtime without editing code. A user-created table is tenant data, so it lives in the tenant
|
| 10 |
+
store and is MERGED into the nav beside the registry rows. That also keeps the archived/allowed
|
| 11 |
+
machinery honest: a user table is never "archived", it is deleted, and it is never in
|
| 12 |
+
`allowed_modules` because it is not a module.
|
| 13 |
+
|
| 14 |
+
SHAPE (one store key, `user_tables`):
|
| 15 |
+
|
| 16 |
+
{table_key: {'key', 'label', 'source', 'createdBy', 'created', 'fields': [...], 'rows': {...}}}
|
| 17 |
+
|
| 18 |
+
`source` is deliberately NOT 'Odoo' β the nav badge is generated from it, so a blank table reads
|
| 19 |
+
"Blank" and can never be mistaken for connected data. `rows` is `{row_id: {field_key: value}}`,
|
| 20 |
+
which is the same overlay shape `TableStore` already uses, so the grid's write path needs no new
|
| 21 |
+
storage concept.
|
| 22 |
+
|
| 23 |
+
β ROW WRITES ARE ONLY EVER LEGAL HERE. An Odoo-sourced table is READ-ONLY at the source and the
|
| 24 |
+
row endpoints refuse unless `is_user_table(key)` β a user must not be able to invent a customer
|
| 25 |
+
in Odoo by typing into a grid. (Wave 18 note: the events seam has NO row event types; row
|
| 26 |
+
add/delete are REST endpoints on `aios-web/api/routes_tables.py`, and this predicate is their
|
| 27 |
+
wall.)
|
| 28 |
+
|
| 29 |
+
WAVE 18 (C3-UT): every function takes an optional `st` STORE HANDLE β anything exposing
|
| 30 |
+
`get(name)` / `update(name, fn, flush=)` / `exists(name)`. Default = `core.store` (tenant #0,
|
| 31 |
+
the Streamlit host, unchanged). The API passes the session's `TenantRuntime`, whose accessors
|
| 32 |
+
apply the tenant prefix / repo binding, so a Nurilab table lands in Nurilab's store.
|
| 33 |
+
"""
|
| 34 |
+
import datetime as _dt
|
| 35 |
+
import re
|
| 36 |
+
|
| 37 |
+
import core.store as store
|
| 38 |
+
|
| 39 |
+
STORE_KEY = 'user_tables'
|
| 40 |
+
MAX_TABLES = 40
|
| 41 |
+
MAX_LABEL = 60
|
| 42 |
+
MAX_ROWS = 5000
|
| 43 |
+
MAX_FIELDS = 60
|
| 44 |
+
#: the badge the nav paints for a table with no connector behind it
|
| 45 |
+
BLANK_SOURCE = 'Blank'
|
| 46 |
+
#: automation-created tables carry their maker instead (wave 18 C4-AUTO)
|
| 47 |
+
AUTOMATION_SOURCE = 'Automation'
|
| 48 |
+
#: every user table's key is prefixed, so it can never collide with a registry module key
|
| 49 |
+
KEY_PREFIX = 'ut_'
|
| 50 |
+
|
| 51 |
+
#: The field types a user table may declare. β Kept a SUBSET of
|
| 52 |
+
#: `aios_grid.CUSTOM_FIELD_TYPES` (gated in verify_api's W18-UT section) β minus the two
|
| 53 |
+
#: machine-written kinds (`formula` computes client-side over a workspace stratum; a base
|
| 54 |
+
#: column of that type would have no author). Local literal rather than an aios_grid import so
|
| 55 |
+
#: this module stays dependency-light for the API's boot path.
|
| 56 |
+
UT_FIELD_TYPES = {'text', 'select', 'multiselect', 'user', 'int', 'currency', 'pct', 'date',
|
| 57 |
+
'checkbox', 'phone', 'email', 'url', 'rating'}
|
| 58 |
+
|
| 59 |
+
|
| 60 |
+
def _st(st):
|
| 61 |
+
return st if st is not None else store
|
| 62 |
+
|
| 63 |
+
|
| 64 |
+
def _slug(label):
|
| 65 |
+
s = re.sub(r'[^a-z0-9]+', '_', str(label or '').strip().lower()).strip('_')
|
| 66 |
+
return (s or 'table')[:40]
|
| 67 |
+
|
| 68 |
+
|
| 69 |
+
def clean_fields(raw):
|
| 70 |
+
"""Validate a caller-supplied field list into the stored shape β refuse junk, never store
|
| 71 |
+
it. Returns a list of `{key,label,type,source}` dicts (source is always 'overlay': every
|
| 72 |
+
base column of a user table is user-editable by construction), or None when nothing
|
| 73 |
+
survives. Unknown types are DROPPED, not coerced β a column silently retyped is a lie."""
|
| 74 |
+
out, seen = [], set()
|
| 75 |
+
for f in (raw or [])[:MAX_FIELDS]:
|
| 76 |
+
if not isinstance(f, dict):
|
| 77 |
+
continue
|
| 78 |
+
label = ' '.join(str(f.get('label') or f.get('key') or '').split())[:80]
|
| 79 |
+
key = re.sub(r'[^a-z0-9_]+', '_', str(f.get('key') or _slug(label)).strip().lower())
|
| 80 |
+
key = key.strip('_')[:60]
|
| 81 |
+
ftype = str(f.get('type') or 'text').strip().lower()
|
| 82 |
+
if not key or key in seen or ftype not in UT_FIELD_TYPES or not label:
|
| 83 |
+
continue
|
| 84 |
+
seen.add(key)
|
| 85 |
+
entry = {'key': key, 'label': label, 'type': ftype, 'source': 'overlay'}
|
| 86 |
+
if ftype in ('select', 'multiselect'):
|
| 87 |
+
opts = [' '.join(str(o).split())[:60] for o in (f.get('options') or [])
|
| 88 |
+
if str(o).strip()][:50]
|
| 89 |
+
if opts:
|
| 90 |
+
entry['options'] = opts
|
| 91 |
+
if f.get('default') is True:
|
| 92 |
+
entry['default'] = True
|
| 93 |
+
out.append(entry)
|
| 94 |
+
return out or None
|
| 95 |
+
|
| 96 |
+
|
| 97 |
+
def all_tables(st=None):
|
| 98 |
+
"""{key: definition} β every user-created table in this tenant."""
|
| 99 |
+
try:
|
| 100 |
+
return dict(_st(st).get(STORE_KEY) or {})
|
| 101 |
+
except Exception:
|
| 102 |
+
return {}
|
| 103 |
+
|
| 104 |
+
|
| 105 |
+
def get(table_key, st=None):
|
| 106 |
+
return all_tables(st).get(str(table_key))
|
| 107 |
+
|
| 108 |
+
|
| 109 |
+
def is_user_table(table_key, st=None):
|
| 110 |
+
"""THE permission predicate for row writes. Cheap and total: a key that is not in this store
|
| 111 |
+
is not a user table, so it cannot accept invented rows."""
|
| 112 |
+
return str(table_key or '').startswith(KEY_PREFIX) and get(table_key, st) is not None
|
| 113 |
+
|
| 114 |
+
|
| 115 |
+
def create(label, username, fields=None, source=None, st=None):
|
| 116 |
+
"""Create a blank table. Returns its key, or None if refused.
|
| 117 |
+
|
| 118 |
+
Refuses rather than raises on the ordinary conditions (no name, duplicate, cap reached) β
|
| 119 |
+
the caller is a UI form and a refusal is a message, not an exception.
|
| 120 |
+
"""
|
| 121 |
+
label = str(label or '').strip()[:MAX_LABEL]
|
| 122 |
+
if not label:
|
| 123 |
+
return None
|
| 124 |
+
existing = all_tables(st)
|
| 125 |
+
if len(existing) >= MAX_TABLES:
|
| 126 |
+
return None
|
| 127 |
+
key = KEY_PREFIX + _slug(label)
|
| 128 |
+
if key in existing: # a second table of the same name gets a suffix
|
| 129 |
+
for n in range(2, 50):
|
| 130 |
+
if f'{key}_{n}' not in existing:
|
| 131 |
+
key = f'{key}_{n}'
|
| 132 |
+
break
|
| 133 |
+
else:
|
| 134 |
+
return None
|
| 135 |
+
# A blank table still needs ONE column to be a table at all: a row with no fields cannot be
|
| 136 |
+
# displayed, edited or identified. 'name' is the identity column, the same role `customer`
|
| 137 |
+
# plays in the Customer table.
|
| 138 |
+
defn = {
|
| 139 |
+
'key': key,
|
| 140 |
+
'label': label,
|
| 141 |
+
'source': (source if source in (BLANK_SOURCE, AUTOMATION_SOURCE) else BLANK_SOURCE),
|
| 142 |
+
'createdBy': username,
|
| 143 |
+
'created': _dt.datetime.now().strftime('%Y-%m-%dT%H:%M:%S'),
|
| 144 |
+
'fields': clean_fields(fields) or [{'key': 'name', 'label': 'Name', 'type': 'text',
|
| 145 |
+
'source': 'overlay', 'default': True}],
|
| 146 |
+
'rows': {},
|
| 147 |
+
}
|
| 148 |
+
|
| 149 |
+
def _add(cur):
|
| 150 |
+
cur[key] = defn
|
| 151 |
+
return cur
|
| 152 |
+
|
| 153 |
+
_st(st).update(STORE_KEY, _add, flush='sync') # a creation is not a hot path; commit it
|
| 154 |
+
return key
|
| 155 |
+
|
| 156 |
+
|
| 157 |
+
def delete(table_key, st=None):
|
| 158 |
+
def _drop(cur):
|
| 159 |
+
cur.pop(str(table_key), None)
|
| 160 |
+
return cur
|
| 161 |
+
_st(st).update(STORE_KEY, _drop, flush='sync')
|
| 162 |
+
|
| 163 |
+
|
| 164 |
+
def set_fields(table_key, fields, st=None):
|
| 165 |
+
"""Replace a table's base-field contract (wave 18 β the automation engine grows tables it
|
| 166 |
+
creates). Cleaned with the same validator as `create`; refuses to leave a table fieldless."""
|
| 167 |
+
clean = clean_fields(fields)
|
| 168 |
+
if not clean or not is_user_table(table_key, st):
|
| 169 |
+
return False
|
| 170 |
+
|
| 171 |
+
def _set(cur):
|
| 172 |
+
t = cur.get(str(table_key))
|
| 173 |
+
if t is not None:
|
| 174 |
+
t['fields'] = clean
|
| 175 |
+
return cur
|
| 176 |
+
|
| 177 |
+
_st(st).update(STORE_KEY, _set, flush='sync')
|
| 178 |
+
return True
|
| 179 |
+
|
| 180 |
+
|
| 181 |
+
def add_row(table_key, values=None, username=None, st=None):
|
| 182 |
+
"""Append one row. Returns the new row id, or None if refused."""
|
| 183 |
+
if not is_user_table(table_key, st):
|
| 184 |
+
return None # β never on a connector-backed table
|
| 185 |
+
defn = get(table_key, st) or {}
|
| 186 |
+
rows = defn.get('rows') or {}
|
| 187 |
+
if len(rows) >= MAX_ROWS:
|
| 188 |
+
return None
|
| 189 |
+
rid = str(max((int(r) for r in rows if str(r).isdigit()), default=0) + 1)
|
| 190 |
+
valid = {f['key'] for f in (defn.get('fields') or [])}
|
| 191 |
+
clean = {k: str(v) for k, v in (values or {}).items() if k in valid}
|
| 192 |
+
|
| 193 |
+
def _add(cur):
|
| 194 |
+
t = cur.get(str(table_key))
|
| 195 |
+
if t is not None:
|
| 196 |
+
t.setdefault('rows', {})[rid] = clean
|
| 197 |
+
return cur
|
| 198 |
+
|
| 199 |
+
_st(st).update(STORE_KEY, _add, flush='async')
|
| 200 |
+
return rid
|
| 201 |
+
|
| 202 |
+
|
| 203 |
+
def upsert_rows(table_key, rows_by_key, key_field, username=None, st=None):
|
| 204 |
+
"""Bulk UPSERT for the automation engine (wave 18 C4-AUTO): match on `key_field`'s value,
|
| 205 |
+
update matched rows, insert the rest. Returns `{'updated': n, 'inserted': n, 'orphans': n}`
|
| 206 |
+
where orphans = existing rows whose key value did not appear this run β COUNTED, never
|
| 207 |
+
deleted (the contract's no-silent-delete rule). Refuses on a non-user table."""
|
| 208 |
+
if not is_user_table(table_key, st):
|
| 209 |
+
return None
|
| 210 |
+
defn = get(table_key, st) or {}
|
| 211 |
+
valid = {f['key'] for f in (defn.get('fields') or [])}
|
| 212 |
+
if key_field not in valid:
|
| 213 |
+
return None
|
| 214 |
+
incoming = {}
|
| 215 |
+
for kv, vals in (rows_by_key or {}).items():
|
| 216 |
+
kv = str(kv).strip()
|
| 217 |
+
if kv:
|
| 218 |
+
incoming[kv] = {k: str(v) for k, v in (vals or {}).items() if k in valid}
|
| 219 |
+
counts = {'updated': 0, 'inserted': 0, 'orphans': 0}
|
| 220 |
+
|
| 221 |
+
def _apply(cur):
|
| 222 |
+
t = cur.get(str(table_key))
|
| 223 |
+
if t is None:
|
| 224 |
+
return cur
|
| 225 |
+
rows = t.setdefault('rows', {})
|
| 226 |
+
by_val = {}
|
| 227 |
+
for rid, row in rows.items():
|
| 228 |
+
v = str((row or {}).get(key_field) or '').strip()
|
| 229 |
+
if v and v not in by_val:
|
| 230 |
+
by_val[v] = rid
|
| 231 |
+
next_id = max((int(r) for r in rows if str(r).isdigit()), default=0)
|
| 232 |
+
for kv, vals in incoming.items():
|
| 233 |
+
rid = by_val.get(kv)
|
| 234 |
+
if rid is not None:
|
| 235 |
+
rows[rid].update(vals)
|
| 236 |
+
counts['updated'] += 1
|
| 237 |
+
elif len(rows) < MAX_ROWS:
|
| 238 |
+
next_id += 1
|
| 239 |
+
rows[str(next_id)] = vals
|
| 240 |
+
counts['inserted'] += 1
|
| 241 |
+
counts['orphans'] = sum(1 for v in by_val if v not in incoming)
|
| 242 |
+
return cur
|
| 243 |
+
|
| 244 |
+
_st(st).update(STORE_KEY, _apply, flush='sync')
|
| 245 |
+
return counts
|
| 246 |
+
|
| 247 |
+
|
| 248 |
+
def delete_row(table_key, row_id, st=None):
|
| 249 |
+
if not is_user_table(table_key, st):
|
| 250 |
+
return False
|
| 251 |
+
|
| 252 |
+
def _drop(cur):
|
| 253 |
+
t = cur.get(str(table_key))
|
| 254 |
+
if t is not None:
|
| 255 |
+
(t.get('rows') or {}).pop(str(row_id), None)
|
| 256 |
+
return cur
|
| 257 |
+
|
| 258 |
+
_st(st).update(STORE_KEY, _drop, flush='async')
|
| 259 |
+
return True
|
| 260 |
+
|
| 261 |
+
|
| 262 |
+
def may_open(table_key, viewer, is_admin=False, st=None):
|
| 263 |
+
"""FAIL-CLOSED visibility: the creator, or an admin. Nothing else.
|
| 264 |
+
|
| 265 |
+
A user table cannot be gated by `allowed_modules` β a module grant written last month cannot
|
| 266 |
+
describe a table created this morning β so it needs its own rule, and the safe rule is the
|
| 267 |
+
narrow one. Tables are stored tenant-wide under one key, so without this ANY user (including
|
| 268 |
+
a BU-scoped sales agent) could open a table somebody else created just by knowing its key.
|
| 269 |
+
Sharing a user table with named colleagues is a follow-on, and it should reuse the shared-
|
| 270 |
+
VIEW vocabulary rather than inventing a second one.
|
| 271 |
+
"""
|
| 272 |
+
t = get(table_key, st)
|
| 273 |
+
if not t:
|
| 274 |
+
return False
|
| 275 |
+
return bool(is_admin) or t.get('createdBy') == viewer
|
| 276 |
+
|
| 277 |
+
|
| 278 |
+
def nav_entries(viewer=None, is_admin=False, st=None):
|
| 279 |
+
"""Registry-SHAPED dicts so the nav can render user tables beside real modules with no
|
| 280 |
+
special-casing: the same keys the flat nav reads (`key`, `label`, `source`).
|
| 281 |
+
|
| 282 |
+
Filtered by `may_open`, so the nav cannot offer a row the dispatcher would refuse.
|
| 283 |
+
"""
|
| 284 |
+
out = []
|
| 285 |
+
for key, t in sorted(all_tables(st).items(),
|
| 286 |
+
key=lambda kv: (kv[1].get('label') or '').lower()):
|
| 287 |
+
if viewer is not None and not may_open(key, viewer, is_admin, st=st):
|
| 288 |
+
continue
|
| 289 |
+
out.append({'key': key, 'label': t.get('label') or key,
|
| 290 |
+
'source': t.get('source') or BLANK_SOURCE,
|
| 291 |
+
'user_table': True, 'nav': True, 'validate': False})
|
| 292 |
+
return out
|
platform/core/users.py
ADDED
|
@@ -0,0 +1,361 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Per-user accounts for the platform, persisted in the HF Dataset store (users.json).
|
| 2 |
+
|
| 3 |
+
Passwords are salted + PBKDF2-HMAC-SHA256 (200k iterations) β never stored or logged in plaintext.
|
| 4 |
+
A bootstrap 'admin' account is seeded from APP_PASSWORD so the owner can always log in and create
|
| 5 |
+
users; APP_PASSWORD also works as an emergency master for 'admin' if the registry is unreachable.
|
| 6 |
+
|
| 7 |
+
Each account carries BU access ('all' or a list of team-ids [5=Fisch, 6=Royal]) which drives
|
| 8 |
+
allowed_bus() β the basis for per-Business-Unit permissioning (a Royal-only user never sees Fisch).
|
| 9 |
+
"""
|
| 10 |
+
import os
|
| 11 |
+
import hmac
|
| 12 |
+
import hashlib
|
| 13 |
+
import secrets
|
| 14 |
+
|
| 15 |
+
import core.store as store
|
| 16 |
+
|
| 17 |
+
BU_LABELS = {5: 'Fisch', 6: 'Royal'}
|
| 18 |
+
_ITER = 200_000
|
| 19 |
+
|
| 20 |
+
#: Wave 15 C-PERM β the explicit-resolution marker. Mirrors `core.perm_scope.PERMS_VERSION`;
|
| 21 |
+
#: kept as a literal here so `users` does not import the permission layer it is read by.
|
| 22 |
+
PERMS_VERSION = 1
|
| 23 |
+
|
| 24 |
+
|
| 25 |
+
def _hash(pw, salt):
|
| 26 |
+
return hashlib.pbkdf2_hmac('sha256', str(pw).encode('utf-8'), bytes.fromhex(salt), _ITER).hex()
|
| 27 |
+
|
| 28 |
+
|
| 29 |
+
def _record(pw, name, role, bus, active=True, modules='all', agent=None, email=None,
|
| 30 |
+
perms=None, tenant='royal-imports'):
|
| 31 |
+
salt = secrets.token_hex(16)
|
| 32 |
+
rec = {'salt': salt, 'hash': _hash(pw, salt), 'name': name, 'role': role,
|
| 33 |
+
'bus': bus, 'active': active, 'modules': modules,
|
| 34 |
+
'agent': agent or None, 'email': email or None,
|
| 35 |
+
# Wave 18 (C1-TENANT, R1): the account's COMPANY. Absent == 'royal-imports' on every
|
| 36 |
+
# pre-wave record β no migration. Login binds the session to THIS value; the posted
|
| 37 |
+
# tenant field can hint but never override it.
|
| 38 |
+
'tenant': str(tenant or 'royal-imports').strip().lower()}
|
| 39 |
+
if perms is not None:
|
| 40 |
+
# Wave 15 C-PERM. A record written WITH perms is migrated by construction β the marker
|
| 41 |
+
# and the block are set together, here, so no writer can create one without the other.
|
| 42 |
+
# (`core.perm_scope` reads an unmarked record as legacy, so a block without its marker
|
| 43 |
+
# would be silently ignored; a marker without a block would deny everything.)
|
| 44 |
+
rec['perms'] = perms
|
| 45 |
+
rec['perms_v'] = PERMS_VERSION
|
| 46 |
+
return rec
|
| 47 |
+
|
| 48 |
+
|
| 49 |
+
def registry():
|
| 50 |
+
return store.get('users')
|
| 51 |
+
|
| 52 |
+
|
| 53 |
+
def ensure_bootstrap():
|
| 54 |
+
"""Seed an 'admin' account from APP_PASSWORD ONLY on a truly fresh store (no users file yet).
|
| 55 |
+
Idempotent; no-op if the store is unavailable (the app then falls back to the master-password
|
| 56 |
+
path in verify()).
|
| 57 |
+
|
| 58 |
+
Critically, this NEVER overwrites an existing registry: it seeds only when store.exists('users')
|
| 59 |
+
is definitively False. A transient read failure at startup used to return {} and make this
|
| 60 |
+
re-seed just {admin} over the real accounts β that is the bug that wiped users on restart."""
|
| 61 |
+
if not store.available():
|
| 62 |
+
return
|
| 63 |
+
if store.exists('users'): # present, or uncertain -> never clobber
|
| 64 |
+
return
|
| 65 |
+
try:
|
| 66 |
+
reg = store.get('users', fresh=True)
|
| 67 |
+
except Exception:
|
| 68 |
+
return
|
| 69 |
+
if reg:
|
| 70 |
+
return
|
| 71 |
+
master = os.environ.get('APP_PASSWORD', '')
|
| 72 |
+
if not master:
|
| 73 |
+
return
|
| 74 |
+
try:
|
| 75 |
+
store.put('users', {'admin': _record(master, 'Administrator', 'admin', 'all')})
|
| 76 |
+
except Exception:
|
| 77 |
+
pass
|
| 78 |
+
|
| 79 |
+
|
| 80 |
+
def _public(username, u):
|
| 81 |
+
return {'username': username, 'name': u.get('name', username),
|
| 82 |
+
'role': u.get('role', 'user'), 'bus': u.get('bus', 'all'),
|
| 83 |
+
'modules': u.get('modules', 'all'),
|
| 84 |
+
'agent': u.get('agent'), 'email': u.get('email'),
|
| 85 |
+
# Wave 18 (C1-TENANT): the session's tenant binding travels on the projection or it
|
| 86 |
+
# does not travel β the same rule the perms block states below.
|
| 87 |
+
'tenant': str(u.get('tenant') or 'royal-imports').strip().lower(),
|
| 88 |
+
# Wave 14 C-AVATAR: the profile photo is public-safe by definition (it is served
|
| 89 |
+
# to every grid session via the workspace map); without it here the API session's
|
| 90 |
+
# user record silently drops it and /me can never show your own photo.
|
| 91 |
+
'avatar': u.get('avatar') or None,
|
| 92 |
+
# β WAVE 15 C-PERM β THE WALL TRAVELS ON THIS PROJECTION OR IT DOES NOT TRAVEL.
|
| 93 |
+
# `deps._user_for` builds every API session from `_public()`, so a `perms` block
|
| 94 |
+
# dropped here is a restricted account served as an unrestricted one β silently, on
|
| 95 |
+
# every route, with nothing to notice. `perms_v` must ride ALONG WITH it and for the
|
| 96 |
+
# same reason inverted: the marker without the block denies everything, the block
|
| 97 |
+
# without the marker is ignored. Two keys, one fact, never separated.
|
| 98 |
+
# `verify_api` asserts a restricted user's SESSION OBJECT carries both, at the mount
|
| 99 |
+
# rather than by grep β a projection is exactly the kind of wiring that looks
|
| 100 |
+
# present in three files and is absent in the one that runs.
|
| 101 |
+
**({'perms': u['perms']} if isinstance(u.get('perms'), dict) else {}),
|
| 102 |
+
**({'perms_v': int(u['perms_v'] or 0)} if u.get('perms_v') else {}),
|
| 103 |
+
'epoch': int(u.get('epoch') or 0)}
|
| 104 |
+
|
| 105 |
+
|
| 106 |
+
# ------------------------------------------------------------------ session revocation (X3)
|
| 107 |
+
# The API's session cookie is SIGNED AND STATELESS: there is no server-side session table to
|
| 108 |
+
# delete from, so "log this user out everywhere" needs a number that lives with the account. The
|
| 109 |
+
# cookie carries the epoch it was minted under; bumping the account's epoch makes every
|
| 110 |
+
# outstanding cookie for that user fail verification on its next use. Absent == 0, so every
|
| 111 |
+
# record written before this wave is valid without a migration.
|
| 112 |
+
def epoch(username):
|
| 113 |
+
"""The current session epoch for `username`. None when there is no such account.
|
| 114 |
+
|
| 115 |
+
None is NOT 0. 0 is "this account exists and has never been revoked"; None is "no record" β
|
| 116 |
+
which the session verifier must treat as a reason to refuse, not as a default to compare
|
| 117 |
+
against. (The APP_PASSWORD emergency-master admin has no record at all; the verifier handles
|
| 118 |
+
that case explicitly rather than inventing an epoch for it here.)
|
| 119 |
+
"""
|
| 120 |
+
username = (username or '').strip().lower()
|
| 121 |
+
try:
|
| 122 |
+
u = (store.get('users') or {}).get(username)
|
| 123 |
+
except Exception:
|
| 124 |
+
return None
|
| 125 |
+
return int((u or {}).get('epoch') or 0) if u else None
|
| 126 |
+
|
| 127 |
+
|
| 128 |
+
def bump_epoch(username):
|
| 129 |
+
"""Revoke every outstanding API session for this account."""
|
| 130 |
+
username = (username or '').strip().lower()
|
| 131 |
+
|
| 132 |
+
def _set(reg):
|
| 133 |
+
u = reg.get(username)
|
| 134 |
+
if u:
|
| 135 |
+
u['epoch'] = int(u.get('epoch') or 0) + 1
|
| 136 |
+
return reg
|
| 137 |
+
store.update('users', _set)
|
| 138 |
+
|
| 139 |
+
|
| 140 |
+
def verify(username, pw):
|
| 141 |
+
"""Return a public user dict on success, else None. APP_PASSWORD is an emergency master for the
|
| 142 |
+
'admin' login even if the store is unreachable, so the owner is never locked out."""
|
| 143 |
+
username = (username or '').strip().lower()
|
| 144 |
+
if not username or not pw:
|
| 145 |
+
return None
|
| 146 |
+
master = os.environ.get('APP_PASSWORD', '')
|
| 147 |
+
try:
|
| 148 |
+
# read fresh so accounts created moments ago (UI or out-of-band) are recognised at once
|
| 149 |
+
reg = store.get('users', fresh=True)
|
| 150 |
+
except Exception:
|
| 151 |
+
reg = {}
|
| 152 |
+
u = reg.get(username)
|
| 153 |
+
if u is None and '@' in username:
|
| 154 |
+
# Wave 18 (R1): the login box takes a username OR an email β admin@nurilab.id signs in
|
| 155 |
+
# without knowing the slug an admin chose. First case-insensitive email match wins;
|
| 156 |
+
# ambiguity is an admin data problem, not a login feature.
|
| 157 |
+
for k, r in reg.items():
|
| 158 |
+
if isinstance(r, dict) and str(r.get('email') or '').strip().lower() == username:
|
| 159 |
+
username, u = k, r
|
| 160 |
+
break
|
| 161 |
+
if u and u.get('active', True) and hmac.compare_digest(_hash(pw, u['salt']), u['hash']):
|
| 162 |
+
return _public(username, u)
|
| 163 |
+
# emergency master: admin + APP_PASSWORD always works (covers first run / store outage)
|
| 164 |
+
if username == 'admin' and master and hmac.compare_digest(str(pw), master):
|
| 165 |
+
# β CARRY THE RECORD'S CURRENT EPOCH when there is a record to read, so the session cookie
|
| 166 |
+
# the API mints from this dict AGREES with the stored account.
|
| 167 |
+
#
|
| 168 |
+
# This is not what stops the emergency lockout β `deps._user_for`'s master fallback does
|
| 169 |
+
# that, and a negative control confirmed the lockout is gone with or without this line.
|
| 170 |
+
# What it fixes is subtler and is a SCOPE question: a cookie whose epoch disagrees with the
|
| 171 |
+
# record falls through to that master fallback, which hands back a SYNTHETIC identity
|
| 172 |
+
# (`bus: 'all'`, `modules: 'all'`). An admin whose record narrows either field would
|
| 173 |
+
# therefore be silently WIDENED to consolidated, all-module access for the life of that
|
| 174 |
+
# session. Matching the epoch means the record branch wins and the account's real scope
|
| 175 |
+
# applies, leaving the master fallback as the true last resort it is meant to be.
|
| 176 |
+
# 0 when the store is unreachable, which is the case this branch was written for.
|
| 177 |
+
return {'username': 'admin', 'name': 'Administrator', 'role': 'admin', 'bus': 'all',
|
| 178 |
+
'modules': 'all', 'epoch': int((reg.get('admin') or {}).get('epoch') or 0)}
|
| 179 |
+
return None
|
| 180 |
+
|
| 181 |
+
|
| 182 |
+
def create_user(username, pw, name, role='user', bus='all', modules='all',
|
| 183 |
+
agent=None, email=None, tenant=None):
|
| 184 |
+
"""Create β or, from app.py's dialog, OVERWRITE β an account.
|
| 185 |
+
|
| 186 |
+
β X3: OVERWRITING AN ACCOUNT MUST NOT RESURRECT ITS OLD SESSIONS. `_record()` builds a fresh
|
| 187 |
+
record with no `epoch` key, i.e. absent == 0. So re-saving an existing username used to reset
|
| 188 |
+
the epoch to 0, and every cookie minted before that account's last password rotation started
|
| 189 |
+
verifying again β a silent un-revocation. `app.py`'s "Add / update a user" calls this function
|
| 190 |
+
for BOTH add and update, so the hole was reachable from the shipped UI.
|
| 191 |
+
|
| 192 |
+
Epoch revocation is only ever as strong as the NARROWEST write path that touches the record, so
|
| 193 |
+
the carry-and-bump lives here rather than in each caller: an overwrite is at least as
|
| 194 |
+
session-invalidating as a password change, and it usually IS one.
|
| 195 |
+
"""
|
| 196 |
+
username = (username or '').strip().lower()
|
| 197 |
+
if not username or not pw:
|
| 198 |
+
raise ValueError('username and password are required')
|
| 199 |
+
|
| 200 |
+
def _add(reg):
|
| 201 |
+
prior = reg.get(username) or {}
|
| 202 |
+
rec = _record(pw, name or username, role, bus, modules=modules,
|
| 203 |
+
agent=agent, email=email,
|
| 204 |
+
# An overwrite that names no tenant KEEPS the account's company β a
|
| 205 |
+
# rename must never quietly move a user between tenants.
|
| 206 |
+
tenant=(tenant or prior.get('tenant') or 'royal-imports'))
|
| 207 |
+
if prior:
|
| 208 |
+
rec['epoch'] = int(prior.get('epoch') or 0) + 1
|
| 209 |
+
reg[username] = rec
|
| 210 |
+
return reg
|
| 211 |
+
store.update('users', _add)
|
| 212 |
+
|
| 213 |
+
|
| 214 |
+
def set_password(username, pw):
|
| 215 |
+
username = (username or '').strip().lower()
|
| 216 |
+
|
| 217 |
+
def _set(reg):
|
| 218 |
+
u = reg.get(username)
|
| 219 |
+
if u:
|
| 220 |
+
u['salt'] = secrets.token_hex(16)
|
| 221 |
+
u['hash'] = _hash(pw, u['salt'])
|
| 222 |
+
# X3: a password change revokes every outstanding API session for the account. Bumped
|
| 223 |
+
# INSIDE the same read-modify-write as the hash so the two can never disagree β a
|
| 224 |
+
# separate update() could rotate the password and leave old cookies live if the second
|
| 225 |
+
# write failed.
|
| 226 |
+
u['epoch'] = int(u.get('epoch') or 0) + 1
|
| 227 |
+
return reg
|
| 228 |
+
store.update('users', _set)
|
| 229 |
+
|
| 230 |
+
|
| 231 |
+
def set_active(username, active):
|
| 232 |
+
username = (username or '').strip().lower()
|
| 233 |
+
|
| 234 |
+
def _set(reg):
|
| 235 |
+
if username in reg:
|
| 236 |
+
reg[username]['active'] = bool(active)
|
| 237 |
+
# X3: DEACTIVATION must kill live sessions, not just future logins β otherwise a
|
| 238 |
+
# disabled account keeps working until its cookie expires. Bumped on reactivation too:
|
| 239 |
+
# cheap, and it means a re-enabled account never resurrects a stale cookie.
|
| 240 |
+
reg[username]['epoch'] = int(reg[username].get('epoch') or 0) + 1
|
| 241 |
+
return reg
|
| 242 |
+
store.update('users', _set)
|
| 243 |
+
|
| 244 |
+
|
| 245 |
+
def set_access(username, role=None, bus=None, modules=None, agent=None, email=None,
|
| 246 |
+
name=None, perms=None):
|
| 247 |
+
"""Update access fields. agent/email: pass '' to clear, None to leave unchanged β
|
| 248 |
+
the userβagent link scopes the Customer List page / digests to that agent's book.
|
| 249 |
+
|
| 250 |
+
`name` follows the same None-means-unchanged idiom. It is here because it had no setter at all:
|
| 251 |
+
a display name could previously only be changed by re-creating the record through
|
| 252 |
+
`create_user`, i.e. by also resetting the password (and, before the fix above, the session
|
| 253 |
+
epoch). Y4's `PATCH {name?}` needs the narrow write, not the destructive one."""
|
| 254 |
+
username = (username or '').strip().lower()
|
| 255 |
+
|
| 256 |
+
def _set(reg):
|
| 257 |
+
u = reg.get(username)
|
| 258 |
+
if u:
|
| 259 |
+
if name is not None:
|
| 260 |
+
u['name'] = name
|
| 261 |
+
if role is not None:
|
| 262 |
+
u['role'] = role
|
| 263 |
+
if bus is not None:
|
| 264 |
+
u['bus'] = bus
|
| 265 |
+
if modules is not None:
|
| 266 |
+
u['modules'] = modules
|
| 267 |
+
if agent is not None:
|
| 268 |
+
u['agent'] = agent or None
|
| 269 |
+
if email is not None:
|
| 270 |
+
u['email'] = email or None
|
| 271 |
+
if perms is not None:
|
| 272 |
+
# Wave 15 C-PERM. Writing perms MIGRATES the record: the marker goes on in the
|
| 273 |
+
# same read-modify-write, so a record can never end up with one and not the
|
| 274 |
+
# other (see `_record`). Whole-block replace, matching the PUT route's shape β
|
| 275 |
+
# a merge would make "remove this restriction" unexpressible.
|
| 276 |
+
u['perms'] = perms
|
| 277 |
+
u['perms_v'] = PERMS_VERSION
|
| 278 |
+
return reg
|
| 279 |
+
store.update('users', _set)
|
| 280 |
+
|
| 281 |
+
|
| 282 |
+
def allowed_bus_labels(user):
|
| 283 |
+
"""BU labels this user may select. 'all' -> All+Fisch+Royal; a single BU -> just that BU (no
|
| 284 |
+
'All', so the other BU is never reachable); multiple -> All + each."""
|
| 285 |
+
bus = (user or {}).get('bus', 'all')
|
| 286 |
+
if bus == 'all':
|
| 287 |
+
return ['All', 'Fisch', 'Royal']
|
| 288 |
+
labels = [BU_LABELS[b] for b in bus if b in BU_LABELS]
|
| 289 |
+
if not labels:
|
| 290 |
+
return ['All', 'Fisch', 'Royal']
|
| 291 |
+
return (['All'] + labels) if len(labels) > 1 else labels
|
| 292 |
+
|
| 293 |
+
|
| 294 |
+
def assignable_people(tenant=None):
|
| 295 |
+
"""Display names for `user`-typed overlay columns β the tenant's ACTIVE accounts.
|
| 296 |
+
|
| 297 |
+
Moved from app.py (2026-07-31) so both hosts serve the same choices. Resolved on every
|
| 298 |
+
call rather than persisted with the column: a snapshot would keep offering people who
|
| 299 |
+
have left and never offer people who joined. Deactivated accounts are excluded; a value
|
| 300 |
+
already stored on a row is untouched β history should still say who owned something.
|
| 301 |
+
|
| 302 |
+
Wave 18 (C1-TENANT): pass `tenant` to scope the choices to ONE company β the user registry
|
| 303 |
+
is a global control-plane bucket, and a Nurilab picker offering Royal's staff is a
|
| 304 |
+
cross-tenant name leak. None = unscoped (the Streamlit host, tenant #0's process).
|
| 305 |
+
"""
|
| 306 |
+
try:
|
| 307 |
+
reg = registry() or {}
|
| 308 |
+
except Exception:
|
| 309 |
+
return []
|
| 310 |
+
want = str(tenant or '').strip().lower()
|
| 311 |
+
out = []
|
| 312 |
+
for username, u in reg.items():
|
| 313 |
+
if not isinstance(u, dict) or u.get('active') is False:
|
| 314 |
+
continue
|
| 315 |
+
if want and str(u.get('tenant') or 'royal-imports').strip().lower() != want:
|
| 316 |
+
continue
|
| 317 |
+
out.append(str(u.get('name') or username))
|
| 318 |
+
return sorted(set(out))
|
| 319 |
+
|
| 320 |
+
|
| 321 |
+
def set_avatar(username, data_url):
|
| 322 |
+
"""Set (or clear, with None/'') the user's profile photo β a data URL (wave 14 C-AVATAR,
|
| 323 |
+
[[loopable-wave14-split]] item 11). Stored VERBATIM; the API route owns validation (mime +
|
| 324 |
+
decoded size) because this value is served back to every grid session. Cleared by removing
|
| 325 |
+
the key, so records without a photo keep their pre-wave shape."""
|
| 326 |
+
username = (username or '').strip().lower()
|
| 327 |
+
|
| 328 |
+
def _set(reg):
|
| 329 |
+
u = reg.get(username)
|
| 330 |
+
if u:
|
| 331 |
+
if data_url:
|
| 332 |
+
u['avatar'] = str(data_url)
|
| 333 |
+
else:
|
| 334 |
+
u.pop('avatar', None)
|
| 335 |
+
return reg
|
| 336 |
+
store.update('users', _set)
|
| 337 |
+
|
| 338 |
+
|
| 339 |
+
def avatar_map(tenant=None):
|
| 340 |
+
"""{display name -> avatar data URL} for ACTIVE accounts with a photo β the companion of
|
| 341 |
+
`assignable_people()`, keyed by the SAME vocabulary: a `user` cell stores the display
|
| 342 |
+
name, so the display name is the only join a renderer has. Two active accounts sharing a
|
| 343 |
+
display name share one option; the first WITH a photo wins the key rather than a coin
|
| 344 |
+
flip deciding whether the option has a face. `tenant` scopes it exactly as
|
| 345 |
+
`assignable_people(tenant)` does, and for the same leak."""
|
| 346 |
+
try:
|
| 347 |
+
reg = registry() or {}
|
| 348 |
+
except Exception:
|
| 349 |
+
return {}
|
| 350 |
+
want = str(tenant or '').strip().lower()
|
| 351 |
+
out = {}
|
| 352 |
+
for username, u in sorted(reg.items()):
|
| 353 |
+
if not isinstance(u, dict) or u.get('active') is False:
|
| 354 |
+
continue
|
| 355 |
+
if want and str(u.get('tenant') or 'royal-imports').strip().lower() != want:
|
| 356 |
+
continue
|
| 357 |
+
av = u.get('avatar')
|
| 358 |
+
nm = str(u.get('name') or username)
|
| 359 |
+
if av and nm not in out:
|
| 360 |
+
out[nm] = str(av)
|
| 361 |
+
return out
|
platform/evals/analyst_golden.yml
ADDED
|
@@ -0,0 +1,104 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Analyst golden eval set v1 (OM-4 gate, 2026-07-11; truths REFRESHED 2026-07-17 after the
|
| 2 |
+
# Jul-15 Odoo mass-recompute + hard-delete reconciliation β closed windows DO drift when docs
|
| 3 |
+
# are edited/deleted; refresh truths from the parity-proven layer, never loosen tolerances).
|
| 4 |
+
# Original header: truths computed from the PARITY-PROVEN
|
| 5 |
+
# semantic layer over CLOSED windows (H1 2026 = 2026-01-01..2026-06-30) so live data drift cannot
|
| 6 |
+
# break them. Checks are DETERMINISTIC (number within tolerance + required substrings + tool-use
|
| 7 |
+
# assertions) β the validate() culture applied to AI answers. Run: harness/evals.py.
|
| 8 |
+
# expect.value tolerance: abs or rel (fraction). expect.contains: ALL substrings (case-insensitive).
|
| 9 |
+
# expect.tools: tool names that MUST appear in the trace (the answer must come FROM the tools).
|
| 10 |
+
# expect.not_tools: tool names that must NOT appear (the over-refusal guard β an answerable ask
|
| 11 |
+
# must never end in report_gap; a gap ask must never end in a forced wrong query).
|
| 12 |
+
items:
|
| 13 |
+
- id: rev_h1
|
| 14 |
+
ask: "What was our wholesale revenue for the first half of 2026 (Jan 1 through Jun 30)?"
|
| 15 |
+
expect: {value: 3501508.33, rel: 0.001, tools: [run_semantic_query]}
|
| 16 |
+
- id: rev_h1_fisch
|
| 17 |
+
ask: "Fisch revenue only, H1 2026 (Jan 1 - Jun 30)?"
|
| 18 |
+
expect: {value: 2494244.70, rel: 0.001, tools: [run_semantic_query]}
|
| 19 |
+
- id: rev_h1_royal
|
| 20 |
+
ask: "And Royal's revenue for the same H1 2026 window?"
|
| 21 |
+
expect: {value: 1007263.63, rel: 0.001, tools: [run_semantic_query]}
|
| 22 |
+
- id: rev_june
|
| 23 |
+
ask: "Revenue for June 2026?"
|
| 24 |
+
expect: {value: 535046.06, rel: 0.001, tools: [run_semantic_query]}
|
| 25 |
+
- id: margin_h1
|
| 26 |
+
ask: "Gross margin dollars for H1 2026 (Jan-Jun), wholesale?"
|
| 27 |
+
expect: {value: 2067509.45, rel: 0.001, tools: [run_semantic_query]}
|
| 28 |
+
- id: margin_pct_h1
|
| 29 |
+
ask: "What gross margin percentage did we run in H1 2026?"
|
| 30 |
+
expect: {value: 0.59, abs: 0.005, allow_pct_form: true, tools: [run_semantic_query]}
|
| 31 |
+
- id: orders_h1
|
| 32 |
+
ask: "How many confirmed wholesale orders did we take in H1 2026 (Jan 1 - Jun 30)?"
|
| 33 |
+
expect: {value: 4989, abs: 1, tools: [run_semantic_query]}
|
| 34 |
+
- id: customers_h1
|
| 35 |
+
ask: "How many distinct customers bought from us in H1 2026?"
|
| 36 |
+
expect: {value: 1037, abs: 1, tools: [run_semantic_query]}
|
| 37 |
+
- id: top_customer
|
| 38 |
+
ask: "Who was our biggest customer by revenue in H1 2026?"
|
| 39 |
+
expect: {contains: ["Poppy Flowers"], tools: [run_semantic_query], not_tools: [report_gap]}
|
| 40 |
+
- id: opex_h1
|
| 41 |
+
ask: "Total operating expense on the GL for H1 2026 (Jan 1 - Jun 30)?"
|
| 42 |
+
expect: {value: 4683428.81, rel: 0.001, tools: [run_semantic_query]}
|
| 43 |
+
- id: top_expense
|
| 44 |
+
ask: "What is our single largest expense account in H1 2026?"
|
| 45 |
+
expect: {contains: ["FBA"], tools: [run_semantic_query]}
|
| 46 |
+
- id: chart_request
|
| 47 |
+
ask: "Make me a line chart of monthly wholesale revenue for H1 2026."
|
| 48 |
+
expect: {artifact: chart, tools: [run_semantic_query, make_chart]}
|
| 49 |
+
- id: bu_guard
|
| 50 |
+
ask: "Show one single revenue line combining Fisch and Royal together per month for H1 2026."
|
| 51 |
+
expect: {tools: [run_semantic_query]} # combined-total ok; the guard is it must NOT error
|
| 52 |
+
- id: name_resolution
|
| 53 |
+
ask: "How much revenue did the customer 'flower' β I don't remember the exact name, something with FLOWERS in it β bring in H1 2026? Pick the top match."
|
| 54 |
+
expect: {tools: [get_field_values, run_semantic_query]}
|
| 55 |
+
- id: map_request
|
| 56 |
+
ask: "Put my top 20 customers by H1 2026 revenue on a map."
|
| 57 |
+
expect: {artifact: chart, tools: [run_semantic_query, make_chart]}
|
| 58 |
+
# --- the gap loop + location/YoY competence (added 2026-07-16 with the debug fixes) ---
|
| 59 |
+
- id: city_filter
|
| 60 |
+
ask: "Which customer in Brooklyn brought us the most revenue in H1 2026 (Jan 1 - Jun 30)?"
|
| 61 |
+
expect: {value: 48785.00, rel: 0.001, contains: ["J KAY"],
|
| 62 |
+
tools: [get_field_values, run_semantic_query], not_tools: [report_gap]}
|
| 63 |
+
- id: yoy_decliners
|
| 64 |
+
ask: "Which Fisch customer declined the most in revenue in H1 2026 (Jan 1 - Jun 30) versus the same window of 2025? Give the dollar decline."
|
| 65 |
+
expect: {value: 16451.25, rel: 0.005, contains: ["JUNE"],
|
| 66 |
+
tools: [run_semantic_query, transform_result], not_tools: [report_gap]}
|
| 67 |
+
# --- the agent dimension (added 2026-07-17: res.partner.agent_ids m2m sync β the #1 logged
|
| 68 |
+
# gap, closed). agent_book is VERBATIM the old gap_refusal ask: yesterday's honest refusal
|
| 69 |
+
# must become today's answer.
|
| 70 |
+
- id: agent_book
|
| 71 |
+
ask: "Who are the top customers handled by our salesperson Naomi in H1 2026?"
|
| 72 |
+
expect: {contains: ["Poppy Flowers"], tools: [get_field_values, run_semantic_query],
|
| 73 |
+
not_tools: [report_gap]}
|
| 74 |
+
- id: agent_revenue
|
| 75 |
+
ask: "How much revenue did agent Naomi's book of customers bring in H1 2026 (Jan 1 - Jun 30)?"
|
| 76 |
+
expect: {value: 872869.98, rel: 0.005, tools: [run_semantic_query], not_tools: [report_gap]}
|
| 77 |
+
# --- the TWO agent sources (added 2026-07-28). Odoo holds a customer-master book AND a
|
| 78 |
+
# per-invoice-line commission agent; they name different people, and the commission table also
|
| 79 |
+
# carries internal SALESPEOPLE who are not agents. These three lock the distinction: the
|
| 80 |
+
# unallocated bucket must be answerable at all, and Anna/Shantal must never be called agents.
|
| 81 |
+
- id: agent_unallocated
|
| 82 |
+
ask: "Using the invoice-line basis, how much of Fisch's 2025 sales was NOT allocated to an agent?"
|
| 83 |
+
expect: {value: 1447579.59, rel: 0.01, tools: [run_semantic_query], not_tools: [report_gap]}
|
| 84 |
+
- id: agent_not_salesperson
|
| 85 |
+
ask: "Who are our agents on the Fisch invoice lines in 2025, and what did each generate?"
|
| 86 |
+
expect: {contains: ["Pasternak"], not_contains: ["Anna", "Shantal"],
|
| 87 |
+
tools: [run_semantic_query], not_tools: [report_gap]}
|
| 88 |
+
- id: agent_allocated_2025
|
| 89 |
+
ask: "On the invoice-line basis, how much of Fisch's 2025 sales was allocated to a real agent?"
|
| 90 |
+
expect: {value: 2995664.75, rel: 0.01, tools: [run_semantic_query], not_tools: [report_gap]}
|
| 91 |
+
- id: gap_refusal
|
| 92 |
+
ask: "How many units of our best-selling SKU do we physically have in stock right now?"
|
| 93 |
+
expect: {tools: [report_gap]} # stock-on-hand has NO topic in the store β must self-report, never guess
|
| 94 |
+
# --- the analytics transform library (added 2026-07-18: 60-op Tableau-class set). Truths from
|
| 95 |
+
# the parity-proven semantic layer over the CLOSED H1 2026 window.
|
| 96 |
+
- id: abc_a_tier
|
| 97 |
+
ask: "Using an ABC / Pareto analysis of H1 2026 revenue (Jan 1 - Jun 30), how many customers fall in the A tier β the vital few that make up the first 80% of revenue?"
|
| 98 |
+
expect: {value: 311, abs: 6, tools: [run_semantic_query, transform_result], not_tools: [report_gap]}
|
| 99 |
+
- id: ytd_customers
|
| 100 |
+
ask: "How many distinct customers had we sold to cumulatively from Jan 1 through the end of March 2026?"
|
| 101 |
+
expect: {value: 790, abs: 1, tools: [run_semantic_query], not_tools: [report_gap]}
|
| 102 |
+
- id: concentration_top5
|
| 103 |
+
ask: "What percent of our H1 2026 revenue (Jan 1 - Jun 30) is accounted for by our top 5 customers?"
|
| 104 |
+
expect: {value: 7.84, abs: 0.5, tools: [run_semantic_query], not_tools: [report_gap]}
|
platform/harness/__init__.py
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Harness β the multi-source financial-dashboarding platform (connectors + canonical model + skills)."""
|
| 2 |
+
from .canonical import Capability, Entity, Query, Filter
|
| 3 |
+
from .base import Connector, Tenant, Skill, register_skill, skills_for
|
platform/harness/analyst.py
ADDED
|
@@ -0,0 +1,318 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""harness/analyst.py β the AIOS Analyst loop (OM-4, 2026-07-11).
|
| 2 |
+
|
| 3 |
+
A CHEAP small model (via OpenRouter, OpenAI-compatible) drives the compounding tool registry
|
| 4 |
+
(harness/tools.py) to answer questions, build charts, and spawn dashboards over the semantic
|
| 5 |
+
layer. The model NEVER writes SQL β it picks registry keys, guided by the in-product SKILL
|
| 6 |
+
recipes (model/skills/*.skill.yml) injected into its system prompt. Plan: omni-adoption addendum.
|
| 7 |
+
|
| 8 |
+
Architecture (mirrors the Omni coordinator pattern):
|
| 9 |
+
ask(question) -> bounded tool loop: chat -> tool_calls -> tools.dispatch() -> results -> chat β¦
|
| 10 |
+
until a plain answer or MAX_ITERS. chat_fn is INJECTABLE: live = OpenRouter POST; tests = a
|
| 11 |
+
scripted double β the loop mechanics are provable offline (no key, no tokens).
|
| 12 |
+
|
| 13 |
+
Correctness posture: bounded iterations (never endless); tool errors go BACK to the model as
|
| 14 |
+
readable envelopes (it self-corrects); the final answer carries the tool trace (auditability +
|
| 15 |
+
the drill guarantee); exhausted-budget is reported AS exhausted, never as success (loop rules).
|
| 16 |
+
Cost posture: DEFAULT_MODEL is a cheap tool-calling model; override per-tenant/env; prompt = the
|
| 17 |
+
skills + topic summary (small, cacheable), NOT raw data.
|
| 18 |
+
"""
|
| 19 |
+
import json
|
| 20 |
+
import os
|
| 21 |
+
from pathlib import Path
|
| 22 |
+
|
| 23 |
+
import harness.tools as T
|
| 24 |
+
|
| 25 |
+
# Provider LADDER (owner 2026-07-12: "use Groq and Cerebras"): all OpenAI-compatible chat
|
| 26 |
+
# endpoints; tried in order, failing over on payment/rate/server errors β an exhausted or
|
| 27 |
+
# rate-limited provider degrades to the next instead of taking the Analyst down (the
|
| 28 |
+
# OpenRouter-402 incident). Pin one with ANALYST_PROVIDER=groq|cerebras|openrouter; override
|
| 29 |
+
# its model with ANALYST_MODEL. Every model here must re-pass the eval gate (Β§4b).
|
| 30 |
+
PROVIDERS = [
|
| 31 |
+
{"name": "groq", "env": "GROQ_API_KEY",
|
| 32 |
+
"url": "https://api.groq.com/openai/v1/chat/completions",
|
| 33 |
+
"model": "llama-3.3-70b-versatile"},
|
| 34 |
+
{"name": "cerebras", "env": "CEREBRAS_API_KEY",
|
| 35 |
+
"url": "https://api.cerebras.ai/v1/chat/completions",
|
| 36 |
+
"model": "gpt-oss-120b"}, # this account's tool-capable model (from /v1/models)
|
| 37 |
+
{"name": "openrouter", "env": "OPENROUTER_API_KEY",
|
| 38 |
+
"url": "https://openrouter.ai/api/v1/chat/completions",
|
| 39 |
+
"model": "openai/gpt-4o-mini"},
|
| 40 |
+
]
|
| 41 |
+
DEFAULT_MODEL = None # None = the selected provider's default model
|
| 42 |
+
MAX_ITERS = 8
|
| 43 |
+
|
| 44 |
+
|
| 45 |
+
def available_providers():
|
| 46 |
+
pin = os.environ.get("ANALYST_PROVIDER")
|
| 47 |
+
ps = [p for p in PROVIDERS if os.environ.get(p["env"])]
|
| 48 |
+
if pin:
|
| 49 |
+
ps = [p for p in ps if p["name"] == pin] or ps
|
| 50 |
+
return ps
|
| 51 |
+
|
| 52 |
+
|
| 53 |
+
def llm_configured():
|
| 54 |
+
return bool(available_providers())
|
| 55 |
+
SKILLS_DIR = Path(__file__).resolve().parents[1] / "model" / "skills"
|
| 56 |
+
|
| 57 |
+
|
| 58 |
+
def _skills_text():
|
| 59 |
+
parts = []
|
| 60 |
+
for f in sorted(SKILLS_DIR.glob("*.skill.yml")):
|
| 61 |
+
parts.append(f.read_text(encoding="utf-8"))
|
| 62 |
+
return "\n\n".join(parts) or "(no skill files installed)"
|
| 63 |
+
|
| 64 |
+
|
| 65 |
+
def system_prompt():
|
| 66 |
+
"""The small model's standing instructions: the platform rules + the skill recipes.
|
| 67 |
+
Stable content (cache-friendly); per-question content goes in the user turn."""
|
| 68 |
+
topics = T.list_topics()
|
| 69 |
+
topic_summary = "\n".join(
|
| 70 |
+
f"- {t['topic']}: {t['label']} | metrics: {', '.join(t['metrics'])} | dims: {', '.join(t['dims'])}"
|
| 71 |
+
for t in topics)
|
| 72 |
+
return f"""You are the AIOS Analyst for a wholesale business. You answer data questions by
|
| 73 |
+
calling TOOLS over a governed semantic layer. THE RULES (non-negotiable):
|
| 74 |
+
1. You NEVER write SQL or invent field names β only registry keys from the tools' schemas.
|
| 75 |
+
2. Start unknown asks with list_topics/describe_topic. Resolve typed names (customers, products)
|
| 76 |
+
with get_field_values BEFORE filtering. Business units: Fisch=5, Royal=6 β never mix them in
|
| 77 |
+
one series unless explicitly comparing.
|
| 78 |
+
3. Every number you present must come from a tool result and keep its result_id (drillable).
|
| 79 |
+
If a tool errors, read the error and correct your call β do not make numbers up, ever.
|
| 80 |
+
4. Artifact actions (save_view, compose_dashboard, schedule_report, create_alert) need the
|
| 81 |
+
user's explicit confirmation first.
|
| 82 |
+
5. Answer plainly and lead with the finding. State the window and scope you used β READ the
|
| 83 |
+
window from the tool result's 'window' field, never guess it: a query without dates covers
|
| 84 |
+
ALL recorded history (there is no hidden default window). Time-bound asks ("this year",
|
| 85 |
+
"past 4 weeks") must pass explicit date_from/date_to.
|
| 86 |
+
6. A single-number question gets THE SINGLE NUMBER first (the total for the asked window/scope),
|
| 87 |
+
THEN any breakdown. Never answer a "what was X" question with only components: no monthly
|
| 88 |
+
lists without the total, no per-BU splits when the consolidated figure was asked. If the ask
|
| 89 |
+
is consolidated, query WITHOUT group_by (or sum before answering).
|
| 90 |
+
7. NEVER state an aggregate you did not obtain from a tool result. If you want to mention a
|
| 91 |
+
total across the rows you fetched, either run the scalar query for it or ADD UP those exact
|
| 92 |
+
rows β a round "approximately" figure is a fabrication and forbidden. When in doubt, present
|
| 93 |
+
the rows without a total. Joining or diffing two result sets BY HAND (e.g. this year vs last
|
| 94 |
+
year per customer) is equally forbidden β that is transform_result's job ({{"op":"yoy"}} adds
|
| 95 |
+
<m>_ly, <m>_delta, <m>_yoy_pct per row); if no transform can compute it, say you cannot.
|
| 96 |
+
8. THE GAP PROTOCOL. If β and only if β after checking the schema (list_topics/describe_topic)
|
| 97 |
+
no registered dimension, metric, transform or chart kind can answer, do NOT guess and do NOT
|
| 98 |
+
force a wrong query: call report_gap(kind, missing, question, workaround) ONCE, then answer
|
| 99 |
+
in one honest sentence what is missing and offer the nearest ask that IS answerable. Before
|
| 100 |
+
concluding a gap, check the TRANSFORMS: year-over-year / decline / growth comparisons (yoy),
|
| 101 |
+
rankings (top_n/sort/head), shares, running totals and distributions are ALL answerable via
|
| 102 |
+
transform_result β a derived-analytics question is never a gap. Both directions are
|
| 103 |
+
failures: refusing without report_gap, and calling report_gap for something the tools
|
| 104 |
+
support.
|
| 105 |
+
9. NEVER emit markdown images, data: URIs, or links to charts β you cannot draw. Charts exist
|
| 106 |
+
ONLY through make_chart / make_table; the platform renders them below your answer. Refer to
|
| 107 |
+
a chart as "the chart below". Derived analytics (top-N, YoY, shares, running totals, moving
|
| 108 |
+
averages, ranks, bins) exist ONLY through transform_result β never arithmetic in your head.
|
| 109 |
+
|
| 110 |
+
CHART PICKER (the form follows the comparison β pick by what the question compares):
|
| 111 |
+
- trend over time -> line (few periods -> bar); seasonal/two-dim intensity -> heatmap
|
| 112 |
+
- this year vs last year by period -> transform_result [{{"op":"yoy"}}] then yoy_bars
|
| 113 |
+
- ranking / "top N" -> transform_result top_n then ranked_bar (sorted, labeled)
|
| 114 |
+
- part-to-whole -> pie|donut ONLY when <=6 slices (top_n first); over time -> stacked_bar;
|
| 115 |
+
shares of a shifting total -> stacked_pct; hierarchy of shares -> treemap
|
| 116 |
+
- what drove the change / contribution -> waterfall (signed steps; platform adds the Total)
|
| 117 |
+
- concentration ("do the top X carry the book?") -> pareto
|
| 118 |
+
- correlation of two measures -> scatter; + a third measure -> bubble (size)
|
| 119 |
+
- distribution of a measure -> histogram (or transform bin -> bar)
|
| 120 |
+
- stage conversion -> funnel; actual vs target -> bullet (y=actual, y2=target)
|
| 121 |
+
- level + rate together (e.g. GM$ + GM%) -> combo (y=bars left, y2=line right)
|
| 122 |
+
- same small chart per segment -> facet on line|bar|area|scatter
|
| 123 |
+
- exact figures, many columns, lists -> make_table (totals row included)
|
| 124 |
+
- customer geography -> map. When two forms tie, pick the simpler one.
|
| 125 |
+
|
| 126 |
+
DATASETS:
|
| 127 |
+
{topic_summary}
|
| 128 |
+
|
| 129 |
+
YOUR SKILL RECIPES (follow these plans; adjust the [bracketed] parameters):
|
| 130 |
+
{_skills_text()}"""
|
| 131 |
+
|
| 132 |
+
|
| 133 |
+
_DEAD_STATUS = {401, 402, 403} # payment/auth: skip the provider immediately, no retry
|
| 134 |
+
_RETRY_STATUS = {429, 500, 502, 503} # rate/server: retry with backoff before failing over
|
| 135 |
+
_MAX_ANSWER_TOKENS = 1500 # answers are short; also keeps low-credit providers usable
|
| 136 |
+
_COOLDOWN = {} # provider name -> unix ts until which we skip it (in-proc)
|
| 137 |
+
|
| 138 |
+
|
| 139 |
+
def _live_chat(messages, tools, model, tool_choice="auto"):
|
| 140 |
+
"""POST down the provider ladder with LATENCY-FIRST failover (a chat user is waiting):
|
| 141 |
+
a rate-limited provider is SKIPPED IMMEDIATELY in favor of the next one β we only sleep
|
| 142 |
+
when the ENTIRE ladder is limited (then short ladder-level backoffs). A LONG Retry-After
|
| 143 |
+
(daily quota, e.g. Groq TPD) puts the provider in an in-process COOLDOWN; 401/402/403 skip
|
| 144 |
+
the provider outright."""
|
| 145 |
+
import time as _time
|
| 146 |
+
|
| 147 |
+
import requests
|
| 148 |
+
provs = available_providers()
|
| 149 |
+
if not provs:
|
| 150 |
+
raise RuntimeError("no LLM provider configured β set GROQ_API_KEY / CEREBRAS_API_KEY / "
|
| 151 |
+
"OPENROUTER_API_KEY in platform/.env")
|
| 152 |
+
last_err = None
|
| 153 |
+
# Two patience profiles: CHAT (default) fails over fast β a user is waiting; BATCH
|
| 154 |
+
# (ANALYST_PATIENT=1, set by the eval gate / insight runner) waits out per-minute rate
|
| 155 |
+
# windows across more ladder rounds instead of exhausting the ladder in seconds.
|
| 156 |
+
patient = os.environ.get("ANALYST_PATIENT") == "1"
|
| 157 |
+
round_sleeps = (5, 10, 20, 30, 45, 60) if patient else (2, 6, 12)
|
| 158 |
+
for rnd in range(len(round_sleeps)): # ladder rounds
|
| 159 |
+
saw_retryable = False
|
| 160 |
+
for p in provs:
|
| 161 |
+
if _COOLDOWN.get(p["name"], 0) > _time.time():
|
| 162 |
+
last_err = RuntimeError(f"{p['name']}: cooling down (quota)")
|
| 163 |
+
continue
|
| 164 |
+
try:
|
| 165 |
+
r = requests.post(p["url"], timeout=120,
|
| 166 |
+
headers={"Authorization": f"Bearer {os.environ[p['env']]}",
|
| 167 |
+
"HTTP-Referer": "https://aios.local",
|
| 168 |
+
"X-Title": "AIOS Analyst"},
|
| 169 |
+
json={"model": (model or os.environ.get("ANALYST_MODEL")
|
| 170 |
+
or p["model"]),
|
| 171 |
+
"messages": messages, "tools": tools,
|
| 172 |
+
"tool_choice": tool_choice, "temperature": 0.1,
|
| 173 |
+
"max_tokens": _MAX_ANSWER_TOKENS})
|
| 174 |
+
if r.status_code in _DEAD_STATUS:
|
| 175 |
+
last_err = RuntimeError(f"{p['name']}: HTTP {r.status_code}")
|
| 176 |
+
_COOLDOWN[p["name"]] = _time.time() + 300 # don't re-poke dead auth/payment
|
| 177 |
+
continue
|
| 178 |
+
if r.status_code in _RETRY_STATUS:
|
| 179 |
+
last_err = RuntimeError(f"{p['name']}: HTTP {r.status_code}")
|
| 180 |
+
try:
|
| 181 |
+
wait = float(r.headers.get("retry-after") or 0)
|
| 182 |
+
except ValueError:
|
| 183 |
+
wait = 0
|
| 184 |
+
if wait > 60: # daily/long quota β cool down
|
| 185 |
+
_COOLDOWN[p["name"]] = _time.time() + wait
|
| 186 |
+
else:
|
| 187 |
+
saw_retryable = True # NO sleep β next provider NOW
|
| 188 |
+
continue
|
| 189 |
+
r.raise_for_status()
|
| 190 |
+
usage = r.json().get("usage", {})
|
| 191 |
+
return r.json()["choices"][0]["message"], usage
|
| 192 |
+
except requests.RequestException as e:
|
| 193 |
+
last_err = e
|
| 194 |
+
continue # network blip β next provider
|
| 195 |
+
if not saw_retryable:
|
| 196 |
+
break # nothing retryable left anywhere
|
| 197 |
+
_time.sleep(round_sleeps[rnd]) # whole ladder limited β pause a round
|
| 198 |
+
raise RuntimeError(f"all LLM providers failed (last: {last_err})")
|
| 199 |
+
|
| 200 |
+
|
| 201 |
+
#: Wave 15 R2 β what a restricted account is told instead of an answer.
|
| 202 |
+
ANALYST_DENIED = ("The Analyst is not available on your account. It answers from the full "
|
| 203 |
+
"governed model, which is wider than what your account is permitted to see.")
|
| 204 |
+
|
| 205 |
+
|
| 206 |
+
def may_ask(user):
|
| 207 |
+
"""β R2 β THE AI WALL, AND IT IS DELIBERATELY BLUNT.
|
| 208 |
+
|
| 209 |
+
Owner, verbatim: *"If the AI is triggered by a user, the AI ITSELF should not be able to see
|
| 210 |
+
the data."* Today the Analyst's tools run UNSCOPED β `harness/tools.run_semantic_query` takes
|
| 211 |
+
no subject and no filter, so every question it answers is answered over the whole tenant.
|
| 212 |
+
A restricted account asking "what were our top customers?" would therefore receive, in prose,
|
| 213 |
+
precisely the rows the grid's permanent filter exists to withhold. Prose is not a lesser leak
|
| 214 |
+
than a table; it is a harder one to notice.
|
| 215 |
+
|
| 216 |
+
So the wall is REFUSAL, not narrowing: an account with ANY declared restriction β or any
|
| 217 |
+
non-admin on a migrated record β does not reach the model at all. An AI that never runs
|
| 218 |
+
cannot leak what it never read. Fail-closed on an unreadable record for the same reason.
|
| 219 |
+
|
| 220 |
+
β’ NAMED DEBT, owner-mandated (R2, doc Β§Debt, log 2026-08-02). The RIGHT answer is to thread
|
| 221 |
+
the subject's permanent filter into every semantic query the Analyst runs β `filter_sql`
|
| 222 |
+
already compiles that tree to SQL, so the shape exists; what does not exist is a per-subject
|
| 223 |
+
query path through `harness/semantic.py`, which takes `team_id` and nothing else. Until that
|
| 224 |
+
lands this refusal is the whole enforcement, and it must NOT be relaxed into "narrow the
|
| 225 |
+
prompt" or "tell the model what it may not say" β a prompt is not a permission boundary.
|
| 226 |
+
"""
|
| 227 |
+
import core.perms as perms
|
| 228 |
+
import core.perm_scope as perm_scope
|
| 229 |
+
|
| 230 |
+
if not isinstance(user, dict):
|
| 231 |
+
return False # no identity resolved -> no Analyst
|
| 232 |
+
if perms.is_admin(user):
|
| 233 |
+
return True
|
| 234 |
+
if perm_scope.is_migrated(user):
|
| 235 |
+
# Migrated and not an admin: the account is governed by a wall the Analyst cannot honour.
|
| 236 |
+
return False
|
| 237 |
+
# Legacy record: restricted the old way is still restricted.
|
| 238 |
+
return (perms.allowed_modules(user) is None
|
| 239 |
+
and perms.scope_team_id(user) is None
|
| 240 |
+
and not perms.scope_agent(user))
|
| 241 |
+
|
| 242 |
+
|
| 243 |
+
def ask(question, history=None, chat_fn=None, model=DEFAULT_MODEL, max_iters=MAX_ITERS,
|
| 244 |
+
telemetry_kind="analyst", user=None):
|
| 245 |
+
"""Run one Analyst turn. Returns {answer, artifacts, tool_trace, iterations, exhausted,
|
| 246 |
+
usage}. chat_fn(messages, tools, model) -> (assistant_message_dict, usage_dict).
|
| 247 |
+
|
| 248 |
+
`user` is the SUBJECT asking. Passing it engages the R2 wall (`may_ask`); omitting it keeps
|
| 249 |
+
the unscoped behaviour that `harness/evals.py` and `harness/routines.py` rely on β those run
|
| 250 |
+
as the SYSTEM, not on behalf of a person, and gating them on a user record they do not have
|
| 251 |
+
would break the eval ladder to no security benefit. Every call that IS on behalf of somebody
|
| 252 |
+
passes it, and `app.py`'s chat entry point does.
|
| 253 |
+
"""
|
| 254 |
+
if user is not None and not may_ask(user):
|
| 255 |
+
return {"answer": ANALYST_DENIED, "artifacts": [], "tool_trace": [],
|
| 256 |
+
"iterations": 0, "exhausted": False, "denied": True,
|
| 257 |
+
"usage": {"prompt_tokens": 0, "completion_tokens": 0}}
|
| 258 |
+
chat = chat_fn or _live_chat
|
| 259 |
+
tools = T.openai_tools()
|
| 260 |
+
messages = [{"role": "system", "content": system_prompt()}]
|
| 261 |
+
messages += list(history or [])
|
| 262 |
+
# The model has no clock: anchor relative windows ("last week", "this month") to today.
|
| 263 |
+
# Lives in the USER turn so the system prompt stays byte-stable (prompt-cache friendly).
|
| 264 |
+
import datetime as _dt
|
| 265 |
+
messages.append({"role": "user",
|
| 266 |
+
"content": f"(today is {_dt.date.today().isoformat()})\n{question}"})
|
| 267 |
+
trace, artifacts, usage_total = [], [], {"prompt_tokens": 0, "completion_tokens": 0}
|
| 268 |
+
|
| 269 |
+
for i in range(max_iters):
|
| 270 |
+
msg, usage = chat(messages, tools, model)
|
| 271 |
+
for k in usage_total:
|
| 272 |
+
usage_total[k] += usage.get(k, 0) or 0
|
| 273 |
+
tool_calls = msg.get("tool_calls") or []
|
| 274 |
+
# Free-tier models INTERMITTENTLY answer a data question without calling any tool
|
| 275 |
+
# (~1 in 4 observed on gpt-oss-120b β the "no chart" defect). Deterministic fix: if
|
| 276 |
+
# the VERY FIRST turn used no tools, redo it once with tool_choice='required'. Meta
|
| 277 |
+
# questions just get a harmless schema lookup before answering.
|
| 278 |
+
if i == 0 and not tool_calls and chat is _live_chat:
|
| 279 |
+
msg, usage = _live_chat(messages, tools, model, tool_choice="required")
|
| 280 |
+
for k in usage_total:
|
| 281 |
+
usage_total[k] += usage.get(k, 0) or 0
|
| 282 |
+
tool_calls = msg.get("tool_calls") or []
|
| 283 |
+
messages.append({"role": "assistant", "content": msg.get("content") or "",
|
| 284 |
+
**({"tool_calls": tool_calls} if tool_calls else {})})
|
| 285 |
+
if not tool_calls:
|
| 286 |
+
answer = (msg.get("content") or "").strip()
|
| 287 |
+
if not answer:
|
| 288 |
+
# A model occasionally returns NO content and NO tool calls (observed live
|
| 289 |
+
# 2026-07-16: the reply bubble was simply empty and the next turn hallucinated).
|
| 290 |
+
# Never surface silence β report the failure honestly instead.
|
| 291 |
+
answer = ("(the model returned an empty answer β nothing was computed; "
|
| 292 |
+
"please rephrase or ask again)")
|
| 293 |
+
import harness.telemetry as _tel
|
| 294 |
+
_tel.log(telemetry_kind, question=question[:200],
|
| 295 |
+
model=model or (available_providers() or [{}])[0].get("name", "auto"),
|
| 296 |
+
iterations=i + 1,
|
| 297 |
+
tools=[t["tool"] for t in trace],
|
| 298 |
+
tokens=usage_total["prompt_tokens"] + usage_total["completion_tokens"])
|
| 299 |
+
return {"answer": answer, "artifacts": artifacts,
|
| 300 |
+
"tool_trace": trace, "iterations": i + 1, "exhausted": False,
|
| 301 |
+
"usage": usage_total}
|
| 302 |
+
for tc in tool_calls:
|
| 303 |
+
name = tc["function"]["name"]
|
| 304 |
+
args = tc["function"].get("arguments") or "{}"
|
| 305 |
+
res = T.dispatch(name, args)
|
| 306 |
+
trace.append({"tool": name, "args": args if isinstance(args, str) else json.dumps(args),
|
| 307 |
+
"ok": res["ok"], "error": res.get("error")})
|
| 308 |
+
if res["ok"] and isinstance(res["data"], dict) and (
|
| 309 |
+
"chart" in res["data"] or "kpi" in res["data"] or "table" in res["data"]
|
| 310 |
+
or "dashboard" in res["data"]):
|
| 311 |
+
artifacts.append(res["data"])
|
| 312 |
+
messages.append({"role": "tool", "tool_call_id": tc.get("id", name),
|
| 313 |
+
"content": json.dumps(res, default=str)[:8000]})
|
| 314 |
+
# budget exhausted β report honestly (never as success)
|
| 315 |
+
return {"answer": "(stopped: tool-call budget exhausted before a final answer β "
|
| 316 |
+
"narrow the question or raise max_iters)",
|
| 317 |
+
"artifacts": artifacts, "tool_trace": trace, "iterations": max_iters,
|
| 318 |
+
"exhausted": True, "usage": usage_total}
|
platform/harness/base.py
ADDED
|
@@ -0,0 +1,123 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Harness core β the platform contract that turns 'a dashboard' into 'a product'.
|
| 2 |
+
|
| 3 |
+
Three concepts:
|
| 4 |
+
Connector a source adapter (Odoo / QuickBooks / Amazon / β¦). Declares its capabilities and
|
| 5 |
+
answers canonical Queries. Adding a source = subclass Connector.
|
| 6 |
+
Tenant one client (Royal Imports, β¦). Bundles the connectors they've hooked up + their scope
|
| 7 |
+
config (business units, excluded accounts, channel map, branding). Multi-tenant lives here.
|
| 8 |
+
Skill a portable dashboard/metric written ONCE against the canonical model; it declares the
|
| 9 |
+
capabilities it needs so a tenant only ever sees skills its sources can serve.
|
| 10 |
+
|
| 11 |
+
The UI (app.py) becomes a thin renderer over (tenant β available skills). Nothing above the
|
| 12 |
+
connector layer knows what Odoo is.
|
| 13 |
+
"""
|
| 14 |
+
from __future__ import annotations
|
| 15 |
+
from abc import ABC, abstractmethod
|
| 16 |
+
from dataclasses import dataclass, field
|
| 17 |
+
from typing import Callable
|
| 18 |
+
from .canonical import Capability, Query, Entity
|
| 19 |
+
|
| 20 |
+
|
| 21 |
+
class Connector(ABC):
|
| 22 |
+
"""A data source. Implement `capabilities()` + `aggregate()` (and optionally `records()`), map the
|
| 23 |
+
canonical model to the source, and the source is usable by every compatible skill."""
|
| 24 |
+
key: str = 'connector'
|
| 25 |
+
name: str = 'Connector'
|
| 26 |
+
|
| 27 |
+
@abstractmethod
|
| 28 |
+
def capabilities(self) -> set[Capability]:
|
| 29 |
+
"""The canonical capabilities this source can serve for the given account."""
|
| 30 |
+
|
| 31 |
+
@abstractmethod
|
| 32 |
+
def aggregate(self, q: Query) -> list[dict]:
|
| 33 |
+
"""Answer an aggregate Query β list of canonical dicts (keys = measures/group_by, + 'count')."""
|
| 34 |
+
|
| 35 |
+
def records(self, q: Query) -> list[dict]:
|
| 36 |
+
"""Optional row-level fetch (canonical dicts). Default: not supported."""
|
| 37 |
+
raise NotImplementedError(f'{self.key}: row-level records() not implemented')
|
| 38 |
+
|
| 39 |
+
def distinct_count(self, q: Query, fieldname: str) -> int:
|
| 40 |
+
"""Number of distinct values of a canonical field over the query's scope (e.g. active
|
| 41 |
+
customers). Default: derive from an aggregate group_by; connectors may override efficiently."""
|
| 42 |
+
from dataclasses import replace
|
| 43 |
+
rows = self.aggregate(replace(q, measures=(), aggregate='count', group_by=(fieldname,)))
|
| 44 |
+
return sum(1 for r in rows if r.get(fieldname) not in (None, False))
|
| 45 |
+
|
| 46 |
+
def supports(self, cap: Capability) -> bool:
|
| 47 |
+
return cap in self.capabilities()
|
| 48 |
+
|
| 49 |
+
def entities(self) -> set[Entity]:
|
| 50 |
+
"""Canonical entities this connector can materialize (advertised by subclasses)."""
|
| 51 |
+
return set()
|
| 52 |
+
|
| 53 |
+
|
| 54 |
+
@dataclass
|
| 55 |
+
class Tenant:
|
| 56 |
+
"""One client of the platform. `sources` maps a connector key β Connector instance. `config` holds
|
| 57 |
+
the client's scope semantics (which is what used to be hard-coded in core/odoo.py):
|
| 58 |
+
business_units {label: source_scope_value} e.g. {'Fisch': 5, 'Royal': 6}
|
| 59 |
+
confirmed_statuses [...] which order states count as 'real'
|
| 60 |
+
excluded_customers [ids] house / inter-co accounts to drop
|
| 61 |
+
channels {label: match} e.g. {'Amazon': <partner>} (marketplace split)
|
| 62 |
+
branding {...} palette, name, logo
|
| 63 |
+
A skill reads tenant.config for scope and asks tenant.source(capability) for the right connector."""
|
| 64 |
+
key: str
|
| 65 |
+
name: str
|
| 66 |
+
sources: dict[str, Connector] = field(default_factory=dict)
|
| 67 |
+
config: dict = field(default_factory=dict)
|
| 68 |
+
|
| 69 |
+
def add(self, conn: Connector) -> 'Tenant':
|
| 70 |
+
self.sources[conn.key] = conn
|
| 71 |
+
return self
|
| 72 |
+
|
| 73 |
+
def capabilities(self) -> set[Capability]:
|
| 74 |
+
caps: set[Capability] = set()
|
| 75 |
+
for c in self.sources.values():
|
| 76 |
+
caps |= c.capabilities()
|
| 77 |
+
return caps
|
| 78 |
+
|
| 79 |
+
def source(self, cap: Capability, prefer: str | None = None) -> Connector:
|
| 80 |
+
"""The connector that serves `cap` (a tenant may hook several sources; `prefer` picks one,
|
| 81 |
+
else first that supports it). Raises if no connected source provides the capability."""
|
| 82 |
+
if prefer and prefer in self.sources and self.sources[prefer].supports(cap):
|
| 83 |
+
return self.sources[prefer]
|
| 84 |
+
for c in self.sources.values():
|
| 85 |
+
if c.supports(cap):
|
| 86 |
+
return c
|
| 87 |
+
raise LookupError(f"tenant {self.key!r} has no source for capability {cap.value!r}")
|
| 88 |
+
|
| 89 |
+
def has(self, *caps: Capability) -> bool:
|
| 90 |
+
mine = self.capabilities()
|
| 91 |
+
return all(c in mine for c in caps)
|
| 92 |
+
|
| 93 |
+
|
| 94 |
+
# --------------------------------------------------------------------------- skill registry
|
| 95 |
+
@dataclass(frozen=True)
|
| 96 |
+
class Skill:
|
| 97 |
+
"""A portable dashboard. `requires` are the capabilities it needs; the harness only offers it to a
|
| 98 |
+
tenant whose connected sources cover them. `render` is the UI entrypoint (given tenant + context)."""
|
| 99 |
+
key: str
|
| 100 |
+
label: str
|
| 101 |
+
requires: tuple[Capability, ...]
|
| 102 |
+
render: Callable | None = None
|
| 103 |
+
hq: bool = False # consolidated / company-level (ignores the business-unit filter)
|
| 104 |
+
|
| 105 |
+
def available_for(self, tenant: Tenant) -> bool:
|
| 106 |
+
return tenant.has(*self.requires)
|
| 107 |
+
|
| 108 |
+
|
| 109 |
+
_SKILLS: dict[str, Skill] = {}
|
| 110 |
+
|
| 111 |
+
|
| 112 |
+
def register_skill(skill: Skill) -> Skill:
|
| 113 |
+
_SKILLS[skill.key] = skill
|
| 114 |
+
return skill
|
| 115 |
+
|
| 116 |
+
|
| 117 |
+
def skills_for(tenant: Tenant) -> list[Skill]:
|
| 118 |
+
"""The skills a tenant can actually run, given its connected sources β this is the dynamic nav."""
|
| 119 |
+
return [s for s in _SKILLS.values() if s.available_for(tenant)]
|
| 120 |
+
|
| 121 |
+
|
| 122 |
+
def all_skills() -> list[Skill]:
|
| 123 |
+
return list(_SKILLS.values())
|
platform/harness/canonical.py
ADDED
|
@@ -0,0 +1,101 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Canonical financial data model β the SHARED vocabulary of the harness.
|
| 2 |
+
|
| 3 |
+
Skills speak ONLY in these terms; connectors (Odoo / QuickBooks / Amazon / β¦) translate them to
|
| 4 |
+
their own source. This is the seam that decouples the dashboards from any single system: add a new
|
| 5 |
+
source by writing a connector that maps its API onto these entities/fields β every skill that needs
|
| 6 |
+
only the capabilities that source provides then works, unchanged.
|
| 7 |
+
|
| 8 |
+
Design rules:
|
| 9 |
+
- Entities and canonical field names are source-agnostic (no `amount_untaxed`, no `sale.order`).
|
| 10 |
+
- A skill builds a `Query` (what business question), a connector answers it (how, per source).
|
| 11 |
+
- Capabilities let a skill declare what it needs and a tenant show only the dashboards its
|
| 12 |
+
connected sources can actually serve.
|
| 13 |
+
"""
|
| 14 |
+
from __future__ import annotations
|
| 15 |
+
from dataclasses import dataclass, field
|
| 16 |
+
from enum import Enum
|
| 17 |
+
from typing import Any
|
| 18 |
+
|
| 19 |
+
|
| 20 |
+
class Capability(str, Enum):
|
| 21 |
+
"""What a connector can serve. Skills require a set of these; a tenant's nav is the union of
|
| 22 |
+
what its connectors provide. (A pure-accounting source like QuickBooks has LEDGER+INVOICING+
|
| 23 |
+
CUSTOMERS but not SKU-level SALES velocity; Amazon has SALES but no LEDGER; Odoo has all.)"""
|
| 24 |
+
SALES = 'sales' # orders / order lines (SKU-level revenue, units)
|
| 25 |
+
INVOICING = 'invoicing' # customer invoices + credit notes (AR, returns)
|
| 26 |
+
LEDGER = 'ledger' # GL accounts + journal lines (P&L, balance sheet)
|
| 27 |
+
PAYABLES = 'payables' # vendor bills / AP
|
| 28 |
+
INVENTORY = 'inventory' # on-hand stock levels + valuation
|
| 29 |
+
PRODUCTS = 'products' # product master (SKU, category, cost)
|
| 30 |
+
CUSTOMERS = 'customers' # customer master (name, geo, assigned agent)
|
| 31 |
+
|
| 32 |
+
|
| 33 |
+
class Entity(str, Enum):
|
| 34 |
+
"""Canonical business objects. Each connector declares which it can materialize."""
|
| 35 |
+
SALES_ORDER = 'sales_order'
|
| 36 |
+
SALES_ORDER_LINE = 'sales_order_line'
|
| 37 |
+
INVOICE = 'invoice' # customer invoice / credit note
|
| 38 |
+
JOURNAL_LINE = 'journal_line' # GL posting
|
| 39 |
+
GL_ACCOUNT = 'gl_account'
|
| 40 |
+
VENDOR_BILL = 'vendor_bill'
|
| 41 |
+
STOCK_LEVEL = 'stock_level'
|
| 42 |
+
PRODUCT = 'product'
|
| 43 |
+
CUSTOMER = 'customer'
|
| 44 |
+
|
| 45 |
+
|
| 46 |
+
# Canonical field names per entity β the ONLY field identifiers a skill may use. Connectors own the
|
| 47 |
+
# mapping to their native fields. `date` is the entity's primary time axis (used by Query windows).
|
| 48 |
+
FIELDS: dict[Entity, set[str]] = {
|
| 49 |
+
Entity.SALES_ORDER: {'date', 'amount', 'customer', 'salesperson', 'agent', 'status', 'business_unit', 'channel'},
|
| 50 |
+
Entity.SALES_ORDER_LINE: {'date', 'amount', 'qty', 'margin', 'product', 'category', 'customer', 'status', 'business_unit'},
|
| 51 |
+
Entity.INVOICE: {'date', 'amount', 'customer', 'doc_type', 'status', 'salesperson', 'business_unit'},
|
| 52 |
+
Entity.JOURNAL_LINE: {'date', 'amount', 'account', 'account_type', 'business_unit', 'status'},
|
| 53 |
+
Entity.GL_ACCOUNT: {'code', 'name', 'type'},
|
| 54 |
+
Entity.VENDOR_BILL: {'date', 'amount', 'vendor', 'status'},
|
| 55 |
+
Entity.STOCK_LEVEL: {'product', 'qty', 'value', 'location'},
|
| 56 |
+
Entity.PRODUCT: {'id', 'code', 'name', 'category', 'cost', 'active'},
|
| 57 |
+
Entity.CUSTOMER: {'id', 'name', 'city', 'state', 'agent', 'active'},
|
| 58 |
+
}
|
| 59 |
+
|
| 60 |
+
# Allowed filter/aggregate vocabulary (connectors translate these; they never see source operators).
|
| 61 |
+
OPS = frozenset({'eq', 'ne', 'in', 'not_in', 'gte', 'lte', 'gt', 'lt', 'like'})
|
| 62 |
+
AGGS = frozenset({'sum', 'count', 'avg', 'min', 'max'})
|
| 63 |
+
GRAINS = frozenset({'day', 'week', 'month', 'quarter', 'year'}) # time-bucket for a trend
|
| 64 |
+
|
| 65 |
+
|
| 66 |
+
@dataclass(frozen=True)
|
| 67 |
+
class Filter:
|
| 68 |
+
"""A canonical predicate: (canonical_field, op, value). op β OPS."""
|
| 69 |
+
fieldname: str
|
| 70 |
+
op: str
|
| 71 |
+
value: Any
|
| 72 |
+
|
| 73 |
+
def __post_init__(self):
|
| 74 |
+
if self.op not in OPS:
|
| 75 |
+
raise ValueError(f'unknown op {self.op!r}; allowed {sorted(OPS)}')
|
| 76 |
+
|
| 77 |
+
|
| 78 |
+
@dataclass(frozen=True)
|
| 79 |
+
class Query:
|
| 80 |
+
"""A source-agnostic analytical request. `measures` are canonical fields aggregated by `aggregate`,
|
| 81 |
+
optionally grouped by canonical `group_by` fields, filtered by `filters`, over [date_from, date_to]
|
| 82 |
+
on the entity's `date` axis. A connector returns a list of plain dicts keyed by the canonical
|
| 83 |
+
measure/group names (+ 'count'). Skills post-process those dicts β never source rows."""
|
| 84 |
+
entity: Entity
|
| 85 |
+
measures: tuple[str, ...] = ()
|
| 86 |
+
aggregate: str = 'sum'
|
| 87 |
+
group_by: tuple[str, ...] = ()
|
| 88 |
+
filters: tuple[Filter, ...] = ()
|
| 89 |
+
date_from: str | None = None
|
| 90 |
+
date_to: str | None = None
|
| 91 |
+
grain: str | None = None # bucket the date axis into a trend; result rows carry 'period' (e.g. '2026-05')
|
| 92 |
+
|
| 93 |
+
def __post_init__(self):
|
| 94 |
+
if self.aggregate not in AGGS:
|
| 95 |
+
raise ValueError(f'unknown aggregate {self.aggregate!r}')
|
| 96 |
+
if self.grain is not None and self.grain not in GRAINS:
|
| 97 |
+
raise ValueError(f'unknown grain {self.grain!r}; allowed {sorted(GRAINS)}')
|
| 98 |
+
valid = FIELDS.get(self.entity, set())
|
| 99 |
+
for f in list(self.measures) + list(self.group_by) + [x.fieldname for x in self.filters]:
|
| 100 |
+
if f not in valid:
|
| 101 |
+
raise ValueError(f'{f!r} is not a canonical field of {self.entity.value} (have {sorted(valid)})')
|
platform/harness/chatlog.py
ADDED
|
@@ -0,0 +1,102 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""harness/chatlog.py β durable Analyst conversations (2026-07-16).
|
| 2 |
+
|
| 3 |
+
Every Analyst conversation is a logged SESSION that survives reruns, page navigation and Space
|
| 4 |
+
restarts. Persistence rides the platform's only writable store (core/store.py β the private HF
|
| 5 |
+
dataset), the SAME mechanism as users/notes β Odoo stays read-only.
|
| 6 |
+
|
| 7 |
+
Shape (store key 'analyst_sessions'): { username: [ session, ... ] } (newest first), each
|
| 8 |
+
session = {id, title, started_at, updated_at, ui:[[role,content,artifacts],...],
|
| 9 |
+
llm:[{role,content},...]}.
|
| 10 |
+
ui = the redisplay transcript; llm = the trimmed model-context transcript.
|
| 11 |
+
|
| 12 |
+
Write posture (per the design review): the LIVE turn stays in st.session_state for a snappy chat;
|
| 13 |
+
we persist in a BACKGROUND daemon thread (core/store.update is a full dataset commit β never block
|
| 14 |
+
the UI on it), and bound growth (MAX_SESSIONS per user, MAX_TURNS per session) so the JSON stays
|
| 15 |
+
small. A failed/absent store degrades to in-memory only (chat still works this session).
|
| 16 |
+
"""
|
| 17 |
+
import threading
|
| 18 |
+
import time
|
| 19 |
+
import uuid
|
| 20 |
+
|
| 21 |
+
import core.store as store
|
| 22 |
+
|
| 23 |
+
KEY = "analyst_sessions"
|
| 24 |
+
MAX_SESSIONS = 40 # keep the last N conversations per user (bounds the JSON)
|
| 25 |
+
MAX_TURNS = 60 # ui/llm entries kept per conversation
|
| 26 |
+
TITLE_LEN = 60
|
| 27 |
+
|
| 28 |
+
|
| 29 |
+
def new_id():
|
| 30 |
+
return uuid.uuid4().hex[:12]
|
| 31 |
+
|
| 32 |
+
|
| 33 |
+
def new_session(sid=None):
|
| 34 |
+
now = time.strftime("%Y-%m-%d %H:%M:%S")
|
| 35 |
+
return {"id": sid or new_id(), "title": "", "started_at": now, "updated_at": now,
|
| 36 |
+
"ui": [], "llm": []}
|
| 37 |
+
|
| 38 |
+
|
| 39 |
+
def _safe(obj):
|
| 40 |
+
"""JSON round-trip a value (chart/kpi artifacts) so only serializable content is stored;
|
| 41 |
+
anything that won't serialize is dropped rather than breaking the whole write."""
|
| 42 |
+
import json
|
| 43 |
+
try:
|
| 44 |
+
return json.loads(json.dumps(obj, default=str))
|
| 45 |
+
except Exception:
|
| 46 |
+
return None
|
| 47 |
+
|
| 48 |
+
|
| 49 |
+
def load_sessions(username):
|
| 50 |
+
"""The user's conversations, newest first (lenient read β never raises on a store blip)."""
|
| 51 |
+
if not store.available():
|
| 52 |
+
return []
|
| 53 |
+
try:
|
| 54 |
+
all_s = store.get(KEY) or {}
|
| 55 |
+
except Exception:
|
| 56 |
+
return []
|
| 57 |
+
return list(all_s.get(username, []))
|
| 58 |
+
|
| 59 |
+
|
| 60 |
+
def touch(session, question):
|
| 61 |
+
"""Stamp updated_at and derive a title from the first user question."""
|
| 62 |
+
session["updated_at"] = time.strftime("%Y-%m-%d %H:%M:%S")
|
| 63 |
+
if not session.get("title") and question:
|
| 64 |
+
t = " ".join(str(question).split())
|
| 65 |
+
session["title"] = t[:TITLE_LEN] + ("β¦" if len(t) > TITLE_LEN else "")
|
| 66 |
+
# bound in-memory transcript length (keep the most recent turns)
|
| 67 |
+
if len(session.get("ui", [])) > MAX_TURNS:
|
| 68 |
+
session["ui"] = session["ui"][-MAX_TURNS:]
|
| 69 |
+
if len(session.get("llm", [])) > MAX_TURNS:
|
| 70 |
+
session["llm"] = session["llm"][-MAX_TURNS:]
|
| 71 |
+
return session
|
| 72 |
+
|
| 73 |
+
|
| 74 |
+
def upsert(sessions, session):
|
| 75 |
+
"""Return the sessions list with `session` moved to the front (deduped by id), capped."""
|
| 76 |
+
out = [s for s in (sessions or []) if s.get("id") != session.get("id")]
|
| 77 |
+
out.insert(0, session)
|
| 78 |
+
return out[:MAX_SESSIONS]
|
| 79 |
+
|
| 80 |
+
|
| 81 |
+
def _persist(username, session):
|
| 82 |
+
# sanitize artifacts for storage (charts are plain dicts; drop any non-serializable stragglers)
|
| 83 |
+
stored = dict(session)
|
| 84 |
+
stored["ui"] = [[role, content, [a for a in (_safe(x) for x in (arts or [])) if a is not None]]
|
| 85 |
+
for (role, content, arts) in session.get("ui", [])]
|
| 86 |
+
|
| 87 |
+
def _fn(data):
|
| 88 |
+
data = data or {}
|
| 89 |
+
data[username] = upsert(data.get(username, []), stored)
|
| 90 |
+
return data
|
| 91 |
+
try:
|
| 92 |
+
store.update(KEY, _fn)
|
| 93 |
+
except Exception:
|
| 94 |
+
pass # store blip / no token β in-memory only this run
|
| 95 |
+
|
| 96 |
+
|
| 97 |
+
def save_async(username, session):
|
| 98 |
+
"""Persist in a daemon thread so the chat never waits on a dataset commit."""
|
| 99 |
+
if not store.available():
|
| 100 |
+
return
|
| 101 |
+
threading.Thread(target=_persist, args=(username, dict(session)),
|
| 102 |
+
daemon=True, name="analyst-chatlog").start()
|
platform/harness/connectors/__init__.py
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
"""Source connectors β one module per system (odoo, quickbooks, amazon, ...)."""
|
platform/harness/connectors/odoo.py
ADDED
|
@@ -0,0 +1,147 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Odoo connector β the reference implementation.
|
| 2 |
+
|
| 3 |
+
ALL Odoo specifics (model names, field names, XML-RPC quirks, the read_group dot-path gotchas) live
|
| 4 |
+
here and nowhere else. It maps the canonical model onto the existing, battle-tested `core.odoo` data
|
| 5 |
+
layer (read-only, write-guarded) so nothing is re-plumbed β the harness just gives it a source-agnostic
|
| 6 |
+
face. Every other connector (QuickBooks, Amazon, β¦) is a sibling of this file.
|
| 7 |
+
"""
|
| 8 |
+
from __future__ import annotations
|
| 9 |
+
import sys
|
| 10 |
+
from dataclasses import replace
|
| 11 |
+
from functools import lru_cache
|
| 12 |
+
from pathlib import Path
|
| 13 |
+
sys.path.insert(0, str(Path(__file__).resolve().parents[2])) # platform/
|
| 14 |
+
import core.odoo as O
|
| 15 |
+
from harness.canonical import Capability, Entity, Query, Filter
|
| 16 |
+
|
| 17 |
+
|
| 18 |
+
@lru_cache(maxsize=1)
|
| 19 |
+
def _product_category_map():
|
| 20 |
+
"""product_id -> main category name (Odoo-side enrichment; reuses the existing resolver)."""
|
| 21 |
+
import modules.sales as _sales
|
| 22 |
+
return _sales._product_cat()
|
| 23 |
+
|
| 24 |
+
|
| 25 |
+
def _replace_group(q: Query, new_group_by):
|
| 26 |
+
return replace(q, group_by=new_group_by, grain=None)
|
| 27 |
+
|
| 28 |
+
_OPS = {'eq': '=', 'ne': '!=', 'in': 'in', 'not_in': 'not in',
|
| 29 |
+
'gte': '>=', 'lte': '<=', 'gt': '>', 'lt': '<', 'like': 'ilike'}
|
| 30 |
+
|
| 31 |
+
# Canonical entity -> Odoo model + (canonical field -> Odoo field) + its time axis.
|
| 32 |
+
_MAP = {
|
| 33 |
+
Entity.SALES_ORDER: {
|
| 34 |
+
'model': 'sale.order', 'date': 'date_order', 'dt': True,
|
| 35 |
+
'f': {'amount': 'amount_untaxed', 'customer': 'partner_id', 'salesperson': 'user_id',
|
| 36 |
+
'status': 'state', 'business_unit': 'team_id'}},
|
| 37 |
+
Entity.SALES_ORDER_LINE: {
|
| 38 |
+
'model': 'sale.order.line', 'date': 'order_id.date_order', 'dt': True,
|
| 39 |
+
'f': {'amount': 'price_subtotal', 'qty': 'product_uom_qty', 'margin': 'margin',
|
| 40 |
+
'product': 'product_id', 'customer': 'order_partner_id',
|
| 41 |
+
'status': 'order_id.state', 'business_unit': 'order_id.team_id'}},
|
| 42 |
+
Entity.INVOICE: {
|
| 43 |
+
'model': 'account.move', 'date': 'invoice_date', 'dt': False,
|
| 44 |
+
'f': {'amount': 'amount_untaxed_signed', 'customer': 'partner_id', 'doc_type': 'move_type',
|
| 45 |
+
'status': 'state', 'salesperson': 'invoice_user_id', 'business_unit': 'team_id'}},
|
| 46 |
+
Entity.STOCK_LEVEL: {
|
| 47 |
+
'model': 'stock.quant', 'date': None, 'dt': False,
|
| 48 |
+
'f': {'product': 'product_id', 'qty': 'quantity', 'location': 'location_id'}},
|
| 49 |
+
Entity.PRODUCT: {
|
| 50 |
+
'model': 'product.product', 'date': None, 'dt': False,
|
| 51 |
+
'f': {'id': 'id', 'code': 'default_code', 'name': 'name', 'category': 'categ_id',
|
| 52 |
+
'cost': 'standard_price', 'active': 'active'}},
|
| 53 |
+
Entity.CUSTOMER: {
|
| 54 |
+
'model': 'res.partner', 'date': None, 'dt': False,
|
| 55 |
+
'f': {'id': 'id', 'name': 'name', 'city': 'city', 'state': 'state_id',
|
| 56 |
+
'agent': 'agent_ids', 'active': 'active'}},
|
| 57 |
+
}
|
| 58 |
+
|
| 59 |
+
|
| 60 |
+
class OdooConnector:
|
| 61 |
+
key = 'odoo'
|
| 62 |
+
name = 'Odoo ERP'
|
| 63 |
+
|
| 64 |
+
def __init__(self, doc_mode: str = 'order'):
|
| 65 |
+
self._doc_mode = doc_mode
|
| 66 |
+
|
| 67 |
+
def capabilities(self) -> set[Capability]:
|
| 68 |
+
return {Capability.SALES, Capability.INVOICING, Capability.LEDGER, Capability.PAYABLES,
|
| 69 |
+
Capability.INVENTORY, Capability.PRODUCTS, Capability.CUSTOMERS}
|
| 70 |
+
|
| 71 |
+
def entities(self) -> set[Entity]:
|
| 72 |
+
return set(_MAP)
|
| 73 |
+
|
| 74 |
+
def supports(self, cap: Capability) -> bool:
|
| 75 |
+
return cap in self.capabilities()
|
| 76 |
+
|
| 77 |
+
# ------------------------------------------------------------------ query translation
|
| 78 |
+
def _domain(self, q: Query, meta):
|
| 79 |
+
dom = []
|
| 80 |
+
if q.date_from and meta['date']:
|
| 81 |
+
dom.append((meta['date'], '>=', q.date_from + (' 00:00:00' if meta['dt'] else '')))
|
| 82 |
+
if q.date_to and meta['date']:
|
| 83 |
+
dom.append((meta['date'], '<=', q.date_to + (' 23:59:59' if meta['dt'] else '')))
|
| 84 |
+
for flt in q.filters:
|
| 85 |
+
dom.append((meta['f'].get(flt.fieldname, flt.fieldname), _OPS[flt.op], flt.value))
|
| 86 |
+
return dom
|
| 87 |
+
|
| 88 |
+
def aggregate(self, q: Query) -> list[dict]:
|
| 89 |
+
meta = _MAP[q.entity]
|
| 90 |
+
O.set_doc_mode(self._doc_mode)
|
| 91 |
+
# 'category' isn't a native line field (dot-path groupby fails) β enrich from the product master.
|
| 92 |
+
if 'category' in q.group_by:
|
| 93 |
+
return self._by_category(q, meta)
|
| 94 |
+
dom = self._domain(q, meta)
|
| 95 |
+
measures = [f"{meta['f'].get(m, m)}:{q.aggregate}" for m in q.measures]
|
| 96 |
+
groupby = [meta['f'].get(g, g) for g in q.group_by]
|
| 97 |
+
if q.grain and meta['date']: # time-bucket -> Odoo's date:grain groupby
|
| 98 |
+
groupby.append(f"{meta['date']}:{q.grain}")
|
| 99 |
+
rows = O.read_group(meta['model'], dom, measures, groupby, lazy=False)
|
| 100 |
+
return [self._canon_row(r, q, meta) for r in rows]
|
| 101 |
+
|
| 102 |
+
def _canon_row(self, r, q, meta):
|
| 103 |
+
rec = {}
|
| 104 |
+
for m in q.measures:
|
| 105 |
+
rec[m] = r.get(meta['f'].get(m, m)) or 0.0
|
| 106 |
+
for g in q.group_by:
|
| 107 |
+
v = r.get(meta['f'].get(g, g))
|
| 108 |
+
rec[g] = ({'id': O.m2o_id(v), 'name': O.m2o_name(v)} if isinstance(v, (list, tuple)) else v)
|
| 109 |
+
if q.grain and meta['date']:
|
| 110 |
+
rng = (r.get('__range') or {}).get(f"{meta['date']}:{q.grain}") or {}
|
| 111 |
+
rec['period'] = (rng.get('from') or '')[:7]
|
| 112 |
+
rec['count'] = r.get('__count') or 0
|
| 113 |
+
return rec
|
| 114 |
+
|
| 115 |
+
def _by_category(self, q, meta):
|
| 116 |
+
"""Group SALES_ORDER_LINE by main product category by grouping on product then folding
|
| 117 |
+
product->category in-connector (source-specific enrichment lives here, not in the skill)."""
|
| 118 |
+
catmap = _product_category_map()
|
| 119 |
+
sub = _replace_group(q, tuple(g for g in q.group_by if g != 'category') + ('product',))
|
| 120 |
+
agg = {}
|
| 121 |
+
for r in self.aggregate(sub):
|
| 122 |
+
prod = r.get('product') or {}
|
| 123 |
+
cat = catmap.get(prod.get('id') if isinstance(prod, dict) else prod, '(uncategorized)')
|
| 124 |
+
e = agg.setdefault(cat, {'category': cat, 'count': 0})
|
| 125 |
+
for m in q.measures:
|
| 126 |
+
e[m] = e.get(m, 0.0) + (r.get(m) or 0.0)
|
| 127 |
+
e['count'] += r.get('count') or 0
|
| 128 |
+
return sorted(agg.values(), key=lambda x: -(x.get(q.measures[0]) if q.measures else 0))
|
| 129 |
+
|
| 130 |
+
def distinct_count(self, q: Query, fieldname: str) -> int:
|
| 131 |
+
meta = _MAP[q.entity]
|
| 132 |
+
O.set_doc_mode(self._doc_mode)
|
| 133 |
+
return O.distinct_count(meta['model'], self._domain(q, meta), meta['f'].get(fieldname, fieldname))
|
| 134 |
+
|
| 135 |
+
def records(self, q: Query) -> list[dict]:
|
| 136 |
+
meta = _MAP[q.entity]
|
| 137 |
+
O.set_doc_mode(self._doc_mode)
|
| 138 |
+
fields = [meta['f'].get(m, m) for m in (q.measures or tuple(meta['f']))]
|
| 139 |
+
rows = O.search_read(meta['model'], self._domain(q, meta), fields)
|
| 140 |
+
inv = {v: k for k, v in meta['f'].items()}
|
| 141 |
+
return [{inv.get(k, k): (O.m2o_name(v) if isinstance(v, (list, tuple)) else v)
|
| 142 |
+
for k, v in r.items()} for r in rows]
|
| 143 |
+
|
| 144 |
+
# convenience for tenant setup: resolve names β ids for scope config
|
| 145 |
+
@staticmethod
|
| 146 |
+
def excluded_customer_ids() -> list[int]:
|
| 147 |
+
return list(O.excluded_partner_ids())
|
platform/harness/connectors/quickbooks.py
ADDED
|
@@ -0,0 +1,37 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""QuickBooks Online connector β STUB (scaffolding for the next real integration).
|
| 2 |
+
|
| 3 |
+
It exists to prove the harness is source-agnostic: it declares the capabilities QBO actually has
|
| 4 |
+
(accounting + AR + customers, but NOT SKU-level sales velocity or warehouse inventory), so the
|
| 5 |
+
platform will automatically OFFER ledger/AR/customer skills on a QuickBooks-only tenant and HIDE the
|
| 6 |
+
SKU/inventory skills β no per-skill if-statements anywhere. Fill in aggregate() against the QBO
|
| 7 |
+
Reports/Query API (OAuth2) to make it live.
|
| 8 |
+
"""
|
| 9 |
+
from __future__ import annotations
|
| 10 |
+
from harness.canonical import Capability, Entity, Query
|
| 11 |
+
|
| 12 |
+
|
| 13 |
+
class QuickBooksConnector:
|
| 14 |
+
key = 'quickbooks'
|
| 15 |
+
name = 'QuickBooks Online'
|
| 16 |
+
|
| 17 |
+
def __init__(self, realm_id: str | None = None, token: str | None = None):
|
| 18 |
+
self.realm_id = realm_id
|
| 19 |
+
self._token = token # OAuth2 access token (never logged)
|
| 20 |
+
|
| 21 |
+
def capabilities(self) -> set[Capability]:
|
| 22 |
+
# QBO gives you the books + receivables + the customer list; it does not model SKU-level
|
| 23 |
+
# order lines or physical stock the way an ERP/marketplace does.
|
| 24 |
+
return {Capability.LEDGER, Capability.INVOICING, Capability.PAYABLES, Capability.CUSTOMERS}
|
| 25 |
+
|
| 26 |
+
def entities(self) -> set[Entity]:
|
| 27 |
+
return {Entity.INVOICE, Entity.JOURNAL_LINE, Entity.GL_ACCOUNT, Entity.VENDOR_BILL, Entity.CUSTOMER}
|
| 28 |
+
|
| 29 |
+
def supports(self, cap: Capability) -> bool:
|
| 30 |
+
return cap in self.capabilities()
|
| 31 |
+
|
| 32 |
+
def aggregate(self, q: Query) -> list[dict]:
|
| 33 |
+
raise NotImplementedError(
|
| 34 |
+
"QuickBooksConnector.aggregate: TODO β translate the canonical Query to the QBO "
|
| 35 |
+
"Query/Reports API (GET /v3/company/{realm}/query) and map rows back to canonical dicts. "
|
| 36 |
+
"Entity map: INVOICE->Invoice/CreditMemo, JOURNAL_LINE->JournalEntry lines, "
|
| 37 |
+
"GL_ACCOUNT->Account, VENDOR_BILL->Bill, CUSTOMER->Customer.")
|
platform/harness/datastore.py
ADDED
|
@@ -0,0 +1,750 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""harness/store.py β the tenant data store (OM-1, 2026-07-11).
|
| 2 |
+
|
| 3 |
+
Incremental Odoo β DuckDB mirror: the local analytical store that saved views/dashboards (OM-3)
|
| 4 |
+
and the AI Analyst's tools (OM-4) query at warehouse speed, instead of hammering live XML-RPC
|
| 5 |
+
per question.
|
| 6 |
+
Plan: .claude/wiki/research/omni-adoption.md Part IV (OM-1) + [[odoo-open-source]] BUILD-NOW #1.
|
| 7 |
+
|
| 8 |
+
Design rules:
|
| 9 |
+
- The store is a RAW mirror (unscoped). Business scope (wholesale teams, GIFTWARE exclusion,
|
| 10 |
+
confirmed-only) is applied by the SEMANTIC layer at query time (OM-2) β one source of truth.
|
| 11 |
+
- Sync is CHECKPOINTED + BOUNDED + RESUMABLE (loop-library discipline): backfill paginates by
|
| 12 |
+
id; live mode advances a write_date cursor; every run is bounded by max_batches and safe to
|
| 13 |
+
kill/re-run (upserts are idempotent).
|
| 14 |
+
- validate() compares the store against LIVE Odoo (raw fidelity: counts + monthly sums to the
|
| 15 |
+
cent) β the store never validates itself. Odoo hard-deletes (unlink) don't move write_date;
|
| 16 |
+
the count checks are the drift detector for those.
|
| 17 |
+
- Client data: the .duckdb file lives under data/store/ (git-ignored). READ-ONLY on Odoo.
|
| 18 |
+
"""
|
| 19 |
+
import datetime as dt
|
| 20 |
+
import json
|
| 21 |
+
import os
|
| 22 |
+
import sys
|
| 23 |
+
import time
|
| 24 |
+
from pathlib import Path
|
| 25 |
+
|
| 26 |
+
import duckdb
|
| 27 |
+
|
| 28 |
+
import core.odoo as O
|
| 29 |
+
|
| 30 |
+
_STORE_DIR = Path(__file__).resolve().parents[1] / "data" / "store"
|
| 31 |
+
|
| 32 |
+
# β MUTABLE ON PURPOSE, and read LATE by every function below (Python resolves a module global at
|
| 33 |
+
# CALL time), which is how `provision_tenant.py` already points a fresh instance at its own file.
|
| 34 |
+
# EXIT-4a (X7) formalises that: `AIOS_DUCKDB_PATH` lets a container name the file without an edit,
|
| 35 |
+
# `path_for()` owns the per-tenant naming convention, and `use_path()` is the SAFE way to switch β
|
| 36 |
+
# see its docstring for why a bare assignment is not.
|
| 37 |
+
# The DEFAULT is unchanged: <app root>/data/store/royal.duckdb.
|
| 38 |
+
DB_PATH = Path(os.environ.get("AIOS_DUCKDB_PATH") or (_STORE_DIR / "royal.duckdb"))
|
| 39 |
+
|
| 40 |
+
|
| 41 |
+
def path_for(tenant_key):
|
| 42 |
+
"""The canonical DuckDB file for a tenant slug. `royal-imports` keeps the historical
|
| 43 |
+
`royal.duckdb` name so tenant #0's existing 175 MB file is not orphaned by a rename.
|
| 44 |
+
|
| 45 |
+
File-per-tenant IS the isolation model (C1c): DuckDB has no row-level security, so the only
|
| 46 |
+
boundary that holds is the operating system's β one file, one tenant.
|
| 47 |
+
"""
|
| 48 |
+
key = str(tenant_key or "").strip().lower()
|
| 49 |
+
if key in ("", "royal-imports"):
|
| 50 |
+
return DB_PATH if key == "" else Path(
|
| 51 |
+
os.environ.get("AIOS_DUCKDB_PATH") or (_STORE_DIR / "royal.duckdb"))
|
| 52 |
+
safe = "".join(c if (c.isalnum() or c in "-_") else "_" for c in key)[:60]
|
| 53 |
+
return _STORE_DIR / f"{safe}.duckdb"
|
| 54 |
+
|
| 55 |
+
|
| 56 |
+
def use_path(path):
|
| 57 |
+
"""Repoint the store at `path`, CLOSING the process singleton first.
|
| 58 |
+
|
| 59 |
+
β A bare `datastore.DB_PATH = β¦` is not enough and is the more dangerous half of this
|
| 60 |
+
operation. `_instance()` caches ONE open connection for the life of the process, so after a
|
| 61 |
+
plain reassignment every read keeps being served from the PREVIOUSLY opened file β a silent
|
| 62 |
+
cross-tenant read, returning real rows that belong to somebody else. Closing the connection
|
| 63 |
+
and clearing the readiness memo (which is also per-file) is what makes the switch honest.
|
| 64 |
+
|
| 65 |
+
β AND CLOSING THE CONNECTION IS STILL NOT ENOUGH ON ITS OWN, because `ro_con()` caches a
|
| 66 |
+
cursor in a `threading.local`: this function can only clear the CALLING thread's, and
|
| 67 |
+
`ro_con`'s liveness probe (`SELECT 1`) SUCCEEDS on a cursor whose underlying connection was
|
| 68 |
+
closed out from under it in some cases β so another thread would keep answering from the old
|
| 69 |
+
file with no error to notice. The generation counter below is what closes that: `ro_con`
|
| 70 |
+
compares the generation its cursor was opened under and reopens when it has moved, which is
|
| 71 |
+
a check no individual thread can forget to do.
|
| 72 |
+
|
| 73 |
+
Returns the new path. Callers: `provision_tenant.py`, and any single-tenant worker process.
|
| 74 |
+
NOT a per-request operation β one process serves one analytical store at a time, which is why
|
| 75 |
+
`harness.runtime` hands out the PATH rather than switching the global underneath a request.
|
| 76 |
+
"""
|
| 77 |
+
global DB_PATH
|
| 78 |
+
with _INSTANCE_LOCK:
|
| 79 |
+
con = _INSTANCE.get("con")
|
| 80 |
+
if con is not None:
|
| 81 |
+
try:
|
| 82 |
+
con.close()
|
| 83 |
+
except Exception:
|
| 84 |
+
pass
|
| 85 |
+
_INSTANCE["con"] = None
|
| 86 |
+
_RO_TLS.__dict__.pop("cur", None)
|
| 87 |
+
_READY["ok"] = False
|
| 88 |
+
_GENERATION[0] += 1 # every other thread's cached cursor is now stale
|
| 89 |
+
DB_PATH = Path(path)
|
| 90 |
+
return DB_PATH
|
| 91 |
+
|
| 92 |
+
# Entity specs: Odoo model β store table. m2o fields land as <name>_id + <name>_name.
|
| 93 |
+
# 'archivable' entities are pulled with active in [True, False] (the re-SKU/merge rule).
|
| 94 |
+
ENTITIES = {
|
| 95 |
+
"sale_order": {
|
| 96 |
+
"model": "sale.order", "archivable": False,
|
| 97 |
+
"fields": ["name", "date_order", "partner_id", "team_id", "user_id", "state",
|
| 98 |
+
"amount_untaxed", "invoice_status", "write_date"],
|
| 99 |
+
},
|
| 100 |
+
"sale_order_line": {
|
| 101 |
+
"model": "sale.order.line", "archivable": False,
|
| 102 |
+
"fields": ["order_id", "order_partner_id", "product_id", "product_uom_qty",
|
| 103 |
+
"price_subtotal", "margin", "purchase_price", "write_date"],
|
| 104 |
+
},
|
| 105 |
+
"res_partner": {
|
| 106 |
+
"model": "res.partner", "archivable": True,
|
| 107 |
+
# `agent` (bool, labelled "Creditor/Agent") is THE discriminator between a real sales
|
| 108 |
+
# AGENT and an internal SALESPERSON who merely carries commission lines β the commission
|
| 109 |
+
# table itself cannot tell them apart. `salesman_as_agent` ("convert salesman into agent")
|
| 110 |
+
# marks agents who came from staff; the owner ruling on whether those count as agents is
|
| 111 |
+
# OUTSTANDING, so both flags are synced and neither reading is baked in.
|
| 112 |
+
"fields": ["name", "team_id", "city", "state_id", "country_id", "active",
|
| 113 |
+
"agent", "salesman_as_agent", "write_date"],
|
| 114 |
+
"backfill_fields": ["agent", "salesman_as_agent"], # added 2026-07-28
|
| 115 |
+
# m2m fields land as <base>_id (the FIRST linked id β the Customers-module convention:
|
| 116 |
+
# a customer's assigned agent is agent_ids[0], ~one per customer, MECE) + <field> as a
|
| 117 |
+
# JSON list of ALL ids. Declared explicitly: a 2-element m2m read would otherwise be
|
| 118 |
+
# indistinguishable from an m2o (id, name) pair in _flatten.
|
| 119 |
+
"m2m": ["agent_ids"],
|
| 120 |
+
},
|
| 121 |
+
"product_product": {
|
| 122 |
+
"model": "product.product", "archivable": True,
|
| 123 |
+
"fields": ["default_code", "name", "categ_id", "type", "standard_price", "active",
|
| 124 |
+
"write_date"],
|
| 125 |
+
},
|
| 126 |
+
"account_move": {
|
| 127 |
+
"model": "account.move", "archivable": False,
|
| 128 |
+
"fields": ["name", "move_type", "state", "invoice_date", "invoice_date_due",
|
| 129 |
+
"partner_id", "amount_untaxed_signed", "amount_residual_signed",
|
| 130 |
+
"payment_state", "write_date"],
|
| 131 |
+
},
|
| 132 |
+
"account_move_line": {
|
| 133 |
+
"model": "account.move.line", "archivable": False,
|
| 134 |
+
# display_type/move_type/parent_state/price_subtotal/product_id added 2026-07-28 so the
|
| 135 |
+
# invoice-LINE grain is queryable (topic `invoice_lines`).
|
| 136 |
+
# β display_type values here are 'product', 'cogs', 'payment_term', 'line_note',
|
| 137 |
+
# 'line_section'. Commission rows attach to 'cogs' lines as well as 'product' ones
|
| 138 |
+
# (96,936 of 147,160 in 2025). Filtering to 'product' is a GRAIN guard: measured
|
| 139 |
+
# 2026-07-29, omitting it does not move revenue (non-product lines carry
|
| 140 |
+
# price_subtotal = 0) but it inflates commission ROW COUNTS ~80% and would corrupt any
|
| 141 |
+
# count-of-lines measure.
|
| 142 |
+
"fields": ["move_id", "account_id", "partner_id", "date", "debit", "credit",
|
| 143 |
+
"balance", "display_type", "move_type", "parent_state", "price_subtotal",
|
| 144 |
+
"product_id", "write_date"],
|
| 145 |
+
"backfill_fields": ["display_type", "move_type", "parent_state", "price_subtotal",
|
| 146 |
+
"product_id"], # added 2026-07-28
|
| 147 |
+
# the BU bridge: account.move carries NO business unit (every invoice sits on team 1 β
|
| 148 |
+
# see [[invoice-bu-attribution]]), so Fisch/Royal on the invoice basis is only reachable
|
| 149 |
+
# through the originating sale order. sale_line_id (the FIRST linked sale line) is
|
| 150 |
+
# lossless here: ZERO 2025 invoice lines span more than one team (measured).
|
| 151 |
+
"m2m": ["sale_line_ids"],
|
| 152 |
+
},
|
| 153 |
+
"account_invoice_line_agent": {
|
| 154 |
+
# The OCA sale-commission module: one row per (invoice line Γ agent). This is the SECOND
|
| 155 |
+
# agent source β per-line commission attribution β and it names a different population
|
| 156 |
+
# than res_partner.agent_ids (the customer-master book). See topic `invoice_lines`.
|
| 157 |
+
# β TWO similarly-named FKs, do not confuse them:
|
| 158 |
+
# object_id -> account_move_line (THE grain; join on this)
|
| 159 |
+
# invoice_id -> account_move (the parent document)
|
| 160 |
+
"model": "account.invoice.line.agent", "archivable": False,
|
| 161 |
+
"fields": ["agent_id", "commission_id", "amount", "invoice_id", "object_id",
|
| 162 |
+
"invoice_date", "settled", "write_date"],
|
| 163 |
+
},
|
| 164 |
+
"account_account": {
|
| 165 |
+
# NOTE: no `active` field on account.account in this Odoo version (learned 2026-07-11 β
|
| 166 |
+
# a ('active','in',...) domain 500s); treat as non-archivable.
|
| 167 |
+
"model": "account.account", "archivable": False,
|
| 168 |
+
"fields": ["code", "name", "account_type", "write_date"],
|
| 169 |
+
},
|
| 170 |
+
}
|
| 171 |
+
|
| 172 |
+
BATCH = 2000
|
| 173 |
+
|
| 174 |
+
# Fields that are genuinely BOOLEAN. This list is load-bearing in TWO places: _cols types the
|
| 175 |
+
# column, and _flatten must NOT collapse a real False into NULL. Odoo returns False both for
|
| 176 |
+
# "empty" and for "boolean false", so a bool field missing from here silently becomes NULL β and
|
| 177 |
+
# for res_partner.agent that would erase the ONE flag separating an AGENT from a SALESPERSON
|
| 178 |
+
# (see [[invoice-line-agent-commission]]): every row would read "not an agent" indistinguishably
|
| 179 |
+
# from "unknown".
|
| 180 |
+
BOOL_FIELDS = ("active", "agent", "salesman_as_agent", "settled", "commission_free")
|
| 181 |
+
|
| 182 |
+
|
| 183 |
+
import threading as _thr
|
| 184 |
+
|
| 185 |
+
_RO_TLS = _thr.local()
|
| 186 |
+
_READY = {"ok": False}
|
| 187 |
+
_INSTANCE = {"con": None}
|
| 188 |
+
_INSTANCE_LOCK = _thr.Lock()
|
| 189 |
+
#: Bumped by `use_path()` whenever the store FILE changes. Every per-thread cursor is stamped with
|
| 190 |
+
#: the generation it was opened under, so a thread that never called `use_path` still cannot keep
|
| 191 |
+
#: reading the previous tenant's file β see `ro_con`. A plain "is the cursor alive?" probe cannot
|
| 192 |
+
#: answer that question, which is why this counter exists rather than more probing.
|
| 193 |
+
_GENERATION = [0]
|
| 194 |
+
|
| 195 |
+
|
| 196 |
+
def mark_ready():
|
| 197 |
+
"""In-process signal that the store is fully synced (every entity live). The sync writer calls
|
| 198 |
+
this after an all-live pass so a reader never has to (re)discover usability by touching the
|
| 199 |
+
file. Belt-and-suspenders alongside the shared-instance model below."""
|
| 200 |
+
_READY["ok"] = True
|
| 201 |
+
|
| 202 |
+
|
| 203 |
+
def _instance():
|
| 204 |
+
"""THE one process-wide read-write DuckDB connection. Readers take .cursor() off it and the
|
| 205 |
+
sync writer uses it too β DuckDB serves concurrent reads + a single writer from ONE instance
|
| 206 |
+
via MVCC, so a read NEVER contends with a sync pass for the file lock.
|
| 207 |
+
|
| 208 |
+
This is the fix for the 'connection error unless I refresh' bug (2026-07-16): an independent
|
| 209 |
+
read-only `duckdb.connect()` FAILS while the writer holds the lock ('Can't open a connection
|
| 210 |
+
to same database file with a different configuration than existing connections'); a cursor on
|
| 211 |
+
the shared instance does not. Openers must ensure the file is fully present first β ensure_seed
|
| 212 |
+
writes atomically (os.replace) and every read path guards on DB_PATH.exists() β so the instance
|
| 213 |
+
never binds a partial or absent file. (Trade-off: the app now holds the write lock for its whole
|
| 214 |
+
life, so a SEPARATE process β e.g. mcp_server.py β cannot open the same store file concurrently;
|
| 215 |
+
in production only the app runs, and that was already true during any sync pass.)"""
|
| 216 |
+
con = _INSTANCE["con"]
|
| 217 |
+
if con is not None:
|
| 218 |
+
return con
|
| 219 |
+
with _INSTANCE_LOCK:
|
| 220 |
+
if _INSTANCE["con"] is None:
|
| 221 |
+
DB_PATH.parent.mkdir(parents=True, exist_ok=True)
|
| 222 |
+
_INSTANCE["con"] = duckdb.connect(str(DB_PATH))
|
| 223 |
+
return _INSTANCE["con"]
|
| 224 |
+
|
| 225 |
+
|
| 226 |
+
def ro_cursor():
|
| 227 |
+
"""A read cursor on the shared instance β the contention-free way to read the store."""
|
| 228 |
+
return _instance().cursor()
|
| 229 |
+
|
| 230 |
+
|
| 231 |
+
def ready():
|
| 232 |
+
"""True once EVERY entity has completed its backfill (phase 'live'). Gate all store READS on
|
| 233 |
+
this: a mid-backfill store would return PARTIAL totals silently β worse than an error. Positive
|
| 234 |
+
results cache for the process (backfill never regresses). Reads via a shared-instance cursor,
|
| 235 |
+
so it is never mis-reported as 'not ready' just because a background sync pass is mid-flight."""
|
| 236 |
+
if _READY["ok"]:
|
| 237 |
+
return True
|
| 238 |
+
if not DB_PATH.exists():
|
| 239 |
+
return False
|
| 240 |
+
try:
|
| 241 |
+
cur = ro_cursor()
|
| 242 |
+
try:
|
| 243 |
+
phases = {r[0]: r[1] for r in
|
| 244 |
+
cur.execute("SELECT entity, phase FROM _sync_state").fetchall()}
|
| 245 |
+
finally:
|
| 246 |
+
cur.close()
|
| 247 |
+
except Exception:
|
| 248 |
+
return False
|
| 249 |
+
if all(phases.get(k) == "live" for k in ENTITIES):
|
| 250 |
+
_READY["ok"] = True
|
| 251 |
+
return True
|
| 252 |
+
return False
|
| 253 |
+
|
| 254 |
+
|
| 255 |
+
def ro_con():
|
| 256 |
+
"""Per-thread cached read CURSOR on the shared instance for page/module store reads (the
|
| 257 |
+
open cost dominates repeated small queries β the Expenses retrofit measured 7.7sβ1.0s). No
|
| 258 |
+
lock contention with the sync writer; a cursor that has gone bad is reopened once. REFUSES
|
| 259 |
+
until ready() β callers either fall back to live reads (modules) or relay a readable message
|
| 260 |
+
(Analyst).
|
| 261 |
+
|
| 262 |
+
β THE GENERATION CHECK IS A CORRECTNESS GUARD, NOT AN OPTIMISATION (EXIT-4a). The liveness
|
| 263 |
+
probe below asks "does this cursor still work?", which is a different question from "is this
|
| 264 |
+
cursor still pointing at the file we are supposed to be reading?". After `use_path()` switches
|
| 265 |
+
tenants, a cursor cached in ANOTHER thread's `threading.local` can still answer `SELECT 1` β
|
| 266 |
+
and would then serve that thread real, plausible rows from the PREVIOUS tenant's file, with no
|
| 267 |
+
exception anywhere. `use_path` bumps the generation; a cursor opened under an older one is
|
| 268 |
+
discarded here, which is the only place every thread is guaranteed to pass through."""
|
| 269 |
+
cur = getattr(_RO_TLS, "cur", None)
|
| 270 |
+
if cur is not None and getattr(_RO_TLS, "gen", None) != _GENERATION[0]:
|
| 271 |
+
try:
|
| 272 |
+
cur.close()
|
| 273 |
+
except Exception:
|
| 274 |
+
pass
|
| 275 |
+
cur, _RO_TLS.cur = None, None # the store moved underneath this thread
|
| 276 |
+
if cur is not None:
|
| 277 |
+
try:
|
| 278 |
+
cur.execute("SELECT 1") # cheap liveness probe
|
| 279 |
+
return cur
|
| 280 |
+
except Exception:
|
| 281 |
+
_RO_TLS.cur = None # stale β reopen below
|
| 282 |
+
if not ready():
|
| 283 |
+
raise RuntimeError("the tenant data store is completing its FIRST sync on this "
|
| 284 |
+
"deployment β dashboards work meanwhile (live reads); retry "
|
| 285 |
+
"store queries in a few minutes")
|
| 286 |
+
cur = ro_cursor()
|
| 287 |
+
_RO_TLS.cur = cur
|
| 288 |
+
_RO_TLS.gen = _GENERATION[0] # stamp WHICH file this cursor belongs to
|
| 289 |
+
return cur
|
| 290 |
+
|
| 291 |
+
|
| 292 |
+
SEED_DATASET = os.environ.get("STORE_SEED_DATASET", "royal-imports/cfo-os-data")
|
| 293 |
+
|
| 294 |
+
|
| 295 |
+
def ensure_seed():
|
| 296 |
+
"""Fresh/ephemeral deployment (HF Space disks reset on every rebuild): hydrate the store
|
| 297 |
+
from the PRIVATE dataset's seed snapshot β Analyst-ready in ~a minute instead of a 20-40 min
|
| 298 |
+
XML-RPC backfill; the auto-sync then closes the gap via write_date cursors. Also restores
|
| 299 |
+
the pilot views/routines when absent. Fail-quiet: no seed/token β normal backfill."""
|
| 300 |
+
import shutil
|
| 301 |
+
if DB_PATH.exists():
|
| 302 |
+
return False
|
| 303 |
+
tok = os.environ.get("HF_TOKEN")
|
| 304 |
+
if not tok:
|
| 305 |
+
return False
|
| 306 |
+
try:
|
| 307 |
+
from huggingface_hub import hf_hub_download
|
| 308 |
+
DB_PATH.parent.mkdir(parents=True, exist_ok=True)
|
| 309 |
+
p = hf_hub_download(SEED_DATASET, "store_seed/royal.duckdb",
|
| 310 |
+
repo_type="dataset", token=tok)
|
| 311 |
+
# Atomic install: copy to a temp path then os.replace β a reader that guards on
|
| 312 |
+
# DB_PATH.exists() must never bind a half-written file into the shared instance.
|
| 313 |
+
tmp = DB_PATH.with_name(DB_PATH.name + ".tmp")
|
| 314 |
+
shutil.copyfile(p, tmp)
|
| 315 |
+
os.replace(tmp, DB_PATH)
|
| 316 |
+
for fn in ("views.json", "routines.json"):
|
| 317 |
+
tgt = DB_PATH.parent / fn
|
| 318 |
+
if not tgt.exists():
|
| 319 |
+
try:
|
| 320 |
+
q = hf_hub_download(SEED_DATASET, f"store_seed/{fn}",
|
| 321 |
+
repo_type="dataset", token=tok)
|
| 322 |
+
shutil.copyfile(q, tgt)
|
| 323 |
+
except Exception:
|
| 324 |
+
pass
|
| 325 |
+
return True
|
| 326 |
+
except Exception:
|
| 327 |
+
return False
|
| 328 |
+
|
| 329 |
+
|
| 330 |
+
def connect():
|
| 331 |
+
"""A cursor on the shared instance for the SYNC WRITER (and status/validate). Returned as a
|
| 332 |
+
cursor so callers' con.close() frees the cursor without closing the process-wide instance;
|
| 333 |
+
one writer cursor is active at a time (the sync thread is sequential), readers use others."""
|
| 334 |
+
cur = _instance().cursor()
|
| 335 |
+
cur.execute("""CREATE TABLE IF NOT EXISTS _sync_state (
|
| 336 |
+
entity VARCHAR PRIMARY KEY, phase VARCHAR, last_id BIGINT,
|
| 337 |
+
cursor_wd VARCHAR, rows BIGINT, updated_at VARCHAR)""")
|
| 338 |
+
return cur
|
| 339 |
+
|
| 340 |
+
|
| 341 |
+
def _flatten(rec, fields, m2m=()):
|
| 342 |
+
out = {"id": rec["id"]}
|
| 343 |
+
for f in m2m:
|
| 344 |
+
v = rec.get(f) or []
|
| 345 |
+
base = f.removesuffix("_ids")
|
| 346 |
+
out[f"{base}_id"] = v[0] if v else None
|
| 347 |
+
out[f] = json.dumps(list(v)) if v else None
|
| 348 |
+
for f in fields:
|
| 349 |
+
v = rec.get(f)
|
| 350 |
+
if isinstance(v, (list, tuple)) and len(v) == 2 and isinstance(v[0], int):
|
| 351 |
+
out[f"{f.removesuffix('_id')}_id"] = v[0]
|
| 352 |
+
out[f"{f.removesuffix('_id')}_name"] = str(v[1])
|
| 353 |
+
elif isinstance(v, (list, tuple)):
|
| 354 |
+
out[f] = json.dumps(v)
|
| 355 |
+
elif v is False and f not in BOOL_FIELDS: # Odoo False = NULL for non-bool fields
|
| 356 |
+
out[f] = None
|
| 357 |
+
else:
|
| 358 |
+
out[f] = v
|
| 359 |
+
return out
|
| 360 |
+
|
| 361 |
+
|
| 362 |
+
def _cols(spec):
|
| 363 |
+
cols = ["id BIGINT PRIMARY KEY"]
|
| 364 |
+
for f in spec["fields"]:
|
| 365 |
+
base = f.removesuffix("_id")
|
| 366 |
+
if f.endswith("_id"):
|
| 367 |
+
cols += [f"{base}_id BIGINT", f"{base}_name VARCHAR"]
|
| 368 |
+
elif f in BOOL_FIELDS:
|
| 369 |
+
cols.append(f"{f} BOOLEAN")
|
| 370 |
+
elif f in ("amount_untaxed", "amount_untaxed_signed", "amount_residual_signed",
|
| 371 |
+
"price_subtotal", "margin", "purchase_price", "product_uom_qty",
|
| 372 |
+
"standard_price", "debit", "credit", "balance", "amount"):
|
| 373 |
+
cols.append(f"{f} DOUBLE")
|
| 374 |
+
else:
|
| 375 |
+
cols.append(f"{f} VARCHAR")
|
| 376 |
+
for f in spec.get("m2m") or []:
|
| 377 |
+
base = f.removesuffix("_ids")
|
| 378 |
+
cols += [f"{base}_id BIGINT", f"{f} VARCHAR"]
|
| 379 |
+
return cols
|
| 380 |
+
|
| 381 |
+
|
| 382 |
+
def _ensure_table(con, key, spec):
|
| 383 |
+
con.execute(f"CREATE TABLE IF NOT EXISTS {key} ({', '.join(_cols(spec))})")
|
| 384 |
+
|
| 385 |
+
|
| 386 |
+
def _ensure_columns(con, key, spec):
|
| 387 |
+
"""Schema evolution: existing stores/seeds predate columns a newer spec introduces β
|
| 388 |
+
CREATE TABLE IF NOT EXISTS won't add them. Returns the newly added column names."""
|
| 389 |
+
have = {r[1] for r in con.execute(f"PRAGMA table_info('{key}')").fetchall()}
|
| 390 |
+
added = []
|
| 391 |
+
for c in _cols(spec):
|
| 392 |
+
name = c.split()[0]
|
| 393 |
+
if name not in have:
|
| 394 |
+
con.execute(f"ALTER TABLE {key} ADD COLUMN {c}")
|
| 395 |
+
added.append(name)
|
| 396 |
+
return added
|
| 397 |
+
|
| 398 |
+
|
| 399 |
+
def _field_cols(f):
|
| 400 |
+
"""The store column(s) one spec field materialises into (m2o fields become _id + _name)."""
|
| 401 |
+
if f.endswith("_id"):
|
| 402 |
+
base = f.removesuffix("_id")
|
| 403 |
+
return [f"{base}_id", f"{base}_name"]
|
| 404 |
+
return [f]
|
| 405 |
+
|
| 406 |
+
|
| 407 |
+
def _backfill_columns(con, key, spec, log=print):
|
| 408 |
+
"""Scalar/m2o schema evolution β the twin of _backfill_m2m, and for the same reason.
|
| 409 |
+
|
| 410 |
+
_ensure_columns ALTERs a new column in, but existing rows NEVER re-sync (WE moved the schema;
|
| 411 |
+
their write_date didn't move), so a newly added field stays NULL forever β silently, and NULL
|
| 412 |
+
is indistinguishable from a legitimately empty value. That is how a store starts answering
|
| 413 |
+
'no rows match' instead of erroring. Re-pull just the affected fields for every row,
|
| 414 |
+
paginated by id, marked durably so a crash retries rather than declaring completion.
|
| 415 |
+
|
| 416 |
+
The fields are DECLARED in the spec (`backfill_fields`), never inferred from "which columns
|
| 417 |
+
did _ensure_columns just add" β that inference is wrong on the second run, when the ALTER has
|
| 418 |
+
already happened and the list comes back empty while the data is still missing. Declaring it
|
| 419 |
+
makes the migration re-entrant and reviewable. Leave the entry in place after it completes;
|
| 420 |
+
the durable marker, not the absence of the declaration, is what stops it re-running.
|
| 421 |
+
"""
|
| 422 |
+
fields = [f for f in (spec.get("backfill_fields") or []) if f != "write_date"]
|
| 423 |
+
if not fields:
|
| 424 |
+
return
|
| 425 |
+
marker = f"{key}.cols.{','.join(sorted(fields))}"
|
| 426 |
+
st = con.execute("SELECT phase, last_id, rows FROM _sync_state WHERE entity=?",
|
| 427 |
+
[marker]).fetchone()
|
| 428 |
+
if st and st[0] == "done":
|
| 429 |
+
return
|
| 430 |
+
# RESUMABLE, not restart-from-zero: these passes run to ~1M rows, and a migration that loses
|
| 431 |
+
# an hour of work to one dropped connection gets skipped by the next person under time
|
| 432 |
+
# pressure. Progress is checkpointed every page; 'done' is written only at the end, so an
|
| 433 |
+
# interrupted run resumes at the last committed id instead of re-pulling or, worse, declaring
|
| 434 |
+
# completion it never reached.
|
| 435 |
+
last, n = (st[1] or 0, st[2] or 0) if st else (0, 0)
|
| 436 |
+
if last:
|
| 437 |
+
log(f" {key}: column backfill resuming at id {last:,} ({n:,} rows already done)")
|
| 438 |
+
cols = [c for f in fields for c in _field_cols(f)]
|
| 439 |
+
setter = ", ".join(f"{c}=?" for c in cols)
|
| 440 |
+
while True:
|
| 441 |
+
recs = O.search_read(spec["model"], _base_domain(spec) + [("id", ">", last)],
|
| 442 |
+
fields, limit=5000, order="id asc")
|
| 443 |
+
if not recs:
|
| 444 |
+
break
|
| 445 |
+
rows = [_flatten(r, fields) for r in recs]
|
| 446 |
+
con.executemany(f"UPDATE {key} SET {setter} WHERE id=?",
|
| 447 |
+
[[r.get(c) for c in cols] + [r["id"]] for r in rows])
|
| 448 |
+
last, n = recs[-1]["id"], n + len(recs)
|
| 449 |
+
con.execute("INSERT OR REPLACE INTO _sync_state VALUES (?,?,?,?,?,?)",
|
| 450 |
+
[marker, "running", last, None, n, time.strftime("%Y-%m-%d %H:%M:%S")])
|
| 451 |
+
log(f" {key}: column backfill {'+'.join(fields)} -> id {last:,} ({n:,} rows)")
|
| 452 |
+
con.execute("INSERT OR REPLACE INTO _sync_state VALUES (?,?,?,?,?,?)",
|
| 453 |
+
[marker, "done", last, None, n, time.strftime("%Y-%m-%d %H:%M:%S")])
|
| 454 |
+
log(f" {key}: column backfill complete -> {n:,} rows")
|
| 455 |
+
|
| 456 |
+
|
| 457 |
+
def _backfill_m2m(con, key, spec, log=print):
|
| 458 |
+
"""One-time targeted backfill after a schema migration: existing rows never re-sync (their
|
| 459 |
+
write_date didn't move when WE added a column), so pull every record where the m2m field is
|
| 460 |
+
SET and update in place. Rows with the field empty keep NULL β correct and free. Completion
|
| 461 |
+
is a durable _sync_state marker ('<key>.m2m.<field>') β NOT 'column just added': a crash
|
| 462 |
+
between the ALTER and this backfill (seen live: the SSL-failed first run) must retry."""
|
| 463 |
+
for f in spec.get("m2m") or []:
|
| 464 |
+
marker = f"{key}.m2m.{f}"
|
| 465 |
+
if con.execute("SELECT 1 FROM _sync_state WHERE entity=?", [marker]).fetchone():
|
| 466 |
+
continue
|
| 467 |
+
base = f.removesuffix("_ids")
|
| 468 |
+
# PAGINATED by id. A single limit=100000 call silently TRUNCATED here: account_move_line
|
| 469 |
+
# has 228,067 rows with sale_line_ids set, so 56% of the BU bridge would have gone
|
| 470 |
+
# missing with no error β a partial backfill that reads as a complete one (the
|
| 471 |
+
# [[no-unverifiable-aggregates]] no-silent-caps rule).
|
| 472 |
+
rows, last, page = [], 0, 20000
|
| 473 |
+
while True:
|
| 474 |
+
recs = O.search_read(spec["model"],
|
| 475 |
+
_base_domain(spec) + [(f, "!=", False), ("id", ">", last)],
|
| 476 |
+
[f], limit=page, order="id asc")
|
| 477 |
+
if not recs:
|
| 478 |
+
break
|
| 479 |
+
rows += [(r[f][0], json.dumps(list(r[f])), r["id"]) for r in recs if r.get(f)]
|
| 480 |
+
last = recs[-1]["id"]
|
| 481 |
+
log(f" {key}: m2m {f} scanned to id {last:,} ({len(rows):,} rows)")
|
| 482 |
+
if rows:
|
| 483 |
+
con.executemany(f"UPDATE {key} SET {base}_id=?, {f}=? WHERE id=?", rows)
|
| 484 |
+
con.execute("INSERT OR REPLACE INTO _sync_state VALUES (?,?,?,?,?,?)",
|
| 485 |
+
[marker, "done", 0, None, len(rows), time.strftime("%Y-%m-%d %H:%M:%S")])
|
| 486 |
+
log(f" {key}: m2m backfill {f} -> {len(rows):,} rows updated")
|
| 487 |
+
|
| 488 |
+
|
| 489 |
+
def _upsert(con, key, spec, recs):
|
| 490 |
+
if not recs:
|
| 491 |
+
return
|
| 492 |
+
rows = [_flatten(r, spec["fields"], spec.get("m2m") or ()) for r in recs]
|
| 493 |
+
colnames = [c.split()[0] for c in _cols(spec)]
|
| 494 |
+
ids = [r["id"] for r in rows]
|
| 495 |
+
con.execute(f"DELETE FROM {key} WHERE id IN ({','.join(map(str, ids))})")
|
| 496 |
+
con.executemany(
|
| 497 |
+
f"INSERT INTO {key} ({', '.join(colnames)}) VALUES ({', '.join('?' for _ in colnames)})",
|
| 498 |
+
[[r.get(c) for c in colnames] for r in rows])
|
| 499 |
+
|
| 500 |
+
|
| 501 |
+
def _state(con, key):
|
| 502 |
+
row = con.execute("SELECT phase, last_id, cursor_wd, rows FROM _sync_state WHERE entity=?",
|
| 503 |
+
[key]).fetchone()
|
| 504 |
+
return {"phase": row[0], "last_id": row[1], "cursor_wd": row[2], "rows": row[3]} if row else None
|
| 505 |
+
|
| 506 |
+
|
| 507 |
+
def _save_state(con, key, phase, last_id, cursor_wd):
|
| 508 |
+
n = con.execute(f"SELECT count(*) FROM {key}").fetchone()[0]
|
| 509 |
+
con.execute("""INSERT OR REPLACE INTO _sync_state VALUES (?,?,?,?,?,?)""",
|
| 510 |
+
[key, phase, last_id, cursor_wd, n, time.strftime("%Y-%m-%d %H:%M:%S")])
|
| 511 |
+
|
| 512 |
+
|
| 513 |
+
def _base_domain(spec):
|
| 514 |
+
return [("active", "in", [True, False])] if spec["archivable"] else []
|
| 515 |
+
|
| 516 |
+
|
| 517 |
+
def sync_entity(key, max_batches=200, batch=BATCH, log=print):
|
| 518 |
+
"""One bounded, resumable sync pass for an entity. Backfill (by id) β live (by write_date).
|
| 519 |
+
Returns (phase, pulled_rows)."""
|
| 520 |
+
spec = ENTITIES[key]
|
| 521 |
+
con = connect()
|
| 522 |
+
_ensure_table(con, key, spec)
|
| 523 |
+
_ensure_columns(con, key, spec)
|
| 524 |
+
_backfill_columns(con, key, spec, log=log)
|
| 525 |
+
_backfill_m2m(con, key, spec, log=log)
|
| 526 |
+
st = _state(con, key) or {"phase": "backfill", "last_id": 0, "cursor_wd": None, "rows": 0}
|
| 527 |
+
fields = spec["fields"] + list(spec.get("m2m") or [])
|
| 528 |
+
pulled = 0
|
| 529 |
+
try:
|
| 530 |
+
if st["phase"] == "backfill":
|
| 531 |
+
last_id = st["last_id"] or 0
|
| 532 |
+
for i in range(max_batches):
|
| 533 |
+
dom = _base_domain(spec) + [("id", ">", last_id)]
|
| 534 |
+
recs = O.search_read(spec["model"], dom, ["id"] + fields,
|
| 535 |
+
limit=batch, order="id asc")
|
| 536 |
+
if not recs:
|
| 537 |
+
# backfill complete β switch to live mode anchored at max write_date seen
|
| 538 |
+
wd = con.execute(f"SELECT max(write_date) FROM {key}").fetchone()[0]
|
| 539 |
+
_save_state(con, key, "live", last_id, wd or "1970-01-01 00:00:00")
|
| 540 |
+
log(f" {key}: BACKFILL COMPLETE ({st['rows'] + pulled:,} rows) -> live mode")
|
| 541 |
+
return "live", pulled
|
| 542 |
+
_upsert(con, key, spec, recs)
|
| 543 |
+
last_id = recs[-1]["id"]
|
| 544 |
+
pulled += len(recs)
|
| 545 |
+
_save_state(con, key, "backfill", last_id, None)
|
| 546 |
+
if (i + 1) % 10 == 0:
|
| 547 |
+
log(f" {key}: backfill β¦{pulled:,} rows (idβ€{last_id})")
|
| 548 |
+
log(f" {key}: backfill PAUSED at {pulled:,} rows this run (bounded; resumes)")
|
| 549 |
+
return "backfill", pulled
|
| 550 |
+
# live mode: write_date >= cursor (idempotent overlap; upsert dedupes). MASS-STAMP
|
| 551 |
+
# FALLBACK (found live 2026-07-17): a batch job can stamp >batch rows with ONE
|
| 552 |
+
# write_date second (an Odoo recompute stamped ~88k account_move rows '2026-07-15
|
| 553 |
+
# 16:35:03'; stored values carry microseconds, XML-RPC strings don't) β then the
|
| 554 |
+
# write_date cursor can NEVER advance and every pass re-pulls the same first page
|
| 555 |
+
# forever. When a full batch leaves the cursor unchanged we switch to an ID-WALK over
|
| 556 |
+
# write_date >= cursor (order id asc, id > tie), checkpointed as 'cursor|<id>' in
|
| 557 |
+
# cursor_wd so a killed walk resumes; on completion the cursor jumps one second past
|
| 558 |
+
# the stamp (safe: the walk covered every row in that second, and the stamp is in the
|
| 559 |
+
# past β guarded by a 2-minute recency check).
|
| 560 |
+
raw = st["cursor_wd"] or "1970-01-01 00:00:00"
|
| 561 |
+
cursor, _pipe, _tie = raw.partition("|")
|
| 562 |
+
walking, tie_id = bool(_pipe), int(_tie or 0)
|
| 563 |
+
walk_max = cursor
|
| 564 |
+
for _ in range(max_batches):
|
| 565 |
+
if walking:
|
| 566 |
+
dom = _base_domain(spec) + [("write_date", ">=", cursor), ("id", ">", tie_id)]
|
| 567 |
+
recs = O.search_read(spec["model"], dom, ["id"] + fields,
|
| 568 |
+
limit=batch, order="id asc")
|
| 569 |
+
if not recs: # walk complete β advance past the stamp
|
| 570 |
+
try:
|
| 571 |
+
wm = dt.datetime.strptime(walk_max, "%Y-%m-%d %H:%M:%S")
|
| 572 |
+
if wm < dt.datetime.utcnow() - dt.timedelta(minutes=2):
|
| 573 |
+
cursor = (wm + dt.timedelta(seconds=1)).strftime("%Y-%m-%d %H:%M:%S")
|
| 574 |
+
else:
|
| 575 |
+
cursor = walk_max
|
| 576 |
+
except ValueError:
|
| 577 |
+
cursor = walk_max
|
| 578 |
+
walking = False
|
| 579 |
+
log(f" {key}: id-walk complete -> cursor {cursor}")
|
| 580 |
+
break
|
| 581 |
+
_upsert(con, key, spec, recs)
|
| 582 |
+
pulled += len(recs)
|
| 583 |
+
tie_id = recs[-1]["id"]
|
| 584 |
+
walk_max = max(walk_max, max(str(r["write_date"]) for r in recs))
|
| 585 |
+
_save_state(con, key, "live", st["last_id"], f"{cursor}|{tie_id}")
|
| 586 |
+
continue
|
| 587 |
+
dom = _base_domain(spec) + [("write_date", ">=", cursor)]
|
| 588 |
+
recs = O.search_read(spec["model"], dom, ["id"] + fields,
|
| 589 |
+
limit=batch, order="write_date asc, id asc")
|
| 590 |
+
if not recs or (len(recs) == 1 and str(recs[0].get("write_date")) == cursor
|
| 591 |
+
and pulled == 0 and _already(con, key, recs[0])):
|
| 592 |
+
break
|
| 593 |
+
new_cursor = str(recs[-1]["write_date"])
|
| 594 |
+
if new_cursor == cursor and len(recs) == batch:
|
| 595 |
+
# full batch stuck on one second: enter the id-walk from id 0 (the write_date
|
| 596 |
+
# ordering shuffles ids within the stamp second, so the batch just pulled is
|
| 597 |
+
# NOT an id prefix β restart coverage by id; upserts make the overlap free)
|
| 598 |
+
walking, tie_id, walk_max = True, 0, cursor
|
| 599 |
+
_save_state(con, key, "live", st["last_id"], f"{cursor}|0")
|
| 600 |
+
log(f" {key}: mass-stamp second at {cursor} -> switching to id-walk")
|
| 601 |
+
continue
|
| 602 |
+
_upsert(con, key, spec, recs)
|
| 603 |
+
pulled += len(recs)
|
| 604 |
+
if new_cursor == cursor and len(recs) < batch:
|
| 605 |
+
cursor = new_cursor
|
| 606 |
+
break
|
| 607 |
+
cursor = new_cursor
|
| 608 |
+
# Checkpoint the cursor EVERY batch: a big delta (88k account_move rows after the
|
| 609 |
+
# 2026-07-15 recompute) means a killed pass otherwise re-pulls everything β
|
| 610 |
+
# upserts are idempotent, but the wasted RPC is real (learned 2026-07-17).
|
| 611 |
+
_save_state(con, key, "live", st["last_id"], cursor)
|
| 612 |
+
_save_state(con, key, "live", st["last_id"],
|
| 613 |
+
f"{cursor}|{tie_id}" if walking else cursor)
|
| 614 |
+
if pulled:
|
| 615 |
+
log(f" {key}: live sync +{pulled:,} rows (cursor {cursor}"
|
| 616 |
+
+ (f" walking id>{tie_id}" if walking else "") + ")")
|
| 617 |
+
return "live", pulled
|
| 618 |
+
finally:
|
| 619 |
+
con.close()
|
| 620 |
+
|
| 621 |
+
|
| 622 |
+
def _already(con, key, rec):
|
| 623 |
+
return bool(con.execute(f"SELECT 1 FROM {key} WHERE id=?", [rec["id"]]).fetchone())
|
| 624 |
+
|
| 625 |
+
|
| 626 |
+
def sync_all(entities=None, max_batches=200, log=print):
|
| 627 |
+
out = {}
|
| 628 |
+
for key in (entities or list(ENTITIES)):
|
| 629 |
+
t0 = time.time()
|
| 630 |
+
phase, pulled = sync_entity(key, max_batches=max_batches, log=log)
|
| 631 |
+
out[key] = {"phase": phase, "pulled": pulled, "secs": round(time.time() - t0, 1)}
|
| 632 |
+
# A full pass that lands every entity in live mode latches process readiness in-process β so
|
| 633 |
+
# readers never race the writer's lock to (re)discover the store is usable.
|
| 634 |
+
if entities is None and out and all(v["phase"] == "live" for v in out.values()):
|
| 635 |
+
mark_ready()
|
| 636 |
+
return out
|
| 637 |
+
|
| 638 |
+
|
| 639 |
+
def reconcile_deletes(entities=None, log=print):
|
| 640 |
+
"""Remove store rows hard-deleted in Odoo. unlink() doesn't move write_date, so the cursor
|
| 641 |
+
sync can never see a deletion β the count parity DETECTS the drift (by design); this repairs
|
| 642 |
+
it. Pulls the full live id set per entity in bounded id-ascending pages (id-only reads are
|
| 643 |
+
cheap) and deletes store ids not present live. Run from sync_runner / maintenance, not the
|
| 644 |
+
app's frequent passes."""
|
| 645 |
+
con = connect()
|
| 646 |
+
try:
|
| 647 |
+
out = {}
|
| 648 |
+
for key in (entities or list(ENTITIES)):
|
| 649 |
+
spec = ENTITIES[key]
|
| 650 |
+
live_ids, last = set(), 0
|
| 651 |
+
while True:
|
| 652 |
+
recs = O.search_read(spec["model"], _base_domain(spec) + [("id", ">", last)],
|
| 653 |
+
["id"], limit=50000, order="id asc")
|
| 654 |
+
if not recs:
|
| 655 |
+
break
|
| 656 |
+
live_ids.update(r["id"] for r in recs)
|
| 657 |
+
last = recs[-1]["id"]
|
| 658 |
+
store_ids = {r[0] for r in con.execute(f"SELECT id FROM {key}").fetchall()}
|
| 659 |
+
dead = store_ids - live_ids
|
| 660 |
+
if dead:
|
| 661 |
+
con.execute("CREATE TEMP TABLE IF NOT EXISTS _dead(id BIGINT)")
|
| 662 |
+
con.execute("DELETE FROM _dead")
|
| 663 |
+
con.executemany("INSERT INTO _dead VALUES (?)", [[i] for i in dead])
|
| 664 |
+
con.execute(f"DELETE FROM {key} WHERE id IN (SELECT id FROM _dead)")
|
| 665 |
+
con.execute("DELETE FROM _dead")
|
| 666 |
+
n = con.execute(f"SELECT count(*) FROM {key}").fetchone()[0]
|
| 667 |
+
con.execute("UPDATE _sync_state SET rows=? WHERE entity=?", [n, key])
|
| 668 |
+
out[key] = len(dead)
|
| 669 |
+
log(f" {key}: reconciled {len(dead):,} hard-deleted rows")
|
| 670 |
+
return out
|
| 671 |
+
finally:
|
| 672 |
+
con.close()
|
| 673 |
+
|
| 674 |
+
|
| 675 |
+
def status():
|
| 676 |
+
con = connect()
|
| 677 |
+
try:
|
| 678 |
+
return {r[0]: {"phase": r[1], "cursor": r[3], "rows": r[4], "updated": r[5]}
|
| 679 |
+
for r in con.execute("SELECT * FROM _sync_state").fetchall()}
|
| 680 |
+
finally:
|
| 681 |
+
con.close()
|
| 682 |
+
|
| 683 |
+
|
| 684 |
+
# ------------------------------------------------------------------ parity (store vs LIVE Odoo)
|
| 685 |
+
|
| 686 |
+
def _check(name, ours, live, tol=0.01):
|
| 687 |
+
gap = abs((ours or 0) - (live or 0))
|
| 688 |
+
return {"check": name, "ok": gap <= tol, "gap": round(gap, 4), "ours": ours, "live": live}
|
| 689 |
+
|
| 690 |
+
|
| 691 |
+
def validate(pre=None, months=("2026-01-01", "2026-07-01")):
|
| 692 |
+
"""Raw-fidelity parity: the store must mirror live Odoo β row counts per synced entity +
|
| 693 |
+
unscoped monetary sums over a window, to the cent. Only checks entities in LIVE phase."""
|
| 694 |
+
con = connect()
|
| 695 |
+
out = []
|
| 696 |
+
try:
|
| 697 |
+
live_state = status()
|
| 698 |
+
d0, d1 = months
|
| 699 |
+
for key, spec in ENTITIES.items():
|
| 700 |
+
if live_state.get(key, {}).get("phase") != "live":
|
| 701 |
+
continue
|
| 702 |
+
ours = con.execute(f"SELECT count(*) FROM {key}").fetchone()[0]
|
| 703 |
+
live = O.get_odoo().search_count(spec["model"], _base_domain(spec))
|
| 704 |
+
out.append(_check(f"store.{key} row count == live search_count", ours, live, tol=0))
|
| 705 |
+
if live_state.get("sale_order_line", {}).get("phase") == "live":
|
| 706 |
+
ours = con.execute(
|
| 707 |
+
"SELECT coalesce(sum(l.price_subtotal),0) FROM sale_order_line l "
|
| 708 |
+
"JOIN sale_order o ON o.id = l.order_id "
|
| 709 |
+
"WHERE o.date_order >= ? AND o.date_order < ?", [d0, d1]).fetchone()[0]
|
| 710 |
+
live = O.sum_field("sale.order.line",
|
| 711 |
+
[("order_id.date_order", ">=", f"{d0} 00:00:00"),
|
| 712 |
+
("order_id.date_order", "<", f"{d1} 00:00:00")],
|
| 713 |
+
"price_subtotal")
|
| 714 |
+
out.append(_check(f"store Ξ£ line revenue [{d0}..{d1}) == live (unscoped)", ours, live))
|
| 715 |
+
if live_state.get("sale_order", {}).get("phase") == "live":
|
| 716 |
+
ours = con.execute(
|
| 717 |
+
"SELECT coalesce(sum(amount_untaxed),0) FROM sale_order "
|
| 718 |
+
"WHERE date_order >= ? AND date_order < ?", [d0, d1]).fetchone()[0]
|
| 719 |
+
live = O.sum_field("sale.order",
|
| 720 |
+
[("date_order", ">=", f"{d0} 00:00:00"),
|
| 721 |
+
("date_order", "<", f"{d1} 00:00:00")], "amount_untaxed")
|
| 722 |
+
out.append(_check(f"store Ξ£ order amount_untaxed [{d0}..{d1}) == live (unscoped)",
|
| 723 |
+
ours, live))
|
| 724 |
+
finally:
|
| 725 |
+
con.close()
|
| 726 |
+
return out
|
| 727 |
+
|
| 728 |
+
|
| 729 |
+
if __name__ == "__main__":
|
| 730 |
+
args = sys.argv[1:]
|
| 731 |
+
if args and args[0] == "status":
|
| 732 |
+
for k, v in status().items():
|
| 733 |
+
print(f" {k:18s} {v['phase']:9s} rows={v['rows']:>9,} cursor={v['cursor']} ({v['updated']})")
|
| 734 |
+
elif args and args[0] == "reconcile":
|
| 735 |
+
ents = args[1].split(",") if len(args) > 1 else None
|
| 736 |
+
for k, n in reconcile_deletes(ents).items():
|
| 737 |
+
print(f" {k:18s} -{n:,} hard-deleted rows")
|
| 738 |
+
elif args and args[0] == "validate":
|
| 739 |
+
ok = True
|
| 740 |
+
for c in validate():
|
| 741 |
+
mark = "OK " if c["ok"] else "XX "
|
| 742 |
+
ok &= c["ok"]
|
| 743 |
+
print(f" {mark}{c['check'][:80]} gap={c['gap']}")
|
| 744 |
+
print("PARITY:", "GREEN" if ok else "RED")
|
| 745 |
+
sys.exit(0 if ok else 1)
|
| 746 |
+
else:
|
| 747 |
+
ents = args[0].split(",") if args else None
|
| 748 |
+
mb = int(args[1]) if len(args) > 1 else 200
|
| 749 |
+
for k, v in sync_all(ents, max_batches=mb).items():
|
| 750 |
+
print(f" {k:18s} {v['phase']:9s} +{v['pulled']:,} rows in {v['secs']}s")
|
platform/harness/evals.py
ADDED
|
@@ -0,0 +1,134 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""harness/evals.py β the Analyst eval gate (OM-4, 2026-07-11).
|
| 2 |
+
|
| 3 |
+
Runs the golden set (evals/analyst_golden.yml β truths from the parity-proven semantic layer over
|
| 4 |
+
CLOSED windows) against the Analyst and scores DETERMINISTICALLY: expected number found in the
|
| 5 |
+
answer (tolerance-aware, handles $/commas/% forms), required substrings, required tools in the
|
| 6 |
+
trace, required artifacts. No LLM judge in v1 β the validate() culture applied to AI answers.
|
| 7 |
+
|
| 8 |
+
Gate rule (from the plan): run before any deploy that touches model/ or the Analyst; ship only on
|
| 9 |
+
pass-rate >= the bar (start 80%, tighten as the model/skills improve).
|
| 10 |
+
|
| 11 |
+
Usage:
|
| 12 |
+
python -X utf8 -c "import ssl; ssl._create_default_https_context=ssl._create_unverified_context;
|
| 13 |
+
import harness.evals as E; E.run()" # live (needs OPENROUTER_API_KEY)
|
| 14 |
+
E.run(chat_fn=scripted) # offline harness test
|
| 15 |
+
"""
|
| 16 |
+
import re
|
| 17 |
+
from pathlib import Path
|
| 18 |
+
|
| 19 |
+
import yaml
|
| 20 |
+
|
| 21 |
+
GOLDEN = Path(__file__).resolve().parents[1] / "evals" / "analyst_golden.yml"
|
| 22 |
+
|
| 23 |
+
|
| 24 |
+
def _numbers(text):
|
| 25 |
+
"""Every number in the answer, normalized: $1,234.56 -> 1234.56; '59%'/'59.0 %' -> 0.59 too."""
|
| 26 |
+
out = []
|
| 27 |
+
for m in re.finditer(r"\$?([\d,]+(?:\.\d+)?)\s*(%?)", text):
|
| 28 |
+
try:
|
| 29 |
+
v = float(m.group(1).replace(",", ""))
|
| 30 |
+
except ValueError:
|
| 31 |
+
continue
|
| 32 |
+
out.append(v)
|
| 33 |
+
if m.group(2): # percent form: also offer the fraction
|
| 34 |
+
out.append(v / 100.0)
|
| 35 |
+
return out
|
| 36 |
+
|
| 37 |
+
|
| 38 |
+
def _norm_ws(s):
|
| 39 |
+
"""Collapse ALL unicode whitespace to single spaces. Models emit U+202F (narrow no-break
|
| 40 |
+
space) and friends inside names β 'Poppy\\u202fFlowers' failed a plain substring check on
|
| 41 |
+
two different models (the 'top_customer flake', diagnosed 2026-07-12)."""
|
| 42 |
+
return re.sub(r"\s+", " ", s)
|
| 43 |
+
|
| 44 |
+
|
| 45 |
+
def _check(item, result):
|
| 46 |
+
exp = item.get("expect") or {}
|
| 47 |
+
answer = _norm_ws(result.get("answer") or "")
|
| 48 |
+
reasons = []
|
| 49 |
+
if result.get("exhausted"):
|
| 50 |
+
reasons.append("tool budget exhausted")
|
| 51 |
+
if "value" in exp:
|
| 52 |
+
want = float(exp["value"])
|
| 53 |
+
tol = exp.get("abs", exp.get("rel", 0.001) * abs(want) or 0.01)
|
| 54 |
+
nums = _numbers(answer)
|
| 55 |
+
if exp.get("allow_pct_form"):
|
| 56 |
+
nums += [n for n in list(nums)]
|
| 57 |
+
if not any(abs(n - want) <= tol for n in nums):
|
| 58 |
+
reasons.append(f"expected {want:,.2f} (Β±{tol:,.2f}) not in answer")
|
| 59 |
+
for s in exp.get("contains", []):
|
| 60 |
+
if _norm_ws(s).lower() not in answer.lower():
|
| 61 |
+
reasons.append(f"missing substring {s!r}")
|
| 62 |
+
# WRONG-ANSWER guard (2026-07-28). `contains` can only prove an answer said the right thing;
|
| 63 |
+
# some failures are about saying a WRONG thing that a right answer never says β e.g. naming an
|
| 64 |
+
# internal salesperson as an agent ([[invoice-line-agent-commission]]). Those need a negative
|
| 65 |
+
# assertion or the eval passes on an answer that is confidently incorrect.
|
| 66 |
+
for s in exp.get("not_contains", []):
|
| 67 |
+
if _norm_ws(s).lower() in answer.lower():
|
| 68 |
+
reasons.append(f"forbidden substring {s!r} present")
|
| 69 |
+
used = [t["tool"] for t in result.get("tool_trace", [])]
|
| 70 |
+
for t in exp.get("tools", []):
|
| 71 |
+
if t not in used:
|
| 72 |
+
reasons.append(f"tool {t!r} not used (trace: {used})")
|
| 73 |
+
for t in exp.get("not_tools", []): # over-refusal guard: these must NOT appear
|
| 74 |
+
if t in used:
|
| 75 |
+
reasons.append(f"tool {t!r} used but forbidden here (trace: {used})")
|
| 76 |
+
if exp.get("artifact") == "chart":
|
| 77 |
+
if not any("chart" in a for a in result.get("artifacts", [])):
|
| 78 |
+
reasons.append("no chart artifact produced")
|
| 79 |
+
return (not reasons), reasons
|
| 80 |
+
|
| 81 |
+
|
| 82 |
+
def run(chat_fn=None, model=None, only=None, bar=0.8, verbose=True):
|
| 83 |
+
"""Run the golden set. Returns {passed, total, pass_rate, gate_ok, results}."""
|
| 84 |
+
import harness.analyst as A
|
| 85 |
+
items = yaml.safe_load(GOLDEN.read_text(encoding="utf-8"))["items"]
|
| 86 |
+
if only:
|
| 87 |
+
items = [i for i in items if i["id"] in only]
|
| 88 |
+
results, passed = [], 0
|
| 89 |
+
import os as _os
|
| 90 |
+
import time as _time
|
| 91 |
+
_os.environ["ANALYST_PATIENT"] = "1" # batch mode: wait out rate windows, don't fail fast
|
| 92 |
+
infra_fails = 0
|
| 93 |
+
aborted = False
|
| 94 |
+
for item in items:
|
| 95 |
+
_time.sleep(6) # pace the free-tier per-minute windows (Groq/Cerebras)
|
| 96 |
+
kw = {"chat_fn": chat_fn} if chat_fn else {}
|
| 97 |
+
if model:
|
| 98 |
+
kw["model"] = model
|
| 99 |
+
try:
|
| 100 |
+
res = A.ask(item["ask"], telemetry_kind="analyst_eval", **kw)
|
| 101 |
+
ok, reasons = _check(item, res)
|
| 102 |
+
infra_fails = 0
|
| 103 |
+
except Exception as e:
|
| 104 |
+
res, ok, reasons = {"answer": "", "tool_trace": []}, False, [f"RUN ERROR: {e}"]
|
| 105 |
+
# Provider exhaustion is NOT a model failure: two in a row means the ladder is out
|
| 106 |
+
# of quota β abort and report INCONCLUSIVE instead of burning the remaining items
|
| 107 |
+
# into a misleading FAIL (learned 2026-07-16: 14 wasted items, 4/18 "22%").
|
| 108 |
+
if "all LLM providers failed" in str(e):
|
| 109 |
+
infra_fails += 1
|
| 110 |
+
if infra_fails >= 2:
|
| 111 |
+
aborted = True
|
| 112 |
+
passed += ok
|
| 113 |
+
results.append({"id": item["id"], "ok": ok, "reasons": reasons,
|
| 114 |
+
"usage": res.get("usage"), "iterations": res.get("iterations")})
|
| 115 |
+
if verbose:
|
| 116 |
+
mark = "OK " if ok else "XX "
|
| 117 |
+
print(f" {mark}{item['id']:18s}" + ("" if ok else f" β {'; '.join(reasons)[:90]}"))
|
| 118 |
+
if aborted:
|
| 119 |
+
break
|
| 120 |
+
if aborted:
|
| 121 |
+
ran = [r for r in results if not str(r["reasons"])[:60].startswith("['RUN ERROR")]
|
| 122 |
+
if verbose:
|
| 123 |
+
print(f"\nEVAL ABORTED β LLM providers exhausted after {len(results)} item(s); "
|
| 124 |
+
f"{passed}/{len(ran) or 1} of the items that actually ran passed. "
|
| 125 |
+
"Gate is INCONCLUSIVE (not a model failure) β re-run when quota recovers.")
|
| 126 |
+
return {"passed": passed, "total": len(items), "pass_rate": None, "gate_ok": False,
|
| 127 |
+
"inconclusive": True, "results": results}
|
| 128 |
+
rate = passed / len(items) if items else 0.0
|
| 129 |
+
gate = rate >= bar
|
| 130 |
+
if verbose:
|
| 131 |
+
print(f"\nEVAL: {passed}/{len(items)} = {rate:.0%} GATE({bar:.0%}): "
|
| 132 |
+
f"{'PASS' if gate else 'FAIL'}")
|
| 133 |
+
return {"passed": passed, "total": len(items), "pass_rate": rate, "gate_ok": gate,
|
| 134 |
+
"results": results}
|
platform/harness/filter_eval.py
ADDED
|
@@ -0,0 +1,485 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""harness/filter_eval.py β the filter tree evaluated against ROWS, in Python (wave 15, C-EVAL).
|
| 2 |
+
|
| 3 |
+
THE THIRD ENGINE, and the one with a different job. There are now three readers of the same
|
| 4 |
+
filter tree and it is worth being precise about why:
|
| 5 |
+
|
| 6 |
+
* `customer-grid/useVisibleRows.ts` β the CLIENT, over in-memory rows. The reference.
|
| 7 |
+
* `harness/filter_sql.py` β compiles the tree to SQL for the DuckDB semantic tables.
|
| 8 |
+
* **this module** β evaluates the tree against wire-shaped row dicts in the
|
| 9 |
+
API/host process, where there is no SQL and the rows are
|
| 10 |
+
already in hand.
|
| 11 |
+
|
| 12 |
+
It exists because a PERMANENT (permission) filter has to be applied server-side, to rows the
|
| 13 |
+
server is about to send, on a surface whose columns are not a DuckDB table. Neither existing
|
| 14 |
+
engine can do that: the client one is TypeScript, and the SQL one needs a table.
|
| 15 |
+
|
| 16 |
+
β TWO ENTRY POINTS, OPPOSITE DEFAULTS, AND THE DIFFERENCE IS THE WHOLE POINT.
|
| 17 |
+
|
| 18 |
+
matches(...) DISPLAY semantics. A faithful mirror of the client, including the places
|
| 19 |
+
where the client IGNORES a leaf. Refereed against the client by
|
| 20 |
+
`aios-web/verify_filter_parity.py` β if this disagrees with the browser about
|
| 21 |
+
one row, a saved view means two different things in two places.
|
| 22 |
+
|
| 23 |
+
permits(...) PERMISSION semantics. Same evaluator, `strict=True`: anything the display
|
| 24 |
+
engine would IGNORE, this DENIES.
|
| 25 |
+
|
| 26 |
+
Why they cannot be the same function (C-EVAL amendment 1, and the reason this module has a
|
| 27 |
+
docstring this long). The client engine is TRI-STATE β `True` / `False` / `None`, where `None`
|
| 28 |
+
means "this leaf is INACTIVE, parent ignores it entirely". A leaf naming a column that does not
|
| 29 |
+
exist returns `None` (`useVisibleRows.ts:318`), and a group whose children are all inactive is
|
| 30 |
+
itself inactive (:273). For a DISPLAY filter that is exactly right: a half-built condition must
|
| 31 |
+
not zero the grid, and a stale column reference must not either.
|
| 32 |
+
|
| 33 |
+
For a PERMISSION filter it is a catastrophe. `agent is Tara` is the only thing standing between
|
| 34 |
+
a sales rep and the whole book; rename or delete the `agent` field and that leaf becomes
|
| 35 |
+
`None` β ignored β and the rep silently receives every customer. The wall would be one field
|
| 36 |
+
rename away from open, and nothing would go red. So `permits()` inverts every ignore into a
|
| 37 |
+
denial: an unanswerable permission filter is an OUTAGE, not a licence.
|
| 38 |
+
|
| 39 |
+
The asymmetry is a PARAMETER (`strict`) rather than a second implementation, deliberately: a
|
| 40 |
+
mirror must copy the original's asymmetries, and the only way to be sure it still does next
|
| 41 |
+
month is for both behaviours to come out of the same walk.
|
| 42 |
+
|
| 43 |
+
β WHAT THIS MODULE DOES NOT RE-IMPLEMENT. Activeness (`is_rule_active`), the JS-`Number`
|
| 44 |
+
coercion (`to_num`), the op vocabularies, cohort-id parsing and the date-window/anchor resolvers
|
| 45 |
+
all already exist in `filter_sql.py` and `windows.py`, and are already held against the client by
|
| 46 |
+
`verify_filter_engine.py` (525 checks) and `verify_windows.py` (123). Importing them is not
|
| 47 |
+
laziness β a second copy of "what counts as an active rule" is precisely the drift those gates
|
| 48 |
+
exist to catch, and it would be a copy the gates do not cover.
|
| 49 |
+
"""
|
| 50 |
+
from harness.filter_sql import ( # noqa: F401 (COHORT_OPS re-exported for callers)
|
| 51 |
+
COHORT_FIELD, COHORT_OPS, COHORT_OP_ALIASES, FILTER_OPS, RANK_OPS,
|
| 52 |
+
is_rule_active, parse_cohort_ids, to_num, _resolve_anchor, _window_bounds,
|
| 53 |
+
)
|
| 54 |
+
|
| 55 |
+
#: The CLIENT's numeric family, which is WIDER than `filter_sql.NUMERIC_TYPES`. `rating` (an
|
| 56 |
+
#: integer 1..max) and `formula` (computed in the browser and injected at the field's key) compare
|
| 57 |
+
#: as numbers here because this engine sees the same assembled row the client does. They are
|
| 58 |
+
#: absent from the SQL port for a good reason β they are client-only strata that can never reach a
|
| 59 |
+
#: DuckDB table β so the two lists differing is correct, not drift. Mirrors `isNumericType`.
|
| 60 |
+
NUMERIC_TYPES = frozenset({'currency', 'int', 'pct', 'rating', 'formula'})
|
| 61 |
+
|
| 62 |
+
#: `created_time` is a DATETIME rendered read-only, and it filters as a DATE (truncated to the
|
| 63 |
+
#: day below). Mirrors `isDateType`.
|
| 64 |
+
DATE_TYPES = frozenset({'date', 'created_time'})
|
| 65 |
+
|
| 66 |
+
#: The date ops that actually compare against an ANCHOR. Everything else falls to the
|
| 67 |
+
#: active-and-always-true default WITHOUT resolving one β ordering matters, see `_match_date`.
|
| 68 |
+
DATE_CMP_OPS = frozenset({'eq', 'neq', 'lt', 'lte', 'gt', 'gte'})
|
| 69 |
+
|
| 70 |
+
|
| 71 |
+
class EvalCtx:
|
| 72 |
+
"""Everything the engine needs besides the row and the columns.
|
| 73 |
+
|
| 74 |
+
All four members share one property β each is an answer a single ROW cannot compute β and
|
| 75 |
+
therefore share one failure mode: absent, the condition matches NOTHING rather than
|
| 76 |
+
everything. That is the widening sin the cohort/measure/rank leaves are each written to
|
| 77 |
+
avoid, and it is why none of these defaults to a permissive value.
|
| 78 |
+
"""
|
| 79 |
+
|
| 80 |
+
__slots__ = ('cohort_sets', 'measure_sets', 'rank_sets', 'today')
|
| 81 |
+
|
| 82 |
+
def __init__(self, cohort_sets=None, measure_sets=None, rank_sets=None, today=None):
|
| 83 |
+
#: {cohort_id: set(pid)} β the cohorts THIS reader may see.
|
| 84 |
+
self.cohort_sets = cohort_sets or {}
|
| 85 |
+
#: {rule_id: set(pid)} β resolved measure conditions (CG-8).
|
| 86 |
+
self.measure_sets = measure_sets or {}
|
| 87 |
+
#: {id(leaf) or leaf-key: set(pid)} β resolved rank slices (C-OPS).
|
| 88 |
+
self.rank_sets = rank_sets or {}
|
| 89 |
+
#: The TENANT's today, ISO. Never the server clock at call time β a date filter that
|
| 90 |
+
#: means something different per process is not a filter.
|
| 91 |
+
self.today = today
|
| 92 |
+
|
| 93 |
+
|
| 94 |
+
_EMPTY_CTX = EvalCtx()
|
| 95 |
+
|
| 96 |
+
|
| 97 |
+
def _is_group(node):
|
| 98 |
+
return isinstance(node, dict) and isinstance(node.get('children'), list)
|
| 99 |
+
|
| 100 |
+
|
| 101 |
+
def _is_cohort(node):
|
| 102 |
+
return isinstance(node, dict) and node.get('colId') == COHORT_FIELD
|
| 103 |
+
|
| 104 |
+
|
| 105 |
+
def _is_measure(node):
|
| 106 |
+
"""Mirrors `isMeasureRule`: a leaf carrying a `window` IS a measure condition."""
|
| 107 |
+
return isinstance(node, dict) and node.get('window') is not None
|
| 108 |
+
|
| 109 |
+
|
| 110 |
+
def _rhs_col(rhs):
|
| 111 |
+
"""Mirrors `rhsColId`: a STATISTIC names no column."""
|
| 112 |
+
if not isinstance(rhs, dict) or rhs.get('kind') == 'stat':
|
| 113 |
+
return None
|
| 114 |
+
return rhs.get('colId')
|
| 115 |
+
|
| 116 |
+
|
| 117 |
+
def _as_text(v):
|
| 118 |
+
"""The client's `String(raw ?? "").toLowerCase()`. `None` becomes '' (never 'None')."""
|
| 119 |
+
return ('' if v is None else str(v)).lower()
|
| 120 |
+
|
| 121 |
+
|
| 122 |
+
def _as_day(v):
|
| 123 |
+
"""Day-granularity truncation. A `created_time` cell is a DATETIME and every comparand in
|
| 124 |
+
this vocabulary is a DATE β lexically the longer string sorts AFTER its own day, so without
|
| 125 |
+
this `on or before <today>` would silently exclude today's rows. Identity for a pure date."""
|
| 126 |
+
return ('' if v is None else str(v))[:10]
|
| 127 |
+
|
| 128 |
+
|
| 129 |
+
def _cmp_num(op, a, b):
|
| 130 |
+
"""`default: True` is the documented ACTIVE-and-always-true case β an operator that exists
|
| 131 |
+
but does not apply to this type (`contains` on a number) narrows nothing and asks nothing.
|
| 132 |
+
An operator in NO vocabulary never reaches here; `_eval` refuses it first."""
|
| 133 |
+
if op == 'gt':
|
| 134 |
+
return a > b
|
| 135 |
+
if op == 'gte':
|
| 136 |
+
return a >= b
|
| 137 |
+
if op == 'lt':
|
| 138 |
+
return a < b
|
| 139 |
+
if op == 'lte':
|
| 140 |
+
return a <= b
|
| 141 |
+
if op == 'eq':
|
| 142 |
+
return a == b
|
| 143 |
+
if op == 'neq':
|
| 144 |
+
return a != b
|
| 145 |
+
return True
|
| 146 |
+
|
| 147 |
+
|
| 148 |
+
def _cmp_str(op, a, v):
|
| 149 |
+
if op == 'eq':
|
| 150 |
+
return a == v
|
| 151 |
+
if op == 'neq':
|
| 152 |
+
return a != v
|
| 153 |
+
if op == 'lt':
|
| 154 |
+
return a < v
|
| 155 |
+
if op == 'lte':
|
| 156 |
+
return a <= v
|
| 157 |
+
if op == 'gt':
|
| 158 |
+
return a > v
|
| 159 |
+
if op == 'gte':
|
| 160 |
+
return a >= v
|
| 161 |
+
return True
|
| 162 |
+
|
| 163 |
+
|
| 164 |
+
def _cmp_text(op, a, v):
|
| 165 |
+
if op == 'contains':
|
| 166 |
+
return v in a
|
| 167 |
+
if op == 'doesNotContain':
|
| 168 |
+
return v not in a
|
| 169 |
+
if op == 'eq':
|
| 170 |
+
return a == v
|
| 171 |
+
if op == 'neq':
|
| 172 |
+
return a != v
|
| 173 |
+
return True
|
| 174 |
+
|
| 175 |
+
|
| 176 |
+
def _is_blank(raw):
|
| 177 |
+
"""Blank = None or ''. A numeric 0 is NOT blank β `isEmpty` must not swallow a real zero,
|
| 178 |
+
and the wire preserves null for a missing value."""
|
| 179 |
+
return raw is None or raw == ''
|
| 180 |
+
|
| 181 |
+
|
| 182 |
+
def _match_numeric(rule, raw, other, has_rhs):
|
| 183 |
+
a = to_num(raw)
|
| 184 |
+
op = rule.get('op')
|
| 185 |
+
if has_rhs:
|
| 186 |
+
return _cmp_num(op, a, to_num(other))
|
| 187 |
+
if op == 'between':
|
| 188 |
+
v, v2 = rule.get('value'), rule.get('value2')
|
| 189 |
+
if v == '' or v2 is None or v2 == '':
|
| 190 |
+
return True # both bounds required
|
| 191 |
+
lo, hi = to_num(v), to_num(v2)
|
| 192 |
+
if lo > hi:
|
| 193 |
+
lo, hi = hi, lo
|
| 194 |
+
return lo <= a <= hi
|
| 195 |
+
if rule.get('value') == '':
|
| 196 |
+
return True # inactive until a value is typed
|
| 197 |
+
return _cmp_num(op, a, to_num(rule.get('value')))
|
| 198 |
+
|
| 199 |
+
|
| 200 |
+
def _match_date(rule, raw, other, has_rhs, today):
|
| 201 |
+
a = _as_day(raw)
|
| 202 |
+
op = rule.get('op')
|
| 203 |
+
|
| 204 |
+
# "is within [the past month]" β a RANGE test, and the only date op reading `dateWindow`.
|
| 205 |
+
if op == 'within':
|
| 206 |
+
if not today:
|
| 207 |
+
return False
|
| 208 |
+
bounds = _window_bounds(rule.get('dateWindow'), today)
|
| 209 |
+
if not bounds:
|
| 210 |
+
return False # unresolvable range -> matches nothing
|
| 211 |
+
if a == '':
|
| 212 |
+
# A blank date is not "within" any range. Without this, '' >= a null lower bound is
|
| 213 |
+
# vacuously true and every never-ordered customer lands inside every window.
|
| 214 |
+
return False
|
| 215 |
+
lo, hi = bounds
|
| 216 |
+
return (lo is None or a >= lo) and (hi is None or a <= hi)
|
| 217 |
+
|
| 218 |
+
if has_rhs:
|
| 219 |
+
return _cmp_str(op, a, _as_day(other))
|
| 220 |
+
if op == 'between':
|
| 221 |
+
v, v2 = rule.get('value'), rule.get('value2')
|
| 222 |
+
if v == '' or v2 is None or v2 == '':
|
| 223 |
+
return True
|
| 224 |
+
lo, hi = (v, v2) if v < v2 else (v2, v)
|
| 225 |
+
return lo <= a <= hi
|
| 226 |
+
|
| 227 |
+
# An op that does not apply to a date (`contains`) is ACTIVE-and-always-true, decided BEFORE
|
| 228 |
+
# any anchor is resolved β there is no date in that question to resolve. Ordering this after
|
| 229 |
+
# the anchor would turn `contains` on a date from always-true into matches-nothing, which is
|
| 230 |
+
# a different answer under `or`.
|
| 231 |
+
if op not in DATE_CMP_OPS:
|
| 232 |
+
return True
|
| 233 |
+
if not rule.get('dateMode') and rule.get('value') == '':
|
| 234 |
+
return True # inactive until a value is typed
|
| 235 |
+
mode = rule.get('dateMode') or 'exact'
|
| 236 |
+
# `today` is required by a RELATIVE anchor and by nothing else, which is what keeps every
|
| 237 |
+
# pre-anchor saved view working in a payload that carries no today.
|
| 238 |
+
if mode != 'exact' and not today:
|
| 239 |
+
return False
|
| 240 |
+
v = _resolve_anchor(mode, rule.get('value'), today or '1970-01-01')
|
| 241 |
+
if v is None:
|
| 242 |
+
# Unresolvable (junk date, missing N) -> matches nothing. The honest answer to a
|
| 243 |
+
# question with no date in it.
|
| 244 |
+
return False
|
| 245 |
+
return _cmp_str(op, a, v)
|
| 246 |
+
|
| 247 |
+
|
| 248 |
+
def _match_filter(rule, row, columns, today):
|
| 249 |
+
"""One LEAF against one row, dispatched on the field's declared type.
|
| 250 |
+
|
| 251 |
+
`isEmpty`/`isNotEmpty` are handled FIRST and type-independently: they must never fall
|
| 252 |
+
through to the "blank value -> inactive" guards below, which would turn them into
|
| 253 |
+
always-true no-ops.
|
| 254 |
+
"""
|
| 255 |
+
field = columns.get(rule.get('colId')) or {}
|
| 256 |
+
ftype = field.get('type')
|
| 257 |
+
raw = row.get(rule.get('colId'))
|
| 258 |
+
op = rule.get('op')
|
| 259 |
+
|
| 260 |
+
if op == 'isEmpty':
|
| 261 |
+
return _is_blank(raw)
|
| 262 |
+
if op == 'isNotEmpty':
|
| 263 |
+
return not _is_blank(raw)
|
| 264 |
+
|
| 265 |
+
# A RANK op can never be answered from one row. `_eval` routes rank leaves to their resolved
|
| 266 |
+
# set and one should never arrive here; this is the SECOND layer, and it exists because of
|
| 267 |
+
# what the first layer's absence would do β `_cmp_num`'s `default: True` would make `topN`
|
| 268 |
+
# match EVERY row, a filter that widens under a count that agrees with it.
|
| 269 |
+
if op in RANK_OPS:
|
| 270 |
+
return False
|
| 271 |
+
|
| 272 |
+
rhs = rule.get('rhs')
|
| 273 |
+
has_rhs = rhs is not None
|
| 274 |
+
other = row.get(_rhs_col(rhs) or '') if has_rhs else None
|
| 275 |
+
|
| 276 |
+
if ftype in NUMERIC_TYPES:
|
| 277 |
+
return _match_numeric(rule, raw, other, has_rhs)
|
| 278 |
+
if ftype in DATE_TYPES:
|
| 279 |
+
return _match_date(rule, raw, other, has_rhs, today)
|
| 280 |
+
|
| 281 |
+
a = _as_text(raw)
|
| 282 |
+
if has_rhs:
|
| 283 |
+
return _cmp_text(op, a, _as_text(other))
|
| 284 |
+
if rule.get('value') == '':
|
| 285 |
+
return True
|
| 286 |
+
return _cmp_text(op, a, _as_text(rule.get('value')))
|
| 287 |
+
|
| 288 |
+
|
| 289 |
+
def _eval(node, row, columns, ctx, strict):
|
| 290 |
+
"""One node against one row. `True` / `False` / `None`, where `None` = INACTIVE.
|
| 291 |
+
|
| 292 |
+
`strict` is the permission mode: every `None` this function would return becomes `False`
|
| 293 |
+
instead. Read the two behaviours side by side rather than trusting either alone β the
|
| 294 |
+
difference between them IS the security property (see the module docstring).
|
| 295 |
+
"""
|
| 296 |
+
ignore = False if strict else None
|
| 297 |
+
|
| 298 |
+
if _is_group(node):
|
| 299 |
+
saw_active = False
|
| 300 |
+
all_true = True
|
| 301 |
+
any_true = False
|
| 302 |
+
for child in node['children']:
|
| 303 |
+
v = _eval(child, row, columns, ctx, strict)
|
| 304 |
+
if v is None:
|
| 305 |
+
continue # inactive -> ignored entirely
|
| 306 |
+
saw_active = True
|
| 307 |
+
all_true = all_true and v
|
| 308 |
+
any_true = any_true or v
|
| 309 |
+
if not saw_active:
|
| 310 |
+
# A group with no active children is itself inactive. In strict mode an empty or
|
| 311 |
+
# wholly-unanswerable permission group DENIES: a wall that evaluates to "nothing to
|
| 312 |
+
# say" must not mean "let them through".
|
| 313 |
+
return ignore
|
| 314 |
+
return any_true if node.get('conj') == 'or' else all_true
|
| 315 |
+
|
| 316 |
+
if not isinstance(node, dict):
|
| 317 |
+
return ignore
|
| 318 |
+
|
| 319 |
+
# A COHORT leaf, BEFORE the column lookup: `__cohort__` is not a column, so the lookup below
|
| 320 |
+
# would call it INACTIVE β and inactive means every row on screen under a count nobody doubts.
|
| 321 |
+
if _is_cohort(node):
|
| 322 |
+
if not is_rule_active(node, columns):
|
| 323 |
+
return ignore
|
| 324 |
+
ids = parse_cohort_ids(node.get('value'))
|
| 325 |
+
sets = [ctx.cohort_sets.get(cid) for cid in ids]
|
| 326 |
+
# ANY named cohort we were given no membership for makes the WHOLE condition
|
| 327 |
+
# unanswerable, and unanswerable matches nothing β for `is none of` as well. Answering
|
| 328 |
+
# "everything" to `is none of [a cohort we cannot see]` is the widening sin wearing a
|
| 329 |
+
# plausible face, and PARTIAL evaluation is worse: `is none of [A, B]` quietly becoming
|
| 330 |
+
# `is none of [A]` would show every row in B under a count nobody would doubt.
|
| 331 |
+
if any(s is None for s in sets):
|
| 332 |
+
return False
|
| 333 |
+
pid = row.get('pid')
|
| 334 |
+
op = node.get('op')
|
| 335 |
+
op = op if op in COHORT_OPS else COHORT_OP_ALIASES.get(op, 'anyOf')
|
| 336 |
+
if op == 'allOf':
|
| 337 |
+
return all(pid in s for s in sets)
|
| 338 |
+
in_any = any(pid in s for s in sets)
|
| 339 |
+
return (not in_any) if op == 'noneOf' else in_any
|
| 340 |
+
|
| 341 |
+
# A MEASURE leaf, also before the column lookup and for the same reason: a measure's colId
|
| 342 |
+
# ("revenue") is not a column of this table. An ACTIVE rule whose set has not been resolved
|
| 343 |
+
# is PENDING and matches NOTHING β never everything.
|
| 344 |
+
if _is_measure(node):
|
| 345 |
+
if not is_rule_active(node, columns):
|
| 346 |
+
return ignore
|
| 347 |
+
s = ctx.measure_sets.get(node.get('id')) if node.get('id') else None
|
| 348 |
+
if s is None:
|
| 349 |
+
return False
|
| 350 |
+
return row.get('pid') in s
|
| 351 |
+
|
| 352 |
+
# ββ a column leaf ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 353 |
+
if node.get('colId') not in columns:
|
| 354 |
+
# THE LEAF THIS WHOLE MODULE'S `strict` FLAG EXISTS FOR. Display: ignore it. Permission:
|
| 355 |
+
# deny β see the module docstring's field-rename walkthrough.
|
| 356 |
+
return ignore
|
| 357 |
+
if node.get('rhs') is not None and _rhs_col(node['rhs']) not in columns:
|
| 358 |
+
# An rhs naming a column this table does not have is ignored by the SAME rule as an
|
| 359 |
+
# unknown LEFT column. Treating one side as "ignore" and the other as "match nothing"
|
| 360 |
+
# would make `a > deleted` behave differently from `deleted > a`.
|
| 361 |
+
return ignore
|
| 362 |
+
if not is_rule_active(node, columns):
|
| 363 |
+
return ignore
|
| 364 |
+
|
| 365 |
+
# A RANK leaf is a MEMBERSHIP test, like a cohort or a measure: the set was computed in a
|
| 366 |
+
# prior pass over the other rows. No set = REFUSE, never "ignore" and never "true" β both of
|
| 367 |
+
# those SHOW rows.
|
| 368 |
+
if node.get('op') in RANK_OPS:
|
| 369 |
+
s = ctx.rank_sets.get(_rank_key(node))
|
| 370 |
+
return (row.get('pid') in s) if s is not None else False
|
| 371 |
+
|
| 372 |
+
# An operator in NO vocabulary at all. The comparators end in `default: True`, right for an
|
| 373 |
+
# operator that exists but does not apply to this type; NOT right for one that exists
|
| 374 |
+
# nowhere (`inTertile` out of a stale saved view), which would match every row.
|
| 375 |
+
if node.get('op') not in FILTER_OPS:
|
| 376 |
+
return False
|
| 377 |
+
|
| 378 |
+
return _match_filter(node, row, columns, ctx.today)
|
| 379 |
+
|
| 380 |
+
|
| 381 |
+
def _rank_key(node):
|
| 382 |
+
"""The client keys `rankSets` by leaf IDENTITY (a Map keyed on the node object). Python
|
| 383 |
+
dicts cannot key on a dict, so the caller that resolves rank sets and this evaluator agree
|
| 384 |
+
on a derived key. Kept in ONE place so the two cannot drift apart."""
|
| 385 |
+
return (node.get('id') or '', node.get('colId') or '', node.get('op') or '',
|
| 386 |
+
str(node.get('value') if node.get('value') is not None else ''))
|
| 387 |
+
|
| 388 |
+
|
| 389 |
+
def _columns_map(fields):
|
| 390 |
+
"""`[{key,type,β¦}]` -> `{key: field}`. Accepts a mapping unchanged, so callers holding
|
| 391 |
+
either shape need no adapter."""
|
| 392 |
+
if isinstance(fields, dict):
|
| 393 |
+
return fields
|
| 394 |
+
out = {}
|
| 395 |
+
for f in fields or ():
|
| 396 |
+
if isinstance(f, dict) and f.get('key'):
|
| 397 |
+
out[f['key']] = f
|
| 398 |
+
return out
|
| 399 |
+
|
| 400 |
+
|
| 401 |
+
def evaluate(tree, row, fields, ctx=None):
|
| 402 |
+
"""The TRI-STATE the client actually has: `True` / `False` / `None`, where `None` means the
|
| 403 |
+
tree is INACTIVE β it asks nothing, so it narrows nothing.
|
| 404 |
+
|
| 405 |
+
Exposed because collapsing `None` into `True` (which `matches` does, correctly, for its own
|
| 406 |
+
purpose) destroys a distinction some callers must keep. The `memberPids` union is the one
|
| 407 |
+
that bit: the client's rule is `members.has(pid) || matchFilterTree(β¦)` and that union sits
|
| 408 |
+
INSIDE `if (filters.length)`, so an INACTIVE tree must show every row and ignore the member
|
| 409 |
+
list β whereas an ACTIVE tree that merely fails must still admit hand-picked members. A
|
| 410 |
+
caller holding only `matches` cannot tell those two apart and will narrow an inactive view
|
| 411 |
+
down to its hand-picked rows. (`filter_sql.compile_filter_tree` keeps the same distinction
|
| 412 |
+
by returning None rather than a predicate; this is the row-engine's equivalent.)
|
| 413 |
+
|
| 414 |
+
Activeness is ROW-INDEPENDENT β every `None` here comes from `is_rule_active` or an empty
|
| 415 |
+
group, none of which reads the row β so a caller may compute it once per tree if it wishes.
|
| 416 |
+
"""
|
| 417 |
+
columns = _columns_map(fields)
|
| 418 |
+
nodes, conj = tree_parts(tree)
|
| 419 |
+
return _eval({'conj': conj, 'children': nodes}, row, columns, ctx or _EMPTY_CTX, False)
|
| 420 |
+
|
| 421 |
+
|
| 422 |
+
def matches(tree, row, fields, ctx=None):
|
| 423 |
+
"""DISPLAY semantics β does `row` survive this filter tree? Mirrors the client exactly.
|
| 424 |
+
|
| 425 |
+
A wholly INACTIVE tree returns True: no narrowing, every row shows. That is what the browser
|
| 426 |
+
does with a half-built condition, and this function's whole job is to agree with the browser.
|
| 427 |
+
Use `evaluate()` instead when you need to tell "inactive" from "matched" (see its docstring).
|
| 428 |
+
"""
|
| 429 |
+
v = evaluate(tree, row, fields, ctx)
|
| 430 |
+
return True if v is None else bool(v)
|
| 431 |
+
|
| 432 |
+
|
| 433 |
+
def visible_pids(tree, rows, fields, ctx=None, member_pids=None):
|
| 434 |
+
"""The client's whole row pipeline for one tree, member union included β the shape a caller
|
| 435 |
+
should reach for instead of assembling `matches` in a loop.
|
| 436 |
+
|
| 437 |
+
Mirrors `useVisibleRows`: an INACTIVE tree yields every row (and IGNORES `member_pids`,
|
| 438 |
+
because the union lives inside `if (filters.length)`); an ACTIVE tree yields the rows it
|
| 439 |
+
matches UNION the hand-picked members.
|
| 440 |
+
"""
|
| 441 |
+
ctx = ctx or _EMPTY_CTX
|
| 442 |
+
members = set(member_pids or ())
|
| 443 |
+
out, active = [], False
|
| 444 |
+
verdicts = []
|
| 445 |
+
for r in rows:
|
| 446 |
+
v = evaluate(tree, r, fields, ctx)
|
| 447 |
+
if v is not None:
|
| 448 |
+
active = True
|
| 449 |
+
verdicts.append((r, v))
|
| 450 |
+
for r, v in verdicts:
|
| 451 |
+
if not active:
|
| 452 |
+
out.append(r.get('pid'))
|
| 453 |
+
elif v or (members and r.get('pid') in members):
|
| 454 |
+
out.append(r.get('pid'))
|
| 455 |
+
return out
|
| 456 |
+
|
| 457 |
+
|
| 458 |
+
def permits(tree, row, fields, ctx=None):
|
| 459 |
+
"""PERMISSION semantics β may this reader SEE `row` under this permanent filter?
|
| 460 |
+
|
| 461 |
+
Same evaluator, every ignore inverted into a denial. An empty/absent tree means "no
|
| 462 |
+
restriction declared" and admits (the caller decides whether a module with no entry is
|
| 463 |
+
reachable at all β that is `perm_scope`'s question, not this one); anything PRESENT but
|
| 464 |
+
unanswerable DENIES.
|
| 465 |
+
"""
|
| 466 |
+
nodes, conj = tree_parts(tree)
|
| 467 |
+
if not nodes:
|
| 468 |
+
return True # nothing declared -> nothing to enforce here
|
| 469 |
+
columns = _columns_map(fields)
|
| 470 |
+
v = _eval({'conj': conj, 'children': nodes}, row, columns, ctx or _EMPTY_CTX, True)
|
| 471 |
+
return bool(v)
|
| 472 |
+
|
| 473 |
+
|
| 474 |
+
def tree_parts(tree):
|
| 475 |
+
"""Read C-PERM amendment 2's `FilterTree` β `{conj?, nodes}` β into `(nodes, conj)`.
|
| 476 |
+
|
| 477 |
+
Accepts a BARE LIST too, because that is what `ViewConfig.filters` is and what
|
| 478 |
+
`clean_filter_tree` returns; an absent conj is `'and'`, mirroring `ViewConfig.filterConj`.
|
| 479 |
+
One reader, so the absent-conj default lives here instead of at every call site.
|
| 480 |
+
"""
|
| 481 |
+
if isinstance(tree, dict):
|
| 482 |
+
nodes = tree.get('nodes')
|
| 483 |
+
conj = 'or' if tree.get('conj') == 'or' else 'and'
|
| 484 |
+
return (nodes if isinstance(nodes, list) else []), conj
|
| 485 |
+
return (tree if isinstance(tree, list) else []), 'and'
|
platform/harness/filter_sql.py
ADDED
|
@@ -0,0 +1,673 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""harness/filter_sql.py β the grid's filter/sort engine, compiled to SQL (CG-1).
|
| 2 |
+
|
| 3 |
+
THE PORT. `customer-grid/useVisibleRows.ts` evaluates the Airtable-parity filter TREE and the
|
| 4 |
+
multi-level sort in TypeScript, per row, in the browser. That works at customer grain (1,550 rows,
|
| 5 |
+
576 KB) and does NOT work at line grain, so a line-grain table has to filter server-side. This
|
| 6 |
+
module is the second implementation of that contract, emitting parameterized SQL instead of
|
| 7 |
+
booleans. `aios_grid.clean_filter_tree` (the validator) is the third. All three are held in
|
| 8 |
+
lock-step by `aios-web/verify_filter_engine.py`, which runs the SAME cases through the TS engine,
|
| 9 |
+
this compiler (executed on DuckDB), and the validator, and compares row sets.
|
| 10 |
+
|
| 11 |
+
Design: this file is PURE β no model, no store, no app imports. It takes a filter tree plus a
|
| 12 |
+
`columns` spec and returns SQL + params. `harness/semantic.store_columns` builds the spec from a
|
| 13 |
+
topic; nothing else here knows what a topic is.
|
| 14 |
+
|
| 15 |
+
columns = {colId: {"sql": <SQL expression>, "type": <field type>, "aggregate": bool}}
|
| 16 |
+
|
| 17 |
+
`sql` is a trusted IDENTIFIER/expression from the semantic model (never user input); every VALUE
|
| 18 |
+
is a bound parameter. That split is the whole injection story.
|
| 19 |
+
|
| 20 |
+
--------------------------------------------------------------------------------------------
|
| 21 |
+
TRI-STATE, AND WHY IT BECOMES COMPILE-TIME PRUNING
|
| 22 |
+
--------------------------------------------------------------------------------------------
|
| 23 |
+
`evalNode` in TS returns true | false | null, where null means "this node is INACTIVE and its
|
| 24 |
+
parent must IGNORE it". Get that wrong and a half-typed condition returns true, makes its `or`
|
| 25 |
+
group true, and shows every record.
|
| 26 |
+
|
| 27 |
+
Activeness reads ONLY the rule (op, value, value2, is-the-column-known) β never the row. So in
|
| 28 |
+
SQL it is not a third truth value at all: it is decided while COMPILING. An inactive node emits
|
| 29 |
+
no SQL, a group with no active children emits no SQL, and a root with nothing active emits NO
|
| 30 |
+
PREDICATE AT ALL (`compile_filter_tree` returns None) β which is exactly `matchFilterTree`
|
| 31 |
+
returning true for every row.
|
| 32 |
+
|
| 33 |
+
--------------------------------------------------------------------------------------------
|
| 34 |
+
THE FOUR PLACES A NAIVE PORT SILENTLY DIVERGES
|
| 35 |
+
--------------------------------------------------------------------------------------------
|
| 36 |
+
JavaScript coerces null to 0 / "" before comparing; SQL propagates NULL and drops the row.
|
| 37 |
+
|
| 38 |
+
1. `toNum(null)` is 0, so a NULL revenue PASSES `>= 0`. SQL: NULL >= 0 -> NULL -> dropped.
|
| 39 |
+
2. `String(null)` is "", so a NULL state PASSES `is not FL`. SQL: NULL <> 'fl' -> dropped.
|
| 40 |
+
3. `"" <= "2026-01-01"` is true, so a NULL date PASSES `is before`. SQL: dropped.
|
| 41 |
+
4. `contains` is a LITERAL substring test. `LIKE '%v%'` treats % and _ as wildcards, so a
|
| 42 |
+
value containing either matches the wrong rows. Hence `strpos`, not LIKE.
|
| 43 |
+
|
| 44 |
+
Every column is therefore read through `_value_sql`, which COALESCEs to the same zero-value
|
| 45 |
+
JavaScript would have coerced to. The one thing that must NOT be coalesced away is the
|
| 46 |
+
blank test itself β `isBlank` looks at the RAW value, and numeric 0 is NOT blank.
|
| 47 |
+
|
| 48 |
+
--------------------------------------------------------------------------------------------
|
| 49 |
+
AND THE FIFTH: `default: return true` IS ACTIVE, NOT INACTIVE
|
| 50 |
+
--------------------------------------------------------------------------------------------
|
| 51 |
+
`matchFilter`'s per-type switch ends in `default: return true`, reachable whenever an operator
|
| 52 |
+
does not apply to the field type β `contains` on a currency/date field, or `gt`/`between` on a
|
| 53 |
+
text field. A persisted tree can hold such a pair. That rule is ACTIVE and evaluates TRUE for
|
| 54 |
+
every row, which is NOT the same as inactive: under `or` it makes the whole group true and shows
|
| 55 |
+
everything. We emit `1=1` for those, and the gate asserts the difference is observable.
|
| 56 |
+
|
| 57 |
+
--------------------------------------------------------------------------------------------
|
| 58 |
+
INPUT CONTRACT
|
| 59 |
+
--------------------------------------------------------------------------------------------
|
| 60 |
+
Feed this CLEANED trees (`aios_grid.clean_filter_tree`). It is deliberately faithful to the TS
|
| 61 |
+
engine rather than fail-closed on garbage β an unknown OP compiles to `1=1` exactly as TS's
|
| 62 |
+
`default:` branch returns true β because the validator's job is to reject garbage and this
|
| 63 |
+
module's job is to mean the same thing as the client. The gate proves the composition.
|
| 64 |
+
|
| 65 |
+
Ported ahead of their consumer (CG-2) and gated at the same fidelity as the tree:
|
| 66 |
+
`compile_search` (`matchSearch`) and `member_ids` (the hand-picked-members union in
|
| 67 |
+
`useVisibleRows`).
|
| 68 |
+
"""
|
| 69 |
+
import math
|
| 70 |
+
import re
|
| 71 |
+
from collections import namedtuple
|
| 72 |
+
|
| 73 |
+
# Mirrors customer-grid/types.ts FilterOp. verify_filter_engine.py asserts this set is
|
| 74 |
+
# byte-identical to aios_grid.FILTER_OPS β three copies of a vocabulary need a drift gate.
|
| 75 |
+
FILTER_OPS = frozenset({
|
| 76 |
+
'contains', 'doesNotContain', 'eq', 'neq', 'isEmpty', 'isNotEmpty',
|
| 77 |
+
'gt', 'gte', 'lt', 'lte', 'between', 'within',
|
| 78 |
+
'topN', 'bottomN', 'inTopPct', 'inBottomPct',
|
| 79 |
+
'aboveAvg', 'belowAvg', 'inQuartile', 'inDecile',
|
| 80 |
+
})
|
| 81 |
+
#: Wave 2026-08-02 (C-OPS): the RANK subset β set-ranked over the sibling-filtered domain.
|
| 82 |
+
#: Vocabulary here, execution NOWHERE in this module: a per-row WHERE cannot express Top-N,
|
| 83 |
+
#: and compiling to 1=0 would answer "nothing" to a question that has an answer, under a
|
| 84 |
+
#: row_count that still looks authoritative. `_compile_leaf` therefore REFUSES (ValueError),
|
| 85 |
+
#: the same server-side posture as a relative anchor with no `today`. The TS engine
|
| 86 |
+
#: evaluates these as a set pass in useVisibleRows (gated by web/verify_rank_filters.py).
|
| 87 |
+
RANK_OPS = frozenset({'topN', 'bottomN', 'inTopPct', 'inBottomPct',
|
| 88 |
+
'aboveAvg', 'belowAvg', 'inQuartile', 'inDecile'})
|
| 89 |
+
#: The RESERVED pseudo-column of a cohort-membership leaf. Mirrors types.ts COHORT_FIELD.
|
| 90 |
+
COHORT_FIELD = '__cohort__'
|
| 91 |
+
#: Set operators over a SET of cohorts. Mirrors types.ts COHORT_OPS; DISJOINT from FILTER_OPS.
|
| 92 |
+
COHORT_OPS = frozenset({'anyOf', 'allOf', 'noneOf'})
|
| 93 |
+
#: The single-cohort ops the leaf shipped with, kept as permanent aliases. types.ts mirrors this.
|
| 94 |
+
COHORT_OP_ALIASES = {'eq': 'anyOf', 'neq': 'noneOf'}
|
| 95 |
+
#: Mirrors types.ts MAX_COHORT_IDS.
|
| 96 |
+
MAX_COHORT_IDS = 20
|
| 97 |
+
|
| 98 |
+
|
| 99 |
+
def parse_cohort_ids(value):
|
| 100 |
+
"""The cohorts a leaf names. Mirrors types.ts `cohortIds()` and aios_grid.parse_cohort_ids."""
|
| 101 |
+
out = []
|
| 102 |
+
for raw in ('' if value is None else str(value)).split(','):
|
| 103 |
+
cid = raw.strip()[:120]
|
| 104 |
+
if not cid or cid in out:
|
| 105 |
+
continue
|
| 106 |
+
out.append(cid)
|
| 107 |
+
if len(out) >= MAX_COHORT_IDS:
|
| 108 |
+
break
|
| 109 |
+
return out
|
| 110 |
+
# types.ts VALUE_FREE_OPS: no value control is rendered, and they must never be judged
|
| 111 |
+
# "inactive because the value is blank" β that would turn them into no-ops.
|
| 112 |
+
VALUE_FREE_OPS = frozenset({'isEmpty', 'isNotEmpty', 'aboveAvg', 'belowAvg'})
|
| 113 |
+
# types.ts isNumericType()
|
| 114 |
+
NUMERIC_TYPES = frozenset({'currency', 'int', 'pct'})
|
| 115 |
+
TEXTUAL_TYPES = frozenset({'text', 'status'})
|
| 116 |
+
|
| 117 |
+
SQL_TRUE = '1=1'
|
| 118 |
+
|
| 119 |
+
_NUM_CMP = {'gt': '>', 'gte': '>=', 'lt': '<', 'lte': '<=', 'eq': '=', 'neq': '<>'}
|
| 120 |
+
_DATE_CMP = dict(_NUM_CMP)
|
| 121 |
+
|
| 122 |
+
#: sql = the predicate (already parenthesised); params = bound values, positional;
|
| 123 |
+
#: uses_aggregate = the predicate references an aggregate column, so it belongs in HAVING
|
| 124 |
+
#: rather than WHERE; columns_used = colIds actually referenced in the emitted SQL (an
|
| 125 |
+
#: always-true `1=1` branch references nothing, and must not force HAVING routing).
|
| 126 |
+
Compiled = namedtuple('Compiled', 'sql params uses_aggregate columns_used')
|
| 127 |
+
|
| 128 |
+
|
| 129 |
+
# --------------------------------------------------------------------------- value coercion
|
| 130 |
+
|
| 131 |
+
# JS `Number(string)` trims exactly this set (WhiteSpace + LineTerminator). Python's float() and
|
| 132 |
+
# str.strip() use a DIFFERENT set -- notably U+0085 NEL, which Python treats as whitespace and JS
|
| 133 |
+
# does not -- so the set is spelled out rather than left to strip().
|
| 134 |
+
_JS_WS = "".join(chr(c) for c in (9, 10, 11, 12, 13, 32, 160, 5760, 8232, 8233,
|
| 135 |
+
8239, 8287, 12288, 65279)) + \
|
| 136 |
+
"".join(chr(c) for c in range(8192, 8203))
|
| 137 |
+
# ASCII-ONLY on purpose: `\d` in a Python str pattern matches every Unicode decimal digit, so
|
| 138 |
+
# `\d` here would parse the fullwidth "οΌοΌοΌ" that JS rejects -- reintroducing the exact bug
|
| 139 |
+
# this function exists to avoid.
|
| 140 |
+
_JS_DEC = re.compile(r"^[+-]?(?:[0-9]+\.?[0-9]*|\.[0-9]+)(?:[eE][+-]?[0-9]+)?$")
|
| 141 |
+
_JS_INF = re.compile(r"^[+-]?Infinity$")
|
| 142 |
+
_JS_RADIX = ((re.compile(r"^0[xX][0-9a-fA-F]+$"), 16),
|
| 143 |
+
(re.compile(r"^0[oO][0-7]+$"), 8),
|
| 144 |
+
(re.compile(r"^0[bB][01]+$"), 2))
|
| 145 |
+
|
| 146 |
+
|
| 147 |
+
def to_num(v):
|
| 148 |
+
"""Port of `toNum`: JS `Number(v)`, then `Number.isFinite` or 0.
|
| 149 |
+
|
| 150 |
+
NOT `float(v)`. Python's float() is MORE permissive than JS's Number() in ways that make SQL
|
| 151 |
+
narrow where the client shows everything -- the worse direction, because at line grain that
|
| 152 |
+
is a silent under-fetch sitting beside an honest-looking row_count:
|
| 153 |
+
|
| 154 |
+
"1_000" float() -> 1000.0 (PEP-515 underscores) Number() -> NaN -> 0
|
| 155 |
+
"οΌοΌοΌ" float() -> 250.0 (any Unicode Nd) Number() -> NaN -> 0
|
| 156 |
+
"\\u0085" + "12" float() -> 12.0 (NEL is Python whitespace) Number() -> NaN -> 0
|
| 157 |
+
|
| 158 |
+
and LESS permissive in one family, which errs the other way:
|
| 159 |
+
|
| 160 |
+
"0x10" float() -> raises -> 0 Number() -> 16 (also 0b, 0o)
|
| 161 |
+
|
| 162 |
+
A tree carrying any of these is reachable: the React builder renders <input type="number">,
|
| 163 |
+
but persisted / seeded / imported view JSON goes through `clean_filter_tree`, which passes
|
| 164 |
+
the value string through untouched by design (it validates structure, not numeric syntax).
|
| 165 |
+
|
| 166 |
+
Agreed by construction on the rest: ""/None -> 0, "abc"/"12,000" -> 0, " 250 " -> 250,
|
| 167 |
+
".5"/"5."/"+5", "1e400" -> Infinity -> 0, "-0" -> 0, and float64 truncation of
|
| 168 |
+
"9007199254740993" -> 9007199254740992.
|
| 169 |
+
"""
|
| 170 |
+
if isinstance(v, bool):
|
| 171 |
+
return 1.0 if v else 0.0 # JS Number(true) === 1
|
| 172 |
+
if isinstance(v, (int, float)):
|
| 173 |
+
n = float(v)
|
| 174 |
+
return n if math.isfinite(n) else 0.0
|
| 175 |
+
if v is None:
|
| 176 |
+
return 0.0 # JS Number(null) === 0
|
| 177 |
+
s = str(v).strip(_JS_WS)
|
| 178 |
+
if s == "":
|
| 179 |
+
return 0.0 # JS Number("") === 0
|
| 180 |
+
if _JS_INF.match(s):
|
| 181 |
+
return 0.0 # Infinity is not finite -> 0
|
| 182 |
+
for pattern, base in _JS_RADIX:
|
| 183 |
+
if pattern.match(s):
|
| 184 |
+
return float(int(s[2:], base))
|
| 185 |
+
if not _JS_DEC.match(s):
|
| 186 |
+
return 0.0 # JS NaN -> 0
|
| 187 |
+
n = float(s)
|
| 188 |
+
return n if math.isfinite(n) else 0.0
|
| 189 |
+
|
| 190 |
+
|
| 191 |
+
def _value_sql(spec):
|
| 192 |
+
"""The ONE expression for a column, used by BOTH the filter and the sort.
|
| 193 |
+
|
| 194 |
+
Two expressions would let two rows tie in the grid and not tie in SQL (dates truncated for
|
| 195 |
+
comparison but ordered by the raw timestamp is the obvious way to get that wrong), so the
|
| 196 |
+
next sort key would break the tie in one engine and not the other.
|
| 197 |
+
|
| 198 |
+
numeric: `round_even(x, 0)` reproduces `aios_grid._round`, which is Python's round() =
|
| 199 |
+
BANKER'S rounding. DuckDB's plain ROUND() rounds half AWAY from zero, so ROUND(1234.5)=1235
|
| 200 |
+
while the grid shows 1234 β `eq 1234` would then match on screen and miss in SQL. The grid
|
| 201 |
+
displays rounded values; filters must agree with what is on screen.
|
| 202 |
+
date: truncate to the 10-char ISO shape the payload carries, so lexical compare is
|
| 203 |
+
chronological and a timestamp column cannot smuggle ' 00:00:00' into the comparison.
|
| 204 |
+
text: lowercased, matching `String(raw ?? "").toLowerCase()`.
|
| 205 |
+
"""
|
| 206 |
+
e = spec['sql']
|
| 207 |
+
t = spec['type']
|
| 208 |
+
if t in NUMERIC_TYPES:
|
| 209 |
+
return f"COALESCE(round_even(TRY_CAST({e} AS DOUBLE), 0), 0)"
|
| 210 |
+
if t == 'date':
|
| 211 |
+
return f"COALESCE(SUBSTR(CAST({e} AS VARCHAR), 1, 10), '')"
|
| 212 |
+
return f"LOWER(COALESCE(CAST({e} AS VARCHAR), ''))"
|
| 213 |
+
|
| 214 |
+
|
| 215 |
+
def _blank_sql(spec):
|
| 216 |
+
"""Port of `isBlank`: null/undefined/"" are blank β numeric 0 is NOT.
|
| 217 |
+
|
| 218 |
+
Reads the RAW column, never `_value_sql` (which has already coalesced blanks into 0/"",
|
| 219 |
+
the very distinction this test exists to make).
|
| 220 |
+
"""
|
| 221 |
+
e = spec['sql']
|
| 222 |
+
if spec['type'] in NUMERIC_TYPES:
|
| 223 |
+
# 0 stays non-blank: CAST(0 AS VARCHAR) is '0'. The '' arm covers a text-typed column
|
| 224 |
+
# holding "", which `_round` passes through unchanged into the payload.
|
| 225 |
+
return f"(({e}) IS NULL OR CAST({e} AS VARCHAR) = '')"
|
| 226 |
+
if spec['type'] == 'date':
|
| 227 |
+
return f"(COALESCE(SUBSTR(CAST({e} AS VARCHAR), 1, 10), '') = '')"
|
| 228 |
+
return f"(COALESCE(CAST({e} AS VARCHAR), '') = '')"
|
| 229 |
+
|
| 230 |
+
|
| 231 |
+
# --------------------------------------------------------------------------- activeness
|
| 232 |
+
|
| 233 |
+
def is_rule_active(rule, columns):
|
| 234 |
+
"""Port of `isRuleActive` + `evalNode`'s two leaf guards. Row-independent by construction.
|
| 235 |
+
|
| 236 |
+
Unknown column -> inactive (evalNode returns null before matchFilter is ever called).
|
| 237 |
+
Value-free ops are always active. `between` needs BOTH bounds.
|
| 238 |
+
|
| 239 |
+
β The four VALUE-FREE ANCHOR MODES (`is before [today]`) carry no value, and the historical
|
| 240 |
+
"blank value = inactive" rule would IGNORE them β widening the result under a count nobody
|
| 241 |
+
would doubt. Same shape as the isEmpty/isNotEmpty no-op bug, arriving by a different door.
|
| 242 |
+
"""
|
| 243 |
+
if not isinstance(rule, dict):
|
| 244 |
+
return False
|
| 245 |
+
col = rule.get('colId')
|
| 246 |
+
if col == COHORT_FIELD: # a cohort leaf has no column to look up
|
| 247 |
+
# Driven by the SET it names, so a value of "," is INACTIVE rather than active-and-
|
| 248 |
+
# unanswerable. Mirrors the cohort branch at the top of types.ts isRuleActive.
|
| 249 |
+
return bool(parse_cohort_ids(rule.get('value')))
|
| 250 |
+
if col not in columns:
|
| 251 |
+
return False
|
| 252 |
+
op = rule.get('op')
|
| 253 |
+
if op in VALUE_FREE_OPS:
|
| 254 |
+
return True
|
| 255 |
+
if op == 'within':
|
| 256 |
+
return _window_bounds(rule.get('dateWindow'), None, probe=True)
|
| 257 |
+
if rule.get('rhs') is not None:
|
| 258 |
+
rhs = rule['rhs']
|
| 259 |
+
# An rhs naming a column this table does not have is IGNORED, by the same rule as an
|
| 260 |
+
# unknown left column two lines above β for EITHER kind. Ignoring one side and hiding
|
| 261 |
+
# every row for the other would make the comparison asymmetric in a way nobody could
|
| 262 |
+
# predict. Doing it here rather than in _compile_leaf keeps activeness the single place
|
| 263 |
+
# that decides, which is what makes the compiled branch below provably unreachable.
|
| 264 |
+
if not isinstance(rhs, dict) or rhs.get('colId') not in columns:
|
| 265 |
+
return False
|
| 266 |
+
return _rhs_complete(rhs)
|
| 267 |
+
if rule.get('dateMode') in _ANCHOR_VALUE_FREE:
|
| 268 |
+
return True
|
| 269 |
+
value = _as_str(rule.get('value'))
|
| 270 |
+
if op == 'between':
|
| 271 |
+
v2 = rule.get('value2')
|
| 272 |
+
return value != '' and v2 is not None and _as_str(v2) != ''
|
| 273 |
+
return value != ''
|
| 274 |
+
|
| 275 |
+
|
| 276 |
+
def _rhs_complete(rhs):
|
| 277 |
+
"""Port of types.ts isRhsComplete. An incomplete rhs makes the rule INACTIVE, exactly as a
|
| 278 |
+
half-typed value does."""
|
| 279 |
+
if not isinstance(rhs, dict) or not rhs.get('colId'):
|
| 280 |
+
return False
|
| 281 |
+
if rhs.get('kind') == 'measure':
|
| 282 |
+
return _windows().normalize(rhs.get('window')) is not None
|
| 283 |
+
return rhs.get('kind') == 'field'
|
| 284 |
+
|
| 285 |
+
|
| 286 |
+
def _windows():
|
| 287 |
+
"""`harness.windows` imported lazily, so this module stays importable on its own.
|
| 288 |
+
|
| 289 |
+
filter_sql is otherwise PURE (no model/store/app imports) and that property is what lets
|
| 290 |
+
aios_grid embed the grid anywhere. windows.py is the one exception and it is a deliberate
|
| 291 |
+
one: the date vocabulary is a two-ENGINE contract, and a third copy of "what the past month
|
| 292 |
+
means" is precisely the drift verify_windows.py exists to prevent. It imports nothing but
|
| 293 |
+
datetime, so purity is preserved in substance.
|
| 294 |
+
"""
|
| 295 |
+
from harness import windows as _w
|
| 296 |
+
return _w
|
| 297 |
+
|
| 298 |
+
|
| 299 |
+
#: Mirrors windows.ANCHOR_VALUE_FREE, resolved lazily on first use (see _windows).
|
| 300 |
+
class _AnchorValueFree:
|
| 301 |
+
def __contains__(self, mode):
|
| 302 |
+
return mode in _windows().ANCHOR_VALUE_FREE
|
| 303 |
+
|
| 304 |
+
|
| 305 |
+
_ANCHOR_VALUE_FREE = _AnchorValueFree()
|
| 306 |
+
|
| 307 |
+
|
| 308 |
+
def _window_bounds(spec, today, probe=False):
|
| 309 |
+
"""Resolve a date-condition RANGE, or refuse.
|
| 310 |
+
|
| 311 |
+
`probe=True` answers only "is this a resolvable window?" without needing `today` β which is
|
| 312 |
+
what activeness has to know while compiling, before any date is in hand.
|
| 313 |
+
|
| 314 |
+
β A missing `today` on the real path RAISES rather than widening. The client's answer to the
|
| 315 |
+
same gap is to match nothing; the server's must be to refuse, because a server that quietly
|
| 316 |
+
drops a condition reports an authoritative row_count for a query nobody asked for. Same rule
|
| 317 |
+
as the sibling/depth caps: truncate on the client, refuse on the server.
|
| 318 |
+
"""
|
| 319 |
+
w = _windows()
|
| 320 |
+
if probe:
|
| 321 |
+
return w.normalize(spec) is not None
|
| 322 |
+
if today is None:
|
| 323 |
+
raise ValueError(
|
| 324 |
+
'a relative-date condition needs `today` β compile_filter_tree(today=...). '
|
| 325 |
+
'Refusing rather than dropping the condition, which would widen the result while '
|
| 326 |
+
'the count beside it still looked authoritative.')
|
| 327 |
+
return w.resolve(spec, today)
|
| 328 |
+
|
| 329 |
+
|
| 330 |
+
def _as_str(v):
|
| 331 |
+
"""The builder stores values as raw input strings; be tolerant of a number that survived
|
| 332 |
+
an older payload. `str(500) != ''` is active and `to_num` reads it back, matching JS where
|
| 333 |
+
`500 === ""` is false."""
|
| 334 |
+
return '' if v is None else str(v)
|
| 335 |
+
|
| 336 |
+
|
| 337 |
+
# --------------------------------------------------------------------------- leaf
|
| 338 |
+
|
| 339 |
+
def _compile_leaf(rule, columns, state):
|
| 340 |
+
"""Port of `matchFilter`. Returns (sql, params) or None when the rule is INACTIVE."""
|
| 341 |
+
if not is_rule_active(rule, columns):
|
| 342 |
+
return None
|
| 343 |
+
col_id = rule['colId']
|
| 344 |
+
op = rule.get('op')
|
| 345 |
+
value = _as_str(rule.get('value'))
|
| 346 |
+
|
| 347 |
+
# --- owner item 5: a COHORT leaf, before any column lookup -------------------------------
|
| 348 |
+
# `__cohort__` is not a column. Handled first for the same reason evalNode handles it first:
|
| 349 |
+
# falling through to the column path would make it INACTIVE, and inactive means every row.
|
| 350 |
+
if col_id == COHORT_FIELD:
|
| 351 |
+
return _compile_cohort(rule, state)
|
| 352 |
+
|
| 353 |
+
# --- C-OPS (wave 2026-08-02): a RANK leaf is structurally unanswerable at row grain ------
|
| 354 |
+
# Top-N / quantile membership is a property of the SET, not the row; the client engine
|
| 355 |
+
# answers it with a set pass over the sibling-filtered domain. Refuse loudly rather than
|
| 356 |
+
# compile something that means a different question (see RANK_OPS).
|
| 357 |
+
if op in RANK_OPS:
|
| 358 |
+
raise ValueError(
|
| 359 |
+
f'rank operator {op!r} cannot compile to row SQL - evaluate it as a set pass '
|
| 360 |
+
f'(the client engine) or pre-resolve the matching ids before compiling')
|
| 361 |
+
|
| 362 |
+
spec = columns[col_id]
|
| 363 |
+
|
| 364 |
+
def used(sql, params=()):
|
| 365 |
+
state['used'].add(col_id)
|
| 366 |
+
if spec.get('aggregate'):
|
| 367 |
+
state['aggregate'] = True
|
| 368 |
+
return (sql, list(params))
|
| 369 |
+
|
| 370 |
+
# --- value-free ops, BEFORE any value-based short-circuit (the no-op bug) ---------------
|
| 371 |
+
if op == 'isEmpty':
|
| 372 |
+
return used(_blank_sql(spec))
|
| 373 |
+
if op == 'isNotEmpty':
|
| 374 |
+
return used(f"NOT {_blank_sql(spec)}")
|
| 375 |
+
|
| 376 |
+
v = _value_sql(spec)
|
| 377 |
+
t = spec['type']
|
| 378 |
+
|
| 379 |
+
# --- CG-9: the comparand is another COLUMN, coerced through the LEFT field's type --------
|
| 380 |
+
# `_value_sql` on a spec whose `sql` is the RIGHT column and whose `type` is the LEFT one β
|
| 381 |
+
# exactly what TS does (`toNum(other)` for a numeric LHS, `String(other ?? "")` for a date,
|
| 382 |
+
# `.toLowerCase()` for text). Reading the right column through its OWN type would compare a
|
| 383 |
+
# rounded number against an unrounded one and disagree on the ties.
|
| 384 |
+
rhs_v = None
|
| 385 |
+
if rule.get('rhs') is not None:
|
| 386 |
+
# No `rspec is None` branch: `is_rule_active` already rejected an rhs naming a column
|
| 387 |
+
# this table does not have, so it cannot be missing here. A defensive fallback would be
|
| 388 |
+
# unreachable, and the negative control showed exactly that β deleting it changed
|
| 389 |
+
# nothing, which is how dead code hides behind a careful-looking line.
|
| 390 |
+
rspec = columns[rule['rhs']['colId']]
|
| 391 |
+
rhs_v = _value_sql({'sql': rspec['sql'], 'type': t})
|
| 392 |
+
state['used'].add(rule['rhs']['colId'])
|
| 393 |
+
if rspec.get('aggregate'):
|
| 394 |
+
state['aggregate'] = True
|
| 395 |
+
|
| 396 |
+
if t in NUMERIC_TYPES:
|
| 397 |
+
if rhs_v is not None:
|
| 398 |
+
return used(f"({v} {_NUM_CMP[op]} {rhs_v})") if op in _NUM_CMP else (SQL_TRUE, [])
|
| 399 |
+
if op == 'between':
|
| 400 |
+
a, b = to_num(value), to_num(rule.get('value2'))
|
| 401 |
+
return used(f"({v} >= ? AND {v} <= ?)", [min(a, b), max(a, b)])
|
| 402 |
+
if op in _NUM_CMP:
|
| 403 |
+
return used(f"({v} {_NUM_CMP[op]} ?)", [to_num(value)])
|
| 404 |
+
return (SQL_TRUE, []) # contains/doesNotContain on a number -> default: true
|
| 405 |
+
|
| 406 |
+
if t == 'date':
|
| 407 |
+
if op == 'within':
|
| 408 |
+
bounds = _window_bounds(rule.get('dateWindow'), state.get('today'))
|
| 409 |
+
if bounds is None:
|
| 410 |
+
return ('1=0', []) # unreachable: activeness probed it. Never widen.
|
| 411 |
+
lo, hi = bounds
|
| 412 |
+
# `{v} <> ''` is NOT redundant. `_value_sql` COALESCEs a NULL date to '', and
|
| 413 |
+
# '' <= any upper bound is true β so an open-ended window would sweep in every
|
| 414 |
+
# never-ordered customer. TS guards it with `if (a === "") return false`.
|
| 415 |
+
parts, params = [f"{v} <> ''"], []
|
| 416 |
+
if lo is not None:
|
| 417 |
+
parts.append(f"{v} >= ?")
|
| 418 |
+
params.append(lo)
|
| 419 |
+
if hi is not None:
|
| 420 |
+
parts.append(f"{v} <= ?")
|
| 421 |
+
params.append(hi)
|
| 422 |
+
return used('(' + ' AND '.join(parts) + ')', params)
|
| 423 |
+
if rhs_v is not None:
|
| 424 |
+
return used(f"({v} {_DATE_CMP[op]} {rhs_v})") if op in _DATE_CMP else (SQL_TRUE, [])
|
| 425 |
+
if op == 'between':
|
| 426 |
+
v2 = _as_str(rule.get('value2'))
|
| 427 |
+
lo, hi = (value, v2) if value < v2 else (v2, value)
|
| 428 |
+
return used(f"({v} >= ? AND {v} <= ?)", [lo, hi])
|
| 429 |
+
if op in _DATE_CMP:
|
| 430 |
+
anchor = _resolve_anchor(rule.get('dateMode'), value, state.get('today'))
|
| 431 |
+
if anchor is None:
|
| 432 |
+
return ('1=0', []) # unresolvable anchor -> matches nothing (TS: false)
|
| 433 |
+
return used(f"({v} {_DATE_CMP[op]} ?)", [anchor])
|
| 434 |
+
return (SQL_TRUE, []) # contains/doesNotContain on a date -> default: true
|
| 435 |
+
|
| 436 |
+
# text / status β case-insensitive. strpos, not LIKE: `includes` is a LITERAL substring
|
| 437 |
+
# test, so a value holding % or _ must not act as a wildcard.
|
| 438 |
+
#
|
| 439 |
+
# BOTH SIDES fold in SQL (`LOWER(?)`, never Python's str.lower()). Python and JS agree on
|
| 440 |
+
# case folding; DuckDB does NOT β LOWER('ISTANBUL' with a dotted capital I) drops the dot
|
| 441 |
+
# while both languages keep it as i + combining dot. Folding the needle in Python and the
|
| 442 |
+
# haystack in DuckDB therefore produced two different strings, and a value could fail to
|
| 443 |
+
# match ITSELF. One folder for both operands is the only version that cannot do that.
|
| 444 |
+
if rhs_v is not None:
|
| 445 |
+
# Both operands are already folded by `_value_sql` (text branch), so no LOWER(?) here.
|
| 446 |
+
if op == 'contains':
|
| 447 |
+
return used(f"(strpos({v}, {rhs_v}) > 0)")
|
| 448 |
+
if op == 'doesNotContain':
|
| 449 |
+
return used(f"(strpos({v}, {rhs_v}) = 0)")
|
| 450 |
+
if op == 'eq':
|
| 451 |
+
return used(f"({v} = {rhs_v})")
|
| 452 |
+
if op == 'neq':
|
| 453 |
+
return used(f"({v} <> {rhs_v})")
|
| 454 |
+
return (SQL_TRUE, [])
|
| 455 |
+
if op == 'contains':
|
| 456 |
+
return used(f"(strpos({v}, LOWER(?)) > 0)", [value])
|
| 457 |
+
if op == 'doesNotContain':
|
| 458 |
+
return used(f"(strpos({v}, LOWER(?)) = 0)", [value])
|
| 459 |
+
if op == 'eq':
|
| 460 |
+
return used(f"({v} = LOWER(?))", [value])
|
| 461 |
+
if op == 'neq':
|
| 462 |
+
return used(f"({v} <> LOWER(?))", [value])
|
| 463 |
+
return (SQL_TRUE, []) # gt/gte/lt/lte/between/within on text -> default: true
|
| 464 |
+
|
| 465 |
+
|
| 466 |
+
def _resolve_anchor(mode, value, today):
|
| 467 |
+
"""A date condition's anchor -> one ISO date, or None. `today` may be absent ONLY for the
|
| 468 |
+
`exact` mode, which does not use it.
|
| 469 |
+
|
| 470 |
+
The sentinel is deliberate and load-bearing for BACK-COMPAT: every date filter that existed
|
| 471 |
+
before anchors is `exact`, and every caller that compiles one (store_query, store_rows)
|
| 472 |
+
passes no `today`. Requiring it unconditionally would turn a working filter into an
|
| 473 |
+
exception. A RELATIVE anchor still refuses β the server never widens.
|
| 474 |
+
"""
|
| 475 |
+
mode = mode or 'exact'
|
| 476 |
+
if today is None:
|
| 477 |
+
if mode != 'exact':
|
| 478 |
+
raise ValueError(
|
| 479 |
+
f"the date anchor {mode!r} is relative and needs `today` β "
|
| 480 |
+
f"compile_filter_tree(today=...). Refusing rather than dropping the condition, "
|
| 481 |
+
f"which would widen the result under an authoritative-looking count.")
|
| 482 |
+
# `exact` ignores `today` entirely (verify_windows resolves it across 11 probe dates and
|
| 483 |
+
# gets the same answer every time), so any valid date serves as the unused argument.
|
| 484 |
+
today = '1970-01-01'
|
| 485 |
+
return _windows().resolve_anchor(mode, value, today)
|
| 486 |
+
|
| 487 |
+
|
| 488 |
+
def _compile_cohort(rule, state):
|
| 489 |
+
"""Port of evalNode's cohort branch: membership in a SET of hand-curated sets of row ids.
|
| 490 |
+
|
| 491 |
+
Owner 2026-07-27: the leaf names one OR MORE cohorts and asks `anyOf` / `allOf` / `noneOf`.
|
| 492 |
+
A single id is a one-element set, so the shipped `is part of` compiles down this same path.
|
| 493 |
+
|
| 494 |
+
States, mirroring TS exactly:
|
| 495 |
+
every set present -> the union (anyOf / noneOf) or the intersection (allOf)
|
| 496 |
+
a set present but EMPTY -> that member contributes 1=0, because nobody is in an empty
|
| 497 |
+
cohort. So `allOf` collapses to 1=0 and `noneOf` to 1=1 β
|
| 498 |
+
which is what `set.has(pid)` on an empty Set does in TS.
|
| 499 |
+
ANY set ABSENT -> 1=0 for ALL THREE ops. The condition is unanswerable, and
|
| 500 |
+
unanswerable matches nothing: answering "everything" to
|
| 501 |
+
`is none of [a cohort we cannot see]` is the widening sin
|
| 502 |
+
wearing a plausible face, and answering it for the members we
|
| 503 |
+
DO have would silently ask a different question.
|
| 504 |
+
|
| 505 |
+
COALESCE(... , FALSE) because a NULL row id makes `id IN (...)` NULL, and `NOT NULL` is NULL
|
| 506 |
+
β SQL would drop the row where TS keeps it.
|
| 507 |
+
"""
|
| 508 |
+
sets = state.get('cohort_sets')
|
| 509 |
+
if sets is None:
|
| 510 |
+
raise ValueError(
|
| 511 |
+
'a cohort condition needs cohort_sets β compile_filter_tree(cohort_sets=...). '
|
| 512 |
+
'Refusing rather than dropping it: a dropped condition widens the result while the '
|
| 513 |
+
'count beside it still looks authoritative.')
|
| 514 |
+
id_sql = state.get('id_sql')
|
| 515 |
+
if not id_sql:
|
| 516 |
+
raise ValueError('a cohort condition needs id_sql (the row-identity column)')
|
| 517 |
+
op = COHORT_OP_ALIASES.get(rule.get('op'), rule.get('op'))
|
| 518 |
+
named = parse_cohort_ids(rule.get('value'))
|
| 519 |
+
members = [sets.get(cid) for cid in named]
|
| 520 |
+
if not named or any(m is None for m in members):
|
| 521 |
+
return ('1=0', [])
|
| 522 |
+
|
| 523 |
+
parts, params = [], []
|
| 524 |
+
for ids in members:
|
| 525 |
+
ids = [int(x) for x in ids]
|
| 526 |
+
if not ids:
|
| 527 |
+
parts.append('1=0')
|
| 528 |
+
continue
|
| 529 |
+
parts.append(f"COALESCE({id_sql} IN ({','.join('?' for _ in ids)}), FALSE)")
|
| 530 |
+
params.extend(ids)
|
| 531 |
+
if op == 'allOf':
|
| 532 |
+
return ('(' + ' AND '.join(parts) + ')', params)
|
| 533 |
+
union = '(' + ' OR '.join(parts) + ')'
|
| 534 |
+
return ((f"(NOT {union})" if op == 'noneOf' else union), params)
|
| 535 |
+
|
| 536 |
+
|
| 537 |
+
# --------------------------------------------------------------------------- tree
|
| 538 |
+
|
| 539 |
+
def _compile_node(node, columns, state):
|
| 540 |
+
"""Port of `evalNode`. Returns (sql, params), or None when the node is INACTIVE.
|
| 541 |
+
|
| 542 |
+
A group combines only its ACTIVE children with its OWN conjunction; a group with no active
|
| 543 |
+
children is itself inactive and disappears. That is what stops a half-built group from
|
| 544 |
+
either zeroing the grid (`and`) or disabling it (`or`).
|
| 545 |
+
"""
|
| 546 |
+
if not isinstance(node, dict):
|
| 547 |
+
return None
|
| 548 |
+
if isinstance(node.get('children'), list):
|
| 549 |
+
parts, params = [], []
|
| 550 |
+
for child in node['children']:
|
| 551 |
+
got = _compile_node(child, columns, state)
|
| 552 |
+
if got is None:
|
| 553 |
+
continue # inactive -> ignored entirely
|
| 554 |
+
parts.append(got[0])
|
| 555 |
+
params.extend(got[1])
|
| 556 |
+
if not parts:
|
| 557 |
+
return None
|
| 558 |
+
joiner = ' OR ' if node.get('conj') == 'or' else ' AND '
|
| 559 |
+
return ('(' + joiner.join(parts) + ')', params)
|
| 560 |
+
return _compile_leaf(node, columns, state)
|
| 561 |
+
|
| 562 |
+
|
| 563 |
+
def compile_filter_tree(nodes, conj='and', columns=None, member_ids=None, id_sql=None,
|
| 564 |
+
today=None, cohort_sets=None):
|
| 565 |
+
"""Port of `matchFilterTree` (+ the `memberPids` union in `useVisibleRows`).
|
| 566 |
+
|
| 567 |
+
Returns a `Compiled`, or **None when nothing is active** β meaning no predicate at all, not
|
| 568 |
+
a false one. `matchFilterTree` returns true when the tree is inactive, so emitting anything
|
| 569 |
+
here would narrow rows the client would have shown.
|
| 570 |
+
|
| 571 |
+
`member_ids` reproduces `members.has(r.pid) || matchFilterTree(...)`. Note the union lives
|
| 572 |
+
INSIDE `if (filters.length)` and is OR'd with a tree that is true for everything when
|
| 573 |
+
inactive β so an inactive tree must emit NOTHING, never `pid IN (...)`, which would narrow
|
| 574 |
+
to just the hand-picked members. Same family of bug as the tri-state one, inverted.
|
| 575 |
+
|
| 576 |
+
`today` is the TENANT'S today, required by any relative date condition (`is within β¦`,
|
| 577 |
+
`is before [one month ago]`) and by nothing else β an `exact` date needs none, which is what
|
| 578 |
+
keeps every pre-anchor caller working unchanged. `cohort_sets` is `{cohortId: [pid, ...]}`
|
| 579 |
+
for cohort-membership leaves. Both REFUSE when needed and missing; neither defaults.
|
| 580 |
+
"""
|
| 581 |
+
state = {'aggregate': False, 'used': set(),
|
| 582 |
+
'today': today, 'cohort_sets': cohort_sets, 'id_sql': id_sql}
|
| 583 |
+
got = _compile_node({'conj': conj, 'children': list(nodes or [])}, columns or {}, state)
|
| 584 |
+
if got is None:
|
| 585 |
+
return None
|
| 586 |
+
sql, params = got
|
| 587 |
+
if member_ids:
|
| 588 |
+
if not id_sql:
|
| 589 |
+
raise ValueError('member_ids requires id_sql (the row-identity column)')
|
| 590 |
+
ids = [int(x) for x in member_ids]
|
| 591 |
+
sql = f"({sql} OR {id_sql} IN ({','.join('?' for _ in ids)}))"
|
| 592 |
+
params = list(params) + ids
|
| 593 |
+
return Compiled(sql, list(params), state['aggregate'], frozenset(state['used']))
|
| 594 |
+
|
| 595 |
+
|
| 596 |
+
def compile_search(q, columns):
|
| 597 |
+
"""Port of `matchSearch`: the narrowing search box.
|
| 598 |
+
|
| 599 |
+
True if ANY human-readable (text/status) column contains `q`, case-insensitively. NUMERIC
|
| 600 |
+
columns are skipped on purpose so a query like "10" does not match revenue or order counts.
|
| 601 |
+
Dates are skipped too β `matchSearch` tests only text and status.
|
| 602 |
+
|
| 603 |
+
The caller ANDs this with the filter predicate, matching the TS pipeline order
|
| 604 |
+
(filter -> search). It is deliberately OUTSIDE the `member_ids` union: in `useVisibleRows`
|
| 605 |
+
the union applies to the filter step only, so a hand-picked member is still searchable away.
|
| 606 |
+
"""
|
| 607 |
+
# `.strip(_JS_WS)`, not `.strip()`: the client trims with JS String.trim(), whose whitespace
|
| 608 |
+
# set differs from Python's in BOTH directions (Python strips U+0085 NEL and U+001C..1F,
|
| 609 |
+
# JS does not; JS strips U+FEFF, Python does not). A query of just those characters is
|
| 610 |
+
# blank to one engine and a real search to the other. Folding is left to SQL β see the
|
| 611 |
+
# LOWER(?) note in _compile_leaf.
|
| 612 |
+
q = (q or '').strip(_JS_WS)
|
| 613 |
+
if not q:
|
| 614 |
+
return None
|
| 615 |
+
parts, params, used = [], [], set()
|
| 616 |
+
aggregate = False
|
| 617 |
+
for col_id, spec in (columns or {}).items():
|
| 618 |
+
if spec['type'] not in TEXTUAL_TYPES:
|
| 619 |
+
continue
|
| 620 |
+
parts.append(f"(strpos({_value_sql(spec)}, LOWER(?)) > 0)")
|
| 621 |
+
params.append(q)
|
| 622 |
+
used.add(col_id)
|
| 623 |
+
aggregate = aggregate or bool(spec.get('aggregate'))
|
| 624 |
+
if not parts:
|
| 625 |
+
# No searchable column: `matchSearch` returns false for every row, so the search
|
| 626 |
+
# narrows to nothing. That is a real predicate, not an absent one.
|
| 627 |
+
return Compiled('1=0', [], False, frozenset())
|
| 628 |
+
return Compiled('(' + ' OR '.join(parts) + ')', params, aggregate, frozenset(used))
|
| 629 |
+
|
| 630 |
+
|
| 631 |
+
def compile_order_by(sorts, columns, tiebreak_sql=None):
|
| 632 |
+
"""Port of `makeComparator`. Returns an ORDER BY body, or None when nothing sorts.
|
| 633 |
+
|
| 634 |
+
`tiebreak_sql` (the row-identity column) is appended as a final ASC key and is REQUIRED for
|
| 635 |
+
any windowed fetch. `Array.prototype.sort` is stable, so TS leaves tied rows in their
|
| 636 |
+
incoming order; SQL guarantees NOTHING for ties, so LIMIT/OFFSET paging over a non-total
|
| 637 |
+
order can repeat a row on one page and drop it from another. Appending row identity makes
|
| 638 |
+
the order total. It is a STRENGTHENING of the TS guarantee, not a divergence: the payload a
|
| 639 |
+
windowed fetch produces is itself in this order, so the client never re-sorts it differently.
|
| 640 |
+
|
| 641 |
+
BLANKS SORT LAST IN BOTH DIRECTIONS (Airtable's behavior, and the reason descending
|
| 642 |
+
"Last order" leads with the most recent rather than with never-ordered customers). Each key
|
| 643 |
+
contributes TWO terms: an always-ASC blank flag, then the value in the requested direction.
|
| 644 |
+
Two blanks tie on both terms, so the NEXT key decides β matching the comparator's
|
| 645 |
+
`if (aBlank && bBlank) continue`.
|
| 646 |
+
|
| 647 |
+
An unknown column is skipped: in TS its values are `undefined` on both sides, so both are
|
| 648 |
+
blank and the key is a no-op tiebreak.
|
| 649 |
+
|
| 650 |
+
Direction: `s.dir === "asc" ? c : -c` β only the exact string "asc" ascends, anything else
|
| 651 |
+
descends. Faithfully reproduced; do not "fix" it to default ASC.
|
| 652 |
+
|
| 653 |
+
Residual divergence, deliberate: TS uses `localeCompare`, we use LOWER() + DuckDB's binary
|
| 654 |
+
collation. They agree on case-insensitive ordering of ASCII and on ISO dates (unaffected by
|
| 655 |
+
LOWER), and differ on locale tie-breaks between case-variants of the same string ("a" vs
|
| 656 |
+
"A" sort adjacent under localeCompare, tie under LOWER) and on accent ordering.
|
| 657 |
+
"""
|
| 658 |
+
terms = []
|
| 659 |
+
for s in (sorts or []):
|
| 660 |
+
if not isinstance(s, dict):
|
| 661 |
+
continue
|
| 662 |
+
spec = (columns or {}).get(s.get('colId'))
|
| 663 |
+
if spec is None:
|
| 664 |
+
continue
|
| 665 |
+
terms.append(f"CASE WHEN {_blank_sql(spec)} THEN 1 ELSE 0 END ASC")
|
| 666 |
+
terms.append(f"{_value_sql(spec)} {'ASC' if s.get('dir') == 'asc' else 'DESC'}")
|
| 667 |
+
# Append the tiebreak even when NO sort key survived (all unknown, or none given). A
|
| 668 |
+
# windowed fetch needs a total order regardless of what the user sorted by β an unsorted
|
| 669 |
+
# LIMIT/OFFSET is exactly as unstable as a tied one. Callers wanting pure client parity
|
| 670 |
+
# pass no tiebreak_sql and still get None.
|
| 671 |
+
if tiebreak_sql:
|
| 672 |
+
terms.append(f"{tiebreak_sql} ASC")
|
| 673 |
+
return ', '.join(terms) or None
|
platform/harness/measure_filter.py
ADDED
|
@@ -0,0 +1,788 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""harness/measure_filter.py β answering `Where [Sales] [in the last 90 days] [>] [5,000]` (CG-8).
|
| 2 |
+
|
| 3 |
+
GATED 2026-07-26 by `platform/verify_measure_filter.py` β 119 assertions against the real
|
| 4 |
+
store (every resolved id set compared to a per-customer aggregate summed in Python from raw rows)
|
| 5 |
+
plus a synthetic store for the cases live data cannot produce. Negative control: 24 faithful
|
| 6 |
+
mutations, 24 caught. The gate found two things the reasoning below asserted and the code did not
|
| 7 |
+
do, both now fixed and both re-caught by the control:
|
| 8 |
+
|
| 9 |
+
- a rule with NO window resolved to ALL TIME. The error message four lines below says it
|
| 10 |
+
refuses to do exactly that; the line above it did it anyway.
|
| 11 |
+
- `sum()` over an all-NULL group is NULL, and `NULL > x` and `NULL <= x` are BOTH false, so
|
| 12 |
+
such a customer fell out of the two halves of the same question. Now COALESCEd to 0.
|
| 13 |
+
|
| 14 |
+
THE PROBLEM. `revenue_ytd` is a pre-computed column in `customer_data.pool()`. "Sales in the last
|
| 15 |
+
90 days" is a DIFFERENT aggregate over a DIFFERENT window, and no column holds it. So a measure
|
| 16 |
+
condition cannot be answered by looking at the row in hand β it has to be a query.
|
| 17 |
+
|
| 18 |
+
THE SHAPE. Resolve each measure condition server-side to a **set of customer ids**, and let the
|
| 19 |
+
existing client engine treat that set as a membership test. Per condition, one query:
|
| 20 |
+
|
| 21 |
+
SELECT rp_id FROM <topic scope> WHERE <window> GROUP BY rp_id
|
| 22 |
+
HAVING <the metric's OWN aggregate expression> <op> ?
|
| 23 |
+
|
| 24 |
+
Three properties that fall out, none incidental:
|
| 25 |
+
|
| 26 |
+
* **No second definition of a number.** The aggregate comes from `semantic._measure_sql` β the
|
| 27 |
+
same expression the Analyst and every metric already use. This cannot disagree with "revenue"
|
| 28 |
+
elsewhere because it IS that computation.
|
| 29 |
+
* **The displayed columns are untouched.** `revenue_ytd` still comes from `pool()`. We ADD a
|
| 30 |
+
predicate; we do not re-source the table. So this needs no `pool()`βstore migration, and does
|
| 31 |
+
not inherit that migration's parity problem.
|
| 32 |
+
* **Counts stay honest.** The client still holds every row, so the table stays whole-book and
|
| 33 |
+
`countLabel` keeps counting what it actually has (CG-3 `whole-book` mode).
|
| 34 |
+
|
| 35 |
+
β WHAT MAY BE OFFERED, AND WHY IT IS AN ALLOWLIST
|
| 36 |
+
`store_parity()` validates exactly five metrics against live Odoo. Every OTHER sum-metric on the
|
| 37 |
+
topic would get an expression with no independent check behind it β self-consistent with the rest
|
| 38 |
+
of the app and still wrong if the expression is wrong. Against "robust for EVERY numerically
|
| 39 |
+
driven object", that gap is load-bearing, so admission is gated on HAVING A CONTRACT and spelled
|
| 40 |
+
as a list. An allowlist fails CLOSED when a metric is added; iterating the topic's metrics would
|
| 41 |
+
fail open, silently offering an unvalidated number in the one place a number looks authoritative.
|
| 42 |
+
|
| 43 |
+
β AN UNRESOLVABLE CONDITION RAISES. It never returns "everything" or "nothing": both are answers,
|
| 44 |
+
and a wrong answer here is invisible β the grid just shows a different set of customers under a
|
| 45 |
+
count that looks authoritative. `ModelError` is the only safe failure.
|
| 46 |
+
"""
|
| 47 |
+
from harness import semantic as _sem
|
| 48 |
+
from harness import windows as _win
|
| 49 |
+
|
| 50 |
+
#: metric key -> how it reads in the condition builder. ONLY metrics with an independent
|
| 51 |
+
#: validation contract (semantic.store_parity / a metric `validate:` block) appear here.
|
| 52 |
+
#: `orders` lives on sales_orders and `customers` is a property of a SET, not of a customer β
|
| 53 |
+
#: neither is a per-customer measure, so neither is offered.
|
| 54 |
+
#: Each entry carries its TOPIC, because admission is no longer single-topic. `orders` lives on
|
| 55 |
+
#: sales_orders (order-header grain) while the other three live on sales_lines β and the topic
|
| 56 |
+
#: decides the FROM clause, the scope SQL and which dim identifies a customer. Letting the caller
|
| 57 |
+
#: pass the topic (the old shape) meant a caller could pass a wrong one; deriving it from the
|
| 58 |
+
#: measure means there is nothing to get wrong.
|
| 59 |
+
ADMITTED = {
|
| 60 |
+
'revenue': {'label': 'Sales', 'topic': 'sales_lines'},
|
| 61 |
+
'margin': {'label': 'Gross Margin $', 'topic': 'sales_lines'},
|
| 62 |
+
'units': {'label': 'Units', 'topic': 'sales_lines'},
|
| 63 |
+
# `orders` is admitted on the SAME rule as the other three and not by exception:
|
| 64 |
+
# `semantic.store_parity()` checks it store-vs-live per BU, and the metric carries an
|
| 65 |
+
# `order_count` validate contract. It is the replacement for the retired `Orders 24M`
|
| 66 |
+
# column β same question, with the period as a parameter instead of frozen at 24 months.
|
| 67 |
+
'orders': {'label': 'Orders', 'topic': 'sales_orders'},
|
| 68 |
+
# β The first COMPOSITE measure (2026-07-27, owner's follow-up): a RATIO, not an aggregate.
|
| 69 |
+
# `aov` is `revenue` (line grain, sales_lines) over `orders` (header grain, sales_orders), so
|
| 70 |
+
# it has no single query and `_measure_sql` REFUSES it β which is the fail-closed shape that
|
| 71 |
+
# makes a mis-wired ratio raise instead of quietly computing something else. It is resolved
|
| 72 |
+
# by computing each BASE per customer and combining them through `semantic._post_compute`,
|
| 73 |
+
# the same function every other surface uses, so AOV here cannot disagree with AOV anywhere.
|
| 74 |
+
# `topic` names where the metric is DECLARED; the bases each resolve on their own.
|
| 75 |
+
'aov': {'label': 'Avg order $', 'topic': 'sales_lines'},
|
| 76 |
+
# β `input_scale` SETTLES the question this measure was withheld over. The metric stays a
|
| 77 |
+
# FRACTION (margin/revenue, 0-1) β it is the semantic model's number and every other surface
|
| 78 |
+
# reads it β while the USER'S TYPED VALUE is divided by 100 on the way in, so
|
| 79 |
+
# `Gross Margin % > 20` compares against 0.20. The factor moves exactly once, at the input
|
| 80 |
+
# boundary, and never touches the metric. Scaling a COMPUTED comparand (a statistic, or the
|
| 81 |
+
# other side of a measure-vs-measure) would divide by 100 twice, so it is applied to the
|
| 82 |
+
# literal alone and the gate proves that by construction.
|
| 83 |
+
'margin_pct': {'label': 'Gross Margin %', 'topic': 'sales_lines', 'input_scale': 100.0},
|
| 84 |
+
# β COMPANY-LEVEL (2026-07-27, owner's ask). Credit notes carry team_id = 1 for everything,
|
| 85 |
+
# so a business-unit total is not a thing that exists β `_query_parts` refuses one and the
|
| 86 |
+
# live domain builder refuses one. `measure_fields(team_id)` therefore stops OFFERING these
|
| 87 |
+
# to a BU-scoped caller rather than letting them build a condition that can only error: a
|
| 88 |
+
# dead entry in the picker is worse than an absent one, and hiding what cannot be answered
|
| 89 |
+
# is what `filterable: false` already does for columns.
|
| 90 |
+
'returns': {'label': 'Returns $', 'topic': 'credit_notes'},
|
| 91 |
+
'returns_pct': {'label': 'Return %', 'topic': 'credit_notes', 'input_scale': 100.0},
|
| 92 |
+
}
|
| 93 |
+
|
| 94 |
+
#: The metrics `semantic.store_parity()` actually exercises against live Odoo. ADMITTED must be
|
| 95 |
+
#: a SUBSET of this β that is the whole admission rule ("does this measure carry an independent
|
| 96 |
+
#: validation contract"), and stating it as data lets the gate assert it instead of trusting the
|
| 97 |
+
#: comment above. Adding a measure here without adding it to store_parity's loop fails the gate.
|
| 98 |
+
PARITY_CHECKED = frozenset({'revenue', 'margin', 'units', 'orders', 'customers',
|
| 99 |
+
'returns', 'invoiced'})
|
| 100 |
+
|
| 101 |
+
#: numeric comparison ops, mirroring filter_sql._NUM_CMP. `between` is expressed as two
|
| 102 |
+
#: conditions by the caller rather than a third operator shape.
|
| 103 |
+
OPS = {'gt': '>', 'gte': '>=', 'lt': '<', 'lte': '<=', 'eq': '=', 'neq': '<>'}
|
| 104 |
+
|
| 105 |
+
#: CG-9 β the ops allowed when the right-hand side is ANOTHER MEASURE. `eq`/`neq` are withheld,
|
| 106 |
+
#: and not out of caution: both sides are sums of floats the user never sees, so `=` would match
|
| 107 |
+
#: nobody and `β ` would match everybody, and both would LOOK like they worked. CG-8 could offer
|
| 108 |
+
#: `=` honestly because the other side was a number a person typed. Mirrors types.ts
|
| 109 |
+
#: MEASURE_PAIR_OPS; the gate asserts the two lists are the same.
|
| 110 |
+
PAIR_OPS = ('gte', 'gt', 'lte', 'lt')
|
| 111 |
+
|
| 112 |
+
#: The statistics a condition may compare against β "who is in the top quartile" (owner,
|
| 113 |
+
#: 2026-07-27). Each is computed over the CALLER'S POOL for the same measure and window.
|
| 114 |
+
#: The labels are the sentence the user reads; the client mirrors them.
|
| 115 |
+
STATS = {
|
| 116 |
+
'avg': 'the average',
|
| 117 |
+
'median': 'the median',
|
| 118 |
+
'p10': 'the bottom 10%',
|
| 119 |
+
'p25': 'the bottom quartile',
|
| 120 |
+
'p75': 'the top quartile',
|
| 121 |
+
'p90': 'the top 10%',
|
| 122 |
+
}
|
| 123 |
+
# β The owner's OWN words, and short enough to fit: "the 75th percentile (top quartile)" clipped
|
| 124 |
+
# to "the 75th perce" in the value select, and a truncated parenthetical reads as a rendering
|
| 125 |
+
# bug. `> the top quartile` is also how a person says it out loud, so the shorter label is the
|
| 126 |
+
# better one twice over. Seen in a live screenshot of a run whose every assertion was green.
|
| 127 |
+
|
| 128 |
+
#: β THE POPULATION IS THE CALLER'S POOL, INCLUDING ZEROS. A customer who bought nothing has a
|
| 129 |
+
#: measure of 0, not of nothing β that is the convention the zero-group already establishes
|
| 130 |
+
#: everywhere else, and "the median customer" means the median of the customers you can see.
|
| 131 |
+
#: Two consequences worth knowing before reading a result:
|
| 132 |
+
#: Β· it is a PERMISSION boundary as well as a definition β a scoped user's median is theirs,
|
| 133 |
+
#: and a statistic over the whole book would leak the shape of data they cannot open;
|
| 134 |
+
#: Β· on a measure most customers score 0 on, the 75th percentile IS 0, so "top quartile"
|
| 135 |
+
#: degenerates to "anyone with any activity at all". True, and surprising.
|
| 136 |
+
|
| 137 |
+
|
| 138 |
+
def percentile(values, q):
|
| 139 |
+
"""The `q` quantile of `values` by LINEAR INTERPOLATION between closest ranks β the R-7
|
| 140 |
+
convention, which is what Excel's PERCENTILE.INC computes. `q` may also be a STATS key.
|
| 141 |
+
|
| 142 |
+
Pinned to one convention on purpose: nearest-rank and interpolation disagree on any even
|
| 143 |
+
population (the median of [1,2,3,4] is 2.5 here and 2 or 3 under nearest-rank), and the gate
|
| 144 |
+
checks it against hand-computed values rather than against `statistics.median` β a reference
|
| 145 |
+
borrowed from a library only proves we match that library.
|
| 146 |
+
|
| 147 |
+
Returns None for an empty population: there is no median of nobody, and 0 would be a number
|
| 148 |
+
the caller could act on.
|
| 149 |
+
"""
|
| 150 |
+
if isinstance(q, str):
|
| 151 |
+
if q == 'avg':
|
| 152 |
+
xs = list(values)
|
| 153 |
+
return (sum(xs) / len(xs)) if xs else None
|
| 154 |
+
if q not in STATS:
|
| 155 |
+
raise _sem.ModelError(
|
| 156 |
+
f"{q!r} is not a statistic. Offered: {sorted(STATS)}")
|
| 157 |
+
q = 0.5 if q == 'median' else int(q[1:]) / 100.0
|
| 158 |
+
xs = sorted(float(v) for v in values)
|
| 159 |
+
if not xs:
|
| 160 |
+
return None
|
| 161 |
+
if len(xs) == 1:
|
| 162 |
+
return xs[0]
|
| 163 |
+
pos = (len(xs) - 1) * float(q)
|
| 164 |
+
lo = int(pos)
|
| 165 |
+
hi = min(lo + 1, len(xs) - 1)
|
| 166 |
+
return xs[lo] + (xs[hi] - xs[lo]) * (pos - lo)
|
| 167 |
+
|
| 168 |
+
|
| 169 |
+
#: The same six comparisons as OPS, as Python callables β used wherever the comparison happens in
|
| 170 |
+
#: Python rather than in a HAVING (a composite measure, or a two-measure pair).
|
| 171 |
+
_CMP = {'gt': lambda a, b: a > b, 'gte': lambda a, b: a >= b,
|
| 172 |
+
'lt': lambda a, b: a < b, 'lte': lambda a, b: a <= b,
|
| 173 |
+
'eq': lambda a, b: a == b, 'neq': lambda a, b: a != b}
|
| 174 |
+
|
| 175 |
+
|
| 176 |
+
def is_company_level(key):
|
| 177 |
+
"""True when this measure cannot be scoped to a business unit.
|
| 178 |
+
|
| 179 |
+
Checked over every BASE the measure decomposes into, not just the declared topic: `Return %`
|
| 180 |
+
names credit_notes but also reads customer_invoices, and one un-scopable half is enough to
|
| 181 |
+
make the whole ratio un-scopable.
|
| 182 |
+
"""
|
| 183 |
+
bases, _ = _sem._expand_measures([key])
|
| 184 |
+
tps = _sem.topics()
|
| 185 |
+
for b in bases:
|
| 186 |
+
store = (tps.get(_sem.metrics()[b].get('topic')) or {}).get('store') or {}
|
| 187 |
+
if not store.get('team_col'):
|
| 188 |
+
return True
|
| 189 |
+
return False
|
| 190 |
+
|
| 191 |
+
|
| 192 |
+
def measure_fields(team_id=None):
|
| 193 |
+
"""The measures the condition builder may offer, in the payload's field shape.
|
| 194 |
+
|
| 195 |
+
These are FILTERABLE BUT NOT DISPLAYABLE β a measure is a question about a window, not a
|
| 196 |
+
column of the table, and rendering it as a column would silently pick a window for the user.
|
| 197 |
+
|
| 198 |
+
β `team_id` NARROWS THE OFFER. A COMPANY-LEVEL measure (returns, and anything else built on
|
| 199 |
+
documents that carry no trustworthy BU tag) cannot be answered for one business unit β
|
| 200 |
+
`_query_parts` refuses, and refusing is right. But a picker entry whose every use ends in an
|
| 201 |
+
error is worse than no entry, so a BU-scoped caller is not offered them at all. Same
|
| 202 |
+
principle as `filterable: false` on a column: do not offer what cannot be answered.
|
| 203 |
+
"""
|
| 204 |
+
mts = _sem.metrics()
|
| 205 |
+
out = []
|
| 206 |
+
for key, spec in ADMITTED.items():
|
| 207 |
+
m = mts.get(key)
|
| 208 |
+
if not m:
|
| 209 |
+
continue
|
| 210 |
+
if team_id and is_company_level(key):
|
| 211 |
+
continue
|
| 212 |
+
out.append({'key': key, 'label': spec['label'],
|
| 213 |
+
'type': _sem._FORMAT_TYPE.get(m.get('format'), 'currency')})
|
| 214 |
+
return out
|
| 215 |
+
|
| 216 |
+
|
| 217 |
+
def topic_for(key):
|
| 218 |
+
"""The topic a measure is answered on. Raises for anything not admitted."""
|
| 219 |
+
spec = ADMITTED.get(key)
|
| 220 |
+
if not spec:
|
| 221 |
+
raise _sem.ModelError(
|
| 222 |
+
f"{key!r} is not an admitted measure. Offered: {sorted(ADMITTED)} β a measure "
|
| 223 |
+
f"without an independent validation contract is not filterable (see this module's "
|
| 224 |
+
f"docstring).")
|
| 225 |
+
return spec['topic']
|
| 226 |
+
|
| 227 |
+
|
| 228 |
+
def _entity_sql(topic):
|
| 229 |
+
"""(from-clause, the id column customers are grouped by, the store binding) for a topic."""
|
| 230 |
+
t = _sem.topics().get(topic)
|
| 231 |
+
if not t or 'store' not in t:
|
| 232 |
+
raise _sem.ModelError(f"topic {topic!r} has no store binding")
|
| 233 |
+
s = t['store']
|
| 234 |
+
dims = s.get('dims') or {}
|
| 235 |
+
# sales_lines calls it `order_partner` (the line's parent order's partner); sales_orders
|
| 236 |
+
# calls the same thing `partner`. One name per topic, both meaning "the customer this row
|
| 237 |
+
# belongs to" β which is the only dim this module ever groups by.
|
| 238 |
+
dim = dims.get('order_partner') or dims.get('partner')
|
| 239 |
+
if not dim:
|
| 240 |
+
raise _sem.ModelError(f"topic {topic!r} has no customer dim to group by")
|
| 241 |
+
return f"{s['table']} {s['alias']} {s.get('join', '')}", dim['col'], s
|
| 242 |
+
|
| 243 |
+
|
| 244 |
+
def is_composite(key):
|
| 245 |
+
"""True for a measure with no single aggregate expression β a `ratio` or a `derived`.
|
| 246 |
+
|
| 247 |
+
These are answered by computing each BASE per customer and combining in Python; they can
|
| 248 |
+
never be a HAVING, which is why `_measure_sql` refusing them is a feature.
|
| 249 |
+
"""
|
| 250 |
+
return _sem.metrics().get(key, {}).get('agg') in ('ratio', 'derived')
|
| 251 |
+
|
| 252 |
+
|
| 253 |
+
def _base_topic(key):
|
| 254 |
+
"""The topic a BASE metric aggregates on β read from the METRIC, never from ADMITTED.
|
| 255 |
+
|
| 256 |
+
A composite decomposes into bases that need not themselves be offerable (and whose topics may
|
| 257 |
+
differ from each other: `aov`'s numerator is on sales_lines and its denominator on
|
| 258 |
+
sales_orders). Resolving the topic from the allowlist would make that undecidable.
|
| 259 |
+
"""
|
| 260 |
+
m = _sem.metrics().get(key)
|
| 261 |
+
if not m:
|
| 262 |
+
raise _sem.ModelError(f"unknown metric {key!r}")
|
| 263 |
+
topic = m.get('topic')
|
| 264 |
+
if not topic:
|
| 265 |
+
raise _sem.ModelError(f"metric {key!r} declares no topic to aggregate on")
|
| 266 |
+
return topic
|
| 267 |
+
|
| 268 |
+
|
| 269 |
+
def _query_parts(key, window, today, team_id):
|
| 270 |
+
"""The FROM / GROUP-BY id / aggregate / WHERE for one (measure, window), shared by the
|
| 271 |
+
one-sided and the two-sided resolver so there is one definition of the scope."""
|
| 272 |
+
topic = _base_topic(key)
|
| 273 |
+
# NO DEFAULT WINDOW. A rule that carries no window is not "all time" β it is a rule whose
|
| 274 |
+
# window we do not know, and answering it over all time WIDENS the condition while the count
|
| 275 |
+
# beside it still looks authoritative. `all_time` is a legitimate ANSWER the user can pick;
|
| 276 |
+
# it is not a legitimate GUESS. (This line shipped as `rule.get('window') or {...}` and the
|
| 277 |
+
# gate caught it contradicting the error message two lines below.)
|
| 278 |
+
bounds = _win.resolve(window, today)
|
| 279 |
+
if bounds is None:
|
| 280 |
+
raise _sem.ModelError(
|
| 281 |
+
f"unresolvable window {window!r} β refusing rather than defaulting to all "
|
| 282 |
+
f"time, which would widen the condition while the count still looked authoritative")
|
| 283 |
+
date_from, date_to = bounds
|
| 284 |
+
|
| 285 |
+
frm, id_col, s = _entity_sql(topic)
|
| 286 |
+
m = _sem.metrics()[key]
|
| 287 |
+
# COALESCE, because SQL's sum() over an all-NULL group is NULL, and `NULL > x` and
|
| 288 |
+
# `NULL <= x` are BOTH false: the customer would vanish from the two halves of the same
|
| 289 |
+
# question, so `margin <= 0` and `margin > 0` would not cover the book. The zero-group below
|
| 290 |
+
# cannot rescue them either β they DO have rows in the window, just no values. Python's sum
|
| 291 |
+
# of the same rows is 0.0, and 0 is what the grid would show, so 0 is the answer.
|
| 292 |
+
agg = f"COALESCE({_sem._measure_sql(m, s['alias'])}, 0)"
|
| 293 |
+
|
| 294 |
+
where, params = [_sem.render_scope(s['scope_sql'].strip())], []
|
| 295 |
+
if date_from:
|
| 296 |
+
where.append(f"CAST({s['date_col']} AS TIMESTAMP) >= ?")
|
| 297 |
+
params.append(f"{date_from} 00:00:00")
|
| 298 |
+
if date_to:
|
| 299 |
+
where.append(f"CAST({s['date_col']} AS TIMESTAMP) <= ?")
|
| 300 |
+
params.append(f"{date_to} 23:59:59")
|
| 301 |
+
if team_id:
|
| 302 |
+
if not s.get('team_col'):
|
| 303 |
+
raise _sem.ModelError(f"topic {topic!r} is company-level β it has no business-unit "
|
| 304 |
+
f"filter, so a BU-scoped caller cannot be answered safely")
|
| 305 |
+
where.append(f"{s['team_col']} = ?")
|
| 306 |
+
params.append(int(team_id))
|
| 307 |
+
return frm, id_col, agg, ' AND '.join(where), params
|
| 308 |
+
|
| 309 |
+
|
| 310 |
+
def _scaled(key, value):
|
| 311 |
+
"""A TYPED value in the units the metric actually holds.
|
| 312 |
+
|
| 313 |
+
`Gross Margin % > 20` is typed in percentage points and the metric is a fraction, so the 20
|
| 314 |
+
becomes 0.20. Applied to a LITERAL only β a statistic and the other side of a
|
| 315 |
+
measure-vs-measure are already on the metric's scale, and dividing them too would apply the
|
| 316 |
+
factor twice for a result that still looks plausible.
|
| 317 |
+
"""
|
| 318 |
+
scale = ADMITTED.get(key, {}).get('input_scale')
|
| 319 |
+
return (value / scale) if scale else value
|
| 320 |
+
|
| 321 |
+
|
| 322 |
+
def resolve_stat(rule, today, team_id=None, allowed_pids=None):
|
| 323 |
+
"""`Where [Sales] [this year] [>] [the 75th percentile]` β the top quartile of the pool.
|
| 324 |
+
|
| 325 |
+
One totals query for the measure, the statistic taken over the CALLER'S POOL (see the STATS
|
| 326 |
+
note on why the population is the pool and what that costs), then a comparison per customer.
|
| 327 |
+
"""
|
| 328 |
+
stat = (rule.get('rhs') or {}).get('stat')
|
| 329 |
+
if stat not in STATS:
|
| 330 |
+
raise _sem.ModelError(f"{stat!r} is not a statistic. Offered: {sorted(STATS)}")
|
| 331 |
+
key = rule.get('colId')
|
| 332 |
+
topic_for(key)
|
| 333 |
+
op = rule.get('op')
|
| 334 |
+
if op not in PAIR_OPS:
|
| 335 |
+
raise _sem.ModelError(
|
| 336 |
+
f"{op!r} cannot compare a measure with a statistic ({sorted(PAIR_OPS)}). Equality "
|
| 337 |
+
f"against a computed cut-off matches nobody except by coincidence.")
|
| 338 |
+
pool = _pool_of(allowed_pids)
|
| 339 |
+
if pool is None:
|
| 340 |
+
raise _sem.ModelError(
|
| 341 |
+
"a statistic needs a population to be computed over, and none was supplied β "
|
| 342 |
+
"refusing rather than taking it over whichever customers happen to have rows, "
|
| 343 |
+
"which would be a different question with an authoritative-looking answer")
|
| 344 |
+
con = _sem._store_con()
|
| 345 |
+
try:
|
| 346 |
+
totals = _totals(con, key, rule.get('window'), today, team_id)
|
| 347 |
+
finally:
|
| 348 |
+
con.close()
|
| 349 |
+
# β THE POPULATION IS THE CUSTOMERS WHO HAVE THIS MEASURE AT ALL β owner's decision,
|
| 350 |
+
# 2026-07-27: *"top quartile of customers who actually has returned something"*.
|
| 351 |
+
#
|
| 352 |
+
# It was the whole pool including zeros, which is the convention everywhere else and was
|
| 353 |
+
# wrong HERE. The pool carries lapsed customers, so ~30% of it sits at zero for any 12-month
|
| 354 |
+
# window β that dragged the 25th percentile to exactly 0.0, made `< the bottom quartile`
|
| 355 |
+
# match NOBODY, and would have been worse for Return %, where most customers never return
|
| 356 |
+
# anything and "the top quartile" would have degenerated to "anyone with a single refund".
|
| 357 |
+
#
|
| 358 |
+
# NON-ZERO, not has-rows: a zero Return % means nothing was returned, which is precisely the
|
| 359 |
+
# customer the owner is excluding. It also reads as one sentence β the question is asked
|
| 360 |
+
# among customers who have that measure β and it generalises: "the median Sales" means the
|
| 361 |
+
# median among customers who sold something.
|
| 362 |
+
#
|
| 363 |
+
# The ANSWER is restricted to the same population, so `< the bottom quartile` returns a
|
| 364 |
+
# quarter of the returners rather than every customer who never returned.
|
| 365 |
+
population = {pid: v for pid, v in totals.items() if pid in pool and v}
|
| 366 |
+
cut = percentile(list(population.values()), stat)
|
| 367 |
+
if cut is None: # nobody has this measure -> nobody qualifies
|
| 368 |
+
return set()
|
| 369 |
+
return {pid for pid, v in population.items() if _CMP[op](v, cut)}
|
| 370 |
+
|
| 371 |
+
|
| 372 |
+
def _pool_of(allowed_pids):
|
| 373 |
+
"""The caller's visible pool as a set of plain ints, or None.
|
| 374 |
+
|
| 375 |
+
β The pool arrives from `pool()`/a cohort and may hold numpy ints or id strings; the store
|
| 376 |
+
returns BIGINTs. A set difference across two int types silently finds nothing in common,
|
| 377 |
+
which here would look like "no lapsed customers" rather than an error.
|
| 378 |
+
"""
|
| 379 |
+
return None if allowed_pids is None else {int(p) for p in allowed_pids if p is not None}
|
| 380 |
+
|
| 381 |
+
|
| 382 |
+
def rule_complete(rule):
|
| 383 |
+
"""Is this rule's right-hand side FULLY present β a question that has actually been asked?
|
| 384 |
+
|
| 385 |
+
THE GAP THIS CLOSES (2026-07-27, the cold-container race). The component's view state
|
| 386 |
+
passes through the host while the user β or the client's own initialisation β is mid-edit,
|
| 387 |
+
and a rule can transiently arrive with its window, statistic or value missing. `to_num(None)`
|
| 388 |
+
is 0 by the engines' shared contract, so resolution used to ANSWER that half-asked question
|
| 389 |
+
as `<op> 0`: the id landed in `measureSets`, the client read the rule as answered and
|
| 390 |
+
cleared its pending marker, and the grid stood behind a count for a question nobody asked.
|
| 391 |
+
Refusing instead leaves the id OUT of the payload, which the client renders as PENDING
|
| 392 |
+
("Calculatingβ¦") β the honest state for a question still being typed. Exactly the class the
|
| 393 |
+
module docstring already names: an unresolvable condition never becomes an answer.
|
| 394 |
+
|
| 395 |
+
INCOMPLETE is distinct from MALFORMED, on purpose. An ABSENT window / stat / value is a
|
| 396 |
+
normal transient (the caller skips it silently β mid-edit is not a telemetry event); a
|
| 397 |
+
PRESENT-but-bogus one is a real defect and still RAISES downstream (`unresolvable window`,
|
| 398 |
+
`not a statistic`, `not admitted`). `value` presence is textual, not numeric: "0" is a
|
| 399 |
+
typed answer, while parseability stays `to_num`'s business so the two engines keep one
|
| 400 |
+
contract for garbage input.
|
| 401 |
+
"""
|
| 402 |
+
if not isinstance(rule, dict) or not rule.get('op'):
|
| 403 |
+
return False
|
| 404 |
+
if rule.get('window') is None:
|
| 405 |
+
return False
|
| 406 |
+
rhs = rule.get('rhs')
|
| 407 |
+
if isinstance(rhs, dict) and rhs.get('kind') == 'measure':
|
| 408 |
+
return bool(rhs.get('colId')) and rhs.get('window') is not None
|
| 409 |
+
if isinstance(rhs, dict) and rhs.get('kind') == 'stat':
|
| 410 |
+
return bool(rhs.get('stat'))
|
| 411 |
+
v = rule.get('value')
|
| 412 |
+
return v is not None and str(v).strip() != ''
|
| 413 |
+
|
| 414 |
+
|
| 415 |
+
def resolve_rule(rule, today, team_id=None, allowed_pids=None):
|
| 416 |
+
"""One measure condition -> the SET OF CUSTOMER IDS satisfying it.
|
| 417 |
+
|
| 418 |
+
`rule` = {colId: <measure key>, window: <window spec>, op: <numeric op>, value: <str|num>},
|
| 419 |
+
or β CG-9 β the same with `rhs: {kind:'measure', colId, window}` instead of a value, which
|
| 420 |
+
hands off to `resolve_pair`.
|
| 421 |
+
|
| 422 |
+
β Customers with NO rows in the window are absent from the GROUP BY, so a condition like
|
| 423 |
+
`revenue < 100` would miss them even though 0 < 100 is true. That is the single most
|
| 424 |
+
dangerous thing about this query shape: the missing rows are exactly the lapsed customers a
|
| 425 |
+
"who has gone quiet" filter is FOR. So the caller passes `allowed_pids` (the pool) and the
|
| 426 |
+
zero-group is reconstructed explicitly below rather than left to the SQL.
|
| 427 |
+
"""
|
| 428 |
+
if not rule_complete(rule):
|
| 429 |
+
# An absent value would coerce to 0 and resolve as a DIFFERENT question under an
|
| 430 |
+
# authoritative-looking count β the same failure class as defaulting a missing window
|
| 431 |
+
# to all time, caught by the same principle: refuse, never guess.
|
| 432 |
+
raise _sem.ModelError(
|
| 433 |
+
'incomplete measure condition β its window, statistic or value is not present; '
|
| 434 |
+
'refusing to answer a question that has not been fully asked')
|
| 435 |
+
rhs = rule.get('rhs')
|
| 436 |
+
if isinstance(rhs, dict) and rhs.get('kind') == 'measure':
|
| 437 |
+
return resolve_pair(rule, today, team_id, allowed_pids)
|
| 438 |
+
if isinstance(rhs, dict) and rhs.get('kind') == 'stat':
|
| 439 |
+
return resolve_stat(rule, today, team_id, allowed_pids)
|
| 440 |
+
key = rule.get('colId')
|
| 441 |
+
topic_for(key) # raises for anything not admitted
|
| 442 |
+
op = rule.get('op')
|
| 443 |
+
if op not in OPS:
|
| 444 |
+
raise _sem.ModelError(f"{op!r} is not a numeric comparison ({sorted(OPS)})")
|
| 445 |
+
|
| 446 |
+
from harness import filter_sql as _fs
|
| 447 |
+
# β THE SCALE IS APPLIED HERE, TO THE TYPED VALUE, AND NOWHERE ELSE.
|
| 448 |
+
#
|
| 449 |
+
# Its position relative to `_zero_satisfies` turns out NOT to matter, and the negative
|
| 450 |
+
# control is what established that rather than reasoning: dividing by a POSITIVE constant
|
| 451 |
+
# cannot change how a number compares to zero, so the zero-group is identical either way.
|
| 452 |
+
# (An earlier version of this comment claimed the opposite. It was wrong.) What DOES matter
|
| 453 |
+
# is that a COMPUTED comparand never passes through here β see `_scaled`.
|
| 454 |
+
value = _scaled(key, _fs.to_num(rule.get('value')))
|
| 455 |
+
|
| 456 |
+
pool = _pool_of(allowed_pids)
|
| 457 |
+
if is_composite(key):
|
| 458 |
+
# A ratio has no HAVING to push the comparison into: compute the value per customer, then
|
| 459 |
+
# compare in Python β the same shape `resolve_pair` uses, and for the same reason.
|
| 460 |
+
con = _sem._store_con()
|
| 461 |
+
try:
|
| 462 |
+
totals = _totals(con, key, rule.get('window'), today, team_id)
|
| 463 |
+
finally:
|
| 464 |
+
con.close()
|
| 465 |
+
# No pool means we can only speak about customers a base has seen. WITH a pool, everyone
|
| 466 |
+
# else scores 0 through the ratio's guard β which is what makes the zero-group work for a
|
| 467 |
+
# ratio exactly as it does for a sum ("AOV <= 250" includes those who bought nothing).
|
| 468 |
+
universe = pool if pool is not None else set(totals)
|
| 469 |
+
return {pid for pid in universe if _CMP[op](totals.get(pid, 0.0), value)}
|
| 470 |
+
|
| 471 |
+
frm, id_col, agg, where_sql, params = _query_parts(
|
| 472 |
+
key, rule.get('window'), today, team_id)
|
| 473 |
+
sql = (f"SELECT {id_col} AS pid, {agg} AS v FROM {frm} WHERE {where_sql} "
|
| 474 |
+
f"GROUP BY {id_col} HAVING {agg} {OPS[op]} ?")
|
| 475 |
+
|
| 476 |
+
con = _sem._store_con()
|
| 477 |
+
try:
|
| 478 |
+
matched = {int(r[0]) for r in con.execute(sql, params + [value]).fetchall()
|
| 479 |
+
if r[0] is not None}
|
| 480 |
+
# The zero-group: every customer in scope with NO rows in the window aggregates to 0, and
|
| 481 |
+
# SQL cannot return a group that has no rows. Reconstruct it from the pool rather than
|
| 482 |
+
# pretending those customers do not exist.
|
| 483 |
+
if pool is not None:
|
| 484 |
+
present = {int(r[0]) for r in con.execute(
|
| 485 |
+
f"SELECT DISTINCT {id_col} FROM {frm} WHERE {where_sql}",
|
| 486 |
+
params).fetchall() if r[0] is not None}
|
| 487 |
+
absent = pool - present
|
| 488 |
+
if absent and _zero_satisfies(op, value):
|
| 489 |
+
matched |= absent
|
| 490 |
+
finally:
|
| 491 |
+
con.close()
|
| 492 |
+
if pool is not None:
|
| 493 |
+
matched &= pool
|
| 494 |
+
return matched
|
| 495 |
+
|
| 496 |
+
|
| 497 |
+
def _totals(con, key, window, today, team_id):
|
| 498 |
+
"""`{pid: value}` for one (measure, window), COMPOSITES included.
|
| 499 |
+
|
| 500 |
+
For a plain aggregate this is resolve_rule's query minus the HAVING. For a ratio or a derived
|
| 501 |
+
metric it is one query per BASE β each on that base's own topic β combined per customer by
|
| 502 |
+
`semantic._post_compute`, the same function `store_query` uses for the un-grouped number. So
|
| 503 |
+
AOV per customer is the app's AOV, computed once and defined once.
|
| 504 |
+
|
| 505 |
+
β The pid keys come from DIFFERENT dims for a cross-topic ratio (`l.order_partner_id` and
|
| 506 |
+
`o.partner_id`). The union is taken rather than the numerator's keys, so a customer either
|
| 507 |
+
side has seen gets a number; the ratio's own `den == 0 -> 0.0` guard covers the rest.
|
| 508 |
+
"""
|
| 509 |
+
if is_composite(key):
|
| 510 |
+
bases, _ = _sem._expand_measures([key])
|
| 511 |
+
parts = {b: _totals(con, b, window, today, team_id) for b in bases}
|
| 512 |
+
pids = set()
|
| 513 |
+
for got in parts.values():
|
| 514 |
+
pids |= set(got)
|
| 515 |
+
return {pid: _sem._post_compute({b: parts[b].get(pid, 0.0) for b in bases}, key)
|
| 516 |
+
for pid in pids}
|
| 517 |
+
frm, id_col, agg, where_sql, params = _query_parts(key, window, today, team_id)
|
| 518 |
+
sql = (f"SELECT {id_col} AS pid, {agg} AS v FROM {frm} WHERE {where_sql} GROUP BY {id_col}")
|
| 519 |
+
# `float(r[1])`, not `float(r[1] or 0.0)`: `_query_parts` already wraps the aggregate in
|
| 520 |
+
# COALESCE(..., 0), so a NULL cannot arrive here β a second guard would be unreachable, and
|
| 521 |
+
# the negative control proved it (deleting it changed nothing). If one ever DID arrive, a
|
| 522 |
+
# TypeError is the right outcome: it would mean the SQL-level guard had been removed, which
|
| 523 |
+
# is the bug worth hearing about rather than papering over.
|
| 524 |
+
return {int(r[0]): float(r[1])
|
| 525 |
+
for r in con.execute(sql, params).fetchall() if r[0] is not None}
|
| 526 |
+
|
| 527 |
+
|
| 528 |
+
def resolve_pair(rule, today, team_id=None, allowed_pids=None):
|
| 529 |
+
"""CG-9 β `Where [Sales] [this year] [>] [Sales] [year to date, last year]`.
|
| 530 |
+
|
| 531 |
+
TWO QUERIES, compared in Python. The obvious alternative β one query with two conditional
|
| 532 |
+
aggregates and a HAVING comparing them β would have to reach inside `_sem._measure_sql` to
|
| 533 |
+
inject a per-side window into an expression that is already a `sum(CASE WHEN β¦ )` for
|
| 534 |
+
filtered measures. Two queries keep ONE definition of each number (the metric's own
|
| 535 |
+
aggregate, unmodified), let the two sides sit on DIFFERENT TOPICS, and make the zero-group
|
| 536 |
+
fall out for free: absent from a side's result means 0 on that side.
|
| 537 |
+
|
| 538 |
+
β The zero-group is LOUDER here than it is for a one-sided rule. Every customer with no rows
|
| 539 |
+
in EITHER window compares 0 against 0, so `<=` and `>=` match all of them β correct, and it
|
| 540 |
+
will look wrong on a narrow window. That is why `eq`/`neq` are not offered at all
|
| 541 |
+
(PAIR_OPS): two float sums are never exactly equal except at zero, where they always are.
|
| 542 |
+
"""
|
| 543 |
+
rhs = rule.get('rhs') or {}
|
| 544 |
+
lkey, rkey = rule.get('colId'), rhs.get('colId')
|
| 545 |
+
topic_for(lkey), topic_for(rkey) # raises for anything not admitted
|
| 546 |
+
op = rule.get('op')
|
| 547 |
+
if op not in PAIR_OPS:
|
| 548 |
+
raise _sem.ModelError(
|
| 549 |
+
f"{op!r} cannot compare two measures ({sorted(PAIR_OPS)}). Equality between two sums "
|
| 550 |
+
f"of floats matches nobody and inequality matches everybody; neither answers the "
|
| 551 |
+
f"question, and both look like they work.")
|
| 552 |
+
cmp = _CMP[op]
|
| 553 |
+
|
| 554 |
+
pool = _pool_of(allowed_pids)
|
| 555 |
+
con = _sem._store_con()
|
| 556 |
+
try:
|
| 557 |
+
left = _totals(con, lkey, rule.get('window'), today, team_id)
|
| 558 |
+
right = _totals(con, rkey, rhs.get('window'), today, team_id)
|
| 559 |
+
finally:
|
| 560 |
+
con.close()
|
| 561 |
+
# With no pool we can only speak about customers at least one side has seen β the zero-vs-
|
| 562 |
+
# zero group is unknowable without being told who exists. With a pool, it is every member.
|
| 563 |
+
universe = pool if pool is not None else (set(left) | set(right))
|
| 564 |
+
return {pid for pid in universe if cmp(left.get(pid, 0.0), right.get(pid, 0.0))}
|
| 565 |
+
|
| 566 |
+
|
| 567 |
+
def _zero_satisfies(op, value):
|
| 568 |
+
"""Does a customer with NO activity in the window (measure = 0) satisfy this condition?"""
|
| 569 |
+
return _CMP[op](0.0, value)
|
| 570 |
+
|
| 571 |
+
|
| 572 |
+
def resolve_values(key, window, today, team_id=None, allowed_pids=None):
|
| 573 |
+
"""`{pid: display value}` for one (measure, window) over the caller's pool β the COLUMN
|
| 574 |
+
version of `resolve_rule` (owner item 7, 2026-07-27: a formula-measure FIELD).
|
| 575 |
+
|
| 576 |
+
Same `_totals` the filter conditions and statistics use, so the column and the condition
|
| 577 |
+
cannot disagree. Three display decisions live here, not in the client:
|
| 578 |
+
|
| 579 |
+
* ZERO-FILLED over the pool. A customer with no rows in the window has a measure of 0 β
|
| 580 |
+
the zero-group convention everywhere else β and 0 is a real value the cell shows. Blank
|
| 581 |
+
is reserved for "could not compute" (the caller omits the column entirely on failure).
|
| 582 |
+
* `input_scale` applied OUTWARD. `margin_pct` is a fraction (0-1) internally; the user
|
| 583 |
+
types percentage points into the filter (input divides by 100), so the CELL shows points
|
| 584 |
+
too (value Γ 100) β one boundary, both directions, and `yoy_pct` beside it already reads
|
| 585 |
+
in points.
|
| 586 |
+
* Rounded like every other numeric column: whole numbers for money/counts (the pool's own
|
| 587 |
+
`_round` convention), one decimal for a percent.
|
| 588 |
+
|
| 589 |
+
Raises for anything not admitted / an unresolvable window β the caller degrades that column
|
| 590 |
+
to blank rather than to a number nobody computed.
|
| 591 |
+
"""
|
| 592 |
+
topic_for(key) # raises for anything not admitted
|
| 593 |
+
# The window must RESOLVE before any query runs β same refusal as `_query_parts`, checked
|
| 594 |
+
# here too because a composite's `_totals` path resolves per base and would otherwise raise
|
| 595 |
+
# deeper with a less useful message.
|
| 596 |
+
if _win.resolve(window, today) is None:
|
| 597 |
+
raise _sem.ModelError(
|
| 598 |
+
f"unresolvable window {window!r} for the {key!r} column β refusing rather than "
|
| 599 |
+
f"defaulting to all time")
|
| 600 |
+
pool = _pool_of(allowed_pids)
|
| 601 |
+
con = _sem._store_con()
|
| 602 |
+
try:
|
| 603 |
+
totals = _totals(con, key, window, today, team_id)
|
| 604 |
+
finally:
|
| 605 |
+
con.close()
|
| 606 |
+
scale = ADMITTED.get(key, {}).get('input_scale') or 1.0
|
| 607 |
+
mtype = _sem._FORMAT_TYPE.get(_sem.metrics().get(key, {}).get('format'), 'currency')
|
| 608 |
+
nd = 1 if mtype == 'pct' else None # money/counts whole, percents one decimal
|
| 609 |
+
universe = pool if pool is not None else set(totals)
|
| 610 |
+
# round(x, None) returns an int β exactly the pool's `_round` convention for money/counts.
|
| 611 |
+
return {int(p): round(float(totals.get(p, 0.0)) * scale, nd) for p in universe}
|
| 612 |
+
|
| 613 |
+
|
| 614 |
+
#: The series channel's calendar buckets. `date_trunc` names in DuckDB; 'week' truncates to
|
| 615 |
+
#: Monday, matching the date-window vocabulary's week convention.
|
| 616 |
+
SERIES_BUCKETS = frozenset({'week', 'month', 'quarter', 'year'})
|
| 617 |
+
|
| 618 |
+
|
| 619 |
+
def resolve_series(key, bucket, date_from, date_to, today, team_id=None, allowed_pids=None):
|
| 620 |
+
"""POOLED value of one measure per calendar bucket β the time-series channel (C-TS,
|
| 621 |
+
wave 2026-08-02). Returns `{'values': {bucket_start_iso: value}, 'agg': kind}`.
|
| 622 |
+
|
| 623 |
+
ONE grouped query per measure: the aggregate expression is `semantic._measure_sql`'s own
|
| 624 |
+
(no second definition of a number), evaluated over the CALLER'S POOL per `date_trunc`
|
| 625 |
+
bucket. That is the pooled aggregate β a non-additive measure (an average) is computed at
|
| 626 |
+
pool grain, never averaged over per-customer answers (the additivity law).
|
| 627 |
+
|
| 628 |
+
Composites (ratio/derived) are REFUSED: they have no single aggregate expression, and
|
| 629 |
+
combining their bases per bucket is a correctness project, not a default β the caller
|
| 630 |
+
degrades that field honestly rather than showing a number nobody computed.
|
| 631 |
+
|
| 632 |
+
Scale and rounding mirror `resolve_values` exactly, so a bucket cell and the grid's own
|
| 633 |
+
measure column can never disagree on units. Buckets with no rows are ABSENT from the
|
| 634 |
+
answer β zero-filling is the CALLER'S decision because it depends on the agg kind: an
|
| 635 |
+
empty month of a SUM is a real 0, an empty month of an AVERAGE is unanswerable.
|
| 636 |
+
"""
|
| 637 |
+
if bucket not in SERIES_BUCKETS:
|
| 638 |
+
raise _sem.ModelError(
|
| 639 |
+
f"unknown series bucket {bucket!r} - offered: week, month, quarter, year")
|
| 640 |
+
topic_for(key) # raises for anything not admitted
|
| 641 |
+
if is_composite(key):
|
| 642 |
+
raise _sem.ModelError(
|
| 643 |
+
f"{key!r} is a ratio/derived measure - the series channel computes "
|
| 644 |
+
f"single-aggregate measures only")
|
| 645 |
+
window = {'kind': 'custom', 'from': str(date_from), 'to': str(date_to)}
|
| 646 |
+
frm, id_col, agg, where, params = _query_parts(key, window, today, team_id)
|
| 647 |
+
_, _, s = _entity_sql(_base_topic(key))
|
| 648 |
+
meta = _sem.metrics().get(key, {})
|
| 649 |
+
agg_kind = str(meta.get('agg') or 'sum')
|
| 650 |
+
pool = _pool_of(allowed_pids)
|
| 651 |
+
sql_params = list(params)
|
| 652 |
+
pool_sql = ''
|
| 653 |
+
if pool is not None:
|
| 654 |
+
if not pool:
|
| 655 |
+
return {'values': {}, 'agg': agg_kind}
|
| 656 |
+
marks = ','.join('?' for _ in pool)
|
| 657 |
+
pool_sql = f" AND {id_col} IN ({marks})"
|
| 658 |
+
sql_params.extend(int(p) for p in sorted(pool))
|
| 659 |
+
sql = (f"SELECT CAST(date_trunc('{bucket}', CAST({s['date_col']} AS TIMESTAMP)) AS DATE)"
|
| 660 |
+
f" AS b, {agg} AS v FROM {frm} WHERE {where}{pool_sql} GROUP BY b")
|
| 661 |
+
con = _sem._store_con()
|
| 662 |
+
try:
|
| 663 |
+
rows = con.execute(sql, sql_params).fetchall()
|
| 664 |
+
finally:
|
| 665 |
+
con.close()
|
| 666 |
+
scale = ADMITTED.get(key, {}).get('input_scale') or 1.0
|
| 667 |
+
mtype = _sem._FORMAT_TYPE.get(meta.get('format'), 'currency')
|
| 668 |
+
nd = 1 if mtype == 'pct' else None
|
| 669 |
+
values = {str(b)[:10]: round(float(v) * scale, nd) for b, v in rows if b is not None}
|
| 670 |
+
return {'values': values, 'agg': agg_kind}
|
| 671 |
+
|
| 672 |
+
|
| 673 |
+
def resolve_series_slid(key, window_spec, buckets, today, team_id=None, allowed_pids=None):
|
| 674 |
+
"""AS-OF series (C-TSWIN, wave 14 β [[loopable-wave14-split]] R1): the measure's OWN
|
| 675 |
+
window re-resolved per bucket with `today := min(bucket end, real today)` β so `ytd`
|
| 676 |
+
becomes cumulative from Jan 1, `last_90_days` becomes trailing-90, `all_time` cumulative
|
| 677 |
+
ever. Returns `{'values': {bucket_start_iso: value}, 'agg': kind}`.
|
| 678 |
+
|
| 679 |
+
`buckets` is `[(start_iso, end_iso), β¦]`. One aggregate query per bucket rather than one
|
| 680 |
+
grouped query β deliberately: every part (scope, aggregate expression, BU wall, scale,
|
| 681 |
+
rounding) is `_query_parts`' own, so **bucket N is exactly what the grid's measure column
|
| 682 |
+
would show if the tenant's today were bucket N's end** β the two surfaces cannot disagree
|
| 683 |
+
by construction, which is the property the pooled/date_trunc shape could not give a
|
| 684 |
+
sliding window.
|
| 685 |
+
|
| 686 |
+
Absence semantics (the caller's fill law, same as `resolve_series`): a bucket whose
|
| 687 |
+
window holds NO ROWS is ABSENT from the answer (`COALESCE` would have manufactured a 0
|
| 688 |
+
for an empty AVERAGE, so the query carries a COUNT(*) to tell the two apart); a bucket
|
| 689 |
+
whose START is after `today` is absent too β an unstarted period's YTD is not 0, it is
|
| 690 |
+
unanswered. `custom` (fixed-range) windows cannot slide and are REFUSED β the caller
|
| 691 |
+
drops them as `window_fixed` before ever calling here.
|
| 692 |
+
"""
|
| 693 |
+
topic_for(key) # raises for anything not admitted
|
| 694 |
+
if is_composite(key):
|
| 695 |
+
raise _sem.ModelError(
|
| 696 |
+
f"{key!r} is a ratio/derived measure - the series channel computes "
|
| 697 |
+
f"single-aggregate measures only")
|
| 698 |
+
spec = _win.normalize(window_spec)
|
| 699 |
+
if spec is None or spec.get('kind') == 'custom':
|
| 700 |
+
raise _sem.ModelError(
|
| 701 |
+
f"{key!r}: a fixed-range window cannot slide across buckets (window_fixed)")
|
| 702 |
+
meta = _sem.metrics().get(key, {})
|
| 703 |
+
agg_kind = str(meta.get('agg') or 'sum')
|
| 704 |
+
pool = _pool_of(allowed_pids)
|
| 705 |
+
if pool is not None and not pool:
|
| 706 |
+
return {'values': {}, 'agg': agg_kind}
|
| 707 |
+
scale = ADMITTED.get(key, {}).get('input_scale') or 1.0
|
| 708 |
+
mtype = _sem._FORMAT_TYPE.get(meta.get('format'), 'currency')
|
| 709 |
+
nd = 1 if mtype == 'pct' else None
|
| 710 |
+
today_s = str(today)[:10]
|
| 711 |
+
values = {}
|
| 712 |
+
con = _sem._store_con()
|
| 713 |
+
try:
|
| 714 |
+
for start, end in buckets:
|
| 715 |
+
start_s, end_s = str(start)[:10], str(end)[:10]
|
| 716 |
+
if start_s > today_s:
|
| 717 |
+
continue # unstarted period -> unanswered, never 0
|
| 718 |
+
eff = min(end_s, today_s)
|
| 719 |
+
frm, id_col, agg, where, params = _query_parts(key, spec, eff, team_id)
|
| 720 |
+
sql_params = list(params)
|
| 721 |
+
pool_sql = ''
|
| 722 |
+
if pool is not None:
|
| 723 |
+
marks = ','.join('?' for _ in pool)
|
| 724 |
+
pool_sql = f" AND {id_col} IN ({marks})"
|
| 725 |
+
sql_params.extend(int(p) for p in sorted(pool))
|
| 726 |
+
sql = f"SELECT {agg} AS v, COUNT(*) AS n FROM {frm} WHERE {where}{pool_sql}"
|
| 727 |
+
v, nrows = con.execute(sql, sql_params).fetchone()
|
| 728 |
+
if not nrows:
|
| 729 |
+
continue # empty window -> the caller's per-agg fill law
|
| 730 |
+
values[start_s] = round(float(v) * scale, nd)
|
| 731 |
+
finally:
|
| 732 |
+
con.close()
|
| 733 |
+
return {'values': values, 'agg': agg_kind}
|
| 734 |
+
|
| 735 |
+
|
| 736 |
+
def resolve_tree(rules, today, team_id=None, allowed_pids=None):
|
| 737 |
+
"""{rule_id: sorted pids} for every measure condition in a tree.
|
| 738 |
+
|
| 739 |
+
Keyed by the rule's client-generated `id` so the browser can map an answer back to the
|
| 740 |
+
condition that asked it β positional matching would silently re-associate every answer the
|
| 741 |
+
moment a user deleted a condition.
|
| 742 |
+
"""
|
| 743 |
+
out = {}
|
| 744 |
+
for r in rules or []:
|
| 745 |
+
rid = str(r.get('id') or '')
|
| 746 |
+
if not rid:
|
| 747 |
+
raise _sem.ModelError('a measure condition needs a stable id to carry its answer back')
|
| 748 |
+
out[rid] = sorted(resolve_rule(r, today, team_id, allowed_pids))
|
| 749 |
+
return out
|
| 750 |
+
|
| 751 |
+
|
| 752 |
+
def signature(rule):
|
| 753 |
+
"""The identity of the QUESTION a rule asks, ignoring which condition row asked it.
|
| 754 |
+
|
| 755 |
+
Two conditions with the same measure, window, operator and value have the same answer, so
|
| 756 |
+
they cost one query between them β and a saved view re-rendered unchanged costs none, since
|
| 757 |
+
the caller can memoise on this. `id` is deliberately absent: it names the asker, not the
|
| 758 |
+
question.
|
| 759 |
+
|
| 760 |
+
β The RHS is part of the question. Without it, `Sales[ytd] > Sales[last year]` and
|
| 761 |
+
`Sales[ytd] > Sales[ytd_last_year]` would share one memo entry β same measure, same op, same
|
| 762 |
+
(empty) value β and the second would be served the first one's answer. Silent, and wrong in
|
| 763 |
+
exactly the comparison this was built for.
|
| 764 |
+
"""
|
| 765 |
+
w = _win.normalize(rule.get('window'))
|
| 766 |
+
rhs = rule.get('rhs')
|
| 767 |
+
rsig = None
|
| 768 |
+
if isinstance(rhs, dict):
|
| 769 |
+
rw = _win.normalize(rhs.get('window'))
|
| 770 |
+
# `stat` rides the same tuple: `> the median` and `> the 75th percentile` are different
|
| 771 |
+
# questions with the same measure, window, op and (empty) value.
|
| 772 |
+
rsig = (rhs.get('kind'), rhs.get('colId'), rhs.get('stat'),
|
| 773 |
+
None if rw is None else tuple(sorted(rw.items())))
|
| 774 |
+
return (rule.get('colId'), rule.get('op'), str(rule.get('value') or ''),
|
| 775 |
+
None if w is None else tuple(sorted(w.items())), rsig)
|
| 776 |
+
|
| 777 |
+
|
| 778 |
+
def collect(nodes):
|
| 779 |
+
"""Every MEASURE leaf in a filter tree, flattened (groups contribute their children)."""
|
| 780 |
+
found = []
|
| 781 |
+
for n in nodes or []:
|
| 782 |
+
if not isinstance(n, dict):
|
| 783 |
+
continue
|
| 784 |
+
if isinstance(n.get('children'), list):
|
| 785 |
+
found.extend(collect(n['children']))
|
| 786 |
+
elif n.get('colId') in ADMITTED:
|
| 787 |
+
found.append(n)
|
| 788 |
+
return found
|
platform/harness/pg/schema.sql
ADDED
|
@@ -0,0 +1,180 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
-- harness/pg/schema.sql β the control DB + per-tenant schema (X4 / EXIT-2b, 2026-07-30).
|
| 2 |
+
--
|
| 3 |
+
-- WHY POSTGRES AT ALL (C1c, and it is not a preference). The HF Dataset store is a FILE HOST:
|
| 4 |
+
-- no transactions, no PITR, no row-level anything, and `store.get()` is cache-first β so a write
|
| 5 |
+
-- from another process is invisible to a running app until restart. That was observed LIVE in
|
| 6 |
+
-- wave 11, and it is the whole reason the store leaves the product. Application/control data
|
| 7 |
+
-- (users, sessions, tenants, audit, saved views, overlay fields, cohorts, folders, prefs,
|
| 8 |
+
-- billing mirror) is transactional and needs joins; the ANALYTICAL mirror stays in DuckDB,
|
| 9 |
+
-- one file per tenant, because file-per-tenant IS the isolation model there.
|
| 10 |
+
--
|
| 11 |
+
-- β NOT YET RUN AGAINST A REAL SERVER. Owner blocker **B-3** (choose + provision managed
|
| 12 |
+
-- Postgres, US-region, co-located with Ashburn). This file and `core/store_pg.py` are the
|
| 13 |
+
-- sequenced-around work; the integration test is written but PENDING and is NOT reported as
|
| 14 |
+
-- passing. Nothing here is claimed to be verified beyond "it parses and the shapes match the
|
| 15 |
+
-- interface `core/store.py` already exposes".
|
| 16 |
+
--
|
| 17 |
+
-- β CO-LOCATE (C1b). An Ashburn app with a European database adds ~90ms to every query and undoes
|
| 18 |
+
-- the reason the market pivot happened.
|
| 19 |
+
--
|
| 20 |
+
-- psql "$DATABASE_URL" -v ON_ERROR_STOP=1 -f harness/pg/schema.sql
|
| 21 |
+
-- psql "$DATABASE_URL" -v ON_ERROR_STOP=1 -v slug=royal_imports -f harness/pg/schema.sql
|
| 22 |
+
-- (the second form additionally provisions one tenant schema β see the bottom block)
|
| 23 |
+
|
| 24 |
+
BEGIN;
|
| 25 |
+
|
| 26 |
+
-- βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 27 |
+
-- CONTROL: what the platform knows about itself. One row per tenant, one per account.
|
| 28 |
+
-- βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 29 |
+
CREATE SCHEMA IF NOT EXISTS control;
|
| 30 |
+
|
| 31 |
+
CREATE TABLE IF NOT EXISTS control.tenants (
|
| 32 |
+
-- The SLUG is the identity everywhere: the session cookie's `t` claim, the store namespace,
|
| 33 |
+
-- the DuckDB filename and the schema name are all derived from it. Constrained to what is
|
| 34 |
+
-- safe in all four positions, which is stricter than Postgres alone would require.
|
| 35 |
+
slug text PRIMARY KEY CHECK (slug ~ '^[a-z0-9][a-z0-9-]{0,58}[a-z0-9]$'),
|
| 36 |
+
name text NOT NULL,
|
| 37 |
+
-- `harness/tenants.py`'s literal, as data. Its own docstring says this becomes "a tenants
|
| 38 |
+
-- store (DB / HF dataset)"; this is that store. jsonb rather than columns because the shape
|
| 39 |
+
-- is per-connector and adding a source must not be a migration.
|
| 40 |
+
config jsonb NOT NULL DEFAULT '{}'::jsonb,
|
| 41 |
+
-- Connector CREDENTIALS never live here in plaintext. The Odoo key is the customer's,
|
| 42 |
+
-- stored encrypted at rest by the secrets layer (C2: SOPS+age per tenant) and shown once.
|
| 43 |
+
-- This column holds only a REFERENCE to where the secret lives.
|
| 44 |
+
secret_ref text,
|
| 45 |
+
active boolean NOT NULL DEFAULT true,
|
| 46 |
+
created_at timestamptz NOT NULL DEFAULT now(),
|
| 47 |
+
updated_at timestamptz NOT NULL DEFAULT now()
|
| 48 |
+
);
|
| 49 |
+
|
| 50 |
+
CREATE TABLE IF NOT EXISTS control.users (
|
| 51 |
+
-- Mirrors `core/users.py`'s record exactly, so the HFβPG cutover (C-4) is a copy, not a
|
| 52 |
+
-- redesign: salt + PBKDF2-HMAC-SHA256 hash (200k), never a plaintext password.
|
| 53 |
+
tenant_slug text NOT NULL REFERENCES control.tenants(slug) ON DELETE CASCADE,
|
| 54 |
+
username text NOT NULL CHECK (username = lower(username)),
|
| 55 |
+
salt text NOT NULL,
|
| 56 |
+
hash text NOT NULL,
|
| 57 |
+
name text NOT NULL DEFAULT '',
|
| 58 |
+
role text NOT NULL DEFAULT 'user',
|
| 59 |
+
-- 'all' or a JSON array of Odoo team ids ([5]=Fisch, [6]=Royal) β the BU isolation input.
|
| 60 |
+
bus jsonb NOT NULL DEFAULT '"all"'::jsonb,
|
| 61 |
+
-- 'all' or a JSON array of registry keys β the module grant `may_open` reads.
|
| 62 |
+
modules jsonb NOT NULL DEFAULT '"all"'::jsonb,
|
| 63 |
+
agent text, -- own-book scope (res.partner.agent_ids name)
|
| 64 |
+
email text,
|
| 65 |
+
active boolean NOT NULL DEFAULT true,
|
| 66 |
+
-- X3's stateless-session revocation handle. A signed cookie cannot be deleted server-side,
|
| 67 |
+
-- so it carries the epoch it was minted under and every verification re-reads THIS number;
|
| 68 |
+
-- a password change or a deactivation bumps it and every outstanding cookie dies at once.
|
| 69 |
+
epoch integer NOT NULL DEFAULT 0,
|
| 70 |
+
created_at timestamptz NOT NULL DEFAULT now(),
|
| 71 |
+
updated_at timestamptz NOT NULL DEFAULT now(),
|
| 72 |
+
PRIMARY KEY (tenant_slug, username)
|
| 73 |
+
);
|
| 74 |
+
|
| 75 |
+
-- D2's SESSIONS MIRROR β an AUDIT view, deliberately NOT the source of truth (X3 keeps sessions
|
| 76 |
+
-- stateless so the API process holds no per-tenant state). It answers "who is signed in, from
|
| 77 |
+
-- where, since when", and it is what makes per-DEVICE revocation possible later. Arrives with
|
| 78 |
+
-- C-2; the table is defined now so the cutover does not need a second migration.
|
| 79 |
+
CREATE TABLE IF NOT EXISTS control.sessions (
|
| 80 |
+
id bigserial PRIMARY KEY,
|
| 81 |
+
tenant_slug text NOT NULL,
|
| 82 |
+
username text NOT NULL,
|
| 83 |
+
epoch integer NOT NULL,
|
| 84 |
+
issued_at timestamptz NOT NULL DEFAULT now(),
|
| 85 |
+
last_seen_at timestamptz NOT NULL DEFAULT now(),
|
| 86 |
+
absolute_expiry timestamptz NOT NULL,
|
| 87 |
+
-- Truncated/derived client facts only. A full UA string plus an exact IP is more personal
|
| 88 |
+
-- data than an audit trail needs, and the DPA is easier to keep when the row is smaller.
|
| 89 |
+
ip_prefix text,
|
| 90 |
+
user_agent text,
|
| 91 |
+
revoked_at timestamptz,
|
| 92 |
+
FOREIGN KEY (tenant_slug, username)
|
| 93 |
+
REFERENCES control.users(tenant_slug, username) ON DELETE CASCADE
|
| 94 |
+
);
|
| 95 |
+
CREATE INDEX IF NOT EXISTS sessions_live_idx
|
| 96 |
+
ON control.sessions (tenant_slug, username) WHERE revoked_at IS NULL;
|
| 97 |
+
|
| 98 |
+
CREATE TABLE IF NOT EXISTS control.audit (
|
| 99 |
+
id bigserial PRIMARY KEY,
|
| 100 |
+
at timestamptz NOT NULL DEFAULT now(),
|
| 101 |
+
tenant_slug text,
|
| 102 |
+
username text,
|
| 103 |
+
action text NOT NULL, -- 'login' | 'store.put' | 'user.deactivate' | β¦
|
| 104 |
+
target text,
|
| 105 |
+
detail jsonb NOT NULL DEFAULT '{}'::jsonb
|
| 106 |
+
);
|
| 107 |
+
CREATE INDEX IF NOT EXISTS audit_at_idx ON control.audit (at DESC);
|
| 108 |
+
CREATE INDEX IF NOT EXISTS audit_tenant_idx ON control.audit (tenant_slug, at DESC);
|
| 109 |
+
|
| 110 |
+
-- βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 111 |
+
-- PER-TENANT: one schema `t_<slug>` per tenant, holding what the HF store holds today.
|
| 112 |
+
--
|
| 113 |
+
-- A SCHEMA PER TENANT, not a tenant_id column with RLS. Both are defensible; this one is chosen
|
| 114 |
+
-- because the isolation is then structural β a query that forgets its tenant predicate cannot
|
| 115 |
+
-- silently return another tenant's rows, it names a table that is not in the search_path. RLS
|
| 116 |
+
-- makes the same guarantee only while every policy is correct on every table, forever, and
|
| 117 |
+
-- [[aios-permissioning]] already records that a forgotten predicate is the failure mode we keep
|
| 118 |
+
-- writing gates against. The cost is DDL per tenant, which is one function call below.
|
| 119 |
+
-- βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 120 |
+
CREATE OR REPLACE FUNCTION control.provision_tenant_schema(p_slug text)
|
| 121 |
+
RETURNS text
|
| 122 |
+
LANGUAGE plpgsql
|
| 123 |
+
AS $$
|
| 124 |
+
DECLARE
|
| 125 |
+
-- β `-` IS NOT LEGAL UNQUOTED, and the slug carries them (`royal-imports`). Normalising to
|
| 126 |
+
-- `_` here rather than quoting everywhere keeps every generated identifier plain, and
|
| 127 |
+
-- `format('%I')` is what makes the interpolation injection-safe regardless.
|
| 128 |
+
v_schema text := 't_' || replace(lower(p_slug), '-', '_');
|
| 129 |
+
BEGIN
|
| 130 |
+
IF p_slug IS NULL OR p_slug !~ '^[a-z0-9][a-z0-9-]{0,58}[a-z0-9]$' THEN
|
| 131 |
+
RAISE EXCEPTION 'refusing to provision an unsafe tenant slug: %', p_slug;
|
| 132 |
+
END IF;
|
| 133 |
+
|
| 134 |
+
EXECUTE format('CREATE SCHEMA IF NOT EXISTS %I', v_schema);
|
| 135 |
+
|
| 136 |
+
-- store_kv β the JSON-per-key store `core/store.py` exposes, one row per key. `rev` is what
|
| 137 |
+
-- the file host never had: an optimistic-concurrency counter, so a read-modify-write can
|
| 138 |
+
-- detect that somebody else wrote between its read and its write instead of overwriting
|
| 139 |
+
-- them. That is the concrete thing "no transactions" cost us.
|
| 140 |
+
EXECUTE format($f$
|
| 141 |
+
CREATE TABLE IF NOT EXISTS %I.store_kv (
|
| 142 |
+
key text PRIMARY KEY,
|
| 143 |
+
value jsonb NOT NULL,
|
| 144 |
+
updated_at timestamptz NOT NULL DEFAULT now(),
|
| 145 |
+
rev bigint NOT NULL DEFAULT 1
|
| 146 |
+
)$f$, v_schema);
|
| 147 |
+
|
| 148 |
+
-- store_blobs β `upload_bytes`/`download_bytes`/`delete_path`. Customer documents live here
|
| 149 |
+
-- rather than base64'd into a JSON value, for the reason `core.store.upload_bytes` gives:
|
| 150 |
+
-- the workspace blob is READ ON EVERY RENDER, so a few MB of base64 inside it would be paid
|
| 151 |
+
-- on every keystroke by every user of that table.
|
| 152 |
+
EXECUTE format($f$
|
| 153 |
+
CREATE TABLE IF NOT EXISTS %I.store_blobs (
|
| 154 |
+
path text PRIMARY KEY,
|
| 155 |
+
bytes bytea NOT NULL,
|
| 156 |
+
message text,
|
| 157 |
+
updated_at timestamptz NOT NULL DEFAULT now()
|
| 158 |
+
)$f$, v_schema);
|
| 159 |
+
|
| 160 |
+
EXECUTE format('CREATE INDEX IF NOT EXISTS %I ON %I.store_kv (updated_at DESC)',
|
| 161 |
+
v_schema || '_kv_updated_idx', v_schema);
|
| 162 |
+
RETURN v_schema;
|
| 163 |
+
END;
|
| 164 |
+
$$;
|
| 165 |
+
|
| 166 |
+
-- Tenant #0, always. Its slug is the one the session cookie already carries.
|
| 167 |
+
INSERT INTO control.tenants (slug, name)
|
| 168 |
+
VALUES ('royal-imports', 'Royal Imports')
|
| 169 |
+
ON CONFLICT (slug) DO NOTHING;
|
| 170 |
+
SELECT control.provision_tenant_schema('royal-imports');
|
| 171 |
+
|
| 172 |
+
-- Optional: `psql -v slug=<a-slug>` provisions one more tenant schema in the same transaction.
|
| 173 |
+
-- `:'slug'` is unset in the plain invocation, so this block is skipped there.
|
| 174 |
+
\if :{?slug}
|
| 175 |
+
INSERT INTO control.tenants (slug, name) VALUES (:'slug', :'slug')
|
| 176 |
+
ON CONFLICT (slug) DO NOTHING;
|
| 177 |
+
SELECT control.provision_tenant_schema(:'slug');
|
| 178 |
+
\endif
|
| 179 |
+
|
| 180 |
+
COMMIT;
|
platform/harness/routines.py
ADDED
|
@@ -0,0 +1,259 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""harness/routines.py β OM-5 v0: scheduled reports + KPI alerts over the semantic layer.
|
| 2 |
+
|
| 3 |
+
Two of the four productization user jobs (Scheduled Reporting, KPI monitoring). Definitions live
|
| 4 |
+
in data/store/routines.json (tenant data, git-ignored); evaluation happens ON READ against the
|
| 5 |
+
tenant store through the same governed queries as everything else β an alert is a metric key +
|
| 6 |
+
a mode, never a hand-rolled query.
|
| 7 |
+
|
| 8 |
+
v0 boundaries (deliberate):
|
| 9 |
+
- Channel is 'in-app' only. Email/Slack delivery is OUTBOUND and ships only with owner approval
|
| 10 |
+
(the Odoo mail client is whitelisted for Collections statements ONLY β nothing else sends).
|
| 11 |
+
- No background scheduler: reports record cadence + show due-state in-app; alerts evaluate live
|
| 12 |
+
whenever the Routines page (or a tool) asks. The enterprise scheduler lands with OM-5 delivery.
|
| 13 |
+
|
| 14 |
+
Alert modes:
|
| 15 |
+
- 'xmr' Wheeler individuals chart on complete monthly values (promoted from
|
| 16 |
+
modules/warehouse._xmr, generalized: no 0/100 clamps β metrics aren't percentages).
|
| 17 |
+
Three detection rules only (more = alarm fatigue, per the warehouse brief).
|
| 18 |
+
- 'threshold' last complete month above/below a fixed value.
|
| 19 |
+
"""
|
| 20 |
+
import json
|
| 21 |
+
import time
|
| 22 |
+
from datetime import date, timedelta
|
| 23 |
+
|
| 24 |
+
import harness.semantic as SEM
|
| 25 |
+
from harness.tools import VIEWS_PATH
|
| 26 |
+
|
| 27 |
+
ROUTINES_PATH = VIEWS_PATH.parent / "routines.json"
|
| 28 |
+
|
| 29 |
+
CADENCES = ("daily", "weekly", "monthly")
|
| 30 |
+
CHANNELS = ("in-app",) # email/slack gate on owner approval (outbound send)
|
| 31 |
+
ALERT_MODES = ("xmr", "threshold")
|
| 32 |
+
XMR_MONTHS = 18 # complete months fed to the individuals chart
|
| 33 |
+
|
| 34 |
+
|
| 35 |
+
def load():
|
| 36 |
+
if ROUTINES_PATH.exists():
|
| 37 |
+
return json.loads(ROUTINES_PATH.read_text(encoding="utf-8"))
|
| 38 |
+
return {"reports": {}, "alerts": {}}
|
| 39 |
+
|
| 40 |
+
|
| 41 |
+
def _save(d):
|
| 42 |
+
ROUTINES_PATH.parent.mkdir(parents=True, exist_ok=True)
|
| 43 |
+
ROUTINES_PATH.write_text(json.dumps(d, indent=1), encoding="utf-8")
|
| 44 |
+
|
| 45 |
+
|
| 46 |
+
# ------------------------------------------------------------------ scheduled reports
|
| 47 |
+
|
| 48 |
+
def schedule_report(view, cadence, channel="in-app"):
|
| 49 |
+
import harness.views as V
|
| 50 |
+
if view not in V.views() and view not in V.dashboards():
|
| 51 |
+
raise SEM.ModelError(f"unknown view/dashboard {view!r} β save_view/compose_dashboard first")
|
| 52 |
+
if cadence not in CADENCES:
|
| 53 |
+
raise SEM.ModelError(f"cadence must be one of {CADENCES}")
|
| 54 |
+
if channel not in CHANNELS:
|
| 55 |
+
raise SEM.ModelError(f"channel {channel!r} not available yet β v0 delivers in-app "
|
| 56 |
+
"(email/Slack ship with the delivery gate); use channel='in-app'")
|
| 57 |
+
d = load()
|
| 58 |
+
d["reports"][view] = {"cadence": cadence, "channel": channel,
|
| 59 |
+
"created_at": time.strftime("%Y-%m-%d %H:%M")}
|
| 60 |
+
_save(d)
|
| 61 |
+
return {"scheduled": view, "cadence": cadence, "channel": channel,
|
| 62 |
+
"note": "runs in-app (Workflows > Routines); external delivery ships with OM-5 email approval"}
|
| 63 |
+
|
| 64 |
+
|
| 65 |
+
def delete_report(view):
|
| 66 |
+
d = load()
|
| 67 |
+
d["reports"].pop(view, None)
|
| 68 |
+
_save(d)
|
| 69 |
+
|
| 70 |
+
|
| 71 |
+
_CADENCE_DAYS = {"daily": 1, "weekly": 7, "monthly": 30}
|
| 72 |
+
|
| 73 |
+
|
| 74 |
+
def mark_viewed(view):
|
| 75 |
+
"""Called when the scheduled view/dashboard is actually opened β resets its due-ness."""
|
| 76 |
+
d = load()
|
| 77 |
+
r = d.get("reports", {}).get(view)
|
| 78 |
+
if r is not None:
|
| 79 |
+
r["last_viewed"] = time.strftime("%Y-%m-%d %H:%M")
|
| 80 |
+
_save(d)
|
| 81 |
+
|
| 82 |
+
|
| 83 |
+
def due_reports():
|
| 84 |
+
"""Scheduled reports whose cadence has elapsed since last viewed (v0 in-app delivery via
|
| 85 |
+
the Routines page). Never-viewed ages from created_at."""
|
| 86 |
+
from datetime import datetime
|
| 87 |
+
out = []
|
| 88 |
+
for name, r in load().get("reports", {}).items():
|
| 89 |
+
days = _CADENCE_DAYS.get(r.get("cadence"), 7)
|
| 90 |
+
anchor = r.get("last_viewed") or r.get("created_at") or ""
|
| 91 |
+
try:
|
| 92 |
+
age = (datetime.now() - datetime.strptime(anchor, "%Y-%m-%d %H:%M")).days
|
| 93 |
+
except Exception:
|
| 94 |
+
age = days
|
| 95 |
+
if age >= days:
|
| 96 |
+
out.append({"name": name, "cadence": r.get("cadence"),
|
| 97 |
+
"last_viewed": r.get("last_viewed"), "age_days": age})
|
| 98 |
+
return out
|
| 99 |
+
|
| 100 |
+
|
| 101 |
+
# ------------------------------------------------------------------ insight routines (OM-5)
|
| 102 |
+
# A saved PROMPT on a cadence: the Analyst runs it in the background and the narrative (plus its
|
| 103 |
+
# governed artifacts) lands in-app on the Routines page. The narrative is a deliberate
|
| 104 |
+
# point-in-time SNAPSHOT (it narrates a moment); its charts still carry re-runnable queries.
|
| 105 |
+
|
| 106 |
+
def schedule_insight(name, prompt, cadence):
|
| 107 |
+
if cadence not in CADENCES:
|
| 108 |
+
raise SEM.ModelError(f"cadence must be one of {CADENCES}")
|
| 109 |
+
if not (prompt or "").strip():
|
| 110 |
+
raise SEM.ModelError("an insight needs a prompt β what should the Analyst look at?")
|
| 111 |
+
d = load()
|
| 112 |
+
d.setdefault("insights", {})[name] = {"prompt": prompt.strip(), "cadence": cadence,
|
| 113 |
+
"created_at": time.strftime("%Y-%m-%d %H:%M")}
|
| 114 |
+
_save(d)
|
| 115 |
+
return {"insight": name, "cadence": cadence,
|
| 116 |
+
"note": "runs in the background when due; the narrative lands on "
|
| 117 |
+
"Workflows > Routines"}
|
| 118 |
+
|
| 119 |
+
|
| 120 |
+
def delete_insight(name):
|
| 121 |
+
d = load()
|
| 122 |
+
(d.get("insights") or {}).pop(name, None)
|
| 123 |
+
_save(d)
|
| 124 |
+
|
| 125 |
+
|
| 126 |
+
def due_insights():
|
| 127 |
+
from datetime import datetime
|
| 128 |
+
out = []
|
| 129 |
+
for name, i in (load().get("insights") or {}).items():
|
| 130 |
+
days = _CADENCE_DAYS.get(i.get("cadence"), 7)
|
| 131 |
+
anchor = i.get("last_run_at") or ""
|
| 132 |
+
try:
|
| 133 |
+
age = (datetime.now() - datetime.strptime(anchor, "%Y-%m-%d %H:%M")).days
|
| 134 |
+
except Exception:
|
| 135 |
+
age = days # never run -> due now
|
| 136 |
+
if age >= days:
|
| 137 |
+
out.append(name)
|
| 138 |
+
return out
|
| 139 |
+
|
| 140 |
+
|
| 141 |
+
def run_insight(name):
|
| 142 |
+
"""Execute one insight through the Analyst and persist the narrative + artifacts."""
|
| 143 |
+
d = load()
|
| 144 |
+
ins = (d.get("insights") or {}).get(name)
|
| 145 |
+
if not ins:
|
| 146 |
+
raise SEM.ModelError(f"unknown insight {name!r}")
|
| 147 |
+
import harness.analyst as A
|
| 148 |
+
r = A.ask(ins["prompt"])
|
| 149 |
+
now = time.strftime("%Y-%m-%d %H:%M")
|
| 150 |
+
ins["last_run_at"] = now
|
| 151 |
+
ins["last"] = {"at": now, "answer": r["answer"], "artifacts": r.get("artifacts") or [],
|
| 152 |
+
"tokens": (r.get("usage") or {}).get("prompt_tokens", 0)
|
| 153 |
+
+ (r.get("usage") or {}).get("completion_tokens", 0)}
|
| 154 |
+
_save(d)
|
| 155 |
+
return ins["last"]
|
| 156 |
+
|
| 157 |
+
|
| 158 |
+
# ------------------------------------------------------------------ KPI alerts
|
| 159 |
+
|
| 160 |
+
def create_alert(metric, mode="xmr", team_id=None, threshold=None, direction=None):
|
| 161 |
+
m = SEM.metrics().get(metric)
|
| 162 |
+
if not m:
|
| 163 |
+
raise SEM.ModelError(f"unknown metric {metric!r} (use list_topics)")
|
| 164 |
+
if mode not in ALERT_MODES:
|
| 165 |
+
raise SEM.ModelError(f"mode must be one of {ALERT_MODES}")
|
| 166 |
+
if mode == "threshold":
|
| 167 |
+
if threshold is None or direction not in ("above", "below"):
|
| 168 |
+
raise SEM.ModelError("threshold mode needs threshold=<number> and "
|
| 169 |
+
"direction='above'|'below'")
|
| 170 |
+
d = load()
|
| 171 |
+
key = metric + (f"_team{team_id}" if team_id else "")
|
| 172 |
+
d["alerts"][key] = {"metric": metric, "mode": mode, "team_id": team_id,
|
| 173 |
+
"threshold": threshold, "direction": direction,
|
| 174 |
+
"created_at": time.strftime("%Y-%m-%d %H:%M")}
|
| 175 |
+
_save(d)
|
| 176 |
+
return {"alert": key, **evaluate_alert(d["alerts"][key])}
|
| 177 |
+
|
| 178 |
+
|
| 179 |
+
def delete_alert(key):
|
| 180 |
+
d = load()
|
| 181 |
+
d["alerts"].pop(key, None)
|
| 182 |
+
_save(d)
|
| 183 |
+
|
| 184 |
+
|
| 185 |
+
def xmr(series):
|
| 186 |
+
"""Wheeler XmR individuals limits + the three detection rules β the model-level primitive
|
| 187 |
+
(generalized from modules/warehouse._xmr; no percentage clamps). series=[(label, value)]."""
|
| 188 |
+
vals = [v for _, v in series]
|
| 189 |
+
if len(vals) < 8:
|
| 190 |
+
return {"center": None, "unpl": None, "lnpl": None,
|
| 191 |
+
"points": [{"label": l, "value": v, "signal": ""} for l, v in series]}
|
| 192 |
+
center = sum(vals) / len(vals)
|
| 193 |
+
mrs = [abs(vals[i] - vals[i - 1]) for i in range(1, len(vals))]
|
| 194 |
+
mr_bar = sum(mrs) / len(mrs) if mrs else 0.0
|
| 195 |
+
unpl = center + 2.66 * mr_bar
|
| 196 |
+
lnpl = center - 2.66 * mr_bar
|
| 197 |
+
pts = []
|
| 198 |
+
for i, (l, v) in enumerate(series):
|
| 199 |
+
sig = ""
|
| 200 |
+
if v > unpl or v < lnpl:
|
| 201 |
+
sig = "outside limits"
|
| 202 |
+
if not sig and i >= 7 and all((x - center) > 0 for _, x in series[i - 7:i + 1]):
|
| 203 |
+
sig = "sustained shift (8 above center)"
|
| 204 |
+
if not sig and i >= 7 and all((x - center) < 0 for _, x in series[i - 7:i + 1]):
|
| 205 |
+
sig = "sustained shift (8 below center)"
|
| 206 |
+
if not sig and i >= 3:
|
| 207 |
+
last4 = [x for _, x in series[i - 3:i + 1]]
|
| 208 |
+
near = sum(1 for x in last4
|
| 209 |
+
if abs(x - center) > 0.5 * ((unpl - center) if x >= center
|
| 210 |
+
else (center - lnpl)))
|
| 211 |
+
if near >= 3:
|
| 212 |
+
sig = "moderate shift emerging"
|
| 213 |
+
pts.append({"label": l, "value": v, "signal": sig})
|
| 214 |
+
return {"center": center, "unpl": unpl, "lnpl": lnpl, "points": pts}
|
| 215 |
+
|
| 216 |
+
|
| 217 |
+
def _month_window(n=XMR_MONTHS):
|
| 218 |
+
"""Last n COMPLETE months (the in-progress month would false-signal every alert)."""
|
| 219 |
+
first_cur = date.today().replace(day=1)
|
| 220 |
+
y, mo = first_cur.year, first_cur.month
|
| 221 |
+
for _ in range(n):
|
| 222 |
+
mo -= 1
|
| 223 |
+
if mo == 0:
|
| 224 |
+
y, mo = y - 1, 12
|
| 225 |
+
start = date(y, mo, 1)
|
| 226 |
+
return start.isoformat(), (first_cur - timedelta(days=1)).isoformat()
|
| 227 |
+
|
| 228 |
+
|
| 229 |
+
def evaluate_alert(a):
|
| 230 |
+
"""Live evaluation: monthly series of the metric (complete months only) -> mode verdict."""
|
| 231 |
+
metric = a["metric"]
|
| 232 |
+
topic = SEM.metrics()[metric]["topic"]
|
| 233 |
+
date_from, date_to = _month_window()
|
| 234 |
+
res = SEM.store_query(topic, [metric], grain="month", date_from=date_from, date_to=date_to,
|
| 235 |
+
team_id=a.get("team_id"))
|
| 236 |
+
series = [(r["period"][:7], r.get(metric) or 0) for r in res["rows"]]
|
| 237 |
+
out = {"metric": metric, "mode": a["mode"], "team_id": a.get("team_id"),
|
| 238 |
+
"window": f"{date_from}..{date_to}", "series": series}
|
| 239 |
+
if not series:
|
| 240 |
+
return {**out, "status": "no data", "fired": False}
|
| 241 |
+
last_label, last_val = series[-1]
|
| 242 |
+
if a["mode"] == "threshold":
|
| 243 |
+
thr, direc = a.get("threshold"), a.get("direction")
|
| 244 |
+
fired = (last_val > thr) if direc == "above" else (last_val < thr)
|
| 245 |
+
return {**out, "last": {"label": last_label, "value": last_val},
|
| 246 |
+
"threshold": thr, "direction": direc,
|
| 247 |
+
"status": (f"FIRED β {last_label} is {direc} {thr:,.0f}" if fired else "OK"),
|
| 248 |
+
"fired": bool(fired)}
|
| 249 |
+
lim = xmr(series)
|
| 250 |
+
sig = lim["points"][-1]["signal"] if lim["points"] else ""
|
| 251 |
+
return {**out, "last": {"label": last_label, "value": last_val},
|
| 252 |
+
"center": lim["center"], "unpl": lim["unpl"], "lnpl": lim["lnpl"],
|
| 253 |
+
"points": lim["points"],
|
| 254 |
+
"status": (f"SIGNAL β {sig} ({last_label})" if sig else "OK"),
|
| 255 |
+
"fired": bool(sig)}
|
| 256 |
+
|
| 257 |
+
|
| 258 |
+
def evaluate_all():
|
| 259 |
+
return {k: evaluate_alert(a) for k, a in load().get("alerts", {}).items()}
|
platform/harness/runtime.py
ADDED
|
@@ -0,0 +1,248 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""harness/runtime.py β X7: per-REQUEST tenant resolution for the shared stateless API (EXIT-4a).
|
| 2 |
+
|
| 3 |
+
THE COST ARGUMENT, in one rule: *no per-tenant state may be resident in the app process.* Today
|
| 4 |
+
each tenant carries its own interpreter, its own never-evicted pool, its own 9 prewarmed bundles
|
| 5 |
+
and its own DuckDB β EXIT-0 measured a **~0.6 GB commit floor per tenant**, duplicated exactly
|
| 6 |
+
(tenant B came within 2% of tenant A; nothing is shared). The target is ~30β80 MB. That is only
|
| 7 |
+
reachable if the API resolves the tenant from the REQUEST and holds nothing tenant-shaped between
|
| 8 |
+
requests except a bounded cache it is willing to throw away.
|
| 9 |
+
|
| 10 |
+
WHAT THIS REPLACES. `ui/cache.py`'s `st.cache_resource` singleton β one process, one tenant,
|
| 11 |
+
caches that are never evicted by design. That is correct for Streamlit and fatal for a shared
|
| 12 |
+
process. The Streamlit side keeps its singleton untouched this wave (two processes, two caches,
|
| 13 |
+
accepted for the strangler period; `ui/cache.py` dies with app.py at EXIT-6).
|
| 14 |
+
|
| 15 |
+
THREE RULES THIS FILE OBEYS, each of which is a way the old shape leaked:
|
| 16 |
+
1. **No module-level per-tenant globals.** Every tenant-shaped value hangs off a TenantRuntime
|
| 17 |
+
reached through `get_runtime(slug)`; nothing is stashed at import time. A module global is
|
| 18 |
+
exactly what makes "which tenant is this process serving?" unanswerable.
|
| 19 |
+
2. **LRU-BOUNDED.** `_MAX_RUNTIMES` runtimes live at once; the least recently used is evicted.
|
| 20 |
+
An unbounded registry cache is a memory leak with a tenant-count-shaped growth curve β the
|
| 21 |
+
thing this whole program exists to remove.
|
| 22 |
+
3. **The store handle is tenant-bound**, so a write cannot land in another tenant's namespace
|
| 23 |
+
even if a caller passes the wrong key downstream (EXIT-4b proof #3).
|
| 24 |
+
|
| 25 |
+
β THE DUCKDB CAVEAT, STATED NOT HIDDEN. `harness.datastore` holds ONE process-wide connection
|
| 26 |
+
(the fix for the 2026-07-16 "connection error unless I refresh" bug), so one PROCESS can serve
|
| 27 |
+
one analytical store at a time. This module therefore hands out `datastore_path` β the file a
|
| 28 |
+
tenant's analytical reads belong to β and does NOT switch the global underneath a live request.
|
| 29 |
+
Until the singleton is per-tenant (EXIT-5), an API process answering measure/Analyst queries must
|
| 30 |
+
be pinned to one tenant's file, and `assert_datastore_matches()` below is what turns a
|
| 31 |
+
mis-pinned deployment into a loud failure instead of a silent cross-tenant read.
|
| 32 |
+
"""
|
| 33 |
+
from __future__ import annotations
|
| 34 |
+
|
| 35 |
+
import os
|
| 36 |
+
import threading
|
| 37 |
+
from collections import OrderedDict
|
| 38 |
+
from dataclasses import dataclass, field
|
| 39 |
+
|
| 40 |
+
import core.store as store
|
| 41 |
+
from harness import datastore as _ds
|
| 42 |
+
from harness import tenants as _tenants
|
| 43 |
+
|
| 44 |
+
#: How many tenant runtimes stay resident. Small on purpose: a runtime is cheap to rebuild (a
|
| 45 |
+
#: Tenant literal + a couple of handles) and expensive to leak. Raise only with a measurement.
|
| 46 |
+
_MAX_RUNTIMES = int(os.environ.get("AIOS_MAX_RUNTIMES") or 32)
|
| 47 |
+
|
| 48 |
+
#: slug -> builder. The literal registry `harness/tenants.py` describes; per its own docstring
|
| 49 |
+
#: this becomes "a tenants store (DB / HF dataset)" β X4's control DB is where it goes, and
|
| 50 |
+
#: `register()` below is the seam that makes the swap a one-liner rather than a refactor.
|
| 51 |
+
_BUILDERS = {"royal-imports": _tenants.royal_imports}
|
| 52 |
+
|
| 53 |
+
# X8's isolation fixture, admitted ONLY on request. A stub tenant that is always present is a
|
| 54 |
+
# slug an attacker can mint a session against in production; gated, it exists exactly where it is
|
| 55 |
+
# needed (the TestClient isolation proofs) and nowhere else. Read at import so the flag cannot be
|
| 56 |
+
# flipped by a request.
|
| 57 |
+
if os.environ.get("AIOS_ENABLE_QA_TENANT") == "1":
|
| 58 |
+
_BUILDERS["qa-b"] = _tenants.qa_tenant_b
|
| 59 |
+
|
| 60 |
+
_CACHE: "OrderedDict[str, TenantRuntime]" = OrderedDict()
|
| 61 |
+
_LOCK = threading.RLock()
|
| 62 |
+
|
| 63 |
+
|
| 64 |
+
@dataclass
|
| 65 |
+
class TenantRuntime:
|
| 66 |
+
"""Everything a request needs to serve ONE tenant, and nothing about any other.
|
| 67 |
+
|
| 68 |
+
`store_namespace` is the prefix every product-data key is written under. Tenant #0 keeps the
|
| 69 |
+
EMPTY prefix so its existing keys (`users`, `customer_lists`, `customer_docs`, β¦) are found
|
| 70 |
+
exactly where they already live β a namespacing change that renamed tenant #0's keys would be
|
| 71 |
+
a silent data loss dressed as a refactor. Every OTHER tenant is prefixed, which is what makes
|
| 72 |
+
proof #3 hold.
|
| 73 |
+
"""
|
| 74 |
+
key: str
|
| 75 |
+
tenant: object # harness.base.Tenant
|
| 76 |
+
store_namespace: str = ""
|
| 77 |
+
pool_cache: dict = field(default_factory=dict)
|
| 78 |
+
#: EXIT-5 β the API adapter's memos for `core.measure_resolve` (the Streamlit adapter keeps
|
| 79 |
+
#: its own in st.session_state). Bounded by the module's own clear-past-cap rule; keyed on
|
| 80 |
+
#: (stamp, scope, pool identity, question) so nothing user-shaped lives in them.
|
| 81 |
+
measure_memo: dict = field(default_factory=dict)
|
| 82 |
+
mset_memo: dict = field(default_factory=dict)
|
| 83 |
+
#: Wave 2026-08-02 (C-TS): the time-series channel's own memo β separate from
|
| 84 |
+
#: measure_memo so column churn and series churn cannot evict each other's answers.
|
| 85 |
+
series_memo: dict = field(default_factory=dict)
|
| 86 |
+
_lock: threading.RLock = field(default_factory=threading.RLock, repr=False)
|
| 87 |
+
|
| 88 |
+
@property
|
| 89 |
+
def name(self):
|
| 90 |
+
return getattr(self.tenant, "name", self.key)
|
| 91 |
+
|
| 92 |
+
@property
|
| 93 |
+
def datastore_path(self):
|
| 94 |
+
"""The DuckDB file this tenant's analytical reads belong to (file-per-tenant IS the
|
| 95 |
+
isolation model β DuckDB has no RLS, so the OS boundary is the only one that holds)."""
|
| 96 |
+
return _ds.path_for(self.key)
|
| 97 |
+
|
| 98 |
+
# ---------------------------------------------------------------- the store handle
|
| 99 |
+
#: Wave 18 (C1-TENANT, R2): a tenant that OWNS its dataset repo carries its bound Store
|
| 100 |
+
#: instance here (namespace stays EMPTY β a whole repo needs no prefix). None = the shared
|
| 101 |
+
#: default repo through the MODULE functions, which is what keeps the gates' fake-store
|
| 102 |
+
#: installation effective (the fake patches module attributes, and royal-imports + the
|
| 103 |
+
#: prefix tenants resolve through exactly those).
|
| 104 |
+
store_handle: object = None
|
| 105 |
+
|
| 106 |
+
def _store(self):
|
| 107 |
+
return self.store_handle if self.store_handle is not None else store
|
| 108 |
+
|
| 109 |
+
def store_key(self, name):
|
| 110 |
+
"""Namespace a product-data key for this tenant. The ONE place the prefix is applied."""
|
| 111 |
+
return f"{self.store_namespace}{name}" if self.store_namespace else str(name)
|
| 112 |
+
|
| 113 |
+
def get(self, name, fresh=False):
|
| 114 |
+
return self._store().get(self.store_key(name), fresh=fresh)
|
| 115 |
+
|
| 116 |
+
def put(self, name, data):
|
| 117 |
+
return self._store().put(self.store_key(name), data)
|
| 118 |
+
|
| 119 |
+
def update(self, name, fn, flush="sync"):
|
| 120 |
+
return self._store().update(self.store_key(name), fn, flush=flush)
|
| 121 |
+
|
| 122 |
+
def exists(self, name):
|
| 123 |
+
return self._store().exists(self.store_key(name))
|
| 124 |
+
|
| 125 |
+
def available(self):
|
| 126 |
+
return self._store().available()
|
| 127 |
+
|
| 128 |
+
def assert_datastore_matches(self):
|
| 129 |
+
"""RAISE if the process-wide DuckDB file is not this tenant's.
|
| 130 |
+
|
| 131 |
+
The alternative is the failure mode that matters: an analytical read served from another
|
| 132 |
+
tenant's file returns real, plausible, wrong rows β no exception, no telemetry, and the
|
| 133 |
+
number reconciles against the wrong book. A loud failure is strictly better, and this is
|
| 134 |
+
the assertion EXIT-5 deletes when the connection becomes per-tenant.
|
| 135 |
+
"""
|
| 136 |
+
want, have = self.datastore_path.resolve(), _ds.DB_PATH.resolve()
|
| 137 |
+
if want != have:
|
| 138 |
+
raise RuntimeError(
|
| 139 |
+
f"tenant {self.key!r} expects the analytical store at {want}, but this process "
|
| 140 |
+
f"has {have} open. Refusing the read β a cross-tenant answer would reconcile "
|
| 141 |
+
f"against the wrong book. (Pin the process with AIOS_DUCKDB_PATH, or call "
|
| 142 |
+
f"harness.datastore.use_path before serving this tenant.)")
|
| 143 |
+
|
| 144 |
+
|
| 145 |
+
def register(key, builder):
|
| 146 |
+
"""Add a tenant to the registry (a stub connector is a legitimate builder β see X8's qa-b).
|
| 147 |
+
Invalidates any cached runtime for the slug so a re-registration cannot be shadowed."""
|
| 148 |
+
_BUILDERS[str(key)] = builder
|
| 149 |
+
invalidate(key)
|
| 150 |
+
|
| 151 |
+
|
| 152 |
+
def known_tenants():
|
| 153 |
+
keys = set(_BUILDERS)
|
| 154 |
+
try:
|
| 155 |
+
keys |= set(_tenant_records())
|
| 156 |
+
except Exception:
|
| 157 |
+
pass
|
| 158 |
+
return sorted(keys)
|
| 159 |
+
|
| 160 |
+
|
| 161 |
+
#: Wave 18 (C1-TENANT) β the CONTROL-PLANE bucket: tenants provisioned at runtime rather than
|
| 162 |
+
#: compiled in. Lives in the DEFAULT store (tenant #0's repo) deliberately: it answers "which
|
| 163 |
+
#: tenants exist", which is a platform fact, not any one tenant's data.
|
| 164 |
+
TENANTS_KEY = "tenants"
|
| 165 |
+
|
| 166 |
+
|
| 167 |
+
def _tenant_records():
|
| 168 |
+
"""{slug: record} from the control-plane bucket. {} on any failure β an unreachable bucket
|
| 169 |
+
must degrade to "only the compiled-in tenants exist", never to an exception at login."""
|
| 170 |
+
try:
|
| 171 |
+
recs = store.get(TENANTS_KEY) or {}
|
| 172 |
+
return {str(k).strip().lower(): v for k, v in recs.items() if isinstance(v, dict)}
|
| 173 |
+
except Exception:
|
| 174 |
+
return {}
|
| 175 |
+
|
| 176 |
+
|
| 177 |
+
def _builder_from_record(key, rec):
|
| 178 |
+
"""A Tenant for a bucket-provisioned client: no sources yet (connectors arrive via the
|
| 179 |
+
Keychain flow), config carries the platform fields the request path reads."""
|
| 180 |
+
def _build():
|
| 181 |
+
from harness.base import Tenant
|
| 182 |
+
return Tenant(key=key, name=str(rec.get("name") or key),
|
| 183 |
+
config={"modules": rec.get("modules", []),
|
| 184 |
+
"store_repo": rec.get("store_repo") or None})
|
| 185 |
+
return _build
|
| 186 |
+
|
| 187 |
+
|
| 188 |
+
def get_runtime(tenant_key):
|
| 189 |
+
"""The runtime for `tenant_key`, built on first use and LRU-cached.
|
| 190 |
+
|
| 191 |
+
Raises KeyError for an unknown slug β the caller turns that into a 401 (fail-closed: an
|
| 192 |
+
unknown tenant is not a 404, because confirming which slugs exist answers a question the
|
| 193 |
+
request was not entitled to ask).
|
| 194 |
+
"""
|
| 195 |
+
key = str(tenant_key or "").strip().lower()
|
| 196 |
+
if not key:
|
| 197 |
+
raise KeyError("no tenant")
|
| 198 |
+
with _LOCK:
|
| 199 |
+
rt = _CACHE.get(key)
|
| 200 |
+
if rt is not None:
|
| 201 |
+
_CACHE.move_to_end(key) # LRU: touch on read
|
| 202 |
+
return rt
|
| 203 |
+
builder = _BUILDERS.get(key)
|
| 204 |
+
# Wave 18 (C1-TENANT): a slug the compiled registry does not know may be a PROVISIONED
|
| 205 |
+
# tenant β consult the control-plane bucket before failing closed. A suspended record is
|
| 206 |
+
# the same KeyError as an unknown one: "which tenants exist" is not this caller's question.
|
| 207 |
+
if builder is None:
|
| 208 |
+
rec = _tenant_records().get(key)
|
| 209 |
+
if not rec or rec.get("status", "active") != "active":
|
| 210 |
+
raise KeyError(key)
|
| 211 |
+
builder = _builder_from_record(key, rec)
|
| 212 |
+
# Built OUTSIDE the lock: a builder may do real work (a connector handshake), and holding the
|
| 213 |
+
# registry lock through it would serialise every other tenant's requests behind it.
|
| 214 |
+
tenant = builder()
|
| 215 |
+
with _LOCK:
|
| 216 |
+
existing = _CACHE.get(key)
|
| 217 |
+
if existing is not None: # another thread won the race β keep ONE runtime
|
| 218 |
+
_CACHE.move_to_end(key)
|
| 219 |
+
return existing
|
| 220 |
+
# R2: a tenant with its OWN repo gets a bound Store instance and an EMPTY prefix (the
|
| 221 |
+
# repo boundary is the namespace); everyone else keeps the t/<slug>/ prefix in the
|
| 222 |
+
# shared repo. Tenant #0 keeps both defaults β empty prefix, shared repo.
|
| 223 |
+
repo = (getattr(tenant, "config", {}) or {}).get("store_repo")
|
| 224 |
+
rt = TenantRuntime(
|
| 225 |
+
key=key, tenant=tenant,
|
| 226 |
+
store_namespace=("" if (key == "royal-imports" or repo) else f"t/{key}/"),
|
| 227 |
+
store_handle=(store.for_repo(repo) if repo else None))
|
| 228 |
+
_CACHE[key] = rt
|
| 229 |
+
while len(_CACHE) > _MAX_RUNTIMES:
|
| 230 |
+
_CACHE.popitem(last=False) # evict least recently used
|
| 231 |
+
return rt
|
| 232 |
+
|
| 233 |
+
|
| 234 |
+
def invalidate(tenant_key=None):
|
| 235 |
+
"""Drop one tenant's runtime, or all of them. Explicit, per X7 β a cache with no documented
|
| 236 |
+
way to clear it becomes a restart."""
|
| 237 |
+
with _LOCK:
|
| 238 |
+
if tenant_key is None:
|
| 239 |
+
_CACHE.clear()
|
| 240 |
+
else:
|
| 241 |
+
_CACHE.pop(str(tenant_key or "").strip().lower(), None)
|
| 242 |
+
|
| 243 |
+
|
| 244 |
+
def resident():
|
| 245 |
+
"""The tenant slugs currently holding a runtime β LRU order, oldest first. For the EXIT-4
|
| 246 |
+
measurement and for asserting the bound actually binds."""
|
| 247 |
+
with _LOCK:
|
| 248 |
+
return list(_CACHE)
|
platform/harness/semantic.py
ADDED
|
@@ -0,0 +1,936 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""harness/semantic.py β the semantic seam (OM-0a, 2026-07-11).
|
| 2 |
+
|
| 3 |
+
The ONE governed registry of topics + metrics (platform/model/*.yml) and the resolver that
|
| 4 |
+
turns a metric key into a number through the SAME proven data layer the validated modules use
|
| 5 |
+
(core/odoo domain builders + aggregate helpers) β so scope parity is by construction, and every
|
| 6 |
+
metric carries its definition, ai_context, and an independent validate contract.
|
| 7 |
+
|
| 8 |
+
Consumers (the point of the seam): the app's pages, the metric dictionary (OM-0c), the saved-
|
| 9 |
+
dashboard viewer (OM-3; the Explore page was retired 2026-07-23), the AI Analyst's tools (OM-4:
|
| 10 |
+
list_topics / describe_topic / run_semantic_query), and the MCP server (OM-6). Plan:
|
| 11 |
+
.claude/wiki/research/omni-adoption.md (Part IV + the productization addendum).
|
| 12 |
+
|
| 13 |
+
Design constraints honored (from /odoo-api):
|
| 14 |
+
- read_group dot-path GROUPBY fails β scalar aggregates here use sum_field/distinct_count;
|
| 15 |
+
grouped queries (OM-3+) must group by direct m2o fields only.
|
| 16 |
+
- Scope is never hand-rolled: topics BIND to a named domain builder (sale_line_domain, ...);
|
| 17 |
+
the YAML `scope:` block is the declarative statement of what that builder bakes in.
|
| 18 |
+
- READ-ONLY on Odoo, always.
|
| 19 |
+
"""
|
| 20 |
+
import ast
|
| 21 |
+
import functools
|
| 22 |
+
import operator
|
| 23 |
+
from pathlib import Path
|
| 24 |
+
|
| 25 |
+
import yaml
|
| 26 |
+
|
| 27 |
+
import core.odoo as O
|
| 28 |
+
|
| 29 |
+
MODEL_DIR = Path(__file__).resolve().parents[1] / "model"
|
| 30 |
+
|
| 31 |
+
# Topic scope binds to a NAMED, reviewed domain builder β never a YAML-assembled domain (one
|
| 32 |
+
# source of truth for scope until OM-2 makes topics executable against the store).
|
| 33 |
+
def _order_domain(*a, **kw):
|
| 34 |
+
# lazy: order scope currently lives in modules/sales.py; OM-2 lifts it into core/the store
|
| 35 |
+
import modules.sales as S
|
| 36 |
+
return S.order_domain(*a, **kw)
|
| 37 |
+
|
| 38 |
+
|
| 39 |
+
def _customer_move_domain(move_type):
|
| 40 |
+
"""A posted customer document domain, matching `modules/returns.py`'s `_mdom` exactly.
|
| 41 |
+
|
| 42 |
+
β COMPANY-LEVEL, and it REFUSES a team rather than ignoring one. Credit notes all carry
|
| 43 |
+
team_id = 1, so a BU filter on the document is meaningless β silently dropping the argument
|
| 44 |
+
would hand a business-unit user the company's number under their own name, which is the
|
| 45 |
+
widening sin in a different currency.
|
| 46 |
+
"""
|
| 47 |
+
def build(date_from=None, date_to=None, team_id=None, **kw):
|
| 48 |
+
if team_id:
|
| 49 |
+
raise ModelError(
|
| 50 |
+
f"customer {move_type} documents are COMPANY-LEVEL β credit notes and invoices "
|
| 51 |
+
f"carry no trustworthy business-unit tag, so a team-scoped total cannot be "
|
| 52 |
+
f"produced. Refusing rather than returning the company number.")
|
| 53 |
+
dom = [("move_type", "=", move_type), ("state", "=", "posted")]
|
| 54 |
+
if date_from:
|
| 55 |
+
dom.append(("invoice_date", ">=", date_from))
|
| 56 |
+
if date_to:
|
| 57 |
+
dom.append(("invoice_date", "<=", date_to))
|
| 58 |
+
return dom
|
| 59 |
+
return build
|
| 60 |
+
|
| 61 |
+
|
| 62 |
+
_BUILDERS = {
|
| 63 |
+
"sale_line_domain": O.sale_line_domain,
|
| 64 |
+
"sale_order_domain": _order_domain,
|
| 65 |
+
"credit_note_domain": _customer_move_domain("out_refund"),
|
| 66 |
+
"customer_invoice_domain": _customer_move_domain("out_invoice"),
|
| 67 |
+
}
|
| 68 |
+
|
| 69 |
+
|
| 70 |
+
class ModelError(Exception):
|
| 71 |
+
"""A model-file problem (unknown key, bad reference, unparseable expr)."""
|
| 72 |
+
|
| 73 |
+
|
| 74 |
+
# ---------------------------------------------------------------- loading
|
| 75 |
+
|
| 76 |
+
@functools.lru_cache(maxsize=1)
|
| 77 |
+
def _tenant():
|
| 78 |
+
f = MODEL_DIR / "tenant.yml"
|
| 79 |
+
return yaml.safe_load(f.read_text(encoding="utf-8")) if f.exists() else {"params": {}}
|
| 80 |
+
|
| 81 |
+
|
| 82 |
+
def _sql_value(v):
|
| 83 |
+
"""Render a tenant param into SQL safely (params are versioned config, not user input β
|
| 84 |
+
but escape anyway): ints as-is, lists comma-joined, strings single-quoted + escaped."""
|
| 85 |
+
if isinstance(v, bool):
|
| 86 |
+
raise ModelError("boolean tenant params not supported in scope_sql")
|
| 87 |
+
if isinstance(v, (int, float)):
|
| 88 |
+
return str(v)
|
| 89 |
+
if isinstance(v, (list, tuple)):
|
| 90 |
+
return ", ".join(_sql_value(x) for x in v)
|
| 91 |
+
return "'" + str(v).replace("'", "''") + "'"
|
| 92 |
+
|
| 93 |
+
|
| 94 |
+
def render_scope(sql):
|
| 95 |
+
"""Substitute {param} placeholders in a store scope/filter SQL fragment from tenant.yml."""
|
| 96 |
+
params = _tenant().get("params") or {}
|
| 97 |
+
out = sql
|
| 98 |
+
for k, v in params.items():
|
| 99 |
+
out = out.replace("{" + k + "}", _sql_value(v))
|
| 100 |
+
if "{" in out and "}" in out:
|
| 101 |
+
import re as _re
|
| 102 |
+
missing = _re.findall(r"\{([a-z_]+)\}", out)
|
| 103 |
+
if missing:
|
| 104 |
+
raise ModelError(f"scope_sql references unknown tenant params: {missing}")
|
| 105 |
+
return out
|
| 106 |
+
|
| 107 |
+
|
| 108 |
+
@functools.lru_cache(maxsize=1)
|
| 109 |
+
def _model():
|
| 110 |
+
topics, metrics = {}, {}
|
| 111 |
+
for f in sorted((MODEL_DIR / "topics").glob("*.yml")):
|
| 112 |
+
d = yaml.safe_load(f.read_text(encoding="utf-8"))
|
| 113 |
+
if not d or "key" not in d:
|
| 114 |
+
raise ModelError(f"topic file {f.name} lacks a 'key'")
|
| 115 |
+
if d.get("domain_builder") and d["domain_builder"] not in _BUILDERS:
|
| 116 |
+
raise ModelError(f"topic {d['key']}: unknown domain_builder {d.get('domain_builder')!r}")
|
| 117 |
+
# a topic may be STORE-ONLY (no live-path builder yet, e.g. gl_lines) β metric() raises
|
| 118 |
+
# a clear error if a live resolution is attempted on it
|
| 119 |
+
topics[d["key"]] = d
|
| 120 |
+
for f in sorted((MODEL_DIR / "metrics").glob("*.yml")):
|
| 121 |
+
d = yaml.safe_load(f.read_text(encoding="utf-8"))
|
| 122 |
+
topic = d.get("topic")
|
| 123 |
+
if topic not in topics:
|
| 124 |
+
raise ModelError(f"metrics file {f.name}: unknown topic {topic!r}")
|
| 125 |
+
for m in d.get("metrics", []):
|
| 126 |
+
if "key" not in m:
|
| 127 |
+
raise ModelError(f"metrics file {f.name}: a metric lacks a 'key'")
|
| 128 |
+
if m["key"] in metrics:
|
| 129 |
+
raise ModelError(f"duplicate metric key {m['key']!r}")
|
| 130 |
+
m.setdefault("topic", topic) # a metric may name its own topic (rare)
|
| 131 |
+
if m["topic"] not in topics:
|
| 132 |
+
raise ModelError(f"metric {m['key']!r}: unknown topic {m['topic']!r}")
|
| 133 |
+
metrics[m["key"]] = m
|
| 134 |
+
return {"topics": topics, "metrics": metrics}
|
| 135 |
+
|
| 136 |
+
|
| 137 |
+
def reload_model():
|
| 138 |
+
_model.cache_clear()
|
| 139 |
+
_tenant.cache_clear()
|
| 140 |
+
return _model()
|
| 141 |
+
|
| 142 |
+
|
| 143 |
+
def topics():
|
| 144 |
+
return dict(_model()["topics"])
|
| 145 |
+
|
| 146 |
+
|
| 147 |
+
def metrics():
|
| 148 |
+
return dict(_model()["metrics"])
|
| 149 |
+
|
| 150 |
+
|
| 151 |
+
def describe(key):
|
| 152 |
+
"""Full metadata for a metric (the dictionary page / AI describe_topic feed)."""
|
| 153 |
+
m = _model()["metrics"].get(key)
|
| 154 |
+
if not m:
|
| 155 |
+
raise ModelError(f"unknown metric {key!r}")
|
| 156 |
+
return {**m, "topic_def": _model()["topics"][m["topic"]]}
|
| 157 |
+
|
| 158 |
+
|
| 159 |
+
# ---------------------------------------------------------------- resolving
|
| 160 |
+
|
| 161 |
+
def _domain(topic_def, date_from, date_to, team_id):
|
| 162 |
+
b = topic_def.get("domain_builder")
|
| 163 |
+
if not b:
|
| 164 |
+
raise ModelError(f"topic {topic_def['key']!r} is store-only β use store_query() "
|
| 165 |
+
f"(no live-path domain builder bound yet)")
|
| 166 |
+
return _BUILDERS[b](date_from, date_to, team_id=team_id)
|
| 167 |
+
|
| 168 |
+
|
| 169 |
+
_OPS = {ast.Add: operator.add, ast.Sub: operator.sub,
|
| 170 |
+
ast.Mult: operator.mul, ast.Div: operator.truediv}
|
| 171 |
+
|
| 172 |
+
|
| 173 |
+
def _eval_expr(expr, resolve):
|
| 174 |
+
"""Safely evaluate a derived-metric expression: metric keys, numbers, + - * / and parens."""
|
| 175 |
+
def ev(node):
|
| 176 |
+
if isinstance(node, ast.Expression):
|
| 177 |
+
return ev(node.body)
|
| 178 |
+
if isinstance(node, ast.BinOp) and type(node.op) in _OPS:
|
| 179 |
+
return _OPS[type(node.op)](ev(node.left), ev(node.right))
|
| 180 |
+
if isinstance(node, ast.UnaryOp) and isinstance(node.op, ast.USub):
|
| 181 |
+
return -ev(node.operand)
|
| 182 |
+
if isinstance(node, ast.Num): # py<3.8 compat name; Constant below
|
| 183 |
+
return node.n
|
| 184 |
+
if isinstance(node, ast.Constant) and isinstance(node.value, (int, float)):
|
| 185 |
+
return node.value
|
| 186 |
+
if isinstance(node, ast.Name):
|
| 187 |
+
return resolve(node.id)
|
| 188 |
+
raise ModelError(f"disallowed token in derived expr: {ast.dump(node)}")
|
| 189 |
+
try:
|
| 190 |
+
return ev(ast.parse(expr, mode="eval"))
|
| 191 |
+
except SyntaxError as e:
|
| 192 |
+
raise ModelError(f"unparseable derived expr {expr!r}: {e}")
|
| 193 |
+
|
| 194 |
+
|
| 195 |
+
def metric(key, date_from=None, date_to=None, team_id=None, _seen=None):
|
| 196 |
+
"""Resolve a registered metric to a number for a window (+optional BU). The only public
|
| 197 |
+
compute path β every surface that shows this metric goes through here."""
|
| 198 |
+
_seen = _seen or set()
|
| 199 |
+
if key in _seen:
|
| 200 |
+
raise ModelError(f"circular metric reference at {key!r}")
|
| 201 |
+
_seen = _seen | {key}
|
| 202 |
+
m = _model()["metrics"].get(key)
|
| 203 |
+
if not m:
|
| 204 |
+
raise ModelError(f"unknown metric {key!r}")
|
| 205 |
+
agg = m.get("agg")
|
| 206 |
+
if agg == "ratio":
|
| 207 |
+
num = metric(m["numerator"], date_from, date_to, team_id, _seen)
|
| 208 |
+
den = metric(m["denominator"], date_from, date_to, team_id, _seen)
|
| 209 |
+
return (num / den) if den else 0.0
|
| 210 |
+
if agg == "derived":
|
| 211 |
+
return _eval_expr(m["expr"], lambda k: metric(k, date_from, date_to, team_id, _seen))
|
| 212 |
+
t = _model()["topics"][m["topic"]]
|
| 213 |
+
dom = _domain(t, date_from, date_to, team_id)
|
| 214 |
+
if agg == "sum":
|
| 215 |
+
# β `negate` applies HERE too, not only in `_measure_sql`. It existed for GL income on a
|
| 216 |
+
# STORE-ONLY topic, so the live path never met it β the first metric that is both negated
|
| 217 |
+
# and parity-checked (`returns`) would otherwise compare a negative live number against a
|
| 218 |
+
# positive store one, and the "parity failure" would look like a data problem.
|
| 219 |
+
got = O.sum_field(t["entity"], dom, m["field"])
|
| 220 |
+
return -got if m.get("negate") else got
|
| 221 |
+
if agg == "count_distinct":
|
| 222 |
+
return O.distinct_count(t["entity"], dom, m["field"])
|
| 223 |
+
if agg == "count":
|
| 224 |
+
return O.get_odoo().search_count(t["entity"], dom)
|
| 225 |
+
raise ModelError(f"metric {key!r}: unknown agg {agg!r}")
|
| 226 |
+
|
| 227 |
+
|
| 228 |
+
# ---------------------------------------------------------------- validate contracts
|
| 229 |
+
|
| 230 |
+
def _v_order_level_revenue(date_from, date_to, team_id):
|
| 231 |
+
"""Independent check: line-level Ξ£ price_subtotal == order-level Ξ£ amount_untaxed."""
|
| 232 |
+
import modules.sales as S
|
| 233 |
+
line = metric("revenue", date_from, date_to, team_id)
|
| 234 |
+
order = O.sum_field("sale.order", S.order_domain(date_from, date_to, team_id), "amount_untaxed")
|
| 235 |
+
return line, order
|
| 236 |
+
|
| 237 |
+
|
| 238 |
+
def _v_order_count(date_from, date_to, team_id):
|
| 239 |
+
"""Exact reconciliation: order-level count == distinct line-parent orders + line-LESS orders.
|
| 240 |
+
(Confirmed orders with zero lines exist in the data β 2 found at first proof, 2026-07-11;
|
| 241 |
+
counting them explicitly makes the identity exact instead of hiding them in a tolerance,
|
| 242 |
+
and surfaces them as a data-health signal.)"""
|
| 243 |
+
import modules.sales as S
|
| 244 |
+
order_level = metric("orders", date_from, date_to, team_id)
|
| 245 |
+
dom = S.order_domain(date_from, date_to, team_id)
|
| 246 |
+
from_lines = O.distinct_count("sale.order.line",
|
| 247 |
+
O.sale_line_domain(date_from, date_to, team_id), "order_id")
|
| 248 |
+
empty = O.get_odoo().search_count("sale.order", dom + [("order_line", "=", False)])
|
| 249 |
+
return order_level, from_lines + empty
|
| 250 |
+
|
| 251 |
+
|
| 252 |
+
def _v_gl_opex(date_from, date_to, team_id):
|
| 253 |
+
"""Store-path opex == the live Odoo aggregate with the Expenses-module domain (posted,
|
| 254 |
+
expense-type accounts), to the cent. team_id N/A (company-level)."""
|
| 255 |
+
res = store_query("gl_lines", ["opex"], date_from=date_from, date_to=date_to)
|
| 256 |
+
ours = res["rows"][0]["opex"] if res["rows"] else 0
|
| 257 |
+
dom = [("move_id.state", "=", "posted"),
|
| 258 |
+
("account_id.account_type", "in", ["expense", "expense_depreciation"])]
|
| 259 |
+
if date_from:
|
| 260 |
+
dom.append(("date", ">=", date_from))
|
| 261 |
+
if date_to:
|
| 262 |
+
dom.append(("date", "<=", date_to))
|
| 263 |
+
live = O.sum_field("account.move.line", dom, "balance")
|
| 264 |
+
return ours, live
|
| 265 |
+
|
| 266 |
+
|
| 267 |
+
def _v_ar_outstanding(date_from, date_to, team_id):
|
| 268 |
+
"""Store-path AR outstanding == live Odoo aggregate (posted out_invoice/out_refund residuals).
|
| 269 |
+
No date window β outstanding is an as-of-now balance."""
|
| 270 |
+
res = store_query("receivables", ["ar_outstanding"])
|
| 271 |
+
ours = res["rows"][0]["ar_outstanding"] if res["rows"] else 0
|
| 272 |
+
live = O.sum_field("account.move",
|
| 273 |
+
[("state", "=", "posted"), ("move_type", "in", ["out_invoice", "out_refund"])],
|
| 274 |
+
"amount_residual_signed")
|
| 275 |
+
return ours, live
|
| 276 |
+
|
| 277 |
+
|
| 278 |
+
def _v_returns_sign(date_from, date_to, team_id):
|
| 279 |
+
"""Independent check on the NEGATION: `returns` must equal the ABSOLUTE untaxed total of the
|
| 280 |
+
same posted credit notes, read straight from Odoo without going through the metric.
|
| 281 |
+
|
| 282 |
+
A lost negation makes returns negative; a doubled one makes it negative again. Both read as
|
| 283 |
+
"no returns" rather than as an error, and neither is visible to any self-consistency check β
|
| 284 |
+
the store and the live path would agree with each other perfectly while both being wrong.
|
| 285 |
+
"""
|
| 286 |
+
ours = metric("returns", date_from, date_to, team_id)
|
| 287 |
+
raw = O.sum_field("account.move",
|
| 288 |
+
_BUILDERS["credit_note_domain"](date_from, date_to, team_id),
|
| 289 |
+
"amount_untaxed_signed")
|
| 290 |
+
return ours, abs(raw)
|
| 291 |
+
|
| 292 |
+
|
| 293 |
+
def _v_line_vs_move_grain(date_from, date_to, team_id):
|
| 294 |
+
"""Independent check on the invoice-LINE topic: the store's netted product-line total must
|
| 295 |
+
equal the same figure read straight from LIVE Odoo.
|
| 296 |
+
|
| 297 |
+
Two things this catches that self-consistency cannot. (1) A LOST CREDIT-NOTE NEGATION β
|
| 298 |
+
`price_subtotal` is stored POSITIVE on a refund line, so a dropped subtraction inflates
|
| 299 |
+
revenue while every internal check still agrees. (2) A WRONG `display_type` FILTER β the
|
| 300 |
+
invoice tables carry 'cogs' and 'payment_term' rows alongside 'product' ones, and including
|
| 301 |
+
them changes the total silently rather than erroring.
|
| 302 |
+
|
| 303 |
+
Company-level (team_id None) uses the strongest possible reference: the MOVE-grain
|
| 304 |
+
`amount_untaxed_signed`, a different table entirely. Per-BU there is no move-grain analogue
|
| 305 |
+
(account.move carries no business unit β every invoice sits on team 1), so the reference is
|
| 306 |
+
the live line-level aggregate reached through the sale-order dot path: still an independent
|
| 307 |
+
engine and query path from the DuckDB store, which is what the contract requires.
|
| 308 |
+
"""
|
| 309 |
+
ours = (store_query("invoice_lines", ["invoiced_line_sales"],
|
| 310 |
+
date_from=date_from, date_to=date_to, team_id=team_id)
|
| 311 |
+
.get("rows") or [{}])[0].get("invoiced_line_sales") or 0.0
|
| 312 |
+
if team_id is None:
|
| 313 |
+
raw = O.sum_field("account.move",
|
| 314 |
+
[("state", "=", "posted"),
|
| 315 |
+
("move_type", "in", ["out_invoice", "out_refund"]),
|
| 316 |
+
("invoice_date", ">=", date_from), ("invoice_date", "<=", date_to)],
|
| 317 |
+
"amount_untaxed_signed")
|
| 318 |
+
return ours, raw
|
| 319 |
+
dom = [("parent_state", "=", "posted"), ("display_type", "=", "product"),
|
| 320 |
+
("date", ">=", date_from), ("date", "<=", date_to),
|
| 321 |
+
("sale_line_ids.order_id.team_id", "=", team_id)]
|
| 322 |
+
gross = O.sum_field("account.move.line", dom + [("move_type", "=", "out_invoice")],
|
| 323 |
+
"price_subtotal")
|
| 324 |
+
refunds = O.sum_field("account.move.line", dom + [("move_type", "=", "out_refund")],
|
| 325 |
+
"price_subtotal")
|
| 326 |
+
return ours, (gross or 0) - (refunds or 0)
|
| 327 |
+
|
| 328 |
+
|
| 329 |
+
_VALIDATORS = {
|
| 330 |
+
"returns_sign": _v_returns_sign,
|
| 331 |
+
"line_vs_move_grain": _v_line_vs_move_grain,
|
| 332 |
+
"order_level_revenue": _v_order_level_revenue,
|
| 333 |
+
"order_count": _v_order_count,
|
| 334 |
+
"gl_opex": _v_gl_opex,
|
| 335 |
+
"ar_outstanding": _v_ar_outstanding,
|
| 336 |
+
}
|
| 337 |
+
|
| 338 |
+
|
| 339 |
+
def validate_metric(key, date_from=None, date_to=None, team_id=None, tol=0.01):
|
| 340 |
+
"""Run a metric's declared independent cross-check. Returns the standard check dict
|
| 341 |
+
(the validate.py convention: check/ok/gap)."""
|
| 342 |
+
m = _model()["metrics"].get(key)
|
| 343 |
+
if not m:
|
| 344 |
+
raise ModelError(f"unknown metric {key!r}")
|
| 345 |
+
contract = m.get("validate")
|
| 346 |
+
if not contract:
|
| 347 |
+
return {"check": f"semantic.{key}: no independent contract declared", "ok": True, "gap": 0.0,
|
| 348 |
+
"note": "declare one in model/metrics when an independent aggregate exists"}
|
| 349 |
+
fn = _VALIDATORS.get(contract["method"])
|
| 350 |
+
if not fn:
|
| 351 |
+
raise ModelError(f"metric {key!r}: unknown validate method {contract['method']!r}")
|
| 352 |
+
ours, independent = fn(date_from, date_to, team_id)
|
| 353 |
+
gap = abs((ours or 0) - (independent or 0))
|
| 354 |
+
return {"check": f"semantic.{key} == {contract['method']} ({date_from}..{date_to}, team={team_id})",
|
| 355 |
+
"ok": gap <= tol, "gap": round(gap, 4), "ours": ours, "independent": independent}
|
| 356 |
+
|
| 357 |
+
|
| 358 |
+
def validate(pre=None):
|
| 359 |
+
"""Module-convention validate(): run every declared contract over YTD, all-BU + per-BU."""
|
| 360 |
+
import core.periods as P
|
| 361 |
+
yf, yt = P.ytd()
|
| 362 |
+
out = []
|
| 363 |
+
for key, m in _model()["metrics"].items():
|
| 364 |
+
if not m.get("validate"):
|
| 365 |
+
continue
|
| 366 |
+
out.append(validate_metric(key, yf, yt, None))
|
| 367 |
+
topic = _model()["topics"][m["topic"]]
|
| 368 |
+
if (topic.get("store") or {}).get("team_col"): # company-level topics have no BU runs
|
| 369 |
+
for tid in O.TEAM_IDS:
|
| 370 |
+
out.append(validate_metric(key, yf, yt, tid))
|
| 371 |
+
return out
|
| 372 |
+
|
| 373 |
+
|
| 374 |
+
# ---------------------------------------------------------------- store path (OM-2)
|
| 375 |
+
# The compiler behind run_semantic_query: registered metrics + whitelisted dims β DuckDB SQL over
|
| 376 |
+
# the tenant store (harness/datastore.py). EVERY identifier comes from the versioned model files
|
| 377 |
+
# (trusted); every VALUE is parameterized β the small model only ever picks keys, so there is no
|
| 378 |
+
# injection surface. Live XML-RPC stays the validate() path (the store never validates itself).
|
| 379 |
+
|
| 380 |
+
def _store_con():
|
| 381 |
+
import harness.datastore as DS
|
| 382 |
+
if not DS.ready(): # a mid-backfill store returns PARTIAL totals silently
|
| 383 |
+
raise ModelError("the data cache is still warming up after a restart (a one-time first "
|
| 384 |
+
"sync) β the dashboards work now from live data; ask me again in a "
|
| 385 |
+
"moment and I'll have it")
|
| 386 |
+
return DS.ro_cursor() # cursor on the shared instance β no sync-writer contention
|
| 387 |
+
|
| 388 |
+
|
| 389 |
+
def _measure_sql(m, alias):
|
| 390 |
+
"""SQL for a base measure. Supports FILTERED measures (`store_filter_sql` β the Omni
|
| 391 |
+
filtered-measure concept: sum(CASE WHEN β¦ )) and `negate: true` (e.g. GL income, stored
|
| 392 |
+
credit-negative, reported positive)."""
|
| 393 |
+
agg, f = m.get("agg"), m.get("field")
|
| 394 |
+
flt = render_scope(m["store_filter_sql"]) if m.get("store_filter_sql") else None
|
| 395 |
+
if agg == "sum":
|
| 396 |
+
core = f"sum(CASE WHEN {flt} THEN {alias}.{f} ELSE 0 END)" if flt else f"sum({alias}.{f})"
|
| 397 |
+
elif agg == "count_distinct":
|
| 398 |
+
core = (f"count(DISTINCT CASE WHEN {flt} THEN {alias}.{f} END)" if flt
|
| 399 |
+
else f"count(DISTINCT {alias}.{f})")
|
| 400 |
+
elif agg == "count":
|
| 401 |
+
core = f"count(CASE WHEN {flt} THEN 1 END)" if flt else "count(*)"
|
| 402 |
+
else:
|
| 403 |
+
raise ModelError(f"metric {m['key']!r}: agg {agg!r} has no store expression")
|
| 404 |
+
return f"-({core})" if m.get("negate") else core
|
| 405 |
+
|
| 406 |
+
|
| 407 |
+
def _expand_measures(keys):
|
| 408 |
+
"""Expand ratio/derived metrics into the base measures SQL must aggregate, keeping the
|
| 409 |
+
requested keys for post-compute. Returns (base_keys, requested_keys)."""
|
| 410 |
+
mts = _model()["metrics"]
|
| 411 |
+
base, seen = [], set()
|
| 412 |
+
|
| 413 |
+
def need(k):
|
| 414 |
+
if k in seen:
|
| 415 |
+
return
|
| 416 |
+
seen.add(k)
|
| 417 |
+
m = mts.get(k)
|
| 418 |
+
if not m:
|
| 419 |
+
raise ModelError(f"unknown metric {k!r}")
|
| 420 |
+
if m["agg"] == "ratio":
|
| 421 |
+
need(m["numerator"]); need(m["denominator"])
|
| 422 |
+
elif m["agg"] == "derived":
|
| 423 |
+
import ast as _ast
|
| 424 |
+
for node in _ast.walk(_ast.parse(m["expr"], mode="eval")):
|
| 425 |
+
if isinstance(node, _ast.Name):
|
| 426 |
+
need(node.id)
|
| 427 |
+
else:
|
| 428 |
+
if k not in base:
|
| 429 |
+
base.append(k)
|
| 430 |
+
for k in keys:
|
| 431 |
+
need(k)
|
| 432 |
+
return base, list(keys)
|
| 433 |
+
|
| 434 |
+
|
| 435 |
+
def _post_compute(row, key):
|
| 436 |
+
m = _model()["metrics"][key]
|
| 437 |
+
if m["agg"] == "ratio":
|
| 438 |
+
num, den = _post_compute(row, m["numerator"]), _post_compute(row, m["denominator"])
|
| 439 |
+
return (num / den) if den else 0.0
|
| 440 |
+
if m["agg"] == "derived":
|
| 441 |
+
return _eval_expr(m["expr"], lambda k: _post_compute(row, k))
|
| 442 |
+
return row.get(key) or 0.0
|
| 443 |
+
|
| 444 |
+
|
| 445 |
+
# A metric's declared `format` -> the grid field type the compiler and the cells both read.
|
| 446 |
+
# One mapping, so a new metric cannot arrive as an untyped column.
|
| 447 |
+
_FORMAT_TYPE = {"usd": "currency", "int": "int", "pct": "pct"}
|
| 448 |
+
|
| 449 |
+
|
| 450 |
+
def _clean_tree(tree, cols):
|
| 451 |
+
"""Run a filter tree through the SAME validator the client's view state goes through.
|
| 452 |
+
|
| 453 |
+
`filter_sql`'s documented input contract is "feed me CLEANED trees" β it is faithful to the
|
| 454 |
+
TS engine rather than fail-closed on garbage, because rejecting garbage is the validator's
|
| 455 |
+
job. The UI path always cleans (view state is sanitised on the way into the store), but
|
| 456 |
+
store_query/store_rows are callable directly, and an uncleaned tree is where the two
|
| 457 |
+
diverge: `{"value": null}` is ACTIVE in TS (and throws on a text field) and INACTIVE here.
|
| 458 |
+
|
| 459 |
+
Normalising at the entry point makes that unreachable rather than merely unlikely, and
|
| 460 |
+
brings the depth / width / node-count caps to the server side too. `aios_grid` is a leaf
|
| 461 |
+
module with zero imports of its own, so this cannot cycle.
|
| 462 |
+
"""
|
| 463 |
+
from aios_grid import (clean_filter_tree, COHORT_FIELD as _COHORT_FIELD,
|
| 464 |
+
MAX_FILTER_DEPTH, MAX_FILTER_NODES, MAX_FILTER_SIBLINGS)
|
| 465 |
+
|
| 466 |
+
# REFUSE an over-large tree rather than TRUNCATE it. clean_filter_tree caps siblings per
|
| 467 |
+
# level, total nodes and depth by DROPPING the excess β correct on the client, where the
|
| 468 |
+
# cap is a DoS guard on a per-row render loop and a dropped condition is the lesser evil.
|
| 469 |
+
# On the server it is not: dropping AND-conditions WIDENS the result, and store_rows then
|
| 470 |
+
# reports a row_count and scope-wide totals that look authoritative for a query the caller
|
| 471 |
+
# never asked for. That is precisely the silent [:N] that [[no-unverifiable-aggregates]]
|
| 472 |
+
# forbids. A caller that exceeds the caps gets an error, not a quietly different answer.
|
| 473 |
+
total = 0
|
| 474 |
+
|
| 475 |
+
def _walk(nodes, depth):
|
| 476 |
+
nonlocal total
|
| 477 |
+
nodes = list(nodes or [])
|
| 478 |
+
if len(nodes) > MAX_FILTER_SIBLINGS:
|
| 479 |
+
raise ModelError(f"filter tree has {len(nodes)} conditions at one level; the limit "
|
| 480 |
+
f"is {MAX_FILTER_SIBLINGS}. Refusing rather than dropping the "
|
| 481 |
+
f"excess, which would silently widen the result.")
|
| 482 |
+
for node in nodes:
|
| 483 |
+
if not isinstance(node, dict):
|
| 484 |
+
continue
|
| 485 |
+
total += 1
|
| 486 |
+
if isinstance(node.get('children'), list):
|
| 487 |
+
if depth >= MAX_FILTER_DEPTH:
|
| 488 |
+
raise ModelError(f"filter tree nests deeper than {MAX_FILTER_DEPTH} levels; "
|
| 489 |
+
f"refusing rather than dropping the deepest group.")
|
| 490 |
+
_walk(node['children'], depth + 1)
|
| 491 |
+
|
| 492 |
+
_walk(tree, 1)
|
| 493 |
+
if total > MAX_FILTER_NODES:
|
| 494 |
+
raise ModelError(f"filter tree has {total} nodes; the limit is {MAX_FILTER_NODES}. "
|
| 495 |
+
f"Refusing rather than truncating, which would silently widen it.")
|
| 496 |
+
|
| 497 |
+
# CG-8. A MEASURE condition ("Sales in the last 90 days > 5,000") is answered by
|
| 498 |
+
# harness/measure_filter.py, which resolves it to a SET OF CUSTOMER IDS. It has no meaning
|
| 499 |
+
# on this path, and both ways it could arrive here are silent:
|
| 500 |
+
# - on a topic without that column, `clean_filter_tree` DROPS it as unknown β widening the
|
| 501 |
+
# query while store_rows reports an authoritative row_count for something nobody asked
|
| 502 |
+
# for (the same class as the sibling/depth caps above);
|
| 503 |
+
# - on sales_lines at row grain, `revenue` IS a real column, so it would compile as a
|
| 504 |
+
# per-LINE predicate with the window silently discarded β a different question answered
|
| 505 |
+
# confidently, which is worse than dropping it.
|
| 506 |
+
# The discriminator is the `window`, not the column name: no column condition has one.
|
| 507 |
+
windowed = []
|
| 508 |
+
cohorts = []
|
| 509 |
+
|
| 510 |
+
def _find(nodes):
|
| 511 |
+
for node in nodes or []:
|
| 512 |
+
if not isinstance(node, dict):
|
| 513 |
+
continue
|
| 514 |
+
if isinstance(node.get('children'), list):
|
| 515 |
+
_find(node['children'])
|
| 516 |
+
elif node.get('window') is not None:
|
| 517 |
+
windowed.append(str(node.get('colId')))
|
| 518 |
+
elif node.get('colId') == _COHORT_FIELD:
|
| 519 |
+
cohorts.append(str(node.get('value')))
|
| 520 |
+
|
| 521 |
+
_find(tree)
|
| 522 |
+
# Owner item 5, and the same refusal for the same reason. A cohort leaf names a
|
| 523 |
+
# hand-curated set of CUSTOMERS held per user in the tenant store β this path knows nothing
|
| 524 |
+
# about it, and `clean_filter_tree` below would drop it as an unknown key, widening the
|
| 525 |
+
# query while store_rows still reported an authoritative row_count. If a caller ever needs
|
| 526 |
+
# it here, the fix is to pass the membership in, not to let it fall through.
|
| 527 |
+
if cohorts:
|
| 528 |
+
raise ModelError(
|
| 529 |
+
f"filter tree carries cohort-membership condition(s) {sorted(set(cohorts))} β "
|
| 530 |
+
f"cohort membership is host state, not a column of this topic. Refusing rather than "
|
| 531 |
+
f"dropping it, which would silently widen the result under an authoritative count.")
|
| 532 |
+
if windowed:
|
| 533 |
+
raise ModelError(
|
| 534 |
+
f"filter tree carries measure condition(s) over a date window {sorted(set(windowed))}"
|
| 535 |
+
f" β those are resolved to a set of ids by harness.measure_filter, not compiled into "
|
| 536 |
+
f"this query. Refusing rather than dropping or mis-compiling them, either of which "
|
| 537 |
+
f"would silently answer a different question under an authoritative count.")
|
| 538 |
+
return clean_filter_tree(tree, set(cols))
|
| 539 |
+
|
| 540 |
+
|
| 541 |
+
def _measure_row_sql(m, alias):
|
| 542 |
+
"""A sum-metric's per-ROW value: the same expression `_measure_sql` aggregates, without the
|
| 543 |
+
aggregate wrapper. At line grain `revenue` IS `l.price_subtotal` for that line β deriving it
|
| 544 |
+
from the metric rather than hand-writing the column is what keeps ONE definition of the
|
| 545 |
+
number (the drift the semantic layer exists to prevent). Only `sum` has a row value; a
|
| 546 |
+
count/count_distinct metric is a property of a SET, not of a row."""
|
| 547 |
+
if m.get("agg") != "sum":
|
| 548 |
+
return None
|
| 549 |
+
f = m.get("field")
|
| 550 |
+
flt = render_scope(m["store_filter_sql"]) if m.get("store_filter_sql") else None
|
| 551 |
+
core = f"CASE WHEN {flt} THEN {alias}.{f} ELSE 0 END" if flt else f"{alias}.{f}"
|
| 552 |
+
return f"-({core})" if m.get("negate") else core
|
| 553 |
+
|
| 554 |
+
|
| 555 |
+
def store_columns(topic, include_measures=True, grain="aggregate"):
|
| 556 |
+
"""The `{colId: {sql, type, aggregate}}` spec `harness.filter_sql` compiles against (CG-1).
|
| 557 |
+
|
| 558 |
+
`grain="row"` is the LINE-grain projection (CG-2): sum-metrics resolve to their own raw
|
| 559 |
+
field with `aggregate=False`, so a line table filters and sorts on real row values in WHERE
|
| 560 |
+
rather than needing HAVING. count-style metrics are dropped β they have no row value.
|
| 561 |
+
|
| 562 |
+
This is the seam between the grid's field contract and the semantic model: the grid speaks
|
| 563 |
+
field KEYS, the compiler needs SQL expressions and a field TYPE, and only the model may say
|
| 564 |
+
what a key resolves to. Nothing about a topic leaks into filter_sql itself.
|
| 565 |
+
|
| 566 |
+
<dim> -> the dim's NAME column (what the grid displays and what a user means when
|
| 567 |
+
they type "contains floral"), filtered PRE-aggregation in WHERE. Correct
|
| 568 |
+
even in a grouped query: every row of a group shares its group's name.
|
| 569 |
+
<dim>_id -> the raw id, for exact machine-keyed filtering.
|
| 570 |
+
date -> the topic's date column, so a grid can filter/sort it like any other field.
|
| 571 |
+
<metric> -> the aggregate expression, marked `aggregate` so the caller routes it to
|
| 572 |
+
HAVING rather than WHERE (see store_query β that path is not built yet).
|
| 573 |
+
"""
|
| 574 |
+
t = _model()["topics"].get(topic)
|
| 575 |
+
if not t or "store" not in t:
|
| 576 |
+
raise ModelError(f"topic {topic!r} has no store binding")
|
| 577 |
+
s = t["store"]
|
| 578 |
+
cols = {}
|
| 579 |
+
for key, d in (s.get("dims") or {}).items():
|
| 580 |
+
cols[key] = {"sql": d.get("name_col") or d["col"], "type": "text", "aggregate": False}
|
| 581 |
+
# Only expose a separate id column when the dim actually HAS one. `city` is its own
|
| 582 |
+
# name (col == name_col), so a `city_id` would be a text column typed int β filtering
|
| 583 |
+
# it numerically would quietly compare 0 against 0 for every row.
|
| 584 |
+
if d.get("name_col") and d["name_col"] != d["col"]:
|
| 585 |
+
cols[f"{key}_id"] = {"sql": d["col"], "type": "int", "aggregate": False}
|
| 586 |
+
if s.get("date_col"):
|
| 587 |
+
cols["date"] = {"sql": s["date_col"], "type": "date", "aggregate": False}
|
| 588 |
+
if include_measures:
|
| 589 |
+
for k, m in _model()["metrics"].items():
|
| 590 |
+
if m.get("topic") != topic or m.get("agg") in ("ratio", "derived"):
|
| 591 |
+
continue # ratio/derived are post-computed, not SQL
|
| 592 |
+
# The field TYPE comes from the metric's declared format, never a guess: `units` is
|
| 593 |
+
# format:int, and typing it currency would render 791,960 units as dollars.
|
| 594 |
+
ftype = _FORMAT_TYPE.get(m.get("format"), "currency")
|
| 595 |
+
if grain == "row":
|
| 596 |
+
row_sql = _measure_row_sql(m, s["alias"])
|
| 597 |
+
if row_sql:
|
| 598 |
+
cols[k] = {"sql": row_sql, "type": ftype, "aggregate": False}
|
| 599 |
+
else:
|
| 600 |
+
cols[k] = {"sql": _measure_sql(m, s["alias"]),
|
| 601 |
+
"type": ftype, "aggregate": True}
|
| 602 |
+
return cols
|
| 603 |
+
|
| 604 |
+
|
| 605 |
+
def store_query(topic, measures, group_by=None, grain=None, date_from=None, date_to=None,
|
| 606 |
+
team_id=None, filters=None, sort=None, limit=1000, exclude_services=False,
|
| 607 |
+
filter_tree=None, filter_conj="and", today=None):
|
| 608 |
+
"""The semantic query over the tenant store β the engine behind the Analyst's
|
| 609 |
+
run_semantic_query tool and the saved-view re-runner. All keys whitelisted against the model."""
|
| 610 |
+
t = _model()["topics"].get(topic)
|
| 611 |
+
if not t or "store" not in t:
|
| 612 |
+
raise ModelError(f"topic {topic!r} has no store binding")
|
| 613 |
+
s = t["store"]
|
| 614 |
+
alias = s["alias"]
|
| 615 |
+
dims = s.get("dims") or {}
|
| 616 |
+
limit = max(1, min(int(limit or 1000), 5000))
|
| 617 |
+
|
| 618 |
+
base_keys, requested = _expand_measures(list(measures or []))
|
| 619 |
+
if not base_keys:
|
| 620 |
+
raise ModelError("at least one measure required")
|
| 621 |
+
# Cross-topic base metrics (e.g. aov = revenue Γ· orders, where orders lives on sales_orders):
|
| 622 |
+
# in a SCALAR query they resolve via a nested scalar query on their HOME topic (correct grain β
|
| 623 |
+
# never count(*) on the wrong table); in a GROUPED query they are refused (split the request).
|
| 624 |
+
foreign = [k for k in base_keys if _model()["metrics"][k]["topic"] != topic]
|
| 625 |
+
if foreign and (group_by or grain):
|
| 626 |
+
raise ModelError(f"metrics {foreign} belong to another topic β cross-topic measures are "
|
| 627 |
+
f"scalar-only; run them against their own topic when grouping")
|
| 628 |
+
base_keys = [k for k in base_keys if k not in foreign]
|
| 629 |
+
foreign_vals = {}
|
| 630 |
+
for k in foreign:
|
| 631 |
+
sub = store_query(_model()["metrics"][k]["topic"], [k], date_from=date_from,
|
| 632 |
+
date_to=date_to, team_id=team_id)
|
| 633 |
+
foreign_vals[k] = sub["rows"][0][k] if sub["rows"] else 0
|
| 634 |
+
if not base_keys: # purely foreign scalar request
|
| 635 |
+
return {"topic": topic, "rows": [foreign_vals], "row_count": 1, "sql": "(nested)",
|
| 636 |
+
"measures": requested, "group_by": [], "grain": None}
|
| 637 |
+
|
| 638 |
+
select, group_cols, params = [], [], []
|
| 639 |
+
gb = [g for g in (group_by or []) if g]
|
| 640 |
+
for g in gb:
|
| 641 |
+
if g not in dims:
|
| 642 |
+
raise ModelError(f"group_by {g!r} not a dim of {topic!r} (allowed: {list(dims)})")
|
| 643 |
+
d = dims[g]
|
| 644 |
+
select.append(f"{d['col']} AS {g}_id" if d.get("name_col") else f"{d['col']} AS {g}")
|
| 645 |
+
group_cols.append(d["col"])
|
| 646 |
+
if d.get("name_col"):
|
| 647 |
+
select.append(f"max({d['name_col']}) AS {g}")
|
| 648 |
+
if grain:
|
| 649 |
+
if grain not in ("month", "week", "day"):
|
| 650 |
+
raise ModelError("grain must be month|week|day")
|
| 651 |
+
select.insert(0, f"date_trunc('{grain}', CAST({s['date_col']} AS TIMESTAMP)) AS period")
|
| 652 |
+
group_cols.insert(0, f"date_trunc('{grain}', CAST({s['date_col']} AS TIMESTAMP))")
|
| 653 |
+
for k in base_keys:
|
| 654 |
+
select.append(f"{_measure_sql(_model()['metrics'][k], alias)} AS {k}")
|
| 655 |
+
|
| 656 |
+
where = [render_scope(s["scope_sql"].strip())]
|
| 657 |
+
# TYPE-CORRECT date bounds (the 2025-07-01 lesson): columns hold ISO strings in TWO shapes β
|
| 658 |
+
# datetimes ('YYYY-MM-DD HH:MM:SS', sale date_order) and bare dates ('YYYY-MM-DD', GL date).
|
| 659 |
+
# Lexical string comparison EXCLUDES the window's first day for bare dates ('2025-07-01' <
|
| 660 |
+
# '2025-07-01 00:00:00'), which silently dropped a whole day of GL ($49,467). Cast both sides.
|
| 661 |
+
if date_from:
|
| 662 |
+
where.append(f"CAST({s['date_col']} AS TIMESTAMP) >= ?")
|
| 663 |
+
params.append(f"{date_from} 00:00:00")
|
| 664 |
+
if date_to:
|
| 665 |
+
where.append(f"CAST({s['date_col']} AS TIMESTAMP) <= ?")
|
| 666 |
+
params.append(f"{date_to} 23:59:59")
|
| 667 |
+
if team_id:
|
| 668 |
+
if not s.get("team_col"):
|
| 669 |
+
raise ModelError(f"topic {topic!r} is company-level β it has no business-unit filter")
|
| 670 |
+
where.append(f"{s['team_col']} = ?"); params.append(int(team_id))
|
| 671 |
+
if exclude_services and s.get("service_filter"):
|
| 672 |
+
where.append(render_scope(s["service_filter"]))
|
| 673 |
+
for dim_key, vals in (filters or {}).items():
|
| 674 |
+
if dim_key not in dims:
|
| 675 |
+
raise ModelError(f"filter dim {dim_key!r} not allowed (use: {list(dims)})")
|
| 676 |
+
# id dims filter by int; TEXT dims (e.g. city) filter by the value itself β either way
|
| 677 |
+
# every VALUE stays parameterized (the whitelist covers identifiers only)
|
| 678 |
+
def _coerce(v):
|
| 679 |
+
try:
|
| 680 |
+
return int(v)
|
| 681 |
+
except (TypeError, ValueError):
|
| 682 |
+
return str(v)
|
| 683 |
+
ids = [_coerce(v) for v in (vals if isinstance(vals, (list, tuple)) else [vals])]
|
| 684 |
+
where.append(f"{dims[dim_key]['col']} IN ({','.join('?' for _ in ids)})")
|
| 685 |
+
params.extend(ids)
|
| 686 |
+
|
| 687 |
+
# The grid's filter TREE (CG-1). `filters` above stays the Analyst's flat dim=IN shape;
|
| 688 |
+
# this is the nested, 11-operator contract the table UI emits. Compiled by
|
| 689 |
+
# harness/filter_sql, which is held in lock-step with the TS engine and the validator by
|
| 690 |
+
# aios-web/verify_filter_engine.py. Appended AFTER the dim filters so params stay in
|
| 691 |
+
# positional order with `where`.
|
| 692 |
+
if filter_tree:
|
| 693 |
+
from harness import filter_sql as _fs
|
| 694 |
+
cols = store_columns(topic)
|
| 695 |
+
pred = _fs.compile_filter_tree(_clean_tree(filter_tree, cols), filter_conj, cols,
|
| 696 |
+
today=today)
|
| 697 |
+
if pred is not None:
|
| 698 |
+
if pred.uses_aggregate:
|
| 699 |
+
bad = sorted(c for c in pred.columns_used if cols[c].get("aggregate"))
|
| 700 |
+
raise ModelError(
|
| 701 |
+
f"filter_tree references measure(s) {bad} β a measure filter has to land in "
|
| 702 |
+
f"HAVING, and that path is deliberately NOT built: CG-1 ships the WHERE path "
|
| 703 |
+
f"because its consumer (CG-2, the line-grain sales table) does not aggregate. "
|
| 704 |
+
f"Filter on dims, or aggregate first and filter the result.")
|
| 705 |
+
where.append(pred.sql)
|
| 706 |
+
params.extend(pred.params)
|
| 707 |
+
|
| 708 |
+
sql = (f"SELECT {', '.join(select)} FROM {s['table']} {alias} {s.get('join','')} "
|
| 709 |
+
f"WHERE {' AND '.join(where)}")
|
| 710 |
+
if group_cols:
|
| 711 |
+
sql += f" GROUP BY {', '.join(group_cols)}"
|
| 712 |
+
if sort:
|
| 713 |
+
key = sort.lstrip("-")
|
| 714 |
+
if key not in base_keys + gb + (["period"] if grain else []):
|
| 715 |
+
raise ModelError(f"sort {sort!r} must reference a selected measure/dim")
|
| 716 |
+
sql += f" ORDER BY {key} {'DESC' if sort.startswith('-') else 'ASC'}"
|
| 717 |
+
elif grain:
|
| 718 |
+
sql += " ORDER BY period"
|
| 719 |
+
sql += f" LIMIT {limit}"
|
| 720 |
+
|
| 721 |
+
con = _store_con()
|
| 722 |
+
try:
|
| 723 |
+
cur = con.execute(sql, params)
|
| 724 |
+
cols = [d[0] for d in cur.description]
|
| 725 |
+
rows = [dict(zip(cols, r)) for r in cur.fetchall()]
|
| 726 |
+
finally:
|
| 727 |
+
con.close()
|
| 728 |
+
for row in rows: # ratio/derived post-compute per group
|
| 729 |
+
row.update(foreign_vals) # scalar cross-topic components
|
| 730 |
+
for k in requested:
|
| 731 |
+
if _model()["metrics"][k]["agg"] in ("ratio", "derived"):
|
| 732 |
+
row[k] = _post_compute(row, k)
|
| 733 |
+
if "period" in row and row["period"] is not None:
|
| 734 |
+
row["period"] = str(row["period"])[:10]
|
| 735 |
+
return {"topic": topic, "rows": rows, "row_count": len(rows), "sql": sql,
|
| 736 |
+
"measures": requested, "group_by": gb, "grain": grain}
|
| 737 |
+
|
| 738 |
+
|
| 739 |
+
def store_rows(topic, date_from=None, date_to=None, team_id=None, filter_tree=None,
|
| 740 |
+
filter_conj="and", search=None, sorts=None, limit=200, offset=0,
|
| 741 |
+
aggregates=None, exclude_services=False, id_sql=None, member_ids=None,
|
| 742 |
+
today=None):
|
| 743 |
+
"""ROW-grain fetch β the line-grain counterpart to store_query's aggregate path (CG-2).
|
| 744 |
+
|
| 745 |
+
store_query answers "what is the number"; this answers "which rows". At line grain the
|
| 746 |
+
payload is the binding constraint (254,837 sale_order_lines against 1,550 customers β the
|
| 747 |
+
whole book would be ~96 MB at the customer table's measured 377 B/row), so rows come back
|
| 748 |
+
WINDOWED.
|
| 749 |
+
|
| 750 |
+
A window is only honest if the counts and totals are not windowed with it
|
| 751 |
+
([[no-unverifiable-aggregates]]). So this returns THREE independent numbers, each from its
|
| 752 |
+
own query over the FULL scope, never from `len(rows)`:
|
| 753 |
+
|
| 754 |
+
row_count rows matching the filter across the whole scope -> the N of "N of M"
|
| 755 |
+
total_count rows in the scope with no filter at all -> the M
|
| 756 |
+
aggregates the topic's OWN metric definitions, summed over the whole FILTERED scope
|
| 757 |
+
|
| 758 |
+
That is the difference between a windowed fetch and a silent `[:N]` cap: the user is told
|
| 759 |
+
how many rows exist and shown totals for all of them, while receiving only the page they
|
| 760 |
+
can see.
|
| 761 |
+
|
| 762 |
+
Filters/sorts/search compile through harness.filter_sql, so a line table means EXACTLY what
|
| 763 |
+
the client engine means at customer grain β that equivalence is what verify_filter_engine.py
|
| 764 |
+
exists to hold.
|
| 765 |
+
|
| 766 |
+
Dates are returned already truncated to the 10-char ISO shape the filter compares against,
|
| 767 |
+
so what is displayed and what is filtered are the same string. Numerics come back RAW; the
|
| 768 |
+
payload layer rounds them (aios_grid._round), and the compiler mirrors that rounding.
|
| 769 |
+
"""
|
| 770 |
+
t = _model()["topics"].get(topic)
|
| 771 |
+
if not t or "store" not in t:
|
| 772 |
+
raise ModelError(f"topic {topic!r} has no store binding")
|
| 773 |
+
from harness import filter_sql as _fs
|
| 774 |
+
s = t["store"]
|
| 775 |
+
alias = s["alias"]
|
| 776 |
+
cols = store_columns(topic, grain="row")
|
| 777 |
+
id_sql = id_sql or f"{alias}.id"
|
| 778 |
+
limit = max(1, min(int(limit or 200), 5000))
|
| 779 |
+
offset = max(0, int(offset or 0))
|
| 780 |
+
|
| 781 |
+
# --- the scope every one of the three queries shares -------------------------------
|
| 782 |
+
scope, scope_params = [render_scope(s["scope_sql"].strip())], []
|
| 783 |
+
if date_from:
|
| 784 |
+
scope.append(f"CAST({s['date_col']} AS TIMESTAMP) >= ?")
|
| 785 |
+
scope_params.append(f"{date_from} 00:00:00")
|
| 786 |
+
if date_to:
|
| 787 |
+
scope.append(f"CAST({s['date_col']} AS TIMESTAMP) <= ?")
|
| 788 |
+
scope_params.append(f"{date_to} 23:59:59")
|
| 789 |
+
if team_id:
|
| 790 |
+
if not s.get("team_col"):
|
| 791 |
+
raise ModelError(f"topic {topic!r} is company-level β it has no business-unit filter")
|
| 792 |
+
scope.append(f"{s['team_col']} = ?")
|
| 793 |
+
scope_params.append(int(team_id))
|
| 794 |
+
if exclude_services and s.get("service_filter"):
|
| 795 |
+
scope.append(render_scope(s["service_filter"]))
|
| 796 |
+
|
| 797 |
+
# --- the user's narrowing, on top of the scope --------------------------------------
|
| 798 |
+
narrow, narrow_params = [], []
|
| 799 |
+
pred = _fs.compile_filter_tree(_clean_tree(filter_tree or [], cols), filter_conj, cols,
|
| 800 |
+
today=today,
|
| 801 |
+
member_ids=member_ids or None, id_sql=id_sql)
|
| 802 |
+
if pred is not None:
|
| 803 |
+
if pred.uses_aggregate: # cannot happen at grain="row"; guard anyway
|
| 804 |
+
raise ModelError(f"row-grain filter referenced an aggregate: {sorted(pred.columns_used)}")
|
| 805 |
+
narrow.append(pred.sql)
|
| 806 |
+
narrow_params.extend(pred.params)
|
| 807 |
+
spred = _fs.compile_search(search, cols)
|
| 808 |
+
if spred is not None:
|
| 809 |
+
narrow.append(spred.sql)
|
| 810 |
+
narrow_params.extend(spred.params)
|
| 811 |
+
|
| 812 |
+
frm = f"{s['table']} {alias} {s.get('join', '')}"
|
| 813 |
+
scope_where = " AND ".join(scope)
|
| 814 |
+
all_where = " AND ".join(scope + narrow)
|
| 815 |
+
con = _store_con()
|
| 816 |
+
try:
|
| 817 |
+
# 1. the WINDOW of rows
|
| 818 |
+
select = [f"{id_sql} AS _rid"] + [
|
| 819 |
+
(f"SUBSTR(CAST({c['sql']} AS VARCHAR), 1, 10) AS {k}" if c["type"] == "date"
|
| 820 |
+
else f"{c['sql']} AS {k}")
|
| 821 |
+
for k, c in cols.items()]
|
| 822 |
+
order = _fs.compile_order_by(sorts, cols, tiebreak_sql=id_sql) or f"{id_sql} ASC"
|
| 823 |
+
sql = (f"SELECT {', '.join(select)} FROM {frm} WHERE {all_where} "
|
| 824 |
+
f"ORDER BY {order} LIMIT {limit} OFFSET {offset}")
|
| 825 |
+
cur = con.execute(sql, scope_params + narrow_params)
|
| 826 |
+
names = [d[0] for d in cur.description]
|
| 827 |
+
rows = [dict(zip(names, r)) for r in cur.fetchall()]
|
| 828 |
+
|
| 829 |
+
# 2. the two counts β over the WHOLE scope, never len(rows)
|
| 830 |
+
count_sql = f"SELECT count(*) FROM {frm} WHERE {all_where}"
|
| 831 |
+
row_count = con.execute(count_sql, scope_params + narrow_params).fetchone()[0]
|
| 832 |
+
total_count = con.execute(f"SELECT count(*) FROM {frm} WHERE {scope_where}",
|
| 833 |
+
scope_params).fetchone()[0]
|
| 834 |
+
|
| 835 |
+
# 3. the aggregates β the topic's OWN metric definitions over the whole FILTERED scope
|
| 836 |
+
mts = _model()["metrics"]
|
| 837 |
+
keys = [k for k in (aggregates if aggregates is not None
|
| 838 |
+
else [k for k, m in mts.items()
|
| 839 |
+
if m.get("topic") == topic and m.get("agg") == "sum"])]
|
| 840 |
+
aggs = {}
|
| 841 |
+
if keys:
|
| 842 |
+
for k in keys:
|
| 843 |
+
if k not in mts or mts[k].get("topic") != topic:
|
| 844 |
+
raise ModelError(f"aggregate {k!r} is not a metric of topic {topic!r}")
|
| 845 |
+
agg_sql = ("SELECT " + ", ".join(f"{_measure_sql(mts[k], alias)} AS {k}" for k in keys)
|
| 846 |
+
+ f" FROM {frm} WHERE {all_where}")
|
| 847 |
+
cur = con.execute(agg_sql, scope_params + narrow_params)
|
| 848 |
+
aggs = dict(zip([d[0] for d in cur.description], cur.fetchone()))
|
| 849 |
+
finally:
|
| 850 |
+
con.close()
|
| 851 |
+
|
| 852 |
+
return {
|
| 853 |
+
"topic": topic,
|
| 854 |
+
"rows": rows,
|
| 855 |
+
"columns": {k: {"type": c["type"]} for k, c in cols.items()},
|
| 856 |
+
"window": {"offset": offset, "limit": limit, "returned": len(rows)},
|
| 857 |
+
"row_count": row_count, # N β matches the filter, across the WHOLE scope
|
| 858 |
+
"total_count": total_count, # M β the unfiltered scope
|
| 859 |
+
"aggregates": aggs, # over the whole FILTERED scope, not the window
|
| 860 |
+
"sql": sql,
|
| 861 |
+
"count_sql": count_sql,
|
| 862 |
+
}
|
| 863 |
+
|
| 864 |
+
|
| 865 |
+
def store_field_values(topic, dim, search=None, limit=25):
|
| 866 |
+
"""Resolve real filterable values for a dim (the Analyst's get_field_values tool β fixes
|
| 867 |
+
'NYC Florist' vs the actual record name BEFORE querying)."""
|
| 868 |
+
t = _model()["topics"].get(topic)
|
| 869 |
+
if not t or "store" not in t:
|
| 870 |
+
raise ModelError(f"topic {topic!r} has no store binding")
|
| 871 |
+
d = (t["store"].get("dims") or {}).get(dim)
|
| 872 |
+
if not d:
|
| 873 |
+
raise ModelError(f"unknown dim {dim!r} for topic {topic!r}")
|
| 874 |
+
if not d.get("name_col"):
|
| 875 |
+
raise ModelError(f"dim {dim!r} has no name column (filter by id)")
|
| 876 |
+
s = t["store"]
|
| 877 |
+
sql = (f"SELECT DISTINCT {d['col']} AS id, {d['name_col']} AS name "
|
| 878 |
+
f"FROM {s['table']} {s['alias']} {s.get('join','')} "
|
| 879 |
+
f"WHERE {render_scope(s['scope_sql'].strip())}")
|
| 880 |
+
params = []
|
| 881 |
+
if search:
|
| 882 |
+
sql += f" AND {d['name_col']} ILIKE ?"
|
| 883 |
+
params.append(f"%{search}%")
|
| 884 |
+
sql += f" ORDER BY name LIMIT {max(1, min(int(limit), 100))}"
|
| 885 |
+
con = _store_con()
|
| 886 |
+
try:
|
| 887 |
+
return [{"id": r[0], "name": r[1]} for r in con.execute(sql, params).fetchall()]
|
| 888 |
+
finally:
|
| 889 |
+
con.close()
|
| 890 |
+
|
| 891 |
+
|
| 892 |
+
def store_parity(date_from=None, date_to=None):
|
| 893 |
+
"""THE OM-2 gate: the store path must equal the live path to the cent, per metric per scope."""
|
| 894 |
+
import core.periods as P
|
| 895 |
+
if not date_from:
|
| 896 |
+
date_from, date_to = P.ytd()
|
| 897 |
+
out = []
|
| 898 |
+
for key in ("revenue", "margin", "units", "orders", "customers", "returns", "invoiced"):
|
| 899 |
+
m = _model()["metrics"][key]
|
| 900 |
+
# β A COMPANY-LEVEL topic is checked at company scope ONLY. `store_query` RAISES when a
|
| 901 |
+
# team is asked of a topic with no team_col, and the live builder refuses too β so
|
| 902 |
+
# looping the BUs here would fail the gate for a topic that is correct. Parity for a
|
| 903 |
+
# company-level number means company-level parity; pretending otherwise would either
|
| 904 |
+
# crash or, worse, compare two numbers that silently ignored the team.
|
| 905 |
+
_store = (_model()["topics"].get(m["topic"]) or {}).get("store") or {}
|
| 906 |
+
scopes = (None, *O.TEAM_IDS) if _store.get("team_col") else (None,)
|
| 907 |
+
for tid in scopes:
|
| 908 |
+
live = metric(key, date_from, date_to, tid)
|
| 909 |
+
res = store_query(m["topic"], [key], date_from=date_from, date_to=date_to, team_id=tid)
|
| 910 |
+
ours = res["rows"][0][key] if res["rows"] else 0
|
| 911 |
+
gap = abs((ours or 0) - (live or 0))
|
| 912 |
+
out.append({"check": f"store.{key} == live.{key} (team={tid})",
|
| 913 |
+
"ok": gap <= 0.01, "gap": round(gap, 4), "store": ours, "live": live})
|
| 914 |
+
# The agent dim (2026-07-17): store agent-filtered revenue must equal live revenue over the
|
| 915 |
+
# agent's FIRST-agent book β a fully independent path (live partner read + line-domain sum)
|
| 916 |
+
# against the store's res_partner.agent_id attribution. Largest book = the sharpest check.
|
| 917 |
+
recs = O.search_read("res.partner", [("active", "in", [True, False]),
|
| 918 |
+
("agent_ids", "!=", False)], ["agent_ids"], limit=100000)
|
| 919 |
+
books = {}
|
| 920 |
+
for r in recs:
|
| 921 |
+
a = (r.get("agent_ids") or [None])[0]
|
| 922 |
+
if a:
|
| 923 |
+
books.setdefault(a, []).append(r["id"])
|
| 924 |
+
if books:
|
| 925 |
+
top = max(books, key=lambda k: len(books[k]))
|
| 926 |
+
live = O.sum_field("sale.order.line",
|
| 927 |
+
O.sale_line_domain(date_from, date_to, partner_ids=books[top]),
|
| 928 |
+
"price_subtotal")
|
| 929 |
+
res = store_query("sales_lines", ["revenue"], date_from=date_from, date_to=date_to,
|
| 930 |
+
filters={"agent": [top]})
|
| 931 |
+
ours = res["rows"][0]["revenue"] if res["rows"] else 0
|
| 932 |
+
gap = abs((ours or 0) - (live or 0))
|
| 933 |
+
out.append({"check": f"store.revenue[agent={top}] == live book sum (first-agent, "
|
| 934 |
+
f"{len(books[top])} customers)",
|
| 935 |
+
"ok": gap <= 0.01, "gap": round(gap, 4), "store": ours, "live": live})
|
| 936 |
+
return out
|
platform/harness/skills/__init__.py
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
"""Portable skills β written once against the canonical model, run on any compatible connector."""
|
platform/harness/skills/sales.py
ADDED
|
@@ -0,0 +1,168 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Sales skill β PORTABLE. The de-Odoo'd replacement for modules/sales.py: every metric is expressed
|
| 2 |
+
against the canonical model and runs on any connector with the SALES capability (INVOICING for
|
| 3 |
+
returns). No model names, no `amount_untaxed`, no Odoo β the connector translates.
|
| 4 |
+
|
| 5 |
+
`validate(tenant)` parity-checks each metric against the current Odoo module, so a port only replaces
|
| 6 |
+
the original once it ties out to the cent (same discipline as the modules' own validate()).
|
| 7 |
+
|
| 8 |
+
Date windows come from core.periods (calendar math is source-agnostic).
|
| 9 |
+
"""
|
| 10 |
+
from __future__ import annotations
|
| 11 |
+
import sys
|
| 12 |
+
from pathlib import Path
|
| 13 |
+
sys.path.insert(0, str(Path(__file__).resolve().parents[2])) # platform/
|
| 14 |
+
import core.periods as P
|
| 15 |
+
from harness.canonical import Capability, Entity, Query, Filter
|
| 16 |
+
from harness.base import Tenant
|
| 17 |
+
|
| 18 |
+
REQUIRES = (Capability.SALES,)
|
| 19 |
+
|
| 20 |
+
|
| 21 |
+
# ------------------------------------------------------------------ tenant scope -> canonical filters
|
| 22 |
+
def _scope(tenant: Tenant, business_unit: str | None):
|
| 23 |
+
"""The client's scope CONFIG (was hard-coded Odoo constants) as canonical filters β valid on both
|
| 24 |
+
SALES_ORDER and SALES_ORDER_LINE (status/business_unit/customer exist on both)."""
|
| 25 |
+
cfg = tenant.config
|
| 26 |
+
fs = []
|
| 27 |
+
if cfg.get('confirmed_statuses'):
|
| 28 |
+
fs.append(Filter('status', 'in', list(cfg['confirmed_statuses'])))
|
| 29 |
+
bus = cfg.get('business_units') or {}
|
| 30 |
+
if business_unit and business_unit in bus:
|
| 31 |
+
fs.append(Filter('business_unit', 'in', [bus[business_unit]]))
|
| 32 |
+
elif bus:
|
| 33 |
+
fs.append(Filter('business_unit', 'in', list(bus.values())))
|
| 34 |
+
if cfg.get('excluded_customer_ids'):
|
| 35 |
+
fs.append(Filter('customer', 'not_in', list(cfg['excluded_customer_ids'])))
|
| 36 |
+
return tuple(fs)
|
| 37 |
+
|
| 38 |
+
|
| 39 |
+
def _sum(conn, entity, measure, df, dt, filters):
|
| 40 |
+
return sum(r[measure] for r in conn.aggregate(
|
| 41 |
+
Query(entity, (measure,), 'sum', date_from=df, date_to=dt, filters=filters)))
|
| 42 |
+
|
| 43 |
+
|
| 44 |
+
def _count(conn, entity, df, dt, filters):
|
| 45 |
+
return sum(r['count'] for r in conn.aggregate(
|
| 46 |
+
Query(entity, (), 'count', date_from=df, date_to=dt, filters=filters)))
|
| 47 |
+
|
| 48 |
+
|
| 49 |
+
# ------------------------------------------------------------------ metrics (portable)
|
| 50 |
+
def revenue(tenant, date_from, date_to, business_unit=None):
|
| 51 |
+
return _sum(tenant.source(Capability.SALES), Entity.SALES_ORDER, 'amount',
|
| 52 |
+
date_from, date_to, _scope(tenant, business_unit))
|
| 53 |
+
|
| 54 |
+
|
| 55 |
+
def headline(tenant, business_unit=None, t=None):
|
| 56 |
+
conn = tenant.source(Capability.SALES)
|
| 57 |
+
t = t or P.today()
|
| 58 |
+
yf, yt = P.ytd(t)
|
| 59 |
+
lf, lt = P.ytd_last_year(t)
|
| 60 |
+
sc = _scope(tenant, business_unit)
|
| 61 |
+
rev = _sum(conn, Entity.SALES_ORDER, 'amount', yf, yt, sc)
|
| 62 |
+
rev_ly = _sum(conn, Entity.SALES_ORDER, 'amount', lf, lt, sc)
|
| 63 |
+
orders = _count(conn, Entity.SALES_ORDER, yf, yt, sc)
|
| 64 |
+
custs = conn.distinct_count(Query(Entity.SALES_ORDER, date_from=yf, date_to=yt, filters=sc), 'customer')
|
| 65 |
+
return {'ytd_revenue': rev, 'ytd_revenue_ly': rev_ly, 'yoy_pct': P.yoy_pct(rev, rev_ly),
|
| 66 |
+
'ytd_orders': orders, 'ytd_customers': custs, 'aov': (rev / orders) if orders else 0.0}
|
| 67 |
+
|
| 68 |
+
|
| 69 |
+
def monthly_trend(tenant, n=13, business_unit=None, t=None):
|
| 70 |
+
conn = tenant.source(Capability.SALES)
|
| 71 |
+
t = t or P.today()
|
| 72 |
+
first = P.month_starts(n, t)[0][1]
|
| 73 |
+
span_from = f"{int(first[:4]) - 1:04d}{first[4:]}" # a year before the first shown month
|
| 74 |
+
q = Query(Entity.SALES_ORDER, ('amount',), 'sum', grain='month',
|
| 75 |
+
date_from=span_from, date_to=t.isoformat(), filters=_scope(tenant, business_unit))
|
| 76 |
+
m = {r['period']: r['amount'] for r in conn.aggregate(q)}
|
| 77 |
+
rows = []
|
| 78 |
+
for ym, _s, _e in P.month_starts(n, t):
|
| 79 |
+
y, mo = int(ym[:4]) - 1, int(ym[5:7])
|
| 80 |
+
this, last = m.get(ym, 0.0), m.get(f'{y:04d}-{mo:02d}', 0.0)
|
| 81 |
+
rows.append({'month': ym, 'revenue': this, 'revenue_ly': last, 'yoy_pct': P.yoy_pct(this, last)})
|
| 82 |
+
return rows
|
| 83 |
+
|
| 84 |
+
|
| 85 |
+
def top_customers(tenant, limit=25, business_unit=None, t=None):
|
| 86 |
+
conn = tenant.source(Capability.SALES)
|
| 87 |
+
yf, yt = P.ytd(t or P.today())
|
| 88 |
+
q = Query(Entity.SALES_ORDER, ('amount',), 'sum', group_by=('customer',),
|
| 89 |
+
date_from=yf, date_to=yt, filters=_scope(tenant, business_unit))
|
| 90 |
+
rows = [{'pid': r['customer']['id'], 'customer': r['customer']['name'],
|
| 91 |
+
'revenue': r['amount'], 'orders': r['count']}
|
| 92 |
+
for r in conn.aggregate(q) if r.get('customer')]
|
| 93 |
+
return sorted(rows, key=lambda x: -x['revenue'])[:limit]
|
| 94 |
+
|
| 95 |
+
|
| 96 |
+
def top_skus(tenant, limit=25, business_unit=None, t=None):
|
| 97 |
+
conn = tenant.source(Capability.SALES)
|
| 98 |
+
yf, yt = P.ytd(t or P.today())
|
| 99 |
+
q = Query(Entity.SALES_ORDER_LINE, ('amount', 'qty', 'margin'), 'sum', group_by=('product',),
|
| 100 |
+
date_from=yf, date_to=yt, filters=_scope(tenant, business_unit))
|
| 101 |
+
rows = []
|
| 102 |
+
for r in conn.aggregate(q):
|
| 103 |
+
if not r.get('product'):
|
| 104 |
+
continue
|
| 105 |
+
lines = r['count'] or 0
|
| 106 |
+
rows.append({'product': r['product']['name'], 'pid': r['product']['id'], 'revenue': r['amount'],
|
| 107 |
+
'qty': r['qty'], 'margin': r['margin'],
|
| 108 |
+
'profit_per_order': (r['margin'] / lines) if lines else 0.0})
|
| 109 |
+
return sorted(rows, key=lambda x: -x['revenue'])[:limit]
|
| 110 |
+
|
| 111 |
+
|
| 112 |
+
def by_category(tenant, limit=20, business_unit=None, t=None):
|
| 113 |
+
conn = tenant.source(Capability.SALES)
|
| 114 |
+
yf, yt = P.ytd(t or P.today())
|
| 115 |
+
q = Query(Entity.SALES_ORDER_LINE, ('amount',), 'sum', group_by=('category',),
|
| 116 |
+
date_from=yf, date_to=yt, filters=_scope(tenant, business_unit))
|
| 117 |
+
rows = [{'category': r['category'], 'revenue': r['amount']} for r in conn.aggregate(q)]
|
| 118 |
+
return sorted(rows, key=lambda x: -x['revenue'])[:limit]
|
| 119 |
+
|
| 120 |
+
|
| 121 |
+
def returns_monthly(tenant, n=13, t=None):
|
| 122 |
+
"""Returns (credit notes) per month β needs INVOICING, consolidated (no BU split)."""
|
| 123 |
+
conn = tenant.source(Capability.INVOICING)
|
| 124 |
+
t = t or P.today()
|
| 125 |
+
first = P.month_starts(n, t)[0][1]
|
| 126 |
+
span_from = f"{int(first[:4]) - 1:04d}{first[4:]}"
|
| 127 |
+
ex = tenant.config.get('excluded_customer_ids') or []
|
| 128 |
+
filters = [Filter('doc_type', 'eq', 'out_refund'), Filter('status', 'eq', 'posted')]
|
| 129 |
+
if ex:
|
| 130 |
+
filters.append(Filter('customer', 'not_in', list(ex)))
|
| 131 |
+
q = Query(Entity.INVOICE, ('amount',), 'sum', grain='month',
|
| 132 |
+
date_from=span_from, date_to=t.isoformat(), filters=tuple(filters))
|
| 133 |
+
m = {r['period']: -r['amount'] for r in conn.aggregate(q)} # out_refund amount is negative -> flip
|
| 134 |
+
rows = []
|
| 135 |
+
for ym, _s, _e in P.month_starts(n, t):
|
| 136 |
+
y, mo = int(ym[:4]) - 1, int(ym[5:7])
|
| 137 |
+
this, last = m.get(ym, 0.0), m.get(f'{y:04d}-{mo:02d}', 0.0)
|
| 138 |
+
rows.append({'month': ym, 'revenue': this, 'revenue_ly': last, 'yoy_pct': P.yoy_pct(this, last)})
|
| 139 |
+
return rows
|
| 140 |
+
|
| 141 |
+
|
| 142 |
+
# ------------------------------------------------------------------ parity validation (port gate)
|
| 143 |
+
def validate(tenant, t=None):
|
| 144 |
+
"""Reconcile the ported skill to the current Odoo module β a port ships only when it ties out."""
|
| 145 |
+
import modules.sales as M
|
| 146 |
+
t = t or P.today()
|
| 147 |
+
yf, yt = P.ytd(t)
|
| 148 |
+
checks = []
|
| 149 |
+
|
| 150 |
+
def chk(name, a, b, tol=1.0):
|
| 151 |
+
checks.append({'check': name, 'a': round(a, 2), 'b': round(b, 2),
|
| 152 |
+
'gap': round(a - b, 2), 'ok': abs(a - b) <= tol})
|
| 153 |
+
|
| 154 |
+
h, mh = headline(tenant), M.headline()
|
| 155 |
+
chk('headline YTD revenue == module', h['ytd_revenue'], mh['ytd_revenue'])
|
| 156 |
+
chk('headline YTD orders == module', h['ytd_orders'], mh['ytd_orders'])
|
| 157 |
+
chk('headline YTD customers == module', h['ytd_customers'], mh['ytd_customers'])
|
| 158 |
+
chk('Ξ£ top_customers == module', sum(r['revenue'] for r in top_customers(tenant, 10**9)),
|
| 159 |
+
sum(r['revenue'] for r in M.top_customers(limit=10**9)))
|
| 160 |
+
chk('Ξ£ by_category == module', sum(r['revenue'] for r in by_category(tenant, 10**9)),
|
| 161 |
+
sum(r['revenue'] for r in M.by_category(limit=10**9)))
|
| 162 |
+
chk('Ξ£ top_skus revenue == module', sum(r['revenue'] for r in top_skus(tenant, 10**9)),
|
| 163 |
+
sum(r['revenue'] for r in M.top_skus(limit=10**9)))
|
| 164 |
+
chk('monthly_trend Ξ£ this-yr == module', sum(r['revenue'] for r in monthly_trend(tenant)),
|
| 165 |
+
sum(r['revenue'] for r in M.monthly_trend()))
|
| 166 |
+
chk('returns_monthly Ξ£ this-yr == module', sum(r['revenue'] for r in returns_monthly(tenant)),
|
| 167 |
+
sum(r['revenue'] for r in M.returns_monthly()))
|
| 168 |
+
return checks
|
platform/harness/tables.py
ADDED
|
@@ -0,0 +1,129 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""harness/tables.py β a topic-backed TABLE for the grid (CG-2).
|
| 2 |
+
|
| 3 |
+
The grid's first table (customers) is built from `modules.customer_data.pool()`: one row per
|
| 4 |
+
customer, the whole book shipped as component args, filtered client-side. That works at 1,550
|
| 5 |
+
rows and does not work at line grain β `sales_lines` is 201,558 rows / ~89 MB.
|
| 6 |
+
|
| 7 |
+
This module builds the SECOND table: rows come from `semantic.store_rows`, so the fetch is
|
| 8 |
+
WINDOWED and the filtering is SQL. Its whole job is to hand the grid a payload in the same
|
| 9 |
+
host-neutral shape the React component already speaks, plus the one thing a windowed table must
|
| 10 |
+
carry that a whole-book table never needed:
|
| 11 |
+
|
| 12 |
+
counts = {matched, total, shown, windowed}
|
| 13 |
+
|
| 14 |
+
`matched` and `total` come from their own queries over the FULL scope; `shown` is the size of
|
| 15 |
+
this window. The component renders "showing SHOWN of MATCHED (filtered from TOTAL)" from those
|
| 16 |
+
numbers and NEVER from `rows.length` β which is the difference between an honest window and a
|
| 17 |
+
silent `[:N]` wearing a total ([[no-unverifiable-aggregates]]).
|
| 18 |
+
|
| 19 |
+
The field CONTRACT is derived from the topic, not hand-written: labels and types come from the
|
| 20 |
+
model's dims and metrics, so a new metric cannot arrive as an untyped, unlabelled column and the
|
| 21 |
+
grid can never disagree with the semantic layer about what a column means.
|
| 22 |
+
"""
|
| 23 |
+
from harness import semantic as _sem
|
| 24 |
+
|
| 25 |
+
#: Topic-backed tables the grid can open. `key` is the storage key its saved views live under β
|
| 26 |
+
#: separate from the customer table's, so the two tables' views can never collide.
|
| 27 |
+
TOPIC_TABLES = {
|
| 28 |
+
"sales_lines": {
|
| 29 |
+
"label": "Sales lines",
|
| 30 |
+
"storage_key": "sales_lines_table_workspace",
|
| 31 |
+
# the columns worth defaulting to visible, in order; everything else stays available
|
| 32 |
+
"default_visible": ["date", "order_partner", "product", "category", "state", "agent",
|
| 33 |
+
"team", "revenue", "units", "margin"],
|
| 34 |
+
# a line-grain table is unusable sorted by nothing in particular
|
| 35 |
+
"default_sorts": [{"colId": "date", "dir": "desc"}],
|
| 36 |
+
},
|
| 37 |
+
}
|
| 38 |
+
|
| 39 |
+
#: how many rows one window carries. Small enough that the payload stays a few hundred KB at
|
| 40 |
+
#: ~440 B/row, large enough to scroll. The honest counts make the window visible rather than
|
| 41 |
+
#: silent, so this is a performance knob and not a correctness one.
|
| 42 |
+
PAGE_ROWS = 200
|
| 43 |
+
|
| 44 |
+
|
| 45 |
+
def _labels(topic):
|
| 46 |
+
"""colId -> human label, taken from the MODEL (dim labels, metric labels)."""
|
| 47 |
+
t = _sem.topics()[topic]
|
| 48 |
+
out = {}
|
| 49 |
+
for key, d in ((t.get("store") or {}).get("dims") or {}).items():
|
| 50 |
+
out[key] = d.get("label") or key.replace("_", " ").title()
|
| 51 |
+
out[f"{key}_id"] = f"{out[key]} id"
|
| 52 |
+
out["date"] = "Date"
|
| 53 |
+
for k, m in _sem.metrics().items():
|
| 54 |
+
if m.get("topic") == topic:
|
| 55 |
+
out[k] = m.get("label") or k
|
| 56 |
+
return out
|
| 57 |
+
|
| 58 |
+
|
| 59 |
+
def table_fields(topic):
|
| 60 |
+
"""The grid field contract for a topic, derived from the semantic model.
|
| 61 |
+
|
| 62 |
+
Every column is `source='odoo'` β read-only. A line is a fact from the ERP; unlike the
|
| 63 |
+
customer table there is no overlay stratum here (notes/tags hang off a CUSTOMER, not off an
|
| 64 |
+
individual order line), so the two-strata split simply does not arise.
|
| 65 |
+
"""
|
| 66 |
+
cfg = TOPIC_TABLES[topic]
|
| 67 |
+
cols = _sem.store_columns(topic, grain="row")
|
| 68 |
+
labels = _labels(topic)
|
| 69 |
+
order = [k for k in cfg["default_visible"] if k in cols]
|
| 70 |
+
order += [k for k in cols if k not in order]
|
| 71 |
+
return [{
|
| 72 |
+
"key": k,
|
| 73 |
+
"label": labels.get(k, k.replace("_", " ").title()),
|
| 74 |
+
"type": cols[k]["type"],
|
| 75 |
+
"source": "odoo",
|
| 76 |
+
"default": k in cfg["default_visible"],
|
| 77 |
+
} for k in order]
|
| 78 |
+
|
| 79 |
+
|
| 80 |
+
def table_payload(topic, config=None, offset=0, limit=None, team_id=None,
|
| 81 |
+
date_from=None, date_to=None):
|
| 82 |
+
"""Fetch ONE window of a topic-backed table plus its scope-wide counts and totals.
|
| 83 |
+
|
| 84 |
+
`config` is a grid ViewConfig (filters / filterConj / sorts / search) β the same shape the
|
| 85 |
+
customer table persists, so a saved view means the same thing on either table.
|
| 86 |
+
|
| 87 |
+
Returns the component payload: {fields, rows, counts, aggregates, storageKey}.
|
| 88 |
+
"""
|
| 89 |
+
if topic not in TOPIC_TABLES:
|
| 90 |
+
raise _sem.ModelError(f"no topic-backed table registered for {topic!r}")
|
| 91 |
+
cfg = TOPIC_TABLES[topic]
|
| 92 |
+
config = config or {}
|
| 93 |
+
limit = PAGE_ROWS if limit is None else limit
|
| 94 |
+
|
| 95 |
+
got = _sem.store_rows(
|
| 96 |
+
topic,
|
| 97 |
+
filter_tree=config.get("filters") or [],
|
| 98 |
+
filter_conj=config.get("filterConj") or "and",
|
| 99 |
+
search=config.get("search") or None,
|
| 100 |
+
sorts=config.get("sorts") or cfg["default_sorts"],
|
| 101 |
+
member_ids=config.get("memberPids") or None,
|
| 102 |
+
limit=limit, offset=offset,
|
| 103 |
+
team_id=team_id, date_from=date_from, date_to=date_to,
|
| 104 |
+
)
|
| 105 |
+
|
| 106 |
+
rows = []
|
| 107 |
+
for i, r in enumerate(got["rows"]):
|
| 108 |
+
# `pid` is the grid's row identity contract (selection, detail, overlay all key on it).
|
| 109 |
+
# At line grain that identity is the order LINE id, which store_rows returns as _rid.
|
| 110 |
+
row = {k: v for k, v in r.items() if k != "_rid"}
|
| 111 |
+
row["pid"] = r["_rid"]
|
| 112 |
+
rows.append(row)
|
| 113 |
+
|
| 114 |
+
return {
|
| 115 |
+
"fields": table_fields(topic),
|
| 116 |
+
"rows": rows,
|
| 117 |
+
# THE honest window. `matched`/`total` are scope-wide queries; `shown` is this window.
|
| 118 |
+
# A caller must never recompute `matched` from len(rows).
|
| 119 |
+
"counts": {
|
| 120 |
+
"shown": len(rows),
|
| 121 |
+
"matched": got["row_count"],
|
| 122 |
+
"total": got["total_count"],
|
| 123 |
+
"windowed": True,
|
| 124 |
+
},
|
| 125 |
+
"aggregates": got["aggregates"], # over the whole FILTERED scope, not the window
|
| 126 |
+
"storageKey": cfg["storage_key"],
|
| 127 |
+
"label": cfg["label"],
|
| 128 |
+
"window": got["window"],
|
| 129 |
+
}
|